Repository: dbt-labs/dbt-core Branch: main Commit: 7acf053cb99a Files: 1075 Total size: 12.5 MB Directory structure: gitextract_6zoc49gl/ ├── .changes/ │ ├── 0.0.0.md │ ├── README.md │ ├── header.tpl.md │ └── unreleased/ │ ├── .gitkeep │ ├── Dependencies-20251118-155354.yaml │ ├── Dependencies-20251217-151349.yaml │ ├── Dependencies-20251219-153804.yaml │ ├── Dependencies-20260113-135442.yaml │ ├── Dependencies-20260113-143057.yaml │ ├── Dependencies-20260121-171712.yaml │ ├── Dependencies-20260302-130425.yaml │ ├── Docs-20240311-140344.yaml │ ├── Docs-20240501-021050.yaml │ ├── Docs-20240516-223036.yaml │ ├── Docs-20240613-151048.yaml │ ├── Docs-20250728-162542.yaml │ ├── Docs-20260319-112630.yaml │ ├── Features-20251006-140352.yaml │ ├── Features-20251111-103504.yaml │ ├── Features-20251117-141053.yaml │ ├── Features-20251201-165209.yaml │ ├── Features-20251203-122926.yaml │ ├── Features-20251210-202001.yaml │ ├── Features-20260115-131115.yaml │ ├── Features-20260119-210143.yaml │ ├── Features-20260123-173805.yaml │ ├── Features-20260126-164458.yaml │ ├── Features-20260127-114149.yaml │ ├── Features-20260127-214617.yaml │ ├── Features-20260128-125727.yaml │ ├── Features-20260128-172052.yaml │ ├── Features-20260129-110711.yaml │ ├── Features-20260129-114832.yaml │ ├── Features-20260129-175432.yaml │ ├── Features-20260129-185659.yaml │ ├── Features-20260205-105228.yaml │ ├── Features-20260212-232728.yaml │ ├── Features-20260219-133321.yaml │ ├── Features-20260302-123311.yaml │ ├── Features-20260303-214328.yaml │ ├── Features-9041.yaml │ ├── Fixes-20250922-151726.yaml │ ├── Fixes-20251117-140649.yaml │ ├── Fixes-20251117-185025.yaml │ ├── Fixes-20251118-171106.yaml │ ├── Fixes-20251119-195034.yaml │ ├── Fixes-20251124-155629.yaml │ ├── Fixes-20251124-155756.yaml │ ├── Fixes-20251124-170855.yaml │ ├── Fixes-20251125-120246.yaml │ ├── Fixes-20251125-122020.yaml │ ├── Fixes-20251127-141308.yaml │ ├── Fixes-20251127-145929.yaml │ ├── Fixes-20251127-170124.yaml │ ├── Fixes-20251128-102129.yaml │ ├── Fixes-20251128-122838.yaml │ ├── Fixes-20251128-161937.yaml │ ├── Fixes-20251128-163144.yaml │ ├── Fixes-20251202-133705.yaml │ ├── Fixes-20251204-094753.yaml │ ├── Fixes-20251209-175031.yaml │ ├── Fixes-20251210-143935.yaml │ ├── Fixes-20251216-120727.yaml │ ├── Fixes-20251217-002813.yaml │ ├── Fixes-20251217-105918.yaml │ ├── Fixes-20251219-184405.yaml │ ├── Fixes-20260109-141332.yaml │ ├── Fixes-20260113-161742.yaml │ ├── Fixes-20260124-212300.yaml │ ├── Fixes-20260125-113244.yaml │ ├── Fixes-20260126-160659.yaml │ ├── Fixes-20260202-123453.yaml │ ├── Fixes-20260202-153835.yaml │ ├── Fixes-20260204-184553.yaml │ ├── Fixes-20260204-211128.yaml │ ├── Fixes-20260205-120000.yaml │ ├── Fixes-20260206-204257.yaml │ ├── Fixes-20260210-154042.yaml │ ├── Fixes-20260216-230817.yaml │ ├── Fixes-20260219-012414.yaml │ ├── Fixes-20260219-131833.yaml │ ├── Fixes-20260219-170000.yaml │ ├── Fixes-20260223-132342.yaml │ ├── Fixes-20260223-161820.yaml │ ├── Fixes-20260224-115546.yaml │ ├── Fixes-20260224-180000.yaml │ ├── Fixes-20260224-231047.yaml │ ├── Fixes-20260225-203131.yaml │ ├── Fixes-20260226-163836.yaml │ ├── Fixes-20260227-133424.yaml │ ├── Fixes-20260227-140009.yaml │ ├── Fixes-20260302-000739.yaml │ ├── Fixes-20260303-114528.yaml │ ├── Fixes-20260305-133929.yaml │ ├── Fixes-20260306-034008.yaml │ ├── Fixes-20260306-071359.yaml │ ├── Fixes-20260306-192013.yaml │ ├── Fixes-20260310-120000.yaml │ ├── Fixes-20260317-094953.yaml │ ├── Fixes-20260318-000000.yaml │ ├── Fixes-20260318-091800.yaml │ ├── Fixes-20260318-173426.yaml │ ├── Fixes-20260318-182358.yaml │ ├── Fixes-20260318-191153.yaml │ ├── Fixes-20260318-200626.yaml │ ├── Under the Hood-20250929-151159.yaml │ ├── Under the Hood-20251119-110110.yaml │ ├── Under the Hood-20251121-140515.yaml │ ├── Under the Hood-20251209-131857.yaml │ ├── Under the Hood-20251215-155046.yaml │ ├── Under the Hood-20260205-200835.yaml │ ├── Under the Hood-20260210-151244.yaml │ ├── Under the Hood-20260214-181659.yaml │ ├── Under the Hood-20260220-170741.yaml │ ├── Under the Hood-20260223-121653.yaml │ ├── Under the Hood-20260225-092728.yaml │ ├── Under the Hood-20260302-101022.yaml │ ├── Under the Hood-20260306-182611.yaml │ └── Under the Hood-20260318-120715.yaml ├── .changie.yaml ├── .dockerignore ├── .flake8 ├── .git-blame-ignore-revs ├── .gitattributes ├── .github/ │ ├── CODEOWNERS │ ├── ISSUE_TEMPLATE/ │ │ ├── bug-report.yml │ │ ├── code-docs.yml │ │ ├── config.yml │ │ ├── feature-request.yml │ │ ├── implementation-ticket.yml │ │ └── regression-report.yml │ ├── _README.md │ ├── actions/ │ │ ├── latest-wrangler/ │ │ │ ├── Dockerfile │ │ │ ├── README.md │ │ │ ├── action.yml │ │ │ ├── examples/ │ │ │ │ ├── example_workflow.yml │ │ │ │ └── example_workflow_dispatch.json │ │ │ └── main.py │ │ └── setup-postgres-windows/ │ │ └── action.yml │ ├── dbt-postgres-testing.yml │ ├── dependabot.yml │ ├── pull_request_template.md │ └── workflows/ │ ├── artifact-reviews.yml │ ├── auto-respond-bug-reports.yml │ ├── backport.yml │ ├── bot-changelog.yml │ ├── changelog-existence.yml │ ├── check-artifact-changes.yml │ ├── community-label.yml │ ├── cut-release-branch.yml │ ├── docs-issue.yml │ ├── main.yml │ ├── nightly-release.yml │ ├── release-branch-tests.yml │ ├── release.yml │ ├── repository-cleanup.yml │ ├── schema-check.yml │ ├── stale.yml │ ├── structured-logging-schema-check.yml │ ├── test-repeater.yml │ ├── triage-labels.yml │ └── update-test-durations.yml ├── .gitignore ├── .pre-commit-config.yaml ├── AGENTS.md ├── ARCHITECTURE.md ├── CHANGELOG.md ├── CONTRIBUTING.md ├── Dockerfile.test ├── Makefile ├── README.md ├── SECURITY.md ├── codecov.yml ├── core/ │ ├── .test_durations │ ├── LICENSE │ ├── README.md │ ├── dbt/ │ │ ├── README.md │ │ ├── __init__.py │ │ ├── __version__.py │ │ ├── _pydantic_shim.py │ │ ├── artifacts/ │ │ │ ├── __init__.py │ │ │ ├── exceptions/ │ │ │ │ ├── __init__.py │ │ │ │ └── schemas.py │ │ │ ├── resources/ │ │ │ │ ├── __init__.py │ │ │ │ ├── base.py │ │ │ │ ├── types.py │ │ │ │ └── v1/ │ │ │ │ ├── analysis.py │ │ │ │ ├── catalog.py │ │ │ │ ├── components.py │ │ │ │ ├── config.py │ │ │ │ ├── documentation.py │ │ │ │ ├── exposure.py │ │ │ │ ├── function.py │ │ │ │ ├── generic_test.py │ │ │ │ ├── group.py │ │ │ │ ├── hook.py │ │ │ │ ├── macro.py │ │ │ │ ├── metric.py │ │ │ │ ├── model.py │ │ │ │ ├── owner.py │ │ │ │ ├── saved_query.py │ │ │ │ ├── seed.py │ │ │ │ ├── semantic_layer_components.py │ │ │ │ ├── semantic_model.py │ │ │ │ ├── singular_test.py │ │ │ │ ├── snapshot.py │ │ │ │ ├── source_definition.py │ │ │ │ ├── sql_operation.py │ │ │ │ └── unit_test_definition.py │ │ │ ├── schemas/ │ │ │ │ ├── __init__.py │ │ │ │ ├── base.py │ │ │ │ ├── batch_results.py │ │ │ │ ├── catalog/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── v1/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── catalog.py │ │ │ │ ├── freshness/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── v3/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── freshness.py │ │ │ │ ├── manifest/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── v12/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── manifest.py │ │ │ │ ├── results.py │ │ │ │ ├── run/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── v5/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── run.py │ │ │ │ └── upgrades/ │ │ │ │ ├── __init__.py │ │ │ │ ├── upgrade_manifest.py │ │ │ │ └── upgrade_manifest_dbt_version.py │ │ │ └── utils/ │ │ │ └── validation.py │ │ ├── cli/ │ │ │ ├── __init__.py │ │ │ ├── context.py │ │ │ ├── exceptions.py │ │ │ ├── flags.py │ │ │ ├── main.py │ │ │ ├── option_types.py │ │ │ ├── options.py │ │ │ ├── params.py │ │ │ ├── requires.py │ │ │ ├── resolvers.py │ │ │ └── types.py │ │ ├── clients/ │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── checked_load.py │ │ │ ├── git.py │ │ │ ├── jinja.py │ │ │ ├── jinja_static.py │ │ │ ├── registry.py │ │ │ └── yaml_helper.py │ │ ├── compilation.py │ │ ├── config/ │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── catalogs.py │ │ │ ├── profile.py │ │ │ ├── project.py │ │ │ ├── renderer.py │ │ │ ├── runtime.py │ │ │ ├── selectors.py │ │ │ └── utils.py │ │ ├── constants.py │ │ ├── context/ │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── base.py │ │ │ ├── configured.py │ │ │ ├── context_config.py │ │ │ ├── docs.py │ │ │ ├── exceptions_jinja.py │ │ │ ├── macro_resolver.py │ │ │ ├── macros.py │ │ │ ├── manifest.py │ │ │ ├── providers.py │ │ │ ├── query_header.py │ │ │ ├── secret.py │ │ │ └── target.py │ │ ├── contracts/ │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── files.py │ │ │ ├── graph/ │ │ │ │ ├── __init__.py │ │ │ │ ├── manifest.py │ │ │ │ ├── metrics.py │ │ │ │ ├── model_config.py │ │ │ │ ├── node_args.py │ │ │ │ ├── nodes.py │ │ │ │ ├── semantic_manifest.py │ │ │ │ └── unparsed.py │ │ │ ├── project.py │ │ │ ├── results.py │ │ │ ├── selection.py │ │ │ ├── sql.py │ │ │ ├── state.py │ │ │ └── util.py │ │ ├── deprecations.py │ │ ├── deps/ │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── base.py │ │ │ ├── git.py │ │ │ ├── local.py │ │ │ ├── registry.py │ │ │ ├── resolver.py │ │ │ └── tarball.py │ │ ├── docs/ │ │ │ ├── Makefile │ │ │ ├── build/ │ │ │ │ ├── doctrees/ │ │ │ │ │ ├── environment.pickle │ │ │ │ │ └── index.doctree │ │ │ │ └── html/ │ │ │ │ ├── .buildinfo │ │ │ │ ├── _sources/ │ │ │ │ │ └── index.rst.txt │ │ │ │ ├── _static/ │ │ │ │ │ ├── _sphinx_javascript_frameworks_compat.js │ │ │ │ │ ├── alabaster.css │ │ │ │ │ ├── basic.css │ │ │ │ │ ├── custom.css │ │ │ │ │ ├── doctools.js │ │ │ │ │ ├── documentation_options.js │ │ │ │ │ ├── jquery-3.6.0.js │ │ │ │ │ ├── jquery.js │ │ │ │ │ ├── language_data.js │ │ │ │ │ ├── pygments.css │ │ │ │ │ ├── searchtools.js │ │ │ │ │ ├── sphinx_highlight.js │ │ │ │ │ ├── underscore-1.13.1.js │ │ │ │ │ └── underscore.js │ │ │ │ ├── genindex.html │ │ │ │ ├── objects.inv │ │ │ │ ├── search.html │ │ │ │ └── searchindex.js │ │ │ ├── make.bat │ │ │ └── source/ │ │ │ ├── _ext/ │ │ │ │ └── dbt_click.py │ │ │ ├── conf.py │ │ │ └── index.rst │ │ ├── env_vars.py │ │ ├── event_time/ │ │ │ ├── event_time.py │ │ │ └── sample_window.py │ │ ├── events/ │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── base_types.py │ │ │ ├── core_types_pb2.py │ │ │ ├── logging.py │ │ │ └── types.py │ │ ├── exceptions.py │ │ ├── flags.py │ │ ├── graph/ │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── cli.py │ │ │ ├── graph.py │ │ │ ├── queue.py │ │ │ ├── selector.py │ │ │ ├── selector_methods.py │ │ │ ├── selector_spec.py │ │ │ └── thread_pool.py │ │ ├── hooks.py │ │ ├── include/ │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ └── starter_project/ │ │ │ ├── .gitignore │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── analyses/ │ │ │ │ └── .gitkeep │ │ │ ├── dbt_project.yml │ │ │ ├── macros/ │ │ │ │ └── .gitkeep │ │ │ ├── models/ │ │ │ │ └── example/ │ │ │ │ ├── my_first_dbt_model.sql │ │ │ │ ├── my_second_dbt_model.sql │ │ │ │ └── schema.yml │ │ │ ├── seeds/ │ │ │ │ └── .gitkeep │ │ │ ├── snapshots/ │ │ │ │ └── .gitkeep │ │ │ └── tests/ │ │ │ └── .gitkeep │ │ ├── internal_deprecations.py │ │ ├── jsonschemas/ │ │ │ ├── __init__.py │ │ │ ├── jsonschemas.py │ │ │ ├── project/ │ │ │ │ ├── 0.0.110.json │ │ │ │ └── 0.0.85.json │ │ │ └── resources/ │ │ │ ├── 0.0.110.json │ │ │ ├── 0.0.85.json │ │ │ └── latest.json │ │ ├── links.py │ │ ├── materializations/ │ │ │ ├── __init__.py │ │ │ └── incremental/ │ │ │ ├── __init__.py │ │ │ └── microbatch.py │ │ ├── mp_context.py │ │ ├── node_types.py │ │ ├── parser/ │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── analysis.py │ │ │ ├── base.py │ │ │ ├── common.py │ │ │ ├── docs.py │ │ │ ├── fixtures.py │ │ │ ├── functions.py │ │ │ ├── generic_test.py │ │ │ ├── generic_test_builders.py │ │ │ ├── hooks.py │ │ │ ├── macros.py │ │ │ ├── manifest.py │ │ │ ├── models.py │ │ │ ├── partial.py │ │ │ ├── read_files.py │ │ │ ├── schema_generic_tests.py │ │ │ ├── schema_renderer.py │ │ │ ├── schema_yaml_readers.py │ │ │ ├── schemas.py │ │ │ ├── search.py │ │ │ ├── seeds.py │ │ │ ├── singular_test.py │ │ │ ├── snapshots.py │ │ │ ├── sources.py │ │ │ ├── sql.py │ │ │ └── unit_tests.py │ │ ├── plugins/ │ │ │ ├── __init__.py │ │ │ ├── contracts.py │ │ │ ├── exceptions.py │ │ │ ├── manager.py │ │ │ └── manifest.py │ │ ├── profiler.py │ │ ├── py.typed │ │ ├── runners/ │ │ │ ├── __init__.py │ │ │ ├── exposure_runner.py │ │ │ ├── no_op_runner.py │ │ │ └── saved_query_runner.py │ │ ├── selected_resources.py │ │ ├── task/ │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── base.py │ │ │ ├── build.py │ │ │ ├── clean.py │ │ │ ├── clone.py │ │ │ ├── compile.py │ │ │ ├── debug.py │ │ │ ├── deps.py │ │ │ ├── docs/ │ │ │ │ ├── __init__.py │ │ │ │ ├── generate.py │ │ │ │ └── serve.py │ │ │ ├── freshness.py │ │ │ ├── function.py │ │ │ ├── group_lookup.py │ │ │ ├── init.py │ │ │ ├── list.py │ │ │ ├── printer.py │ │ │ ├── retry.py │ │ │ ├── run.py │ │ │ ├── run_operation.py │ │ │ ├── runnable.py │ │ │ ├── seed.py │ │ │ ├── show.py │ │ │ ├── snapshot.py │ │ │ ├── sql.py │ │ │ └── test.py │ │ ├── tests/ │ │ │ ├── fixtures/ │ │ │ │ ├── __init__.py │ │ │ │ └── project.py │ │ │ └── util.py │ │ ├── tracking.py │ │ ├── utils/ │ │ │ ├── __init__.py │ │ │ ├── artifact_upload.py │ │ │ └── utils.py │ │ └── version.py │ ├── hatch.toml │ └── pyproject.toml ├── docker/ │ ├── Dockerfile │ ├── README.md │ └── test.sh ├── docker-compose.yml ├── docs/ │ ├── arch/ │ │ ├── 1_Overview.md │ │ ├── 2_CLI.md │ │ ├── 3.1_Partial_Parsing.md │ │ ├── 3.2_Deferral.md │ │ ├── 3_Parsing.md │ │ ├── 4.1_Task_Framework.md │ │ ├── 4.2_Graph_Compilation.md │ │ ├── 4.3_Node_Selection.md │ │ ├── 4.4_Node_Compilation.md │ │ ├── 4.5_Node_Materialization.md │ │ ├── 4_Execution.md │ │ ├── 5_Adapter.md │ │ ├── 6.10_dbt_compile.md │ │ ├── 6.11_dbt_source.md │ │ ├── 6.12_dbt_run-operation.md │ │ ├── 6.13_dbt_init.md │ │ ├── 6.14_dbt_list.md │ │ ├── 6.15_dbt_retry.md │ │ ├── 6.16_dbt_clone.md │ │ ├── 6.17_dbt_debug.md │ │ ├── 6.18_dbt_clean.md │ │ ├── 6.1_dbt_parse.md │ │ ├── 6.2_dbt_run.md │ │ ├── 6.3_dbt_build.md │ │ ├── 6.4_dbt_seed.md │ │ ├── 6.5_dbt_snapshot.md │ │ ├── 6.6_dbt_test.md │ │ ├── 6.7_dbt_show.md │ │ ├── 6.8_dbt_deps.md │ │ ├── 6.9_dbt_docs.md │ │ ├── 6_Commands.md │ │ ├── 7_Artifacts.md │ │ └── 8_Versioning_Branching_Strategy.md │ ├── guides/ │ │ ├── behavior-change-flags.md │ │ └── parsing-vs-compilation-vs-runtime.md │ └── roadmap/ │ ├── 2022-05-dbt-a-core-story.md │ ├── 2022-08-back-for-more.md │ ├── 2023-02-back-to-basics.md │ ├── 2023-11-dbt-tng.md │ ├── 2024-12-play-on.md │ ├── 2025-05-new-engine-same-language.md │ └── 2025-12-magic-to-do.md ├── hatch.toml ├── pyproject.toml ├── pytest.ini ├── requirements.txt ├── schemas/ │ └── dbt/ │ ├── catalog/ │ │ └── v1.json │ ├── manifest/ │ │ ├── v10.json │ │ ├── v11.json │ │ ├── v12.json │ │ ├── v5.json │ │ ├── v6.json │ │ ├── v7.json │ │ ├── v8.json │ │ └── v9.json │ ├── run-results/ │ │ ├── v4.json │ │ ├── v5.json │ │ └── v6.json │ └── sources/ │ └── v3.json ├── scripts/ │ ├── check_libyaml.py │ ├── collect-artifact-schema.py │ ├── collect-dbt-contexts.py │ ├── env-setup.sh │ ├── migrate-adapters.py │ ├── pre-commit-hooks/ │ │ └── no_versioned_artifact_resource_imports.py │ ├── setup_db.sh │ └── update_dev_packages.sh ├── tests/ │ ├── __init__.py │ ├── conftest.py │ ├── data/ │ │ └── __init__.py │ ├── fixtures/ │ │ ├── __init__.py │ │ ├── dbt_integration_project.py │ │ ├── jaffle_shop.py │ │ └── jaffle_shop_data/ │ │ ├── .gitkeep │ │ ├── raw_customers.csv │ │ ├── raw_orders.csv │ │ └── raw_payments.csv │ ├── functional/ │ │ ├── README.md │ │ ├── __init__.py │ │ ├── access/ │ │ │ └── test_access.py │ │ ├── analysis/ │ │ │ └── test_analyses.py │ │ ├── artifacts/ │ │ │ ├── data/ │ │ │ │ ├── results/ │ │ │ │ │ ├── v4/ │ │ │ │ │ │ └── run_results.json │ │ │ │ │ ├── v5/ │ │ │ │ │ │ └── run_results.json │ │ │ │ │ └── v6/ │ │ │ │ │ └── run_results.json │ │ │ │ └── state/ │ │ │ │ ├── v1/ │ │ │ │ │ └── manifest.json │ │ │ │ ├── v10/ │ │ │ │ │ └── manifest.json │ │ │ │ ├── v11/ │ │ │ │ │ └── manifest.json │ │ │ │ ├── v12/ │ │ │ │ │ └── manifest.json │ │ │ │ ├── v2/ │ │ │ │ │ └── manifest.json │ │ │ │ ├── v3/ │ │ │ │ │ └── manifest.json │ │ │ │ ├── v4/ │ │ │ │ │ └── manifest.json │ │ │ │ ├── v5/ │ │ │ │ │ └── manifest.json │ │ │ │ ├── v6/ │ │ │ │ │ └── manifest.json │ │ │ │ ├── v7/ │ │ │ │ │ └── manifest.json │ │ │ │ ├── v8/ │ │ │ │ │ └── manifest.json │ │ │ │ └── v9/ │ │ │ │ └── manifest.json │ │ │ ├── expected_manifest.py │ │ │ ├── expected_run_results.py │ │ │ ├── test_artifact_fields.py │ │ │ ├── test_artifacts.py │ │ │ ├── test_docs_generate_defer.py │ │ │ ├── test_override.py │ │ │ ├── test_previous_version_state.py │ │ │ ├── test_run_execution_result.py │ │ │ └── test_run_results.py │ │ ├── assertions/ │ │ │ └── test_runner.py │ │ ├── basic/ │ │ │ ├── data/ │ │ │ │ ├── seed-initial.csv │ │ │ │ ├── seed-update.csv │ │ │ │ ├── summary_expected.csv │ │ │ │ ├── summary_expected_update.csv │ │ │ │ ├── varchar10_seed.sql │ │ │ │ └── varchar300_seed.sql │ │ │ ├── test_basic.py │ │ │ ├── test_invalid_reference.py │ │ │ ├── test_jaffle_shop.py │ │ │ ├── test_mixed_case_db.py │ │ │ ├── test_project.py │ │ │ ├── test_simple_reference.py │ │ │ └── test_varchar_widening.py │ │ ├── build_command/ │ │ │ ├── fixtures.py │ │ │ └── test_build.py │ │ ├── catalogs/ │ │ │ └── test_catalogs_parsing.py │ │ ├── clean/ │ │ │ └── test_clean.py │ │ ├── cli/ │ │ │ ├── test_cli_exit_codes.py │ │ │ ├── test_click_flags.py │ │ │ ├── test_env_var_deprecations.py │ │ │ ├── test_error_handling.py │ │ │ ├── test_multioption.py │ │ │ ├── test_option_interaction_validations.py │ │ │ ├── test_requires.py │ │ │ └── test_resolvers.py │ │ ├── colors/ │ │ │ └── test_colors.py │ │ ├── column_quoting/ │ │ │ └── test_column_quotes.py │ │ ├── compile/ │ │ │ ├── fixtures.py │ │ │ └── test_compile.py │ │ ├── configs/ │ │ │ ├── fixtures.py │ │ │ ├── test_configs.py │ │ │ ├── test_configs_in_schema_files.py │ │ │ ├── test_contract_configs.py │ │ │ ├── test_custom_node_colors_configs.py │ │ │ ├── test_disabled_configs.py │ │ │ ├── test_disabled_model.py │ │ │ ├── test_dupe_paths.py │ │ │ ├── test_get_default.py │ │ │ ├── test_grant_configs.py │ │ │ ├── test_indiv_tests.py │ │ │ ├── test_unused_configs.py │ │ │ ├── test_vars_file.py │ │ │ ├── test_versioned_model_constraint.py │ │ │ └── test_warn_error_options.py │ │ ├── conftest.py │ │ ├── constraints/ │ │ │ ├── fixtures.py │ │ │ └── test_foreign_key_constraints.py │ │ ├── context_methods/ │ │ │ ├── first_dependency.py │ │ │ ├── test_builtin_functions.py │ │ │ ├── test_cli_var_override.py │ │ │ ├── test_cli_vars.py │ │ │ ├── test_custom_env_vars.py │ │ │ ├── test_env_vars.py │ │ │ ├── test_secret_env_vars.py │ │ │ ├── test_var_dependency.py │ │ │ ├── test_var_in_generate_name.py │ │ │ └── test_yaml_functions.py │ │ ├── contracts/ │ │ │ ├── test_contract_enforcement.py │ │ │ ├── test_contract_precision.py │ │ │ └── test_nonstandard_data_type.py │ │ ├── custom_aliases/ │ │ │ ├── fixtures.py │ │ │ └── test_custom_aliases.py │ │ ├── custom_schemas/ │ │ │ └── test_custom_schemas.py │ │ ├── custom_singular_tests/ │ │ │ ├── data/ │ │ │ │ └── seed_expected.sql │ │ │ └── test_custom_singular_tests.py │ │ ├── custom_target_path/ │ │ │ └── test_custom_target_path.py │ │ ├── cycles/ │ │ │ └── test_cycles.py │ │ ├── data_test_patch/ │ │ │ ├── fixtures.py │ │ │ └── test_singular_test_patch.py │ │ ├── data_tests/ │ │ │ └── test_hooks.py │ │ ├── dbt_runner/ │ │ │ └── test_dbt_runner.py │ │ ├── defer_state/ │ │ │ ├── data/ │ │ │ │ └── manifest.json │ │ │ ├── fixtures.py │ │ │ ├── test_defer_state.py │ │ │ ├── test_group_updates.py │ │ │ ├── test_modified_state.py │ │ │ ├── test_modified_state_environment_vars.py │ │ │ ├── test_modified_state_jinja.py │ │ │ ├── test_modified_state_schema_evolution.py │ │ │ ├── test_modified_state_sources_unrendered.py │ │ │ ├── test_modified_state_vars.py │ │ │ ├── test_removed_test_state.py │ │ │ ├── test_run_results_state.py │ │ │ └── test_unrendered_config.py │ │ ├── dependencies/ │ │ │ ├── data/ │ │ │ │ ├── seed.sql │ │ │ │ └── update.sql │ │ │ ├── duplicate_dependency/ │ │ │ │ └── dbt_project.yml │ │ │ ├── early_hook_dependency/ │ │ │ │ └── dbt_project.yml │ │ │ ├── inverted_ref_dependency/ │ │ │ │ ├── dbt_project.yml │ │ │ │ └── models/ │ │ │ │ ├── a.sql │ │ │ │ ├── b.sql │ │ │ │ └── b_root_package_in_ref.sql │ │ │ ├── late_hook_dependency/ │ │ │ │ └── dbt_project.yml │ │ │ ├── local_dependency/ │ │ │ │ ├── dbt_project.yml │ │ │ │ ├── macros/ │ │ │ │ │ ├── dep_macro.sql │ │ │ │ │ └── generate_schema_name.sql │ │ │ │ ├── models/ │ │ │ │ │ ├── model_to_import.sql │ │ │ │ │ └── schema.yml │ │ │ │ └── seeds/ │ │ │ │ └── seed.csv │ │ │ ├── models_local/ │ │ │ │ ├── dep_source_model.sql │ │ │ │ ├── my_configured_model.sql │ │ │ │ ├── my_model.sql │ │ │ │ ├── schema.yml │ │ │ │ └── source_override_model.sql │ │ │ ├── nested_dependency/ │ │ │ │ ├── dbt_project.yml │ │ │ │ ├── models/ │ │ │ │ │ └── model.sql │ │ │ │ └── packages.yml │ │ │ ├── test_add_package_edge_cases.py │ │ │ ├── test_dependency_inverted_ref.py │ │ │ ├── test_dependency_options.py │ │ │ ├── test_dependency_secrets.py │ │ │ ├── test_local_dependency.py │ │ │ ├── test_simple_dependency.py │ │ │ ├── test_simple_dependency_with_configs.py │ │ │ └── test_uninstalled_package_found_error.py │ │ ├── deprecations/ │ │ │ ├── fixtures.py │ │ │ ├── test_config_deprecations.py │ │ │ ├── test_deprecations.py │ │ │ ├── test_missing_plus_in_config_deprecations.py │ │ │ └── test_model_deprecations.py │ │ ├── deps/ │ │ │ └── test_deps_with_vars.py │ │ ├── docs/ │ │ │ ├── test_doc_blocks_backcompat.py │ │ │ ├── test_doc_blocks_formatting.py │ │ │ ├── test_doc_concat_arg.py │ │ │ ├── test_doc_variable_arg.py │ │ │ ├── test_duplicate_docs_block.py │ │ │ ├── test_generate.py │ │ │ ├── test_good_docs_blocks.py │ │ │ ├── test_invalid_doc_ref.py │ │ │ ├── test_missing_docs_blocks.py │ │ │ ├── test_model_version_docs_blocks.py │ │ │ └── test_static.py │ │ ├── duplicates/ │ │ │ ├── test_duplicate_analysis.py │ │ │ ├── test_duplicate_exposure.py │ │ │ ├── test_duplicate_macro.py │ │ │ ├── test_duplicate_metric.py │ │ │ ├── test_duplicate_model.py │ │ │ ├── test_duplicate_resource.py │ │ │ ├── test_duplicate_resource_names.py │ │ │ └── test_duplicate_source.py │ │ ├── events/ │ │ │ └── events.py │ │ ├── exit_codes/ │ │ │ ├── fixtures.py │ │ │ └── test_exit_codes.py │ │ ├── experimental_parser/ │ │ │ └── test_all_experimental_parser.py │ │ ├── exposures/ │ │ │ ├── fixtures.py │ │ │ ├── test_exposure_configs.py │ │ │ └── test_exposures.py │ │ ├── external_reference/ │ │ │ └── test_external_reference.py │ │ ├── fail_fast/ │ │ │ └── test_fail_fast_run.py │ │ ├── fixtures/ │ │ │ ├── __init__.py │ │ │ ├── happy_path_fixture.py │ │ │ └── happy_path_project/ │ │ │ ├── analyses/ │ │ │ │ ├── a.sql │ │ │ │ └── a.yml │ │ │ ├── dbt_project.yml │ │ │ ├── functions/ │ │ │ │ ├── area_of_circle.sql │ │ │ │ └── area_of_circle.yml │ │ │ ├── macros/ │ │ │ │ ├── expression_is_true.sql │ │ │ │ └── macro_stuff.sql │ │ │ ├── models/ │ │ │ │ ├── docs.md │ │ │ │ ├── e.yml │ │ │ │ ├── ephemeral.sql │ │ │ │ ├── g.yml │ │ │ │ ├── incremental.sql │ │ │ │ ├── m.yml │ │ │ │ ├── macros.yml │ │ │ │ ├── metricflow_time_spine.sql │ │ │ │ ├── metricflow_time_spine_second.sql │ │ │ │ ├── model_to_unit_test.sql │ │ │ │ ├── model_with_lots_of_schema_configs.sql │ │ │ │ ├── outer.sql │ │ │ │ ├── schema.yml │ │ │ │ ├── sm.yml │ │ │ │ ├── snapshot_source.sql │ │ │ │ ├── sq.yml │ │ │ │ ├── sub/ │ │ │ │ │ └── inner.sql │ │ │ │ └── unit_tests.yml │ │ │ ├── seeds/ │ │ │ │ ├── s.yml │ │ │ │ └── seed.csv │ │ │ ├── snapshots/ │ │ │ │ ├── snapshot.sql │ │ │ │ ├── snapshot_2.yml │ │ │ │ └── snapshot_3.yml │ │ │ └── tests/ │ │ │ ├── fixtures/ │ │ │ │ └── test_incremental_fixture.csv │ │ │ ├── generic/ │ │ │ │ └── my_generic_test.sql │ │ │ ├── schema.yml │ │ │ └── t.sql │ │ ├── fixtures.py │ │ ├── functions/ │ │ │ ├── test_udafs.py │ │ │ └── test_udfs.py │ │ ├── generic_test_description/ │ │ │ ├── fixtures.py │ │ │ └── test_generic_test_description.py │ │ ├── graph_selection/ │ │ │ ├── data/ │ │ │ │ ├── seed.csv │ │ │ │ └── summary_expected.csv │ │ │ ├── fixtures.py │ │ │ ├── test_graph_selection.py │ │ │ ├── test_group_selection.py │ │ │ ├── test_inline.py │ │ │ ├── test_intersection_syntax.py │ │ │ ├── test_schema_test_graph_selection.py │ │ │ ├── test_tag_selection.py │ │ │ └── test_version_selection.py │ │ ├── incremental_schema_tests/ │ │ │ ├── fixtures.py │ │ │ └── test_incremental_schema.py │ │ ├── init/ │ │ │ └── test_init.py │ │ ├── invalid_model_tests/ │ │ │ ├── test_invalid_models.py │ │ │ └── test_model_logging.py │ │ ├── list/ │ │ │ ├── fixtures.py │ │ │ ├── test_commands.py │ │ │ └── test_list.py │ │ ├── logging/ │ │ │ ├── test_logging.py │ │ │ └── test_meta_logging.py │ │ ├── macros/ │ │ │ ├── data/ │ │ │ │ └── seed.sql │ │ │ ├── fixtures.py │ │ │ ├── package_macro_overrides/ │ │ │ │ ├── dbt_project.yml │ │ │ │ └── macros/ │ │ │ │ └── macros.sql │ │ │ ├── test_macro_annotations.py │ │ │ └── test_macros.py │ │ ├── manifest_validations/ │ │ │ └── test_check_for_spaces_in_model_names.py │ │ ├── materializations/ │ │ │ ├── conftest.py │ │ │ ├── fixtures.py │ │ │ ├── test_custom_materialization.py │ │ │ ├── test_ephemeral_compilation.py │ │ │ ├── test_incremental.py │ │ │ ├── test_incremental_with_contract.py │ │ │ ├── test_runtime_materialization.py │ │ │ └── test_supported_languages.py │ │ ├── metrics/ │ │ │ ├── fixtures.py │ │ │ ├── test_metric_configs.py │ │ │ ├── test_metric_deferral.py │ │ │ ├── test_metric_helper_functions.py │ │ │ └── test_metrics.py │ │ ├── microbatch/ │ │ │ ├── test_microbatch.py │ │ │ └── test_microbatch_config_validation.py │ │ ├── minimal_cli/ │ │ │ ├── fixtures.py │ │ │ └── test_minimal_cli.py │ │ ├── model_config/ │ │ │ └── test_freshness_config.py │ │ ├── partial_parsing/ │ │ │ ├── fixtures.py │ │ │ ├── test_file_diff.py │ │ │ ├── test_partial_parsing.py │ │ │ ├── test_pp_disabled_config.py │ │ │ ├── test_pp_docs.py │ │ │ ├── test_pp_functions.py │ │ │ ├── test_pp_groups.py │ │ │ ├── test_pp_metrics.py │ │ │ ├── test_pp_schema_file_order.py │ │ │ ├── test_pp_semantic_models.py │ │ │ ├── test_pp_undefined_serialization.py │ │ │ ├── test_pp_vars.py │ │ │ └── test_versioned_models.py │ │ ├── permission/ │ │ │ ├── data/ │ │ │ │ └── seed.sql │ │ │ └── fixtures.py │ │ ├── postgres/ │ │ │ ├── fixtures.py │ │ │ ├── test_postgres_indexes.py │ │ │ └── test_postgres_unlogged_table.py │ │ ├── primary_keys/ │ │ │ ├── fixtures.py │ │ │ └── test_primary_keys.py │ │ ├── profiles/ │ │ │ ├── test_profile_dir.py │ │ │ └── test_profiles_yml.py │ │ ├── record/ │ │ │ └── test_record.py │ │ ├── ref_override/ │ │ │ ├── test_custom_ref_kwargs.py │ │ │ └── test_ref_override.py │ │ ├── relation_names/ │ │ │ └── test_relation_name.py │ │ ├── relation_quoting/ │ │ │ └── test_relation_quoting.py │ │ ├── retry/ │ │ │ ├── fixtures.py │ │ │ ├── test_retry.py │ │ │ └── test_retry_threads.py │ │ ├── run_operations/ │ │ │ ├── fixtures.py │ │ │ └── test_run_operations.py │ │ ├── run_query/ │ │ │ └── test_types.py │ │ ├── sample_mode/ │ │ │ └── test_sample_mode.py │ │ ├── saved_queries/ │ │ │ ├── __init__.py │ │ │ ├── fixtures.py │ │ │ ├── test_configs.py │ │ │ ├── test_saved_query_build.py │ │ │ └── test_saved_query_parsing.py │ │ ├── schema/ │ │ │ ├── fixtures/ │ │ │ │ ├── macros.py │ │ │ │ └── sql.py │ │ │ └── test_custom_schema.py │ │ ├── schema_tests/ │ │ │ ├── data/ │ │ │ │ ├── seed.sql │ │ │ │ └── seed_failure.sql │ │ │ ├── fixtures.py │ │ │ ├── test_custom_test_config.py │ │ │ ├── test_schema_v2_tests.py │ │ │ └── test_sql_header_config.py │ │ ├── seeds/ │ │ │ └── test_seed_column_type_validation.py │ │ ├── selected_resources/ │ │ │ ├── fixtures.py │ │ │ └── test_selected_resources.py │ │ ├── selectors/ │ │ │ ├── test_default_selectors.py │ │ │ └── test_selector_selector_method.py │ │ ├── semantic_models/ │ │ │ ├── fixtures.py │ │ │ ├── test_semantic_model_configs.py │ │ │ ├── test_semantic_model_parsing.py │ │ │ ├── test_semantic_model_v2_parsing.py │ │ │ └── test_semantic_models.py │ │ ├── severity/ │ │ │ └── test_severity.py │ │ ├── show/ │ │ │ ├── fixtures.py │ │ │ └── test_show.py │ │ ├── snapshots/ │ │ │ ├── data/ │ │ │ │ ├── invalidate_postgres.sql │ │ │ │ ├── seed_cn.sql │ │ │ │ ├── seed_dbt_valid_to.sql │ │ │ │ ├── seed_pg.sql │ │ │ │ ├── shared_macros.sql │ │ │ │ └── update.sql │ │ │ ├── fixtures.py │ │ │ ├── test_basic_snapshot.py │ │ │ ├── test_changing_check_cols_snapshot.py │ │ │ ├── test_changing_strategy_snapshot.py │ │ │ ├── test_check_cols_snapshot.py │ │ │ ├── test_check_cols_updated_at_snapshot.py │ │ │ ├── test_comment_ending_snapshot.py │ │ │ ├── test_cross_schema_snapshot.py │ │ │ ├── test_hard_delete_snapshot.py │ │ │ ├── test_invalid_namespace_snapshot.py │ │ │ ├── test_long_text_snapshot.py │ │ │ ├── test_missing_strategy_snapshot.py │ │ │ ├── test_renamed_source_snapshot.py │ │ │ ├── test_select_exclude_snapshot.py │ │ │ ├── test_slow_query_snapshot.py │ │ │ ├── test_snapshot_column_names.py │ │ │ ├── test_snapshot_config.py │ │ │ ├── test_snapshot_empty.py │ │ │ └── test_snapshot_timestamps.py │ │ ├── source_overrides/ │ │ │ ├── fixtures.py │ │ │ ├── test_simple_source_override.py │ │ │ └── test_source_overrides_duplicate_model.py │ │ ├── sources/ │ │ │ ├── common_source_setup.py │ │ │ ├── data/ │ │ │ │ └── seed.sql │ │ │ ├── fixtures.py │ │ │ ├── test_name_chars.py │ │ │ ├── test_simple_source.py │ │ │ ├── test_source_configs.py │ │ │ ├── test_source_fresher_state.py │ │ │ ├── test_source_freshness.py │ │ │ └── test_source_loaded_at_field.py │ │ ├── statements/ │ │ │ ├── fixtures.py │ │ │ └── test_statements.py │ │ ├── test_empty.py │ │ ├── test_project.py │ │ ├── test_selection/ │ │ │ ├── fixtures.py │ │ │ └── test_selection_expansion.py │ │ ├── test_singular_tests.py │ │ ├── threading/ │ │ │ └── test_thread_count.py │ │ ├── time_spines/ │ │ │ ├── fixtures.py │ │ │ └── test_time_spines.py │ │ ├── timezones/ │ │ │ └── test_timezones.py │ │ ├── unit_testing/ │ │ │ ├── fixtures.py │ │ │ ├── test_csv_fixtures.py │ │ │ ├── test_sql_format.py │ │ │ ├── test_state.py │ │ │ ├── test_unit_testing.py │ │ │ ├── test_ut_adapter_hooks.py │ │ │ ├── test_ut_aliases.py │ │ │ ├── test_ut_dependency.py │ │ │ ├── test_ut_diffing.py │ │ │ ├── test_ut_ephemeral.py │ │ │ ├── test_ut_list.py │ │ │ ├── test_ut_macros.py │ │ │ ├── test_ut_names.py │ │ │ ├── test_ut_overrides.py │ │ │ ├── test_ut_resource_types.py │ │ │ ├── test_ut_snapshot_dependency.py │ │ │ ├── test_ut_sources.py │ │ │ ├── test_ut_variables.py │ │ │ └── test_ut_versions.py │ │ └── utils.py │ └── unit/ │ ├── README.md │ ├── __init__.py │ ├── artifacts/ │ │ ├── test_base_resource.py │ │ └── test_run_execution_result.py │ ├── cli/ │ │ ├── test_flags.py │ │ ├── test_main.py │ │ └── test_option_types.py │ ├── clients/ │ │ ├── __init__.py │ │ ├── test_jinja.py │ │ ├── test_jinja_static.py │ │ ├── test_registry.py │ │ └── test_yaml_helper.py │ ├── config/ │ │ ├── __init__.py │ │ ├── test_profile.py │ │ ├── test_project.py │ │ ├── test_renderer_with_vars.py │ │ ├── test_runtime.py │ │ ├── test_selectors.py │ │ ├── test_utils.py │ │ └── test_vars_file.py │ ├── conftest.py │ ├── context/ │ │ ├── __init__.py │ │ ├── test_base.py │ │ ├── test_context.py │ │ ├── test_macro_resolver.py │ │ ├── test_providers.py │ │ └── test_query_header.py │ ├── contracts/ │ │ ├── __init__.py │ │ ├── files/ │ │ │ └── test_schema_source_file.py │ │ ├── graph/ │ │ │ ├── __init__.py │ │ │ ├── test_manifest.py │ │ │ ├── test_node_args.py │ │ │ ├── test_nodes.py │ │ │ ├── test_nodes_parsed.py │ │ │ ├── test_semantic_manifest.py │ │ │ ├── test_udfs.py │ │ │ └── test_unparsed.py │ │ └── test_project.py │ ├── deps/ │ │ ├── __init__.py │ │ └── test_deps.py │ ├── event_time/ │ │ ├── test_event_time.py │ │ └── test_sample_mode.py │ ├── events/ │ │ ├── __init__.py │ │ ├── test_logging.py │ │ └── test_types.py │ ├── fixtures.py │ ├── graph/ │ │ ├── __init__.py │ │ ├── test_cli.py │ │ ├── test_graph.py │ │ ├── test_nodes.py │ │ ├── test_queue.py │ │ ├── test_selector.py │ │ ├── test_selector_methods.py │ │ └── test_selector_spec.py │ ├── materializations/ │ │ └── incremental/ │ │ └── test_microbatch.py │ ├── mock_adapter.py │ ├── parser/ │ │ ├── __init__.py │ │ ├── test_docs.py │ │ ├── test_get_doc_blocks.py │ │ ├── test_manifest.py │ │ ├── test_parser.py │ │ ├── test_partial.py │ │ ├── test_read_files.py │ │ ├── test_schema_renderer.py │ │ ├── test_sources.py │ │ ├── test_unit_tests.py │ │ └── test_v2_column_semantic_parsing.py │ ├── plugins/ │ │ └── test_manager.py │ ├── task/ │ │ ├── __init__.py │ │ ├── docs/ │ │ │ ├── __init__.py │ │ │ └── test_serve.py │ │ ├── test_base.py │ │ ├── test_build.py │ │ ├── test_clone.py │ │ ├── test_docs.py │ │ ├── test_freshness.py │ │ ├── test_list.py │ │ ├── test_retry.py │ │ ├── test_run.py │ │ └── test_test.py │ ├── test_artifact_upload.py │ ├── test_behavior_flags.py │ ├── test_compilation.py │ ├── test_compilation_threading.py │ ├── test_deprecations.py │ ├── test_env_vars.py │ ├── test_events.py │ ├── test_functions.py │ ├── test_graph_selection.py │ ├── test_internal_deprecations.py │ ├── test_invocation_id.py │ ├── test_jsonschemas.py │ ├── test_macro_types.py │ ├── test_node_types.py │ ├── test_semantic_layer_nodes_satisfy_protocols.py │ ├── test_tracking.py │ ├── test_utils.py │ ├── test_version.py │ └── utils/ │ ├── __init__.py │ ├── adapter.py │ ├── config.py │ ├── event_manager.py │ ├── flags.py │ ├── manifest.py │ └── project.py └── third-party-stubs/ ├── agate/ │ ├── __init__.pyi │ └── data_types.pyi ├── cdecimal/ │ └── __init__.pyi ├── daff/ │ └── __init__.pyi ├── mashumaro/ │ ├── __init__.pyi │ ├── config.pyi │ ├── core/ │ │ ├── __init__.pyi │ │ ├── const.pyi │ │ ├── helpers.pyi │ │ └── meta/ │ │ ├── __init__.pyi │ │ ├── code/ │ │ │ ├── __init__.pyi │ │ │ ├── builder.pyi │ │ │ └── lines.pyi │ │ ├── helpers.pyi │ │ ├── mixin.pyi │ │ └── types/ │ │ ├── __init__.pyi │ │ ├── common.pyi │ │ ├── pack.pyi │ │ └── unpack.pyi │ ├── dialect.pyi │ ├── exceptions.pyi │ ├── helper.pyi │ ├── jsonschema/ │ │ ├── __init__.pyi │ │ ├── annotations.pyi │ │ ├── builder.pyi │ │ ├── dialects.pyi │ │ ├── models.pyi │ │ └── schema.pyi │ ├── mixins/ │ │ ├── __init__.pyi │ │ ├── dict.pyi │ │ ├── json.pyi │ │ ├── msgpack.pyi │ │ ├── orjson.pyi │ │ ├── toml.pyi │ │ └── yaml.pyi │ └── types.pyi ├── msgpack/ │ ├── __init__.pyi │ ├── _version.pyi │ ├── exceptions.pyi │ ├── ext.pyi │ └── fallback.pyi ├── snowplow_tracker/ │ └── __init__.pyi └── sqlparse/ ├── __init__.pyi ├── sql.pyi └── tokens.pyi ================================================ FILE CONTENTS ================================================ ================================================ FILE: .changes/0.0.0.md ================================================ ## Previous Releases For information on prior major and minor releases, see their changelogs: * [1.10](https://github.com/dbt-labs/dbt-core/blob/1.10.latest/CHANGELOG.md) * [1.9](https://github.com/dbt-labs/dbt-core/blob/1.9.latest/CHANGELOG.md) * [1.8](https://github.com/dbt-labs/dbt-core/blob/1.8.latest/CHANGELOG.md) * [1.7](https://github.com/dbt-labs/dbt-core/blob/1.7.latest/CHANGELOG.md) * [1.6](https://github.com/dbt-labs/dbt-core/blob/1.6.latest/CHANGELOG.md) * [1.5](https://github.com/dbt-labs/dbt-core/blob/1.5.latest/CHANGELOG.md) * [1.4](https://github.com/dbt-labs/dbt-core/blob/1.4.latest/CHANGELOG.md) * [1.3](https://github.com/dbt-labs/dbt-core/blob/1.3.latest/CHANGELOG.md) * [1.2](https://github.com/dbt-labs/dbt-core/blob/1.2.latest/CHANGELOG.md) * [1.1](https://github.com/dbt-labs/dbt-core/blob/1.1.latest/CHANGELOG.md) * [1.0](https://github.com/dbt-labs/dbt-core/blob/1.0.latest/CHANGELOG.md) * [0.21](https://github.com/dbt-labs/dbt-core/blob/0.21.latest/CHANGELOG.md) * [0.20](https://github.com/dbt-labs/dbt-core/blob/0.20.latest/CHANGELOG.md) * [0.19](https://github.com/dbt-labs/dbt-core/blob/0.19.latest/CHANGELOG.md) * [0.18](https://github.com/dbt-labs/dbt-core/blob/0.18.latest/CHANGELOG.md) * [0.17](https://github.com/dbt-labs/dbt-core/blob/0.17.latest/CHANGELOG.md) * [0.16](https://github.com/dbt-labs/dbt-core/blob/0.16.latest/CHANGELOG.md) * [0.15](https://github.com/dbt-labs/dbt-core/blob/0.15.latest/CHANGELOG.md) * [0.14](https://github.com/dbt-labs/dbt-core/blob/0.14.latest/CHANGELOG.md) * [0.13](https://github.com/dbt-labs/dbt-core/blob/0.13.latest/CHANGELOG.md) * [0.12](https://github.com/dbt-labs/dbt-core/blob/0.12.latest/CHANGELOG.md) * [0.11 and earlier](https://github.com/dbt-labs/dbt-core/blob/0.11.latest/CHANGELOG.md) ================================================ FILE: .changes/README.md ================================================ # CHANGELOG Automation We use [changie](https://changie.dev/) to automate `CHANGELOG` generation. For installation and format/command specifics, see the documentation. ### Quick Tour - All new change entries get generated under `/.changes/unreleased` as a yaml file - `header.tpl.md` contains the contents of the entire CHANGELOG file - `0.0.0.md` contains the contents of the footer for the entire CHANGELOG file. changie looks to be in the process of supporting a footer file the same as it supports a header file. Switch to that when available. For now, the 0.0.0 in the file name forces it to the bottom of the changelog no matter what version we are releasing. - `.changie.yaml` contains the fields in a change, the format of a single change, as well as the format of the Contributors section for each version. ### Workflow #### Daily workflow Almost every code change we make associated with an issue will require a `CHANGELOG` entry. After you have created the PR in GitHub, run `changie new` and follow the command prompts to generate a yaml file with your change details. This only needs to be done once per PR. The `changie new` command will ensure correct file format and file name. There is a one to one mapping of issues to changes. Multiple issues cannot be lumped into a single entry. If you make a mistake, the yaml file may be directly modified and saved as long as the format is preserved. Note: If your PR has been cleared by the Core Team as not needing a changelog entry, the `Skip Changelog` label may be put on the PR to bypass the GitHub action that blacks PRs from being merged when they are missing a `CHANGELOG` entry. #### Prerelease Workflow These commands batch up changes in `/.changes/unreleased` to be included in this prerelease and move those files to a directory named for the release version. The `--move-dir` will be created if it does not exist and is created in `/.changes`. ``` changie batch --move-dir '' --prerelease 'rc1' changie merge ``` Example ``` changie batch 1.0.5 --move-dir '1.0.5' --prerelease 'rc1' changie merge ``` #### Final Release Workflow These commands batch up changes in `/.changes/unreleased` as well as `/.changes/` to be included in this final release and delete all prereleases. This rolls all prereleases up into a single final release. All `yaml` files in `/unreleased` and `` will be deleted at this point. ``` changie batch --include '' --remove-prereleases changie merge ``` Example ``` changie batch 1.0.5 --include '1.0.5' --remove-prereleases changie merge ``` ### A Note on Manual Edits & Gotchas - Changie generates markdown files in the `.changes` directory that are parsed together with the `changie merge` command. Every time `changie merge` is run, it regenerates the entire file. For this reason, any changes made directly to `CHANGELOG.md` will be overwritten on the next run of `changie merge`. - If changes need to be made to the `CHANGELOG.md`, make the changes to the relevant `.md` file located in the `/.changes` directory. You will then run `changie merge` to regenerate the `CHANGELOG.MD`. - Do not run `changie batch` again on released versions. Our final release workflow deletes all of the yaml files associated with individual changes. If for some reason modifications to the `CHANGELOG.md` are required after we've generated the final release `CHANGELOG.md`, the modifications need to be done manually to the `.md` file in the `/.changes` directory. - changie can modify, create and delete files depending on the command you run. This is expected. Be sure to commit everything that has been modified and deleted. ================================================ FILE: .changes/header.tpl.md ================================================ # dbt Core Changelog - This file provides a full account of all changes to `dbt-core` - Changes are listed under the (pre)release in which they first appear. Subsequent releases include changes from previous releases. - "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version. - Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-core/blob/main/CONTRIBUTING.md#adding-changelog-entry) ================================================ FILE: .changes/unreleased/.gitkeep ================================================ ================================================ FILE: .changes/unreleased/Dependencies-20251118-155354.yaml ================================================ kind: Dependencies body: Use EventCatcher from dbt-common instead of maintaining a local copy time: 2025-11-18T15:53:54.284561+05:30 custom: Author: 3loka Issue: "12124" ================================================ FILE: .changes/unreleased/Dependencies-20251217-151349.yaml ================================================ kind: Dependencies body: Upgrading dbt-semantic-interfaces to 0.10.x time: 2025-12-17T15:13:49.721737-05:00 custom: Author: WilliamDee Issue: None ================================================ FILE: .changes/unreleased/Dependencies-20251219-153804.yaml ================================================ kind: Dependencies body: Bump minimum click to 8.2.0 time: 2025-12-19T15:38:04.785842-06:00 custom: Author: QMalcolm Issue: "12305" ================================================ FILE: .changes/unreleased/Dependencies-20260113-135442.yaml ================================================ kind: Dependencies body: Bump DSI minimum to 0.10.2 time: 2026-01-13T13:54:42.340193-06:00 custom: Author: QMalcolm Issue: NA ================================================ FILE: .changes/unreleased/Dependencies-20260113-143057.yaml ================================================ kind: Dependencies body: Bump dbt-protos minimum to 1.0.418 time: 2026-01-13T14:30:57.311017-06:00 custom: Author: QMalcolm Issue: NA ================================================ FILE: .changes/unreleased/Dependencies-20260121-171712.yaml ================================================ kind: Dependencies body: Move `click` minimum to 8.3.0 time: 2026-01-21T17:17:12.251254-06:00 custom: Author: QMalcolm Issue: "12378" ================================================ FILE: .changes/unreleased/Dependencies-20260302-130425.yaml ================================================ kind: Dependencies body: Bump minimum version of dbt-common to 1.37.3 time: 2026-03-02T13:04:25.495699-05:00 custom: Author: emmyoop Issue: "12575" ================================================ FILE: .changes/unreleased/Docs-20240311-140344.yaml ================================================ kind: Docs body: Enable display of unit tests time: 2024-03-11T14:03:44.490834-04:00 custom: Author: gshank Issue: "501" ================================================ FILE: .changes/unreleased/Docs-20240501-021050.yaml ================================================ kind: Docs body: Unit tests not rendering time: 2024-05-01T02:10:50.987412+02:00 custom: Author: aranke Issue: "506" ================================================ FILE: .changes/unreleased/Docs-20240516-223036.yaml ================================================ kind: Docs body: Add support for Saved Query node time: 2024-05-16T22:30:36.206492-07:00 custom: Author: ChenyuLInx Issue: "486" ================================================ FILE: .changes/unreleased/Docs-20240613-151048.yaml ================================================ kind: Docs body: Fix npm security vulnerabilities as of June 2024 time: 2024-06-13T15:10:48.301989+01:00 custom: Author: aranke Issue: "513" ================================================ FILE: .changes/unreleased/Docs-20250728-162542.yaml ================================================ kind: Docs body: Bump form-data from 3.0.1 to 3.0.4 time: 2025-07-28T16:25:42.452808-04:00 custom: Author: michelleark Issue: "554" ================================================ FILE: .changes/unreleased/Docs-20260319-112630.yaml ================================================ kind: Docs body: Add support for UDF (function) resource type in lineage graph time: 2026-03-19T11:26:30.338385+05:30 custom: Author: aahel Issue: "574" ================================================ FILE: .changes/unreleased/Features-20251006-140352.yaml ================================================ kind: Features body: Support partial parsing for function nodes time: 2025-10-06T14:03:52.258104-05:00 custom: Author: QMalcolm Issue: "12072" ================================================ FILE: .changes/unreleased/Features-20251111-103504.yaml ================================================ kind: Features body: Add UnparsedMetricV2 to read in new-style YAML Semantic Layer Metrics. time: 2025-11-11T10:35:04.123144-08:00 custom: Author: theyostalservice Issue: "12157" ================================================ FILE: .changes/unreleased/Features-20251117-141053.yaml ================================================ kind: Features body: Allow for defining funciton arguments with default values time: 2025-11-17T14:10:53.860178-06:00 custom: Author: QMalcolm Issue: "12044" ================================================ FILE: .changes/unreleased/Features-20251201-165209.yaml ================================================ kind: Features body: Raise jsonschema-based deprecation warnings by default time: 2025-12-01T16:52:09.354436-05:00 custom: Author: michelleark Issue: 12240 ================================================ FILE: .changes/unreleased/Features-20251203-122926.yaml ================================================ kind: Features body: ':bug: :snowman: Disable unit tests whose model is disabled' time: 2025-12-03T12:29:26.209248-05:00 custom: Author: michelleark Issue: "10540" ================================================ FILE: .changes/unreleased/Features-20251210-202001.yaml ================================================ kind: Features body: Implement config.meta_get and config.meta_require time: 2025-12-10T20:20:01.354288-05:00 custom: Author: gshank Issue: "12012" ================================================ FILE: .changes/unreleased/Features-20260115-131115.yaml ================================================ kind: Features body: Deprecate null return values from generate_schema_name macro, behind require_valid_schema_from_generate_schema_name flag time: 2026-01-15T13:11:15.787477-05:00 custom: Author: michelleark Issue: "12347" ================================================ FILE: .changes/unreleased/Features-20260119-210143.yaml ================================================ kind: Features body: Add ability to indicate dbt Model also represents a Semantic Model. (Not fully supported yet.) time: 2026-01-19T21:01:43.917805-08:00 custom: Author: theyostalservice Issue: . ================================================ FILE: .changes/unreleased/Features-20260123-173805.yaml ================================================ kind: Features body: Added ability to parse semantic layer dimensions added to columns as part of new YAML. (Not ready for use until more of the new yaml is implemented.) time: 2026-01-23T17:38:05.847998-08:00 custom: Author: theyostalservice Issue: . ================================================ FILE: .changes/unreleased/Features-20260126-164458.yaml ================================================ kind: Features body: Add parsing for new YAML for semantic layer entities attached to dbt models. time: 2026-01-26T16:44:58.041234-08:00 custom: Author: theyostalservice Issue: "12395" ================================================ FILE: .changes/unreleased/Features-20260127-114149.yaml ================================================ kind: Features body: Process semantic metrics in v2 YAML if they are not merged into a model. time: 2026-01-27T11:41:49.912438-08:00 custom: Author: theyostalservice Issue: "12161" ================================================ FILE: .changes/unreleased/Features-20260127-214617.yaml ================================================ kind: Features body: Add parsing for v2 metrics on models and finish implementing their dependency behaviors. time: 2026-01-27T21:46:17.985044-08:00 custom: Author: theyostalservice Issue: "12397" ================================================ FILE: .changes/unreleased/Features-20260128-125727.yaml ================================================ kind: Features body: Add the ability to process derived semantic entities to dbt models, as required by new YAML schema. time: 2026-01-28T12:57:27.576354-08:00 custom: Author: theyostalservice Issue: "12401" ================================================ FILE: .changes/unreleased/Features-20260128-172052.yaml ================================================ kind: Features body: Enable parsing derived dimensions for v2 semantic layer YAML. time: 2026-01-28T17:20:52.400188-08:00 custom: Author: theyostalservice Issue: "12404" ================================================ FILE: .changes/unreleased/Features-20260129-110711.yaml ================================================ kind: Features body: Add unit tests to the Jinja `graph` object, enabling tools like dbt-project-evaluator to run checks on unit tests. time: 2026-01-29T11:00:00.000000-08:00 custom: Author: b-per Issue: "12033" ================================================ FILE: .changes/unreleased/Features-20260129-114832.yaml ================================================ kind: Features body: Implement agg_time_dimension for new semantic YAML. time: 2026-01-29T11:48:32.100015-08:00 custom: Author: theyostalservice Issue: "12410" ================================================ FILE: .changes/unreleased/Features-20260129-175432.yaml ================================================ kind: Features body: Implement parsing of semantic model object-style configuration in v2 semantic YAML. time: 2026-01-29T17:54:32.188082-08:00 custom: Author: theyostalservice Issue: "12413" ================================================ FILE: .changes/unreleased/Features-20260129-185659.yaml ================================================ kind: Features body: Implement primary_entity field for semantic models in semantic YAML v2. time: 2026-01-29T18:56:59.184109-08:00 custom: Author: theyostalservice Issue: "12414" ================================================ FILE: .changes/unreleased/Features-20260205-105228.yaml ================================================ kind: Features body: add config.meta_get to python model parsing time: 2026-02-05T10:52:28.269954-08:00 custom: Author: venkaa28 Issue: "12458" ================================================ FILE: .changes/unreleased/Features-20260212-232728.yaml ================================================ kind: Features body: Added support for vars.yml to declare project variables time: 2026-02-12T23:27:28.294557+05:30 custom: Author: sriramr98 Issue: 11144 2955 ================================================ FILE: .changes/unreleased/Features-20260219-133321.yaml ================================================ kind: Features body: execute dbt debug logic after creating a new project in dbt init time: 2026-02-19T13:33:21.891997+05:30 custom: Author: sriramr98 Issue: "12510" ================================================ FILE: .changes/unreleased/Features-20260302-123311.yaml ================================================ kind: Features body: Write compiled SQL for snapshots to target/compiled/ during dbt compile time: 2026-03-02T12:33:11.456829+05:30 custom: Author: aahel Issue: "7867" ================================================ FILE: .changes/unreleased/Features-20260303-214328.yaml ================================================ kind: Features body: Add "selector" selector method time: 2026-03-03T21:43:28.02659+05:30 custom: Author: ash2shukla Issue: "5009" ================================================ FILE: .changes/unreleased/Features-9041.yaml ================================================ kind: Features body: Add directory change instruction after dbt init time: 2025-11-20T00:00:00Z custom: Author: kalluripradeep Issue: "9041" ================================================ FILE: .changes/unreleased/Fixes-20250922-151726.yaml ================================================ kind: Fixes body: Address Click 8.2+ deprecation warning time: 2025-09-22T15:17:26.983151-06:00 custom: Author: edgarrmondragon Issue: "12038" ================================================ FILE: .changes/unreleased/Fixes-20251117-140649.yaml ================================================ kind: Fixes body: Include macros in unit test parsing time: 2025-11-17T14:06:49.518566-05:00 custom: Author: michelleark nathanskone Issue: "10157" ================================================ FILE: .changes/unreleased/Fixes-20251117-185025.yaml ================================================ kind: Fixes body: Allow dbt deps to run when vars lack defaults in dbt_project.yml time: 2025-11-17T18:50:25.759091+05:30 custom: Author: 3loka Issue: "8913" ================================================ FILE: .changes/unreleased/Fixes-20251118-171106.yaml ================================================ kind: Fixes body: Restore DuplicateResourceNameError for intra-project node name duplication, behind behavior flag `require_unique_project_resource_names` time: 2025-11-18T17:11:06.454784-05:00 custom: Author: michelleark Issue: "12152" ================================================ FILE: .changes/unreleased/Fixes-20251119-195034.yaml ================================================ kind: Fixes body: Allow the usage of `function` with `--exclude-resource-type` flag time: 2025-11-19T19:50:34.703236-06:00 custom: Author: QMalcolm Issue: "12143" ================================================ FILE: .changes/unreleased/Fixes-20251124-155629.yaml ================================================ kind: Fixes body: Fix bug where schemas of functions weren't guaranteed to exist time: 2025-11-24T15:56:29.467004-06:00 custom: Author: QMalcolm Issue: "12142" ================================================ FILE: .changes/unreleased/Fixes-20251124-155756.yaml ================================================ kind: Fixes body: Fix generation of deprecations summary time: 2025-11-24T15:57:56.544123-08:00 custom: Author: asiunov Issue: "12146" ================================================ FILE: .changes/unreleased/Fixes-20251124-170855.yaml ================================================ kind: Fixes body: ':bug: :snowman: Correctly reference foreign key references when --defer and --state provided' time: 2025-11-24T17:08:55.387946-05:00 custom: Author: michellark Issue: "11885" ================================================ FILE: .changes/unreleased/Fixes-20251125-120246.yaml ================================================ kind: Fixes body: ':bug: :snowman: Add exception when using --state and referring to a removed test' time: 2025-11-25T12:02:46.635026-05:00 custom: Author: emmyoop Issue: "10630" ================================================ FILE: .changes/unreleased/Fixes-20251125-122020.yaml ================================================ kind: Fixes body: ':bug: :snowman: Stop emitting `NoNodesForSelectionCriteria` three times during `build` command' time: 2025-11-25T12:20:20.132379-06:00 custom: Author: QMalcolm Issue: "11627" ================================================ FILE: .changes/unreleased/Fixes-20251127-141308.yaml ================================================ kind: Fixes body: ":bug: :snowman: Fix long Python stack traces appearing when package dependencies have incompatible version requirements" time: 2025-11-27T14:13:08.082542-05:00 custom: Author: emmyoop Issue: "12049" ================================================ FILE: .changes/unreleased/Fixes-20251127-145929.yaml ================================================ kind: Fixes body: ':bug: :snowman: Fixed issue where changing data type size/precision/scale (e.g., varchar(3) to varchar(10)) incorrectly triggered a breaking change error fo' time: 2025-11-27T14:59:29.256274-05:00 custom: Author: emmyoop Issue: "11186" ================================================ FILE: .changes/unreleased/Fixes-20251127-170124.yaml ================================================ kind: Fixes body: ':bug: :snowman: Support unit testing models that depend on sources with the same name' time: 2025-11-27T17:01:24.193516-05:00 custom: Author: michelleark Issue: 11975 10433 ================================================ FILE: .changes/unreleased/Fixes-20251128-102129.yaml ================================================ kind: Fixes body: Fix bug in partial parsing when updating a model with a schema file that is referenced by a singular test time: 2025-11-28T10:21:29.911147Z custom: Author: mattogburke Issue: "12223" ================================================ FILE: .changes/unreleased/Fixes-20251128-122838.yaml ================================================ kind: Fixes body: ':bug: :snowman: Avoid retrying successful run-operation commands' time: 2025-11-28T12:28:38.546261-05:00 custom: Author: michelleark Issue: "11850" ================================================ FILE: .changes/unreleased/Fixes-20251128-161937.yaml ================================================ kind: Fixes body: ':bug: :snowman: Fix `dbt deps --add-package` crash when packages.yml contains `warn-unpinned: false`' time: 2025-11-28T16:19:37.608722-05:00 custom: Author: emmyoop Issue: "9104" ================================================ FILE: .changes/unreleased/Fixes-20251128-163144.yaml ================================================ kind: Fixes body: ':bug: :snowman: Improve `dbt deps --add-package` duplicate detection with better cross-source matching and word boundaries' time: 2025-11-28T16:31:44.344099-05:00 custom: Author: emmyoop Issue: "12239" ================================================ FILE: .changes/unreleased/Fixes-20251202-133705.yaml ================================================ kind: Fixes body: ':bug: :snowman: Fix false positive deprecation warning of pre/post-hook SQL configs' time: 2025-12-02T13:37:05.012112-05:00 custom: Author: michelleark Issue: "12244" ================================================ FILE: .changes/unreleased/Fixes-20251204-094753.yaml ================================================ kind: Fixes body: ':bug: :snowman: Fix ref resolution within package when duplicate nodes exist, behind require_ref_searches_node_package_before_root behavior change flag' time: 2025-12-04T09:47:53.349428-08:00 custom: Author: michelleark Issue: "11351" ================================================ FILE: .changes/unreleased/Fixes-20251209-175031.yaml ================================================ kind: Fixes body: Ensure recent deprecation warnings include event name in message time: 2025-12-09T17:50:31.334618-06:00 custom: Author: QMalcolm Issue: "12264" ================================================ FILE: .changes/unreleased/Fixes-20251210-143935.yaml ================================================ kind: Fixes body: Improve error message clarity when detecting nodes with space in name time: 2025-12-10T14:39:35.107841-08:00 custom: Author: michelleark Issue: "11835" ================================================ FILE: .changes/unreleased/Fixes-20251216-120727.yaml ================================================ kind: Fixes body: ':bug: :snowman:Propagate exceptions for NodeFinished callbacks in dbtRunner' time: 2025-12-16T12:07:27.576087-05:00 custom: Author: michelleark Issue: "11612" ================================================ FILE: .changes/unreleased/Fixes-20251217-002813.yaml ================================================ kind: Fixes body: Adds omitted return statement to RuntimeConfigObject.meta_require method time: 2025-12-17T00:28:13.015416197Z custom: Author: mjsqu Issue: "12288" ================================================ FILE: .changes/unreleased/Fixes-20251217-105918.yaml ================================================ kind: Fixes body: Do not raise deprecation warning when encountering dataset or project configs for bigquery time: 2025-12-17T10:59:18.372968-05:00 custom: Author: michelleark Issue: "12285" ================================================ FILE: .changes/unreleased/Fixes-20251219-184405.yaml ================================================ kind: Fixes body: Pin sqlparse <0.5.5 to avoid max tokens issue time: 2025-12-19T18:44:05.216329-05:00 custom: Author: michelleark Issue: "12303" ================================================ FILE: .changes/unreleased/Fixes-20260109-141332.yaml ================================================ kind: Fixes body: Avoid deadlock edgecases of concurret microbatch/batch execution time: 2026-01-09T14:13:32.777942-06:00 custom: Author: QMalcolm Issue: "11420" ================================================ FILE: .changes/unreleased/Fixes-20260113-161742.yaml ================================================ kind: Fixes body: Begin emitting deprecation warnings for DSI produced ValidationFutureError issues time: 2026-01-13T16:17:42.988355-06:00 custom: Author: QMalcolm Issue: NA ================================================ FILE: .changes/unreleased/Fixes-20260124-212300.yaml ================================================ kind: Fixes body: Add @requires.catalogs decorator to compile command to fix REST Catalog-Linked database compilation time: 2026-01-24T21:23:00.00000Z custom: Author: kalluripradeep Issue: "12353" ================================================ FILE: .changes/unreleased/Fixes-20260125-113244.yaml ================================================ kind: Fixes body: Validate and filter out non-existent columns from seed column types time: 2026-01-25T11:32:44.5200661+05:30 custom: Author: abhishek09827 Issue: 12379 ================================================ FILE: .changes/unreleased/Fixes-20260126-160659.yaml ================================================ kind: Fixes body: 'Fix false deprecation warning ' time: 2026-01-26T16:06:59.0157-05:00 custom: Author: alexaustin007 Issue: "12327" ================================================ FILE: .changes/unreleased/Fixes-20260202-123453.yaml ================================================ kind: Fixes body: Return correctly resolved project path when changing dirs time: 2026-02-02T12:34:53.454501+05:30 custom: Author: ash2shukla Issue: "9138" ================================================ FILE: .changes/unreleased/Fixes-20260202-153835.yaml ================================================ kind: Fixes body: Normalize whitespace prior to computing partial parsing checksum when loading model files time: 2026-02-02T15:38:35.220621-05:00 custom: Author: michelleark Issue: "12432" ================================================ FILE: .changes/unreleased/Fixes-20260204-184553.yaml ================================================ kind: Fixes body: Stop raising deprecation warnings for internal python model configs time: 2026-02-04T18:45:53.094578-06:00 custom: Author: QMalcolm Issue: "12314" ================================================ FILE: .changes/unreleased/Fixes-20260204-211128.yaml ================================================ kind: Fixes body: Dont fire config problem event if loaded_at_query is defined for adapters that dont support metadata-based freshness time: 2026-02-04T21:11:28.218659+05:30 custom: Author: ash2shukla Issue: "12451" ================================================ FILE: .changes/unreleased/Fixes-20260205-120000.yaml ================================================ kind: Fixes body: Fix foreign key constraint ref() resolving to deferred relation even when target model is selected for build time: 2026-02-05T12:00:00.00000Z custom: Author: b-per Issue: "12455" ================================================ FILE: .changes/unreleased/Fixes-20260206-204257.yaml ================================================ kind: Fixes body: Skip raising `CustomKeyInConfigDeprecation` for config key alias in sql time: 2026-02-06T20:42:57.134627+05:30 custom: Author: ash2shukla Issue: "12396" ================================================ FILE: .changes/unreleased/Fixes-20260210-154042.yaml ================================================ kind: Fixes body: Provide user-friendly validations that dimensions with 'validity_params' also have granularities. time: 2026-02-10T15:40:42.808774-08:00 custom: Author: theyostalservice Issue: "12473" ================================================ FILE: .changes/unreleased/Fixes-20260216-230817.yaml ================================================ kind: Fixes body: Use resolved profile and target names to allow partial parsing for default profile and target time: 2026-02-16T23:08:17.820862+05:30 custom: Author: akshatha-code71 ash2shukla Issue: "7612" ================================================ FILE: .changes/unreleased/Fixes-20260219-012414.yaml ================================================ kind: Fixes body: Ensure that all locked packages are installed in packages directory time: 2026-02-19T01:24:14.439093+05:30 custom: Author: ash2shukla Issue: 12509 ================================================ FILE: .changes/unreleased/Fixes-20260219-131833.yaml ================================================ kind: Fixes body: Correctly map 'hidden' field of metrics to 'is_private' field in manifests. time: 2026-02-19T13:18:33.912843-08:00 custom: Author: theyostalservice Issue: "12518" ================================================ FILE: .changes/unreleased/Fixes-20260219-170000.yaml ================================================ kind: Fixes body: Set expr to column name for column-based dimensions and entities when the semantic layer name differs from the column name, so MetricFlow queries the correct warehouse column time: 2026-02-19T17:00:00.000000+00:00 custom: Author: b-per Issue: "12512" ================================================ FILE: .changes/unreleased/Fixes-20260223-132342.yaml ================================================ kind: Fixes body: Fix Dimension() jinja in nested metric filters (input_metrics, numerator, denominator) being incorrectly rendered during YAML parsing for v2 semantic layer schema time: 2026-02-23T13:23:42.353928-08:00 custom: Author: theyostalservice Issue: 12529 ================================================ FILE: .changes/unreleased/Fixes-20260223-161820.yaml ================================================ kind: Fixes body: Fix dbt retry for microbatch models to use the original invocation time instead of the current time when recomputing batches time: 2026-02-23T16:18:20.714612+05:30 custom: Author: aahel Issue: "11423" ================================================ FILE: .changes/unreleased/Fixes-20260224-115546.yaml ================================================ kind: Fixes body: Fix v2 metric parsing using model name instead of custom semantic_model.name for generated_from, causing conversion metric validation to fail time: 2026-02-24T11:55:46.515181-08:00 custom: Author: theyostalservice Issue: "12532" ================================================ FILE: .changes/unreleased/Fixes-20260224-180000.yaml ================================================ kind: Fixes body: Fix doc() Jinja in derived_semantics dimension and entity descriptions being incorrectly rendered during v2 YAML parsing time: 2026-02-24T18:00:00.000000-08:00 custom: Author: theyostalservice Issue: 12535 ================================================ FILE: .changes/unreleased/Fixes-20260224-231047.yaml ================================================ kind: Fixes body: Improve logic for detecting config with missing plus prefix in dbt_project.yml time: 2026-02-24T23:10:47.240471+05:30 custom: Author: ash2shukla Issue: "12371" ================================================ FILE: .changes/unreleased/Fixes-20260225-203131.yaml ================================================ kind: Fixes body: Enable sql_header config for data tests gated behind require_sql_header_in_test_configs behavior change flag. time: 2026-02-25T20:31:31.854988+05:30 custom: Author: aahel Issue: "9775" ================================================ FILE: .changes/unreleased/Fixes-20260226-163836.yaml ================================================ kind: Fixes body: Better error formatting for semantic manifest validation errors time: 2026-02-26T16:38:36.858458+05:30 custom: Author: sriramr98 Issue: "9849" ================================================ FILE: .changes/unreleased/Fixes-20260227-133424.yaml ================================================ kind: Fixes body: Allow macros invoked via run-operation to ref() private and protected models time: 2026-02-27T13:34:24.449177+05:30 custom: Author: aahel Issue: 8248 ================================================ FILE: .changes/unreleased/Fixes-20260227-140009.yaml ================================================ kind: Fixes body: Better error and warnings logs with [WARNING] / [ERROR] messages prepended to the logs time: 2026-02-27T14:00:09.850547+05:30 custom: Author: sriramr98 Issue: "9849" ================================================ FILE: .changes/unreleased/Fixes-20260302-000739.yaml ================================================ kind: Fixes body: Add config and allow meta and docs to exist under it for macros time: 2026-03-02T00:07:39.088118+05:30 custom: Author: ash2shukla Issue: 12383 9447 ================================================ FILE: .changes/unreleased/Fixes-20260303-114528.yaml ================================================ kind: Fixes body: '`DBT_ENGINE` prefixed env vars picked up by CLI' time: 2026-03-03T11:45:28.403687-06:00 custom: Author: QMalcolm Issue: "12583" ================================================ FILE: .changes/unreleased/Fixes-20260305-133929.yaml ================================================ kind: Fixes body: Fix duplicate CTE race condition in ephemeral model compilation time: 2026-03-05T13:39:29.780057-08:00 custom: Author: colin-rogers-dbt Issue: "12602" ================================================ FILE: .changes/unreleased/Fixes-20260306-034008.yaml ================================================ kind: Fixes body: Allow deferral for UDFs time: 2026-03-06T03:40:08.773781+05:30 custom: Author: ash2shukla Issue: "12080" ================================================ FILE: .changes/unreleased/Fixes-20260306-071359.yaml ================================================ kind: Fixes body: Update URL and name of behavior change flag for `require_ref_searches_node_package_before_root` time: 2026-03-06T07:13:59.659401-07:00 custom: Author: dbeatty10 Issue: "12324" ================================================ FILE: .changes/unreleased/Fixes-20260306-192013.yaml ================================================ kind: Fixes body: Resolve full node description while running udfs for better logging time: 2026-03-06T19:20:13.395048+05:30 custom: Author: ash2shukla Issue: "12600" ================================================ FILE: .changes/unreleased/Fixes-20260310-120000.yaml ================================================ kind: Fixes body: Fix list-type metric filters under models key being incorrectly rendered at parse time, causing 'Dimension is undefined' errors time: 2026-03-10T12:00:00.000000-08:00 custom: Author: theyostalservice Issue: "12618" ================================================ FILE: .changes/unreleased/Fixes-20260317-094953.yaml ================================================ kind: Fixes body: Add @requires.catalogs decorator to test command to fix custom catalog integration support time: 2026-03-17T09:49:53.183758+01:00 custom: Author: rrittsteiger Issue: "12662" ================================================ FILE: .changes/unreleased/Fixes-20260318-000000.yaml ================================================ kind: Fixes body: Fix IndexError when parsing semantic model that references a disabled or missing model time: 2026-03-18T00:00:00.000000-04:00 custom: Author: MichelleArk Issue: 12671 ================================================ FILE: .changes/unreleased/Fixes-20260318-091800.yaml ================================================ kind: Fixes body: Support custom ref kwargs in unit tests and generic data tests, behind behavior flag `support_custom_ref_kwargs` time: 2026-03-18T09:18:00.000000-05:00 custom: Author: aahel Issue: 12148 ================================================ FILE: .changes/unreleased/Fixes-20260318-173426.yaml ================================================ kind: Fixes body: Fix AttributeError when docs block argument is non-constant time: 2026-03-18T17:34:26.771797-04:00 custom: Author: michelleark Issue: "12673" ================================================ FILE: .changes/unreleased/Fixes-20260318-182358.yaml ================================================ kind: Fixes body: Fix AttributeError when generic test config is non-dictionary time: 2026-03-18T18:23:58.393692-04:00 custom: Author: michelleark Issue: "12674" ================================================ FILE: .changes/unreleased/Fixes-20260318-191153.yaml ================================================ kind: Fixes body: handle jinja2.Undefined in msgpack_encoder time: 2026-03-18T19:11:53.734654-04:00 custom: Author: michelleark Issue: "12677" ================================================ FILE: .changes/unreleased/Fixes-20260318-200626.yaml ================================================ kind: Fixes body: Fix inheritance for defaults.agg_time_dimension in semantic models. time: 2026-03-18T20:06:26.155408-07:00 custom: Author: theyostalservice Issue: "12678" ================================================ FILE: .changes/unreleased/Under the Hood-20250929-151159.yaml ================================================ kind: Under the Hood body: Update schema file order test time: 2025-09-29T15:11:59.611595-04:00 custom: Author: gshank Issue: "11869" ================================================ FILE: .changes/unreleased/Under the Hood-20251119-110110.yaml ================================================ kind: Under the Hood body: Update jsonschemas for schema.yml and dbt_project.yml deprecations time: 2025-11-19T11:01:10.616676-05:00 custom: Author: michelleark Issue: "12180" ================================================ FILE: .changes/unreleased/Under the Hood-20251121-140515.yaml ================================================ kind: Under the Hood body: Replace setuptools and tox with hatch for build, test, and environment management. time: 2025-11-21T14:05:15.838252-05:00 custom: Author: emmyoop Issue: "12151" ================================================ FILE: .changes/unreleased/Under the Hood-20251209-131857.yaml ================================================ kind: Under the Hood body: Add add_catalog_integration call even if we have a pre-existing manifest time: 2025-12-09T13:18:57.043254-08:00 custom: Author: colin-rogers-dbt Issue: "12262" ================================================ FILE: .changes/unreleased/Under the Hood-20251215-155046.yaml ================================================ kind: Under the Hood body: Bump lower bound for dbt-common to 1.37.2 time: 2025-12-15T15:50:46.857793-05:00 custom: Author: michelleark Issue: "12284" ================================================ FILE: .changes/unreleased/Under the Hood-20260205-200835.yaml ================================================ kind: Under the Hood body: sync JSON schemas from dbt-fusion time: 2026-02-05T20:08:35.515361881Z custom: Author: fa-assistant Issue: N/A ================================================ FILE: .changes/unreleased/Under the Hood-20260210-151244.yaml ================================================ kind: Under the Hood body: Handle missing column time granularities during parsing. time: 2026-02-10T15:12:44.460381-08:00 custom: Author: theyostalservice Issue: "12472" ================================================ FILE: .changes/unreleased/Under the Hood-20260214-181659.yaml ================================================ kind: Under the Hood body: sync JSON schemas from dbt-fusion time: 2026-02-14T18:16:59.71097177Z custom: Author: fa-assistant Issue: N/A ================================================ FILE: .changes/unreleased/Under the Hood-20260220-170741.yaml ================================================ kind: Under the Hood body: sync JSON schemas from dbt-fusion time: 2026-02-20T17:07:41.802981778Z custom: Author: fa-assistant Issue: N/A ================================================ FILE: .changes/unreleased/Under the Hood-20260223-121653.yaml ================================================ kind: Under the Hood body: Make test case for parsing metric filters slightly more robust. time: 2026-02-23T12:16:53.395517-08:00 custom: Author: theyostalservice Issue: "12528" ================================================ FILE: .changes/unreleased/Under the Hood-20260225-092728.yaml ================================================ kind: Under the Hood body: sync JSON schemas from dbt-fusion time: 2026-02-25T09:27:28.498892233Z custom: Author: fa-assistant Issue: N/A ================================================ FILE: .changes/unreleased/Under the Hood-20260302-101022.yaml ================================================ kind: Under the Hood body: Unpin sqlparse dependency, and introduce --sqlparse CLI option for configuring sqlparse limits time: 2026-03-02T10:10:22.254214-05:00 custom: Author: michelleark Issue: "12329" ================================================ FILE: .changes/unreleased/Under the Hood-20260306-182611.yaml ================================================ kind: Under the Hood body: sync JSON schemas from dbt-fusion time: 2026-03-06T18:26:11.89001132Z custom: Author: fa-assistant Issue: N/A ================================================ FILE: .changes/unreleased/Under the Hood-20260318-120715.yaml ================================================ kind: Under the Hood body: 'Update jsonschemas for more accurate deprecation warnings: macro.config should not warn' time: 2026-03-18T12:07:15.833316-04:00 custom: Author: michelleark Issue: "12670" ================================================ FILE: .changie.yaml ================================================ changesDir: .changes unreleasedDir: unreleased headerPath: header.tpl.md versionHeaderPath: "" changelogPath: CHANGELOG.md versionExt: md envPrefix: "CHANGIE_" versionFormat: '## dbt-core {{.Version}} - {{.Time.Format "January 02, 2006"}}' kindFormat: '### {{.Kind}}' changeFormat: |- {{- $IssueList := list }} {{- $changes := splitList " " $.Custom.Issue }} {{- range $issueNbr := $changes }} {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-core/issues/nbr)" | replace "nbr" $issueNbr }} {{- $IssueList = append $IssueList $changeLink }} {{- end -}} - {{.Body}} ({{ range $index, $element := $IssueList }}{{if $index}}, {{end}}{{$element}}{{end}}) kinds: - label: Breaking Changes - label: Features - label: Fixes - label: Docs changeFormat: |- {{- $IssueList := list }} {{- $changes := splitList " " $.Custom.Issue }} {{- range $issueNbr := $changes }} {{- $changeLink := "[dbt-docs/#nbr](https://github.com/dbt-labs/dbt-docs/issues/nbr)" | replace "nbr" $issueNbr }} {{- $IssueList = append $IssueList $changeLink }} {{- end -}} - {{.Body}} ({{ range $index, $element := $IssueList }}{{if $index}}, {{end}}{{$element}}{{end}}) - label: Under the Hood - label: Dependencies - label: Security newlines: afterChangelogHeader: 1 afterKind: 1 afterChangelogVersion: 1 beforeKind: 1 endOfVersion: 1 custom: - key: Author label: GitHub Username(s) (separated by a single space if multiple) type: string minLength: 3 - key: Issue label: GitHub Issue Number (separated by a single space if multiple) type: string minLength: 1 footerFormat: | {{- $contributorDict := dict }} {{- /* ensure we always skip snyk and dependabot */}} {{- $bots := list "dependabot[bot]" "snyk-bot"}} {{- range $change := .Changes }} {{- $authorList := splitList " " $change.Custom.Author }} {{- /* loop through all authors for a single changelog */}} {{- range $author := $authorList }} {{- $authorLower := lower $author }} {{- /* we only want to include non-bot contributors */}} {{- if not (has $authorLower $bots)}} {{- $changeList := splitList " " $change.Custom.Author }} {{- $IssueList := list }} {{- $changeLink := $change.Kind }} {{- $changes := splitList " " $change.Custom.Issue }} {{- range $issueNbr := $changes }} {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-core/issues/nbr)" | replace "nbr" $issueNbr }} {{- $IssueList = append $IssueList $changeLink }} {{- end }} {{- /* check if this contributor has other changes associated with them already */}} {{- if hasKey $contributorDict $author }} {{- $contributionList := get $contributorDict $author }} {{- $contributionList = concat $contributionList $IssueList }} {{- $contributorDict := set $contributorDict $author $contributionList }} {{- else }} {{- $contributionList := $IssueList }} {{- $contributorDict := set $contributorDict $author $contributionList }} {{- end }} {{- end}} {{- end}} {{- end }} {{- /* no indentation here for formatting so the final markdown doesn't have unneeded indentations */}} {{- if $contributorDict}} ### Contributors {{- range $k,$v := $contributorDict }} - [@{{$k}}](https://github.com/{{$k}}) ({{ range $index, $element := $v }}{{if $index}}, {{end}}{{$element}}{{end}}) {{- end }} {{- end }} ================================================ FILE: .dockerignore ================================================ * !docker/requirements/*.txt !dist ================================================ FILE: .flake8 ================================================ [flake8] select = E W F ignore = W503 # makes Flake8 work like black W504 E203 # makes Flake8 work like black E704 # makes Flake8 work like black E741 E501 # long line checking is done in black per-file-ignores = */__init__.py: F401 ================================================ FILE: .git-blame-ignore-revs ================================================ # Reformatting dbt-core via black, flake8, mypy, and assorted pre-commit hooks. 43e3fc22c4eae4d3d901faba05e33c40f1f1dc5a ================================================ FILE: .gitattributes ================================================ core/dbt/task/docs/index.html binary tests/functional/artifacts/data/state/*/manifest.json binary core/dbt/docs/build/html/searchindex.js binary core/dbt/docs/build/html/index.html binary performance/runner/Cargo.lock binary core/dbt/events/types_pb2.py binary ================================================ FILE: .github/CODEOWNERS ================================================ # This file contains the code owners for the dbt-core repo. # PRs will be automatically assigned for review to the associated # team(s) or person(s) that touches any files that are mapped to them. # # A statement takes precedence over the statements above it so more general # assignments are found at the top with specific assignments being lower in # the ordering (i.e. catch all assignment should be the first item) # # Consult GitHub documentation for formatting guidelines: # https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners#example-of-a-codeowners-file # As a default for areas with no assignment, # the core team as a whole will be assigned * @dbt-labs/core-team ### ARTIFACTS /schemas/dbt @dbt-labs/cloud-artifacts ================================================ FILE: .github/ISSUE_TEMPLATE/bug-report.yml ================================================ name: 🐞 Bug description: Report a bug or an issue you've found with dbt title: "[Bug] " labels: ["bug", "triage"] body: - type: markdown attributes: value: | Thanks for taking the time to fill out this bug report! - type: checkboxes attributes: label: Is this a new bug in dbt-core? description: > In other words, is this an error, flaw, failure or fault in our software? If this is a bug that broke existing functionality that used to work, please open a regression issue. If this is a bug in an adapter plugin, please open an issue in the adapter's repository. If this is a bug experienced while using dbt Cloud, please report to [support](mailto:support@getdbt.com). If this is a request for help or troubleshooting code in your own dbt project, please join our [dbt Community Slack](https://www.getdbt.com/community/join-the-community/) or open a [Discussion question](https://github.com/dbt-labs/docs.getdbt.com/discussions). Please search to see if an issue already exists for the bug you encountered. options: - label: I believe this is a new bug in dbt-core required: true - label: I have searched the existing issues, and I could not find an existing issue for this bug required: true - type: textarea attributes: label: Current Behavior description: A concise description of what you're experiencing. validations: required: true - type: textarea attributes: label: Expected Behavior description: A concise description of what you expected to happen. validations: required: true - type: textarea attributes: label: Steps To Reproduce description: Steps to reproduce the behavior. placeholder: | 1. In this environment... 2. With this config... 3. Run '...' 4. See error... validations: required: true - type: textarea id: logs attributes: label: Relevant log output description: | If applicable, log output to help explain your problem. render: shell validations: required: false - type: textarea attributes: label: Environment description: | examples: - **OS**: Ubuntu 24.04 - **Python**: 3.10.12 (`python3 --version`) - **dbt-core**: 1.1.1 (`dbt --version`) value: | - OS: - Python: - dbt: render: markdown validations: required: false - type: dropdown id: database attributes: label: Which database adapter are you using with dbt? description: If the bug is specific to the database or adapter, please open the issue in that adapter's repository instead multiple: true options: - postgres - redshift - snowflake - bigquery - spark - other (mention it in "Additional Context") validations: required: false - type: textarea attributes: label: Additional Context description: | Links? References? Anything that will give us more context about the issue you are encountering! Tip: You can attach images or log files by clicking this area to highlight it and then dragging files in. validations: required: false ================================================ FILE: .github/ISSUE_TEMPLATE/code-docs.yml ================================================ name: 📄 Code docs description: Report an issue for markdown files within this repo, such as README, ARCHITECTURE, etc. title: "[Code docs] <title>" labels: ["triage"] body: - type: markdown attributes: value: | Thanks for taking the time to fill out this code docs issue! - type: textarea attributes: label: Please describe the issue and your proposals. description: | Links? References? Anything that will give us more context about the issue you are encountering! Tip: You can attach images by clicking this area to highlight it and then dragging files in. validations: required: false ================================================ FILE: .github/ISSUE_TEMPLATE/config.yml ================================================ blank_issues_enabled: false contact_links: - name: Documentation url: https://github.com/dbt-labs/docs.getdbt.com/issues/new/choose about: Problems and issues with dbt product documentation hosted on docs.getdbt.com. Issues for markdown files within this repo, such as README, should be opened using the "Code docs" template. - name: Ask the community for help url: https://github.com/dbt-labs/docs.getdbt.com/discussions about: Need help troubleshooting? Check out our guide on how to ask - name: Contact dbt Cloud support url: mailto:support@getdbt.com about: Are you using dbt Cloud? Contact our support team for help! - name: Participate in Discussions url: https://github.com/dbt-labs/dbt-core/discussions about: Do you have a Big Idea for dbt? Read open discussions, or start a new one - name: Create an issue for adapters url: https://github.com/dbt-labs/dbt-adapters/issues/new/choose about: Report a bug or request a feature for an adapter ================================================ FILE: .github/ISSUE_TEMPLATE/feature-request.yml ================================================ name: ✨ Feature description: Propose a straightforward extension of dbt functionality title: "[Feature] <title>" labels: ["enhancement", "triage"] body: - type: markdown attributes: value: | Thanks for taking the time to fill out this feature request! - type: checkboxes attributes: label: Is this your first time submitting a feature request? description: > We want to make sure that features are distinct and discoverable, so that other members of the community can find them and offer their thoughts. Issues are the right place to request straightforward extensions of existing dbt functionality. For "big ideas" about future capabilities of dbt, we ask that you open a [discussion](https://github.com/dbt-labs/dbt-core/discussions) in the "Ideas" category instead. options: - label: I have read the [expectations for open source contributors](https://docs.getdbt.com/docs/contributing/oss-expectations) required: true - label: I have searched the existing issues, and I could not find an existing issue for this feature required: true - label: I am requesting a straightforward extension of existing dbt functionality, rather than a Big Idea better suited to a discussion required: true - type: textarea attributes: label: Describe the feature description: A clear and concise description of what you want to happen. validations: required: true - type: textarea attributes: label: Describe alternatives you've considered description: | A clear and concise description of any alternative solutions or features you've considered. validations: required: false - type: textarea attributes: label: Who will this benefit? description: | What kind of use case will this feature be useful for? Please be specific and provide examples, this will help us prioritize properly. validations: required: false - type: input attributes: label: Are you interested in contributing this feature? description: Let us know if you want to write some code, and how we can help. validations: required: false - type: textarea attributes: label: Anything else? description: | Links? References? Anything that will give us more context about the feature you are suggesting! validations: required: false ================================================ FILE: .github/ISSUE_TEMPLATE/implementation-ticket.yml ================================================ name: 🛠️ Implementation description: This is an implementation ticket intended for use by the maintainers of dbt-core title: "[<project>] <title>" labels: ["user docs"] body: - type: markdown attributes: value: This is an implementation ticket intended for use by the maintainers of dbt-core - type: checkboxes attributes: label: Housekeeping description: > A couple friendly reminders: 1. Remove the `user docs` label if the scope of this work does not require changes to https://docs.getdbt.com/docs: no end-user interface (e.g. yml spec, CLI, error messages, etc) or functional changes 2. Link any blocking issues in the "Blocked on" field under the "Core devs & maintainers" project. options: - label: I am a maintainer of dbt-core required: true - type: textarea attributes: label: Short description description: | Describe the scope of the ticket, a high-level implementation approach and any tradeoffs to consider validations: required: true - type: textarea attributes: label: Acceptance criteria description: | What is the definition of done for this ticket? Include any relevant edge cases and/or test cases validations: required: true - type: textarea attributes: label: Suggested Tests description: | Provide scenarios to test. Link to existing similar tests if appropriate. placeholder: | 1. Test with no version specified in the schema file and use selection logic on a versioned model for a specific version. Expect pass. 2. Test with a version specified in the schema file that is no valid. Expect ParsingError. validations: required: true - type: textarea attributes: label: Impact to Other Teams description: | Will this change impact other teams? Include details of the kinds of changes required (new tests, code changes, related tickets) and _add the relevant `Impact:[team]` label_. placeholder: | Example: This change impacts `dbt-redshift` because the tests will need to be modified. The `Impact:[Adapter]` label has been added. validations: required: true - type: textarea attributes: label: Will backports be required? description: | Will this change need to be backported to previous versions? Add details, possible blockers to backporting and _add the relevant backport labels `backport 1.x.latest`_ placeholder: | Example: Backport to 1.6.latest, 1.5.latest and 1.4.latest. Since 1.4 isn't using click, the backport may be complicated. The `backport 1.6.latest`, `backport 1.5.latest` and `backport 1.4.latest` labels have been added. validations: required: true - type: textarea attributes: label: Context description: | Provide the "why", motivation, and alternative approaches considered -- linking to previous refinement issues, spikes and documentation as appropriate validations: required: false ================================================ FILE: .github/ISSUE_TEMPLATE/regression-report.yml ================================================ name: ☣️ Regression description: Report a regression you've observed in a newer version of dbt title: "[Regression] <title>" labels: ["bug", "regression", "triage"] body: - type: markdown attributes: value: | Thanks for taking the time to fill out this regression report! - type: checkboxes attributes: label: Is this a regression in a recent version of dbt-core? description: > A regression is when documented functionality works as expected in an older version of dbt-core, and no longer works after upgrading to a newer version of dbt-core options: - label: I believe this is a regression in dbt-core functionality required: true - label: I have searched the existing issues, and I could not find an existing issue for this regression required: true - type: textarea attributes: label: Current Behavior description: A concise description of what you're experiencing. validations: required: true - type: textarea attributes: label: Expected/Previous Behavior description: A concise description of what you expected to happen. validations: required: true - type: textarea attributes: label: Steps To Reproduce description: Steps to reproduce the behavior. placeholder: | 1. In this environment... 2. With this config... 3. Run '...' 4. See error... validations: required: true - type: textarea id: logs attributes: label: Relevant log output description: | If applicable, log output to help explain your problem. render: shell validations: required: false - type: textarea attributes: label: Environment description: | examples: - **OS**: Ubuntu 24.04 - **Python**: 3.10.12 (`python3 --version`) - **dbt-core (working version)**: 1.1.1 (`dbt --version`) - **dbt-core (regression version)**: 1.2.0 (`dbt --version`) value: | - OS: - Python: - dbt (working version): - dbt (regression version): render: markdown validations: required: true - type: dropdown id: database attributes: label: Which database adapter are you using with dbt? description: If the regression is specific to the database or adapter, please open the issue in that adapter's repository instead multiple: true options: - postgres - redshift - snowflake - bigquery - spark - other (mention it in "Additional Context") validations: required: false - type: textarea attributes: label: Additional Context description: | Links? References? Anything that will give us more context about the issue you are encountering! Tip: You can attach images or log files by clicking this area to highlight it and then dragging files in. validations: required: false ================================================ FILE: .github/_README.md ================================================ <!-- GitHub will publish this readme on the main repo page if the name is `README.md` so we've added the leading underscore to prevent this --> <!-- Do not rename this file `README.md` --> <!-- See https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-readmes --> ## What are GitHub Actions? GitHub Actions are used for many different purposes. We use them to run tests in CI, validate PRs are in an expected state, and automate processes. - [Overview of GitHub Actions](https://docs.github.com/en/actions/learn-github-actions/understanding-github-actions) - [What's a workflow?](https://docs.github.com/en/actions/using-workflows/about-workflows) - [GitHub Actions guides](https://docs.github.com/en/actions/guides) ___ ## Where do actions and workflows live We try to maintain actions that are shared across repositories in a single place so that necesary changes can be made in a single place. [dbt-labs/actions](https://github.com/dbt-labs/actions/) is the central repository of actions and workflows we use across repositories. GitHub Actions also live locally within a repository. The workflows can be found at `.github/workflows` from the root of the repository. These should be specific to that code base. Note: We are actively moving actions into the central Action repository so there is currently some duplication across repositories. ___ ## Basics of Using Actions ### Viewing Output - View the detailed action output for your PR in the **Checks** tab of the PR. This only shows the most recent run. You can also view high level **Checks** output at the bottom on the PR. - View _all_ action output for a repository from the [**Actions**](https://github.com/dbt-labs/dbt-core/actions) tab. Workflow results last 1 year. Artifacts last 90 days, unless specified otherwise in individual workflows. This view often shows what seem like duplicates of the same workflow. This occurs when files are renamed but the workflow name has not changed. These are in fact _not_ duplicates. You can see the branch the workflow runs from in this view. It is listed in the table between the workflow name and the time/duration of the run. When blank, the workflow is running in the context of the `main` branch. ### How to view what workflow file is being referenced from a run - When viewing the output of a specific workflow run, click the 3 dots at the top right of the display. There will be an option to `View workflow file`. ### How to manually run a workflow - If a workflow has the `on: workflow_dispatch` trigger, it can be manually triggered - From the [**Actions**](https://github.com/dbt-labs/dbt-core/actions) tab, find the workflow you want to run, select it and fill in any inputs requied. That's it! ### How to re-run jobs - From the UI you can rerun from failure - You can retrigger the cla check by commenting on the PR with `@cla-bot check` ___ ## General Standards ### Permissions - By default, workflows have read permissions in the repository for the contents scope only when no permissions are explicitly set. - It is best practice to always define the permissions explicitly. This will allow actions to continue to work when the default permissions on the repository are changed. It also allows explicit grants of the least permissions possible. - There are a lot of permissions available. [Read up on them](https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs) if you're unsure what to use. ```yaml permissions: contents: read pull-requests: write ``` ### Secrets - When to use a [Personal Access Token (PAT)](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) vs the [GITHUB_TOKEN](https://docs.github.com/en/actions/security-guides/automatic-token-authentication) generated for the action? The `GITHUB_TOKEN` is used by default. In most cases it is sufficient for what you need. If you expect the workflow to result in a commit to that should retrigger workflows, you will need to use a Personal Access Token for the bot to commit the file. When using the GITHUB_TOKEN, the resulting commit will not trigger another GitHub Actions Workflow run. This is due to limitations set by GitHub. See [the docs](https://docs.github.com/en/actions/security-guides/automatic-token-authentication#using-the-github_token-in-a-workflow) for a more detailed explanation. For example, we must use a PAT in our workflow to commit a new changelog yaml file for bot PRs. Once the file has been committed to the branch, it should retrigger the check to validate that a changelog exists on the PR. Otherwise, it would stay in a failed state since the check would never retrigger. ### Triggers You can configure your workflows to run when specific activity on GitHub happens, at a scheduled time, or when an event outside of GitHub occurs. Read more details in the [GitHub docs](https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows). These triggers are under the `on` key of the workflow and more than one can be listed. ```yaml on: push: branches: - "main" - "*.latest" - "releases/*" pull_request: # catch when the PR is opened with the label or when the label is added types: [opened, labeled] workflow_dispatch: ``` Some triggers of note that we use: - `push` - Runs your workflow when you push a commit or tag. - `pull_request` - Runs your workflow when activity on a pull request in the workflow's repository occurs. Takes in a list of activity types (opened, labeled, etc) if appropriate. - `pull_request_target` - Same as `pull_request` but runs in the context of the PR target branch. - `workflow_call` - used with reusable workflows. Triggered by another workflow calling it. - `workflow_dispatch` - Gives the ability to manually trigger a workflow from the GitHub API, GitHub CLI, or GitHub browser interface. ### Basic Formatting - Add a description of what your workflow does at the top in this format ``` # **what?** # Describe what the action does. # **why?** # Why does this action exist? # **when?** # How/when will it be triggered? ``` - Leave blank lines between steps and jobs ```yaml jobs: dependency_changelog: runs-on: ${{ vars.UBUNTU_LATEST }} steps: - name: Get File Name Timestamp id: filename_time uses: nanzm/get-time-action@v1.1 with: format: 'YYYYMMDD-HHmmss' - name: Get File Content Timestamp id: file_content_time uses: nanzm/get-time-action@v1.1 with: format: 'YYYY-MM-DDTHH:mm:ss.000000-05:00' - name: Generate Filepath id: fp run: | FILEPATH=.changes/unreleased/Dependencies-${{ steps.filename_time.outputs.time }}.yaml echo "FILEPATH=$FILEPATH" >> $GITHUB_OUTPUT ``` - Print out all variables you will reference as the first step of a job. This allows for easier debugging. The first job should log all inputs. Subsequent jobs should reference outputs of other jobs, if present. When possible, generate variables at the top of your workflow in a single place to reference later. This is not always strictly possible since you may generate a value to be used later mid-workflow. Be sure to use quotes around these logs so special characters are not interpreted. ```yaml job1: - name: "[DEBUG] Print Variables" run: | echo "all variables defined as inputs" echo "The last commit sha in the release: ${{ inputs.sha }}" echo "The release version number: ${{ inputs.version_number }}" echo "The changelog_path: ${{ inputs.changelog_path }}" echo "The build_script_path: ${{ inputs.build_script_path }}" echo "The s3_bucket_name: ${{ inputs.s3_bucket_name }}" echo "The package_test_command: ${{ inputs.package_test_command }}" # collect all the variables that need to be used in subsequent jobs - name: Set Variables id: variables run: | echo "important_path='performance/runner/Cargo.toml'" >> $GITHUB_OUTPUT echo "release_id=${{github.event.inputs.release_id}}" >> $GITHUB_OUTPUT echo "open_prs=${{github.event.inputs.open_prs}}" >> $GITHUB_OUTPUT job2: needs: [job1] - name: "[DEBUG] Print Variables" run: | echo "all variables defined in job1 > Set Variables > outputs" echo "important_path: ${{ needs.job1.outputs.important_path }}" echo "release_id: ${{ needs.job1.outputs.release_id }}" echo "open_prs: ${{ needs.job1.outputs.open_prs }}" ``` - When it's not obvious what something does, add a comment! ___ ## Tips ### Context - The [GitHub CLI](https://cli.github.com/) is available in the default runners - Actions run in your context. ie, using an action from the marketplace that uses the GITHUB_TOKEN uses the GITHUB_TOKEN generated by your workflow run. ### Runners - We dynamically set runners based on repository vars. Admins can view repository vars and reset them. Current values are the following but are subject to change: - `vars.UBUNTU_LATEST` -> `ubuntu-latest` - `vars.WINDOWS_LATEST` -> `windows-latest` - `vars.MACOS_LATEST` -> `macos-14` ### Actions from the Marketplace - Don’t use external actions for things that can easily be accomplished manually. - Always read through what an external action does before using it! Often an action in the GitHub Actions Marketplace can be replaced with a few lines in bash. This is much more maintainable (and won’t change under us) and clear as to what’s actually happening. It also prevents any - Pin actions _we don't control_ to tags. ### Connecting to AWS - Authenticate with the aws managed workflow ```yaml - name: Configure AWS credentials from Test account uses: aws-actions/configure-aws-credentials@v2 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: us-east-1 ``` - Then access with the aws command that comes installed on the action runner machines ```yaml - name: Copy Artifacts from S3 via CLI run: aws s3 cp ${{ env.s3_bucket }} . --recursive ``` ### Testing - Depending on what your action does, you may be able to use [`act`](https://github.com/nektos/act) to test the action locally. Some features of GitHub Actions do not work with `act`, among those are reusable workflows. If you can't use `act`, you'll have to push your changes up before being able to test. This can be slow. ================================================ FILE: .github/actions/latest-wrangler/Dockerfile ================================================ FROM python:3-slim AS builder ADD . /app WORKDIR /app # We are installing a dependency here directly into our app source dir RUN pip install --target=/app requests packaging # A distroless container image with Python and some basics like SSL certificates # https://github.com/GoogleContainerTools/distroless FROM gcr.io/distroless/python3-debian10 COPY --from=builder /app /app WORKDIR /app ENV PYTHONPATH /app CMD ["/app/main.py"] ================================================ FILE: .github/actions/latest-wrangler/README.md ================================================ # Github package 'latest' tag wrangler for containers ## Usage Plug in the necessary inputs to determine if the container being built should be tagged 'latest; at the package level, for example `dbt-redshift:latest`. ## Inputs | Input | Description | | - | - | | `package` | Name of the GH package to check against | | `new_version` | Semver of new container | | `gh_token` | GH token with package read scope| | `halt_on_missing` | Return non-zero exit code if requested package does not exist. (defaults to false)| ## Outputs | Output | Description | | - | - | | `latest` | Wether or not the new container should be tagged 'latest'| | `minor_latest` | Wether or not the new container should be tagged major.minor.latest | ## Example workflow ```yaml name: Ship it! on: workflow_dispatch: inputs: package: description: The package to publish required: true version_number: description: The version number required: true jobs: build: runs-on: ${{ vars.UBUNTU_LATEST }} steps: - uses: actions/checkout@v3 - name: Wrangle latest tag id: is_latest uses: ./.github/actions/latest-wrangler with: package: ${{ github.event.inputs.package }} new_version: ${{ github.event.inputs.new_version }} gh_token: ${{ secrets.GITHUB_TOKEN }} - name: Print the results run: | echo "Is it latest? Survey says: ${{ steps.is_latest.outputs.latest }} !" echo "Is it minor.latest? Survey says: ${{ steps.is_latest.outputs.minor_latest }} !" ``` ================================================ FILE: .github/actions/latest-wrangler/action.yml ================================================ name: "GitHub package `latest` tag wrangler for containers" description: "Determines if the published image should include `latest` tags" inputs: package_name: description: "Package being published (i.e. `dbt-core`, `dbt-redshift`, etc.)" required: true new_version: description: "SemVer of the package being published (i.e. 1.7.2, 1.8.0a1, etc.)" required: true github_token: description: "Auth token for GitHub (must have view packages scope)" required: true outputs: tags: description: "A list of tags to associate with this version" runs: using: "docker" image: "Dockerfile" ================================================ FILE: .github/actions/latest-wrangler/examples/example_workflow.yml ================================================ name: Ship it! on: workflow_dispatch: inputs: package: description: The package to publish required: true version_number: description: The version number required: true jobs: build: runs-on: ${{ vars.UBUNTU_LATEST }} steps: - uses: actions/checkout@v3 - name: Wrangle latest tag id: is_latest uses: ./.github/actions/latest-wrangler with: package: ${{ github.event.inputs.package }} new_version: ${{ github.event.inputs.new_version }} gh_token: ${{ secrets.GITHUB_TOKEN }} - name: Print the results run: | echo "Is it latest? Survey says: ${{ steps.is_latest.outputs.latest }} !" ================================================ FILE: .github/actions/latest-wrangler/examples/example_workflow_dispatch.json ================================================ { "inputs": { "version_number": "1.0.1", "package": "dbt-redshift" } } ================================================ FILE: .github/actions/latest-wrangler/main.py ================================================ import os import sys from typing import List import requests from packaging.version import Version, parse def main(): package_name: str = os.environ["INPUT_PACKAGE_NAME"] new_version: Version = parse(os.environ["INPUT_NEW_VERSION"]) github_token: str = os.environ["INPUT_GITHUB_TOKEN"] response = _package_metadata(package_name, github_token) published_versions = _published_versions(response) new_version_tags = _new_version_tags(new_version, published_versions) _register_tags(new_version_tags, package_name) def _package_metadata(package_name: str, github_token: str) -> requests.Response: url = f"https://api.github.com/orgs/dbt-labs/packages/container/{package_name}/versions" return requests.get(url, auth=("", github_token)) def _published_versions(response: requests.Response) -> List[Version]: package_metadata = response.json() return [ parse(tag) for version in package_metadata for tag in version["metadata"]["container"]["tags"] if "latest" not in tag ] def _new_version_tags(new_version: Version, published_versions: List[Version]) -> List[str]: # the package version is always a tag tags = [str(new_version)] # pre-releases don't get tagged with `latest` if new_version.is_prerelease: return tags if new_version > max(published_versions): tags.append("latest") published_patches = [ version for version in published_versions if version.major == new_version.major and version.minor == new_version.minor ] if new_version > max(published_patches): tags.append(f"{new_version.major}.{new_version.minor}.latest") return tags def _register_tags(tags: List[str], package_name: str) -> None: fully_qualified_tags = ",".join([f"ghcr.io/dbt-labs/{package_name}:{tag}" for tag in tags]) github_output = os.environ.get("GITHUB_OUTPUT") with open(github_output, "at", encoding="utf-8") as gh_output: gh_output.write(f"fully_qualified_tags={fully_qualified_tags}") def _validate_response(response: requests.Response) -> None: message = response["message"] if response.status_code != 200: print(f"Call to GitHub API failed: {response.status_code} - {message}") sys.exit(1) if __name__ == "__main__": main() ================================================ FILE: .github/actions/setup-postgres-windows/action.yml ================================================ name: "Set up postgres (windows)" description: "Set up postgres service on windows vm for dbt integration tests" runs: using: "composite" steps: - shell: pwsh run: | Write-Host -Object "Installing PostgreSQL 16 as windows service..." $installerArgs = @("--install_runtimes 0", "--superpassword root", "--enable_acledit 1", "--unattendedmodeui none", "--mode unattended") $filePath = Invoke-DownloadWithRetry -Url "https://get.enterprisedb.com/postgresql/postgresql-16.1-1-windows-x64.exe" -Path "$env:PGROOT/postgresql-16.1-1-windows-x64.exe" Start-Process -FilePath $filePath -ArgumentList $installerArgs -Wait -PassThru Write-Host -Object "Validating PostgreSQL 16 Install..." Get-Service -Name postgresql* $pgReady = Start-Process -FilePath "$env:PGBIN\pg_isready" -Wait -PassThru $exitCode = $pgReady.ExitCode if ($exitCode -ne 0) { Write-Host -Object "PostgreSQL is not ready. Exitcode: $exitCode" exit $exitCode } Write-Host -Object "Starting PostgreSQL 16 Service..." $pgService = Get-Service -Name postgresql-x64-16 Set-Service -InputObject $pgService -Status running -StartupType automatic $env:Path += ";$env:PGBIN" bash ${{ github.action_path }}/setup_db.sh ================================================ FILE: .github/dbt-postgres-testing.yml ================================================ # **what?** # Runs all tests in dbt-postgres with this branch of dbt-core to ensure nothing is broken # **why?** # Ensure dbt-core changes do not break dbt-postgres, as a basic proxy for other adapters # **when?** # This will run when trying to merge a PR into main. # It can also be manually triggered. # This workflow can be skipped by adding the "Skip Postgres Testing" label to the PR. This is # useful when making a change in both `dbt-postgres` and `dbt-core` where the changes are dependant # and cause the other repository to break. name: "dbt-postgres Tests" run-name: >- ${{ (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') && format('dbt-postgres@{0} with dbt-core@{1}', inputs.dbt-postgres-ref, inputs.dbt-core-ref) || 'dbt-postgres@main with dbt-core branch' }} on: push: branches: - "main" - "*.latest" - "releases/*" pull_request: merge_group: types: [checks_requested] workflow_dispatch: inputs: dbt-postgres-ref: description: "The branch of dbt-postgres to test against" default: "main" dbt-core-ref: description: "The branch of dbt-core to test against" default: "main" workflow_call: inputs: dbt-postgres-ref: description: "The branch of dbt-postgres to test against" type: string required: true default: "main" dbt-core-ref: description: "The branch of dbt-core to test against" type: string required: true default: "main" permissions: read-all # will cancel previous workflows triggered by the same event # and for the same ref for PRs/merges or same SHA otherwise # and for the same inputs on workflow_dispatch or workflow_call concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(fromJson('["pull_request", "merge_group"]'), github.event_name) && github.event.pull_request.head.ref || github.sha }}-${{ contains(fromJson('["workflow_call", "workflow_dispatch"]'), github.event_name) && github.event.inputs.dbt-postgres-ref && github.event.inputs.dbt-core-ref || github.sha }} cancel-in-progress: true defaults: run: shell: bash jobs: job-prep: # This allow us to run the workflow on pull_requests as well so we can always run unit tests # and only run integration tests on merge for time purposes name: Setup Repo Refs runs-on: ubuntu-latest outputs: dbt-postgres-ref: ${{ steps.core-ref.outputs.ref }} dbt-core-ref: ${{ steps.common-ref.outputs.ref }} steps: - name: "Input Refs" id: job-inputs run: | echo "inputs.dbt-postgres-ref=${{ inputs.dbt-postgres-ref }}" echo "inputs.dbt-core-ref=${{ inputs.dbt-core-ref }}" - name: "Determine dbt-postgres ref" id: core-ref run: | if [[ -z "${{ inputs.dbt-postgres-ref }}" ]]; then REF="main" else REF=${{ inputs.dbt-postgres-ref }} fi echo "ref=$REF" >> $GITHUB_OUTPUT - name: "Determine dbt-core ref" id: common-ref run: | if [[ -z "${{ inputs.dbt-core-ref }}" ]]; then # these will be commits instead of branches if [[ "${{ github.event_name }}" == "merge_group" ]]; then REF=${{ github.event.merge_group.head_sha }} else REF=${{ github.event.pull_request.base.sha }} fi else REF=${{ inputs.dbt-core-ref }} fi echo "ref=$REF" >> $GITHUB_OUTPUT - name: "Final Refs" run: | echo "dbt-postgres-ref=${{ steps.core-ref.outputs.ref }}" echo "dbt-core-ref=${{ steps.common-ref.outputs.ref }}" # integration-tests-postgres: # name: "dbt-postgres integration tests" # needs: [job-prep] # runs-on: ubuntu-latest # defaults: # run: # working-directory: "./dbt-postgres" # environment: # name: "dbt-postgres" # env: # POSTGRES_TEST_HOST: ${{ vars.POSTGRES_TEST_HOST }} # POSTGRES_TEST_PORT: ${{ vars.POSTGRES_TEST_PORT }} # POSTGRES_TEST_USER: ${{ vars.POSTGRES_TEST_USER }} # POSTGRES_TEST_PASS: ${{ secrets.POSTGRES_TEST_PASS }} # POSTGRES_TEST_DATABASE: ${{ vars.POSTGRES_TEST_DATABASE }} # POSTGRES_TEST_THREADS: ${{ vars.POSTGRES_TEST_THREADS }} # services: # postgres: # image: postgres # env: # POSTGRES_PASSWORD: postgres # options: >- # --health-cmd pg_isready # --health-interval 10s # --health-timeout 5s # --health-retries 5 # ports: # - ${{ vars.POSTGRES_TEST_PORT }}:5432 # steps: # - name: "Check out dbt-adapters@${{ needs.job-prep.outputs.dbt-postgres-ref }}" # uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4 # with: # repository: dbt-labs/dbt-adapters # ref: ${{ needs.job-prep.outputs.dbt-postgres-ref }} # - name: "Set up Python" # uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # actions/setup-python@v5 # with: # python-version: ${{ inputs.python-version }} # - name: "Set environment variables" # run: | # echo "HATCH_PYTHON=${{ inputs.python-version }}" >> $GITHUB_ENV # echo "PIP_ONLY_BINARY=psycopg2-binary" >> $GITHUB_ENV # - name: "Setup test database" # run: psql -f ./scripts/setup_test_database.sql # env: # PGHOST: ${{ vars.POSTGRES_TEST_HOST }} # PGPORT: ${{ vars.POSTGRES_TEST_PORT }} # PGUSER: postgres # PGPASSWORD: postgres # PGDATABASE: postgres # - name: "Install hatch" # uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc # pypa/hatch@install # - name: "Run integration tests" # run: hatch run ${{ inputs.hatch-env }}:integration-tests ================================================ FILE: .github/dependabot.yml ================================================ version: 2 updates: # python dependencies - package-ecosystem: "pip" directory: "/" schedule: interval: "daily" rebase-strategy: "disabled" - package-ecosystem: "pip" directory: "/core" schedule: interval: "daily" rebase-strategy: "disabled" # docker dependencies - package-ecosystem: "docker" directory: "/" schedule: interval: "weekly" rebase-strategy: "disabled" - package-ecosystem: "docker" directory: "/docker" schedule: interval: "weekly" rebase-strategy: "disabled" # github dependencies - package-ecosystem: "github-actions" directory: "/" schedule: interval: "weekly" rebase-strategy: "disabled" ================================================ FILE: .github/pull_request_template.md ================================================ Resolves # <!--- Include the number of the issue addressed by this PR above, if applicable. PRs for code changes without an associated issue *will not be merged*. See CONTRIBUTING.md for more information. Add the `user docs` label to this PR if it will need docs changes. An issue will get opened in docs.getdbt.com upon successful merge of this PR. --> ### Problem <!--- Describe the problem this PR is solving. What is the application state before this PR is merged? --> ### Solution <!--- Describe the way this PR solves the above problem. Add as much detail as you can to help reviewers understand your changes. Include any alternatives and tradeoffs you considered. --> ### Checklist - [ ] I have read [the contributing guide](https://github.com/dbt-labs/dbt-core/blob/main/CONTRIBUTING.md) and understand what's expected of me. - [ ] I have run this code in development, and it appears to resolve the stated issue. - [ ] This PR includes tests, or tests are not required or relevant for this PR. - [ ] This PR has no interface changes (e.g., macros, CLI, logs, JSON artifacts, config files, adapter interface, etc.) or this PR has already received feedback and approval from Product or DX. - [ ] This PR includes [type annotations](https://docs.python.org/3/library/typing.html) for new and modified functions. ================================================ FILE: .github/workflows/artifact-reviews.yml ================================================ # **what?** # Enforces 2 reviews when artifact or validation files are modified. # **why?** # Ensure artifact changes receive proper review from designated team members. GitHub doesn't support # multiple reviews on a single PR based on files changed, so we need to enforce this manually. # **when?** # This will run when reviews are submitted and dismissed. name: "Enforce Additional Reviews on Artifact and Validations Changes" permissions: checks: write pull-requests: write contents: read on: # trigger check on review events. use pull_request_target for forks. pull_request_target: types: [opened, reopened, ready_for_review, synchronize, review_requested] pull_request_review: types: [submitted, edited, dismissed] # only run this once per PR at a time concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number }} cancel-in-progress: true env: required_approvals: 2 team: "core-group" jobs: check-reviews: name: "Validate Additional Reviews" runs-on: ubuntu-latest steps: - name: "Get list of changed files" id: changed_files run: | # Fetch files as JSON and process with jq to sanitize output gh api repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files \ | jq -r '.[].filename' \ | while IFS= read -r file; do # Sanitize the filename by removing any special characters and command injection attempts clean_file=$(echo "$file" | sed 's/[^a-zA-Z0-9\.\/\-_]//g') echo "$clean_file" done > changed_files.txt echo "CHANGED_FILES<<EOF" >> $GITHUB_OUTPUT cat changed_files.txt >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: "Check if any artifact files changed" id: artifact_files_changed run: | artifact_changes=false while IFS= read -r file; do # Only process if file path looks legitimate if [[ "$file" =~ ^[a-zA-Z0-9\.\/\-_]+$ ]]; then if [[ "$file" == "core/dbt/artifacts/"* ]] ; then artifact_changes=true break fi fi done < changed_files.txt echo "artifact_changes=$artifact_changes" >> $GITHUB_OUTPUT - name: "Get Core Team Members" if: steps.artifact_files_changed.outputs.artifact_changes == 'true' id: core_members run: | gh api -H "Accept: application/vnd.github+json" \ /orgs/dbt-labs/teams/${{ env.team }}/members > core_members.json # Extract usernames and set as multiline output echo "membership<<EOF" >> $GITHUB_OUTPUT jq -r '.[].login' core_members.json >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT env: GH_TOKEN: ${{ secrets.IT_TEAM_MEMBERSHIP }} - name: "Verify ${{ env.required_approvals }} core team approvals" if: steps.artifact_files_changed.outputs.artifact_changes == 'true' id: check_approvals run: | # Get all reviews REVIEWS=$(gh api repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/reviews) echo "All reviews:" echo "$REVIEWS" # Count approved reviews from core team members (only most recent review per user) CORE_APPROVALS=0 while IFS= read -r member; do echo "Checking member: $member" APPROVED=$(echo "$REVIEWS" | jq --arg user "$member" ' group_by(.user.login) | map(select(.[0].user.login == $user) | sort_by(.submitted_at) | last) | map(select(.state == "APPROVED" and (.state != "DISMISSED"))) | length') echo "Latest review state for $member: $APPROVED" CORE_APPROVALS=$((CORE_APPROVALS + APPROVED)) echo "Running total: $CORE_APPROVALS" done <<< "${{ steps.core_members.outputs.membership }}" echo "CORE_APPROVALS=$CORE_APPROVALS" >> $GITHUB_OUTPUT echo "CORE_APPROVALS=$CORE_APPROVALS" env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: "Find Comment" if: steps.artifact_files_changed.outputs.artifact_changes == 'true' && steps.check_approvals.outputs.CORE_APPROVALS < env.required_approvals uses: peter-evans/find-comment@a54c31d7fa095754bfef525c0c8e5e5674c4b4b1 # peter-evans/find-comment@v2 id: find-comment with: issue-number: ${{ github.event.pull_request.number }} comment-author: 'github-actions[bot]' body-includes: "### Additional Artifact Review Required" - name: "Create Comment" if: steps.artifact_files_changed.outputs.artifact_changes == 'true' && steps.find-comment.outputs.comment-id == '' && steps.check_approvals.outputs.CORE_APPROVALS < env.required_approvals uses: peter-evans/create-or-update-comment@23ff15729ef2fc348714a3bb66d2f655ca9066f2 # peter-evans/create-or-update-comment@v3 with: issue-number: ${{ github.event.pull_request.number }} body: | ### Additional Artifact Review Required Changes to artifact directory files requires at least ${{ env.required_approvals }} approvals from core team members. - name: "Notify if not enough approvals" if: steps.artifact_files_changed.outputs.artifact_changes == 'true' run: | if [[ "${{ steps.check_approvals.outputs.CORE_APPROVALS }}" -ge "${{ env.required_approvals }}" ]]; then title="Extra requirements met" message="Changes to artifact directory files requires at least ${{ env.required_approvals }} approvals from core team members. Current number of core team approvals: ${{ steps.check_approvals.outputs.CORE_APPROVALS }} " echo "::notice title=$title::$message" echo "REVIEW_STATUS=success" >> $GITHUB_OUTPUT else title="PR Approval Requirements Not Met" message="Changes to artifact directory files requires at least ${{ env.required_approvals }} approvals from core team members. Current number of core team approvals: ${{ steps.check_approvals.outputs.CORE_APPROVALS }} " echo "::notice title=$title::$message" echo "REVIEW_STATUS=neutral" >> $GITHUB_OUTPUT fi id: review_check - name: "Set check status" id: status_check run: | if [[ "${{ steps.artifact_files_changed.outputs.artifact_changes }}" == 'false' ]]; then # no extra review required echo "current_status=success" >> $GITHUB_OUTPUT elif [[ "${{ steps.review_check.outputs.REVIEW_STATUS }}" == "success" ]]; then # we have all the required reviews echo "current_status=success" >> $GITHUB_OUTPUT else # neutral exit - neither success nor failure # we can't fail here because we use multiple triggers for this workflow and they won't reset the check # workaround is to use a neutral exit to skip the check run until it's actually successful echo "current_status=neutral" >> $GITHUB_OUTPUT fi - name: "Post Event" # This step posts the status of the check because the workflow is triggered by multiple events # and we need to ensure the check is always updated. Otherwise we would end up with duplicate # checks in the GitHub UI. run: | if [[ "${{ steps.status_check.outputs.current_status }}" == "success" ]]; then state="success" else state="failure" fi gh api \ --method POST \ -H "Accept: application/vnd.github+json" \ /repos/${{ github.repository }}/statuses/${{ github.event.pull_request.base.sha }} \ -f state="$state" \ -f description="Artifact Review Check" \ -f context="Artifact Review Check" \ -f target_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" env: GH_TOKEN: ${{ secrets.FISHTOWN_BOT_PAT }} ================================================ FILE: .github/workflows/auto-respond-bug-reports.yml ================================================ # **what?** # Check if the an issue is opened near or during an extended holiday period. # If so, post an automatically-generated comment about the holiday for bug reports. # Also provide specific information to customers of dbt Cloud. # **why?** # Explain why responses will be delayed during our holiday period. # **when?** # This will run when new issues are opened. name: Auto-Respond to Bug Reports During Holiday Period on: issues: types: - opened permissions: contents: read issues: write jobs: auto-response: runs-on: ${{ vars.UBUNTU_LATEST }} steps: - name: Check if current date is within holiday period id: date-check run: | current_date=$(date -u +"%Y-%m-%d") start_date="2024-12-23" end_date="2025-01-05" if [[ "$current_date" < "$start_date" || "$current_date" > "$end_date" ]]; then echo "outside_holiday=true" >> $GITHUB_ENV else echo "outside_holiday=false" >> $GITHUB_ENV fi - name: Post comment if: ${{ env.outside_holiday == 'false' && contains(github.event.issue.labels.*.name, 'bug') }} run: | gh issue comment ${{ github.event.issue.number }} --repo ${{ github.repository }} --body "Thank you for your bug report! Our team is will be out of the office for [Christmas and our Global Week of Rest](https://handbook.getdbt.com/docs/time_off#2024-us-holidays), from December 25, 2024, through January 3, 2025. We will review your issue as soon as possible after returning. Thank you for your understanding, and happy holidays! 🎄🎉 If you are a customer of dbt Cloud, please contact our Customer Support team via the dbt Cloud web interface or email **support@dbtlabs.com**." env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} ================================================ FILE: .github/workflows/backport.yml ================================================ # **what?** # When a PR is merged, if it has the backport label, it will create # a new PR to backport those changes to the given branch. If it can't # cleanly do a backport, it will comment on the merged PR of the failure. # # Label naming convention: "backport <branch name to backport to>" # Example: backport 1.0.latest # # You MUST "Squash and merge" the original PR or this won't work. # **why?** # Changes sometimes need to be backported to release branches. # This automates the backporting process # **when?** # Once a PR is "Squash and merge"'d, by adding a backport label, this is triggered name: Backport on: pull_request: types: - labeled permissions: contents: write pull-requests: write jobs: backport: name: Backport runs-on: ${{ vars.UBUNTU_LATEST }} # Only react to merged PRs for security reasons. # See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target. if: > github.event.pull_request.merged && contains(github.event.label.name, 'backport') steps: - uses: tibdex/backport@9565281eda0731b1d20c4025c43339fb0a23812e # tibdex/backport@v2.0.4 with: github_token: ${{ secrets.GITHUB_TOKEN }} ================================================ FILE: .github/workflows/bot-changelog.yml ================================================ # **what?** # When bots create a PR, this action will add a corresponding changie yaml file to that # PR when a specific label is added. # # The file is created off a template: # # kind: <per action matrix> # body: <PR title> # time: <current timestamp> # custom: # Author: <PR User Login (generally the bot)> # Issue: 4904 # PR: <PR number> # # **why?** # Automate changelog generation for more visability with automated bot PRs. # # **when?** # Once a PR is created, label should be added to PR before or after creation. You can also # manually trigger this by adding the appropriate label at any time. # # **how to add another bot?** # Add the label and changie kind to the include matrix. That's it! # name: Bot Changelog on: pull_request: # catch when the PR is opened with the label or when the label is added types: [labeled] permissions: contents: write pull-requests: read jobs: generate_changelog: strategy: matrix: include: - label: "dependencies" changie_kind: "Dependencies" runs-on: ${{ vars.UBUNTU_LATEST }} steps: - name: Create and commit changelog on bot PR if: ${{ contains(github.event.pull_request.labels.*.name, matrix.label) }} id: bot_changelog uses: emmyoop/changie_bot@22b70618b13d0d1c64ea95212bafca2d2bf6b764 # emmyoop/changie_bot@v1.1.0 with: GITHUB_TOKEN: ${{ secrets.FISHTOWN_BOT_PAT }} commit_author_name: "Github Build Bot" commit_author_email: "<buildbot@fishtownanalytics.com>" commit_message: "Add automated changelog yaml from template for bot PR" changie_kind: ${{ matrix.changie_kind }} label: ${{ matrix.label }} custom_changelog_string: "custom:\n Author: ${{ github.event.pull_request.user.login }}\n Issue: ${{ github.event.pull_request.number }}" ================================================ FILE: .github/workflows/changelog-existence.yml ================================================ # **what?** # Checks that a file has been committed under the /.changes directory # as a new CHANGELOG entry. Cannot check for a specific filename as # it is dynamically generated by change type and timestamp. # This workflow runs on pull_request_target because it requires # secrets to post comments. # **why?** # Ensure code change gets reflected in the CHANGELOG. # **when?** # This will run for all PRs going into main and *.latest. It will # run when they are opened, reopened, when any label is added or removed # and when new code is pushed to the branch. The action will then get # skipped if the 'Skip Changelog' label is present is any of the labels. name: Check Changelog Entry on: pull_request_target: types: [opened, reopened, labeled, unlabeled, synchronize] paths-ignore: ['.changes/**', '.github/**', 'tests/**', '**.md', '**.yml'] workflow_dispatch: defaults: run: shell: bash permissions: contents: read pull-requests: write jobs: changelog: uses: dbt-labs/actions/.github/workflows/changelog-existence.yml@main with: changelog_comment: 'Thank you for your pull request! We could not find a changelog entry for this change. For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-core/blob/main/CONTRIBUTING.md#adding-changelog-entry).' skip_label: 'Skip Changelog' secrets: inherit ================================================ FILE: .github/workflows/check-artifact-changes.yml ================================================ name: Check Artifact Changes on: pull_request: types: [ opened, reopened, labeled, unlabeled, synchronize ] paths-ignore: [ '.changes/**', '.github/**', 'tests/**', '**.md', '**.yml' ] merge_group: types: [checks_requested] workflow_dispatch: permissions: contents: read jobs: check-artifact-changes: runs-on: ${{ vars.UBUNTU_LATEST }} if: ${{ !contains(github.event.pull_request.labels.*.name, 'artifact_minor_upgrade') }} steps: - name: Checkout code uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4 with: fetch-depth: 0 - name: Check for changes in core/dbt/artifacts # https://github.com/marketplace/actions/paths-changes-filter uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # dorny/paths-filter@v3 id: check_artifact_changes with: filters: | artifacts_changed: - 'core/dbt/artifacts/**' list-files: shell - name: Fail CI if artifacts have changed if: steps.check_artifact_changes.outputs.artifacts_changed == 'true' run: | echo "CI failure: Artifact changes checked in core/dbt/artifacts directory." echo "Files changed: ${{ steps.check_artifact_changes.outputs.artifacts_changed_files }}" echo "To bypass this check, confirm that the change is not breaking (https://github.com/dbt-labs/dbt-core/blob/main/core/dbt/artifacts/README.md#breaking-changes) and add the 'artifact_minor_upgrade' label to the PR. Modifications and additions to all fields require updates to https://github.com/dbt-labs/dbt-jsonschema." exit 1 - name: CI check passed if: steps.check_artifact_changes.outputs.artifacts_changed == 'false' run: | echo "No prohibited artifact changes found in core/dbt/artifacts. CI check passed." ================================================ FILE: .github/workflows/community-label.yml ================================================ # **what?** # Label a PR with a `community` label when a PR is opened by a user outside core/adapters # **why?** # To streamline triage and ensure that community contributions are recognized and prioritized # **when?** # When a PR is opened, not in draft or moved from draft to ready for review name: Label community PRs on: # have to use pull_request_target since community PRs come from forks pull_request_target: types: [opened, ready_for_review] defaults: run: shell: bash permissions: pull-requests: write # labels PRs contents: read # reads team membership jobs: open_issues: # If this PR already has the community label, no need to relabel it # If this PR is opened and not draft, determine if it needs to be labeled # if the PR is converted out of draft, determine if it needs to be labeled if: | ( !contains(github.event.pull_request.labels.*.name, 'community') && ( (github.event.action == 'opened' && github.event.pull_request.draft == false) || github.event.action == 'ready_for_review' ) && github.event.pull_request.user.type != 'Bot' && github.event.pull_request.user.login != 'dependabot[bot]' ) uses: dbt-labs/actions/.github/workflows/label-community.yml@main with: github_team: 'core-group' label: 'community' secrets: inherit ================================================ FILE: .github/workflows/cut-release-branch.yml ================================================ # **what?** # Cuts the `*.latest` branch, bumps dependencies on it, cleans up all files in `.changes/unreleased` # and `.changes/previous verion on main and bumps main to the input version. # **why?** # Clean up the main branch after a release branch is cut and automate cutting the release branch. # Generally reduces the workload of engineers and reducing error. # **when?** # This will run when called manually or when triggered in another workflow. # Example Usage including required permissions: TODO: update once finalized # permissions: # contents: read # pull-requests: write # # name: Cut Release Branch # jobs: # changelog: # uses: dbt-labs/actions/.github/workflows/cut-release-branch.yml@main # with: # new_branch_name: 1.7.latest # PR_title: "Cleanup main after cutting new 1.7.latest branch" # PR_body: "All adapter PRs will fail CI until the dbt-core PR has been merged due to release version conflicts." # secrets: # FISHTOWN_BOT_PAT: ${{ secrets.FISHTOWN_BOT_PAT }} # TODOs # add note to eventually commit changes directly and bypass checks - same as release - when we move to this model run test action after merge name: Cut new release branch run-name: "Cutting New Branch: ${{ inputs.new_branch_name }}" on: workflow_dispatch: inputs: new_branch_name: description: "The full name of the new branch (ex. 1.5.latest)" required: true type: string defaults: run: shell: bash permissions: contents: write pull-requests: write env: PYTHON_TARGET_VERSION: "3.10" PR_TITLE: "Cleanup main after cutting new ${{ inputs.new_branch_name }} branch" PR_BODY: "All adapter PRs will fail CI until the dbt-core PR has been merged due to release version conflicts." jobs: prep_work: name: "Prep Work" runs-on: ubuntu-latest steps: - name: "[DEBUG] Print Inputs" run: | echo "new_branch_name: ${{ inputs.new_branch_name }}" echo "PR_title: ${{ env.PR_TITLE }}" echo "PR_body: ${{ env.PR_BODY }}" create_temp_branch: name: "Create Temp branch off main" runs-on: ubuntu-latest outputs: temp_branch_name: ${{ steps.variables.outputs.BRANCH_NAME }} steps: - name: "Set Branch Value" id: variables run: | echo "BRANCH_NAME=cutting_release_branch/main_cleanup_$GITHUB_RUN_ID" >> $GITHUB_OUTPUT - name: "Checkout ${{ github.repository }}" uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4 with: ref: "main" token: ${{ secrets.FISHTOWN_BOT_PAT }} - name: "Create PR Branch" run: | user="Github Build Bot" email="buildbot@fishtownanalytics.com" git config user.name "$user" git config user.email "$email" git checkout -b ${{ steps.variables.outputs.BRANCH_NAME }} git push --set-upstream origin ${{ steps.variables.outputs.BRANCH_NAME }} - name: "[Notification] Temp branch created" run: | message="Temp branch ${{ steps.variables.outputs.BRANCH_NAME }} created" echo "::notice title="Temporary branch created": $title::$message" cleanup_changelog: name: "Clean Up Changelog" needs: ["create_temp_branch"] runs-on: ubuntu-latest outputs: next-version: ${{ steps.semver-current.outputs.next-minor-alpha-version }} steps: - name: "Checkout ${{ github.repository }}" uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4 with: ref: ${{ needs.create_temp_branch.outputs.temp_branch_name }} token: ${{ secrets.FISHTOWN_BOT_PAT }} - name: "Add Homebrew To PATH" run: | echo "/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin" >> $GITHUB_PATH - name: "Install Homebrew Packages" run: | brew install pre-commit brew tap miniscruff/changie https://github.com/miniscruff/changie brew install changie - name: "Check Current Version In Code" id: determine_version run: | current_version=$(grep '^version = ' core/pyproject.toml | sed 's/version = "\(.*\)"/\1/') echo "current_version=$current_version" >> $GITHUB_OUTPUT - name: "[Notification] Check Current Version In Code" run: | message="The current version is ${{ steps.determine_version.outputs.current_version }}" echo "::notice title="Version Bump Check": $title::$message" - name: "Parse Current Version Into Parts for Changelog Directories" id: semver-current uses: dbt-labs/actions/parse-semver@v1 with: version: ${{ steps.determine_version.outputs.current_version }} - name: "[Notification] Next Alpha Version" run: | message="The next alpha version is ${{ steps.semver-current.outputs.next-minor-alpha-version }}" echo "::notice title="Version Bump Check": $title::$message" - name: "Delete Unreleased Changelog YAMLs" # removal fails if no files exist. OK to continue since we're just cleaning up the files. continue-on-error: true run: | rm .changes/unreleased/*.yaml || true - name: "Delete Pre Release Changelogs and YAMLs" # removal fails if no files exist. OK to continue since we're just cleaning up the files. continue-on-error: true run: | rm .changes/${{ steps.semver-current.outputs.base-version }}/*.yaml || true rm .changes/${{ steps.semver-current.outputs.major }}.${{ steps.semver-current.outputs.minor }}.*.md || true - name: "Cleanup CHANGELOG.md" run: | changie merge - name: "Commit Changelog Cleanup to Branch" run: | user="Github Build Bot" email="buildbot@fishtownanalytics.com" git config user.name "$user" git config user.email "$email" git status git add . git commit -m "Clean up changelog on main" git push - name: "[Notification] Changelog cleaned up" run: | message="Changelog on ${{ needs.create_temp_branch.outputs.temp_branch_name }} cleaned up" echo "::notice title="Changelog cleaned up": $title::$message" bump_version: name: "Bump to next minor version" needs: ["cleanup_changelog", "create_temp_branch"] runs-on: ubuntu-latest steps: - name: "Checkout ${{ github.repository }}" uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4 with: ref: ${{ needs.create_temp_branch.outputs.temp_branch_name }} token: ${{ secrets.FISHTOWN_BOT_PAT }} - name: "Set up Python - ${{ env.PYTHON_TARGET_VERSION }}" uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # actions/setup-python@v5 with: python-version: "${{ env.PYTHON_TARGET_VERSION }}" - name: "Install Spark Dependencies" if: ${{ contains(github.repository, 'dbt-labs/dbt-spark') }} run: | sudo apt-get update sudo apt-get install libsasl2-dev - name: "Install hatch" uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc # pypa/hatch@install - name: "Bump Version To ${{ needs.cleanup_changelog.outputs.next-version }}" run: | cd core hatch version ${{ needs.cleanup_changelog.outputs.next-version }} hatch run dev-req dbt --version - name: "Commit Version Bump to Branch" run: | user="Github Build Bot" email="buildbot@fishtownanalytics.com" git config user.name "$user" git config user.email "$email" git status git add . git commit -m "Bumping version to ${{ needs.cleanup_changelog.outputs.next-version }}" git push - name: "[Notification] Version Bump completed" run: | message="Version on ${{ needs.create_temp_branch.outputs.temp_branch_name }} bumped to ${{ needs.cleanup_changelog.outputs.next-version }}" echo "::notice title="Version Bump Completed": $title::$message" cleanup: name: "Cleanup Code Quality" needs: ["create_temp_branch", "bump_version"] runs-on: ubuntu-latest steps: - name: "Checkout ${{ github.repository }}" uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4 with: ref: ${{ needs.create_temp_branch.outputs.temp_branch_name }} token: ${{ secrets.FISHTOWN_BOT_PAT }} - name: "Add Homebrew To PATH" run: | echo "/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin" >> $GITHUB_PATH - name: "brew install pre-commit" run: | brew install pre-commit # this step will fail on whitespace errors but also correct them - name: "Cleanup - Remove Trailing Whitespace Via Pre-commit" continue-on-error: true run: | pre-commit run trailing-whitespace --files CHANGELOG.md .changes/* || true # this step will fail on newline errors but also correct them - name: "Cleanup - Remove Extra Newlines Via Pre-commit" continue-on-error: true run: | pre-commit run end-of-file-fixer --files CHANGELOG.md .changes/* || true - name: "Commit Version Bump to Branch" run: | user="Github Build Bot" email="buildbot@fishtownanalytics.com" git config user.name "$user" git config user.email "$email" git status git add . git commit -m "Code quality cleanup" git push open_pr: name: "Open PR Against main" needs: ["cleanup_changelog", "create_temp_branch", "cleanup"] runs-on: ubuntu-latest outputs: pr_number: ${{ steps.create_pr.outputs.pull-request-number }} steps: - name: "Checkout ${{ github.repository }}" uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4 with: ref: ${{ needs.create_temp_branch.outputs.temp_branch_name }} token: ${{ secrets.FISHTOWN_BOT_PAT }} - name: "Determine PR Title" id: pr_title run: | echo "pr_title=${{ env.PR_TITLE }}" >> $GITHUB_OUTPUT if [${{ env.PR_TITLE }} == ""]; then echo "pr_title='Clean up changelogs and bump to version ${{ needs.cleanup_changelog.outputs.next-version }}'" >> $GITHUB_OUTPUT fi - name: "Determine PR Body" id: pr_body run: | echo "pr_body=${{ env.PR_BODY }}" >> $GITHUB_OUTPUT if [${{ env.PR_BODY }} == ""]; then echo "pr_body='Clean up changelogs and bump to version ${{ needs.cleanup_changelog.outputs.next-version }}'" >> $GITHUB_OUTPUT fi - name: "Add Branch Details" id: pr_body_branch run: | branch_details="The workflow that generated this PR also created a new branch: ${{ inputs.new_branch_name }}" full_body="${{ steps.pr_body.outputs.pr_body }} $branch_details" echo "pr_full_body=$full_body" >> $GITHUB_OUTPUT - name: "Open Pull Request" id: create_pr run: | pr_url=$(gh pr create -B main -H ${{ needs.create_temp_branch.outputs.temp_branch_name }} -l "Skip Changelog" -t "${{ steps.pr_title.outputs.pr_title }}" -b "${{ steps.pr_body_branch.outputs.pr_full_body }}") echo "pr_url=$pr_url" >> $GITHUB_OUTPUT env: GH_TOKEN: ${{ secrets.FISHTOWN_BOT_PAT }} - name: "[Notification] Pull Request Opened" run: | message="PR opened at ${{ steps.create_pr.outputs.pr_url }}" echo "::notice title="Pull Request Opened": $title::$message" cut_new_branch: # don't cut the new branch until we're done opening the PR against main name: "Cut New Branch ${{ inputs.new_branch_name }}" needs: [open_pr] runs-on: ubuntu-latest steps: - name: "Checkout ${{ github.repository }}" uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4 with: token: ${{ secrets.FISHTOWN_BOT_PAT }} fetch-depth: 0 - name: "Ensure New Branch Does Not Exist" id: check_new_branch run: | title="Check New Branch Existence" if git show-ref --quiet ${{ inputs.new_branch_name }}; then message="Branch ${{ inputs.new_branch_name }} already exists. Exiting." echo "::error $title::$message" exit 1 fi - name: "Create New Release Branch" run: | git checkout -b ${{ inputs.new_branch_name }} - name: "Push up New Branch" run: | #Data for commit user="Github Build Bot" email="buildbot@fishtownanalytics.com" git config user.name "$user" git config user.email "$email" git push --set-upstream origin ${{ inputs.new_branch_name }} - name: "[Notification] New branch created" run: | message="New branch ${{ inputs.new_branch_name }} created" echo "::notice title="New branch created": $title::$message" - name: "Bump dependencies via script" # This bumps the dependency on dbt-core in the adapters if: ${{ !contains(github.repository, 'dbt-core') }} run: | echo ${{ github.repository }} echo "running update_dependencies script" bash ${GITHUB_WORKSPACE}/.github/scripts/update_dependencies.sh ${{ inputs.new_branch_name }} commit_message="bumping .latest branch variable in update_dependencies.sh to ${{ inputs.new_branch_name }}" git status git add . git commit -m "$commit_message" git push - name: "Bump env variable via script" # bumps the RELEASE_BRANCH variable in nightly-release.yml in adapters if: ${{ !contains(github.repository, 'dbt-core') }} run: | file="./.github/scripts/update_release_branch.sh" if test -f "$file"; then echo ${{ github.repository }} echo "running some script yet to be written now" bash $file ${{ inputs.new_branch_name }} commit_message="updating env variable to ${{ inputs.new_branch_name }} in nightly-release.yml" git status git add . git commit -m "$commit_message" git push else echo "no $file seen skipping step" fi ================================================ FILE: .github/workflows/docs-issue.yml ================================================ # **what?** # Open an issue in docs.getdbt.com when an issue is labeled `user docs` and closed as completed # **why?** # To reduce barriers for keeping docs up to date # **when?** # When an issue is labeled `user docs` and is closed as completed. Can be labeled before or after the issue is closed. name: Open issues in docs.getdbt.com repo when an issue is labeled run-name: "Open an issue in docs.getdbt.com for issue #${{ github.event.issue.number }}" on: issues: types: [labeled, closed] defaults: run: shell: bash permissions: issues: write # comments on issues jobs: get_pr_info: # we only want to run this when the issue is closed as completed and the label `user docs` has been assigned. # If this logic does not exist in this workflow, it runs the # risk of duplicaton of issues being created due to merge and label both triggering this workflow to run and neither having # generating the comment before the other runs. This lives here instead of the shared workflow because this is where we # decide if it should run or not. if: | (github.event.issue.state == 'closed' && github.event.issue.state_reason == 'completed' && contains( github.event.issue.labels.*.name, 'user docs')) runs-on: ubuntu-latest outputs: pr_url: ${{ steps.find_pr.outputs.pr_url }} pr_title: ${{ steps.find_pr.outputs.pr_title }} steps: - name: Find closing PR via timeline id: find_pr uses: actions/github-script@v7 with: script: | const result = await github.graphql(` query($owner: String!, $repo: String!, $number: Int!) { repository(owner: $owner, name: $repo) { issue(number: $number) { timelineItems(itemTypes: [CLOSED_EVENT], last: 1) { nodes { ... on ClosedEvent { closer { ... on PullRequest { title url } } } } } } } }`, { owner: context.repo.owner, repo: context.repo.repo, number: context.issue.number, }); const closer = result.repository.issue.timelineItems.nodes[0]?.closer; core.setOutput('pr_url', closer?.url || 'No linked PR found'); core.setOutput('pr_title', closer?.title || 'N/A'); open_issues: needs: get_pr_info uses: dbt-labs/actions/.github/workflows/open-issue-in-repo.yml@main with: issue_repository: "dbt-labs/docs.getdbt.com" issue_title: "[Core] Docs Changes Needed from ${{ github.event.repository.name }} Issue #${{ github.event.issue.number }}" issue_body: "Originating from this issue: https://github.com/dbt-labs/dbt-core/issues/${{ github.event.issue.number }}\nRelated PR: ${{ needs.get_pr_info.outputs.pr_url }}\nPR title: ${{ needs.get_pr_info.outputs.pr_title }}\n\nAt a minimum, update body to include a link to the page on docs.getdbt.com requiring updates and what part(s) of the page you would like to see updated." secrets: inherit ================================================ FILE: .github/workflows/main.yml ================================================ # **what?** # Runs code quality checks, unit tests, integration tests and # verifies python build on all code commited to the repository. This workflow # should not require any secrets since it runs for PRs from forked repos. By # default, secrets are not passed to workflows running from a forked repos. # **why?** # Ensure code for dbt meets a certain quality standard. # **when?** # This will run for all PRs, when code is pushed to a release # branch, and when manually triggered. name: Tests and Code Checks on: push: branches: - "main" - "*.latest" - "releases/*" pull_request: merge_group: types: [checks_requested] workflow_dispatch: permissions: read-all # will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} cancel-in-progress: true defaults: run: shell: bash # top-level adjustments can be made here env: # number of parallel processes to spawn for python integration testing PYTHON_INTEGRATION_TEST_WORKERS: 15 jobs: code-quality: name: code-quality runs-on: ubuntu-latest timeout-minutes: 10 steps: - name: Check out the repository uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4 - name: Set up Python uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # actions/setup-python@v6 with: python-version: "3.10" - name: "Install hatch" uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc # pypa/hatch@install - name: Install python dependencies run: | cd core hatch -v run setup - name: Verify dbt installation run: | cd core hatch run dbt --version - name: Run pre-commit hooks run: | cd core hatch run code-quality unit: name: "unit test / python ${{ matrix.python-version }}" runs-on: ubuntu-latest timeout-minutes: 10 strategy: fail-fast: false matrix: python-version: ["3.10", "3.11", "3.12", "3.13"] steps: - name: Check out the repository uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - name: "Install hatch" uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc # pypa/hatch@install - name: Run unit tests uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # nick-fields/retry@v3 with: timeout_minutes: 10 max_attempts: 3 command: cd core && hatch -v run ci:unit-tests - name: Get current date if: always() id: date run: | CURRENT_DATE=$(date +'%Y-%m-%dT%H_%M_%S') # no colons allowed for artifacts echo "date=$CURRENT_DATE" >> $GITHUB_OUTPUT - name: Upload Unit Test Coverage to Codecov if: ${{ matrix.python-version == '3.11' }} uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} flags: unit fail_ci_if_error: false integration-metadata: name: integration test metadata generation runs-on: ubuntu-latest outputs: split-groups: ${{ steps.generate-split-groups.outputs.split-groups }} include: ${{ steps.generate-include.outputs.include }} steps: - name: generate split-groups id: generate-split-groups run: | MATRIX_JSON="[" for B in $(seq 1 ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }}); do MATRIX_JSON+=$(sed 's/^/"/;s/$/"/' <<< "${B}") done MATRIX_JSON="${MATRIX_JSON//\"\"/\", \"}" MATRIX_JSON+="]" echo "split-groups=${MATRIX_JSON}" echo "split-groups=${MATRIX_JSON}" >> $GITHUB_OUTPUT - name: generate include id: generate-include run: | INCLUDE=('"python-version":"3.10","os":"windows-latest"' '"python-version":"3.10","os":"macos-14"' ) INCLUDE_GROUPS="[" for include in ${INCLUDE[@]}; do for group in $(seq 1 ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }}); do INCLUDE_GROUPS+=$(sed 's/$/, /' <<< "{\"split-group\":\"${group}\",${include}}") done done INCLUDE_GROUPS=$(echo $INCLUDE_GROUPS | sed 's/,*$//g') INCLUDE_GROUPS+="]" echo "include=${INCLUDE_GROUPS}" echo "include=${INCLUDE_GROUPS}" >> $GITHUB_OUTPUT integration-postgres: name: "(${{ matrix.split-group }}) integration test / python ${{ matrix.python-version }} / ${{ matrix.os }}" runs-on: ${{ matrix.os }} timeout-minutes: 45 needs: - integration-metadata strategy: fail-fast: false matrix: python-version: ["3.10", "3.11", "3.12", "3.13"] os: ["ubuntu-latest"] split-group: ${{ fromJson(needs.integration-metadata.outputs.split-groups) }} env: DBT_INVOCATION_ENV: github-actions DBT_TEST_USER_1: dbt_test_user_1 DBT_TEST_USER_2: dbt_test_user_2 DBT_TEST_USER_3: dbt_test_user_3 DD_CIVISIBILITY_AGENTLESS_ENABLED: true DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} DD_SITE: datadoghq.com DD_ENV: ci DD_SERVICE: ${{ github.event.repository.name }} services: # Label used to access the service container postgres: # Docker Hub image image: postgres # Provide the password for postgres env: POSTGRES_PASSWORD: password POSTGRES_USER: postgres # Set health checks to wait until postgres has started options: >- --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 ports: - 5432:5432 steps: - name: Check out the repository uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - name: Run postgres setup script run: | ./scripts/setup_db.sh env: PGHOST: localhost PGPORT: 5432 PGPASSWORD: password - name: "Install hatch" uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc # pypa/hatch@install - name: Run integration tests uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # nick-fields/retry@v3 with: timeout_minutes: 30 max_attempts: 3 shell: bash command: cd core && hatch -v run ci:integration-tests -- --ddtrace --splits ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }} --group ${{ matrix.split-group }} - name: Get current date if: always() id: date run: | CURRENT_DATE=$(date +'%Y-%m-%dT%H_%M_%S') # no colons allowed for artifacts echo "date=$CURRENT_DATE" >> $GITHUB_OUTPUT - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # actions/upload-artifact@v4 if: always() with: name: logs_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.split-group }}_${{ steps.date.outputs.date }} path: ./logs - name: Upload Integration Test Coverage to Codecov if: ${{ matrix.python-version == '3.11' }} uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} flags: integration fail_ci_if_error: false integration-mac-windows: name: (${{ matrix.split-group }}) integration test / python ${{ matrix.python-version }} / ${{ matrix.os }} runs-on: ${{ matrix.os }} timeout-minutes: 45 needs: - integration-metadata strategy: fail-fast: false matrix: # already includes split group and runs mac + windows include: ${{ fromJson(needs.integration-metadata.outputs.include) }} env: DBT_INVOCATION_ENV: github-actions DBT_TEST_USER_1: dbt_test_user_1 DBT_TEST_USER_2: dbt_test_user_2 DBT_TEST_USER_3: dbt_test_user_3 DD_CIVISIBILITY_AGENTLESS_ENABLED: true DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} DD_SITE: datadoghq.com DD_ENV: ci DD_SERVICE: ${{ github.event.repository.name }} steps: - name: Check out the repository uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - name: Set up postgres (macos) if: runner.os == 'macOS' uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # nick-fields/retry@v3 with: timeout_minutes: 10 max_attempts: 3 command: ./scripts/setup_db.sh - name: Set up postgres (windows) if: runner.os == 'Windows' uses: ./.github/actions/setup-postgres-windows - name: "Install hatch" uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc # pypa/hatch@install - name: Run integration tests uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # nick-fields/retry@v3 with: timeout_minutes: 30 max_attempts: 3 shell: bash command: cd core && hatch -v run ci:integration-tests -- --ddtrace --splits ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }} --group ${{ matrix.split-group }} - name: Get current date if: always() id: date run: | CURRENT_DATE=$(date +'%Y-%m-%dT%H_%M_%S') # no colons allowed for artifacts echo "date=$CURRENT_DATE" >> $GITHUB_OUTPUT - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # actions/upload-artifact@v4 if: always() with: name: logs_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.split-group }}_${{ steps.date.outputs.date }} path: ./logs - name: Upload Integration Test Coverage if: ${{ matrix.python-version == '3.11' }} uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} flags: integration fail_ci_if_error: false integration-report: if: ${{ always() }} name: Integration Test Suite runs-on: ubuntu-latest needs: [integration-mac-windows, integration-postgres] steps: - name: "Integration Tests Failed" if: ${{ contains(needs.integration-mac-windows.result, 'failure') || contains(needs.integration-mac-windows.result, 'cancelled') || contains(needs.integration-postgres.result, 'failure') || contains(needs.integration-postgres.result, 'cancelled') }} # when this is true the next step won't execute run: | echo "::notice title='Integration test suite failed'" exit 1 - name: "Integration Tests Passed" run: | echo "::notice title='Integration test suite passed'" build: name: build packages runs-on: ubuntu-latest steps: - name: Check out the repository uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4 - name: Set up Python uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # actions/setup-python@v6 with: python-version: "3.10" - name: "Install hatch" uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc # pypa/hatch@install - name: Install python dependencies run: | python -m pip install --user --upgrade pip python -m pip install --upgrade twine check-wheel-contents python -m pip --version - name: "Build Distributions" working-directory: core run: hatch build - name: "Show Distributions" working-directory: core run: ls -lh dist/ - name: "Check and Verify Distributions" working-directory: core run: hatch -v run build:check-all ================================================ FILE: .github/workflows/nightly-release.yml ================================================ # **what?** # Nightly releases to GitHub and PyPI. This workflow produces the following outcome: # - generate and validate data for night release (commit SHA, version number, release branch); # - pass data to release workflow; # - night release will be pushed to GitHub as a draft release; # - night build will be pushed to test PyPI; # # **why?** # Ensure an automated and tested release process for nightly builds # # **when?** # This workflow runs on schedule or can be run manually on demand. name: Nightly Test Release to GitHub and PyPI on: workflow_dispatch: # for manual triggering schedule: - cron: 0 9 * * * permissions: contents: write # this is the permission that allows creating a new release packages: write # this is the permission that allows Docker release defaults: run: shell: bash env: RELEASE_BRANCH: "main" jobs: aggregate-release-data: runs-on: ${{ vars.UBUNTU_LATEST }} outputs: version_number: ${{ steps.nightly-release-version.outputs.number }} release_branch: ${{ steps.release-branch.outputs.name }} steps: - name: "Checkout ${{ github.repository }} Branch ${{ env.RELEASE_BRANCH }}" uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4 with: ref: ${{ env.RELEASE_BRANCH }} - name: "Get Current Version Number" id: version-number-sources run: | current_version=$(grep '^version = ' core/dbt/__version__.py | sed 's/version = "\(.*\)"/\1/') echo "current_version=$current_version" >> $GITHUB_OUTPUT - name: "Audit Version And Parse Into Parts" id: semver uses: dbt-labs/actions/parse-semver@v1 with: version: ${{ steps.version-number-sources.outputs.current_version }} - name: "Get Current Date" id: current-date run: echo "date=$(date +'%Y%m%d')" >> $GITHUB_OUTPUT - name: "Generate Nightly Release Version Number" id: nightly-release-version run: | number="${{ steps.semver.outputs.version }}.post${{ steps.current-date.outputs.date }}" echo "number=$number" >> $GITHUB_OUTPUT - name: "Audit Nightly Release Version And Parse Into Parts" uses: dbt-labs/actions/parse-semver@v1 with: version: ${{ steps.nightly-release-version.outputs.number }} - name: "Set Release Branch" id: release-branch run: | echo "name=${{ env.RELEASE_BRANCH }}" >> $GITHUB_OUTPUT log-outputs-aggregate-release-data: runs-on: ${{ vars.UBUNTU_LATEST }} needs: [aggregate-release-data] steps: - name: "[DEBUG] Log Outputs" run: | echo version_number: ${{ needs.aggregate-release-data.outputs.version_number }} echo release_branch: ${{ needs.aggregate-release-data.outputs.release_branch }} release-github-pypi: needs: [aggregate-release-data] uses: ./.github/workflows/release.yml with: target_branch: ${{ needs.aggregate-release-data.outputs.release_branch }} version_number: ${{ needs.aggregate-release-data.outputs.version_number }} test_run: true nightly_release: true secrets: inherit ================================================ FILE: .github/workflows/release-branch-tests.yml ================================================ # **what?** # The purpose of this workflow is to trigger CI to run for each # release branch and main branch on a regular cadence. If the CI workflow # fails for a branch, it will post to #dev-core-alerts to raise awareness. # **why?** # Ensures release branches and main are always shippable and not broken. # Also, can catch any dependencies shifting beneath us that might # introduce breaking changes (could also impact Cloud). # **when?** # Mainly on a schedule of 9:00, 13:00, 18:00 UTC everyday. # Manual trigger can also test on demand name: Release branch scheduled testing on: schedule: - cron: '0 9,13,18 * * *' # 9:00, 13:00, 18:00 UTC workflow_dispatch: # for manual triggering # no special access is needed permissions: read-all jobs: run_tests: uses: dbt-labs/actions/.github/workflows/release-branch-tests.yml@main with: workflows_to_run: '["main.yml"]' secrets: inherit ================================================ FILE: .github/workflows/release.yml ================================================ # **what?** # Release workflow provides the following steps: # - checkout the given commit; # - validate version in sources and changelog file for given version; # - bump the version and generate a changelog if needed; # - merge all changes to the target branch if needed; # - run unit and integration tests against given commit; # - build and package that SHA; # - release it to GitHub and PyPI with that specific build; # - release it to Docker # # **why?** # Ensure an automated and tested release process # # **when?** # This workflow can be run manually on demand or can be called by other workflows name: "Release to GitHub, PyPI & Docker" run-name: "Release ${{ inputs.version_number }} to GitHub, PyPI & Docker" on: workflow_dispatch: inputs: target_branch: description: "The branch to release from" type: string required: true version_number: description: "The release version number (i.e. 1.0.0b1)" type: string required: true test_run: description: "Test run (Publish release as draft)" type: boolean default: true required: false nightly_release: description: "Nightly release to dev environment" type: boolean default: false required: false only_docker: description: "Only release Docker image, skip GitHub & PyPI" type: boolean default: false required: false workflow_call: inputs: target_branch: description: "The branch to release from" type: string required: true version_number: description: "The release version number (i.e. 1.0.0b1)" type: string required: true test_run: description: "Test run (Publish release as draft)" type: boolean default: true required: false nightly_release: description: "Nightly release to dev environment" type: boolean default: false required: false permissions: contents: write # this is the permission that allows creating a new release defaults: run: shell: bash env: MIN_HATCH_VERSION: "1.11.0" jobs: job-setup: name: Log Inputs runs-on: ${{ vars.UBUNTU_LATEST }} outputs: use_hatch: ${{ steps.use_hatch.outputs.use_hatch }} steps: - name: "[DEBUG] Print Variables" run: | echo Inputs echo The branch to release from: ${{ inputs.target_branch }} echo The release version number: ${{ inputs.version_number }} echo Test run: ${{ inputs.test_run }} echo Nightly release: ${{ inputs.nightly_release }} echo Only Docker: ${{ inputs.only_docker }} # In version env.HATCH_VERSION we started to use hatch for build tooling. Before that we used setuptools. # This needs to check if we're using hatch or setuptools based on the version being released. We should # check if the version is greater than or equal to env.HATCH_VERSION. If it is, we use hatch, otherwise we use setuptools. - name: "Check if using hatch" id: use_hatch run: | # Extract major.minor from versions like 1.11.0a1 -> 1.11 INPUT_MAJ_MIN=$(echo "${{ inputs.version_number }}" | sed -E 's/^([0-9]+\.[0-9]+).*/\1/') HATCH_MAJ_MIN=$(echo "${{ env.MIN_HATCH_VERSION }}" | sed -E 's/^([0-9]+\.[0-9]+).*/\1/') if [ $(echo "$INPUT_MAJ_MIN >= $HATCH_MAJ_MIN" | bc) -eq 1 ]; then echo "use_hatch=true" >> $GITHUB_OUTPUT else echo "use_hatch=false" >> $GITHUB_OUTPUT fi - name: "Notify if using hatch" run: | if [ ${{ steps.use_hatch.outputs.use_hatch }} = "true" ]; then echo "::notice title="Using Hatch": $title::Using Hatch for release" else echo "::notice title="Using Setuptools": $title::Using Setuptools for release" fi bump-version-generate-changelog: name: Bump package version, Generate changelog needs: [job-setup] if: ${{ !inputs.only_docker }} uses: dbt-labs/dbt-release/.github/workflows/release-prep.yml@main with: version_number: ${{ inputs.version_number }} hatch_directory: "core" target_branch: ${{ inputs.target_branch }} env_setup_script_path: "scripts/env-setup.sh" test_run: ${{ inputs.test_run }} nightly_release: ${{ inputs.nightly_release }} use_hatch: ${{ needs.job-setup.outputs.use_hatch == 'true' }} # workflow outputs are strings... secrets: inherit log-outputs-bump-version-generate-changelog: name: "[Log output] Bump package version, Generate changelog" if: ${{ !failure() && !cancelled() && !inputs.only_docker }} needs: [bump-version-generate-changelog] runs-on: ${{ vars.UBUNTU_LATEST }} steps: - name: Print variables run: | echo Final SHA : ${{ needs.bump-version-generate-changelog.outputs.final_sha }} echo Changelog path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} build-test-package: name: Build, Test, Package if: ${{ !failure() && !cancelled() && !inputs.only_docker }} needs: [job-setup, bump-version-generate-changelog] uses: dbt-labs/dbt-release/.github/workflows/build.yml@main with: sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} version_number: ${{ inputs.version_number }} hatch_directory: "core" changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} build_script_path: "scripts/build-dist.sh" package_test_command: "dbt --version" test_run: ${{ inputs.test_run }} nightly_release: ${{ inputs.nightly_release }} use_hatch: ${{ needs.job-setup.outputs.use_hatch == 'true' }} # workflow outputs are strings... github-release: name: GitHub Release if: ${{ !failure() && !cancelled() && !inputs.only_docker }} needs: [job-setup, bump-version-generate-changelog, build-test-package] uses: dbt-labs/dbt-release/.github/workflows/github-release.yml@main with: sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} version_number: ${{ inputs.version_number }} changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} test_run: ${{ inputs.test_run }} use_hatch: ${{ needs.job-setup.outputs.use_hatch == 'true' }} # workflow outputs are strings... pypi-release: name: PyPI Release needs: [job-setup, github-release] uses: dbt-labs/dbt-release/.github/workflows/pypi-release.yml@main with: version_number: ${{ inputs.version_number }} test_run: ${{ inputs.test_run }} use_hatch: ${{ needs.job-setup.outputs.use_hatch == 'true' }} # workflow outputs are strings... secrets: PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }} TEST_PYPI_API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} determine-docker-package: # dbt-postgres exists within dbt-core for versions 1.7 and earlier but is a separate package for 1.8 and later. # determine if we need to release dbt-core or both dbt-core and dbt-postgres name: Determine Docker Package if: ${{ !failure() && !cancelled() }} runs-on: ${{ vars.UBUNTU_LATEST }} needs: [pypi-release] outputs: matrix: ${{ steps.determine-docker-package.outputs.matrix }} steps: - name: "Audit Version And Parse Into Parts" id: semver uses: dbt-labs/actions/parse-semver@v1 with: version: ${{ inputs.version_number }} - name: "Determine Packages to Release" id: determine-docker-package run: | if [ ${{ steps.semver.outputs.minor }} -ge 8 ]; then json_output={\"package\":[\"dbt-core\"]} else json_output={\"package\":[\"dbt-core\",\"dbt-postgres\"]} fi echo "matrix=$json_output" >> $GITHUB_OUTPUT docker-release: name: "Docker Release for ${{ matrix.package }}" needs: [determine-docker-package] # We cannot release to docker on a test run because it uses the tag in GitHub as # what we need to release but draft releases don't actually tag the commit so it # finds nothing to release if: ${{ !failure() && !cancelled() && (!inputs.test_run || inputs.only_docker) }} strategy: matrix: ${{fromJson(needs.determine-docker-package.outputs.matrix)}} permissions: packages: write uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@main with: package: ${{ matrix.package }} version_number: ${{ inputs.version_number }} test_run: ${{ inputs.test_run }} slack-notification: name: Slack Notification if: ${{ failure() && (!inputs.test_run || inputs.nightly_release) }} needs: [ bump-version-generate-changelog, build-test-package, github-release, pypi-release, docker-release, ] uses: dbt-labs/dbt-release/.github/workflows/slack-post-notification.yml@main with: status: "failure" secrets: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEV_CORE_ALERTS }} testing-slack-notification: # sends notifications to #slackbot-test name: Testing - Slack Notification if: ${{ failure() && inputs.test_run && !inputs.nightly_release }} needs: [ bump-version-generate-changelog, build-test-package, github-release, pypi-release, docker-release, ] uses: dbt-labs/dbt-release/.github/workflows/slack-post-notification.yml@main with: status: "failure" secrets: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TESTING_WEBHOOK_URL }} ================================================ FILE: .github/workflows/repository-cleanup.yml ================================================ # **what?** # Cleanup branches left over from automation and testing. Also cleanup # draft releases from release testing. # **why?** # The automations are leaving behind branches and releases that clutter # the repository. Sometimes we need them to debug processes so we don't # want them immediately deleted. Running on Saturday to avoid running # at the same time as an actual release to prevent breaking a release # mid-release. # **when?** # Mainly on a schedule of 12:00 Saturday. # Manual trigger can also run on demand name: Repository Cleanup on: schedule: - cron: '0 12 * * SAT' # At 12:00 on Saturday - details in `why` above workflow_dispatch: # for manual triggering permissions: contents: write jobs: cleanup-repo: uses: dbt-labs/actions/.github/workflows/repository-cleanup.yml@main secrets: inherit ================================================ FILE: .github/workflows/schema-check.yml ================================================ # **what?** # Compares the schema of the dbt version of the given ref vs # the latest official schema releases found in schemas.getdbt.com. # If there are differences, the workflow will fail and upload the # diff as an artifact. The metadata team should be alerted to the change. # # **why?** # Reaction work may need to be done if artifact schema changes # occur so we want to proactively alert to it. # # **when?** # Only can be run manually name: Artifact Schema Check on: # pull_request: # types: [ opened, reopened, labeled, unlabeled, synchronize ] # paths-ignore: [ '.changes/**', '.github/**', 'tests/**', '**.md', '**.yml' ] workflow_dispatch: inputs: target_branch: description: "The branch to check against" type: string default: "main" required: true # no special access is needed permissions: read-all env: LATEST_SCHEMA_PATH: ${{ github.workspace }}/new_schemas SCHEMA_DIFF_ARTIFACT: ${{ github.workspace }}/schema_changes.txt DBT_REPO_DIRECTORY: ${{ github.workspace }}/dbt SCHEMA_REPO_DIRECTORY: ${{ github.workspace }}/schemas.getdbt.com jobs: checking-schemas: name: "Post-merge schema changes required" runs-on: ${{ vars.UBUNTU_LATEST }} steps: - name: Set up Python uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # actions/setup-python@v6 with: python-version: "3.10" - name: Checkout dbt repo uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4 with: path: ${{ env.DBT_REPO_DIRECTORY }} ref: ${{ inputs.target_branch }} - name: Check for changes in core/dbt/artifacts # https://github.com/marketplace/actions/paths-changes-filter uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # dorny/paths-filter@v3 id: check_artifact_changes with: filters: | artifacts_changed: - 'core/dbt/artifacts/**' list-files: shell working-directory: ${{ env.DBT_REPO_DIRECTORY }} - name: Succeed if no artifacts have changed if: steps.check_artifact_changes.outputs.artifacts_changed == 'false' run: | echo "No artifact changes found in core/dbt/artifacts. CI check passed." - name: Checkout schemas.getdbt.com repo if: steps.check_artifact_changes.outputs.artifacts_changed == 'true' uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4 with: repository: dbt-labs/schemas.getdbt.com ref: "main" path: ${{ env.SCHEMA_REPO_DIRECTORY }} - name: "Install hatch" uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc # pypa/hatch@install - name: Generate current schema if: steps.check_artifact_changes.outputs.artifacts_changed == 'true' run: | cd ${{ env.DBT_REPO_DIRECTORY }}/core hatch run setup hatch run json-schema -- --path ${{ env.LATEST_SCHEMA_PATH }} # Copy generated schema files into the schemas.getdbt.com repo # Do a git diff to find any changes # Ignore any lines with date-like (yyyy-mm-dd) or version-like (x.y.z) changes - name: Compare schemas if: steps.check_artifact_changes.outputs.artifacts_changed == 'true' run: | cp -r ${{ env.LATEST_SCHEMA_PATH }}/dbt ${{ env.SCHEMA_REPO_DIRECTORY }} cd ${{ env.SCHEMA_REPO_DIRECTORY }} git diff -I='*[0-9]{4}-[0-9]{2}-[0-9]{2}' -I='*[0-9]+\.[0-9]+\.[0-9]+' --exit-code > ${{ env.SCHEMA_DIFF_ARTIFACT }} - name: Upload schema diff uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # actions/upload-artifact@v4 if: ${{ failure() && steps.check_artifact_changes.outputs.artifacts_changed == 'true' }} with: name: "schema_changes.txt" path: "${{ env.SCHEMA_DIFF_ARTIFACT }}" ================================================ FILE: .github/workflows/stale.yml ================================================ name: "Close stale issues and PRs" on: schedule: - cron: "30 1 * * *" permissions: issues: write pull-requests: write jobs: stale: uses: dbt-labs/actions/.github/workflows/stale-bot-matrix.yml@main ================================================ FILE: .github/workflows/structured-logging-schema-check.yml ================================================ # This Action checks makes a dbt run to sample json structured logs # and checks that they conform to the currently documented schema. # # If this action fails it either means we have unintentionally deviated # from our documented structured logging schema, or we need to bump the # version of our structured logging and add new documentation to # communicate these changes. name: Structured Logging Schema Check on: push: branches: - "main" - "*.latest" - "releases/*" pull_request: merge_group: types: [checks_requested] workflow_dispatch: permissions: read-all # top-level adjustments can be made here env: # number of parallel processes to spawn for python testing PYTHON_INTEGRATION_TEST_WORKERS: 15 jobs: integration-metadata: name: integration test metadata generation runs-on: ubuntu-latest outputs: split-groups: ${{ steps.generate-split-groups.outputs.split-groups }} steps: - name: generate split-groups id: generate-split-groups run: | MATRIX_JSON="[" for B in $(seq 1 ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }}); do MATRIX_JSON+=$(sed 's/^/"/;s/$/"/' <<< "${B}") done MATRIX_JSON="${MATRIX_JSON//\"\"/\", \"}" MATRIX_JSON+="]" echo "split-groups=${MATRIX_JSON}" >> $GITHUB_OUTPUT # run the performance measurements on the current or default branch test-schema: name: Test Log Schema runs-on: ubuntu-latest timeout-minutes: 45 needs: - integration-metadata strategy: fail-fast: false matrix: split-group: ${{ fromJson(needs.integration-metadata.outputs.split-groups) }} env: # turns warnings into errors RUSTFLAGS: "-D warnings" # points tests to the log file LOG_DIR: "/home/runner/work/dbt-core/dbt-core/logs" # tells integration tests to output into json format DBT_LOG_FORMAT: "json" # tell eventmgr to convert logging events into bytes DBT_TEST_BINARY_SERIALIZATION: "true" # Additional test users DBT_TEST_USER_1: dbt_test_user_1 DBT_TEST_USER_2: dbt_test_user_2 DBT_TEST_USER_3: dbt_test_user_3 services: # Label used to access the service container postgres: # Docker Hub image image: postgres # Provide the password for postgres env: POSTGRES_PASSWORD: password POSTGRES_USER: postgres # Set health checks to wait until postgres has started options: >- --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 ports: - 5432:5432 steps: - name: checkout dev uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4 with: persist-credentials: false - name: Setup Python uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # actions/setup-python@v6 with: python-version: "3.10" - name: "Install hatch" uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc # pypa/hatch@install - name: Run postgres setup script run: | ./scripts/setup_db.sh env: PGHOST: localhost PGPORT: 5432 PGPASSWORD: password - name: ls run: ls # integration tests generate a ton of logs in different files. the next step will find them all. # we actually care if these pass, because the normal test run doesn't usually include many json log outputs - name: Run integration tests uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # nick-fields/retry@v3 with: timeout_minutes: 30 max_attempts: 3 command: cd core && hatch -v run ci:integration-tests -- --splits ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }} --group ${{ matrix.split-group }} test-schema-report: name: Log Schema Test Suite runs-on: ubuntu-latest needs: test-schema steps: - name: "[Notification] Log test suite passes" run: | echo "::notice title="Log test suite passes"" ================================================ FILE: .github/workflows/test-repeater.yml ================================================ # **what?** # This workflow will test all test(s) at the input path given number of times to determine if it's flaky or not. You can test with any supported OS/Python combination. # This is batched in 10 to allow more test iterations faster. # **why?** # Testing if a test is flaky and if a previously flaky test has been fixed. This allows easy testing on supported python versions and OS combinations. # **when?** # This is triggered manually from dbt-core. name: Flaky Tester on: workflow_dispatch: inputs: branch: description: "Branch to check out" type: string required: true default: "main" test_path: description: "Path to single test to run (ex: tests/functional/retry/test_retry.py::TestRetry::test_fail_fast)" type: string required: true default: "tests/functional/..." python_version: description: "Version of Python to Test Against" type: choice options: - "3.10" - "3.11" os: description: "OS to run test in" type: choice options: - "ubuntu-latest" - "macos-14" - "windows-latest" num_runs_per_batch: description: "Max number of times to run the test per batch. We always run 10 batches." type: number required: true default: "50" permissions: read-all defaults: run: shell: bash jobs: debug: runs-on: ${{ vars.UBUNTU_LATEST }} steps: - name: "[DEBUG] Output Inputs" run: | echo "Branch: ${{ inputs.branch }}" echo "test_path: ${{ inputs.test_path }}" echo "python_version: ${{ inputs.python_version }}" echo "os: ${{ inputs.os }}" echo "num_runs_per_batch: ${{ inputs.num_runs_per_batch }}" pytest: runs-on: ${{ inputs.os }} strategy: # run all batches, even if one fails. This informs how flaky the test may be. fail-fast: false # using a matrix to speed up the jobs since the matrix will run in parallel when runners are available matrix: batch: ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"] env: PYTEST_ADDOPTS: "-v --color=yes -n4 --csv integration_results.csv" DBT_TEST_USER_1: dbt_test_user_1 DBT_TEST_USER_2: dbt_test_user_2 DBT_TEST_USER_3: dbt_test_user_3 DD_CIVISIBILITY_AGENTLESS_ENABLED: true DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} DD_SITE: datadoghq.com DD_ENV: ci DD_SERVICE: ${{ github.event.repository.name }} steps: - name: "Checkout code" uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4 with: ref: ${{ inputs.branch }} - name: "Setup Python" uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # actions/setup-python@v6 with: python-version: "${{ inputs.python_version }}" - name: "Install hatch" uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc # pypa/hatch@install - name: "Setup Dev Environment" run: | cd core hatch run setup - name: "Set up postgres (linux)" if: inputs.os == '${{ vars.UBUNTU_LATEST }}' run: | cd core hatch run setup-db # mac and windows don't use make due to limitations with docker with those runners in GitHub - name: Set up postgres (macos) if: runner.os == 'macOS' uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # nick-fields/retry@v3 with: timeout_minutes: 10 max_attempts: 3 command: ./scripts/setup_db.sh - name: "Set up postgres (windows)" if: inputs.os == 'windows-latest' uses: ./.github/actions/setup-postgres-windows - name: "Test Command" id: command run: | test_command="python -m pytest ${{ inputs.test_path }}" echo "test_command=$test_command" >> $GITHUB_OUTPUT - name: "Run test ${{ inputs.num_runs_per_batch }} times" id: pytest run: | set +e for ((i=1; i<=${{ inputs.num_runs_per_batch }}; i++)) do echo "Running pytest iteration $i..." python -m pytest --ddtrace ${{ inputs.test_path }} exit_code=$? if [[ $exit_code -eq 0 ]]; then success=$((success + 1)) echo "Iteration $i: Success" else failure=$((failure + 1)) echo "Iteration $i: Failure" fi echo echo "===========================" echo "Successful runs: $success" echo "Failed runs: $failure" echo "===========================" echo done echo "failure=$failure" >> $GITHUB_OUTPUT - name: "Success and Failure Summary: ${{ inputs.os }}/Python ${{ inputs.python_version }}" run: | echo "Batch: ${{ matrix.batch }}" echo "Successful runs: ${{ steps.pytest.outputs.success }}" echo "Failed runs: ${{ steps.pytest.outputs.failure }}" - name: "Error for Failures" if: ${{ steps.pytest.outputs.failure }} run: | echo "Batch ${{ matrix.batch }} failed ${{ steps.pytest.outputs.failure }} of ${{ inputs.num_runs_per_batch }} tests" exit 1 ================================================ FILE: .github/workflows/triage-labels.yml ================================================ # **what?** # When the core team triages, we sometimes need more information from the issue creator. In # those cases we remove the `triage` label and add the `awaiting_response` label. Once we # recieve a response in the form of a comment, we want the `awaiting_response` label removed # in favor of the `triage` label so we are aware that the issue needs action. # **why?** # To help with out team triage issue tracking # **when?** # This will run when a comment is added to an issue and that issue has to `awaiting_response` label. name: Update Triage Label on: issue_comment defaults: run: shell: bash permissions: issues: write jobs: triage_label: if: contains(github.event.issue.labels.*.name, 'awaiting_response') uses: dbt-labs/actions/.github/workflows/swap-labels.yml@main with: add_label: "triage" remove_label: "awaiting_response" secrets: inherit ================================================ FILE: .github/workflows/update-test-durations.yml ================================================ # **what?** # Generates pytest-split test duration data for optimized test splitting. # It runs tests with pytest-split to generate duration data, and then merges the data # into a single file to reduce the time it takes to generate the durations file. It # runs for 5+ hours otherwise. The updated file is committed to a branch and a PR is # opened for review. The PR is auto-merged once CI passes. # **why?** # pytest-split uses duration data to balance test workloads across workers. # Without this data, tests are split evenly by count which can cause imbalanced runs. # **when?** # Runs weekly on Sunday at midnight UTC, and can be manually triggered as needed. name: Update Test Durations on: schedule: - cron: "0 0 * * 0" workflow_dispatch: inputs: auto-merge-pr: description: "Auto-merge the PR" required: true type: boolean default: true permissions: contents: write pull-requests: write defaults: run: shell: bash env: # Number of parallel workers - matches main.yml for consistent splitting PYTHON_INTEGRATION_TEST_WORKERS: 15 jobs: generate-split-groups: name: "Generate Split Groups" runs-on: ubuntu-latest outputs: split-groups: ${{ steps.generate.outputs.split-groups }} steps: - name: "Generate split group matrix" id: generate run: | MATRIX_JSON="[" for B in $(seq 1 ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }}); do MATRIX_JSON+=$(sed 's/^/"/;s/$/"/' <<< "${B}") done MATRIX_JSON="${MATRIX_JSON//\"\"/\", \"}" MATRIX_JSON+="]" echo "split-groups=${MATRIX_JSON}" echo "split-groups=${MATRIX_JSON}" >> $GITHUB_OUTPUT generate-durations: name: "Generate Durations (Group ${{ matrix.split-group }})" runs-on: ubuntu-latest timeout-minutes: 60 needs: [generate-split-groups] strategy: fail-fast: false matrix: split-group: ${{ fromJson(needs.generate-split-groups.outputs.split-groups) }} env: DBT_INVOCATION_ENV: github-actions DBT_TEST_USER_1: dbt_test_user_1 DBT_TEST_USER_2: dbt_test_user_2 DBT_TEST_USER_3: dbt_test_user_3 services: postgres: image: postgres env: POSTGRES_PASSWORD: password POSTGRES_USER: postgres options: >- --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 ports: - 5432:5432 steps: - name: "Check out the repository" uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # actions/checkout@v5 - name: "Set up Python 3.11" uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # actions/setup-python@v6 with: python-version: "3.11" - name: "Run postgres setup script" run: ./scripts/setup_db.sh env: PGHOST: localhost PGPORT: 5432 PGPASSWORD: password - name: "Install hatch" uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc # pypa/hatch@install - name: "Run integration tests and store durations" run: | cd core hatch -v run ci:integration-tests-generate-durations -- \ --splits ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }} \ --group ${{ matrix.split-group }} \ || true # Rename to per-group file (pytest-split overwrites with only this group's tests) if [ -f .test_durations ]; then mv .test_durations .test_durations_${{ matrix.split-group }} else echo "ERROR: .test_durations not created - check pytest output above" exit 1 fi - name: "Upload partial durations file" uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # actions/upload-artifact@v6 with: name: durations-${{ matrix.split-group }} path: core/.test_durations_${{ matrix.split-group }} if-no-files-found: error include-hidden-files: true create-pr: name: "Create Pull Request" runs-on: ubuntu-latest timeout-minutes: 10 needs: [generate-durations] outputs: pr_number: ${{ steps.cpr.outputs.pull-request-number }} pr_url: ${{ steps.cpr.outputs.pull-request-url }} steps: - name: "Check out the repository" uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # actions/checkout@v5 - name: "Import build bot GPG key" id: import-gpg uses: crazy-max/ghaction-import-gpg@2dc316deee8e90f13e1a351ab510b4d5bc0c82cd # crazy-max/ghaction-import-gpg@v7 with: gpg_private_key: ${{ secrets.FISHTOWN_BOT_GPG_PRIVATE_KEY }} passphrase: ${{ secrets.FISHTOWN_BOT_GPG_PASSPHRASE }} git_config_global: true git_user_signingkey: true git_commit_gpgsign: true git_committer_name: "Github Build Bot" git_committer_email: "buildbot@dbtlabs.com" - name: "Download all partial duration files" uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # actions/download-artifact@v7 with: pattern: durations-* path: partial-durations merge-multiple: true - name: "Set up Python 3.11" uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # actions/setup-python@v6 with: python-version: "3.11" - name: "Merge duration files" run: | python3 << 'EOF' import json import glob merged = {} for filepath in glob.glob("partial-durations/.test_durations_*"): print(f"Processing {filepath}") with open(filepath, "r") as f: data = json.load(f) merged.update(data) print(f" Added {len(data)} tests") print(f"Total tests: {len(merged)}") # Sort by test name for consistent output sorted_merged = dict(sorted(merged.items())) with open("core/.test_durations", "w") as f: json.dump(sorted_merged, f, indent=4) f.write("\n") EOF - name: "Install hatch" uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc # pypa/hatch@install - name: "Code quality" run: | cd core git add .test_durations hatch run pre-commit run end-of-file-fixer --files .test_durations || true - name: "Create Pull Request" id: cpr uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # peter-evans/create-pull-request@v8.1.0 with: token: ${{ secrets.FISHTOWN_BOT_PAT }} commit-message: "Update test durations for pytest-split" author: "Github Build Bot <buildbot@dbtlabs.com>" committer: "Github Build Bot <buildbot@dbtlabs.com>" branch: update-test-durations base: main delete-branch: true title: "Update test durations for pytest-split" body: | This PR is automatically generated to update the test durations file used by pytest-split. The durations are generated by running the full integration test suite and recording how long each test takes. This data is used to balance test workloads across parallel workers. labels: | Skip Changelog add-paths: | core/.test_durations await-ci: name: "Wait for CI" runs-on: ubuntu-latest needs: [create-pr] if: needs.create-pr.outputs.pr_number != '' && (github.event_name == 'schedule' || inputs.auto-merge-pr == true) env: GH_TOKEN: ${{ secrets.FISHTOWN_BOT_PAT }} steps: - name: "Check out the repository" uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # actions/checkout@v5 - name: "Wait for CI to start" run: | sleep 60 - name: "Wait for CI checks to complete" run: | gh pr checks ${{ needs.create-pr.outputs.pr_number }} --fail-fast --watch merge-pr: name: "Merge Pull Request" runs-on: ubuntu-latest needs: [create-pr, await-ci] if: github.event_name == 'schedule' || inputs.auto-merge-pr == true env: GH_TOKEN: ${{ secrets.FISHTOWN_BOT_PAT }} steps: - name: "Check out the repository" uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # actions/checkout@v5 - name: "Merge PR" run: | gh pr merge ${{ needs.create-pr.outputs.pr_number }} \ --admin \ --squash \ --body "Update test durations for pytest-split" \ --delete-branch slack-notification: name: "Slack Notification" runs-on: ubuntu-latest needs: [create-pr, await-ci, merge-pr] if: always() && contains(join(needs.*.result, ','), 'failure') steps: - name: "Post Slack Notification" run: | python3 <<'PY' > slack-payload.json import json import os failed_jobs = [ job for job in ("create-pr", "await-ci", "merge-pr") if os.environ.get(f"{job.upper().replace('-', '_')}_RESULT") == "failure" ] failed_summary = ", ".join(failed_jobs) if failed_jobs else "unknown job" pr_url = os.environ.get("PR_URL", "") pr_text = f"PR <{pr_url}|opened by the workflow>" if pr_url else "The workflow PR" text = ( f":x: {pr_text} failed to merge for `{os.environ['GITHUB_REPOSITORY']}` " f"on `{os.environ['GITHUB_REF_NAME']}`. Failed job(s): {failed_summary}. " f"<{os.environ['RUN_URL']}|View workflow run>." ) print(json.dumps({"text": text})) PY curl -X POST \ -H "Content-type: application/json" \ --data @slack-payload.json \ "$SLACK_WEBHOOK_URL" env: CREATE_PR_RESULT: ${{ needs.create-pr.result }} AWAIT_CI_RESULT: ${{ needs.await-ci.result }} MERGE_PR_RESULT: ${{ needs.merge-pr.result }} PR_URL: ${{ needs.create-pr.outputs.pr_url }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} SLACK_WEBHOOK_URL: ${{ secrets.SLACK_CORE_PR_CHANNEL_URL }} ================================================ FILE: .gitignore ================================================ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python env*/ dbt_env/ build/ !tests/functional/build !core/dbt/docs/build develop-eggs/ dist/ dist-*/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ *.egg-info/ .installed.cfg *.egg .mypy_cache/ .dmypy.json logs/ .user.yml profiles.yml # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .coverage .coverage.* .cache .env nosetests.xml coverage.xml *,cover .hypothesis/ test.env makefile.test.env *.pytest_cache/ # Unit test artifacts index.html # Translations *.mo *.pot # Django stuff: *.log # Sphinx documentation docs/_build/ # PyBuilder target/ # Ipython Notebook .ipynb_checkpoints # Emacs *~ # Sublime Text *.sublime-* # Vim *.sw* # Pyenv .python-version # Vim *.sw* # pycharm .idea/ venv/ .venv*/ # AWS credentials .aws/ # MacOS .DS_Store # vscode .vscode/ *.code-workspace # poetry poetry.lock # asdf .tool-versions ================================================ FILE: .pre-commit-config.yaml ================================================ # Configuration for pre-commit hooks (see https://pre-commit.com/). # Eventually the hooks described here will be run as tests before merging each PR. exclude: ^(core/dbt/docs/build/|core/dbt/common/events/types_pb2.py|core/dbt/adapters/events/adapter_types_pb2.py) # Force all unspecified python hooks to run python 3.10 default_language_version: python: python3 repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v3.2.0 hooks: - id: check-yaml args: [--unsafe] - id: check-json - id: end-of-file-fixer exclude: schemas/dbt/manifest/ - id: trailing-whitespace exclude_types: - "markdown" - id: check-case-conflict # local hooks are used to run the hooks in the local environment instead of a pre-commit isolated one. # This ensures that the hooks are run with the same version of the dependencies as the local environment # without having to manually keep them in sync. - repo: local hooks: # Formatter/linter/type-checker pins live in the pyproject.dev optional dependency. - id: isort name: isort entry: python -m isort args: [--settings-path, pyproject.toml] language: system types: [python] - id: black name: black entry: python -m black language: system types: [python] - id: black-check name: black-check entry: python -m black args: - "--check" - "--diff" language: system stages: [manual] types: [python] - id: flake8 name: flake8 entry: python -m flake8 language: system types: [python] - id: flake8-check name: flake8-check entry: python -m flake8 language: system stages: [manual] types: [python] # N.B.: Mypy is... a bit fragile. # # By using `language: system` we run this hook in the local # environment instead of a pre-commit isolated one. This is needed # to ensure mypy correctly parses the project. # # It may cause trouble # in that it adds environmental variables out of our control to the # mix. Unfortunately, there's nothing we can do about per pre-commit's # author. # See https://github.com/pre-commit/pre-commit/issues/730 for details. - id: mypy name: mypy entry: python -m mypy args: [--show-error-codes] files: ^core/dbt/ language: system types: [python] - id: mypy-check name: mypy-check entry: python -m mypy args: [--show-error-codes, --pretty] files: ^core/dbt/ language: system stages: [manual] types: [python] - id: no_versioned_artifact_resource_imports name: no_versioned_artifact_resource_imports entry: python scripts/pre-commit-hooks/no_versioned_artifact_resource_imports.py language: system files: ^core/dbt/ types: [python] pass_filenames: true ================================================ FILE: AGENTS.md ================================================ # AGENTS.md — AI Coding Agent Guidelines for dbt-core ## Project Overview dbt-core is the open-source core of [dbt](https://www.getdbt.com/) (data build tool). It transforms data in warehouses by running SQL and Python models, managing dependencies, and producing artifacts. The main Python package lives in `core/` and is built with Hatch/Hatchling. ## Repository Layout ``` core/ # Main dbt-core Python package (pyproject.toml, hatch.toml) dbt/ # Source code artifacts/ # Artifact schemas and versioned resource definitions cli/ # CLI entry point (Click-based) clients/ # Jinja, YAML, git, registry clients config/ # Profile, project, and runtime configuration context/ # Jinja context providers contracts/ # Project and graph contracts (nodes, manifest) deps/ # Dependency resolution events/ # Structured event logging graph/ # DAG selection and selector methods materializations/ # Materialization strategies (e.g. incremental, microbatch) parser/ # Manifest, model, source, macro parsing task/ # One module per dbt command (run, build, test, compile, etc.) tests/ # Test utilities and fixtures (not the test suite) tests/ # Test suite unit/ # Unit tests — no database required functional/ # Functional tests — require Postgres plugins/ # Local Postgres adapter (for integration tests) schemas/ # JSON schemas for dbt artifacts docs/ # Architecture docs and guides .changes/ # Changie changelog entries ``` ### Related Repositories dbt-core depends on packages maintained in separate repos: - **dbt-common** (`dbt-labs/dbt-common`) — shared utilities, `dbtClassMixin` - **dbt-adapters** (`dbt-labs/dbt-adapters`) — adapter interfaces and Postgres adapter - **dbt-semantic-interfaces**, **dbt-extractor**, **dbt-protos** — other ecosystem packages ## Build System and Dev Setup - **Build backend:** Hatchling (`core/pyproject.toml`) - **Python:** ≥3.10 (CI tests 3.10–3.13) - **Setup:** `cd core && hatch run setup` - **Entry point:** `dbt = dbt.cli.main:cli` ## Code Style and Formatting - **Formatter:** Black (line length 99) - **Import sorting:** isort (profile `"black"`) - **Linter:** flake8 - **Type checking:** mypy - **Pre-commit:** Runs all of the above via `hatch run code-quality` Import order should follow isort conventions: 1. `__future__` 2. Standard library 3. Third-party 4. dbt-internal (`dbt`, `dbt_common`, `dbt_adapters`, `dbt_extractor`, `dbt_semantic_interfaces`) ## Key Architectural Conventions ### Artifact Resources: Import from `dbt.artifacts.resources`, Not Versioned Paths **Never** import directly from versioned artifact paths like `dbt.artifacts.resources.v1.model`. Instead, import from `dbt.artifacts.resources`: ```python # WRONG — will fail pre-commit from dbt.artifacts.resources.v1.model import Model # RIGHT from dbt.artifacts.resources import Model ``` The `dbt.artifacts.resources.__init__` re-exports everything from the current version. A pre-commit hook enforces this outside the `artifacts/` module. ### Data Model Layer dbt-core uses `dataclasses` with `dbtClassMixin` (from `dbt-common`) for serialization, backed by mashumaro. It does **not** use pydantic for its data model hierarchy. The node type hierarchy has two layers: 1. **Artifact resources** (`dbt.artifacts.resources`) — serializable data definitions 2. **Contract nodes** (`dbt.contracts.graph.nodes`) — runtime node classes that inherit from artifact resources and add behavior ``` BaseResource → GraphResource → BaseNode → GraphNode → ParsedNode → CompiledNode ├── ModelNode ├── SnapshotNode ├── AnalysisNode ├── SingularTestNode ├── GenericTestNode └── ... ``` Each contract node has a `resource_class()` method returning its corresponding artifact resource type and a `to_resource()` method for conversion. ### Parser Pattern Parsers follow a class hierarchy: `BaseParser → Parser → ConfiguredParser → SQLParser`. The typical flow: `parse_file()` → `parse_node()` → `_create_parsetime_node()` → `parse_from_dict()` → `render_update()` → `add_result_node()` Schema-based parsers (`SchemaParser`, `YamlReader` subclasses) read YAML and apply patches to nodes. ## Testing ### Structure - **Unit tests** (`tests/unit/`): Pure Python, no database. Use mocks and helpers from `tests/unit/utils/`. - **Functional tests** (`tests/functional/`): Require Postgres. Use the `project` fixture from `dbt.tests.fixtures.project`. ### Running Tests ```sh cd core hatch run unit-tests # Unit tests only hatch run integration-tests # Functional tests (requires Postgres) hatch run test # Unit tests + code quality checks hatch run code-quality # Pre-commit hooks on all files ``` Or directly with pytest: ```sh cd core hatch run python3 -m pytest ../tests/unit/path/to/test_file.py hatch run python3 -m pytest ../tests/functional/feature_name ``` ### Functional Test Pattern Functional tests use class-scoped fixtures to define project files and run dbt commands: ```python class TestMyFeature: @pytest.fixture(scope="class") def models(self): return {"my_model.sql": "select 1 as id"} def test_run_succeeds(self, project): results = run_dbt(["run"]) assert len(results) == 1 ``` Key utilities: `run_dbt()`, `run_dbt_and_capture()`, `get_manifest()`, `get_artifact()` from `dbt.tests.util`. Having multiple tests in a functional test class will mean that those tests **will share** the underlying dbt project fixture of the class. This means that if a test modifies the underlying project that will affect other tests in the same class. This leads to flakiness and means it is generally best practice to have one test per functional test class. ### Database Setup for Functional Tests ```sh cd core && hatch run setup-db # or manually: docker-compose up -d database PGHOST=localhost PGUSER=root PGPASSWORD=password PGDATABASE=postgres bash scripts/setup_db.sh ``` ## Contributing Guide General contributing documetatnion can be found in [CONTRIBUTING.md](CONTRIBUTING.md) ## Changelog Use [changie](https://changie.dev/) — do **not** edit `CHANGELOG.md` directly (it is generated). ```sh changie new ``` This creates a YAML entry in `.changes/unreleased/`. Changelog kinds: Breaking Changes, Features, Fixes, Docs, Under the Hood, Dependencies, Security. ## Commit Discipline Separate distinct types of changes into their own commits. Do not combine unrelated changes in a single commit, even if they touch the same file. This keeps the history reviewable and individually revertable. The following categories of change should each be their own commit: - **Tidying** — fixing typos, improving variable names, cleaning up whitespace - **Abstractions** — extracting duplicated logic into a shared function or module - **Refactors** — restructuring code for readability, performance, or maintainability (without changing behavior) - **Bug fixes** — correcting incorrect behavior - **Features** — adding new functionality - **Tests** — adding or improving tests for existing behavior (coverage gaps, edge cases, flaky test fixes). Tests that accompany a new feature or bug fix belong in that commit, but standalone test work is its own category. - **Dependencies** — adding, removing, or upgrading dependencies - **Configuration** — changes to CI workflows, linter settings, build configs, pre-commit hooks, or other tooling When a task involves more than one of these, make separate commits in a logical order. For example, if a bug fix requires a refactor first, commit the refactor, then commit the fix. If a feature benefits from tidying nearby code, commit the tidying first, then the feature. Each commit should make sense in isolation: it should pass tests, not break the build, and have a clear message explaining *what* and *why*. Finally we require all commits to be signed with GPG keys. You can inspect if a GPG is present via `git config --global --get user.signingkey`. If it is not, please help the user setup a github GPG key. ## Pull Requests - Target the `main` branch - Signed commits required (GPG) - CLA signature required for external contributors - Add a changie entry unless the work done was limited to adding/changing tests, adding/changing comments, adding/changing github actions/workflows, or adding/changing markdown files not used during operation of the engine ================================================ FILE: ARCHITECTURE.md ================================================ The core function of dbt is SQL compilation and execution. Users create projects of dbt resources (models, tests, seeds, snapshots, ...), defined in SQL and YAML files, and they invoke dbt to create, update, or query associated views and tables. Today, dbt makes heavy use of Jinja2 to enable the templating of SQL, and to construct a DAG (Directed Acyclic Graph) from all of the resources in a project. Users can also extend their projects by installing resources (including Jinja macros) from other projects, called "packages." ## dbt-core Most of the python code in the repository is within the `core/dbt` directory. - [`single python files`](core/dbt/README.md): A number of individual files, such as 'compilation.py' and 'exceptions.py' The main subdirectories of core/dbt: - [`clients`](core/dbt/clients/README.md): Interface with dependencies (agate, jinja) or across operating systems - [`config`](core/dbt/config/README.md): Reconcile user-supplied configuration from connection profiles, project files, and Jinja macros - [`context`](core/dbt/context/README.md): Build and expose dbt-specific Jinja functionality - [`contracts`](core/dbt/contracts/README.md): Define Python objects (dataclasses) that dbt expects to create and validate - [`deps`](core/dbt/deps/README.md): Package installation and dependency resolution - [`events`](core/dbt/events/README.md): Logging events - [`graph`](core/dbt/graph/README.md): Produce a `networkx` DAG of project resources, and selecting those resources given user-supplied criteria - [`include`](core/dbt/include/README.md): Set up the starter project scaffold. - [`parser`](core/dbt/parser/README.md): Read project files, validate, construct python objects - [`task`](core/dbt/task/README.md): Set forth the actions that dbt can perform when invoked ### Invoking dbt The "tasks" map to top-level dbt commands. So `dbt run` => task.run.RunTask, etc. Some are more like abstract base classes (GraphRunnableTask, for example) but all the concrete types outside of task should map to tasks. Currently one executes at a time. The tasks kick off their “Runners” and those do execute in parallel. The parallelism is managed via a thread pool, in GraphRunnableTask. core/dbt/task/docs/index.html This is the docs website code. It comes from the dbt-docs repository, and is generated when a release is packaged. ## Adapters dbt uses an adapter-plugin pattern to extend support to different databases, warehouses, query engines, etc. Note: dbt-postgres used to exist in dbt-core but is now in [the dbt-adapters repo](https://github.com/dbt-labs/dbt-adapters/tree/main/dbt-postgres) Each adapter is a mix of python, Jinja2, and SQL. The adapter code also makes heavy use of Jinja2 to wrap modular chunks of SQL functionality, define default implementations, and allow plugins to override it. Each adapter plugin is a standalone python package that includes: - `dbt/include/[name]`: A "sub-global" dbt project, of YAML and SQL files, that reimplements Jinja macros to use the adapter's supported SQL syntax - `dbt/adapters/[name]`: Python modules that inherit, and optionally reimplement, the base adapter classes defined in dbt-core - `pyproject.toml` The Postgres adapter code is the most central, and many of its implementations are used as the default defined in the dbt-core global project. The greater the distance of a data technology from Postgres, the more its adapter plugin may need to reimplement. ## Testing dbt The [`tests/`](tests/) subdirectory includes unit and fuctional tests that run as continuous integration checks against open pull requests. Unit tests check mock inputs and outputs of specific python functions. Functional tests perform end-to-end dbt invocations against real adapters (Postgres) and assert that the results match expectations. See [the contributing guide](CONTRIBUTING.md) for a step-by-step walkthrough of setting up a local development and testing environment. ## Everything else - [docker](docker/): All dbt versions are published as Docker images on DockerHub. This subfolder contains the `Dockerfile` (constant) and `requirements.txt` (one for each version). - [scripts](scripts/): Helper scripts for testing, releasing, and producing JSON schemas. These are not included in distributions of dbt, nor are they rigorously tested—they're just handy tools for the dbt maintainers :) ================================================ FILE: CHANGELOG.md ================================================ # dbt Core Changelog - This file provides a full account of all changes to `dbt-core` - Changes are listed under the (pre)release in which they first appear. Subsequent releases include changes from previous releases. - "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version. - Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-core/blob/main/CONTRIBUTING.md#adding-changelog-entry) ## Previous Releases For information on prior major and minor releases, see their changelogs: * [1.11](https://github.com/dbt-labs/dbt-core/blob/1.11.latest/CHANGELOG.md) * [1.10](https://github.com/dbt-labs/dbt-core/blob/1.10.latest/CHANGELOG.md) * [1.9](https://github.com/dbt-labs/dbt-core/blob/1.9.latest/CHANGELOG.md) * [1.8](https://github.com/dbt-labs/dbt-core/blob/1.8.latest/CHANGELOG.md) * [1.7](https://github.com/dbt-labs/dbt-core/blob/1.7.latest/CHANGELOG.md) * [1.6](https://github.com/dbt-labs/dbt-core/blob/1.6.latest/CHANGELOG.md) * [1.5](https://github.com/dbt-labs/dbt-core/blob/1.5.latest/CHANGELOG.md) * [1.4](https://github.com/dbt-labs/dbt-core/blob/1.4.latest/CHANGELOG.md) * [1.3](https://github.com/dbt-labs/dbt-core/blob/1.3.latest/CHANGELOG.md) * [1.2](https://github.com/dbt-labs/dbt-core/blob/1.2.latest/CHANGELOG.md) * [1.1](https://github.com/dbt-labs/dbt-core/blob/1.1.latest/CHANGELOG.md) * [1.0](https://github.com/dbt-labs/dbt-core/blob/1.0.latest/CHANGELOG.md) * [0.21](https://github.com/dbt-labs/dbt-core/blob/0.21.latest/CHANGELOG.md) * [0.20](https://github.com/dbt-labs/dbt-core/blob/0.20.latest/CHANGELOG.md) * [0.19](https://github.com/dbt-labs/dbt-core/blob/0.19.latest/CHANGELOG.md) * [0.18](https://github.com/dbt-labs/dbt-core/blob/0.18.latest/CHANGELOG.md) * [0.17](https://github.com/dbt-labs/dbt-core/blob/0.17.latest/CHANGELOG.md) * [0.16](https://github.com/dbt-labs/dbt-core/blob/0.16.latest/CHANGELOG.md) * [0.15](https://github.com/dbt-labs/dbt-core/blob/0.15.latest/CHANGELOG.md) * [0.14](https://github.com/dbt-labs/dbt-core/blob/0.14.latest/CHANGELOG.md) * [0.13](https://github.com/dbt-labs/dbt-core/blob/0.13.latest/CHANGELOG.md) * [0.12](https://github.com/dbt-labs/dbt-core/blob/0.12.latest/CHANGELOG.md) * [0.11 and earlier](https://github.com/dbt-labs/dbt-core/blob/0.11.latest/CHANGELOG.md) ================================================ FILE: CONTRIBUTING.md ================================================ # Contributing to `dbt-core` `dbt-core` is open source software. It is what it is today because community members have opened issues, provided feedback, and [contributed to the knowledge loop](https://www.getdbt.com/dbt-labs/values/). Whether you are a seasoned open source contributor or a first-time committer, we welcome and encourage you to contribute code, documentation, ideas, or problem statements to this project. - [Contributing to `dbt-core`](#contributing-to-dbt-core) - [About this document](#about-this-document) - [Notes](#notes) - [Getting the code](#getting-the-code) - [Installing git](#installing-git) - [External contributors](#external-contributors) - [dbt Labs contributors](#dbt-labs-contributors) - [Setting up an environment](#setting-up-an-environment) - [Tools](#tools) - [Virtual environments](#virtual-environments) - [Docker and `docker-compose`](#docker-and-docker-compose) - [Postgres (optional)](#postgres-optional) - [Running `dbt-core` in development](#running-dbt-core-in-development) - [Installation](#installation) - [Running `dbt-core`](#running-dbt-core) - [Testing](#testing) - [Initial setup](#initial-setup) - [Test commands](#test-commands) - [Hatch scripts](#hatch-scripts) - [`pre-commit`](#pre-commit) - [`pytest`](#pytest) - [Unit, Integration, Functional?](#unit-integration-functional) - [Debugging](#debugging) - [Assorted development tips](#assorted-development-tips) - [Adding or modifying a CHANGELOG Entry](#adding-or-modifying-a-changelog-entry) - [Submitting a Pull Request](#submitting-a-pull-request) - [Troubleshooting Tips](#troubleshooting-tips) ## About this document There are many ways to contribute to the ongoing development of `dbt-core`, such as by participating in discussions and issues. We encourage you to first read our higher-level document: ["Expectations for Open Source Contributors"](https://docs.getdbt.com/docs/contributing/oss-expectations). The rest of this document serves as a more granular guide for contributing code changes to `dbt-core` (this repository). It is not intended as a guide for using `dbt-core`, and some pieces assume a level of familiarity with Python development and package managers. Specific code snippets in this guide assume you are using macOS or Linux and are comfortable with the command line. If you get stuck, we're happy to help! Drop us a line in the `#dbt-core-development` channel in the [dbt Community Slack](https://community.getdbt.com). ### Notes - **Adapters:** Is your issue or proposed code change related to a specific [database adapter](https://docs.getdbt.com/docs/available-adapters)? If so, please open issues, PRs, and discussions in that adapter's repository instead. - **CLA:** Please note that anyone contributing code to `dbt-core` must sign the [Contributor License Agreement](https://docs.getdbt.com/docs/contributor-license-agreements). If you are unable to sign the CLA, the `dbt-core` maintainers will unfortunately be unable to merge any of your Pull Requests. We welcome you to participate in discussions, open issues, and comment on existing ones. - **Commit Signing:** Contributors to `dbt-core` must submit commits with verified signatures. To set up a GPG key via Github, follow their guides on (1) [generating a new GPG key](https://docs.github.com/en/authentication/managing-commit-signature-verification/generating-a-new-gpg-key), (2) [adding a GPG key to your GitHub account](https://docs.github.com/en/authentication/managing-commit-signature-verification/adding-a-gpg-key-to-your-github-account), and (3) [telling Git about your GPG key](https://docs.github.com/en/authentication/managing-commit-signature-verification/telling-git-about-your-signing-key). If you need to sign existing commits on a branch, [this Stack Overflow post](https://stackoverflow.com/questions/41882919/is-there-a-way-to-gpg-sign-all-previous-commits/41883164#41883164) contains helpful guidance. - **Branches:** All pull requests from community contributors should target the `main` branch (default). If the change is needed as a patch for a minor version of dbt that has already been released (or is already a release candidate), a maintainer will backport the changes in your PR to the relevant "latest" release branch (`1.0.latest`, `1.1.latest`, ...). If an issue fix applies to a release branch, that fix should be first committed to the development branch and then to the release branch (rarely release-branch fixes may not apply to `main`). - **Releases**: Before releasing a new minor version of Core, we prepare a series of alphas and release candidates to allow users (especially employees of dbt Labs!) to test the new version in live environments. This is an important quality assurance step, as it exposes the new code to a wide variety of complicated deployments and can surface bugs before official release. Releases are accessible via our [supported installation methods](https://docs.getdbt.com/docs/core/installation-overview#install-dbt-core). ## Getting the code ### Installing git You will need `git` in order to download and modify the `dbt-core` source code. On macOS, the best way to download git is to just install [Xcode](https://developer.apple.com/support/xcode/). ### External contributors If you are not a member of the `dbt-labs` GitHub organization, you can contribute to `dbt-core` by forking the `dbt-core` repository. For a detailed overview on forking, check out the [GitHub docs on forking](https://help.github.com/en/articles/fork-a-repo). In short, you will need to: 1. Fork the `dbt-core` repository 2. Clone your fork locally 3. Check out a new branch for your proposed changes 4. Push changes to your fork 5. Open a pull request against `dbt-labs/dbt-core` from your forked repository ### dbt Labs contributors If you are a member of the `dbt-labs` GitHub organization, you will have push access to the `dbt-core` repo. Rather than forking `dbt-core` to make your changes, just clone the repository, check out a new branch, and push directly to that branch. ## Setting up an environment There are some tools that will be helpful to you in developing locally. While this is the list relevant for `dbt-core` development, many of these tools are used commonly across open-source python projects. ### Tools These are the tools used in `dbt-core` development and testing: - [`hatch`](https://hatch.pypa.io/) for build backend, environment management, and running tests across Python versions (3.10, 3.11, 3.12, and 3.13) - [`pytest`](https://docs.pytest.org/en/latest/) to define, discover, and run tests - [`flake8`](https://flake8.pycqa.org/en/latest/) for code linting - [`black`](https://github.com/psf/black) for code formatting - [`mypy`](https://mypy.readthedocs.io/en/stable/) for static type checking - [`pre-commit`](https://pre-commit.com) to easily run those checks - [`changie`](https://changie.dev/) to create changelog entries, without merge conflicts - [GitHub Actions](https://github.com/features/actions) for automating tests and checks, once a PR is pushed to the `dbt-core` repository A deep understanding of these tools in not required to effectively contribute to `dbt-core`, but we recommend checking out the attached documentation if you're interested in learning more about each one. #### Virtual environments dbt-core uses [Hatch](https://hatch.pypa.io/) for dependency and environment management. Hatch automatically creates and manages isolated environments for development, testing, and building, so you don't need to manually create virtual environments. For more information on how Hatch manages environments, see the [Hatch environment documentation](https://hatch.pypa.io/latest/environment/). #### Docker and `docker-compose` Docker and `docker-compose` are both used in testing. Specific instructions for you OS can be found [here](https://docs.docker.com/get-docker/). #### Postgres (optional) For testing, and later in the examples in this document, you may want to have `psql` available so you can poke around in the database and see what happened. We recommend that you use [homebrew](https://brew.sh/) for that on macOS, and your package manager on Linux. You can install any version of the postgres client that you'd like. On macOS, with homebrew setup, you can run: ```sh brew install postgresql ``` ## Running `dbt-core` in development ### Installation First make sure you have Python 3.10 or later installed. Ensure you have the latest version of pip installed with `pip install --upgrade pip`. Next, install `hatch`. Finally set up `dbt-core` for development: ```sh cd core hatch run setup ``` This will install all development dependencies and set up pre-commit hooks. By default, hatch will use whatever Python version is active in your environment. To specify a particular Python version, set the `HATCH_PYTHON` environment variable: ```sh export HATCH_PYTHON=3.12 hatch env create ``` Or add it to your shell profile (e.g., `~/.zshrc` or `~/.bashrc`) for persistence. When installed in this way, any changes you make to your local copy of the source code will be reflected immediately in your next `dbt` run. #### Building dbt-core dbt-core uses [Hatch](https://hatch.pypa.io/) (specifically `hatchling`) as its build backend. To build distribution packages: ```sh cd core hatch build ``` This will create both wheel (`.whl`) and source distribution (`.tar.gz`) files in the `dist/` directory. The build configuration is defined in `core/pyproject.toml`. You can also use the standard `python -m build` command if you prefer. ### Running `dbt-core` Once you've run `hatch run setup`, the `dbt` command will be available in your PATH. You can verify this by running `which dbt`. Configure your [profile](https://docs.getdbt.com/docs/configure-your-profile) as necessary to connect to your target databases. It may be a good idea to add a new profile pointing to a local Postgres instance, or a specific test sandbox within your data warehouse if appropriate. Make sure to create a profile before running integration tests. ## Testing Once you're able to manually test that your code change is working as expected, it's important to run existing automated tests, as well as adding some new ones. These tests will ensure that: - Your code changes do not unexpectedly break other established functionality - Your code changes can handle all known edge cases - The functionality you're adding will _keep_ working in the future Although `dbt-core` works with a number of different databases, you won't need to supply credentials for every one of these databases in your test environment. Instead, you can test most `dbt-core` code changes with Python and Postgres. ### Initial setup Postgres offers the easiest way to test most `dbt-core` functionality today. They are the fastest to run, and the easiest to set up. To run the Postgres integration tests, you'll have to do one extra step of setting up the test database: ```sh cd core hatch run setup-db ``` Alternatively, you can run the setup commands directly: ```sh docker-compose up -d database PGHOST=localhost PGUSER=root PGPASSWORD=password PGDATABASE=postgres bash scripts/setup_db.sh ``` ### Test commands There are a few methods for running tests locally. #### Hatch scripts The primary way to run tests and checks is using hatch scripts (defined in `core/hatch.toml`): ```sh cd core # Run all unit tests hatch run unit-tests # Run unit tests and all code quality checks hatch run test # Run integration tests hatch run integration-tests # Run integration tests for test class hatch run integration-tests -k TestVerifyArtifacts # Run integration tests for test method name hatch run integration-tests -k test_load_artifact # Run integration tests in fail-fast mode hatch run integration-tests-fail-fast # Run linting checks only hatch run lint hatch run flake8 hatch run mypy hatch run black # Run all pre-commit hooks hatch run code-quality # Clean build artifacts hatch run clean ``` Hatch manages isolated environments and dependencies automatically. The commands above use the `default` environment which is recommended for most local development. **Using the `ci` environment (optional)** If you need to replicate exactly what runs in GitHub Actions (e.g., with coverage reporting), use the `ci` environment: ```sh cd core # Run unit tests with coverage hatch run ci:unit-tests # Run unit tests with a specific Python version hatch run +py=3.11 ci:unit-tests ``` > **Note:** Most developers should use the default environment (`hatch run unit-tests`). The `ci` environment is primarily for debugging CI failures or running tests with coverage. #### `pre-commit` [`pre-commit`](https://pre-commit.com) takes care of running all code-checks for formatting and linting. Run `hatch run setup` to install `pre-commit` in your local environment (we recommend running this command with a python virtual environment active). This installs several pip executables including black, mypy, and flake8. Once installed, hooks will run automatically on `git commit`, or you can run them manually with `hatch run code-quality`. #### `pytest` Finally, you can also run a specific test or group of tests using [`pytest`](https://docs.pytest.org/en/latest/) directly. After running `hatch run setup`, you can run pytest commands like: ```sh # run all unit tests in a file python3 -m pytest tests/unit/test_invocation_id.py # run a specific unit test python3 -m pytest tests/unit/test_invocation_id.py::TestInvocationId::test_invocation_id # run specific Postgres functional tests python3 -m pytest tests/functional/sources ``` > See [pytest usage docs](https://docs.pytest.org/en/6.2.x/usage.html) for an overview of useful command-line options. ### Unit, Integration, Functional? Here are some general rules for adding tests: * unit tests (`tests/unit`) don’t need to access a database; "pure Python" tests should be written as unit tests * functional tests (`tests/functional`) cover anything that interacts with a database, namely adapter ## Debugging 1. The logs for a `dbt run` have stack traces and other information for debugging errors (in `logs/dbt.log` in your project directory). 2. Try using a debugger, like `ipdb`. For pytest: `--pdb --pdbcls=IPython.terminal.debugger:pdb` 3. Sometimes, it’s easier to debug on a single thread: `dbt --single-threaded run` 4. To make print statements from Jinja macros: `{{ log(msg, info=true) }}` 5. You can also add `{{ debug() }}` statements, which will drop you into some auto-generated code that the macro wrote. 6. The dbt “artifacts” are written out to the ‘target’ directory of your dbt project. They are in unformatted json, which can be hard to read. Format them with: > python -m json.tool target/run_results.json > run_results.json ### Assorted development tips * Append `# type: ignore` to the end of a line if you need to disable `mypy` on that line. * Sometimes flake8 complains about lines that are actually fine, in which case you can put a comment on the line such as: # noqa or # noqa: ANNN, where ANNN is the error code that flake8 issues. * To collect output for `CProfile`, run dbt with the `-r` option and the name of an output file, i.e. `dbt -r dbt.cprof run`. If you just want to profile parsing, you can do: `dbt -r dbt.cprof parse`. `pip` install `snakeviz` to view the output. Run `snakeviz dbt.cprof` and output will be rendered in a browser window. ## Adding or modifying a CHANGELOG Entry We use [changie](https://changie.dev) to generate `CHANGELOG` entries. **Note:** Do not edit the `CHANGELOG.md` directly. Your modifications will be lost. Follow the steps to [install `changie`](https://changie.dev/guide/installation/) for your system. Once changie is installed and your PR is created for a new feature, simply run the following command and changie will walk you through the process of creating a changelog entry: ```shell changie new ``` Commit the file that's created and your changelog entry is complete! If you are contributing to a feature already in progress, you will modify the changie yaml file in dbt/.changes/unreleased/ related to your change. If you need help finding this file, please ask within the discussion for the pull request! You don't need to worry about which `dbt-core` version your change will go into. Just create the changelog entry with `changie`, and open your PR against the `main` branch. All merged changes will be included in the next minor version of `dbt-core`. The Core maintainers _may_ choose to "backport" specific changes in order to patch older minor versions. In that case, a maintainer will take care of that backport after merging your PR, before releasing the new version of `dbt-core`. ## Submitting a Pull Request Code can be merged into the current development branch `main` by opening a pull request. If the proposal looks like it's on the right track, then a `dbt-core` maintainer will triage the PR and label it as `ready_for_review`. From this point, two code reviewers will be assigned with the aim of responding to any updates to the PR within about one week. They may suggest code revision for style or clarity, or request that you add unit or integration test(s). These are good things! We believe that, with a little bit of help, anyone can contribute high-quality code. Once merged, your contribution will be available for the next release of `dbt-core`. Automated tests run via GitHub Actions. If you're a first-time contributor, all tests (including code checks and unit tests) will require a maintainer to approve. Changes in the `dbt-core` repository trigger integration tests against Postgres. dbt Labs also provides CI environments in which to test changes to other adapters, triggered by PRs in those adapters' repositories, as well as periodic maintenance checks of each adapter in concert with the latest `dbt-core` code changes. We require signed git commits. See docs [here](https://docs.github.com/en/authentication/managing-commit-signature-verification/signing-commits) for setting up code signing. Once all tests are passing, all comments are resolved, and your PR has been approved, a `dbt-core` maintainer will merge your changes into the active development branch. And that's it! Happy developing :tada: ## Troubleshooting Tips Sometimes, the content license agreement auto-check bot doesn't find a user's entry in its roster. If you need to force a rerun, add `@cla-bot check` in a comment on the pull request. ================================================ FILE: Dockerfile.test ================================================ ## # This dockerfile is used for local development and adapter testing only. # See `/docker` for a generic and production-ready docker file ## FROM ubuntu:22.04 ENV DEBIAN_FRONTEND noninteractive RUN apt-get update \ && apt-get install -y --no-install-recommends \ software-properties-common gpg-agent \ && add-apt-repository ppa:git-core/ppa -y \ && apt-get dist-upgrade -y \ && apt-get install -y --no-install-recommends \ netcat \ postgresql \ curl \ git \ ssh \ software-properties-common \ make \ build-essential \ ca-certificates \ libpq-dev \ libsasl2-dev \ libsasl2-2 \ libsasl2-modules-gssapi-mit \ libyaml-dev \ unixodbc-dev \ && add-apt-repository ppa:deadsnakes/ppa \ && apt-get install -y \ python-is-python3 \ python-dev-is-python3 \ python3-pip \ python3.10 \ python3.10-dev \ python3.10-venv \ python3.11 \ python3.11-dev \ python3.11-venv \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* ARG DOCKERIZE_VERSION=v0.6.1 RUN curl -LO https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ && tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ && rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz RUN pip3 install -U hatch wheel pre-commit # These args are passed in via docker-compose, which reads then from the .env file. # On Linux, run `make .env` to create the .env file for the current user. # On MacOS and Windows, these can stay unset. ARG USER_ID ARG GROUP_ID RUN if [ ${USER_ID:-0} -ne 0 ] && [ ${GROUP_ID:-0} -ne 0 ]; then \ groupadd -g ${GROUP_ID} dbt_test_user && \ useradd -m -l -u ${USER_ID} -g ${GROUP_ID} dbt_test_user; \ else \ useradd -mU -l dbt_test_user; \ fi RUN mkdir /usr/app && chown dbt_test_user /usr/app WORKDIR /usr/app VOLUME /usr/app USER dbt_test_user ENV PYTHONIOENCODING=utf-8 ENV LANG C.UTF-8 ================================================ FILE: Makefile ================================================ # ============================================================================ # DEPRECATED: This Makefile is maintained for backwards compatibility only. # # dbt-core now uses Hatch for task management and development workflows. # Please migrate to using hatch commands directly: # # make dev → cd core && hatch run setup # make unit → cd core && hatch run unit-tests # make test → cd core && hatch run test # make integration → cd core && hatch run integration-tests # make lint → cd core && hatch run lint # make code_quality → cd core && hatch run code-quality # make setup-db → cd core && hatch run setup-db # make clean → cd core && hatch run clean # # See core/pyproject.toml [tool.hatch.envs.default.scripts] for all available # commands and CONTRIBUTING.md for detailed usage instructions. # # This Makefile will be removed in a future version of dbt-core. # ============================================================================ .DEFAULT_GOAL:=help .PHONY: dev_req dev_req: ## Installs dbt-* packages in develop mode along with only development dependencies. @cd core && hatch run dev-req .PHONY: dev dev: ## Installs dbt-* packages in develop mode along with development dependencies and pre-commit. @cd core && hatch run setup .PHONY: dev-uninstall dev-uninstall: ## Uninstall all packages in venv except for build tools @pip freeze | grep -v "^-e" | cut -d "@" -f1 | xargs pip uninstall -y; \ pip uninstall -y dbt-core .PHONY: mypy mypy: ## Runs mypy against staged changes for static type checking. @cd core && hatch run mypy .PHONY: flake8 flake8: ## Runs flake8 against staged changes to enforce style guide. @cd core && hatch run flake8 .PHONY: black black: ## Runs black against staged changes to enforce style guide. @cd core && hatch run black .PHONY: lint lint: ## Runs flake8 and mypy code checks against staged changes. @cd core && hatch run lint .PHONY: code_quality code_quality: ## Runs all pre-commit hooks against all files. @cd core && hatch run code-quality .PHONY: unit unit: ## Runs unit tests with py @cd core && hatch run unit-tests .PHONY: test test: ## Runs unit tests with py and code checks against staged changes. @cd core && hatch run test .PHONY: integration integration: ## Runs core integration tests using postgres with py-integration @cd core && hatch run integration-tests .PHONY: integration-fail-fast integration-fail-fast: ## Runs core integration tests using postgres with py-integration in "fail fast" mode. @cd core && hatch run integration-tests-fail-fast .PHONY: setup-db setup-db: ## Setup Postgres database with docker-compose for system testing. @cd core && hatch run setup-db .PHONY: clean clean: ## Resets development environment. @cd core && hatch run clean .PHONY: json_schema json_schema: ## Update generated JSON schema using code changes. @cd core && hatch run json-schema .PHONY: help help: ## Show this help message. @echo 'usage: make [target]' @echo @echo 'DEPRECATED: This Makefile is a compatibility shim.' @echo 'Please use "cd core && hatch run <command>" directly.' @echo @echo 'targets:' @grep -E '^[8+a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' @echo @echo 'For more information, see CONTRIBUTING.md' ================================================ FILE: README.md ================================================ <p align="center"> <img src="https://raw.githubusercontent.com/dbt-labs/dbt-core/fa1ea14ddfb1d5ae319d5141844910dd53ab2834/etc/dbt-core.svg" alt="dbt logo" width="750"/> </p> <p align="center"> <a href="https://github.com/dbt-labs/dbt-core/actions/workflows/main.yml"> <img src="https://github.com/dbt-labs/dbt-core/actions/workflows/main.yml/badge.svg?event=push" alt="CI Badge"/> </a> <a href="https://www.bestpractices.dev/projects/11095"><img src="https://www.bestpractices.dev/projects/11095/badge"></a> </p> **[dbt](https://www.getdbt.com/)** enables data analysts and engineers to transform their data using the same practices that software engineers use to build applications. ![architecture](https://github.com/dbt-labs/dbt-core/blob/202cb7e51e218c7b29eb3b11ad058bd56b7739de/etc/dbt-transform.png) ## Understanding dbt Analysts using dbt can transform their data by simply writing select statements, while dbt handles turning these statements into tables and views in a data warehouse. These select statements, or "models", form a dbt project. Models frequently build on top of one another – dbt makes it easy to [manage relationships](https://docs.getdbt.com/docs/ref) between models, and [visualize these relationships](https://docs.getdbt.com/docs/documentation), as well as assure the quality of your transformations through [testing](https://docs.getdbt.com/docs/testing). ![dbt dag](https://raw.githubusercontent.com/dbt-labs/dbt-core/6c6649f9129d5d108aa3b0526f634cd8f3a9d1ed/etc/dbt-dag.png) ## Getting started - [Install dbt Core](https://docs.getdbt.com/docs/get-started/installation) or explore the [dbt Cloud CLI](https://docs.getdbt.com/docs/cloud/cloud-cli-installation), a command-line interface powered by [dbt Cloud](https://docs.getdbt.com/docs/cloud/about-cloud/dbt-cloud-features) that enhances collaboration. - Read the [introduction](https://docs.getdbt.com/docs/introduction/) and [viewpoint](https://docs.getdbt.com/docs/about/viewpoint/) ## Join the dbt Community - Be part of the conversation in the [dbt Community Slack](http://community.getdbt.com/) - Read more on the [dbt Community Discourse](https://discourse.getdbt.com) ## Reporting bugs and contributing code - Want to report a bug or request a feature? Let us know and open [an issue](https://github.com/dbt-labs/dbt-core/issues/new/choose) - Want to help us build dbt? Check out the [Contributing Guide](https://github.com/dbt-labs/dbt-core/blob/HEAD/CONTRIBUTING.md) ## Code of Conduct Everyone interacting in the dbt project's codebases, issue trackers, chat rooms, and mailing lists is expected to follow the [dbt Code of Conduct](https://docs.getdbt.com/community/resources/code-of-conduct). ================================================ FILE: SECURITY.md ================================================ [About dbt Core versions](https://docs.getdbt.com/docs/dbt-versions/core) ================================================ FILE: codecov.yml ================================================ ignore: - ".github" - ".changes" # Disable all status checks to prevent red X's in CI # Coverage data is still uploaded and PR comments are still posted coverage: status: project: off patch: off comment: layout: "header, diff, flags, components" # show component info in the PR comment component_management: individual_components: - component_id: unittests name: "Unit Tests" flag_regexes: - "unit" - component_id: integrationtests name: "Integration Tests" flag_regexes: - "integration" ================================================ FILE: core/.test_durations ================================================ { "tests/functional/access/test_access.py::TestAccess::test_access_attribute": 19.693285994999997, "tests/functional/access/test_access.py::TestAccessDbtProjectConfig::test_dbt_project_access_config": 3.759677324999984, "tests/functional/access/test_access.py::TestGenericTestRestrictAccess::test_generic_tests": 7.516862858999957, "tests/functional/access/test_access.py::TestRestrictedPackageAccess::test_restricted_private_ref": 3.6331608349999556, "tests/functional/access/test_access.py::TestRestrictedPackageAccess::test_restricted_protected_ref": 6.427091412000038, "tests/functional/access/test_access.py::TestUnrestrictedPackageAccess::test_unrestricted_protected_ref": 6.779408883999963, "tests/functional/analysis/test_analyses.py::TestAnalyses::test_postgres_analyses": 1.7279378940000072, "tests/functional/artifacts/test_artifact_fields.py::TestRelationNameInTests::test_relation_name_in_tests": 7.938031587000012, "tests/functional/artifacts/test_artifacts.py::TestVerifyArtifacts::test_load_artifact": 2.982837594999978, "tests/functional/artifacts/test_artifacts.py::TestVerifyArtifacts::test_run_and_generate": 16.822443084999975, "tests/functional/artifacts/test_artifacts.py::TestVerifyArtifactsReferences::test_references": 9.926057194999885, "tests/functional/artifacts/test_artifacts.py::TestVerifyArtifactsVersions::test_versions": 9.94775005200006, "tests/functional/artifacts/test_artifacts.py::TestVerifyRunOperation::test_run_model_with_operation": 3.423703161999981, "tests/functional/artifacts/test_artifacts.py::TestVerifyRunOperation::test_run_operation": 8.988266632999967, "tests/functional/artifacts/test_docs_generate_defer.py::TestDocsGenerateDefer::test_generate_defer": 1.614032256999991, "tests/functional/artifacts/test_override.py::TestDocsGenerateOverride::test_override_used": 9.814491736999969, "tests/functional/artifacts/test_previous_version_state.py::TestPreviousVersionState::test_backwards_compatible_run_results_versions": 8.14310199400012, "tests/functional/artifacts/test_previous_version_state.py::TestPreviousVersionState::test_backwards_compatible_versions": 27.613001499000006, "tests/functional/artifacts/test_previous_version_state.py::TestPreviousVersionState::test_compare_results_current": 3.892034876000025, "tests/functional/artifacts/test_previous_version_state.py::TestPreviousVersionState::test_compare_state_current": 3.334224613999993, "tests/functional/artifacts/test_previous_version_state.py::TestPreviousVersionState::test_get_manifest_schema_version": 0.03290388900001062, "tests/functional/artifacts/test_previous_version_state.py::TestPreviousVersionState::test_nonbackwards_compatible_versions": 10.849718646000042, "tests/functional/artifacts/test_previous_version_state.py::TestPreviousVersionState::test_project": 7.619359766999992, "tests/functional/artifacts/test_run_execution_result.py::test_run_execution_result_compiled_serialization": 8.298854163000101, "tests/functional/artifacts/test_run_results.py::TestRunResultsSerializableInContext::test_results_serializable": 9.274184312000102, "tests/functional/artifacts/test_run_results.py::TestRunResultsTimingFailure::test_timing_exists": 8.796365435999974, "tests/functional/artifacts/test_run_results.py::TestRunResultsTimingSuccess::test_timing_exists": 8.697306241000092, "tests/functional/artifacts/test_run_results.py::TestRunResultsWritesFileOnSignal::test_run_results_are_written_on_signal": 0.0012458909999395473, "tests/functional/basic/test_basic.py::test_basic": 9.023939860000155, "tests/functional/basic/test_invalid_reference.py::test_undefined_value": 8.499570390999907, "tests/functional/basic/test_jaffle_shop.py::TestBasic::test_basic": 17.90248904300006, "tests/functional/basic/test_jaffle_shop.py::TestBasic::test_execution_time_format_is_humanized": 12.56305247499995, "tests/functional/basic/test_mixed_case_db.py::test_basic": 14.851846751999915, "tests/functional/basic/test_project.py::TestArchiveNotAllowed::test_archive_not_allowed": 5.360570413000005, "tests/functional/basic/test_project.py::TestProjectConfigVersionMissing::test_empty_version": 10.43638870000018, "tests/functional/basic/test_project.py::TestProjectDbtCloudConfig::test_dbt_cloud": 4.401225789000023, "tests/functional/basic/test_project.py::TestProjectDbtCloudConfigString::test_dbt_cloud_invalid": 11.372771833000002, "tests/functional/basic/test_project.py::TestProjectYamlVersionInvalid::test_invalid_version": 10.449192945999926, "tests/functional/basic/test_project.py::TestProjectYamlVersionMissing::test_empty_version": 10.180516065000006, "tests/functional/basic/test_project.py::TestProjectYamlVersionValid::test_valid_version": 10.430264007000005, "tests/functional/basic/test_project.py::TestSchemaYmlVersionMissing::test_empty_version": 10.775652831999878, "tests/functional/basic/test_project.py::TestVersionSpecifierChecksComeBeforeYamlValidation::test_version_specifier_checks_before_yaml_validation": 5.235425129999953, "tests/functional/basic/test_simple_reference.py::test_simple_ref_with_models": 20.031065766999973, "tests/functional/basic/test_simple_reference.py::test_simple_reference": 37.87860925199993, "tests/functional/basic/test_simple_reference.py::test_simple_reference_with_models_and_children": 20.939061037999977, "tests/functional/basic/test_varchar_widening.py::test_varchar_widening": 2.2054005790000133, "tests/functional/build_command/test_build.py::TestCircularRelationshipTestsBuild::test_circular_relationship_test_success": 14.430405085000075, "tests/functional/build_command/test_build.py::TestDownstreamSelection::test_downstream_selection": 14.403590032000125, "tests/functional/build_command/test_build.py::TestFailingBuild::test_failing_test_skips_downstream": 14.171351606999906, "tests/functional/build_command/test_build.py::TestFailingTestsBuild::test_failing_test_skips_downstream": 3.4541219389999895, "tests/functional/build_command/test_build.py::TestInterdependentModels::test_interdependent_models": 15.360765523000055, "tests/functional/build_command/test_build.py::TestInterdependentModelsFail::test_interdependent_models_fail": 14.990420995000022, "tests/functional/build_command/test_build.py::TestLimitedUpstreamSelection::test_limited_upstream_selection": 14.407232419000024, "tests/functional/build_command/test_build.py::TestPassingBuild::test_build_happy_path": 14.911859800999878, "tests/functional/build_command/test_build.py::TestSimpleBlockingTest::test_simple_blocking_test": 13.293703752000056, "tests/functional/catalogs/test_catalogs_parsing.py::TestDuplicateWriteIntegration::test_integration": 6.7745474119998335, "tests/functional/catalogs/test_catalogs_parsing.py::TestInvalidWriteIntegration::test_integration": 0.3199430450000307, "tests/functional/catalogs/test_catalogs_parsing.py::TestMultipleWriteIntegration::test_integration": 14.075824718000035, "tests/functional/catalogs/test_catalogs_parsing.py::TestNoActiveWriteIntegration::test_integration": 6.72967783799993, "tests/functional/catalogs/test_catalogs_parsing.py::TestSingleWriteIntegration::test_integration": 13.811159436000139, "tests/functional/clean/test_clean.py::TestCleanPathOutsideProjectAbsolute::test_clean_path_outside_project": 6.962187210000025, "tests/functional/clean/test_clean.py::TestCleanPathOutsideProjectRelative::test_clean_path_outside_project": 6.944305652000139, "tests/functional/clean/test_clean.py::TestCleanPathOutsideProjectWithFlag::test_clean_path_outside_project": 6.980458942999917, "tests/functional/clean/test_clean.py::TestCleanRelativeProjectDir::test_clean_relative_project_dir": 6.891713305000053, "tests/functional/clean/test_clean.py::TestCleanSourcePath::test_clean_source_path": 1.763001666000008, "tests/functional/cli/test_cli_exit_codes.py::TestExitCodeOne::test_exc_thrown": 15.23298833900003, "tests/functional/cli/test_cli_exit_codes.py::TestExitCodeZero::test_no_exc_thrown": 2.4715167580000355, "tests/functional/cli/test_click_flags.py::TestClickCLIFlagsResolveFalsey::test_resolve_falsey": 0.6696680100001231, "tests/functional/cli/test_click_flags.py::TestClickCLIFlagsResolveTruthy::test_resolve_truthy": 7.186096984999949, "tests/functional/cli/test_click_flags.py::TestClickEnvVarFlagsResolveFalsey::test_resolve_falsey[0]": 0.6867838390000429, "tests/functional/cli/test_click_flags.py::TestClickEnvVarFlagsResolveFalsey::test_resolve_falsey[f]": 0.9031602569999961, "tests/functional/cli/test_click_flags.py::TestClickEnvVarFlagsResolveFalsey::test_resolve_falsey[false]": 7.5558693230000245, "tests/functional/cli/test_click_flags.py::TestClickEnvVarFlagsResolveFalsey::test_resolve_falsey[n]": 7.431096890000163, "tests/functional/cli/test_click_flags.py::TestClickEnvVarFlagsResolveFalsey::test_resolve_falsey[no]": 15.229194409000229, "tests/functional/cli/test_click_flags.py::TestClickEnvVarFlagsResolveFalsey::test_resolve_falsey[off]": 0.6196650370000043, "tests/functional/cli/test_click_flags.py::TestClickEnvVarFlagsResolveTruthy::test_resolve_truthy[1]": 0.05747299999984534, "tests/functional/cli/test_click_flags.py::TestClickEnvVarFlagsResolveTruthy::test_resolve_truthy[on]": 0.036709330999883605, "tests/functional/cli/test_click_flags.py::TestClickEnvVarFlagsResolveTruthy::test_resolve_truthy[t]": 0.2084927049999976, "tests/functional/cli/test_click_flags.py::TestClickEnvVarFlagsResolveTruthy::test_resolve_truthy[true]": 0.038485730000047624, "tests/functional/cli/test_click_flags.py::TestClickEnvVarFlagsResolveTruthy::test_resolve_truthy[y]": 0.03741321299980882, "tests/functional/cli/test_click_flags.py::TestClickEnvVarFlagsResolveTruthy::test_resolve_truthy[yes]": 7.547093960999973, "tests/functional/cli/test_env_var_deprecations.py::TestDeprecatedEnvVars::test_defer": 3.296107156000005, "tests/functional/cli/test_env_var_deprecations.py::TestDeprecatedEnvVars::test_favor_state": 1.1239154569999528, "tests/functional/cli/test_env_var_deprecations.py::TestDeprecatedEnvVars::test_print": 3.9606142529999886, "tests/functional/cli/test_env_var_deprecations.py::TestDeprecatedEnvVars::test_state": 1.2487158410000347, "tests/functional/cli/test_error_handling.py::TestHandledExit::test_fail_fast_failed_run_does_not_throw": 1.2742476209999154, "tests/functional/cli/test_error_handling.py::TestHandledExit::test_failed_run_does_not_throw": 2.3287831530000176, "tests/functional/cli/test_multioption.py::TestOutputKeys::test_output_key_args": 1.12989019500003, "tests/functional/cli/test_multioption.py::TestOutputKeys::test_output_key_nested": 1.2242512610000063, "tests/functional/cli/test_multioption.py::TestOutputKeys::test_output_key_nested_deep_meta": 0.5794108019999982, "tests/functional/cli/test_multioption.py::TestOutputKeys::test_output_key_nested_deep_nonexistent": 1.3214415239999653, "tests/functional/cli/test_multioption.py::TestOutputKeys::test_output_key_nested_mixed_existent_nonexistent": 0.9375792609999962, "tests/functional/cli/test_multioption.py::TestOutputKeys::test_output_key_nested_nonexistent": 1.2864939249999452, "tests/functional/cli/test_multioption.py::TestOutputKeys::test_output_key_nested_single_arg": 1.2544720579999762, "tests/functional/cli/test_multioption.py::TestOutputKeys::test_output_key_nested_whole_meta_object": 0.32358837599998935, "tests/functional/cli/test_multioption.py::TestOutputKeys::test_output_key_quoted": 1.1302335889999995, "tests/functional/cli/test_multioption.py::TestOutputKeys::test_output_key_single": 2.645741085999987, "tests/functional/cli/test_multioption.py::TestResourceType::test_resource_type_args": 1.0797773520000078, "tests/functional/cli/test_multioption.py::TestResourceType::test_resource_type_quoted": 1.2743162830000188, "tests/functional/cli/test_multioption.py::TestResourceType::test_resource_type_single": 1.0616604209999991, "tests/functional/cli/test_multioption.py::TestSelectExclude::test_select_exclude_args": 6.061109839999972, "tests/functional/cli/test_multioption.py::TestSelectExclude::test_select_exclude_quoted": 5.288789140000006, "tests/functional/cli/test_multioption.py::TestSelectExclude::test_select_exclude_single": 5.510430188999976, "tests/functional/cli/test_option_interaction_validations.py::TestEventTimeEndEventTimeStart::test_option_combo[2024-10-01-2024-10-02-True]": 0.823348368999973, "tests/functional/cli/test_option_interaction_validations.py::TestEventTimeEndEventTimeStart::test_option_combo[2024-10-02-2024-10-01-False]": 0.10365628400001015, "tests/functional/cli/test_option_interaction_validations.py::TestEventTimeEndEventTimeStartMutuallyRequired::test_option_combo[--event-time-end---event-time-start]": 0.08300517900011073, "tests/functional/cli/test_option_interaction_validations.py::TestEventTimeEndEventTimeStartMutuallyRequired::test_option_combo[--event-time-start---event-time-end]": 2.1814881920000175, "tests/functional/cli/test_requires.py::TestEngineEnvVarPickedUpByClick::test_engine_env_var_picked_up_by_cli_flags": 10.14885321700001, "tests/functional/cli/test_requires.py::TestKnownEngineEnvVarsExplicit::test_allow_list_is_correct": 5.149495886000011, "tests/functional/cli/test_requires.py::TestOldEngineEnvVarPropagation::test_engine_env_var_propagation[False-False-0]": 4.447983458999943, "tests/functional/cli/test_requires.py::TestOldEngineEnvVarPropagation::test_engine_env_var_propagation[False-True-True]": 0.7585075270000061, "tests/functional/cli/test_requires.py::TestOldEngineEnvVarPropagation::test_engine_env_var_propagation[True-False-False]": 0.9553055470000089, "tests/functional/cli/test_requires.py::TestOldEngineEnvVarPropagation::test_engine_env_var_propagation[True-True-True]": 3.655993809000023, "tests/functional/cli/test_resolvers.py::TestDefaultLogPathNoProject::test_default_log_path_no_project": 0.0020984590000807657, "tests/functional/cli/test_resolvers.py::TestDefaultLogPathWithProject::test_default_log_path_with_project": 2.443658099000004, "tests/functional/cli/test_resolvers.py::TestDefaultLogPathWithProjectNoConfiguredLogPath::test_default_log_path_with_project": 2.4112458740000307, "tests/functional/colors/test_colors.py::TestColors::test_no_use_colors": 1.0433806659999902, "tests/functional/colors/test_colors.py::TestColors::test_use_colors": 6.195680938999999, "tests/functional/column_quoting/test_column_quotes.py::TestColumnQuotingDefault::test_column_quotes": 2.852184763000025, "tests/functional/column_quoting/test_column_quotes.py::TestColumnQuotingDisabled::test_column_quotes": 14.150319632999981, "tests/functional/column_quoting/test_column_quotes.py::TestColumnQuotingEnabled::test_column_quotes": 2.5683979560000125, "tests/functional/compile/test_compile.py::TestCompile::test_compile_inline_not_add_node": 8.575035064999838, "tests/functional/compile/test_compile.py::TestCompile::test_compile_inline_ref_node_not_exist": 4.3541807800000925, "tests/functional/compile/test_compile.py::TestCompile::test_compile_inline_syntax_error": 4.393746528999941, "tests/functional/compile/test_compile.py::TestCompile::test_graph_summary_output": 4.844826934000025, "tests/functional/compile/test_compile.py::TestCompile::test_inline_fail": 3.9648541490000753, "tests/functional/compile/test_compile.py::TestCompile::test_inline_fail_database_error": 1.9286313570000004, "tests/functional/compile/test_compile.py::TestCompile::test_inline_pass": 4.394097374000012, "tests/functional/compile/test_compile.py::TestCompile::test_inline_pass_quiet": 0.6285069539999881, "tests/functional/compile/test_compile.py::TestCompile::test_multiline_jinja": 1.7603685540000242, "tests/functional/compile/test_compile.py::TestCompile::test_none": 4.513015250999999, "tests/functional/compile/test_compile.py::TestCompile::test_output_json_inline": 4.434909527000173, "tests/functional/compile/test_compile.py::TestCompile::test_output_json_inline_quiet": 4.501236069000015, "tests/functional/compile/test_compile.py::TestCompile::test_output_json_select": 0.7105877959999845, "tests/functional/compile/test_compile.py::TestCompile::test_output_json_select_quiet": 4.546718135999981, "tests/functional/compile/test_compile.py::TestCompile::test_select_pass": 4.194874711000011, "tests/functional/compile/test_compile.py::TestCompile::test_select_pass_empty": 4.22149414099988, "tests/functional/compile/test_compile.py::TestCompile::test_select_pass_quiet": 4.167139651999946, "tests/functional/compile/test_compile.py::TestEphemeralModelWithAlias::test_compile": 8.302734589999943, "tests/functional/compile/test_compile.py::TestEphemeralModels::test_first_selector": 7.3124343530000715, "tests/functional/compile/test_compile.py::TestEphemeralModels::test_last_selector": 3.713934369999947, "tests/functional/compile/test_compile.py::TestEphemeralModels::test_middle_selector": 3.7627890670000284, "tests/functional/compile/test_compile.py::TestEphemeralModels::test_no_selector": 3.8010051750000002, "tests/functional/compile/test_compile.py::TestEphemeralModels::test_with_recursive_cte": 3.854703838999967, "tests/functional/compile/test_compile.py::TestIntrospectFlag::test_default": 8.733216757000037, "tests/functional/compile/test_compile.py::TestIntrospectFlag::test_no_introspect": 3.469682151000029, "tests/functional/compile/test_compile.py::TestSqlParseGroupingDepthLimit::test_sqlparse_grouping_depth_limit": 1.723351385000015, "tests/functional/compile/test_compile.py::TestSqlParseGroupingTokenLimit::test_sqlparse_grouping_token_limit": 4.526235583999991, "tests/functional/configs/test_configs.py::TestConfigs::test_config_layering": 25.892917640000064, "tests/functional/configs/test_configs.py::TestInvalidSeedsMaterializationProj::test_seeds_materialization_proj_config": 0.6982047689999717, "tests/functional/configs/test_configs.py::TestInvalidSeedsMaterializationSchema::test_seeds_materialization_schema_config": 0.8212266049999641, "tests/functional/configs/test_configs.py::TestInvalidSnapshotsMaterializationProj::test_snapshots_materialization_proj_config": 1.0271297680000089, "tests/functional/configs/test_configs.py::TestInvalidSnapshotsMaterializationSchema::test_snapshots_materialization_schema_config": 10.64628221800001, "tests/functional/configs/test_configs.py::TestInvalidTestsMaterializationProj::test_tests_materialization_proj_config": 9.438830603000042, "tests/functional/configs/test_configs.py::TestTargetConfigs::test_alternative_target_paths": 10.385924960000011, "tests/functional/configs/test_configs_in_schema_files.py::TestLegacySchemaFileConfigs::test_config_layering": 58.362766896999915, "tests/functional/configs/test_configs_in_schema_files.py::TestListSchemaFile::test_list_schema": 11.340628384999945, "tests/functional/configs/test_configs_in_schema_files.py::TestSchemaFileConfigs::test_config_layering": 52.898641360000056, "tests/functional/configs/test_contract_configs.py::TestModelContractEnabledConfigs::test__model_contract": 13.344759579000083, "tests/functional/configs/test_contract_configs.py::TestModelContractEnabledConfigsMissingDataTypes::test_undefined_column_type": 13.544680990999836, "tests/functional/configs/test_contract_configs.py::TestModelContractMissingYAMLColumns::test__missing_column_contract_error": 2.5923054649999813, "tests/functional/configs/test_contract_configs.py::TestModelLevelConstraintsErrorMessages::test__config_errors": 13.269513075999953, "tests/functional/configs/test_contract_configs.py::TestModelLevelConstraintsWarningMessages::test__config_warning": 9.62273693499992, "tests/functional/configs/test_contract_configs.py::TestModelLevelContractDisabledConfigs::test__model_contract_false": 13.158855357999869, "tests/functional/configs/test_contract_configs.py::TestModelLevelContractEnabledConfigs::test__model_contract_true": 19.71907576199999, "tests/functional/configs/test_contract_configs.py::TestModelLevelContractErrorMessages::test__config_errors": 13.356074628999977, "tests/functional/configs/test_contract_configs.py::TestPrimaryKeysModelAndColumnLevelConstraints::test_model_column_pk_error": 0.7245696980000105, "tests/functional/configs/test_contract_configs.py::TestPrimaryKeysMultipleColumns::test_pk_multiple_columns": 2.177795070000002, "tests/functional/configs/test_contract_configs.py::TestProjectContractEnabledConfigs::test_defined_column_type": 12.974213028000122, "tests/functional/configs/test_contract_configs.py::TestProjectContractEnabledConfigsError::test_undefined_column_type": 13.282185422000111, "tests/functional/configs/test_contract_configs.py::TestPythonModelLevelContractErrorMessages::test__python_errors": 1.6309031109999523, "tests/functional/configs/test_contract_configs.py::TestSchemaContractEnabledConfigs::test__schema_error": 1.4336319429999662, "tests/functional/configs/test_custom_node_colors_configs.py::TestCustomNodeColorIncorrectColorHEXYMLConfig::test__invalid_color_docs_under_config": 6.099621764000005, "tests/functional/configs/test_custom_node_colors_configs.py::TestCustomNodeColorIncorrectColorModelConfig::test__invalid_color_config_block": 3.313332834999983, "tests/functional/configs/test_custom_node_colors_configs.py::TestCustomNodeColorIncorrectColorNameYMLConfig::test__invalid_color_docs_not_under_config": 5.256898958999955, "tests/functional/configs/test_custom_node_colors_configs.py::TestCustomNodeColorIncorrectColorProject::test__invalid_color_project": 0.6639624049999924, "tests/functional/configs/test_custom_node_colors_configs.py::TestModelLevelProjectColorConfigs::test__model_override_project": 2.6884690750000004, "tests/functional/configs/test_custom_node_colors_configs.py::TestModelLevelSchemaColorConfigs::test__model_override_schema": 2.911380523000048, "tests/functional/configs/test_custom_node_colors_configs.py::TestModelOverProjectColorConfigs::test__model_show_overrides_dbt_project": 3.351814560999969, "tests/functional/configs/test_custom_node_colors_configs.py::TestSchemaOverProjectColorConfigs::test__schema_override_project": 1.0244241880000118, "tests/functional/configs/test_custom_node_colors_configs.py::TestSubdirectoryColorConfigs::test__project_folder_override_project_root": 3.0257529720000207, "tests/functional/configs/test_disabled_configs.py::TestDisabledConfigs::test_conditional_model": 15.415401803999998, "tests/functional/configs/test_disabled_configs.py::TestDisabledConfigs::test_disable_seed_partial_parse": 11.202385377999974, "tests/functional/configs/test_disabled_configs.py::TestDisabledConfigsSameName::test_disabled_analysis": 4.263107648000073, "tests/functional/configs/test_disabled_model.py::TestInvalidEnabledConfig::test_invalid_config": 6.700385677999975, "tests/functional/configs/test_disabled_model.py::TestManyDisabledNodesSuccess::test_many_disabled_config": 0.6557107710000025, "tests/functional/configs/test_disabled_model.py::TestModelDisabledConfigs::test_disabled_config": 0.6521544539999695, "tests/functional/configs/test_disabled_model.py::TestMultipleDisabledNodesForUniqueIDFailure::test_disabled_config": 6.145755412000028, "tests/functional/configs/test_disabled_model.py::TestMultipleDisabledNodesOverrideModel::test_multiple_disabled_config": 6.647647699999993, "tests/functional/configs/test_disabled_model.py::TestMultipleDisabledNodesSuccess::test_multiple_disabled_config": 0.8877035249999778, "tests/functional/configs/test_disabled_model.py::TestOverrideFalseYAMLConfigsInSQL::test_override_yaml_sql_config": 4.020583670000008, "tests/functional/configs/test_disabled_model.py::TestOverrideProjectConfigsInSQL::test_override_project_sql_config": 5.595645962000049, "tests/functional/configs/test_disabled_model.py::TestOverrideProjectConfigsInYaml::test_override_project_yaml_config": 5.544950577000009, "tests/functional/configs/test_disabled_model.py::TestOverrideTrueYAMLConfigsInSQL::test_override_yaml_sql_config": 6.1709249380000415, "tests/functional/configs/test_disabled_model.py::TestSchemaDisabledConfigs::test_disabled_config": 5.007815124000047, "tests/functional/configs/test_disabled_model.py::TestSchemaDisabledConfigsFailure::test_disabled_config": 5.038891781999951, "tests/functional/configs/test_dupe_paths.py::TestDupeProjectPaths::test_config_with_dupe_paths": 11.528093543000011, "tests/functional/configs/test_dupe_paths.py::TestDupeStrippedProjectPaths::test_config_with_dupe_paths": 11.66505034100004, "tests/functional/configs/test_get_default.py::TestConfigGetDefault::test_config_with_get_default": 7.981411667000032, "tests/functional/configs/test_get_default.py::TestConfigGetMeta::test_config_with_meta_key": 12.29262933900003, "tests/functional/configs/test_get_default.py::TestConfigGetMetaRequire::test_config_with_meta_require": 1.636889019000023, "tests/functional/configs/test_grant_configs.py::TestGrantConfigs::test_model_grant_config": 39.03587114100003, "tests/functional/configs/test_indiv_tests.py::TestConfigIndivTests::test_configuring_individual_tests": 37.198720931000025, "tests/functional/configs/test_unused_configs.py::TestUnusedModelConfigs::test_warn_unused_configuration_paths": 14.602653407000048, "tests/functional/configs/test_vars_file.py::TestComplexVarValues::test_complex_var_values": 12.805695482999909, "tests/functional/configs/test_vars_file.py::TestDbtProjectVarCliOverridesFile::test_cli_overrides_vars_file": 10.501770113999896, "tests/functional/configs/test_vars_file.py::TestDbtProjectVarFromVarsFile::test_dbt_project_var_from_vars_file": 10.362994761000095, "tests/functional/configs/test_vars_file.py::TestDbtProjectVarMissingFromVarsFile::test_error_when_var_missing": 0.04560613100011324, "tests/functional/configs/test_vars_file.py::TestEmptyVarsFileAllowsProjectVars::test_empty_vars_file_uses_project_vars": 1.1186558190000113, "tests/functional/configs/test_vars_file.py::TestMacroVarCliOverridesFile::test_macro_var_cli_overrides_file": 11.158734076999963, "tests/functional/configs/test_vars_file.py::TestMacroVarFromVarsFile::test_macro_var_from_file": 1.1551887620000798, "tests/functional/configs/test_vars_file.py::TestMutualExclusivityError::test_error_when_both_have_vars": 0.042660349999991354, "tests/functional/configs/test_vars_file.py::TestPartialCliOverride::test_partial_cli_override": 12.100688074000004, "tests/functional/configs/test_vars_file.py::TestSqlModelVarCliOverridesFile::test_sql_model_cli_overrides_file": 11.089052496000022, "tests/functional/configs/test_vars_file.py::TestSqlModelVarFromVarsFile::test_sql_model_var_from_file": 11.16906512700018, "tests/functional/configs/test_vars_file.py::TestSqlModelVarMissingFromVarsFile::test_error_when_model_var_missing": 10.899044949000086, "tests/functional/configs/test_vars_file.py::TestVarsFileWithoutVarsKeyAllowsProjectVars::test_vars_file_without_vars_key_uses_project_vars": 11.765256581999893, "tests/functional/configs/test_versioned_model_constraint.py::TestPrimaryKeysModelAndColumnLevelConstraints::test_model_column_pk_error": 25.474653758000045, "tests/functional/configs/test_versioned_model_constraint.py::TestPrimaryKeysMultipleColumns::test_pk_multiple_columns": 25.615617595999993, "tests/functional/configs/test_versioned_model_constraint.py::TestVersionedModelConstraints::test_versioned_model_constraints": 20.631093999999962, "tests/functional/configs/test_warn_error_options.py::TestEmptyWarnError::test_project_flags": 14.867306955000004, "tests/functional/configs/test_warn_error_options.py::TestRequireAllWarningsHandledByWarnErrorBehaviorFlag::test_require_all_warnings_handed_by_warn_error_behavior_flag": 22.369481161999943, "tests/functional/configs/test_warn_error_options.py::TestWarnErrorOptionsFromCLICanExcludeSpecificEvent::test_can_exclude_specific_event": 26.423526234000065, "tests/functional/configs/test_warn_error_options.py::TestWarnErrorOptionsFromCLICanRaiseWarningToError::test_can_raise_warning_to_error": 2.04963370400003, "tests/functional/configs/test_warn_error_options.py::TestWarnErrorOptionsFromCLICanSilence::test_can_silence": 2.320834074999979, "tests/functional/configs/test_warn_error_options.py::TestWarnErrorOptionsFromCLICantSetBothIncludeAndError::test_cant_set_both_exclude_and_warn": 0.06371149100016282, "tests/functional/configs/test_warn_error_options.py::TestWarnErrorOptionsFromCLICantSetBothIncludeAndError::test_cant_set_both_include_and_error": 6.304755018999913, "tests/functional/configs/test_warn_error_options.py::TestWarnErrorOptionsFromProjectCanExcludeSpecificEvent::test_can_exclude_specific_event": 0.0013343959999474464, "tests/functional/configs/test_warn_error_options.py::TestWarnErrorOptionsFromProjectCanRaiseWarningToError::test_can_raise_warning_to_error": 29.41912545699995, "tests/functional/configs/test_warn_error_options.py::TestWarnErrorOptionsFromProjectCanSilence::test_can_silence": 5.462176998999979, "tests/functional/configs/test_warn_error_options.py::TestWarnErrorOptionsFromProjectCantSetBothExcludeAndWarn::test_cant_set_both_exclude_and_warn": 6.721651358999907, "tests/functional/configs/test_warn_error_options.py::TestWarnErrorOptionsFromProjectCantSetBothIncludeAndError::test_cant_set_both_include_and_error": 6.674710779999941, "tests/functional/constraints/test_foreign_key_constraints.py::TestColumnLevelForeignKeyConstraintRefNotFoundError::test_model_level_fk_to_doesnt_exist": 0.8127375680000029, "tests/functional/constraints/test_foreign_key_constraints.py::TestColumnLevelForeignKeyConstraintRefSyntaxError::test_model_level_fk_to": 15.705302438000217, "tests/functional/constraints/test_foreign_key_constraints.py::TestColumnLevelForeignKeyConstraintToRef::test_column_level_fk_to": 23.381145951000008, "tests/functional/constraints/test_foreign_key_constraints.py::TestColumnLevelForeignKeyConstraintToRefDeferRelation::test_column_level_fk_to_current_relation_when_target_selected": 18.505497894999962, "tests/functional/constraints/test_foreign_key_constraints.py::TestColumnLevelForeignKeyConstraintToRefDeferRelation::test_column_level_fk_to_defer_relation_when_target_not_selected": 26.358900947999928, "tests/functional/constraints/test_foreign_key_constraints.py::TestColumnLevelForeignKeyConstraintToSource::test_model_level_fk_to": 23.852429037999855, "tests/functional/constraints/test_foreign_key_constraints.py::TestModelLevelForeignKeyConstraintRefNotFoundError::test_model_level_fk_to_doesnt_exist": 15.055457412000123, "tests/functional/constraints/test_foreign_key_constraints.py::TestModelLevelForeignKeyConstraintRefSyntaxError::test_model_level_fk_to": 0.9532859090000159, "tests/functional/constraints/test_foreign_key_constraints.py::TestModelLevelForeignKeyConstraintRefToDeferRelation::test_model_level_fk_to_current_relation_when_target_selected": 17.989187679999986, "tests/functional/constraints/test_foreign_key_constraints.py::TestModelLevelForeignKeyConstraintRefToDeferRelation::test_model_level_fk_to_defer_relation_when_target_not_selected": 25.373925477000284, "tests/functional/constraints/test_foreign_key_constraints.py::TestModelLevelForeignKeyConstraintToRef::test_model_level_fk_to": 21.87024449499995, "tests/functional/constraints/test_foreign_key_constraints.py::TestModelLevelForeignKeyConstraintToSource::test_model_level_fk_to": 22.921905586999856, "tests/functional/context_methods/test_builtin_functions.py::TestContextBuiltinExceptions::test_builtin_function_exception": 0.9633488140000281, "tests/functional/context_methods/test_builtin_functions.py::TestContextBuiltins::test_builtin_dbt_metadata_envs_function": 9.203543522000018, "tests/functional/context_methods/test_builtin_functions.py::TestContextBuiltins::test_builtin_invocation_args_dict_function": 8.960963966000008, "tests/functional/context_methods/test_builtin_functions.py::TestContextBuiltins::test_builtin_set_function": 16.667912322999882, "tests/functional/context_methods/test_builtin_functions.py::TestContextBuiltins::test_builtin_zip_function": 8.52785550800013, "tests/functional/context_methods/test_cli_var_override.py::TestCLIVarOverride::test__override_vars_global": 29.71246227300003, "tests/functional/context_methods/test_cli_var_override.py::TestCLIVarOverridePorject::test__override_vars_project_level": 2.2715008490000628, "tests/functional/context_methods/test_cli_vars.py::TestCLIVars::test__cli_vars_longform": 9.405296797999938, "tests/functional/context_methods/test_cli_vars.py::TestCLIVarsPackages::test_cli_vars_in_packages": 2.8302657329999192, "tests/functional/context_methods/test_cli_vars.py::TestCLIVarsProfile::test_cli_vars_in_profile": 2.5442399249999994, "tests/functional/context_methods/test_cli_vars.py::TestCLIVarsScrubbing::test__exception_scrubbing": 4.512491621000095, "tests/functional/context_methods/test_cli_vars.py::TestCLIVarsScrubbing::test__run_results_scrubbing": 3.30067886300003, "tests/functional/context_methods/test_cli_vars.py::TestCLIVarsSelectors::test_vars_in_selectors": 5.273423541999932, "tests/functional/context_methods/test_cli_vars.py::TestCLIVarsSimple::test__cli_vars_longer": 2.859908650999955, "tests/functional/context_methods/test_cli_vars.py::TestCLIVarsSimple::test__cli_vars_shorthand": 3.2119398159999832, "tests/functional/context_methods/test_custom_env_vars.py::TestCustomVarInLogs::test_extra_filled": 1.0755296460000068, "tests/functional/context_methods/test_env_vars.py::TestEnvVarInCreateSchema::test_env_var_in_create_schema": 4.919623683000054, "tests/functional/context_methods/test_env_vars.py::TestEnvVars::test_env_vars_dev": 3.473758255000007, "tests/functional/context_methods/test_env_vars.py::TestEnvVars::test_env_vars_prod": 2.898812863000103, "tests/functional/context_methods/test_env_vars.py::TestEnvVars::test_env_vars_secrets": 1.8596417929999802, "tests/functional/context_methods/test_secret_env_vars.py::TestAllowSecretProfilePackage::test_allow_secrets": 7.10889669900007, "tests/functional/context_methods/test_secret_env_vars.py::TestCloneFailSecretNotRendered::test_fail_clone_with_scrubbing": 1.6307092519999742, "tests/functional/context_methods/test_secret_env_vars.py::TestCloneFailSecretScrubbed::test_fail_clone_with_scrubbing": 1.6996358119999968, "tests/functional/context_methods/test_secret_env_vars.py::TestDisallowSecretModel::test_disallow_secret": 5.3389670970000225, "tests/functional/context_methods/test_var_dependency.py::TestVarConfigDependencyInheritance::test_root_var_overrides_package_var": 7.180892410999945, "tests/functional/context_methods/test_var_dependency.py::TestVarDependencyInheritance::test_var_mutual_overrides_v1_conversion": 7.733033945000045, "tests/functional/context_methods/test_var_in_generate_name.py::TestMissingVarGenerateNameMacro::test_generate_schema_name_var": 1.8204441520000216, "tests/functional/context_methods/test_yaml_functions.py::TestContextVars::test_json_data_tests": 5.388633751999976, "tests/functional/contracts/test_contract_enforcement.py::TestIncrementalModelContractEnforcement::test_contracted_incremental": 8.658500275000051, "tests/functional/contracts/test_contract_precision.py::TestModelContractNumericNoPrecision::test_contracted_numeric_without_precision": 8.570239167999944, "tests/functional/contracts/test_contract_precision.py::TestModelContractNumericPrecision::test_contracted_numeric_with_precision": 5.931563314999892, "tests/functional/contracts/test_nonstandard_data_type.py::TestModelContractUnrecognizedTypeCode1::test_nonstandard_data_type": 6.4560003670000015, "tests/functional/contracts/test_nonstandard_data_type.py::TestModelContractUnrecognizedTypeCodeActualMismatch::test_nonstandard_data_type": 6.328724072, "tests/functional/contracts/test_nonstandard_data_type.py::TestModelContractUnrecognizedTypeCodeExpectedMismatch::test_nonstandard_data_type": 6.503988707000019, "tests/functional/custom_aliases/test_custom_aliases.py::TestAliases::test_customer_alias_name": 9.997058941000091, "tests/functional/custom_aliases/test_custom_aliases.py::TestAliasesWithConfig::test_customer_alias_name": 10.64142559000004, "tests/functional/custom_schemas/test_custom_schemas.py::TestCustomSchema::test_custom_schema": 6.828831211000079, "tests/functional/custom_schemas/test_custom_schemas.py::TestCustomSchemaNullReturn::test_custom_schema_null_return": 6.558580455000026, "tests/functional/custom_schemas/test_custom_schemas.py::TestCustomSchemaNullReturnDefault::test_custom_schema_null_return_legacy": 7.017101982999975, "tests/functional/custom_schemas/test_custom_schemas.py::TestCustomSchemaNullReturnLegacy::test_custom_schema_null_return_legacy": 6.840425341000014, "tests/functional/custom_singular_tests/test_custom_singular_tests.py::TestFailingTests::test_data_tests": 12.523478044, "tests/functional/custom_singular_tests/test_custom_singular_tests.py::TestPassingTests::test_data_tests": 1.9780239450001318, "tests/functional/custom_target_path/test_custom_target_path.py::TestTargetPathCliArg::test_target_path": 0.9024194590000008, "tests/functional/custom_target_path/test_custom_target_path.py::TestTargetPathConfig::test_target_path": 7.930154767000033, "tests/functional/custom_target_path/test_custom_target_path.py::TestTargetPathEnvVar::test_target_path": 0.700890194999829, "tests/functional/cycles/test_cycles.py::TestComplexCycle::test_complex_cycle": 8.540429685999925, "tests/functional/cycles/test_cycles.py::TestSimpleCycle::test_simple_cycle": 0.7245702900000879, "tests/functional/data_test_patch/test_singular_test_patch.py::TestPatchSingularTest::test_compile": 13.815726229000006, "tests/functional/data_test_patch/test_singular_test_patch.py::TestPatchSingularTestInvalidName::test_compile": 9.17537908700001, "tests/functional/data_test_patch/test_singular_test_patch.py::TestPatchSingularTestMalformedYaml::test_compile": 9.368079006000016, "tests/functional/data_tests/test_hooks.py::TestSingularTestPostHook::test_data_test_runs_adapter_post_hook_fails": 16.043785289000084, "tests/functional/data_tests/test_hooks.py::TestSingularTestPostHook::test_data_test_runs_adapter_post_hook_pass": 21.66079876899994, "tests/functional/data_tests/test_hooks.py::TestSingularTestPreHook::test_data_test_runs_adapter_pre_hook_fails": 15.145069180000064, "tests/functional/data_tests/test_hooks.py::TestSingularTestPreHook::test_data_test_runs_adapter_pre_hook_pass": 2.5179512160000286, "tests/functional/dbt_runner/test_dbt_runner.py::TestDbtRunner::test_callback_node_finished_exceptions_are_raised": 11.009342442000161, "tests/functional/dbt_runner/test_dbt_runner.py::TestDbtRunner::test_callbacks": 0.22554276500000014, "tests/functional/dbt_runner/test_dbt_runner.py::TestDbtRunner::test_command_invalid_option": 0.004681253999990531, "tests/functional/dbt_runner/test_dbt_runner.py::TestDbtRunner::test_command_mutually_exclusive_option": 0.03389066999989154, "tests/functional/dbt_runner/test_dbt_runner.py::TestDbtRunner::test_directory_does_not_change": 0.10949237999989236, "tests/functional/dbt_runner/test_dbt_runner.py::TestDbtRunner::test_group_invalid_option": 0.006220846000132951, "tests/functional/dbt_runner/test_dbt_runner.py::TestDbtRunner::test_invalid_command": 0.003023223000042208, "tests/functional/dbt_runner/test_dbt_runner.py::TestDbtRunner::test_invoke_kwargs": 5.595669710999914, "tests/functional/dbt_runner/test_dbt_runner.py::TestDbtRunner::test_invoke_kwargs_and_flags": 5.953768584000045, "tests/functional/dbt_runner/test_dbt_runner.py::TestDbtRunner::test_invoke_kwargs_profiles_dir": 0.030260843000064597, "tests/functional/dbt_runner/test_dbt_runner.py::TestDbtRunner::test_invoke_kwargs_project_dir": 0.37493148299998325, "tests/functional/dbt_runner/test_dbt_runner.py::TestDbtRunner::test_invoke_version": 0.16937982699994336, "tests/functional/dbt_runner/test_dbt_runner.py::TestDbtRunner::test_pass_in_args_variable": 0.0048483960000567095, "tests/functional/dbt_runner/test_dbt_runner.py::TestDbtRunner::test_pass_in_manifest": 10.461992227999872, "tests/functional/dbt_runner/test_dbt_runner.py::TestDbtRunnerHooks::test_node_info_non_persistence": 12.267079878000004, "tests/functional/dbt_runner/test_dbt_runner.py::TestDbtRunnerQueryComments::test_query_comment_saved_manifest": 22.213837043000012, "tests/functional/defer_state/test_defer_state.py::TestDeferStateDeletedUpstream::test_run_defer_deleted_upstream": 61.85776184000008, "tests/functional/defer_state/test_defer_state.py::TestDeferStateFlag::test_defer_state_flag": 63.689854235999974, "tests/functional/defer_state/test_defer_state.py::TestDeferStateUnsupportedCommands::test_no_state": 12.90576068900009, "tests/functional/defer_state/test_defer_state.py::TestFunctionDeferral::test_build_with_other_schema_defer": 3.782861483000005, "tests/functional/defer_state/test_defer_state.py::TestRunCompileState::test_run_and_compile_defer": 33.11859059099993, "tests/functional/defer_state/test_defer_state.py::TestRunDeferState::test_run_and_defer": 8.286702338999817, "tests/functional/defer_state/test_defer_state.py::TestRunDeferStateChangedModel::test_run_defer_state_changed_model": 49.603901488999895, "tests/functional/defer_state/test_defer_state.py::TestRunDeferStateIFFNotExists::test_run_defer_iff_not_exists": 59.872256053000115, "tests/functional/defer_state/test_defer_state.py::TestSnapshotState::test_snapshot_state_defer": 44.659071827999924, "tests/functional/defer_state/test_group_updates.py::TestBadGroups::test_changed_groups": 34.54032022500019, "tests/functional/defer_state/test_group_updates.py::TestFullyModifiedGroups::test_changed_groups": 34.23673371799964, "tests/functional/defer_state/test_group_updates.py::TestPartiallyModifiedGroups::test_changed_groups": 33.77331539499983, "tests/functional/defer_state/test_modified_state.py::TestChangedConstraintUnversioned::test_changed_constraint": 80.60591570499992, "tests/functional/defer_state/test_modified_state.py::TestChangedContractUnversioned::test_changed_contract": 115.96679771699996, "tests/functional/defer_state/test_modified_state.py::TestChangedContractVersioned::test_changed_contract_versioned": 8.95739512700004, "tests/functional/defer_state/test_modified_state.py::TestChangedExposure::test_changed_exposure": 55.413003436000054, "tests/functional/defer_state/test_modified_state.py::TestChangedMacroContents::test_changed_macro_contents": 55.10674272599999, "tests/functional/defer_state/test_modified_state.py::TestChangedMaterializationConstraint::test_changed_materialization": 93.97664151799995, "tests/functional/defer_state/test_modified_state.py::TestChangedModelContents::test_changed_model_contents": 65.11376417299994, "tests/functional/defer_state/test_modified_state.py::TestChangedSeedConfig::test_changed_seed_config": 91.34282982499985, "tests/functional/defer_state/test_modified_state.py::TestChangedSeedContents::test_changed_seed_contents_state": 185.61396948100014, "tests/functional/defer_state/test_modified_state.py::TestChangedSemanticModelContents::test_changed_semantic_model_contents": 51.03010479099976, "tests/functional/defer_state/test_modified_state.py::TestDeleteUnversionedContractedModel::test_delete_unversioned_contracted_model": 60.54360014599979, "tests/functional/defer_state/test_modified_state.py::TestDeleteVersionedContractedModel::test_delete_versioned_contracted_model": 60.64387756100041, "tests/functional/defer_state/test_modified_state.py::TestDisableUnversionedContractedModel::test_disable_unversioned_contracted_model": 28.281863183999917, "tests/functional/defer_state/test_modified_state.py::TestDisableUnversionedUncontractedModel::test_delete_versioned_contracted_model": 29.64516934300002, "tests/functional/defer_state/test_modified_state.py::TestDisableVersionedContractedModel::test_disable_versioned_contracted_model": 29.084239192999917, "tests/functional/defer_state/test_modified_state.py::TestDisableVersionedUncontractedModel::test_delete_versioned_contracted_model": 4.162558392000022, "tests/functional/defer_state/test_modified_state.py::TestModifiedAccess::test_changed_access": 54.52696797900012, "tests/functional/defer_state/test_modified_state.py::TestModifiedBodyAndContract::test_modified_body_and_contract": 52.127014975000066, "tests/functional/defer_state/test_modified_state.py::TestModifiedDeprecationDate::test_changed_access": 56.557062286000246, "tests/functional/defer_state/test_modified_state.py::TestModifiedLatestVersion::test_changed_access": 41.55704616800017, "tests/functional/defer_state/test_modified_state.py::TestModifiedVersion::test_changed_access": 40.10616760700009, "tests/functional/defer_state/test_modified_state.py::TestNewMacro::test_new_macro": 63.77843377400018, "tests/functional/defer_state/test_modified_state.py::TestUnrenderedConfigSame::test_unrendered_config_same": 5.892506080000089, "tests/functional/defer_state/test_modified_state.py::TestUnversionedContractVarcharSizeChange::test_varchar_size_increase_not_breaking_unversioned": 43.36008621099995, "tests/functional/defer_state/test_modified_state.py::TestVersionedContractVarcharSizeChange::test_numeric_precision_increase_not_breaking": 33.203787809999994, "tests/functional/defer_state/test_modified_state.py::TestVersionedContractVarcharSizeChange::test_varchar_case_sensitivity_not_breaking": 31.644214640999962, "tests/functional/defer_state/test_modified_state.py::TestVersionedContractVarcharSizeChange::test_varchar_size_increase_not_breaking": 42.405480871000236, "tests/functional/defer_state/test_modified_state_environment_vars.py::TestModelNodeWithEnvVarConfigInProjectYml::test_change_env_var": 43.25435520199994, "tests/functional/defer_state/test_modified_state_environment_vars.py::TestModelNodeWithEnvVarConfigInProjectYmlAndSchemaYml::test_change_env_var": 3.3312798589999915, "tests/functional/defer_state/test_modified_state_environment_vars.py::TestModelNodeWithEnvVarConfigInSchemaYml::test_change_env_var": 43.20262111700026, "tests/functional/defer_state/test_modified_state_environment_vars.py::TestModelNodeWithEnvVarConfigInSqlAndSchemaYml::test_change_env_var": 44.2378646950001, "tests/functional/defer_state/test_modified_state_environment_vars.py::TestModelNodeWithEnvVarConfigInSqlFile::test_change_env_var": 42.95366186099977, "tests/functional/defer_state/test_modified_state_jinja.py::TestModelNodeWithEnvVarConfigInSchemaYml::test_change_jinja_if": 46.20371971999998, "tests/functional/defer_state/test_modified_state_jinja.py::TestModelNodeWithJinjaConfigInProjectYml::test_change_jinja_if": 46.04059499899995, "tests/functional/defer_state/test_modified_state_jinja.py::TestModelNodeWithJinjaConfigInProjectYmlAndSchemaYml::test_change_jinja_if": 48.406280199999856, "tests/functional/defer_state/test_modified_state_jinja.py::TestModelNodeWithJinjaConfigInSqlAndSchemaYml::test_change_jinja_if": 48.053867512000124, "tests/functional/defer_state/test_modified_state_jinja.py::TestModelNodeWithJinjaConfigInSqlFile::test_change_jinja_if": 44.85347691200013, "tests/functional/defer_state/test_modified_state_schema_evolution.py::TestModifiedStateSchemaEvolution::test_modified_state_schema_evolution": 26.930958640000426, "tests/functional/defer_state/test_modified_state_sources_unrendered.py::TestSourceNodeWithEnvVarConfigInDatabase::test_change_env_var": 48.514698134999435, "tests/functional/defer_state/test_modified_state_sources_unrendered.py::TestSourceNodeWithEnvVarConfigInSchema::test_change_env_var": 61.18177458199989, "tests/functional/defer_state/test_modified_state_sources_unrendered.py::TestSourceNodeWithJinjaInDatabase::test_change_jinja_if": 49.90231877100041, "tests/functional/defer_state/test_modified_state_sources_unrendered.py::TestSourceNodeWithJinjaInSchema::test_change_jinja_if": 3.6301306659999, "tests/functional/defer_state/test_modified_state_vars.py::TestStateSelectionVarConfig::test_change_var": 52.277848013000494, "tests/functional/defer_state/test_modified_state_vars.py::TestStateSelectionVarConfigLegacy::test_change_var": 51.33541602200012, "tests/functional/defer_state/test_removed_test_state.py::TestRemovedGenericTest::test_removed_generic_test_with_state_modified": 39.73983506500008, "tests/functional/defer_state/test_removed_test_state.py::TestRemovedGenericTestStateModifiedGracefulError::test_list_with_state_modified_after_test_removal": 39.7702012460004, "tests/functional/defer_state/test_run_results_state.py::TestBuildRunResultsState::test_build_run_results_state": 258.3166066490003, "tests/functional/defer_state/test_run_results_state.py::TestConcurrentSelectionBuildRunResultsState::test_concurrent_selectors_build_run_results_state": 113.7115534909999, "tests/functional/defer_state/test_run_results_state.py::TestConcurrentSelectionRunResultsState::test_concurrent_selection_run_run_results_state": 78.34525452200023, "tests/functional/defer_state/test_run_results_state.py::TestConcurrentSelectionTestRunResultsState::test_concurrent_selection_test_run_results_state": 63.871015166000234, "tests/functional/defer_state/test_run_results_state.py::TestRunRunResultsState::test_run_run_results_state": 15.572167158999946, "tests/functional/defer_state/test_run_results_state.py::TestSeedRunResultsState::test_seed_run_results_state": 135.52150492200008, "tests/functional/defer_state/test_run_results_state.py::TestTestRunResultsState::test_test_run_results_state": 157.52328988399995, "tests/functional/defer_state/test_unrendered_config.py::TestGenericTestUnrenderedConfig::test_unrendered_config": 32.03120717100046, "tests/functional/dependencies/test_add_package_edge_cases.py::TestAddPackageWithWarnUnpinnedInYaml::test_add_package_with_warn_unpinned_in_yaml": 3.0999666530000525, "tests/functional/dependencies/test_dependency_inverted_ref.py::TestInvertedRefDependency::test_inverted_ref_dependency": 32.85723158400015, "tests/functional/dependencies/test_dependency_inverted_ref.py::TestInvertedRefDependencyLegacy::test_inverted_ref_dependency": 1.0653438009999832, "tests/functional/dependencies/test_dependency_options.py::TestDepsOptions::test_deps_add": 1.596422259999997, "tests/functional/dependencies/test_dependency_options.py::TestDepsOptions::test_deps_add_without_install": 0.30580801100040844, "tests/functional/dependencies/test_dependency_options.py::TestDepsOptions::test_deps_default": 0.5980374340001617, "tests/functional/dependencies/test_dependency_options.py::TestDepsOptions::test_deps_lock": 16.19210886199926, "tests/functional/dependencies/test_dependency_options.py::TestDepsOptions::test_deps_upgrade": 0.8616846250000094, "tests/functional/dependencies/test_dependency_secrets.py::TestSecretInPackage::test_mask_secrets": 16.269470662000003, "tests/functional/dependencies/test_local_dependency.py::TestDependencyTestsConfig::test_dependency_tests_config": 36.580165950000264, "tests/functional/dependencies/test_local_dependency.py::TestMissingDependency::test_missing_dependency": 35.868315869999606, "tests/functional/dependencies/test_local_dependency.py::TestSimpleDependency::test_local_dependency": 52.54044688499971, "tests/functional/dependencies/test_local_dependency.py::TestSimpleDependency::test_no_dependency_paths": 53.56587850400001, "tests/functional/dependencies/test_local_dependency.py::TestSimpleDependencyDuplicateName::test_local_dependency_same_name": 0.29325907099999426, "tests/functional/dependencies/test_local_dependency.py::TestSimpleDependencyDuplicateName::test_local_dependency_same_name_sneaky": 17.811623147999853, "tests/functional/dependencies/test_local_dependency.py::TestSimpleDependencyHooks::test_hook_dependency": 35.63815417500018, "tests/functional/dependencies/test_local_dependency.py::TestSimpleDependencyNoVersionCheckConfig::test_local_dependency_out_of_date_no_check": 3.0720469819999607, "tests/functional/dependencies/test_local_dependency.py::TestSimpleDependencyRelativePath::test_local_dependency_relative_path": 17.346668812000644, "tests/functional/dependencies/test_local_dependency.py::TestSimpleDependencyWithSchema::test_local_dependency_out_of_date": 55.8938258029998, "tests/functional/dependencies/test_local_dependency.py::TestSimpleDependencyWithSchema::test_local_dependency_out_of_date_no_check": 36.57688531800022, "tests/functional/dependencies/test_simple_dependency.py::TestBadTarballDependency::test_malformed_tarball_package_causes_exception": 8.129406027999948, "tests/functional/dependencies/test_simple_dependency.py::TestEmptyDependency::test_empty_package": 8.437200644000086, "tests/functional/dependencies/test_simple_dependency.py::TestRekeyedDependencyWithSubduplicates::test_simple_dependency_deps": 3.5478885310000123, "tests/functional/dependencies/test_simple_dependency.py::TestSimpleDependcyTarball::test_deps_simple_tarball_doesnt_error_out": 9.077035645000024, "tests/functional/dependencies/test_simple_dependency.py::TestSimpleDependency::test_simple_dependency": 59.325854435999645, "tests/functional/dependencies/test_simple_dependency.py::TestSimpleDependencyBadProfile::test_deps_bad_profile": 7.960859184000128, "tests/functional/dependencies/test_simple_dependency.py::TestSimpleDependencyBranch::test_simple_dependency": 28.913091971000085, "tests/functional/dependencies/test_simple_dependency.py::TestSimpleDependencyBranchWithEmpty::test_empty_models_not_compiled_in_dependencies": 3.136219147999995, "tests/functional/dependencies/test_simple_dependency.py::TestSimpleDependencyNoProfile::test_simple_dependency_no_profile": 19.026055459999952, "tests/functional/dependencies/test_simple_dependency.py::TestSimpleDependencyUnpinned::test_simple_dependency": 20.250789844999417, "tests/functional/dependencies/test_simple_dependency.py::TestSimpleDependencyWithDependenciesFile::test_dependency_with_dependencies_file": 40.090405611998904, "tests/functional/dependencies/test_simple_dependency.py::TestSimpleDependencyWithDuplicates::test_simple_dependency_deps": 19.765875034999226, "tests/functional/dependencies/test_simple_dependency.py::TestSimpleDependencyWithEmptyPackagesFile::test_dependency_with_empty_packages_file": 18.5043594760009, "tests/functional/dependencies/test_simple_dependency.py::TestSimpleDependencyWithModels::test_simple_dependency_with_models": 39.69027267199999, "tests/functional/dependencies/test_simple_dependency.py::TestSimpleDependencyWithSubdirs::test_git_with_multiple_subdir": 4.301544702999962, "tests/functional/dependencies/test_simple_dependency.py::TestTarballNestedDependencies::test_simple_dependency_deps": 9.450593745999868, "tests/functional/dependencies/test_simple_dependency_with_configs.py::TestSimpleDependencyWithConfigs::test_simple_dependency": 19.92994181600011, "tests/functional/dependencies/test_simple_dependency_with_configs.py::TestSimpleDependencyWithOverriddenConfigs::test_simple_dependency": 20.10618874900001, "tests/functional/dependencies/test_uninstalled_package_found_error.py::TestNoErrorIfPackageYamlDoesNotExist::test_no_error_raised_if_package_yml_does_not_exist": 18.524951326000064, "tests/functional/dependencies/test_uninstalled_package_found_error.py::TestUninstalledPackageWithNestedDependency::test_uninstalled_package_with_nested_dependency": 18.525517385000057, "tests/functional/dependencies/test_uninstalled_package_found_error.py::TestUninstalledPackagesErrorRaisedIfPackageLockDoesNotExist::test_error_raised_if_package_lock_does_not_exist": 18.072258011000258, "tests/functional/deprecations/test_config_deprecations.py::TestBothProjectTestDeprecation::test_tests_config": 0.32122242499994513, "tests/functional/deprecations/test_config_deprecations.py::TestBothSchemaTestDeprecation::test_schema": 20.38593206399969, "tests/functional/deprecations/test_config_deprecations.py::TestSchemaTestDeprecation::test_generic_data_test_parsing": 9.823229470000115, "tests/functional/deprecations/test_config_deprecations.py::TestSchemaTestDeprecation::test_generic_tests_config": 19.400463967999713, "tests/functional/deprecations/test_config_deprecations.py::TestSchemaTestDeprecation::test_generic_tests_fail": 1.1713902639999105, "tests/functional/deprecations/test_config_deprecations.py::TestSourceSchemaTestDeprecation::test_generic_data_tests": 21.072828041000093, "tests/functional/deprecations/test_config_deprecations.py::TestSourceSchemaTestDeprecation::test_source_tests_config": 20.13436093900009, "tests/functional/deprecations/test_config_deprecations.py::TestTestConfigInDependency::test_test_dep": 2.966434448999962, "tests/functional/deprecations/test_config_deprecations.py::TestTestsConfigDeprecation::test_project_tests_config": 5.444153101000097, "tests/functional/deprecations/test_config_deprecations.py::TestTestsConfigDeprecation::test_project_tests_config_fail": 9.782367615000112, "tests/functional/deprecations/test_config_deprecations.py::TestValidateModelConfigOnlyCalledOncePerModel::test_validate_model_config_only_called_once_per_model": 20.86074519900035, "tests/functional/deprecations/test_deprecations.py::TestArgumentsPropertyInGenericTestDeprecationBehaviorChangeDefault::test_arguments_property_in_generic_test_deprecation": 1.1635303539999882, "tests/functional/deprecations/test_deprecations.py::TestArgumentsPropertyInGenericTestDeprecationFalse::test_arguments_property_in_generic_test_deprecation": 1.235795113999984, "tests/functional/deprecations/test_deprecations.py::TestArgumentsPropertyInGenericTestDeprecationTrue::test_arguments_property_in_generic_test_deprecation": 21.688444249000213, "tests/functional/deprecations/test_deprecations.py::TestBaseProjectHasNoDeprecations::test_base_project_has_no_deprecations": 22.609471571000086, "tests/functional/deprecations/test_deprecations.py::TestConfigPathDeprecation::test_data_path": 19.1752317280002, "tests/functional/deprecations/test_deprecations.py::TestConfigPathDeprecation::test_data_path_fail": 0.055538300999614876, "tests/functional/deprecations/test_deprecations.py::TestCustomConfigInDbtProjectYmlNoDeprecation::test_missing_plus_prefix_deprecation_sub_path": 20.275871224999946, "tests/functional/deprecations/test_deprecations.py::TestCustomKeyInConfigComplexSQLDeprecation::test_custom_key_in_config_sql_deprecation": 21.364621893999583, "tests/functional/deprecations/test_deprecations.py::TestCustomKeyInConfigComplexSQLDeprecation::test_custom_key_in_config_sql_deprecation_adapter_specific_config_key_aliases": 11.324098366999806, "tests/functional/deprecations/test_deprecations.py::TestCustomKeyInConfigDeprecation::test_custom_key_in_config_deprecation": 21.478977203000113, "tests/functional/deprecations/test_deprecations.py::TestCustomKeyInConfigSQLDeprecation::test_custom_key_in_config_sql_deprecation": 21.542894689000377, "tests/functional/deprecations/test_deprecations.py::TestCustomKeyInConfigSQLDeprecation::test_custom_key_in_config_sql_deprecation_adapter_specific_config_key_aliases": 11.379860011000346, "tests/functional/deprecations/test_deprecations.py::TestCustomKeyInObjectDeprecation::test_custom_key_in_object_deprecation": 22.23738819099981, "tests/functional/deprecations/test_deprecations.py::TestCustomOutputPathInSourceFreshnessDeprecation::test_jsonschema_validation_deprecations_arent_run_without_env_var": 22.433747362000076, "tests/functional/deprecations/test_deprecations.py::TestDeprecatedInvalidDeprecationDate::test_deprecated_invalid_deprecation_date": 20.658429606000254, "tests/functional/deprecations/test_deprecations.py::TestDeprecatedModelExposure::test_exposure_with_deprecated_model": 1.090100322999973, "tests/functional/deprecations/test_deprecations.py::TestDeprecationSummary::test_package_redirect": 10.843696051999814, "tests/functional/deprecations/test_deprecations.py::TestDuplicateYAMLKeysInSchemaFiles::test_duplicate_yaml_keys_in_schema_files": 21.06595220400027, "tests/functional/deprecations/test_deprecations.py::TestEnvironmentVariableNamespaceDeprecation::test_environment_variable_namespace_deprecation": 20.24838188400031, "tests/functional/deprecations/test_deprecations.py::TestExposureNameDeprecation::test_exposure_name": 21.18804540600013, "tests/functional/deprecations/test_deprecations.py::TestExposureNameDeprecation::test_exposure_name_fail": 10.275663852999969, "tests/functional/deprecations/test_deprecations.py::TestGenerateSchemaNameNullValueDeprecation::test_generate_schema_name_null_value_deprecation": 21.084296618999815, "tests/functional/deprecations/test_deprecations.py::TestHappyPathProjectHasNoDeprecations::test_happy_path_project_has_no_deprecations": 3.7107665290000114, "tests/functional/deprecations/test_deprecations.py::TestJsonSchemaValidationGating::test_jsonschema_validation_gating[False-False-00]": 10.823479230999965, "tests/functional/deprecations/test_deprecations.py::TestJsonSchemaValidationGating::test_jsonschema_validation_gating[False-False-01]": 10.384692801000028, "tests/functional/deprecations/test_deprecations.py::TestJsonSchemaValidationGating::test_jsonschema_validation_gating[False-True-0]": 10.804687088000037, "tests/functional/deprecations/test_deprecations.py::TestJsonSchemaValidationGating::test_jsonschema_validation_gating[True-True-1]": 20.346035234000055, "tests/functional/deprecations/test_deprecations.py::TestJsonschemaValidationDeprecationsArentRunWithoutEnvVar::test_jsonschema_validation_deprecations_arent_run_without_env_var": 22.357440401000076, "tests/functional/deprecations/test_deprecations.py::TestMParamUsageDeprecation::test_m_usage": 19.7771351209999, "tests/functional/deprecations/test_deprecations.py::TestMParamUsageRunnerDeprecation::test_m_usage": 20.082165067999767, "tests/functional/deprecations/test_deprecations.py::TestMissingArgumentsPropertyInGenericTestDeprecation::test_missing_arguments_property_in_generic_test_deprecation": 22.313889760999928, "tests/functional/deprecations/test_deprecations.py::TestModelParamUsageDeprecation::test_model_usage": 24.280937054000333, "tests/functional/deprecations/test_deprecations.py::TestModelParamUsageRunnerDeprecation::test_model_usage": 23.685321349000333, "tests/functional/deprecations/test_deprecations.py::TestModelsParamUsageDeprecation::test_models_usage": 23.59989595800016, "tests/functional/deprecations/test_deprecations.py::TestModelsParamUsageRunnerDeprecation::test_models_usage": 23.992864603999806, "tests/functional/deprecations/test_deprecations.py::TestModulesItertoolsDeprecation::test_models_itertools": 23.83317090100013, "tests/functional/deprecations/test_deprecations.py::TestMultipleCustomKeysInConfigDeprecation::test_multiple_custom_keys_in_config_deprecation": 22.03165513900035, "tests/functional/deprecations/test_deprecations.py::TestNoModulesItertoolsDeprecation::test_models_itertools": 24.04908481500047, "tests/functional/deprecations/test_deprecations.py::TestPackageInstallPathDeprecation::test_package_path": 9.98883508899985, "tests/functional/deprecations/test_deprecations.py::TestPackageInstallPathDeprecation::test_package_path_not_set": 0.04274555200004215, "tests/functional/deprecations/test_deprecations.py::TestPackageRedirectDeprecation::test_package_redirect": 10.226628003000087, "tests/functional/deprecations/test_deprecations.py::TestPackageRedirectDeprecation::test_package_redirect_fail": 0.6062213699999859, "tests/functional/deprecations/test_deprecations.py::TestPrePostHookNoFalsePositiveDeprecation::test_pre_post_hook_no_false_positive_deprecation": 21.38337906300012, "tests/functional/deprecations/test_deprecations.py::TestProjectFlagsMovedDeprecation::test_profile_config_deprecation": 22.26252905499996, "tests/functional/deprecations/test_deprecations.py::TestProjectFlagsMovedDeprecationQuiet::test_profile_config_deprecation": 21.599223906999896, "tests/functional/deprecations/test_deprecations.py::TestProjectFlagsMovedDeprecationWarnErrorOptions::test_profile_config_deprecation": 1.1091793080000798, "tests/functional/deprecations/test_deprecations.py::TestPropertyMovedToConfigDeprecation::test_property_moved_to_config_deprecation": 1.6897038079999902, "tests/functional/deprecations/test_deprecations.py::TestPythonModelConfigAdditionsDontRaiseDeprecations::test_python_model_config_additions_dont_raise_deprecations": 21.242180384999983, "tests/functional/deprecations/test_deprecations.py::TestSelectParamNoModelUsageDeprecation::test_select_usage": 20.4514029139998, "tests/functional/deprecations/test_deprecations.py::TestSelectParamNoModelUsageRunnerDeprecation::test_select_usage": 20.375715364999678, "tests/functional/deprecations/test_deprecations.py::TestShowAllDeprecationsFlag::test_package_redirect": 1.4262878629999989, "tests/functional/deprecations/test_deprecations.py::TestWEOIncludeExcludeDeprecation::test_weo_include_exclude_deprecation[error-exclude-1]": 1.2320054940000205, "tests/functional/deprecations/test_deprecations.py::TestWEOIncludeExcludeDeprecation::test_weo_include_exclude_deprecation[error-warn-0]": 11.690115804000243, "tests/functional/deprecations/test_deprecations.py::TestWEOIncludeExcludeDeprecation::test_weo_include_exclude_deprecation[include-exclude-1]": 22.840482014999907, "tests/functional/deprecations/test_deprecations.py::TestWEOIncludeExcludeDeprecation::test_weo_include_exclude_deprecation[include-warn-1]": 11.459487368000055, "tests/functional/deprecations/test_missing_plus_in_config_deprecations.py::TestEmptyConfig::test_no_warning": 21.21550332700008, "tests/functional/deprecations/test_missing_plus_in_config_deprecations.py::TestEmptyNestedDir::test_no_warning": 0.8941187810000315, "tests/functional/deprecations/test_missing_plus_in_config_deprecations.py::TestMissingPlusPrefixDeprecation::test_missing_plus_prefix_deprecation": 1.0505578570000011, "tests/functional/deprecations/test_missing_plus_in_config_deprecations.py::TestMissingPlusPrefixDeprecationSubPath::test_missing_plus_prefix_deprecation_sub_path": 22.438120303999995, "tests/functional/deprecations/test_missing_plus_in_config_deprecations.py::TestValidConfigKey::test_raises_warning": 21.765928362000068, "tests/functional/deprecations/test_missing_plus_in_config_deprecations.py::TestValidConfigKeyWithNonPlusPrefixInNestedDir::test_raises_warning": 22.186889668000276, "tests/functional/deprecations/test_missing_plus_in_config_deprecations.py::TestValidPlusPrefixConfigKeyWithNonPlusPrefixProperty::test_no_warning": 21.55176104899988, "tests/functional/deprecations/test_missing_plus_in_config_deprecations.py::TestValidPlusPrefixConfigKeyWithPlusPrefixInNestedDir::test_no_warning": 21.814113762999796, "tests/functional/deprecations/test_model_deprecations.py::TestDeprecatedReferenceWarning::test_deprecation_warning": 0.7212648379999962, "tests/functional/deprecations/test_model_deprecations.py::TestDeprecatedReferenceWarning::test_deprecation_warning_error": 11.380211667999902, "tests/functional/deprecations/test_model_deprecations.py::TestDeprecatedReferenceWarning::test_deprecation_warning_error_options": 11.703597500000114, "tests/functional/deprecations/test_model_deprecations.py::TestModelDeprecationWarning::test_deprecation_warning": 1.5684938809999949, "tests/functional/deprecations/test_model_deprecations.py::TestModelDeprecationWarning::test_deprecation_warning_error": 11.089154958000108, "tests/functional/deprecations/test_model_deprecations.py::TestModelDeprecationWarning::test_deprecation_warning_error_options": 11.218567669999857, "tests/functional/deprecations/test_model_deprecations.py::TestUpcomingReferenceDeprecationWarning::test_deprecation_warning": 1.0087990290000164, "tests/functional/deprecations/test_model_deprecations.py::TestUpcomingReferenceDeprecationWarning::test_deprecation_warning_error": 11.13699042899998, "tests/functional/deprecations/test_model_deprecations.py::TestUpcomingReferenceDeprecationWarning::test_deprecation_warning_error_options": 1.203127640000048, "tests/functional/deps/test_deps_with_vars.py::TestBuildFailsWithMissingVar::test_build_fails_with_error": 0.4349237669999866, "tests/functional/deps/test_deps_with_vars.py::TestCompileFailsWithMissingVar::test_compile_fails_with_error": 11.764884287000768, "tests/functional/deps/test_deps_with_vars.py::TestDebugFailsWithMissingVar::test_debug_fails_with_error": 23.543105118000767, "tests/functional/deps/test_deps_with_vars.py::TestDebugSucceedsWithVarDefaults::test_debug_succeeds": 23.202864000999853, "tests/functional/deps/test_deps_with_vars.py::TestDepsSucceedsEvenWhenVarMissing::test_deps_still_succeeds": 11.847928248999779, "tests/functional/deps/test_deps_with_vars.py::TestDepsSucceedsWithVarDefaults::test_deps_succeeds": 11.50544181400005, "tests/functional/deps/test_deps_with_vars.py::TestRunFailsWithMissingVar::test_run_fails_with_error": 11.541906229999768, "tests/functional/deps/test_deps_with_vars.py::TestRunSucceedsWithExplicitVars::test_run_succeeds_with_vars": 24.46075694800038, "tests/functional/deps/test_deps_with_vars.py::TestRunSucceedsWithVarDefaults::test_run_succeeds": 1.3233242549999886, "tests/functional/docs/test_doc_blocks_backcompat.py::TestDocBlocksBackCompat::test_doc_blocks_back_compat": 24.63040471300019, "tests/functional/docs/test_doc_blocks_formatting.py::TestDocBlocksBackCompat::test_doc_blocks_back_compat": 24.50814994400025, "tests/functional/docs/test_duplicate_docs_block.py::TestDuplicateDocsBlock::test_duplicate_doc_ref": 25.39352328399991, "tests/functional/docs/test_generate.py::TestGenerateCatalogWithExternalNodes::test_catalog_with_external_node": 40.900356811000165, "tests/functional/docs/test_generate.py::TestGenerateCatalogWithSources::test_catalog_with_sources": 39.81236420900041, "tests/functional/docs/test_generate.py::TestGenerateEmptyCatalog::test_generate_empty_catalog": 25.63579633800009, "tests/functional/docs/test_generate.py::TestGenerateManifestNotCompiled::test_manifest_not_compiled": 25.082202055999915, "tests/functional/docs/test_generate.py::TestGenerateSelectLimitsCatalog::test_select_limits_catalog": 38.77038824200008, "tests/functional/docs/test_generate.py::TestGenerateSelectLimitsNoMatch::test_select_limits_no_match": 39.3574689080001, "tests/functional/docs/test_generate.py::TestGenerateSelectOverMaxSchemaMetadataRelations::test_select_source": 96.78564224399997, "tests/functional/docs/test_generate.py::TestGenerateSelectSeed::test_select_seed": 57.05880871699992, "tests/functional/docs/test_generate.py::TestGenerateSelectSource::test_select_source": 53.704723122999894, "tests/functional/docs/test_good_docs_blocks.py::TestGoodDocsBlocks::test_valid_doc_ref": 28.075528919000135, "tests/functional/docs/test_good_docs_blocks.py::TestGoodDocsBlocksAltPath::test_alternative_docs_path": 28.326438081000106, "tests/functional/docs/test_invalid_doc_ref.py::TestInvalidDocRef::test_invalid_doc_ref": 27.698858652000126, "tests/functional/docs/test_missing_docs_blocks.py::TestMissingDocsBlocks::test_missing_doc_ref": 28.352783917000124, "tests/functional/docs/test_model_version_docs_blocks.py::TestVersionedModelDocsBlock::test_versioned_doc_ref": 0.7920837240000225, "tests/functional/docs/test_static.py::TestStaticGenerate::test_static_generated": 28.918235899000138, "tests/functional/duplicates/test_duplicate_analysis.py::TestDuplicateAnalysis::test_duplicate_model_enabled": 28.68548046199976, "tests/functional/duplicates/test_duplicate_exposure.py::TestDuplicateExposure::test_duplicate_exposure": 1.4647547040000006, "tests/functional/duplicates/test_duplicate_macro.py::TestDuplicateMacroEnabledDifferentFiles::test_duplicate_macros": 28.646241817000373, "tests/functional/duplicates/test_duplicate_macro.py::TestDuplicateMacroEnabledSameFile::test_duplicate_macros": 28.10583616699978, "tests/functional/duplicates/test_duplicate_metric.py::TestDuplicateMetric::test_duplicate_metric": 29.601348011000027, "tests/functional/duplicates/test_duplicate_model.py::TestDuplicateModelAliasEnabledAcrossPackages::test_duplicate_model_alias_enabled_across_packages": 0.9249040110000237, "tests/functional/duplicates/test_duplicate_model.py::TestDuplicateModelDisabled::test_duplicate_model_disabled": 1.238120804999994, "tests/functional/duplicates/test_duplicate_model.py::TestDuplicateModelDisabled::test_duplicate_model_disabled_partial_parsing": 38.269435807000264, "tests/functional/duplicates/test_duplicate_model.py::TestDuplicateModelDisabledAcrossPackages::test_duplicate_model_disabled_across_packages": 25.5281680209996, "tests/functional/duplicates/test_duplicate_model.py::TestDuplicateModelEnabled::test_duplicate_model_enabled": 23.87504331199989, "tests/functional/duplicates/test_duplicate_model.py::TestDuplicateModelNameWithTestAcrossPackages::test_duplicate_model_name_with_test_across_packages": 25.296530750999864, "tests/functional/duplicates/test_duplicate_model.py::TestDuplicateModelNameWithVersionAcrossPackages::test_duplicate_model_name_with_test_across_packages": 25.416415819999656, "tests/functional/duplicates/test_duplicate_model.py::TestModelTestOverlap::test_duplicate_test_model_paths": 51.639125177000096, "tests/functional/duplicates/test_duplicate_model.py::TestMultipleDisabledModels::test_multiple_disabled_models": 1.3385007279999854, "tests/functional/duplicates/test_duplicate_resource.py::TestDuplicateSchemaResource::test_duplicate_model_and_exposure": 26.352396291000332, "tests/functional/duplicates/test_duplicate_resource_names.py::TestDuplicateNamesDefaultBehavior::test_duplicate_names_with_flag_disabled": 26.858394729999873, "tests/functional/duplicates/test_duplicate_resource_names.py::TestDuplicateNamesDifferentResourceTypesVersionedUnversioned::test_duplicate_names_versioned_unversioned": 26.570472865999818, "tests/functional/duplicates/test_duplicate_resource_names.py::TestDuplicateNamesRequireUniqueResourceNamesFalse::test_duplicate_names_with_flag_disabled": 26.496836588000406, "tests/functional/duplicates/test_duplicate_resource_names.py::TestDuplicateNamesRequireUniqueResourceNamesTrue::test_duplicate_names_with_flag_enabled": 25.893892814000083, "tests/functional/duplicates/test_duplicate_resource_names.py::TestDuplicateNamesRequireUniqueResourceNamesTrueDifferentPackages::test_duplicate_names_with_flag_enabled_different_packages": 26.251696481999716, "tests/functional/duplicates/test_duplicate_source.py::TestDuplicateSourceEnabled::test_duplicate_source_enabled": 27.157224704999862, "tests/functional/exit_codes/test_exit_codes.py::TestExitCodes::test_compile": 14.081638306999594, "tests/functional/exit_codes/test_exit_codes.py::TestExitCodes::test_exit_code_run_fail": 1.974118763999968, "tests/functional/exit_codes/test_exit_codes.py::TestExitCodes::test_exit_code_run_succeed": 1.7424754859999894, "tests/functional/exit_codes/test_exit_codes.py::TestExitCodes::test_schema_test_fail": 28.18309098499958, "tests/functional/exit_codes/test_exit_codes.py::TestExitCodes::test_schema_test_pass": 27.7471505780004, "tests/functional/exit_codes/test_exit_codes.py::TestExitCodes::test_snapshot_pass": 28.559106527999575, "tests/functional/exit_codes/test_exit_codes.py::TestExitCodesDeps::test_deps": 14.88137974700021, "tests/functional/exit_codes/test_exit_codes.py::TestExitCodesDepsFail::test_deps_fail": 14.362762309000118, "tests/functional/exit_codes/test_exit_codes.py::TestExitCodesSeed::test_seed": 1.3676858219999986, "tests/functional/exit_codes/test_exit_codes.py::TestExitCodesSeedFail::test_seed": 29.01156851700034, "tests/functional/exit_codes/test_exit_codes.py::TestExitCodesSnapshotFail::test_snapshot_fail": 43.60882749300026, "tests/functional/experimental_parser/test_all_experimental_parser.py::TestBasicExperimentalParser::test_experimental_parser_basic": 28.981400720000693, "tests/functional/experimental_parser/test_all_experimental_parser.py::TestBasicNoStaticParser::test_static_parser_is_disabled": 29.35429184700024, "tests/functional/experimental_parser/test_all_experimental_parser.py::TestBasicStaticParser::test_static_parser_basic": 29.274852511999597, "tests/functional/exposures/test_exposure_configs.py::TestConfigYamlLevel::test_exposure_config_yaml_level": 29.964520381000057, "tests/functional/exposures/test_exposure_configs.py::TestExposureConfigsInheritence::test_exposure_all_configs": 29.824192048999976, "tests/functional/exposures/test_exposure_configs.py::TestExposureEnabledConfigProjectLevel::test_enabled_exposure_config_dbt_project": 46.25001147100011, "tests/functional/exposures/test_exposure_configs.py::TestInvalidConfig::test_exposure_config_yaml_level": 29.813378384000316, "tests/functional/exposures/test_exposures.py::TestBasicExposures::test_compilation_depends_on": 2.0448871080000117, "tests/functional/exposures/test_exposures.py::TestBasicExposures::test_compilation_names_with_spaces": 2.4895492529999927, "tests/functional/exposures/test_exposures.py::TestBasicExposures::test_execution_default": 16.112158016000194, "tests/functional/exposures/test_exposures.py::TestBasicExposures::test_execution_exclude": 15.807534360000318, "tests/functional/exposures/test_exposures.py::TestBasicExposures::test_execution_select": 1.8320979120001084, "tests/functional/external_reference/test_external_reference.py::TestExternalDependency::test_external_reference": 1.8692533880000042, "tests/functional/external_reference/test_external_reference.py::TestExternalReference::test_external_reference": 2.2516147510000053, "tests/functional/fail_fast/test_fail_fast_run.py::TestFailFastFromConfig::test_fail_fast_run_project_flags": 31.91725944700056, "tests/functional/fail_fast/test_fail_fast_run.py::TestFastFailingDuringRun::test_fail_fast_run": 1.162760456999976, "tests/functional/functions/test_udafs.py::TestBasicSQLUDAF::test_basic_sql_udaf_parsing": 31.75793714099973, "tests/functional/functions/test_udfs.py::TestBasicPythonUDF::test_basic_parsing": 65.13051018799979, "tests/functional/functions/test_udfs.py::TestBasicSQLUDF::test_basic_parsing": 2.4764980189999903, "tests/functional/functions/test_udfs.py::TestCanCallUDFInModel::test_can_call_udf_in_model": 49.52874270300026, "tests/functional/functions/test_udfs.py::TestCanConfigFunctionsFromProjectConfig::test_can_config_functions_from_project_config": 52.7539218779998, "tests/functional/functions/test_udfs.py::TestCanInlineShowUDF::test_can_inline_show_udf": 48.95041919799996, "tests/functional/functions/test_udfs.py::TestCanUseRefInUDF::test_can_use_ref_in_udf": 2.456829693000003, "tests/functional/functions/test_udfs.py::TestCanUseSourceInUDF::test_can_use_ref_in_udf": 74.14871335299995, "tests/functional/functions/test_udfs.py::TestCanUseWithEmptyMode::test_can_use_with_empty_model": 2.809738839000005, "tests/functional/functions/test_udfs.py::TestCreationOfUDFs::test_can_create_udf": 32.91761063800004, "tests/functional/functions/test_udfs.py::TestDefaultArgumentsBasic::test_udfs": 35.512634209000225, "tests/functional/functions/test_udfs.py::TestDefaultArgumentsMustComeLast::test_udfs": 1.2908535799999754, "tests/functional/functions/test_udfs.py::TestFunctionsGetSchemaCreatedIfNecessary::test_functions_get_schema_created_if_necessary": 1.9864689350000049, "tests/functional/functions/test_udfs.py::TestFunctionsIncludeAndExcludeByResourceType::test_udfs": 55.128863868999815, "tests/functional/functions/test_udfs.py::TestPythonFunctionPackagesViaJinjaConfig::test_packages_set_via_jinja_config": 1.2323887480000053, "tests/functional/functions/test_udfs.py::TestPythonFunctionPackagesViaProjectConfig::test_packages_set_via_project_config": 1.219787224000001, "tests/functional/functions/test_udfs.py::TestPythonFunctionPackagesViaYamlConfig::test_packages_set_via_yaml_config": 1.611244054999986, "tests/functional/functions/test_udfs.py::TestPythonFunctionWithJinjaHasCorrectCompiledCode::test_udfs": 34.40929981100044, "tests/functional/functions/test_udfs.py::TestPythonFunctionWithoutJinjaHasEquivalentRawCodeAndCompiledCode::test_udfs": 1.9365318119999984, "tests/functional/generic_test_description/test_generic_test_description.py::TestBuiltinGenericTestDescription::test_compile": 37.24881190300039, "tests/functional/graph_selection/test_graph_selection.py::TestGraphSelection::test_childrens_parents": 2.508250227000019, "tests/functional/graph_selection/test_graph_selection.py::TestGraphSelection::test_concat": 12.909825407999506, "tests/functional/graph_selection/test_graph_selection.py::TestGraphSelection::test_concat_exclude": 0.7179012160000013, "tests/functional/graph_selection/test_graph_selection.py::TestGraphSelection::test_concat_exclude_concat": 1.600413844000002, "tests/functional/graph_selection/test_graph_selection.py::TestGraphSelection::test_concat_exclude_multiple": 13.44328880500052, "tests/functional/graph_selection/test_graph_selection.py::TestGraphSelection::test_concat_multiple": 1.8450229519999652, "tests/functional/graph_selection/test_graph_selection.py::TestGraphSelection::test_exposure_parents": 2.189637500999993, "tests/functional/graph_selection/test_graph_selection.py::TestGraphSelection::test_group": 18.124883124999542, "tests/functional/graph_selection/test_graph_selection.py::TestGraphSelection::test_locally_qualified_name": 4.301605815999892, "tests/functional/graph_selection/test_graph_selection.py::TestGraphSelection::test_locally_qualified_name_model_with_dots": 25.781206155999826, "tests/functional/graph_selection/test_graph_selection.py::TestGraphSelection::test_more_childrens_parents": 25.596005766000417, "tests/functional/graph_selection/test_graph_selection.py::TestGraphSelection::test_specific_model": 37.92465308700048, "tests/functional/graph_selection/test_graph_selection.py::TestGraphSelection::test_specific_model_and_children": 19.502110650999384, "tests/functional/graph_selection/test_graph_selection.py::TestGraphSelection::test_specific_model_and_children_limited": 1.7650639989999775, "tests/functional/graph_selection/test_graph_selection.py::TestGraphSelection::test_specific_model_and_parents": 12.409017880000647, "tests/functional/graph_selection/test_graph_selection.py::TestGraphSelection::test_specific_model_and_parents_limited": 12.416455977000169, "tests/functional/graph_selection/test_graph_selection.py::TestGraphSelection::test_specific_model_with_exclusion": 12.445105512999817, "tests/functional/graph_selection/test_graph_selection.py::TestGraphSelection::test_tags": 19.622023160000936, "tests/functional/graph_selection/test_graph_selection.py::TestGraphSelection::test_tags_and_children": 19.35943232200043, "tests/functional/graph_selection/test_graph_selection.py::TestGraphSelection::test_tags_and_children_limited": 19.2460117920009, "tests/functional/graph_selection/test_graph_selection.py::TestListPathGraphSelection::test_list_select_with_project_dir": 26.28520174499954, "tests/functional/graph_selection/test_group_selection.py::TestGroupSelection::test_select_group_and_children": 13.167205011999613, "tests/functional/graph_selection/test_group_selection.py::TestGroupSelection::test_select_group_and_children_selector_str": 13.272448195999914, "tests/functional/graph_selection/test_group_selection.py::TestGroupSelection::test_select_group_selector_dict": 13.171816516000035, "tests/functional/graph_selection/test_group_selection.py::TestGroupSelection::test_select_group_selector_str": 13.228674113000125, "tests/functional/graph_selection/test_group_selection.py::TestGroupSelection::test_select_models_by_group": 1.361184542999922, "tests/functional/graph_selection/test_group_selection.py::TestGroupSelection::test_select_models_by_group_and_children": 13.299145571000281, "tests/functional/graph_selection/test_group_selection.py::TestGroupSelection::test_select_models_two_groups": 13.193709012, "tests/functional/graph_selection/test_inline.py::TestCompileInlineWithSelector::test_inline_selectors": 42.25843922499962, "tests/functional/graph_selection/test_intersection_syntax.py::TestIntersectionSyncs::test_intersection_concat": 14.995229978000225, "tests/functional/graph_selection/test_intersection_syntax.py::TestIntersectionSyncs::test_intersection_concat_exclude": 14.83617866700024, "tests/functional/graph_selection/test_intersection_syntax.py::TestIntersectionSyncs::test_intersection_concat_exclude_concat": 15.006434442999307, "tests/functional/graph_selection/test_intersection_syntax.py::TestIntersectionSyncs::test_intersection_concat_exclude_intersection_concat": 15.115295890000198, "tests/functional/graph_selection/test_intersection_syntax.py::TestIntersectionSyncs::test_intersection_concat_intersection": 14.510216172000128, "tests/functional/graph_selection/test_intersection_syntax.py::TestIntersectionSyncs::test_intersection_exclude_intersection": 14.356039661999603, "tests/functional/graph_selection/test_intersection_syntax.py::TestIntersectionSyncs::test_intersection_exclude_intersection_lack": 2.0308358759999976, "tests/functional/graph_selection/test_intersection_syntax.py::TestIntersectionSyncs::test_intersection_exclude_intersection_lack_selector": 14.71992816700049, "tests/functional/graph_selection/test_intersection_syntax.py::TestIntersectionSyncs::test_intersection_exclude_intersection_selectors": 14.627233063999483, "tests/functional/graph_selection/test_intersection_syntax.py::TestIntersectionSyncs::test_intersection_exclude_triple_intersection": 14.67519316500011, "tests/functional/graph_selection/test_intersection_syntax.py::TestIntersectionSyncs::test_intersection_triple_ascending": 14.598907127000075, "tests/functional/graph_selection/test_intersection_syntax.py::TestIntersectionSyncs::test_intersection_triple_ascending_schema_selectors": 14.968994184000167, "tests/functional/graph_selection/test_intersection_syntax.py::TestIntersectionSyncs::test_intersection_triple_descending": 14.092702379000002, "tests/functional/graph_selection/test_intersection_syntax.py::TestIntersectionSyncs::test_intersection_triple_descending_schema": 1.58655805799998, "tests/functional/graph_selection/test_intersection_syntax.py::TestIntersectionSyncs::test_intersection_triple_descending_schema_selectors": 14.10705950099964, "tests/functional/graph_selection/test_intersection_syntax.py::TestIntersectionSyncs::test_intersection_with_exclusion": 14.351303169000403, "tests/functional/graph_selection/test_intersection_syntax.py::TestIntersectionSyncs::test_intersection_with_exclusion_selectors": 2.7241375329999755, "tests/functional/graph_selection/test_intersection_syntax.py::TestIntersectionSyncs::test_same_model_intersection": 28.43503246900036, "tests/functional/graph_selection/test_intersection_syntax.py::TestIntersectionSyncs::test_same_model_intersection_selectors": 13.844164738000018, "tests/functional/graph_selection/test_intersection_syntax.py::TestIntersectionSyncs::test_tags_intersection": 14.036897025000144, "tests/functional/graph_selection/test_intersection_syntax.py::TestIntersectionSyncs::test_tags_intersection_selectors": 14.063359466000293, "tests/functional/graph_selection/test_schema_test_graph_selection.py::TestSchemaTestGraphSelection::test_schema_tests_dep_package_only": 51.68071550100012, "tests/functional/graph_selection/test_schema_test_graph_selection.py::TestSchemaTestGraphSelection::test_schema_tests_exclude_pkg": 52.16865292500006, "tests/functional/graph_selection/test_schema_test_graph_selection.py::TestSchemaTestGraphSelection::test_schema_tests_model_in_dep_pkg": 52.11282838900024, "tests/functional/graph_selection/test_schema_test_graph_selection.py::TestSchemaTestGraphSelection::test_schema_tests_no_specifiers": 64.64320401700024, "tests/functional/graph_selection/test_schema_test_graph_selection.py::TestSchemaTestGraphSelection::test_schema_tests_specify_exclude_only": 4.927788882000016, "tests/functional/graph_selection/test_schema_test_graph_selection.py::TestSchemaTestGraphSelection::test_schema_tests_specify_model": 5.523728221999988, "tests/functional/graph_selection/test_schema_test_graph_selection.py::TestSchemaTestGraphSelection::test_schema_tests_specify_model_and_children": 50.22854238400032, "tests/functional/graph_selection/test_schema_test_graph_selection.py::TestSchemaTestGraphSelection::test_schema_tests_specify_model_and_parents": 50.10652645600021, "tests/functional/graph_selection/test_schema_test_graph_selection.py::TestSchemaTestGraphSelection::test_schema_tests_specify_model_and_parents_with_exclude": 4.782106176000013, "tests/functional/graph_selection/test_schema_test_graph_selection.py::TestSchemaTestGraphSelection::test_schema_tests_specify_model_in_pkg": 50.37311680499988, "tests/functional/graph_selection/test_schema_test_graph_selection.py::TestSchemaTestGraphSelection::test_schema_tests_specify_tag": 49.22179712800016, "tests/functional/graph_selection/test_schema_test_graph_selection.py::TestSchemaTestGraphSelection::test_schema_tests_specify_tag_and_children": 4.042665136000011, "tests/functional/graph_selection/test_schema_test_graph_selection.py::TestSchemaTestGraphSelection::test_schema_tests_with_glob": 51.697270744000434, "tests/functional/graph_selection/test_tag_selection.py::TestTagSelection::test_select_tag": 33.416239341000164, "tests/functional/graph_selection/test_tag_selection.py::TestTagSelection::test_select_tag_and_children": 31.180385521000062, "tests/functional/graph_selection/test_tag_selection.py::TestTagSelection::test_select_tag_and_children_selector_dict": 15.542415533000167, "tests/functional/graph_selection/test_tag_selection.py::TestTagSelection::test_select_tag_and_children_selector_str": 15.557976923000297, "tests/functional/graph_selection/test_tag_selection.py::TestTagSelection::test_select_tag_in_model_with_project_config": 15.266220377000991, "tests/functional/graph_selection/test_tag_selection.py::TestTagSelection::test_select_tag_in_model_with_project_config_parents_children": 78.16209385299999, "tests/functional/graph_selection/test_tag_selection.py::TestTagSelection::test_select_tag_in_model_with_project_config_parents_children_selectors": 79.18678878999981, "tests/functional/graph_selection/test_tag_selection.py::TestTagSelection::test_select_tag_in_model_with_project_config_selector": 15.599573576999319, "tests/functional/graph_selection/test_tag_selection.py::TestTagSelection::test_select_tag_selector_dict": 18.79408114599937, "tests/functional/graph_selection/test_tag_selection.py::TestTagSelection::test_select_tag_selector_str": 17.633430820000285, "tests/functional/graph_selection/test_tag_selection.py::TestTagSelectionNoMatch::test_no_match": 62.142076434000046, "tests/functional/graph_selection/test_version_selection.py::TestVersionSelection::test_select_group_and_children_selector_str": 1.6655723169999987, "tests/functional/graph_selection/test_version_selection.py::TestVersionSelection::test_select_latest_versions": 15.785927988999902, "tests/functional/graph_selection/test_version_selection.py::TestVersionSelection::test_select_models_by_version_and_children": 15.688814922000347, "tests/functional/graph_selection/test_version_selection.py::TestVersionSelection::test_select_models_two_versions": 15.784008649000043, "tests/functional/graph_selection/test_version_selection.py::TestVersionSelection::test_select_none_versions": 31.43017837699972, "tests/functional/graph_selection/test_version_selection.py::TestVersionSelection::test_select_old_versions": 15.41188330299974, "tests/functional/graph_selection/test_version_selection.py::TestVersionSelection::test_select_prerelease_versions": 15.761585003000619, "tests/functional/graph_selection/test_version_selection.py::TestVersionSelection::test_select_version_and_children": 15.621994842999811, "tests/functional/graph_selection/test_version_selection.py::TestVersionSelection::test_select_version_selector_dict": 15.54781453500027, "tests/functional/graph_selection/test_version_selection.py::TestVersionSelection::test_select_version_selector_str": 15.499085959999775, "tests/functional/graph_selection/test_version_selection.py::TestVersionZero::test_version_zero": 32.610576246000164, "tests/functional/incremental_schema_tests/test_incremental_schema.py::TestIncrementalSchemaChange::test_run_incremental_append_new_columns": 68.41681122099953, "tests/functional/incremental_schema_tests/test_incremental_schema.py::TestIncrementalSchemaChange::test_run_incremental_fail_on_schema_change": 36.856877812999755, "tests/functional/incremental_schema_tests/test_incremental_schema.py::TestIncrementalSchemaChange::test_run_incremental_ignore": 50.44773877700027, "tests/functional/incremental_schema_tests/test_incremental_schema.py::TestIncrementalSchemaChange::test_run_incremental_sync_all_columns": 69.71002910400057, "tests/functional/init/test_init.py::TestInitDebugFailureDoesNotFailInit::test_debug_failure_does_not_fail_init": 0.5791350929999908, "tests/functional/init/test_init.py::TestInitDebugSkippedWithSkipProfileSetup::test_debug_skipped_with_skip_profile_setup": 17.2339656070003, "tests/functional/init/test_init.py::TestInitInsideProjectAndSkipProfileSetup::test_init_inside_project_and_skip_profile_setup": 17.153582335000465, "tests/functional/init/test_init.py::TestInitInvalidProfileTemplate::test_init_task_in_project_with_invalid_profile_template": 18.39949953400037, "tests/functional/init/test_init.py::TestInitInvalidProjectNameCLI::test_init_invalid_project_name_cli": 18.422810833000312, "tests/functional/init/test_init.py::TestInitInvalidProjectNamePrompt::test_init_invalid_project_name_prompt": 18.742967726999723, "tests/functional/init/test_init.py::TestInitOutsideOfProject::test_init_task_outside_of_project": 19.04777491899995, "tests/functional/init/test_init.py::TestInitOutsideOfProjectSpecifyingInvalidProfile::test_init_task_outside_project_specifying_invalid_profile_errors": 17.796106995999708, "tests/functional/init/test_init.py::TestInitOutsideOfProjectSpecifyingProfileNoProfilesYml::test_init_task_outside_project_specifying_profile_no_profiles_yml_errors": 0.9037674869998682, "tests/functional/init/test_init.py::TestInitOutsideOfProjectWithSpecifiedProfile::test_init_task_outside_of_project_with_specified_profile": 17.525682263000363, "tests/functional/init/test_init.py::TestInitProjectWithExistingProfilesYml::test_init_task_in_project_specifying_profile_errors": 0.037768407999465126, "tests/functional/init/test_init.py::TestInitProjectWithExistingProfilesYml::test_init_task_in_project_with_existing_profiles_yml": 18.100640485999975, "tests/functional/init/test_init.py::TestInitProjectWithProfileTemplateWithoutExistingProfilesYml::test_init_task_in_project_with_profile_template_without_existing_profiles_yml": 17.554543131000173, "tests/functional/init/test_init.py::TestInitProjectWithoutExistingProfilesYml::test_init_task_in_project_without_existing_profiles_yml": 17.863601771999583, "tests/functional/init/test_init.py::TestInitProjectWithoutExistingProfilesYml::test_init_task_in_project_without_profile_yml_specifying_profile_errors": 0.03693259000010585, "tests/functional/init/test_init.py::TestInitProjectWithoutExistingProfilesYmlOrTemplate::test_init_task_in_project_without_existing_profiles_yml_or_profile_template": 17.679810812999676, "tests/functional/init/test_init.py::TestInitProvidedProjectNameAndSkipProfileSetup::test_init_provided_project_name_and_skip_profile_setup": 0.7673309320000214, "tests/functional/init/test_init.py::TestInitRunsDebugAfterInit::test_debug_runs_after_init": 17.08298876900062, "tests/functional/init/test_init.py::TestInitSkipDebugFlag::test_debug_skipped_with_skip_debug_flag": 16.945760626000265, "tests/functional/init/test_init.py::TestInitSkipDebugFlag::test_run_debug_returns_none_when_skip_debug": 0.0059812630001943035, "tests/functional/invalid_model_tests/test_invalid_models.py::TestInvalidDisabledSource::test_postgres_source_disabled": 36.04655223199916, "tests/functional/invalid_model_tests/test_invalid_models.py::TestInvalidMacroCall::test_with_invalid_macro_call": 36.61649233600019, "tests/functional/invalid_model_tests/test_invalid_models.py::TestInvalidMissingSource::test_source_missing": 37.96848639899963, "tests/functional/invalid_model_tests/test_invalid_models.py::TestMalformedEnabledParam::test_view_disabled": 35.979286254000726, "tests/functional/invalid_model_tests/test_invalid_models.py::TestMissingModelReference::test_models_not_found": 36.57908891800071, "tests/functional/invalid_model_tests/test_invalid_models.py::TestReferencingDisabledModel::test_referencing_disabled_model": 1.6028407899999593, "tests/functional/invalid_model_tests/test_model_logging.py::TestModelLogging::test_warn": 38.163236172000325, "tests/functional/list/test_commands.py::TestRunCommands::test_run_commmand": 185.6277160159998, "tests/functional/list/test_commands.py::TestSelectResourceType::test_select_by_resource_type[function]": 2.7868588799998406, "tests/functional/list/test_commands.py::TestSelectResourceType::test_select_by_resource_type[metric]": 3.937126362000015, "tests/functional/list/test_commands.py::TestSelectResourceType::test_select_by_resource_type[model]": 40.602950125999996, "tests/functional/list/test_commands.py::TestSelectResourceType::test_select_by_resource_type[saved_query]": 18.88520880300075, "tests/functional/list/test_commands.py::TestSelectResourceType::test_select_by_resource_type[seed]": 19.36082779299977, "tests/functional/list/test_commands.py::TestSelectResourceType::test_select_by_resource_type[semantic_model]": 19.277891276001355, "tests/functional/list/test_commands.py::TestSelectResourceType::test_select_by_resource_type[snapshot]": 19.32752205700035, "tests/functional/list/test_commands.py::TestSelectResourceType::test_select_by_resource_type[source]": 19.099546781000754, "tests/functional/list/test_commands.py::TestSelectResourceType::test_select_by_resource_type[test]": 19.203873597999518, "tests/functional/list/test_list.py::TestList::test_ls": 1099.951739444, "tests/functional/list/test_list.py::TestList::test_packages_install_path_does_not_exist": 40.05945426699964, "tests/functional/logging/test_logging.py::TestRunResultErrorGroup::test_node_info_on_results": 45.281118392998906, "tests/functional/logging/test_logging.py::TestRunResultErrorNodeInfo::test_node_info_on_results": 45.6950353000002, "tests/functional/logging/test_logging.py::TestRunResultFailureGroup::test_node_info_on_results": 1.7022340180000128, "tests/functional/logging/test_logging.py::TestRunResultGroupWithMultipleEmails::test_node_info_on_results": 43.78835089600034, "tests/functional/logging/test_logging.py::TestRunResultNoGroup::test_node_info_on_results": 46.46921098799976, "tests/functional/logging/test_logging.py::TestRunResultWarningGroup::test_node_info_on_results": 46.4903418340009, "tests/functional/logging/test_logging.py::test_basic": 47.53864436300046, "tests/functional/logging/test_logging.py::test_formatted_logs": 1.5603004649999832, "tests/functional/logging/test_logging.py::test_invalid_event_value": 1.498464998999907, "tests/functional/logging/test_meta_logging.py::test_checksum": 44.08258937299979, "tests/functional/logging/test_meta_logging.py::test_meta": 44.063530516001265, "tests/functional/macros/test_macro_annotations.py::TestMacroDefaultArgMetadata::test_macro_default_arg_metadata": 43.33437836400026, "tests/functional/macros/test_macro_annotations.py::TestMacroNameWarnings::test_macro_name_enforcement": 44.30069975100014, "tests/functional/macros/test_macro_annotations.py::TestMacroNonEnforcement::test_macro_non_enforcement": 43.817898860998866, "tests/functional/macros/test_macro_annotations.py::TestMacroTypeWarnings::test_macro_type_warnings": 1.3724852929999827, "tests/functional/macros/test_macros.py::TestAdapterMacroDeprecated::test_invalid_macro": 35.55750028400007, "tests/functional/macros/test_macros.py::TestAdapterMacroNoDestination::test_invalid_macro": 44.34371805399951, "tests/functional/macros/test_macros.py::TestDispatchMacroOverrideBuiltin::test_overrides": 2.9275026299999922, "tests/functional/macros/test_macros.py::TestInvalidMacros::test_invalid_macro": 45.734582750999834, "tests/functional/macros/test_macros.py::TestMacroMetaDocsMerge::test_both_config_and_top_level_defined": 1.2182601210000144, "tests/functional/macros/test_macros.py::TestMacroMetaDocsMerge::test_only_config_defined": 1.2773016919999804, "tests/functional/macros/test_macros.py::TestMacroMetaDocsMerge::test_only_top_level_defined": 1.4264197600000017, "tests/functional/macros/test_macros.py::TestMacroNotOverridePackage::test_overrides": 69.28586699799871, "tests/functional/macros/test_macros.py::TestMacroOverrideBuiltin::test_overrides": 68.62218654499975, "tests/functional/macros/test_macros.py::TestMacroOverridePackage::test_overrides": 68.23813533700104, "tests/functional/macros/test_macros.py::TestMacros::test_working_macros": 5.3055779669999765, "tests/functional/macros/test_macros.py::TestMacrosNamedMaterialization::test_macro_with_materialization_in_name_works": 44.730292663, "tests/functional/macros/test_macros.py::TestMisnamedMacroNamespace::test_misnamed_macro_namespace": 45.5819768120009, "tests/functional/manifest_validations/test_check_for_spaces_in_model_names.py::TestAllowSpacesInModelNamesFalse::test_require_resource_names_without_spaces": 51.528736794, "tests/functional/manifest_validations/test_check_for_spaces_in_model_names.py::TestSpaceInModelNamesWithDebug::tests_debug_when_spaces_in_name": 2.3961025099999915, "tests/functional/manifest_validations/test_check_for_spaces_in_model_names.py::TestSpacesInModelNamesHappyPath::test_no_warnings_when_no_spaces_in_name": 36.15592363399992, "tests/functional/manifest_validations/test_check_for_spaces_in_model_names.py::TestSpacesInModelNamesSadPath::tests_warning_when_spaces_in_name": 1.566663277999993, "tests/functional/materializations/test_custom_materialization.py::TestCustomMaterializationDependency::test_custom_materialization_deopendency": 1.8580500450000272, "tests/functional/materializations/test_custom_materialization.py::TestOverrideAdapterDependency::test_adapter_dependency": 35.41312095300054, "tests/functional/materializations/test_custom_materialization.py::TestOverrideAdapterDependencyDeprecated::test_adapter_dependency_deprecate_overrides": 35.24775612900066, "tests/functional/materializations/test_custom_materialization.py::TestOverrideAdapterDependencyLegacy::test_adapter_dependency": 36.47106995199965, "tests/functional/materializations/test_custom_materialization.py::TestOverrideAdapterDependencyPassing::test_default_dependency": 1.669941439000013, "tests/functional/materializations/test_custom_materialization.py::TestOverrideAdapterLocal::test_default_dependency": 37.50113029499971, "tests/functional/materializations/test_custom_materialization.py::TestOverrideDefaultDependency::test_default_dependency": 37.01245163500016, "tests/functional/materializations/test_custom_materialization.py::TestOverrideDefaultDependencyDeprecated::test_default_dependency_deprecated": 37.905290189000425, "tests/functional/materializations/test_custom_materialization.py::TestOverrideDefaultDependencyLegacy::test_default_dependency": 36.938370932, "tests/functional/materializations/test_custom_materialization.py::TestOverrideDefaultDependencyRootOverride::test_default_dependency_with_root_override": 37.68423807099953, "tests/functional/materializations/test_custom_materialization.py::TestOverrideDefaultReturn::test_default_dependency": 37.78295711999999, "tests/functional/materializations/test_ephemeral_compilation.py::TestEphemeralCompilation::test__suppress_injected_ctes": 19.31645506500081, "tests/functional/materializations/test_ephemeral_compilation.py::TestEphemeralCompilation::test_ephemeral_compilation": 38.62254519200087, "tests/functional/materializations/test_ephemeral_compilation.py::TestLargeEphemeralCompilation::test_ephemeral_compilation": 42.39926495100008, "tests/functional/materializations/test_incremental.py::test_basic": 40.398111968000194, "tests/functional/materializations/test_incremental_with_contract.py::TestIncremental::test_incremental": 80.8141773289999, "tests/functional/materializations/test_runtime_materialization.py::TestRuntimeMaterialization::test_delete_dbt_tmp_relation": 80.14798397899995, "tests/functional/materializations/test_runtime_materialization.py::TestRuntimeMaterialization::test_full_refresh": 5.311695101999931, "tests/functional/materializations/test_runtime_materialization.py::TestRuntimeMaterializationWithConfig::test_delete_dbt_tmp_relation": 80.55651747700085, "tests/functional/materializations/test_runtime_materialization.py::TestRuntimeMaterializationWithConfig::test_full_refresh": 101.39351926700056, "tests/functional/materializations/test_supported_languages.py::TestSupportedLanguages_SupportsDefault_UsingPython::test_language": 39.40107029200044, "tests/functional/materializations/test_supported_languages.py::TestSupportedLanguages_SupportsDefault_UsingSql::test_language": 40.45087230699937, "tests/functional/materializations/test_supported_languages.py::TestSupportedLanguages_SupportsSql_UsingSql::test_language": 40.2894384859992, "tests/functional/materializations/test_supported_languages.py::TestSupportedLanguages_SuppotsPython_UsingPython::test_language": 42.44475588000023, "tests/functional/materializations/test_supported_languages.py::TestSupportedLanguages_SuppotsPython_UsingSql::test_language": 46.84575000800032, "tests/functional/materializations/test_supported_languages.py::TestSupportedLanguages_SuppotsSqlAndPython_UsingPython::test_language": 48.37612609400094, "tests/functional/materializations/test_supported_languages.py::TestSupportedLanguages_SuppotsSqlAndPython_UsingSql::test_language": 48.06351188900044, "tests/functional/materializations/test_supported_languages.py::TestSupportedLanguages_SuppotsSql_UsingPython::test_language": 43.04030513500038, "tests/functional/metrics/test_metric_configs.py::TestConfigYamlMetricLevel::test_metric_config_yaml_metric_level": 1.4210243919999357, "tests/functional/metrics/test_metric_configs.py::TestDisabledMetric::test_disabling_upstream_metric_errors": 65.37125230400125, "tests/functional/metrics/test_metric_configs.py::TestDisabledMetricRef::test_disabled_metric_ref_model": 74.60683608299951, "tests/functional/metrics/test_metric_configs.py::TestInvalidMetric::test_invalid_config_metric": 48.01344038199932, "tests/functional/metrics/test_metric_configs.py::TestMetricConfigsInheritence::test_metrics_all_configs": 47.10576063699955, "tests/functional/metrics/test_metric_configs.py::TestMetricEnabledConfigProjectLevel::test_enabled_metric_config_dbt_project": 72.71909323699947, "tests/functional/metrics/test_metric_configs.py::TestMetricMetaConfigLevel::test_meta_metric_config_yaml": 43.8280317789995, "tests/functional/metrics/test_metric_configs.py::TestMetricMetaConfigProjectLevel::test_meta_metric_config_dbt_project": 41.78660000899981, "tests/functional/metrics/test_metric_configs.py::TestMetricMetaTopLevel::test_meta_metric_config_yaml": 44.584683468999174, "tests/functional/metrics/test_metric_deferral.py::TestMetricDeferral::test_metric_deferral": 0.0005971929995212122, "tests/functional/metrics/test_metric_helper_functions.py::TestMetricHelperFunctions::test_derived_metric": 45.06806885900005, "tests/functional/metrics/test_metrics.py::TestConversionMetric::test_conversion_metric": 43.48691987300026, "tests/functional/metrics/test_metrics.py::TestCumulativeMetric::test_cumulative_metric": 43.55397571700087, "tests/functional/metrics/test_metrics.py::TestDerivedMetric::test_derived_metric": 64.41477746099918, "tests/functional/metrics/test_metrics.py::TestDuplicateInputMeasures::test_duplicate_input_measures": 43.91750452100041, "tests/functional/metrics/test_metrics.py::TestFilterParsing::test_filter_parsing": 1.0805087550000962, "tests/functional/metrics/test_metrics.py::TestInvalidDerivedMetrics::test_invalid_derived_metrics": 43.00805767900056, "tests/functional/metrics/test_metrics.py::TestInvalidMetricMissingExpression::test_simple_metric": 44.87452756099992, "tests/functional/metrics/test_metrics.py::TestInvalidMetricMissingModel::test_simple_metric": 45.318341484001394, "tests/functional/metrics/test_metrics.py::TestInvalidRefMetrics::test_simple_metric": 44.94730245499977, "tests/functional/metrics/test_metrics.py::TestInvalidTimestampTimeGrainsMetrics::test_simple_metric": 42.52384508099931, "tests/functional/metrics/test_metrics.py::TestInvalidTimestampWindowMetrics::test_simple_metric": 42.79619867999918, "tests/functional/metrics/test_metrics.py::TestLongName::test_long_name": 46.66394922699965, "tests/functional/metrics/test_metrics.py::TestMetricDependsOn::test_metric_depends_on": 2.2376685339999938, "tests/functional/metrics/test_metrics.py::TestNamesWithLeandingNumber::test_names_with_leading_number": 1.4239308189999917, "tests/functional/metrics/test_metrics.py::TestNamesWithSpaces::test_names_with_spaces": 44.89969814699998, "tests/functional/metrics/test_metrics.py::TestNamesWithSpecialChar::test_names_with_special_char": 43.345890454000255, "tests/functional/metrics/test_metrics.py::TestSimpleMetrics::test_simple_metric": 46.22732834999897, "tests/functional/microbatch/test_microbatch.py::TestCanSilenceInvalidConcurrentBatchesConfigWarning::test_microbatch": 61.68408309000006, "tests/functional/microbatch/test_microbatch.py::TestCompilationErrorOnSingleBatchRun::test_microbatch": 1.871858727000017, "tests/functional/microbatch/test_microbatch.py::TestFirstAndLastBatchAlwaysSequential::test_microbatch": 2.093803922999996, "tests/functional/microbatch/test_microbatch.py::TestFirstBatchRunsPreHookLastBatchRunsPostHook::test_microbatch": 41.844506328000534, "tests/functional/microbatch/test_microbatch.py::TestMicrbobatchModelsRunWithSameCurrentTime::test_microbatch": 42.18878241300081, "tests/functional/microbatch/test_microbatch.py::TestMicroBatchBoundsDefault::test_run_with_event_time": 6.730546354000012, "tests/functional/microbatch/test_microbatch.py::TestMicrobatchBuildRetryUsesOriginalInvocationTime::test_build_retry_uses_original_invocation_time": 70.73094428600052, "tests/functional/microbatch/test_microbatch.py::TestMicrobatchCLI::test_run_with_event_time": 68.55835114899946, "tests/functional/microbatch/test_microbatch.py::TestMicrobatchCLIBuild::test_run_with_event_time": 71.50381254700096, "tests/functional/microbatch/test_microbatch.py::TestMicrobatchCLIRunOutputJSON::test_list_output_json": 45.817936219999865, "tests/functional/microbatch/test_microbatch.py::TestMicrobatchCanRunParallelOrSequential::test_microbatch": 60.2004797970003, "tests/functional/microbatch/test_microbatch.py::TestMicrobatchCompiledRunPaths::test_run_with_event_time": 40.059524846000386, "tests/functional/microbatch/test_microbatch.py::TestMicrobatchCompiledRunPathsHourly::test_run_with_event_time": 51.14564189500015, "tests/functional/microbatch/test_microbatch.py::TestMicrobatchCompiledRunPathsMonthly::test_run_with_event_time": 1.6392900529999963, "tests/functional/microbatch/test_microbatch.py::TestMicrobatchCompiledRunPathsYearly::test_run_with_event_time": 38.36248339899885, "tests/functional/microbatch/test_microbatch.py::TestMicrobatchCustomUserStrategyDefault::test_use_custom_microbatch_strategy_by_default": 68.46613417200024, "tests/functional/microbatch/test_microbatch.py::TestMicrobatchCustomUserStrategyProjectFlagTrueNoValidBuiltin::test_use_custom_microbatch_strategy_project_flag_true_invalid_incremental_strategy": 45.130686196999704, "tests/functional/microbatch/test_microbatch.py::TestMicrobatchCustomUserStrategyProjectFlagTrueValid::test_use_custom_microbatch_strategy_project_flag_true_invalid_incremental_strategy": 69.53183550499944, "tests/functional/microbatch/test_microbatch.py::TestMicrobatchFullRefreshConfigFalse::test_run_with_event_time": 5.763186471999973, "tests/functional/microbatch/test_microbatch.py::TestMicrobatchIncrementalBatchFailure::test_run_with_event_time": 48.143064466000396, "tests/functional/microbatch/test_microbatch.py::TestMicrobatchInitialBatchFailure::test_run_with_event_time": 39.21413267099979, "tests/functional/microbatch/test_microbatch.py::TestMicrobatchJinjaContext::test_run_with_event_time": 47.27746463999938, "tests/functional/microbatch/test_microbatch.py::TestMicrobatchJinjaContextVarsAvailable::test_run_with_event_time_logs": 73.00817411300068, "tests/functional/microbatch/test_microbatch.py::TestMicrobatchModelSkipped::test_microbatch_model_skipped": 39.17014909500085, "tests/functional/microbatch/test_microbatch.py::TestMicrobatchMultipleRetries::test_run_with_event_time": 4.641316007, "tests/functional/microbatch/test_microbatch.py::TestMicrobatchRetriesPartialSuccesses::test_run_with_event_time": 73.61377556299976, "tests/functional/microbatch/test_microbatch.py::TestMicrobatchRetryUsesOriginalInvocationTime::test_retry_uses_original_invocation_time": 68.02723471199988, "tests/functional/microbatch/test_microbatch.py::TestMicrobatchSecondBatchFailure::test_run_with_event_time": 2.079216145999993, "tests/functional/microbatch/test_microbatch.py::TestMicrobatchUsingRefRenderSkipsFilter::test_run_with_event_time": 123.13393394800096, "tests/functional/microbatch/test_microbatch.py::TestMicrobatchWithInputWithoutEventTime::test_run_with_event_time": 97.51271705899944, "tests/functional/microbatch/test_microbatch.py::TestMicrobatchWithSource::test_run_with_event_time": 168.65866677299982, "tests/functional/microbatch/test_microbatch.py::TestWhenOnlyOneBatchRunBothPostAndPreHooks::test_microbatch": 1.6867974120000326, "tests/functional/microbatch/test_microbatch_config_validation.py::TestInvaliBegiFormatMicrobatch::test_parsing_error_raised": 49.61423995400037, "tests/functional/microbatch/test_microbatch_config_validation.py::TestInvaliBeginTypeMicrobatch::test_parsing_error_raised": 45.681478436999896, "tests/functional/microbatch/test_microbatch_config_validation.py::TestInvalidBatchSizeMicrobatch::test_parsing_error_raised": 47.00543480400029, "tests/functional/microbatch/test_microbatch_config_validation.py::TestInvalidEventTimeMicrobatch::test_parsing_error_raised": 48.23379116000069, "tests/functional/microbatch/test_microbatch_config_validation.py::TestInvalidInputEventTimeMicrobatch::test_parsing_error_raised": 1.5324863220000111, "tests/functional/microbatch/test_microbatch_config_validation.py::TestMissingBatchSizeMicrobatch::test_parsing_error_raised": 48.58731079300014, "tests/functional/microbatch/test_microbatch_config_validation.py::TestMissingBeginMicrobatch::test_parsing_error_raised": 47.37001400600002, "tests/functional/microbatch/test_microbatch_config_validation.py::TestMissingEventTimeMicrobatch::test_parsing_error_raised": 47.727783499, "tests/functional/microbatch/test_microbatch_config_validation.py::TestValidBeginMicrobatch::test_parsing_error_not_raised": 42.28735269399931, "tests/functional/minimal_cli/test_minimal_cli.py::TestBuild::test_build": 47.305377618999955, "tests/functional/minimal_cli/test_minimal_cli.py::TestBuildFailFast::test_build": 47.45623747299942, "tests/functional/minimal_cli/test_minimal_cli.py::TestBuildFailFastDebug::test_build": 47.646759803999885, "tests/functional/minimal_cli/test_minimal_cli.py::TestClean::test_clean": 20.596570561999215, "tests/functional/minimal_cli/test_minimal_cli.py::TestCleanUpLevel::test_clean_one_level_up": 21.4884069360005, "tests/functional/minimal_cli/test_minimal_cli.py::TestDeps::test_deps": 21.28391110099892, "tests/functional/minimal_cli/test_minimal_cli.py::TestDocsGenerate::test_docs_generate": 3.741823138000001, "tests/functional/minimal_cli/test_minimal_cli.py::TestLS::test_ls": 46.929066123000666, "tests/functional/model_config/test_freshness_config.py::TestModelFreshnessConfig::test_model_freshness_configs": 67.40153810600077, "tests/functional/model_config/test_freshness_config.py::TestModelFreshnessConfigParseBuildAfterOnly::test_model_freshness_configs": 43.433466822999435, "tests/functional/model_config/test_freshness_config.py::TestModelFreshnessConfigParseBuildCountRequiresPeriod::test_model_freshness_configs": 1.3362374740000007, "tests/functional/model_config/test_freshness_config.py::TestModelFreshnessConfigParseBuildPeriodHas0Count::test_model_freshness_configs": 46.19138921600006, "tests/functional/model_config/test_freshness_config.py::TestModelFreshnessConfigParseBuildPeriodRequiresCount::test_model_freshness_configs": 45.27474981100022, "tests/functional/partial_parsing/test_file_diff.py::TestFileDiffPaths::test_file_diffs": 4.706529336999978, "tests/functional/partial_parsing/test_file_diff.py::TestFileDiffs::test_no_file_diffs": 86.154633784, "tests/functional/partial_parsing/test_partial_parsing.py::TestExplicitDefaultProfileAndTarget::test_explicit_default_profile_allows_partial_parse": 2.3602917539999737, "tests/functional/partial_parsing/test_partial_parsing.py::TestExplicitDefaultProfileAndTarget::test_explicit_default_target_allows_partial_parse": 52.71984343099939, "tests/functional/partial_parsing/test_partial_parsing.py::TestExternalModels::test_pp_external_models": 258.72891292200075, "tests/functional/partial_parsing/test_partial_parsing.py::TestGenericTests::test_pp_generic_tests": 102.55774969299819, "tests/functional/partial_parsing/test_partial_parsing.py::TestMacroDescriptionUpdate::test_pp_macro_description_update": 99.31729125599941, "tests/functional/partial_parsing/test_partial_parsing.py::TestModels::test_pp_models": 788.3676428690005, "tests/functional/partial_parsing/test_partial_parsing.py::TestNestedMacros::test_nested_macros": 99.08370542399916, "tests/functional/partial_parsing/test_partial_parsing.py::TestPartialParsingDependency::test_parsing_with_dependency": 124.51294739700097, "tests/functional/partial_parsing/test_partial_parsing.py::TestPortablePartialParsing::test_pp_renamed_project_dir_changed_project_contents": 5.269835572999966, "tests/functional/partial_parsing/test_partial_parsing.py::TestPortablePartialParsing::test_pp_renamed_project_dir_unchanged_project_contents": 132.74717698699897, "tests/functional/partial_parsing/test_partial_parsing.py::TestProfileChanges::test_profile_change": 4.173779966000097, "tests/functional/partial_parsing/test_partial_parsing.py::TestSingularTests::test_pp_singular_tests": 102.38346065500082, "tests/functional/partial_parsing/test_partial_parsing.py::TestSkipMacros::test_skip_macros": 176.99414914099998, "tests/functional/partial_parsing/test_partial_parsing.py::TestSnapshots::test_pp_snapshots": 151.79003548199762, "tests/functional/partial_parsing/test_partial_parsing.py::TestSources::test_pp_sources": 539.2975181890006, "tests/functional/partial_parsing/test_pp_disabled_config.py::TestDisabled::test_pp_disabled": 8.966595016000014, "tests/functional/partial_parsing/test_pp_docs.py::TestDocs::test_pp_docs": 222.60327147799762, "tests/functional/partial_parsing/test_pp_docs.py::TestDocsRemoveReplace::test_remove_replace": 109.55472164000275, "tests/functional/partial_parsing/test_pp_functions.py::TestPartialParsingFunctions::test_pp_functions": 145.96800067899858, "tests/functional/partial_parsing/test_pp_functions.py::TestPartialParsingFunctionsAndCompilationOfDownstreamNodes::test_pp_functions": 3.0146560549998185, "tests/functional/partial_parsing/test_pp_groups.py::TestAddingModelsToNewGroups::test_adding_models_to_new_groups": 86.44163743399986, "tests/functional/partial_parsing/test_pp_groups.py::TestGroups::test_pp_groups": 264.66108741899916, "tests/functional/partial_parsing/test_pp_metrics.py::TestDeleteFileWithMetricsAndSemanticModels::test_metrics": 2.6493146839999895, "tests/functional/partial_parsing/test_pp_metrics.py::TestMetrics::test_metrics": 174.19377571899895, "tests/functional/partial_parsing/test_pp_schema_file_order.py::TestMoreNewVersionedSchemaFile::test_more_schema_file_new_versions": 2.7218378849999, "tests/functional/partial_parsing/test_pp_schema_file_order.py::TestNewVersionedSchemaFile::test_schema_file_order_new_versions": 61.595369394000045, "tests/functional/partial_parsing/test_pp_schema_file_order.py::TestSchemaFileOrder::test_schema_file_order": 3.7725021889999937, "tests/functional/partial_parsing/test_pp_schema_file_order.py::TestSourcesAndSchemaFiles::test_schema_file_order_new_versions": 2.6080510789998925, "tests/functional/partial_parsing/test_pp_vars.py::TestEnvVars::test_env_vars_models": 446.3164038140003, "tests/functional/partial_parsing/test_pp_vars.py::TestProfileEnvVars::test_profile_env_vars": 3.255825217999984, "tests/functional/partial_parsing/test_pp_vars.py::TestProfileSecretEnvVars::test_profile_secret_env_vars": 69.24608755400095, "tests/functional/partial_parsing/test_pp_vars.py::TestProjectEnvVars::test_project_env_vars": 64.19450306199906, "tests/functional/partial_parsing/test_pp_vars.py::TestVarsFileBackwardCompatibility::test_cli_vars_still_trigger_reparse": 51.06528124800025, "tests/functional/partial_parsing/test_pp_vars.py::TestVarsFileBackwardCompatibility::test_no_vars_file_hash_unchanged": 69.65441722599917, "tests/functional/partial_parsing/test_pp_vars.py::TestVarsFilePartialParsing::test_vars_file_partial_parsing": 147.1975664800002, "tests/functional/partial_parsing/test_versioned_models.py::TestAddingVersioningToModel::test_pp_newly_versioned_models": 64.8301448860002, "tests/functional/partial_parsing/test_versioned_models.py::TestVersionedModels::test_pp_versioned_models": 139.65943714700006, "tests/functional/postgres/test_postgres_indexes.py::TestPostgresIndex::test_incremental": 67.72614272999908, "tests/functional/postgres/test_postgres_indexes.py::TestPostgresIndex::test_seed": 67.8726773190001, "tests/functional/postgres/test_postgres_indexes.py::TestPostgresIndex::test_snapshot": 47.1734355159997, "tests/functional/postgres/test_postgres_indexes.py::TestPostgresIndex::test_table": 43.63496663800106, "tests/functional/postgres/test_postgres_indexes.py::TestPostgresInvalidIndex::test_invalid_index_configs": 45.65219084900036, "tests/functional/postgres/test_postgres_unlogged_table.py::TestPostgresUnloggedTable::test_postgres_unlogged_table_catalog": 2.3478930989999753, "tests/functional/primary_keys/test_primary_keys.py::TestInvalidModelCombinationOfColumns::test_invalid_combo_of_columns": 45.34590656199907, "tests/functional/primary_keys/test_primary_keys.py::TestInvalidModelUniqueTest::test_invalid_combo_of_columns": 45.81897478499923, "tests/functional/primary_keys/test_primary_keys.py::TestSimpleModelCombinationOfColumns::test_versioned_simple_combo_of_columns": 46.35722870000063, "tests/functional/primary_keys/test_primary_keys.py::TestSimpleModelConstraints::test_simple_model_constraints": 1.724696487999978, "tests/functional/primary_keys/test_primary_keys.py::TestSimpleModelDisabledUniqueTests::test_simple_model_disabled_unique_test": 1.978174759000126, "tests/functional/primary_keys/test_primary_keys.py::TestSimpleModelNoYml::test_simple_model_no_yml": 44.89601107099952, "tests/functional/primary_keys/test_primary_keys.py::TestSimpleModelUniqueNotNullTests::test_simple_model_unique_not_null_tests": 46.50997733700024, "tests/functional/primary_keys/test_primary_keys.py::TestSimpleModelUniqueTests::test_simple_model_unique_test": 43.63500108600056, "tests/functional/primary_keys/test_primary_keys.py::TestVersionedSimpleModel::test_versioned_simple_model": 43.59115875399948, "tests/functional/primary_keys/test_primary_keys.py::TestVersionedSimpleModelExcludeTests::test_versioned_simple_model_exclude_col": 43.683859021999524, "tests/functional/profiles/test_profile_dir.py::TestProfiles::test_profiles[None-cwd]": 21.77632990299935, "tests/functional/profiles/test_profile_dir.py::TestProfiles::test_profiles[None-cwd_child]": 0.200592833999508, "tests/functional/profiles/test_profile_dir.py::TestProfiles::test_profiles[cwd-cwd_parent]": 0.2147362299992892, "tests/functional/profiles/test_profile_dir.py::TestProfilesMayNotExist::test_debug": 21.4974419880009, "tests/functional/profiles/test_profile_dir.py::TestProfilesMayNotExist::test_deps": 0.04147706699950504, "tests/functional/profiles/test_profiles_yml.py::TestProfileParsing::test_password_jinja_rendered_when_valid": 21.808817881999857, "tests/functional/profiles/test_profiles_yml.py::TestProfileParsing::test_password_not_jinja_rendered_when_invalid": 43.81759927100029, "tests/functional/record/test_record.py::TestRecord::test_record_when_env_var_set": 1.696933209000008, "tests/functional/ref_override/test_ref_override.py::TestAdvancedRefOverride::test_ref_override": 45.67821476099925, "tests/functional/ref_override/test_ref_override.py::TestRefOverride::test_ref_override": 68.1015033879994, "tests/functional/relation_names/test_relation_name.py::TestGeneratedDDLNameRules::test_long_name_passes_when_temp_tables_are_generated": 46.6261673680001, "tests/functional/relation_names/test_relation_name.py::TestGeneratedDDLNameRules::test_name_longer_than_63_does_not_build": 23.91877252700033, "tests/functional/relation_names/test_relation_name.py::TestGeneratedDDLNameRules::test_name_shorter_or_equal_to_63_passes": 3.2090369700000565, "tests/functional/relation_quoting/test_relation_quoting.py::TestModelQuoting::test_models_respect_global_quoting_configs": 45.658229018000384, "tests/functional/relation_quoting/test_relation_quoting.py::TestSourceQuotingGlobalConfigs::test_sources_ignore_global_quoting_configs": 46.12762554700112, "tests/functional/retry/test_retry.py::TestCustomTargetRetry::test_custom_target": 116.06044083400047, "tests/functional/retry/test_retry.py::TestFailFast::test_fail_fast": 171.2848509889991, "tests/functional/retry/test_retry.py::TestFixRetryHook::test_fix_retry_hook": 4.817213597999995, "tests/functional/retry/test_retry.py::TestRetryFullRefresh::test_retry": 74.58556281700021, "tests/functional/retry/test_retry.py::TestRetryHooksAlwaysRun::test_retry_hooks_always_run": 71.02669113800039, "tests/functional/retry/test_retry.py::TestRetryNoPreviousRun::test_no_previous_run": 23.034873092999987, "tests/functional/retry/test_retry.py::TestRetryOverridePath::test_retry": 75.07155892500123, "tests/functional/retry/test_retry.py::TestRetryPreviousRun::test_previous_run": 119.87423673899957, "tests/functional/retry/test_retry.py::TestRetryRemovedFile::test_removed_file": 71.81053844799953, "tests/functional/retry/test_retry.py::TestRetryRemovedFileLeafNode::test_removed_file_leaf_node": 3.2423468590000084, "tests/functional/retry/test_retry.py::TestRetryResourceType::test_resource_type": 7.608399299999974, "tests/functional/retry/test_retry.py::TestRetryRunOperation::test_run_operation": 71.79979037399971, "tests/functional/retry/test_retry.py::TestRetrySuccessfulRunOperation::test_run_operation": 72.19161405999967, "tests/functional/retry/test_retry.py::TestRetryTargetPathEnvVar::test_retry_target_path_env_var": 3.576095366000004, "tests/functional/retry/test_retry.py::TestRetryTargetPathFlag::test_retry_target_path_flag": 70.39241419799964, "tests/functional/retry/test_retry.py::TestRetryVars::test_retry": 125.70006368599934, "tests/functional/retry/test_retry.py::TestRetryWarnError::test_warn_error": 119.3829443249997, "tests/functional/retry/test_retry_threads.py::TestCustomThreadRetry::test_thread_target": 120.06287057600002, "tests/functional/run_operations/test_run_operations.py::TestOperations::test_access_graph": 24.204537547000655, "tests/functional/run_operations/test_run_operations.py::TestOperations::test_cannot_connect": 24.515961968001648, "tests/functional/run_operations/test_run_operations.py::TestOperations::test_macro_args": 2.457786069000008, "tests/functional/run_operations/test_run_operations.py::TestOperations::test_macro_exception": 24.417227221999383, "tests/functional/run_operations/test_run_operations.py::TestOperations::test_macro_missing": 24.187153555000805, "tests/functional/run_operations/test_run_operations.py::TestOperations::test_macro_noargs": 48.49152819699975, "tests/functional/run_operations/test_run_operations.py::TestOperations::test_print": 24.016194951000216, "tests/functional/run_operations/test_run_operations.py::TestOperations::test_run_operation_local_macro": 48.87421876600001, "tests/functional/run_operations/test_run_operations.py::TestOperations::test_select": 23.988189263000095, "tests/functional/run_operations/test_run_operations.py::TestOperations::test_vacuum": 2.6348533470000177, "tests/functional/run_operations/test_run_operations.py::TestOperations::test_vacuum_ref": 48.58166951699968, "tests/functional/run_operations/test_run_operations.py::TestRunOperationRefPrivateModel::test_run_operation_ref_private_model": 73.2422541599999, "tests/functional/run_query/test_types.py::TestTypes::test_nested_types": 48.74362838399975, "tests/functional/sample_mode/test_sample_mode.py::TestBasicSampleMode::test_sample_mode[build-False-3]": 25.79151833600008, "tests/functional/sample_mode/test_sample_mode.py::TestBasicSampleMode::test_sample_mode[build-True-2]": 25.86342170700118, "tests/functional/sample_mode/test_sample_mode.py::TestBasicSampleMode::test_sample_mode[run-False-3]": 26.124693912998737, "tests/functional/sample_mode/test_sample_mode.py::TestBasicSampleMode::test_sample_mode[run-True-2]": 50.47248202400078, "tests/functional/sample_mode/test_sample_mode.py::TestIncrementalModelSampleModeRelative::test_incremental_model_sample[2 days-5]": 3.4631214850000163, "tests/functional/sample_mode/test_sample_mode.py::TestIncrementalModelSampleModeRelative::test_incremental_model_sample[3 days-6]": 5.097192252000127, "tests/functional/sample_mode/test_sample_mode.py::TestIncrementalModelSampleModeRelative::test_incremental_model_sample[None-6]": 77.59621588499976, "tests/functional/sample_mode/test_sample_mode.py::TestIncrementalModelSampleModeSpecific::test_incremental_model_sample[None-6]": 77.27440746000048, "tests/functional/sample_mode/test_sample_mode.py::TestIncrementalModelSampleModeSpecific::test_incremental_model_sample[{'start': '2024-12-31', 'end': '2025-01-03'}-3]": 52.76324909700088, "tests/functional/sample_mode/test_sample_mode.py::TestIncrementalModelSampleModeSpecific::test_incremental_model_sample[{'start': '2025-01-03', 'end': '2025-01-07'}-6]": 52.25839509399884, "tests/functional/sample_mode/test_sample_mode.py::TestIncrementalModelSampleModeSpecific::test_incremental_model_sample[{'start': '2025-01-04', 'end': '2025-01-06'}-5]": 52.672793392999665, "tests/functional/sample_mode/test_sample_mode.py::TestIncrementalModelSampleModeSpecific::test_incremental_model_sample[{'start': '2025-01-05', 'end': '2025-01-07'}-5]": 52.79645216499921, "tests/functional/sample_mode/test_sample_mode.py::TestMicrobatchSampleMode::test_sample_mode": 52.2563432930001, "tests/functional/sample_mode/test_sample_mode.py::TestSampleSeedRefs::test_sample_mode[False-3]": 52.943739364000066, "tests/functional/sample_mode/test_sample_mode.py::TestSampleSeedRefs::test_sample_mode[True-2]": 79.07742516700091, "tests/functional/sample_mode/test_sample_mode.py::TestSamplingModelFromSnapshot::test_sample_mode[False-3]": 26.734211156997844, "tests/functional/sample_mode/test_sample_mode.py::TestSamplingModelFromSnapshot::test_sample_mode[True-2]": 52.99888501199894, "tests/functional/sample_mode/test_sample_mode.py::TestSamplingSnapshot::test_sample_mode[False-3]": 54.98123550700075, "tests/functional/sample_mode/test_sample_mode.py::TestSamplingSnapshot::test_sample_mode[True-2]": 4.526776116000065, "tests/functional/saved_queries/test_configs.py::TestExportConfigsWithAdditionalProperties::test_extra_config_properties_dont_break_parsing": 53.78055426600076, "tests/functional/saved_queries/test_configs.py::TestExportConfigsWithDefaultProperties::test_default_properties": 54.029888948001826, "tests/functional/saved_queries/test_configs.py::TestInheritingExportConfigFromSavedQueryConfig::test_export_config_inherits_from_saved_query": 54.18072945200038, "tests/functional/saved_queries/test_configs.py::TestInheritingExportConfigsFromProject::test_export_config_inherits_from_project": 82.06402399099898, "tests/functional/saved_queries/test_configs.py::TestSavedQueryCacheConfigsOverride::test_override_saved_query_config": 82.90522774700185, "tests/functional/saved_queries/test_configs.py::TestSavedQueryConfigs::test_basic_saved_query_config": 80.25600244999987, "tests/functional/saved_queries/test_configs.py::TestSavedQueryDefaultCacheConfigs::test_basic_saved_query_config": 53.4602600420003, "tests/functional/saved_queries/test_configs.py::TestSavedQueryLevelCacheConfigs::test_basic_saved_query_config": 2.914733303000048, "tests/functional/saved_queries/test_configs.py::TestSavedQueryTagsAdditiveWithConfig::test_saved_query_tags_are_additive_unique_and_sorted": 55.44608239599984, "tests/functional/saved_queries/test_saved_query_build.py::TestSavedQueryBuild::test_build_saved_queries_no_op": 57.6194285499987, "tests/functional/saved_queries/test_saved_query_parsing.py::TestSavedQueryParsing::test_saved_query_error": 30.86176200099908, "tests/functional/saved_queries/test_saved_query_parsing.py::TestSavedQueryParsing::test_semantic_model_parsing": 142.43535239000084, "tests/functional/saved_queries/test_saved_query_parsing.py::TestSavedQueryParsing::test_semantic_model_parsing_change_export": 112.44188117900012, "tests/functional/saved_queries/test_saved_query_parsing.py::TestSavedQueryParsing::test_semantic_model_parsing_with_default_schema": 85.96519105900006, "tests/functional/saved_queries/test_saved_query_parsing.py::TestSavedQueryPartialParsing::test_saved_query_deleted_partial_parsing": 3.5346529350000253, "tests/functional/saved_queries/test_saved_query_parsing.py::TestSavedQueryPartialParsing::test_saved_query_filter_types": 48.3649142080003, "tests/functional/saved_queries/test_saved_query_parsing.py::TestSavedQueryPartialParsing::test_saved_query_metrics_changed": 48.350282712999615, "tests/functional/schema/test_custom_schema.py::TestCustomSchema::test__postgres_handles__custom_schema_with_no_prefix": 74.68811275500047, "tests/functional/schema/test_custom_schema.py::TestCustomSchemaWithCustomMacro::test__postgres_handles__custom_schema_with_custom_macro": 75.236483701, "tests/functional/schema/test_custom_schema.py::TestCustomSchemaWithCustomMacroFromModelName::test__postgres__custom_schema_from_model_name": 74.71315013700041, "tests/functional/schema/test_custom_schema.py::TestCustomSchemaWithPrefix::test__postgres__custom_schema_with_prefix": 74.8875480200004, "tests/functional/schema/test_custom_schema.py::TestCustomSchemaWithPrefixAndDispatch::test__postgres__custom_schema_with_prefix_and_dispatch": 75.57450148299995, "tests/functional/schema_tests/test_custom_test_config.py::TestCustomDataTestConfig::test_custom_config": 126.26984981900023, "tests/functional/schema_tests/test_custom_test_config.py::TestMixedDataTestConfig::test_mixed_config": 2.911730610000035, "tests/functional/schema_tests/test_custom_test_config.py::TestSameKeyErrorDataTestConfig::test_same_key_error": 50.064547424999546, "tests/functional/schema_tests/test_schema_v2_tests.py::TestCliVarsSchemaTests::test_argument_rendering": 3.8451073310000083, "tests/functional/schema_tests/test_schema_v2_tests.py::TestCommentedSchema::test_quoted_schema_file": 57.83621720500014, "tests/functional/schema_tests/test_schema_v2_tests.py::TestConfiguredVarsSchemaTests::test_argument_rendering": 81.6006363939996, "tests/functional/schema_tests/test_schema_v2_tests.py::TestCustomConfigSchemaTests::test_config": 53.53599410499919, "tests/functional/schema_tests/test_schema_v2_tests.py::TestCustomSchemaTestMacroResolutionOrder::test_macro_resolution_test_namespace": 71.21326666999994, "tests/functional/schema_tests/test_schema_v2_tests.py::TestCustomSchemaTests::test_schema_tests": 81.92847736500153, "tests/functional/schema_tests/test_schema_v2_tests.py::TestDefaultBoolType::test_limit_schema_tests": 78.17487069100025, "tests/functional/schema_tests/test_schema_v2_tests.py::TestGenericTestsCollide::test_generic_test_collision": 1.43436082300002, "tests/functional/schema_tests/test_schema_v2_tests.py::TestGenericTestsConfigCustomMacros::test_generic_test_config_custom_macros": 2.647578959999919, "tests/functional/schema_tests/test_schema_v2_tests.py::TestGenericTestsCustomNames::test_generic_tests_with_custom_names": 86.8407421410011, "tests/functional/schema_tests/test_schema_v2_tests.py::TestGenericTestsCustomNamesAltFormat::test_collision_test_names_get_hash": 57.46952650899766, "tests/functional/schema_tests/test_schema_v2_tests.py::TestGenericTestsCustomNamesAltFormat::test_generic_tests_with_custom_names": 85.75782510700083, "tests/functional/schema_tests/test_schema_v2_tests.py::TestHooksForWhich::test_these_hooks_dont_run_for_tests": 53.37430147099985, "tests/functional/schema_tests/test_schema_v2_tests.py::TestHooksInTests::test_hooks_do_run_for_tests": 53.391112824998345, "tests/functional/schema_tests/test_schema_v2_tests.py::TestInvalidSchema::test_invalid_schema_file": 56.53393813800176, "tests/functional/schema_tests/test_schema_v2_tests.py::TestLimitedSchemaTests::test_limit_schema_tests": 78.90498587799993, "tests/functional/schema_tests/test_schema_v2_tests.py::TestMalformedSchemaTests::test_malformed_schema_will_break_run": 2.0592564070000208, "tests/functional/schema_tests/test_schema_v2_tests.py::TestNonBoolType::test_limit_schema_tests": 79.32257230800133, "tests/functional/schema_tests/test_schema_v2_tests.py::TestOtherBoolType::test_limit_schema_tests": 4.101005440000108, "tests/functional/schema_tests/test_schema_v2_tests.py::TestQuotedSchemaTestColumns::test_quote_required_column": 192.117170988, "tests/functional/schema_tests/test_schema_v2_tests.py::TestSchemaCaseInsensitive::test_schema_lowercase_sql": 85.17666384699987, "tests/functional/schema_tests/test_schema_v2_tests.py::TestSchemaCaseInsensitive::test_schema_uppercase_sql": 59.38734428299904, "tests/functional/schema_tests/test_schema_v2_tests.py::TestSchemaTestContext::test_test_context_tests": 88.95719864700004, "tests/functional/schema_tests/test_schema_v2_tests.py::TestSchemaTestContextWhereSubq::test_test_context_tests": 90.77868880499773, "tests/functional/schema_tests/test_schema_v2_tests.py::TestSchemaTestContextWithMacroNamespace::test_test_context_with_macro_namespace": 86.15032830300152, "tests/functional/schema_tests/test_schema_v2_tests.py::TestSchemaTestNameCollision::test_collision_test_names_get_hash": 88.17785598999944, "tests/functional/schema_tests/test_schema_v2_tests.py::TestSchemaTestStoreFailuresFalseHierarchicalParsing::test_parse_store_failures_False_as_ephemeral": 50.428101429999515, "tests/functional/schema_tests/test_schema_v2_tests.py::TestSchemaTestStoreFailuresFalseParsing::test_parse_store_failures_False_as_ephemeral": 59.855075478997605, "tests/functional/schema_tests/test_schema_v2_tests.py::TestSchemaTestStoreFailuresTrueHierarchicalParsing::test_parse_store_failures_True_as_table": 62.23745480099933, "tests/functional/schema_tests/test_schema_v2_tests.py::TestSchemaTestStoreFailuresTrueParsing::test_parse_store_failures_True_as_table": 62.89721251100127, "tests/functional/schema_tests/test_schema_v2_tests.py::TestSchemaTests::test_schema_test_exclude_failures": 79.41838097099935, "tests/functional/schema_tests/test_schema_v2_tests.py::TestSchemaTests::test_schema_test_selection": 103.27117732000079, "tests/functional/schema_tests/test_schema_v2_tests.py::TestSchemaTests::test_schema_tests": 78.74014660999819, "tests/functional/schema_tests/test_schema_v2_tests.py::TestWrongSpecificationBlock::test_wrong_specification_block": 57.86932584399983, "tests/functional/schema_tests/test_sql_header_config.py::TestGenericDataTestSqlHeader::test_generic_test_sql_header_in_config": 4.531009937999954, "tests/functional/schema_tests/test_sql_header_config.py::TestGenericDataTestSqlHeaderDeprecation::test_sql_header_deprecation_warning": 2.883126579999953, "tests/functional/schema_tests/test_sql_header_config.py::TestSingularDataTestSqlHeader::test_singular_test_sql_header_in_config": 4.509966480000031, "tests/functional/seeds/test_seed_column_type_validation.py::TestSeedColumnTypeValidation::test_seed_with_invalid_column_type": 51.2292724859999, "tests/functional/seeds/test_seed_column_type_validation.py::TestSeedColumnTypesBasic::test_seed_basic_column_types": 50.926622969000164, "tests/functional/seeds/test_seed_column_type_validation.py::TestSeedColumnTypesWithMismatchedColumns::test_seed_invalid_column_type_warning": 50.93689672000073, "tests/functional/selected_resources/test_selected_resources.py::TestSelectedResources::test_selected_resources_build_no_model": 53.66736683400086, "tests/functional/selected_resources/test_selected_resources.py::TestSelectedResources::test_selected_resources_build_no_selector": 54.28850211899953, "tests/functional/selected_resources/test_selected_resources.py::TestSelectedResources::test_selected_resources_build_selector": 78.13948855799754, "tests/functional/selected_resources/test_selected_resources.py::TestSelectedResources::test_selected_resources_build_selector_subgraph": 7.200015626000095, "tests/functional/selected_resources/test_selected_resources.py::TestSelectedResources::test_selected_resources_run": 53.82875841199893, "tests/functional/selected_resources/test_selected_resources.py::TestSelectedResources::test_selected_resources_test_no_model": 53.650677725001515, "tests/functional/selectors/test_default_selectors.py::TestDefaultSelectors::test_model__compile": 26.289786381998056, "tests/functional/selectors/test_default_selectors.py::TestDefaultSelectors::test_model__list": 1.909087535000026, "tests/functional/selectors/test_default_selectors.py::TestDefaultSelectors::test_source__freshness": 52.77334636299747, "tests/functional/semantic_models/test_semantic_model_configs.py::TestConfigProjectLevel::test_project_level": 107.87842063700009, "tests/functional/semantic_models/test_semantic_model_configs.py::TestConfigYamlLevel::test_yaml_level": 52.55584869499944, "tests/functional/semantic_models/test_semantic_model_configs.py::TestConfigsInheritence::test_project_plus_yaml_level": 53.62267729099949, "tests/functional/semantic_models/test_semantic_model_configs.py::TestDisabledConfigYamlLevelEnabledMetric::test_yaml_level": 52.42467463399953, "tests/functional/semantic_models/test_semantic_model_configs.py::TestMetaConfig::test_meta_config": 53.81915100100014, "tests/functional/semantic_models/test_semantic_model_configs.py::TestMetaConfigClobbering::test_meta_config_clobbering": 54.51973961499971, "tests/functional/semantic_models/test_semantic_model_configs.py::TestMetaConfigForComponents::test_component_meta_configs": 54.46486899999945, "tests/functional/semantic_models/test_semantic_model_configs.py::TestMismatchesConfigProjectLevel::test_project_level": 79.81348215199978, "tests/functional/semantic_models/test_semantic_model_parsing.py::TestSemanticModelParsingErrors::test_semantic_model_error": 54.541676401000586, "tests/functional/semantic_models/test_semantic_model_parsing.py::TestSemanticModelParsingForConversionMetrics::test_conversion_metric_parsing": 54.95357054500164, "tests/functional/semantic_models/test_semantic_model_parsing.py::TestSemanticModelParsingForCumulativeMetrics::test_cumulative_metric_parsing": 54.55886833600198, "tests/functional/semantic_models/test_semantic_model_parsing.py::TestSemanticModelParsingForDerivedMetrics::test_derived_metric_parsing": 2.266845902, "tests/functional/semantic_models/test_semantic_model_parsing.py::TestSemanticModelParsingForRatioMetrics::test_ratio_metric_parsing": 55.11394714800008, "tests/functional/semantic_models/test_semantic_model_parsing.py::TestSemanticModelParsingWorks::test_semantic_model_parsing": 55.46526735899897, "tests/functional/semantic_models/test_semantic_model_parsing.py::TestSemanticModelPartialParsingGeneratedMetrics::test_generated_metrics": 85.9822180889987, "tests/functional/semantic_models/test_semantic_model_parsing.py::TestSemanticModelPartialParsingWithModelChanged::test_semantic_model_changed_partial_parsing": 2.5982234340000048, "tests/functional/semantic_models/test_semantic_model_parsing.py::TestSemanticModelPartialParsingWithModelDeletedIteratively::test_semantic_model_deleted_partial_parsing": 4.450409737000001, "tests/functional/semantic_models/test_semantic_model_parsing.py::TestSemanticModelPartialParsingWithModelFlippingCreateMetric::test_semantic_model_flipping_create_metric_partial_parsing": 116.06417013899954, "tests/functional/semantic_models/test_semantic_model_v2_parsing.py::TestConversionMetricNoBaseMetricFails::test_conversion_metric_no_base_metric_parsing_fails": 58.117637699000625, "tests/functional/semantic_models/test_semantic_model_v2_parsing.py::TestCumulativeMetricNoInputMetricFails::test_cumulative_metric_no_input_metric_parsing_fails": 58.761033264001526, "tests/functional/semantic_models/test_semantic_model_v2_parsing.py::TestDerivedMetricWithInputMetricsFilterDimensionJinja::test_input_metrics_filter_jinja_not_rendered": 2.15318450700002, "tests/functional/semantic_models/test_semantic_model_v2_parsing.py::TestDerivedSemanticsParsingWorks::test_derived_semantics_parsing": 60.010480524999366, "tests/functional/semantic_models/test_semantic_model_v2_parsing.py::TestDerivedSemanticsWithDocJinjaParsingWorks::test_derived_semantics_doc_jinja_parsing": 59.43265880900253, "tests/functional/semantic_models/test_semantic_model_v2_parsing.py::TestMetricHiddenMapsToIsPrivate::test_metric_hidden_yaml_maps_to_is_private": 62.697041161000016, "tests/functional/semantic_models/test_semantic_model_v2_parsing.py::TestMetricOnModelParsingWorks::test_metric_on_model_parsing": 58.188376295000126, "tests/functional/semantic_models/test_semantic_model_v2_parsing.py::TestMetricOnModelWithCustomSemanticModelName::test_metrics_use_custom_semantic_model_name": 58.923796237000715, "tests/functional/semantic_models/test_semantic_model_v2_parsing.py::TestMetricOnModelWithoutCustomSemanticModelName::test_metrics_use_model_name_as_semantic_model_name": 60.50950170999931, "tests/functional/semantic_models/test_semantic_model_v2_parsing.py::TestRatioMetricWithNumeratorFilterDimensionJinja::test_numerator_filter_jinja_not_rendered": 59.21252180999909, "tests/functional/semantic_models/test_semantic_model_v2_parsing.py::TestSemanticModelConfigDefaultValues::test_semantic_model_parsing_defaults": 56.969397938999464, "tests/functional/semantic_models/test_semantic_model_v2_parsing.py::TestSemanticModelConfigDoesNotExistPassesWithoutParsingSemanticModel::test_semantic_model_parsing": 57.2150586129992, "tests/functional/semantic_models/test_semantic_model_v2_parsing.py::TestSemanticModelConfigOverrides::test_semantic_model_parsing": 57.09212184199896, "tests/functional/semantic_models/test_semantic_model_v2_parsing.py::TestSemanticModelDisabledConfigIsNotParsed::test_semantic_model_parsing": 57.497695056998054, "tests/functional/semantic_models/test_semantic_model_v2_parsing.py::TestSemanticModelFalseConfigIsNotParsed::test_semantic_model_parsing": 57.99841862699941, "tests/functional/semantic_models/test_semantic_model_v2_parsing.py::TestSemanticModelParsingWorks::test_semantic_model_parsing": 57.226948847999665, "tests/functional/semantic_models/test_semantic_model_v2_parsing.py::TestSemanticModelWithPrimaryEntityOnlyOnColumn::test_primary_entity_type_is_id_entity": 58.07909940200079, "tests/functional/semantic_models/test_semantic_model_v2_parsing.py::TestSemanticModelWithPrimaryEntityOnlyOnModel::test_primary_entities_empty": 1.6640911149999624, "tests/functional/semantic_models/test_semantic_model_v2_parsing.py::TestSimpleSemanticModelWithFilterWithFilterDimensionJinja::test_simple_metric_with_filter_with_filter_dimension_jinja_parsing": 58.12395009099964, "tests/functional/semantic_models/test_semantic_model_v2_parsing.py::TestSimpleSemanticModelWithMetricWithDocJinja::test_simple_metric_with_doc_jinja_parsing": 58.23097795800277, "tests/functional/semantic_models/test_semantic_model_v2_parsing.py::TestStandaloneMetricParsingSimpleMetricFails::test_standalone_metric_parsing": 58.324286273000325, "tests/functional/semantic_models/test_semantic_model_v2_parsing.py::TestStandaloneMetricParsingWorks::test_included_metric_parsing": 60.19740994200038, "tests/functional/semantic_models/test_semantic_model_v2_parsing.py::TestTopLevelSemanticsMetricWithDocJinja::test_top_level_metric_with_doc_jinja_parsing": 58.45057118499972, "tests/functional/semantic_models/test_semantic_models.py::TestSemanticModelDependsOn::test_depends_on": 66.67487448200154, "tests/functional/semantic_models/test_semantic_models.py::TestSemanticModelNestedDocs::test_depends_on": 71.11769534000086, "tests/functional/semantic_models/test_semantic_models.py::TestSemanticModelPartialParsing::test_semantic_model_deleted_partial_parsing": 93.21674462900046, "tests/functional/semantic_models/test_semantic_models.py::TestSemanticModelUnknownModel::test_unknown_model_raises_issue": 62.31814279300124, "tests/functional/severity/test_severity.py::TestSeverity::test_generic_default": 130.218340368001, "tests/functional/severity/test_severity.py::TestSeverity::test_generic_strict": 31.606054487998335, "tests/functional/severity/test_severity.py::TestSeverity::test_singular_default": 31.275115849999565, "tests/functional/severity/test_severity.py::TestSeverity::test_singular_strict": 31.3837903929998, "tests/functional/show/test_show.py::TestShowEphemeral::test_ephemeral_model": 4.770717013999985, "tests/functional/show/test_show.py::TestShowInline::test_inline_pass": 6.029213085999999, "tests/functional/show/test_show.py::TestShowInline::test_inline_pass_quiet": 11.347032066999986, "tests/functional/show/test_show.py::TestShowInlineDirect::test_inline_direct_pass": 4.049902450000019, "tests/functional/show/test_show.py::TestShowInlineDirect::test_inline_direct_pass_no_limit": 61.50709874000131, "tests/functional/show/test_show.py::TestShowInlineDirect::test_inline_direct_pass_quiet": 5.852711741000036, "tests/functional/show/test_show.py::TestShowInlineDirectFail::test_inline_fail_database_error": 124.03751825400104, "tests/functional/show/test_show.py::TestShowInlineFail::test_inline_fail": 8.619599016000052, "tests/functional/show/test_show.py::TestShowInlineFailDB::test_inline_fail_database_error": 106.22149159899891, "tests/functional/show/test_show.py::TestShowModelVersions::test_none": 4.215083011000019, "tests/functional/show/test_show.py::TestShowModelVersions::test_version_unspecified": 92.36847730299996, "tests/functional/show/test_show.py::TestShowMultiple::test_select_multiple_model_text": 125.00628722700094, "tests/functional/show/test_show.py::TestShowNone::test_none": 95.32680707100008, "tests/functional/show/test_show.py::TestShowNumeric::test_numeric_values": 4.443285828000029, "tests/functional/show/test_show.py::TestShowNumericNulls::test_numeric_values_with_nulls": 5.245788030999989, "tests/functional/show/test_show.py::TestShowPrivateModel::test_version_unspecified": 91.5191681999986, "tests/functional/show/test_show.py::TestShowSecondEphemeral::test_second_ephemeral_model": 121.40478020699993, "tests/functional/show/test_show.py::TestShowSeed::test_seed": 96.91076028799944, "tests/functional/show/test_show.py::TestShowSelectText::test_select_model_text": 4.6676790470000356, "tests/functional/show/test_show.py::TestShowSingle::test_select_single_model_json": 128.0744835969981, "tests/functional/show/test_show.py::TestShowSingle::test_select_single_model_json_quiet": 10.623800169999981, "tests/functional/snapshots/test_basic_snapshot.py::TestBasicCheckCols::test_basic_snapshot": 172.89177853699948, "tests/functional/snapshots/test_basic_snapshot.py::TestBasicConfiguredCheckCols::test_configured_snapshot": 181.45321795200107, "tests/functional/snapshots/test_basic_snapshot.py::TestBasicCustomNamespace::test_custom_namespace_snapshot": 175.5347446980013, "tests/functional/snapshots/test_basic_snapshot.py::TestBasicCustomSnapshot::test_custom_snapshot": 173.7060016659998, "tests/functional/snapshots/test_basic_snapshot.py::TestBasicRef::test_basic_ref": 91.49162472399985, "tests/functional/snapshots/test_basic_snapshot.py::TestBasicSnapshot::test_basic_snapshot": 156.3946062310024, "tests/functional/snapshots/test_basic_snapshot.py::TestBasicSnapshotYaml::test_basic_snapshot_yaml": 177.77988468200056, "tests/functional/snapshots/test_basic_snapshot.py::TestBasicTargetSchemaConfig::test_target_schema": 101.70318581700121, "tests/functional/snapshots/test_basic_snapshot.py::TestBasicUpdatedAtCheckCols::test_updated_at_snapshot": 178.51089828399745, "tests/functional/snapshots/test_basic_snapshot.py::TestRefCheckCols::test_check_cols_ref": 104.2334418290011, "tests/functional/snapshots/test_basic_snapshot.py::TestRefConfiguredCheckCols::test_configured_ref": 109.90203640600157, "tests/functional/snapshots/test_basic_snapshot.py::TestRefCustomNamespace::test_custom_namespace_ref": 101.03401411400046, "tests/functional/snapshots/test_basic_snapshot.py::TestRefCustomSnapshot::test_custom_ref": 100.64102044499668, "tests/functional/snapshots/test_basic_snapshot.py::TestRefUpdatedAtCheckCols::test_updated_at_ref": 7.190862178000003, "tests/functional/snapshots/test_basic_snapshot.py::TestYamlSnapshotCompiledPath::test_compiled_path": 3.4207499540000015, "tests/functional/snapshots/test_basic_snapshot.py::TestYamlSnapshotPartialParsing::test_snapshot_partial_parsing": 174.00180803200237, "tests/functional/snapshots/test_changing_check_cols_snapshot.py::test_check_cols_snapshot_with_schema_change": 176.6575279610006, "tests/functional/snapshots/test_changing_check_cols_snapshot.py::test_check_cols_snapshot_with_schema_change_and_mismatched_casing": 179.87189774700164, "tests/functional/snapshots/test_changing_strategy_snapshot.py::test_changing_strategy": 185.9991333229991, "tests/functional/snapshots/test_check_cols_snapshot.py::test_snapshots": 196.49388197100052, "tests/functional/snapshots/test_check_cols_updated_at_snapshot.py::test_snapshots": 179.94832851899992, "tests/functional/snapshots/test_comment_ending_snapshot.py::TestSnapshotsWithCommentAtEnd::test_comment_ending": 102.8516368460023, "tests/functional/snapshots/test_cross_schema_snapshot.py::test_cross_schema_snapshot": 90.97828131400092, "tests/functional/snapshots/test_hard_delete_snapshot.py::test_snapshot_hard_delete": 127.98918425499869, "tests/functional/snapshots/test_invalid_namespace_snapshot.py::test_custom_snapshot_invalid_namespace": 59.70147741200162, "tests/functional/snapshots/test_long_text_snapshot.py::test_long_text": 5.868144018000066, "tests/functional/snapshots/test_missing_strategy_snapshot.py::test_missing_strategy": 59.14913008400072, "tests/functional/snapshots/test_renamed_source_snapshot.py::test_renamed_source": 127.87966662799954, "tests/functional/snapshots/test_select_exclude_snapshot.py::TestAllBasic::test_all_snapshots": 94.00559290299861, "tests/functional/snapshots/test_select_exclude_snapshot.py::TestConfigured::test_all_configured_snapshots": 95.86023996599943, "tests/functional/snapshots/test_select_exclude_snapshot.py::TestConfiguredExclude::test_exclude_configured_snapshots": 2.914373089999998, "tests/functional/snapshots/test_select_exclude_snapshot.py::TestConfiguredSelect::test_select_configured_snapshots": 61.21773660799954, "tests/functional/snapshots/test_select_exclude_snapshot.py::TestExcludeBasic::test_exclude_snapshots": 61.27532777599845, "tests/functional/snapshots/test_select_exclude_snapshot.py::TestSelectBasic::test_select_snapshots": 61.48745696199876, "tests/functional/snapshots/test_slow_query_snapshot.py::test_slow": 137.10596646999875, "tests/functional/snapshots/test_snapshot_column_names.py::TestSnapshotColumnNames::test_snapshot_column_names": 108.5104148950013, "tests/functional/snapshots/test_snapshot_column_names.py::TestSnapshotColumnNamesFromDbtProject::test_snapshot_column_names_from_project": 101.97196625699871, "tests/functional/snapshots/test_snapshot_column_names.py::TestSnapshotInvalidColumnNames::test_snapshot_invalid_column_names": 101.64264548700157, "tests/functional/snapshots/test_snapshot_config.py::TestSnapshotConfig::test_config": 5.242415402000006, "tests/functional/snapshots/test_snapshot_empty.py::TestSnapshotEmpty::test_check": 171.2846070789983, "tests/functional/snapshots/test_snapshot_timestamps.py::TestSnapshotConfig::test_timestamp_snapshot": 100.34509776199957, "tests/functional/source_overrides/test_simple_source_override.py::TestSourceOverride::test_source_overrides": 294.4178194999986, "tests/functional/source_overrides/test_source_overrides_duplicate_model.py::TestSourceOverrideDuplicates::test_source_duplicate_overrides": 73.99079210500167, "tests/functional/sources/test_name_chars.py::TestNameChars::test_quotes_in_table_names": 3.0891765860001215, "tests/functional/sources/test_simple_source.py::TestBasicSource::test_basic_source_def": 146.38359287000094, "tests/functional/sources/test_simple_source.py::TestEmptySource::test_empty_source_def": 105.42350240900123, "tests/functional/sources/test_simple_source.py::TestMalformedSources::test_malformed_schema_will_break_run": 2.489531840000012, "tests/functional/sources/test_simple_source.py::TestRenderingInSourceTests::test_render_in_source_tests": 123.0788706459989, "tests/functional/sources/test_simple_source.py::TestSourceChildrenParents::test_source_childrens_parents": 92.32687404799981, "tests/functional/sources/test_simple_source.py::TestSourceDef::test_source_only_def": 127.93349742100145, "tests/functional/sources/test_simple_source.py::TestSourceRunOperation::test_run_operation_source": 94.54722535599831, "tests/functional/sources/test_simple_source.py::TestSourceSelector::test_source_selector": 285.9180681670023, "tests/functional/sources/test_simple_source.py::TestUnquotedSources::test_catalog": 126.18702396299886, "tests/functional/sources/test_source_configs.py::TestConfigYamlSourceLevel::test_source_config_yaml_source_level": 60.67554415400082, "tests/functional/sources/test_source_configs.py::TestConfigYamlSourceTable::test_source_config_yaml_source_table": 60.86672854400058, "tests/functional/sources/test_source_configs.py::TestInvalidSourceConfig::test_invalid_config_source": 69.09736205399895, "tests/functional/sources/test_source_configs.py::TestSourceConfigsInheritence1::test_source_all_configs_source_table": 64.43113302399979, "tests/functional/sources/test_source_configs.py::TestSourceConfigsInheritence2::test_source_two_configs_source_level": 2.7013750989999608, "tests/functional/sources/test_source_configs.py::TestSourceConfigsInheritence3::test_source_two_configs_source_table": 70.98819002899836, "tests/functional/sources/test_source_configs.py::TestSourceEnabledConfigProjectLevel::test_enabled_source_config_dbt_project": 3.186755081000001, "tests/functional/sources/test_source_configs.py::TestSourceLoadedAtFieldConfigLevel::test_loaded_at_field_config": 67.27026138899964, "tests/functional/sources/test_source_configs.py::TestSourceLoadedAtFieldTopLevel::test_loaded_at_field_config": 68.57178109000051, "tests/functional/sources/test_source_configs.py::TestSourceLoadedAtQueryConfigLevel::test_loaded_at_field_config": 68.42528675200265, "tests/functional/sources/test_source_configs.py::TestSourceLoadedAtQueryTopLevel::test_loaded_at_query_config": 68.70141765400149, "tests/functional/sources/test_source_configs.py::TestTableLoadedAtFieldConfigLevel::test_loaded_at_field_config": 68.30800699800056, "tests/functional/sources/test_source_configs.py::TestTableLoadedAtFieldTopLevel::test_loaded_at_field_config": 71.25413081100123, "tests/functional/sources/test_source_configs.py::TestTableLoadedAtQueryConfigLevel::test_loaded_at_field_config": 71.64398513199922, "tests/functional/sources/test_source_configs.py::TestTableLoadedAtQueryNoneWhenFieldSetConfigLevel::test_loaded_at_query_config": 69.18590104799841, "tests/functional/sources/test_source_configs.py::TestTableLoadedAtQueryNoneWhenFieldSetTopLevel::test_loaded_at_query_config": 73.25590512099916, "tests/functional/sources/test_source_configs.py::TestTableLoadedAtQueryTopLevel::test_loaded_at_query_config": 65.88884472599966, "tests/functional/sources/test_source_fresher_state.py::TestSourceFresherBuild::test_source_fresher_build_error": 271.57097632600016, "tests/functional/sources/test_source_fresher_state.py::TestSourceFresherBuild::test_source_fresher_build_pass": 188.58403688100225, "tests/functional/sources/test_source_fresher_state.py::TestSourceFresherBuild::test_source_fresher_build_warn": 191.41722029599987, "tests/functional/sources/test_source_fresher_state.py::TestSourceFresherBuildResultSelectors::test_source_fresher_build_state_modified_pass": 220.9543690240007, "tests/functional/sources/test_source_fresher_state.py::TestSourceFresherBuildStateModified::test_source_fresher_build_state_modified_pass": 12.583257703999948, "tests/functional/sources/test_source_fresher_state.py::TestSourceFresherNoCurrentState::test_intentional_failure_no_previous_state": 183.0358722639994, "tests/functional/sources/test_source_fresher_state.py::TestSourceFresherNoPreviousState::test_intentional_failure_no_previous_state": 147.83519282899942, "tests/functional/sources/test_source_fresher_state.py::TestSourceFresherNothingToDo::test_source_fresher_nothing_to_do": 212.98931883800105, "tests/functional/sources/test_source_fresher_state.py::TestSourceFresherRun::test_source_fresher_run_error": 263.0144736120001, "tests/functional/sources/test_source_fresher_state.py::TestSourceFresherRun::test_source_fresher_run_pass": 11.265075439000043, "tests/functional/sources/test_source_fresher_state.py::TestSourceFresherRun::test_source_fresher_run_warn": 185.07422031900023, "tests/functional/sources/test_source_fresher_state.py::TestSourceFresherRuntimeError::test_runtime_error_states": 16.039504268, "tests/functional/sources/test_source_fresher_state.py::TestSourceFresherTest::test_source_fresher_run_error": 251.44055135300187, "tests/functional/sources/test_source_fresher_state.py::TestSourceFresherTest::test_source_fresher_test_pass": 12.874040330000014, "tests/functional/sources/test_source_fresher_state.py::TestSourceFresherTest::test_source_fresher_test_warn": 192.31558216200028, "tests/functional/sources/test_source_freshness.py::TestHooksInSourceFreshness::test_hooks_do_run_for_source_freshness": 107.76506222799799, "tests/functional/sources/test_source_freshness.py::TestHooksInSourceFreshnessDefault::test_hooks_do_not_run_for_source_freshness": 107.34578038799918, "tests/functional/sources/test_source_freshness.py::TestHooksInSourceFreshnessDisabled::test_hooks_do_not_run_for_source_freshness": 109.31360336499893, "tests/functional/sources/test_source_freshness.py::TestHooksInSourceFreshnessError::test_hooks_do_not_run_for_source_freshness": 70.278961592001, "tests/functional/sources/test_source_freshness.py::TestMetadataFreshnessFails::test_metadata_freshness_unsupported_error_when_run": 3.4563143910002054, "tests/functional/sources/test_source_freshness.py::TestMetadataFreshnessFails::test_metadata_freshness_unsupported_parse_warning": 71.99134183299975, "tests/functional/sources/test_source_freshness.py::TestOverrideSourceFreshness::test_override_source_freshness": 111.01182990599955, "tests/functional/sources/test_source_freshness.py::TestSourceFreshness::test_source_freshness": 182.2602920069985, "tests/functional/sources/test_source_freshness.py::TestSourceFreshnessCustomSQL::test_source_freshness_custom_sql": 111.69172881000122, "tests/functional/sources/test_source_freshness.py::TestSourceFreshnessCustomSQLConfig::test_source_freshness_custom_sql": 115.34302136000042, "tests/functional/sources/test_source_freshness.py::TestSourceFreshnessCustomSQLSourceConfig::test_source_freshness_custom_sql": 111.58725815400066, "tests/functional/sources/test_source_freshness.py::TestSourceFreshnessErrors::test_source_freshness_error": 113.25015244299902, "tests/functional/sources/test_source_freshness.py::TestSourceFreshnessExclude::test_source_freshness_selection_exclude": 142.63405745500313, "tests/functional/sources/test_source_freshness.py::TestSourceFreshnessExplicitNullInSource::test_source_freshness_explicit_null_in_source": 4.322688091000003, "tests/functional/sources/test_source_freshness.py::TestSourceFreshnessExplicitNullInTable::test_source_freshness_explicit_null_in_table": 119.63315542500095, "tests/functional/sources/test_source_freshness.py::TestSourceFreshnessFilter::test_source_freshness_all_records": 183.64217327999904, "tests/functional/sources/test_source_freshness.py::TestSourceFreshnessGraph::test_source_freshness_selection_graph_operation": 124.6483720819997, "tests/functional/sources/test_source_freshness.py::TestSourceFreshnessMacroOverride::test_source_freshness": 109.51772196400088, "tests/functional/sources/test_source_freshness.py::TestSourceFreshnessProjectHooksNotRun::test_hooks_do_run_for_source_freshness": 109.01173660400127, "tests/functional/sources/test_source_freshness.py::TestSourceFreshnessSelection::test_source_freshness_selection_select": 119.06506106000234, "tests/functional/sources/test_source_freshness.py::TestSourceSnapshotFreshness::test_source_snapshot_freshness": 183.7045984460001, "tests/functional/sources/test_source_loaded_at_field.py::TestParsingLoadedAtField::test_loaded_at_field": 7.697236137999994, "tests/functional/statements/test_statements.py::TestStatements::test_duplicated_load_statements": 69.434938907998, "tests/functional/statements/test_statements.py::TestStatements::test_load_statement_on_main_twice": 7.6714405959999965, "tests/functional/statements/test_statements.py::TestStatements::test_postgres_statements": 4.5747430640000175, "tests/functional/test_empty.py::TestEmptyFlag::test_run_with_empty": 16.17108102900005, "tests/functional/test_project.py::TestGenericJsonSchemaValidationDeprecation::test_project": 0.7298283330000004, "tests/functional/test_project.py::TestProjectJsonschemaValidatedOnlyOnce::test_project": 2.9781766890000085, "tests/functional/test_selection/test_selection_expansion.py::TestExpansionWithSelectors::test_all_tests_no_specifiers": 141.95065429699935, "tests/functional/test_selection/test_selection_expansion.py::TestExpansionWithSelectors::test_exclude_column_config_level_tag": 119.2211366340016, "tests/functional/test_selection/test_selection_expansion.py::TestExpansionWithSelectors::test_exclude_column_level_tag": 115.20878721900226, "tests/functional/test_selection/test_selection_expansion.py::TestExpansionWithSelectors::test_exclude_data_test_tag": 9.653047574000027, "tests/functional/test_selection/test_selection_expansion.py::TestExpansionWithSelectors::test_exclude_model_b": 10.316360483000011, "tests/functional/test_selection/test_selection_expansion.py::TestExpansionWithSelectors::test_model_a_alone": 106.81006993200026, "tests/functional/test_selection/test_selection_expansion.py::TestExpansionWithSelectors::test_model_a_exclude_specific_test": 106.01301548300034, "tests/functional/test_selection/test_selection_expansion.py::TestExpansionWithSelectors::test_model_a_exclude_specific_test_buildable": 108.3335359749999, "tests/functional/test_selection/test_selection_expansion.py::TestExpansionWithSelectors::test_model_a_exclude_specific_test_cautious": 106.31086144499932, "tests/functional/test_selection/test_selection_expansion.py::TestExpansionWithSelectors::test_model_a_indirect_selection": 8.845961178999971, "tests/functional/test_selection/test_selection_expansion.py::TestExpansionWithSelectors::test_model_a_indirect_selection_buildable": 138.03266729499592, "tests/functional/test_selection/test_selection_expansion.py::TestExpansionWithSelectors::test_model_a_indirect_selection_cautious": 130.2742062180041, "tests/functional/test_selection/test_selection_expansion.py::TestExpansionWithSelectors::test_model_a_indirect_selection_eager": 112.07709329900172, "tests/functional/test_selection/test_selection_expansion.py::TestExpansionWithSelectors::test_model_a_indirect_selection_empty": 41.55653875000098, "tests/functional/test_selection/test_selection_expansion.py::TestExpansionWithSelectors::test_model_a_indirect_selection_exclude_unique_tests": 10.017910887, "tests/functional/test_selection/test_selection_expansion.py::TestExpansionWithSelectors::test_model_a_model_b": 105.78964403699865, "tests/functional/test_selection/test_selection_expansion.py::TestExpansionWithSelectors::test_model_a_only_singular": 106.76126584200028, "tests/functional/test_selection/test_selection_expansion.py::TestExpansionWithSelectors::test_model_a_only_singular_cautious": 105.15465966299962, "tests/functional/test_selection/test_selection_expansion.py::TestExpansionWithSelectors::test_model_a_only_singular_eager": 106.75551636300224, "tests/functional/test_selection/test_selection_expansion.py::TestExpansionWithSelectors::test_model_a_only_singular_unset": 106.46711188099835, "tests/functional/test_selection/test_selection_expansion.py::TestExpansionWithSelectors::test_model_a_sources": 107.09708416100148, "tests/functional/test_selection/test_selection_expansion.py::TestExpansionWithSelectors::test_model_tag_test_name_intersection": 135.82457860499744, "tests/functional/test_selection/test_selection_expansion.py::TestExpansionWithSelectors::test_only_generic": 141.4812073539979, "tests/functional/test_selection/test_selection_expansion.py::TestExpansionWithSelectors::test_only_singular": 106.70284191899918, "tests/functional/test_selection/test_selection_expansion.py::TestExpansionWithSelectors::test_select_column_config_level_tag": 134.20065341100053, "tests/functional/test_selection/test_selection_expansion.py::TestExpansionWithSelectors::test_select_column_level_tag": 123.04018047400314, "tests/functional/test_selection/test_selection_expansion.py::TestExpansionWithSelectors::test_select_source_column_config_level_tag": 128.33545254800083, "tests/functional/test_selection/test_selection_expansion.py::TestExpansionWithSelectors::test_selector_model_a_buildable_indirect_selection": 111.39159516799918, "tests/functional/test_selection/test_selection_expansion.py::TestExpansionWithSelectors::test_selector_model_a_cautious_indirect_selection": 118.25005439899905, "tests/functional/test_selection/test_selection_expansion.py::TestExpansionWithSelectors::test_selector_model_a_eager_indirect_selection": 111.31943825100097, "tests/functional/test_selection/test_selection_expansion.py::TestExpansionWithSelectors::test_selector_model_a_unset_indirect_selection": 7.305125508000003, "tests/functional/test_selection/test_selection_expansion.py::TestExpansionWithSelectors::test_test_level_tag": 119.76356488300007, "tests/functional/test_selection/test_selection_expansion.py::TestExpansionWithSelectors::test_test_name_intersection": 114.56226429299932, "tests/functional/test_selection/test_selection_expansion.py::TestSelectionExpansion::test_all_tests_no_specifiers": 145.75069392099795, "tests/functional/test_selection/test_selection_expansion.py::TestSelectionExpansion::test_exclude_column_config_level_tag": 106.40668184999959, "tests/functional/test_selection/test_selection_expansion.py::TestSelectionExpansion::test_exclude_column_level_tag": 10.017908178999988, "tests/functional/test_selection/test_selection_expansion.py::TestSelectionExpansion::test_exclude_data_test_tag": 107.11427396499857, "tests/functional/test_selection/test_selection_expansion.py::TestSelectionExpansion::test_exclude_model_b": 6.252533097999958, "tests/functional/test_selection/test_selection_expansion.py::TestSelectionExpansion::test_model_a_alone": 111.18456728799902, "tests/functional/test_selection/test_selection_expansion.py::TestSelectionExpansion::test_model_a_exclude_specific_test": 104.05563873400388, "tests/functional/test_selection/test_selection_expansion.py::TestSelectionExpansion::test_model_a_exclude_specific_test_buildable": 109.03073197399863, "tests/functional/test_selection/test_selection_expansion.py::TestSelectionExpansion::test_model_a_exclude_specific_test_cautious": 108.75725234399943, "tests/functional/test_selection/test_selection_expansion.py::TestSelectionExpansion::test_model_a_indirect_selection": 107.01362749700093, "tests/functional/test_selection/test_selection_expansion.py::TestSelectionExpansion::test_model_a_indirect_selection_buildable": 108.86712989899752, "tests/functional/test_selection/test_selection_expansion.py::TestSelectionExpansion::test_model_a_indirect_selection_cautious": 147.8018395729996, "tests/functional/test_selection/test_selection_expansion.py::TestSelectionExpansion::test_model_a_indirect_selection_eager": 113.41239822299758, "tests/functional/test_selection/test_selection_expansion.py::TestSelectionExpansion::test_model_a_indirect_selection_empty": 35.97894609500145, "tests/functional/test_selection/test_selection_expansion.py::TestSelectionExpansion::test_model_a_indirect_selection_exclude_unique_tests": 7.849806000000029, "tests/functional/test_selection/test_selection_expansion.py::TestSelectionExpansion::test_model_a_model_b": 107.81440000299881, "tests/functional/test_selection/test_selection_expansion.py::TestSelectionExpansion::test_model_a_only_singular": 9.938911569999988, "tests/functional/test_selection/test_selection_expansion.py::TestSelectionExpansion::test_model_a_only_singular_cautious": 5.51844506000009, "tests/functional/test_selection/test_selection_expansion.py::TestSelectionExpansion::test_model_a_only_singular_eager": 108.42982389500139, "tests/functional/test_selection/test_selection_expansion.py::TestSelectionExpansion::test_model_a_only_singular_unset": 106.49227770000107, "tests/functional/test_selection/test_selection_expansion.py::TestSelectionExpansion::test_model_a_sources": 8.371015986999993, "tests/functional/test_selection/test_selection_expansion.py::TestSelectionExpansion::test_model_tag_test_name_intersection": 106.67572221400042, "tests/functional/test_selection/test_selection_expansion.py::TestSelectionExpansion::test_only_generic": 109.30028827100068, "tests/functional/test_selection/test_selection_expansion.py::TestSelectionExpansion::test_only_singular": 146.10128680199887, "tests/functional/test_selection/test_selection_expansion.py::TestSelectionExpansion::test_select_column_config_level_tag": 106.68014139099978, "tests/functional/test_selection/test_selection_expansion.py::TestSelectionExpansion::test_select_column_level_tag": 108.33668895600022, "tests/functional/test_selection/test_selection_expansion.py::TestSelectionExpansion::test_select_source_column_config_level_tag": 105.74667675299861, "tests/functional/test_selection/test_selection_expansion.py::TestSelectionExpansion::test_test_level_tag": 106.38651585599837, "tests/functional/test_selection/test_selection_expansion.py::TestSelectionExpansion::test_test_name_intersection": 106.9687094639994, "tests/functional/test_singular_tests.py::TestSingularTestWarnError::test_singular_test_equals_warn_error": 2.6309978649999834, "tests/functional/test_singular_tests.py::TestSingularTestWarnError::test_singular_test_warn_error": 2.868404886999997, "tests/functional/test_singular_tests.py::TestSingularTestWarnError::test_singular_test_warn_error_options": 1.2043703990000267, "tests/functional/threading/test_thread_count.py::TestThreadCount::test_threading_8x": 76.87732999600121, "tests/functional/time_spines/test_time_spines.py::TestMissingTimeSpine::test_time_spines": 78.73946257500211, "tests/functional/time_spines/test_time_spines.py::TestTimeSpineCustomColumnMissing::test_time_spines": 2.7540323009999383, "tests/functional/time_spines/test_time_spines.py::TestTimeSpineGranularityMissing::test_time_spines": 74.27478839300238, "tests/functional/time_spines/test_time_spines.py::TestTimeSpineStandardColumnMissing::test_time_spines": 75.46766035800101, "tests/functional/time_spines/test_time_spines.py::TestValidLegacyTimeSpine::test_time_spines": 77.84324479399947, "tests/functional/time_spines/test_time_spines.py::TestValidTimeSpines::test_time_spines": 78.52267215500251, "tests/functional/timezones/test_timezones.py::TestTimezones::test_run_started_at": 74.4921795759983, "tests/functional/unit_testing/test_csv_fixtures.py::TestUnitTestsDuplicateCSVFile::test_duplicate": 78.21052728200084, "tests/functional/unit_testing/test_csv_fixtures.py::TestUnitTestsFileCSVEmptyValueIsNull::test_unit_test": 135.5470799370014, "tests/functional/unit_testing/test_csv_fixtures.py::TestUnitTestsInlineCSVEmptyValueIsNull::test_unit_test": 136.62697998799922, "tests/functional/unit_testing/test_csv_fixtures.py::TestUnitTestsMissingCSVFile::test_missing": 2.3512065299999847, "tests/functional/unit_testing/test_csv_fixtures.py::TestUnitTestsWithFileCSV::test_unit_test": 11.741537405000031, "tests/functional/unit_testing/test_csv_fixtures.py::TestUnitTestsWithInlineCSV::test_unit_test": 209.6276906710009, "tests/functional/unit_testing/test_csv_fixtures.py::TestUnitTestsWithMixedCSV::test_unit_test": 204.17008768099913, "tests/functional/unit_testing/test_sql_format.py::TestSQLFormat::test_sql_format": 93.2033532020032, "tests/functional/unit_testing/test_sql_format.py::TestSQLFormatFixtures::test_sql_format_fixtures": 97.35245845700047, "tests/functional/unit_testing/test_state.py::TestUnitTestDeferDoesntOverwrite::test_unit_test_defer_state": 128.1361204720015, "tests/functional/unit_testing/test_state.py::TestUnitTestDeferState::test_unit_test_defer_state": 115.19832038999448, "tests/functional/unit_testing/test_state.py::TestUnitTestRetry::test_unit_test_retry": 150.30434907900053, "tests/functional/unit_testing/test_state.py::TestUnitTestStateModified::test_state_modified": 10.637249088000004, "tests/functional/unit_testing/test_unit_testing.py::TestUnitTestExplicitSeed::test_explicit_seed": 5.671921933999954, "tests/functional/unit_testing/test_unit_testing.py::TestUnitTestExternalPackageNode::test_unit_test_ext_nodes": 149.9469526540015, "tests/functional/unit_testing/test_unit_testing.py::TestUnitTestExternalProjectNode::test_unit_test_ext_nodes": 149.2960233570011, "tests/functional/unit_testing/test_unit_testing.py::TestUnitTestImplicitSeed::test_implicit_seed": 151.22677824200218, "tests/functional/unit_testing/test_unit_testing.py::TestUnitTestIncrementalModelBasic::test_basic": 4.994694784000103, "tests/functional/unit_testing/test_unit_testing.py::TestUnitTestIncrementalModelNoOverride::test_no_override": 73.79452759800188, "tests/functional/unit_testing/test_unit_testing.py::TestUnitTestIncrementalModelNoThisInput::test_no_this_input": 87.73784661399986, "tests/functional/unit_testing/test_unit_testing.py::TestUnitTestIncrementalModelWithAlias::test_basic": 119.83722876900356, "tests/functional/unit_testing/test_unit_testing.py::TestUnitTestIncrementalModelWithVersion::test_basic": 111.91778518799765, "tests/functional/unit_testing/test_unit_testing.py::TestUnitTestIncrementalModelWrongOverride::test_str_override": 2.4922150539999848, "tests/functional/unit_testing/test_unit_testing.py::TestUnitTestInvalidInputConfiguration::test_invalid_input_configuration": 148.5784819019973, "tests/functional/unit_testing/test_unit_testing.py::TestUnitTestModelWithFunction::test_model_with_function": 111.5390882319989, "tests/functional/unit_testing/test_unit_testing.py::TestUnitTestNonexistentSeed::test_nonexistent_seed": 75.89844797499973, "tests/functional/unit_testing/test_unit_testing.py::TestUnitTestRefMissingVersionModel::test_basic": 110.59977783200156, "tests/functional/unit_testing/test_unit_testing.py::TestUnitTestRefWithMissingVersionRef::test_basic": 110.90968626700123, "tests/functional/unit_testing/test_unit_testing.py::TestUnitTestRefWithVersion::test_basic": 111.82149820599443, "tests/functional/unit_testing/test_unit_testing.py::TestUnitTestRefWithVersionDiffLatest::test_basic": 111.34391832700203, "tests/functional/unit_testing/test_unit_testing.py::TestUnitTestRefWithVersionLatestSecond::test_basic": 109.30726277699796, "tests/functional/unit_testing/test_unit_testing.py::TestUnitTestRefWithVersionMissingRefTest::test_basic": 111.00499829599721, "tests/functional/unit_testing/test_unit_testing.py::TestUnitTestSubfolderPath::test_subfolder_unit_test": 74.49816818399995, "tests/functional/unit_testing/test_unit_testing.py::TestUnitTests::test_basic": 584.8818125420039, "tests/functional/unit_testing/test_ut_adapter_hooks.py::TestUnitTestAdapterPostHook::test_unit_test_runs_adapter_post_hook_fails": 112.7039411859987, "tests/functional/unit_testing/test_ut_adapter_hooks.py::TestUnitTestAdapterPostHook::test_unit_test_runs_adapter_post_hook_pass": 4.077167386000042, "tests/functional/unit_testing/test_ut_adapter_hooks.py::TestUnitTestAdapterPreHook::test_unit_test_runs_adapter_pre_hook_fails": 112.3166918930001, "tests/functional/unit_testing/test_ut_adapter_hooks.py::TestUnitTestAdapterPreHook::test_unit_test_runs_adapter_pre_hook_passes": 4.740222096999901, "tests/functional/unit_testing/test_ut_aliases.py::TestUnitTestInputWithAlias::test_input_with_alias": 111.69323447400166, "tests/functional/unit_testing/test_ut_dependency.py::TestUnitTestingInDependency::test_unit_test_in_dependency": 264.8418194640035, "tests/functional/unit_testing/test_ut_diffing.py::TestUnitTestingDiffIsOrderAgnostic::test_unit_testing_diff_is_order_insensitive": 115.13311531499858, "tests/functional/unit_testing/test_ut_ephemeral.py::TestUnitTestEphemeralInput::test_ephemeral_input": 5.880983423999993, "tests/functional/unit_testing/test_ut_list.py::TestUnitTestList::test_unit_test_list": 268.0300040109978, "tests/functional/unit_testing/test_ut_list.py::TestUnitTestListDisabled::test_disabled_unit_tests": 75.15607246699801, "tests/functional/unit_testing/test_ut_macros.py::TestMacroComposition::test_macro_composition_in_unit_test": 112.98634323999795, "tests/functional/unit_testing/test_ut_macros.py::TestMacroWithoutComposition::test_macro_in_unit_test": 5.1601897329999815, "tests/functional/unit_testing/test_ut_names.py::TestUnitTestDuplicateTestNamesAcrossModels::test_duplicate_test_names_across_models": 9.289804296000028, "tests/functional/unit_testing/test_ut_names.py::TestUnitTestDuplicateTestNamesWithinModel::test_duplicate_test_names_within_model": 75.4444905829987, "tests/functional/unit_testing/test_ut_overrides.py::TestUnitTestingMacroOverrides::test_macro_overrides": 75.99408836699877, "tests/functional/unit_testing/test_ut_resource_types.py::TestUnitTestResourceTypes::test_unit_test_list": 616.0432517519985, "tests/functional/unit_testing/test_ut_snapshot_dependency.py::TestUnitTestSnapshotDependency::test_snapshot_dependency": 282.3528574000011, "tests/functional/unit_testing/test_ut_sources.py::TestUnitTestSourceInput::test_source_input": 243.52973640099663, "tests/functional/unit_testing/test_ut_sources.py::TestUnitTestSourceInputSameNames::test_source_input_same_names": 5.345504815999988, "tests/functional/unit_testing/test_ut_variables.py::TestUnitTestOneVariables::test_one_variable_as_input_to_macro": 42.851800518001255, "tests/functional/unit_testing/test_ut_variables.py::TestUnitTestTwoVariables::test_two_variables_as_input_to_macro": 119.55644928000038, "tests/functional/unit_testing/test_ut_versions.py::TestIncludeExcludeSpecified::test_include_exclude_specified": 79.48441015700155, "tests/functional/unit_testing/test_ut_versions.py::TestIncludeUnversioned::test_include_unversioned": 1.6705306679999694, "tests/functional/unit_testing/test_ut_versions.py::TestVersionedFixture::test_versioned_fixture": 66.60719303299993, "tests/functional/unit_testing/test_ut_versions.py::TestVersions::test_versions": 233.78311553499952 } ================================================ FILE: core/LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2021 dbt Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: core/README.md ================================================ <p align="center"> <img src="https://raw.githubusercontent.com/dbt-labs/dbt-core/fa1ea14ddfb1d5ae319d5141844910dd53ab2834/docs/images/dbt-core.svg" alt="dbt logo" width="750"/> </p> <p align="center"> <a href="https://github.com/dbt-labs/dbt-core/actions/workflows/main.yml"> <img src="https://github.com/dbt-labs/dbt-core/actions/workflows/main.yml/badge.svg?event=push" alt="CI Badge"/> </a> </p> **[dbt](https://www.getdbt.com/)** enables data analysts and engineers to transform their data using the same practices that software engineers use to build applications. ![architecture](https://raw.githubusercontent.com/dbt-labs/dbt-core/6c6649f9129d5d108aa3b0526f634cd8f3a9d1ed/docs/images/dbt-arch.png) ## Understanding dbt Analysts using dbt can transform their data by simply writing select statements, while dbt handles turning these statements into tables and views in a data warehouse. These select statements, or "models", form a dbt project. Models frequently build on top of one another – dbt makes it easy to [manage relationships](https://docs.getdbt.com/docs/ref) between models, and [visualize these relationships](https://docs.getdbt.com/docs/documentation), as well as assure the quality of your transformations through [testing](https://docs.getdbt.com/docs/testing). ![dbt dag](https://raw.githubusercontent.com/dbt-labs/dbt-core/6c6649f9129d5d108aa3b0526f634cd8f3a9d1ed/docs/images/dbt-dag.png) ## Getting started - [Install dbt](https://docs.getdbt.com/docs/installation) - Read the [introduction](https://docs.getdbt.com/docs/introduction/) and [viewpoint](https://docs.getdbt.com/docs/about/viewpoint/) ## Join the dbt Community - Be part of the conversation in the [dbt Community Slack](http://community.getdbt.com/) - Read more on the [dbt Community Discourse](https://discourse.getdbt.com) ## Reporting bugs and contributing code - Want to report a bug or request a feature? Let us know on [Slack](http://community.getdbt.com/), or open [an issue](https://github.com/dbt-labs/dbt-core/issues/new) - Want to help us build dbt? Check out the [Contributing Guide](https://github.com/dbt-labs/dbt-core/blob/HEAD/CONTRIBUTING.md) ## Code of Conduct Everyone interacting in the dbt project's codebases, issue trackers, chat rooms, and mailing lists is expected to follow the [dbt Code of Conduct](https://community.getdbt.com/code-of-conduct). ================================================ FILE: core/dbt/README.md ================================================ # core/dbt directory README ## The following are individual files in this directory. ### compilation.py ### constants.py ### dataclass_schema.py ### deprecations.py ### exceptions.py ### flags.py ### helper_types.py ### hooks.py ### lib.py ### links.py ### main.py ### node_types.py ### profiler.py ### selected_resources.py ### semver.py ### tracking.py ### ui.py ### utils.py ### version.py ## The subdirectories will be documented in a README in the subdirectory * adapters * cli * clients * config * context * contracts * deps * docs * events * graph * include * parser * task * tests ================================================ FILE: core/dbt/__init__.py ================================================ # N.B. # This will add to the package’s __path__ all subdirectories of directories on sys.path named after the package which effectively combines both modules into a single namespace (dbt.adapters) # The matching statement is in plugins/postgres/dbt/__init__.py from pkgutil import extend_path __path__ = extend_path(__path__, __name__) ================================================ FILE: core/dbt/__version__.py ================================================ version = "1.12.0a1" ================================================ FILE: core/dbt/_pydantic_shim.py ================================================ # type: ignore """Shim to allow support for both Pydantic 1 and Pydantic 2. dbt-core must support both major versions of Pydantic because dbt-core users might be using an environment with either version, and we can't restrict them to one or the other. Here, we essentially import all Pydantic objects from version 1 that we use. Throughout the repo, we import these objects from this file instead of from Pydantic directly, meaning that we essentially only use Pydantic 1 in dbt-core currently, but without forcing that restriction on dbt users. The development environment for this repo should be pinned to Pydantic 1 to ensure devs get appropriate type hints. """ from importlib.metadata import version pydantic_version = version("pydantic") # Pydantic uses semantic versioning, i.e. <major>.<minor>.<patch>, and we need to know the major pydantic_major = pydantic_version.split(".")[0] if pydantic_major == "1": from pydantic import BaseSettings # noqa: F401 elif pydantic_major == "2": from pydantic.v1 import BaseSettings # noqa: F401 else: raise RuntimeError( f"Currently only pydantic 1 and 2 are supported, found pydantic {pydantic_version}" ) ================================================ FILE: core/dbt/artifacts/__init__.py ================================================ ================================================ FILE: core/dbt/artifacts/exceptions/__init__.py ================================================ from dbt.artifacts.exceptions.schemas import IncompatibleSchemaError ================================================ FILE: core/dbt/artifacts/exceptions/schemas.py ================================================ from typing import Optional from dbt_common.exceptions import DbtRuntimeError class IncompatibleSchemaError(DbtRuntimeError): def __init__(self, expected: str, found: Optional[str] = None) -> None: self.expected = expected self.found = found self.filename = "input file" super().__init__(msg=self.get_message()) def add_filename(self, filename: str): self.filename = filename self.msg = self.get_message() def get_message(self) -> str: found_str = "nothing" if self.found is not None: found_str = f'"{self.found}"' msg = ( f'Expected a schema version of "{self.expected}" in ' f"{self.filename}, but found {found_str}. Are you running with a " f"different version of dbt?" ) return msg CODE = 10014 MESSAGE = "Incompatible Schema" ================================================ FILE: core/dbt/artifacts/resources/__init__.py ================================================ from dbt.artifacts.resources.base import BaseResource, Docs, FileHash, GraphResource from dbt.artifacts.resources.v1.analysis import Analysis from dbt.artifacts.resources.v1.catalog import Catalog, CatalogWriteIntegrationConfig # alias to latest resource definitions from dbt.artifacts.resources.v1.components import ( ColumnConfig, ColumnDimension, ColumnEntity, ColumnInfo, CompiledResource, Contract, DeferRelation, DependsOn, FreshnessThreshold, HasRelationMetadata, InjectedCTE, NodeVersion, ParsedResource, ParsedResourceMandatory, Quoting, RefArgs, Time, ) from dbt.artifacts.resources.v1.config import ( Hook, NodeAndTestConfig, NodeConfig, TestConfig, list_str, metas, ) from dbt.artifacts.resources.v1.documentation import Documentation from dbt.artifacts.resources.v1.exposure import ( Exposure, ExposureConfig, ExposureType, MaturityType, ) from dbt.artifacts.resources.v1.function import ( DeferFunction, Function, FunctionArgument, FunctionConfig, FunctionMandatory, FunctionReturns, ) from dbt.artifacts.resources.v1.generic_test import GenericTest, TestMetadata from dbt.artifacts.resources.v1.group import Group, GroupConfig from dbt.artifacts.resources.v1.hook import HookNode from dbt.artifacts.resources.v1.macro import ( Macro, MacroArgument, MacroConfig, MacroDependsOn, ) from dbt.artifacts.resources.v1.metric import ( ConstantPropertyInput, ConversionTypeParams, CumulativeTypeParams, Metric, MetricAggregationParams, MetricConfig, MetricInput, MetricInputMeasure, MetricTimeWindow, MetricTypeParams, ) from dbt.artifacts.resources.v1.model import ( CustomGranularity, Model, ModelConfig, ModelFreshness, TimeSpine, ) from dbt.artifacts.resources.v1.owner import Owner from dbt.artifacts.resources.v1.saved_query import ( Export, ExportConfig, QueryParams, SavedQuery, SavedQueryConfig, SavedQueryMandatory, ) from dbt.artifacts.resources.v1.seed import Seed, SeedConfig from dbt.artifacts.resources.v1.semantic_layer_components import ( FileSlice, MeasureAggregationParameters, NonAdditiveDimension, SourceFileMetadata, WhereFilter, WhereFilterIntersection, ) from dbt.artifacts.resources.v1.semantic_model import ( Defaults, Dimension, DimensionTypeParams, DimensionValidityParams, Entity, Measure, NodeRelation, SemanticLayerElementConfig, SemanticModel, SemanticModelConfig, ) from dbt.artifacts.resources.v1.singular_test import SingularTest from dbt.artifacts.resources.v1.snapshot import Snapshot, SnapshotConfig from dbt.artifacts.resources.v1.source_definition import ( ExternalPartition, ExternalTable, ParsedSourceMandatory, SourceConfig, SourceDefinition, ) from dbt.artifacts.resources.v1.sql_operation import SqlOperation from dbt.artifacts.resources.v1.unit_test_definition import ( UnitTestConfig, UnitTestDefinition, UnitTestFormat, UnitTestInputFixture, UnitTestNodeVersions, UnitTestOutputFixture, UnitTestOverrides, ) ================================================ FILE: core/dbt/artifacts/resources/base.py ================================================ import hashlib from dataclasses import dataclass from typing import List, Optional from dbt.artifacts.resources.types import NodeType from dbt_common.dataclass_schema import dbtClassMixin @dataclass class BaseResource(dbtClassMixin): name: str resource_type: NodeType package_name: str path: str original_file_path: str unique_id: str @dataclass class GraphResource(BaseResource): fqn: List[str] @dataclass class FileHash(dbtClassMixin): name: str # the hash type name checksum: str # the hashlib.hash_type().hexdigest() of the file contents @classmethod def empty(cls): return FileHash(name="none", checksum="") @classmethod def path(cls, path: str): return FileHash(name="path", checksum=path) def __eq__(self, other): if not isinstance(other, FileHash): return NotImplemented if self.name == "none" or self.name != other.name: return False return self.checksum == other.checksum def compare(self, contents: str) -> bool: """Compare the file contents with the given hash""" if self.name == "none": return False return self.from_contents(contents, name=self.name) == self.checksum @classmethod def from_contents(cls, contents: str, name="sha256") -> "FileHash": """Create a file hash from the given file contents. The hash is always the utf-8 encoding of the contents given, because dbt only reads files as utf-8. """ data = contents.encode("utf-8") checksum = hashlib.new(name, data).hexdigest() return cls(name=name, checksum=checksum) @dataclass class Docs(dbtClassMixin): show: bool = True node_color: Optional[str] = None ================================================ FILE: core/dbt/artifacts/resources/types.py ================================================ from dbt_common.dataclass_schema import StrEnum class AccessType(StrEnum): Private = "private" Protected = "protected" Public = "public" @classmethod def is_valid(cls, item): try: cls(item) except ValueError: return False return True class NodeType(StrEnum): Model = "model" Analysis = "analysis" Test = "test" # renamed to 'data_test'; preserved as 'test' here for back-compat Snapshot = "snapshot" Operation = "operation" Seed = "seed" # TODO: rm? RPCCall = "rpc" SqlOperation = "sql_operation" Documentation = "doc" Source = "source" Macro = "macro" Exposure = "exposure" Metric = "metric" Group = "group" SavedQuery = "saved_query" SemanticModel = "semantic_model" Unit = "unit_test" Fixture = "fixture" Function = "function" def pluralize(self) -> str: if self is self.Analysis: return "analyses" elif self is self.SavedQuery: return "saved_queries" elif self is self.Test: return "data_tests" return f"{self}s" class RunHookType(StrEnum): Start = "on-run-start" End = "on-run-end" class ModelLanguage(StrEnum): python = "python" sql = "sql" class ModelHookType(StrEnum): PreHook = "pre-hook" PostHook = "post-hook" class TimePeriod(StrEnum): minute = "minute" hour = "hour" day = "day" def plural(self) -> str: return str(self) + "s" class BatchSize(StrEnum): hour = "hour" day = "day" month = "month" year = "year" def plural(self) -> str: return str(self) + "s" class FunctionType(StrEnum): Scalar = "scalar" Aggregate = "aggregate" Table = "table" class FunctionVolatility(StrEnum): Deterministic = "deterministic" Stable = "stable" NonDeterministic = "non-deterministic" ================================================ FILE: core/dbt/artifacts/resources/v1/analysis.py ================================================ from dataclasses import dataclass from typing import Literal from dbt.artifacts.resources.types import NodeType from dbt.artifacts.resources.v1.components import CompiledResource @dataclass class Analysis(CompiledResource): resource_type: Literal[NodeType.Analysis] ================================================ FILE: core/dbt/artifacts/resources/v1/catalog.py ================================================ from dataclasses import dataclass, field from typing import Any, Dict, List, Optional from dbt.adapters.catalogs import CatalogIntegrationConfig from dbt_common.dataclass_schema import dbtClassMixin @dataclass class CatalogWriteIntegrationConfig(CatalogIntegrationConfig): name: str catalog_type: str external_volume: Optional[str] = None table_format: Optional[str] = None catalog_name: Optional[str] = None file_format: Optional[str] = None adapter_properties: Dict[str, Any] = field(default_factory=dict) @dataclass class Catalog(dbtClassMixin): name: str active_write_integration: Optional[str] = None write_integrations: List[CatalogWriteIntegrationConfig] = field(default_factory=list) ================================================ FILE: core/dbt/artifacts/resources/v1/components.py ================================================ import time from dataclasses import dataclass, field from datetime import timedelta from typing import Any, Dict, List, Optional, Union from dbt.artifacts.resources.base import Docs, FileHash, GraphResource from dbt.artifacts.resources.types import NodeType, TimePeriod from dbt.artifacts.resources.v1.config import NodeConfig from dbt_common.contracts.config.base import BaseConfig, MergeBehavior from dbt_common.contracts.config.properties import AdditionalPropertiesMixin from dbt_common.contracts.constraints import ColumnLevelConstraint from dbt_common.contracts.util import Mergeable from dbt_common.dataclass_schema import ExtensibleDbtClassMixin, dbtClassMixin from dbt_semantic_interfaces.type_enums import ( DimensionType, EntityType, TimeGranularity, ) NodeVersion = Union[str, float] def _backcompat_doc_blocks(doc_blocks: Any) -> List[str]: """ Make doc_blocks backwards-compatible for scenarios where a user specifies `doc_blocks` on a model or column. Mashumaro will raise a serialization error if the specified `doc_blocks` isn't a list of strings. In such a scenario, this method returns an empty list to avoid a serialization error. Further along, `_get_doc_blocks` in `manifest.py` populates the correct `doc_blocks` for the happy path. """ if isinstance(doc_blocks, list) and all(isinstance(x, str) for x in doc_blocks): return doc_blocks return [] @dataclass class MacroDependsOn(dbtClassMixin): macros: List[str] = field(default_factory=list) # 'in' on lists is O(n) so this is O(n^2) for # of macros def add_macro(self, value: str): if value not in self.macros: self.macros.append(value) @dataclass class DependsOn(MacroDependsOn): nodes: List[str] = field(default_factory=list) def add_node(self, value: str): if value not in self.nodes: self.nodes.append(value) @dataclass class RefArgs(dbtClassMixin): name: str package: Optional[str] = None version: Optional[NodeVersion] = None @property def positional_args(self) -> List[str]: if self.package: return [self.package, self.name] else: return [self.name] @property def keyword_args(self) -> Dict[str, Optional[NodeVersion]]: if self.version: return {"version": self.version} else: return {} @dataclass class ColumnConfig(BaseConfig): meta: Dict[str, Any] = field(default_factory=dict, metadata=MergeBehavior.Update.meta()) tags: List[str] = field(default_factory=list) @dataclass class ColumnDimension(dbtClassMixin): """Used for column-based dimensions for Semantic Models""" @dataclass class ColumnDimensionValidityParams(dbtClassMixin): is_start: bool = False is_end: bool = False name: str type: DimensionType description: Optional[str] = None label: Optional[str] = None is_partition: bool = False config: Dict[str, Any] = field(default_factory=dict) validity_params: Optional[ColumnDimensionValidityParams] = None @dataclass class ColumnEntity(dbtClassMixin): name: str type: EntityType description: Optional[str] = None label: Optional[str] = None config: Dict[str, Any] = field(default_factory=dict) @dataclass class ColumnInfo(AdditionalPropertiesMixin, ExtensibleDbtClassMixin): """Used in all ManifestNodes and SourceDefinition""" name: str description: str = "" meta: Dict[str, Any] = field(default_factory=dict) data_type: Optional[str] = None constraints: List[ColumnLevelConstraint] = field(default_factory=list) quote: Optional[bool] = None config: ColumnConfig = field(default_factory=ColumnConfig) tags: List[str] = field(default_factory=list) _extra: Dict[str, Any] = field(default_factory=dict) granularity: Optional[TimeGranularity] = None dimension: Union[ColumnDimension, DimensionType, None] = None entity: Union[ColumnEntity, EntityType, None] = None doc_blocks: List[str] = field(default_factory=list) def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None) -> dict: dct = super().__post_serialize__(dct, context) dct["doc_blocks"] = _backcompat_doc_blocks(dct["doc_blocks"]) return dct @dataclass class InjectedCTE(dbtClassMixin): """Used in CompiledNodes as part of ephemeral model processing""" id: str sql: str @dataclass class Contract(dbtClassMixin): enforced: bool = False alias_types: bool = True checksum: Optional[str] = None @dataclass class Quoting(dbtClassMixin, Mergeable): database: Optional[bool] = None schema: Optional[bool] = None identifier: Optional[bool] = None column: Optional[bool] = None @dataclass class Time(dbtClassMixin, Mergeable): count: Optional[int] = None period: Optional[TimePeriod] = None def exceeded(self, actual_age: float) -> bool: if self.period is None or self.count is None: return False kwargs: Dict[str, int] = {self.period.plural(): self.count} difference = timedelta(**kwargs).total_seconds() return actual_age > difference def __bool__(self): return self.count is not None and self.period is not None @dataclass class FreshnessThreshold(dbtClassMixin, Mergeable): warn_after: Optional[Time] = field(default_factory=Time) error_after: Optional[Time] = field(default_factory=Time) filter: Optional[str] = None def status(self, age: float) -> "dbt.artifacts.schemas.results.FreshnessStatus": # type: ignore # noqa F821 from dbt.artifacts.schemas.results import FreshnessStatus if self.error_after and self.error_after.exceeded(age): return FreshnessStatus.Error elif self.warn_after and self.warn_after.exceeded(age): return FreshnessStatus.Warn else: return FreshnessStatus.Pass def __bool__(self): return bool(self.warn_after) or bool(self.error_after) @dataclass class HasRelationMetadata(dbtClassMixin): database: Optional[str] schema: str # Can't set database to None like it ought to be # because it messes up the subclasses and default parameters # so hack it here @classmethod def __pre_deserialize__(cls, data): data = super().__pre_deserialize__(data) if "database" not in data: data["database"] = None return data @property def quoting_dict(self) -> Dict[str, bool]: if hasattr(self, "quoting"): return self.quoting.to_dict(omit_none=True) else: return {} @dataclass class DeferRelation(HasRelationMetadata): alias: str relation_name: Optional[str] # The rest of these fields match RelationConfig protocol exactly resource_type: NodeType name: str description: str compiled_code: Optional[str] meta: Dict[str, Any] tags: List[str] config: Optional[NodeConfig] @property def identifier(self): return self.alias @dataclass class ParsedResourceMandatory(GraphResource, HasRelationMetadata): alias: str checksum: FileHash config: NodeConfig = field(default_factory=NodeConfig) @property def identifier(self): return self.alias @dataclass class ParsedResource(ParsedResourceMandatory): tags: List[str] = field(default_factory=list) description: str = field(default="") columns: Dict[str, ColumnInfo] = field(default_factory=dict) meta: Dict[str, Any] = field(default_factory=dict) group: Optional[str] = None docs: Docs = field(default_factory=Docs) patch_path: Optional[str] = None build_path: Optional[str] = None unrendered_config: Dict[str, Any] = field(default_factory=dict) created_at: float = field(default_factory=lambda: time.time()) config_call_dict: Dict[str, Any] = field(default_factory=dict) unrendered_config_call_dict: Dict[str, Any] = field(default_factory=dict) relation_name: Optional[str] = None raw_code: str = "" doc_blocks: List[str] = field(default_factory=list) def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None): dct = super().__post_serialize__(dct, context) if context and context.get("artifact") and "config_call_dict" in dct: del dct["config_call_dict"] if context and context.get("artifact") and "unrendered_config_call_dict" in dct: del dct["unrendered_config_call_dict"] dct["doc_blocks"] = _backcompat_doc_blocks(dct["doc_blocks"]) return dct @dataclass class CompiledResource(ParsedResource): """Contains attributes necessary for SQL files and nodes with refs, sources, etc, so all ManifestNodes except SeedNode.""" language: str = "sql" refs: List[RefArgs] = field(default_factory=list) sources: List[List[str]] = field(default_factory=list) metrics: List[List[str]] = field(default_factory=list) functions: List[List[str]] = field(default_factory=list) depends_on: DependsOn = field(default_factory=DependsOn) compiled_path: Optional[str] = None compiled: bool = False compiled_code: Optional[str] = None extra_ctes_injected: bool = False extra_ctes: List[InjectedCTE] = field(default_factory=list) _pre_injected_sql: Optional[str] = None contract: Contract = field(default_factory=Contract) def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None): dct = super().__post_serialize__(dct, context) if "_pre_injected_sql" in dct: del dct["_pre_injected_sql"] # Remove compiled attributes if "compiled" in dct and dct["compiled"] is False: del dct["compiled"] del dct["extra_ctes_injected"] del dct["extra_ctes"] # "omit_none" means these might not be in the dictionary if "compiled_code" in dct: del dct["compiled_code"] return dct ================================================ FILE: core/dbt/artifacts/resources/v1/config.py ================================================ import re from dataclasses import dataclass, field from typing import Any, Dict, List, Optional, Union from mashumaro.jsonschema.annotations import Pattern from typing_extensions import Annotated from dbt import hooks from dbt.artifacts.resources.base import Docs from dbt.artifacts.resources.types import ModelHookType from dbt.artifacts.utils.validation import validate_color from dbt.flags import get_flags from dbt_common.contracts.config.base import BaseConfig, CompareBehavior, MergeBehavior from dbt_common.contracts.config.materialization import OnConfigurationChangeOption from dbt_common.contracts.config.metadata import Metadata, ShowBehavior from dbt_common.dataclass_schema import ValidationError, dbtClassMixin def list_str() -> List[str]: return [] class Severity(str): pass def metas(*metas: Metadata) -> Dict[str, Any]: existing: Dict[str, Any] = {} for m in metas: existing = m.meta(existing) return existing @dataclass class ContractConfig(dbtClassMixin): enforced: bool = False alias_types: bool = True @dataclass class Hook(dbtClassMixin): sql: str transaction: bool = True index: Optional[int] = None @dataclass class NodeAndTestConfig(BaseConfig): enabled: bool = True # these fields are included in serialized output, but are not part of # config comparison (they are part of database_representation) alias: Optional[str] = field( default=None, metadata=CompareBehavior.Exclude.meta(), ) schema: Optional[str] = field( default=None, metadata=CompareBehavior.Exclude.meta(), ) database: Optional[str] = field( default=None, metadata=CompareBehavior.Exclude.meta(), ) tags: Union[List[str], str] = field( default_factory=list_str, metadata=metas(ShowBehavior.Hide, MergeBehavior.Append, CompareBehavior.Exclude), ) meta: Dict[str, Any] = field( default_factory=dict, metadata=MergeBehavior.Update.meta(), ) group: Optional[str] = field( default=None, metadata=CompareBehavior.Exclude.meta(), ) @dataclass class NodeConfig(NodeAndTestConfig): # Note: if any new fields are added with MergeBehavior, also update the # 'mergebehavior' dictionary materialized: str = "view" incremental_strategy: Optional[str] = None batch_size: Any = None lookback: Any = 1 begin: Any = None persist_docs: Dict[str, Any] = field(default_factory=dict) post_hook: List[Hook] = field( default_factory=list, metadata={"merge": MergeBehavior.Append, "alias": "post-hook"}, ) pre_hook: List[Hook] = field( default_factory=list, metadata={"merge": MergeBehavior.Append, "alias": "pre-hook"}, ) quoting: Dict[str, Any] = field( default_factory=dict, metadata=MergeBehavior.Update.meta(), ) # This is actually only used by seeds. Should it be available to others? # That would be a breaking change! column_types: Dict[str, Any] = field( default_factory=dict, metadata=MergeBehavior.Update.meta(), ) full_refresh: Optional[bool] = None # 'unique_key' doesn't use 'Optional' because typing.get_type_hints was # sometimes getting the Union order wrong, causing serialization failures. unique_key: Union[str, List[str], None] = None on_schema_change: Optional[str] = "ignore" on_configuration_change: OnConfigurationChangeOption = field( default_factory=OnConfigurationChangeOption.default ) grants: Dict[str, Any] = field( default_factory=dict, metadata=MergeBehavior.DictKeyAppend.meta() ) packages: List[str] = field( default_factory=list, metadata=MergeBehavior.Append.meta(), ) docs: Docs = field( default_factory=Docs, metadata=MergeBehavior.Update.meta(), ) contract: ContractConfig = field( default_factory=ContractConfig, metadata=MergeBehavior.Update.meta(), ) event_time: Any = None concurrent_batches: Any = None def __post_init__(self): # we validate that node_color has a suitable value to prevent dbt-docs from crashing if self.docs.node_color: node_color = self.docs.node_color if not validate_color(node_color): raise ValidationError( f"Invalid color name for docs.node_color: {node_color}. " "It is neither a valid HTML color name nor a valid HEX code." ) if ( self.contract.enforced and self.materialized == "incremental" and self.on_schema_change not in ("append_new_columns", "fail") ): raise ValidationError( f"Invalid value for on_schema_change: {self.on_schema_change}. Models " "materialized as incremental with contracts enabled must set " "on_schema_change to 'append_new_columns' or 'fail'" ) @classmethod def __pre_deserialize__(cls, data): data = super().__pre_deserialize__(data) for key in ModelHookType: if key in data: data[key] = [hooks.get_hook_dict(h) for h in data[key]] return data SEVERITY_PATTERN = r"^([Ww][Aa][Rr][Nn]|[Ee][Rr][Rr][Oo][Rr])$" @dataclass class TestConfig(NodeAndTestConfig): __test__ = False # this is repeated because of a different default schema: Optional[str] = field( default="dbt_test__audit", metadata=CompareBehavior.Exclude.meta(), ) materialized: str = "test" # Annotated is used by mashumaro for jsonschema generation severity: Annotated[Severity, Pattern(SEVERITY_PATTERN)] = Severity("ERROR") store_failures: Optional[bool] = None store_failures_as: Optional[str] = None sql_header: Any = None where: Optional[str] = None limit: Optional[int] = None fail_calc: str = "count(*)" warn_if: str = "!= 0" error_if: str = "!= 0" def finalize_and_validate(self): """ The presence of a setting for `store_failures_as` overrides any existing setting for `store_failures`, regardless of level of granularity. If `store_failures_as` is not set, then `store_failures` takes effect. At the time of implementation, `store_failures = True` would always create a table; the user could not configure this. Hence, if `store_failures = True` and `store_failures_as` is not specified, then it should be set to "table" to mimic the existing functionality. A side effect of this overriding functionality is that `store_failures_as="view"` at the project level cannot be turned off at the model level without setting both `store_failures_as` and `store_failures`. The former would cascade down and override `store_failures=False`. The proposal is to include "ephemeral" as a value for `store_failures_as`, which effectively sets `store_failures=False`. The exception handling for this is tricky. If we raise an exception here, the entire run fails at parse time. We would rather well-formed models run successfully, leaving only exceptions to be rerun if necessary. Hence, the exception needs to be raised in the test materialization. In order to do so, we need to make sure that we go down the `store_failures = True` route with the invalid setting for `store_failures_as`. This results in the `.get()` defaulted to `True` below, instead of a normal dictionary lookup as is done in the `if` block. Refer to the test materialization for the exception that is raise as a result of an invalid value. The intention of this block is to behave as if `store_failures_as` is the only setting, but still allow for backwards compatibility for `store_failures`. See https://github.com/dbt-labs/dbt-core/issues/6914 for more information. """ super().finalize_and_validate() # if `store_failures_as` is not set, it gets set by `store_failures` # the settings below mimic existing behavior prior to `store_failures_as` get_store_failures_as_map = { True: "table", False: "ephemeral", None: None, } # if `store_failures_as` is set, it dictates what `store_failures` gets set to # the settings below overrides whatever `store_failures` is set to by the user get_store_failures_map = { "ephemeral": False, "table": True, "view": True, } if self.store_failures_as is None: self.store_failures_as = get_store_failures_as_map[self.store_failures] else: self.store_failures = get_store_failures_map.get(self.store_failures_as, True) return self @classmethod def same_contents(cls, unrendered: Dict[str, Any], other: Dict[str, Any]) -> bool: """This is like __eq__, except it explicitly checks certain fields.""" modifiers = [ "severity", "where", "limit", "fail_calc", "warn_if", "error_if", "store_failures", "store_failures_as", "sql_header", ] seen = set() for _, target_name in cls._get_fields(): key = target_name seen.add(key) if key in modifiers: if not cls.compare_key(unrendered, other, key): return False return True @classmethod def validate(cls, data): if data.get("severity") and not re.match(SEVERITY_PATTERN, data.get("severity")): raise ValidationError( f"Severity must be either 'warn' or 'error'. Got '{data.get('severity')}'" ) super().validate(data) if data.get("materialized") and data.get("materialized") != "test": raise ValidationError("A test must have a materialized value of 'test'") sql_header = data.get("sql_header") if sql_header is not None and get_flags().require_sql_header_in_test_configs: if not isinstance(sql_header, str): raise ValidationError( f"sql_header must be a string. Got '{type(sql_header).__name__}'" ) ================================================ FILE: core/dbt/artifacts/resources/v1/documentation.py ================================================ from dataclasses import dataclass from typing import Literal from dbt.artifacts.resources.base import BaseResource from dbt.artifacts.resources.types import NodeType @dataclass class Documentation(BaseResource): resource_type: Literal[NodeType.Documentation] block_contents: str ================================================ FILE: core/dbt/artifacts/resources/v1/exposure.py ================================================ import time from dataclasses import dataclass, field from typing import Any, Dict, List, Literal, Optional from dbt.artifacts.resources.base import GraphResource from dbt.artifacts.resources.types import NodeType from dbt.artifacts.resources.v1.components import DependsOn, RefArgs from dbt.artifacts.resources.v1.owner import Owner from dbt_common.contracts.config.base import BaseConfig from dbt_common.dataclass_schema import StrEnum class ExposureType(StrEnum): Dashboard = "dashboard" Notebook = "notebook" Analysis = "analysis" ML = "ml" Application = "application" class MaturityType(StrEnum): Low = "low" Medium = "medium" High = "high" @dataclass class ExposureConfig(BaseConfig): enabled: bool = True tags: List[str] = field(default_factory=list) meta: Dict[str, Any] = field(default_factory=dict) @dataclass class Exposure(GraphResource): type: ExposureType owner: Owner resource_type: Literal[NodeType.Exposure] description: str = "" label: Optional[str] = None maturity: Optional[MaturityType] = None meta: Dict[str, Any] = field(default_factory=dict) tags: List[str] = field(default_factory=list) config: ExposureConfig = field(default_factory=ExposureConfig) unrendered_config: Dict[str, Any] = field(default_factory=dict) url: Optional[str] = None depends_on: DependsOn = field(default_factory=DependsOn) refs: List[RefArgs] = field(default_factory=list) sources: List[List[str]] = field(default_factory=list) metrics: List[List[str]] = field(default_factory=list) created_at: float = field(default_factory=lambda: time.time()) ================================================ FILE: core/dbt/artifacts/resources/v1/function.py ================================================ from dataclasses import dataclass, field from typing import Any, Dict, List, Literal, Optional from dbt.artifacts.resources.types import FunctionType, FunctionVolatility, NodeType from dbt.artifacts.resources.v1.components import CompiledResource, HasRelationMetadata from dbt.artifacts.resources.v1.config import NodeConfig from dbt_common.dataclass_schema import dbtClassMixin # ============= # Function config, and supporting classes # ============= @dataclass class FunctionConfig(NodeConfig): # The fact that this is a property, that can be changed, seems wrong. # A function's materialization should never be changed, so why allow for it? materialized: str = "function" type: FunctionType = FunctionType.Scalar volatility: Optional[FunctionVolatility] = None runtime_version: Optional[str] = None entry_point: Optional[str] = None # ============= # Function resource, and supporting classes # ============= @dataclass class FunctionArgument(dbtClassMixin): name: str data_type: str description: Optional[str] = None default_value: Optional[Any] = None @dataclass class FunctionReturns(dbtClassMixin): data_type: str description: Optional[str] = None @dataclass class FunctionMandatory(dbtClassMixin): returns: FunctionReturns @dataclass class DeferFunction(HasRelationMetadata): alias: str resource_type: NodeType name: str description: str compiled_code: Optional[str] meta: Dict[str, Any] tags: List[str] config: Optional[FunctionConfig] arguments: List[FunctionArgument] returns: FunctionReturns @property def identifier(self): return self.name @dataclass class Function(CompiledResource, FunctionMandatory): resource_type: Literal[NodeType.Function] config: FunctionConfig arguments: List[FunctionArgument] = field(default_factory=list) defer_function: Optional[DeferFunction] = None def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None): dct = super().__post_serialize__(dct, context) if context and context.get("artifact") and "defer_function" in dct: del dct["defer_function"] return dct ================================================ FILE: core/dbt/artifacts/resources/v1/generic_test.py ================================================ from dataclasses import dataclass, field from typing import Any, Dict, Literal, Optional from dbt.artifacts.resources.types import NodeType from dbt.artifacts.resources.v1.components import CompiledResource from dbt.artifacts.resources.v1.config import TestConfig from dbt_common.dataclass_schema import dbtClassMixin @dataclass class TestMetadata(dbtClassMixin): __test__ = False name: str = "test" # dummy default to allow default in GenericTestNode. Should always be set. # kwargs are the args that are left in the test builder after # removing configs. They are set from the test builder when # the test node is created. kwargs: Dict[str, Any] = field(default_factory=dict) namespace: Optional[str] = None @dataclass class GenericTest(CompiledResource): resource_type: Literal[NodeType.Test] column_name: Optional[str] = None file_key_name: Optional[str] = None # Was not able to make mypy happy and keep the code working. We need to # refactor the various configs. config: TestConfig = field(default_factory=TestConfig) # type: ignore attached_node: Optional[str] = None test_metadata: TestMetadata = field(default_factory=TestMetadata) ================================================ FILE: core/dbt/artifacts/resources/v1/group.py ================================================ from dataclasses import dataclass, field from typing import Any, Dict, Literal, Optional from dbt.artifacts.resources.base import BaseResource from dbt.artifacts.resources.types import NodeType from dbt.artifacts.resources.v1.owner import Owner from dbt_common.contracts.config.base import BaseConfig, MergeBehavior @dataclass class GroupConfig(BaseConfig): meta: Dict[str, Any] = field(default_factory=dict, metadata=MergeBehavior.Update.meta()) @dataclass class Group(BaseResource): name: str owner: Owner resource_type: Literal[NodeType.Group] description: Optional[str] = None config: GroupConfig = field(default_factory=GroupConfig) ================================================ FILE: core/dbt/artifacts/resources/v1/hook.py ================================================ from dataclasses import dataclass from typing import Literal, Optional from dbt.artifacts.resources.types import NodeType from dbt.artifacts.resources.v1.components import CompiledResource @dataclass class HookNode(CompiledResource): resource_type: Literal[NodeType.Operation] index: Optional[int] = None ================================================ FILE: core/dbt/artifacts/resources/v1/macro.py ================================================ import time from dataclasses import dataclass, field from typing import Any, Dict, List, Literal, Optional from dbt.artifacts.resources.base import BaseResource, Docs from dbt.artifacts.resources.types import ModelLanguage, NodeType from dbt.artifacts.resources.v1.components import MacroDependsOn from dbt_common.contracts.config.base import BaseConfig from dbt_common.dataclass_schema import dbtClassMixin @dataclass class MacroArgument(dbtClassMixin): name: str type: Optional[str] = None description: str = "" @dataclass class MacroConfig(BaseConfig): meta: Dict[str, Any] = field(default_factory=dict) docs: Docs = field(default_factory=Docs) @dataclass class Macro(BaseResource): macro_sql: str resource_type: Literal[NodeType.Macro] depends_on: MacroDependsOn = field(default_factory=MacroDependsOn) description: str = "" meta: Dict[str, Any] = field(default_factory=dict) docs: Docs = field(default_factory=Docs) config: MacroConfig = field(default_factory=MacroConfig) patch_path: Optional[str] = None arguments: List[MacroArgument] = field(default_factory=list) created_at: float = field(default_factory=lambda: time.time()) supported_languages: Optional[List[ModelLanguage]] = None ================================================ FILE: core/dbt/artifacts/resources/v1/metric.py ================================================ import time from dataclasses import dataclass, field from typing import Any, Dict, List, Literal, Optional from dbt.artifacts.resources.base import GraphResource from dbt.artifacts.resources.types import NodeType from dbt.artifacts.resources.v1.components import DependsOn, RefArgs from dbt.artifacts.resources.v1.semantic_layer_components import ( MeasureAggregationParameters, NonAdditiveDimension, SourceFileMetadata, WhereFilterIntersection, ) from dbt_common.contracts.config.base import BaseConfig, CompareBehavior, MergeBehavior from dbt_common.dataclass_schema import dbtClassMixin from dbt_semantic_interfaces.references import MeasureReference, MetricReference from dbt_semantic_interfaces.type_enums import ( AggregationType, ConversionCalculationType, MetricType, PeriodAggregation, TimeGranularity, ) """ The following classes are dataclasses which are used to construct the Metric node in dbt-core. Additionally, these classes need to at a minimum support what is specified in their protocol definitions in dbt-semantic-interfaces. Their protocol definitions can be found here: https://github.com/dbt-labs/dbt-semantic-interfaces/blob/main/dbt_semantic_interfaces/protocols/metric.py """ @dataclass class MetricInputMeasure(dbtClassMixin): name: str filter: Optional[WhereFilterIntersection] = None alias: Optional[str] = None join_to_timespine: bool = False fill_nulls_with: Optional[int] = None def measure_reference(self) -> MeasureReference: return MeasureReference(element_name=self.name) def post_aggregation_measure_reference(self) -> MeasureReference: return MeasureReference(element_name=self.alias or self.name) @dataclass class MetricTimeWindow(dbtClassMixin): count: int granularity: str @property def window_string(self) -> str: # noqa: D return f"{self.count} {self.granularity}" @property def is_standard_granularity(self) -> bool: # noqa: D return self.granularity.casefold() in {item.value.casefold() for item in TimeGranularity} @dataclass class MetricInput(dbtClassMixin): name: str filter: Optional[WhereFilterIntersection] = None alias: Optional[str] = None offset_window: Optional[MetricTimeWindow] = None offset_to_grain: Optional[str] = None def as_reference(self) -> MetricReference: return MetricReference(element_name=self.name) def post_aggregation_reference(self) -> MetricReference: return MetricReference(element_name=self.alias or self.name) @dataclass class ConstantPropertyInput(dbtClassMixin): base_property: str conversion_property: str @dataclass class ConversionTypeParams(dbtClassMixin): entity: str base_measure: Optional[MetricInputMeasure] = None conversion_measure: Optional[MetricInputMeasure] = None base_metric: Optional[MetricInput] = None conversion_metric: Optional[MetricInput] = None calculation: ConversionCalculationType = ConversionCalculationType.CONVERSION_RATE window: Optional[MetricTimeWindow] = None constant_properties: Optional[List[ConstantPropertyInput]] = None @dataclass class CumulativeTypeParams(dbtClassMixin): window: Optional[MetricTimeWindow] = None grain_to_date: Optional[str] = None period_agg: PeriodAggregation = PeriodAggregation.FIRST metric: Optional[MetricInput] = None @dataclass class MetricAggregationParams(dbtClassMixin): semantic_model: str agg: AggregationType agg_params: Optional[MeasureAggregationParameters] = None agg_time_dimension: Optional[str] = None non_additive_dimension: Optional[NonAdditiveDimension] = None @dataclass class MetricTypeParams(dbtClassMixin): # Only used in v1 Semantic YAML measure: Optional[MetricInputMeasure] = None # Only used in v1 Semantic YAML input_measures: List[MetricInputMeasure] = field(default_factory=list) numerator: Optional[MetricInput] = None denominator: Optional[MetricInput] = None expr: Optional[str] = None window: Optional[MetricTimeWindow] = None grain_to_date: Optional[TimeGranularity] = ( None # legacy, use cumulative_type_params.grain_to_date ) metrics: Optional[List[MetricInput]] = None conversion_type_params: Optional[ConversionTypeParams] = None cumulative_type_params: Optional[CumulativeTypeParams] = None metric_aggregation_params: Optional[MetricAggregationParams] = None # Below this point, all fields are only used in v2 Semantic YAML fill_nulls_with: Optional[int] = None join_to_timespine: bool = False is_private: Optional[bool] = None # populated by "hidden" field in YAML def get_semantic_model_name(self) -> Optional[str]: """Simple helper to avoid having to null check intermediate members.""" return ( self.metric_aggregation_params.semantic_model if self.metric_aggregation_params is not None else None ) @dataclass class MetricConfig(BaseConfig): enabled: bool = True group: Optional[str] = field( default=None, metadata=CompareBehavior.Exclude.meta(), ) meta: Dict[str, Any] = field(default_factory=dict, metadata=MergeBehavior.Update.meta()) @dataclass class Metric(GraphResource): name: str description: str label: str type: MetricType type_params: MetricTypeParams filter: Optional[WhereFilterIntersection] = None metadata: Optional[SourceFileMetadata] = None time_granularity: Optional[str] = None resource_type: Literal[NodeType.Metric] config: MetricConfig = field(default_factory=MetricConfig) unrendered_config: Dict[str, Any] = field(default_factory=dict) sources: List[List[str]] = field(default_factory=list) depends_on: DependsOn = field(default_factory=DependsOn) refs: List[RefArgs] = field(default_factory=list) metrics: List[List[str]] = field(default_factory=list) created_at: float = field(default_factory=lambda: time.time()) group: Optional[str] = None # These fields are only used in v1 metrics. meta: Dict[str, Any] = field(default_factory=dict, metadata=MergeBehavior.Update.meta()) tags: List[str] = field(default_factory=list) @property def input_metrics(self) -> List[MetricInput]: return self.type_params.metrics or [] @property def input_measures(self) -> List[MetricInputMeasure]: return self.type_params.input_measures @property def measure_references(self) -> List[MeasureReference]: return [x.measure_reference() for x in self.input_measures] ================================================ FILE: core/dbt/artifacts/resources/v1/model.py ================================================ import enum from dataclasses import dataclass, field from datetime import datetime from typing import Dict, List, Literal, Optional from dbt.artifacts.resources.types import AccessType, NodeType, TimePeriod from dbt.artifacts.resources.v1.components import ( CompiledResource, DeferRelation, NodeVersion, ) from dbt.artifacts.resources.v1.config import NodeConfig from dbt_common.contracts.config.base import MergeBehavior from dbt_common.contracts.constraints import ModelLevelConstraint from dbt_common.contracts.util import Mergeable from dbt_common.dataclass_schema import ( ExtensibleDbtClassMixin, ValidationError, dbtClassMixin, ) class ModelFreshnessUpdatesOnOptions(enum.Enum): all = "all" any = "any" @dataclass class ModelBuildAfter(ExtensibleDbtClassMixin): count: Optional[int] = None period: Optional[TimePeriod] = None updates_on: ModelFreshnessUpdatesOnOptions = ModelFreshnessUpdatesOnOptions.any @dataclass class ModelFreshness(ExtensibleDbtClassMixin, Mergeable): build_after: ModelBuildAfter def merge_model_freshness(*thresholds: Optional[ModelFreshness]) -> Optional[ModelFreshness]: if not thresholds: return None current_merged_value: Optional[ModelFreshness] = thresholds[0] for i in range(1, len(thresholds)): base = current_merged_value update = thresholds[i] if base is not None and update is not None: # When both base and update freshness are defined, # create a new ModelFreshness instance using the build_after from the 'update'. # This effectively means 'update's build_after configuration takes precedence. merged_freshness_obj = base.merged(update) if ( base.build_after.updates_on == ModelFreshnessUpdatesOnOptions.all or update.build_after.updates_on == ModelFreshnessUpdatesOnOptions.all ): merged_freshness_obj.build_after.updates_on = ModelFreshnessUpdatesOnOptions.all current_merged_value = merged_freshness_obj elif base is None and update is not None: # If the current merged value is None but the new update is defined, # take the update. current_merged_value = update else: # This covers cases where 'update' is None (regardless of 'base'), # or both 'base' and 'update' are None. # The result of the pair-merge is None. current_merged_value = base return current_merged_value @dataclass class ModelConfig(NodeConfig): access: AccessType = field( default=AccessType.Protected, metadata=MergeBehavior.Clobber.meta(), ) freshness: Optional[ModelFreshness] = None def __post_init__(self): super().__post_init__() if ( self.freshness and self.freshness.build_after.period and self.freshness.build_after.count is None ): raise ValidationError( "`freshness.build_after` must have a value for `count` if a `period` is provided" ) elif ( self.freshness and self.freshness.build_after.count is not None and not self.freshness.build_after.period ): raise ValidationError( "`freshness.build_after` must have a value for `period` if a `count` is provided" ) @classmethod def __pre_deserialize__(cls, data): data = super().__pre_deserialize__(data) # scrub out model configs where "build_after" is not defined if ( "freshness" in data and isinstance(data["freshness"], dict) and "build_after" in data["freshness"] ): data["freshness"] = ModelFreshness.from_dict(data["freshness"]).to_dict() else: data.pop("freshness", None) return data @dataclass class CustomGranularity(dbtClassMixin): name: str column_name: Optional[str] = None @dataclass class TimeSpine(dbtClassMixin): standard_granularity_column: str custom_granularities: List[CustomGranularity] = field(default_factory=list) @dataclass class Model(CompiledResource): resource_type: Literal[NodeType.Model] access: AccessType = AccessType.Protected config: ModelConfig = field(default_factory=ModelConfig) constraints: List[ModelLevelConstraint] = field(default_factory=list) version: Optional[NodeVersion] = None latest_version: Optional[NodeVersion] = None deprecation_date: Optional[datetime] = None defer_relation: Optional[DeferRelation] = None primary_key: List[str] = field(default_factory=list) time_spine: Optional[TimeSpine] = None def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None): dct = super().__post_serialize__(dct, context) if context and context.get("artifact") and "defer_relation" in dct: del dct["defer_relation"] return dct ================================================ FILE: core/dbt/artifacts/resources/v1/owner.py ================================================ from dataclasses import dataclass from typing import List, Optional, Union from dbt_common.contracts.config.properties import AdditionalPropertiesAllowed @dataclass class Owner(AdditionalPropertiesAllowed): email: Union[str, List[str], None] = None name: Optional[str] = None ================================================ FILE: core/dbt/artifacts/resources/v1/saved_query.py ================================================ from __future__ import annotations import time from dataclasses import dataclass, field from typing import Any, Dict, List, Literal, Optional, Union from dbt.artifacts.resources.base import GraphResource from dbt.artifacts.resources.types import NodeType from dbt.artifacts.resources.v1.components import DependsOn, RefArgs from dbt.artifacts.resources.v1.config import list_str, metas from dbt.artifacts.resources.v1.semantic_layer_components import ( SourceFileMetadata, WhereFilterIntersection, ) from dbt_common.contracts.config.base import BaseConfig, CompareBehavior, MergeBehavior from dbt_common.contracts.config.metadata import ShowBehavior from dbt_common.dataclass_schema import dbtClassMixin from dbt_semantic_interfaces.type_enums.export_destination_type import ( ExportDestinationType, ) @dataclass class ExportConfig(dbtClassMixin): """Nested configuration attributes for exports.""" export_as: ExportDestinationType schema_name: Optional[str] = None alias: Optional[str] = None database: Optional[str] = None @dataclass class Export(dbtClassMixin): """Configuration for writing query results to a table.""" name: str config: ExportConfig unrendered_config: Dict[str, str] = field(default_factory=dict) @dataclass class QueryParams(dbtClassMixin): """The query parameters for the saved query""" metrics: List[str] group_by: List[str] where: Optional[WhereFilterIntersection] order_by: List[str] = field(default_factory=list) limit: Optional[int] = None @dataclass class SavedQueryCache(dbtClassMixin): enabled: bool = False @dataclass class SavedQueryConfig(BaseConfig): """Where config options for SavedQueries are stored. This class is much like many other node config classes. It's likely that this class will expand in the direction of what's in the `NodeAndTestConfig` class. It might make sense to clean the various *Config classes into one at some point. """ enabled: bool = True group: Optional[str] = field( default=None, metadata=CompareBehavior.Exclude.meta(), ) meta: Dict[str, Any] = field( default_factory=dict, metadata=MergeBehavior.Update.meta(), ) export_as: Optional[ExportDestinationType] = None schema: Optional[str] = None cache: SavedQueryCache = field(default_factory=SavedQueryCache) @dataclass class SavedQueryMandatory(GraphResource): query_params: QueryParams exports: List[Export] @dataclass class SavedQuery(SavedQueryMandatory): resource_type: Literal[NodeType.SavedQuery] description: Optional[str] = None label: Optional[str] = None metadata: Optional[SourceFileMetadata] = None config: SavedQueryConfig = field(default_factory=SavedQueryConfig) unrendered_config: Dict[str, Any] = field(default_factory=dict) group: Optional[str] = None depends_on: DependsOn = field(default_factory=DependsOn) created_at: float = field(default_factory=lambda: time.time()) refs: List[RefArgs] = field(default_factory=list) tags: Union[List[str], str] = field( default_factory=list_str, metadata=metas(ShowBehavior.Hide, MergeBehavior.Append, CompareBehavior.Exclude), ) @property def metrics(self) -> List[str]: return self.query_params.metrics @property def depends_on_nodes(self): return self.depends_on.nodes ================================================ FILE: core/dbt/artifacts/resources/v1/seed.py ================================================ from dataclasses import dataclass, field from typing import Dict, Literal, Optional from dbt.artifacts.resources.types import NodeType from dbt.artifacts.resources.v1.components import ( DeferRelation, MacroDependsOn, ParsedResource, ) from dbt.artifacts.resources.v1.config import NodeConfig from dbt_common.dataclass_schema import ValidationError @dataclass class SeedConfig(NodeConfig): materialized: str = "seed" delimiter: str = "," quote_columns: Optional[bool] = None @classmethod def validate(cls, data): super().validate(data) if data.get("materialized") and data.get("materialized") != "seed": raise ValidationError("A seed must have a materialized value of 'seed'") @dataclass class Seed(ParsedResource): # No SQLDefaults! resource_type: Literal[NodeType.Seed] config: SeedConfig = field(default_factory=SeedConfig) # seeds need the root_path because the contents are not loaded initially # and we need the root_path to load the seed later root_path: Optional[str] = None depends_on: MacroDependsOn = field(default_factory=MacroDependsOn) defer_relation: Optional[DeferRelation] = None def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None): dct = super().__post_serialize__(dct, context) if context and context.get("artifact") and "defer_relation" in dct: del dct["defer_relation"] return dct ================================================ FILE: core/dbt/artifacts/resources/v1/semantic_layer_components.py ================================================ from dataclasses import dataclass from typing import List, Optional, Sequence, Tuple from dbt_common.dataclass_schema import dbtClassMixin from dbt_semantic_interfaces.call_parameter_sets import JinjaCallParameterSets from dbt_semantic_interfaces.parsing.where_filter.jinja_object_parser import ( JinjaObjectParser, QueryItemLocation, ) from dbt_semantic_interfaces.type_enums import AggregationType @dataclass class WhereFilter(dbtClassMixin): where_sql_template: str def call_parameter_sets( self, custom_granularity_names: Sequence[str] ) -> JinjaCallParameterSets: return JinjaObjectParser.parse_call_parameter_sets( self.where_sql_template, custom_granularity_names=custom_granularity_names, query_item_location=QueryItemLocation.NON_ORDER_BY, ) @dataclass class WhereFilterIntersection(dbtClassMixin): where_filters: List[WhereFilter] def filter_expression_parameter_sets( self, custom_granularity_names: Sequence[str] ) -> Sequence[Tuple[str, JinjaCallParameterSets]]: raise NotImplementedError @dataclass class FileSlice(dbtClassMixin): """Provides file slice level context about what something was created from. Implementation of the dbt-semantic-interfaces `FileSlice` protocol """ filename: str content: str start_line_number: int end_line_number: int @dataclass class SourceFileMetadata(dbtClassMixin): """Provides file context about what something was created from. Implementation of the dbt-semantic-interfaces `Metadata` protocol """ repo_file_path: str file_slice: FileSlice @dataclass class MeasureAggregationParameters(dbtClassMixin): percentile: Optional[float] = None use_discrete_percentile: bool = False use_approximate_percentile: bool = False @dataclass class NonAdditiveDimension(dbtClassMixin): name: str window_choice: AggregationType window_groupings: List[str] ================================================ FILE: core/dbt/artifacts/resources/v1/semantic_model.py ================================================ import time from dataclasses import dataclass, field from typing import Any, Dict, List, Optional, Sequence from dbt.artifacts.resources import SourceFileMetadata from dbt.artifacts.resources.base import GraphResource from dbt.artifacts.resources.v1.components import DependsOn, RefArgs from dbt.artifacts.resources.v1.metric import Metric from dbt.artifacts.resources.v1.semantic_layer_components import ( MeasureAggregationParameters, NonAdditiveDimension, ) from dbt_common.contracts.config.base import BaseConfig, CompareBehavior, MergeBehavior from dbt_common.dataclass_schema import dbtClassMixin from dbt_semantic_interfaces.references import ( DimensionReference, EntityReference, LinkableElementReference, MeasureReference, SemanticModelReference, TimeDimensionReference, ) from dbt_semantic_interfaces.type_enums import ( AggregationType, DimensionType, EntityType, MetricType, TimeGranularity, ) """ The classes in this file are dataclasses which are used to construct the Semantic Model node in dbt-core. Additionally, these classes need to at a minimum support what is specified in their protocol definitions in dbt-semantic-interfaces. Their protocol definitions can be found here: https://github.com/dbt-labs/dbt-semantic-interfaces/blob/main/dbt_semantic_interfaces/protocols/semantic_model.py """ @dataclass class SemanticLayerElementConfig(dbtClassMixin): meta: Dict[str, Any] = field( default_factory=dict, metadata=MergeBehavior.Update.meta(), ) @dataclass class Defaults(dbtClassMixin): agg_time_dimension: Optional[str] = None @dataclass class NodeRelation(dbtClassMixin): alias: str schema_name: str # TODO: Could this be called simply "schema" so we could reuse StateRelation? database: Optional[str] = None relation_name: Optional[str] = "" # ==================================== # Dimension objects # Dimension protocols: https://github.com/dbt-labs/dbt-semantic-interfaces/blob/main/dbt_semantic_interfaces/protocols/dimension.py # ==================================== @dataclass class DimensionValidityParams(dbtClassMixin): is_start: bool = False is_end: bool = False @dataclass class DimensionTypeParams(dbtClassMixin): time_granularity: TimeGranularity validity_params: Optional[DimensionValidityParams] = None @dataclass class Dimension(dbtClassMixin): name: str type: DimensionType description: Optional[str] = None label: Optional[str] = None is_partition: bool = False type_params: Optional[DimensionTypeParams] = None expr: Optional[str] = None metadata: Optional[SourceFileMetadata] = None config: Optional[SemanticLayerElementConfig] = None @property def reference(self) -> DimensionReference: return DimensionReference(element_name=self.name) @property def time_dimension_reference(self) -> Optional[TimeDimensionReference]: if self.type == DimensionType.TIME: return TimeDimensionReference(element_name=self.name) else: return None @property def validity_params(self) -> Optional[DimensionValidityParams]: if self.type_params: return self.type_params.validity_params else: return None # ==================================== # Entity objects # Entity protocols: https://github.com/dbt-labs/dbt-semantic-interfaces/blob/main/dbt_semantic_interfaces/protocols/entity.py # ==================================== @dataclass class Entity(dbtClassMixin): name: str type: EntityType description: Optional[str] = None label: Optional[str] = None role: Optional[str] = None expr: Optional[str] = None config: Optional[SemanticLayerElementConfig] = None @property def reference(self) -> EntityReference: return EntityReference(element_name=self.name) @property def is_linkable_entity_type(self) -> bool: return self.type in (EntityType.PRIMARY, EntityType.UNIQUE, EntityType.NATURAL) # ==================================== # Measure object # Measure protocols: https://github.com/dbt-labs/dbt-semantic-interfaces/blob/main/dbt_semantic_interfaces/protocols/measure.py # ==================================== @dataclass class Measure(dbtClassMixin): name: str agg: AggregationType description: Optional[str] = None label: Optional[str] = None create_metric: bool = False expr: Optional[str] = None agg_params: Optional[MeasureAggregationParameters] = None non_additive_dimension: Optional[NonAdditiveDimension] = None agg_time_dimension: Optional[str] = None config: Optional[SemanticLayerElementConfig] = None @property def reference(self) -> MeasureReference: return MeasureReference(element_name=self.name) # ==================================== # SemanticModel final parts # ==================================== @dataclass class SemanticModelConfig(BaseConfig): enabled: bool = True group: Optional[str] = field( default=None, metadata=CompareBehavior.Exclude.meta(), ) meta: Dict[str, Any] = field( default_factory=dict, metadata=MergeBehavior.Update.meta(), ) @dataclass class SemanticModel(GraphResource): model: str node_relation: Optional[NodeRelation] description: Optional[str] = None label: Optional[str] = None defaults: Optional[Defaults] = None entities: Sequence[Entity] = field(default_factory=list) measures: Sequence[Measure] = field(default_factory=list) dimensions: Sequence[Dimension] = field(default_factory=list) metadata: Optional[SourceFileMetadata] = None depends_on: DependsOn = field(default_factory=DependsOn) refs: List[RefArgs] = field(default_factory=list) created_at: float = field(default_factory=lambda: time.time()) config: SemanticModelConfig = field(default_factory=SemanticModelConfig) unrendered_config: Dict[str, Any] = field(default_factory=dict) primary_entity: Optional[str] = None group: Optional[str] = None @property def entity_references(self) -> List[LinkableElementReference]: return [entity.reference for entity in self.entities] @property def dimension_references(self) -> List[LinkableElementReference]: return [dimension.reference for dimension in self.dimensions] @property def measure_references(self) -> List[MeasureReference]: return [measure.reference for measure in self.measures] @property def has_validity_dimensions(self) -> bool: return any([dim.validity_params is not None for dim in self.dimensions]) @property def validity_start_dimension(self) -> Optional[Dimension]: validity_start_dims = [ dim for dim in self.dimensions if dim.validity_params and dim.validity_params.is_start ] if not validity_start_dims: return None return validity_start_dims[0] @property def validity_end_dimension(self) -> Optional[Dimension]: validity_end_dims = [ dim for dim in self.dimensions if dim.validity_params and dim.validity_params.is_end ] if not validity_end_dims: return None return validity_end_dims[0] @property def partitions(self) -> List[Dimension]: # noqa: D return [dim for dim in self.dimensions or [] if dim.is_partition] @property def partition(self) -> Optional[Dimension]: partitions = self.partitions if not partitions: return None return partitions[0] @property def reference(self) -> SemanticModelReference: return SemanticModelReference(semantic_model_name=self.name) def checked_agg_time_dimension_for_measure( self, measure_reference: MeasureReference ) -> TimeDimensionReference: measure: Optional[Measure] = None for measure in self.measures: if measure.reference == measure_reference: measure = measure assert ( measure is not None ), f"No measure with name ({measure_reference.element_name}) in semantic_model with name ({self.name})" default_agg_time_dimension = ( self.defaults.agg_time_dimension if self.defaults is not None else None ) agg_time_dimension_name = measure.agg_time_dimension or default_agg_time_dimension assert agg_time_dimension_name is not None, ( f"Aggregation time dimension for measure {measure.name} on semantic model {self.name} is not set! " "To fix this either specify a default `agg_time_dimension` for the semantic model or define an " "`agg_time_dimension` on the measure directly." ) return TimeDimensionReference(element_name=agg_time_dimension_name) def checked_agg_time_dimension_for_simple_metric( self, metric: Metric ) -> TimeDimensionReference: assert ( metric.type == MetricType.SIMPLE ), "Only simple metrics can have an agg time dimension." metric_agg_params = metric.type_params.metric_aggregation_params # There are validations elsewhere to check this for metrics and provide messaging for it. assert metric_agg_params, "Simple metrics must have metric_aggregation_params." # This indicates a validation bug / dev error, not a user error that should appear # in a user's YAML. assert ( metric_agg_params.semantic_model == self.name ), "Cannot retrieve the agg time dimension for a metric from a different model " f"than the one that the metric belongs to. Metric `{metric.name}` belongs to model " f"`{metric_agg_params.semantic_model}`, but we requested the agg time dimension from model `{self.name}`." metric_time_dimension_name = None if ( metric.type_params and metric.type_params.metric_aggregation_params and metric.type_params.metric_aggregation_params.agg_time_dimension ): metric_time_dimension_name = ( metric.type_params.metric_aggregation_params.agg_time_dimension ) default_agg_time_dimension = ( self.defaults.agg_time_dimension if self.defaults is not None else None ) agg_time_dimension_name = metric_time_dimension_name or default_agg_time_dimension assert agg_time_dimension_name is not None, ( f"Aggregation time dimension for metric {metric.name} is not set! This should either be set directly on " f"the metric specification in the model, or else defaulted to the time dimension in the data " f"source containing the metric." ) return TimeDimensionReference(element_name=agg_time_dimension_name) @property def primary_entity_reference(self) -> Optional[EntityReference]: return ( EntityReference(element_name=self.primary_entity) if self.primary_entity is not None else None ) ================================================ FILE: core/dbt/artifacts/resources/v1/singular_test.py ================================================ from dataclasses import dataclass, field from typing import Literal from dbt.artifacts.resources.types import NodeType from dbt.artifacts.resources.v1.components import CompiledResource from dbt.artifacts.resources.v1.config import TestConfig @dataclass class SingularTest(CompiledResource): resource_type: Literal[NodeType.Test] # Was not able to make mypy happy and keep the code working. We need to # refactor the various configs. config: TestConfig = field(default_factory=TestConfig) # type: ignore ================================================ FILE: core/dbt/artifacts/resources/v1/snapshot.py ================================================ from dataclasses import dataclass, field from typing import Dict, List, Literal, Optional, Union from dbt.artifacts.resources.types import NodeType from dbt.artifacts.resources.v1.components import CompiledResource, DeferRelation from dbt.artifacts.resources.v1.config import NodeConfig from dbt_common.dataclass_schema import ValidationError, dbtClassMixin @dataclass class SnapshotMetaColumnNames(dbtClassMixin): dbt_valid_to: Optional[str] = None dbt_valid_from: Optional[str] = None dbt_scd_id: Optional[str] = None dbt_updated_at: Optional[str] = None dbt_is_deleted: Optional[str] = None @dataclass class SnapshotConfig(NodeConfig): materialized: str = "snapshot" strategy: Optional[str] = None unique_key: Union[str, List[str], None] = None target_schema: Optional[str] = None target_database: Optional[str] = None updated_at: Optional[str] = None # Not using Optional because of serialization issues with a Union of str and List[str] check_cols: Union[str, List[str], None] = None snapshot_meta_column_names: SnapshotMetaColumnNames = field( default_factory=SnapshotMetaColumnNames ) dbt_valid_to_current: Optional[str] = None @property def snapshot_table_column_names(self): return { "dbt_valid_from": self.snapshot_meta_column_names.dbt_valid_from or "dbt_valid_from", "dbt_valid_to": self.snapshot_meta_column_names.dbt_valid_to or "dbt_valid_to", "dbt_scd_id": self.snapshot_meta_column_names.dbt_scd_id or "dbt_scd_id", "dbt_updated_at": self.snapshot_meta_column_names.dbt_updated_at or "dbt_updated_at", "dbt_is_deleted": self.snapshot_meta_column_names.dbt_is_deleted or "dbt_is_deleted", } def final_validate(self): if not self.strategy or not self.unique_key: raise ValidationError( "Snapshots must be configured with a 'strategy' and 'unique_key'." ) if self.strategy == "check": if not self.check_cols: raise ValidationError( "A snapshot configured with the check strategy must " "specify a check_cols configuration." ) if isinstance(self.check_cols, str) and self.check_cols != "all": raise ValidationError( f"Invalid value for 'check_cols': {self.check_cols}. " "Expected 'all' or a list of strings." ) elif self.strategy == "timestamp": if not self.updated_at: raise ValidationError( "A snapshot configured with the timestamp strategy " "must specify an updated_at configuration." ) if self.check_cols: raise ValidationError("A 'timestamp' snapshot should not have 'check_cols'") # If the strategy is not 'check' or 'timestamp' it's a custom strategy, # formerly supported with GenericSnapshotConfig if self.materialized and self.materialized != "snapshot": raise ValidationError("A snapshot must have a materialized value of 'snapshot'") # Called by "calculate_node_config_dict" in ContextConfigGenerator def finalize_and_validate(self): data = self.to_dict(omit_none=True) self.validate(data) return self.from_dict(data) @dataclass class Snapshot(CompiledResource): resource_type: Literal[NodeType.Snapshot] config: SnapshotConfig defer_relation: Optional[DeferRelation] = None def __post_serialize__(self, dct, context: Optional[Dict] = None): dct = super().__post_serialize__(dct, context) if context and context.get("artifact") and "defer_relation" in dct: del dct["defer_relation"] return dct ================================================ FILE: core/dbt/artifacts/resources/v1/source_definition.py ================================================ import time from dataclasses import dataclass, field from typing import Any, Dict, List, Literal, Optional, Union from dbt.artifacts.resources.base import GraphResource from dbt.artifacts.resources.types import NodeType from dbt.artifacts.resources.v1.components import ( ColumnInfo, FreshnessThreshold, HasRelationMetadata, Quoting, ) from dbt.artifacts.resources.v1.config import BaseConfig, MergeBehavior from dbt_common.contracts.config.properties import AdditionalPropertiesAllowed from dbt_common.contracts.util import Mergeable from dbt_common.exceptions import CompilationError @dataclass class SourceConfig(BaseConfig): enabled: bool = True event_time: Any = None freshness: Optional[FreshnessThreshold] = field(default_factory=FreshnessThreshold) loaded_at_field: Optional[str] = None loaded_at_query: Optional[str] = None meta: Dict[str, Any] = field(default_factory=dict, metadata=MergeBehavior.Update.meta()) tags: List[str] = field(default_factory=list) @dataclass class ExternalPartition(AdditionalPropertiesAllowed): name: str = "" description: str = "" data_type: str = "" meta: Dict[str, Any] = field(default_factory=dict) def __post_init__(self): if self.name == "" or self.data_type == "": raise CompilationError("External partition columns must have names and data types") @dataclass class ExternalTable(AdditionalPropertiesAllowed, Mergeable): location: Optional[str] = None file_format: Optional[str] = None row_format: Optional[str] = None tbl_properties: Optional[str] = None partitions: Optional[Union[List[str], List[ExternalPartition]]] = None def __bool__(self): return self.location is not None @dataclass class ParsedSourceMandatory(GraphResource, HasRelationMetadata): source_name: str source_description: str loader: str identifier: str resource_type: Literal[NodeType.Source] @dataclass class SourceDefinition(ParsedSourceMandatory): quoting: Quoting = field(default_factory=Quoting) loaded_at_field: Optional[str] = None loaded_at_query: Optional[str] = None freshness: Optional[FreshnessThreshold] = None external: Optional[ExternalTable] = None description: str = "" columns: Dict[str, ColumnInfo] = field(default_factory=dict) meta: Dict[str, Any] = field(default_factory=dict) source_meta: Dict[str, Any] = field(default_factory=dict) tags: List[str] = field(default_factory=list) config: SourceConfig = field(default_factory=SourceConfig) patch_path: Optional[str] = None unrendered_config: Dict[str, Any] = field(default_factory=dict) relation_name: Optional[str] = None created_at: float = field(default_factory=lambda: time.time()) unrendered_database: Optional[str] = None unrendered_schema: Optional[str] = None doc_blocks: List[str] = field(default_factory=list) ================================================ FILE: core/dbt/artifacts/resources/v1/sql_operation.py ================================================ from dataclasses import dataclass from typing import Literal from dbt.artifacts.resources.types import NodeType from dbt.artifacts.resources.v1.components import CompiledResource @dataclass class SqlOperation(CompiledResource): resource_type: Literal[NodeType.SqlOperation] ================================================ FILE: core/dbt/artifacts/resources/v1/unit_test_definition.py ================================================ import time from dataclasses import dataclass, field from typing import Any, Dict, List, Optional, Sequence, Union from dbt.artifacts.resources import DependsOn, NodeVersion from dbt.artifacts.resources.base import GraphResource from dbt.artifacts.resources.v1.config import list_str, metas from dbt_common.contracts.config.base import BaseConfig, CompareBehavior, MergeBehavior from dbt_common.contracts.config.metadata import ShowBehavior from dbt_common.dataclass_schema import StrEnum, dbtClassMixin @dataclass class UnitTestConfig(BaseConfig): tags: Union[str, List[str]] = field( default_factory=list_str, metadata=metas(ShowBehavior.Hide, MergeBehavior.Append, CompareBehavior.Exclude), ) meta: Dict[str, Any] = field( default_factory=dict, metadata=MergeBehavior.Update.meta(), ) enabled: bool = True class UnitTestFormat(StrEnum): CSV = "csv" Dict = "dict" SQL = "sql" @dataclass class UnitTestInputFixture(dbtClassMixin): input: str rows: Optional[Union[str, List[Dict[str, Any]]]] = None format: UnitTestFormat = UnitTestFormat.Dict fixture: Optional[str] = None @dataclass class UnitTestOverrides(dbtClassMixin): macros: Dict[str, Any] = field(default_factory=dict) vars: Dict[str, Any] = field(default_factory=dict) env_vars: Dict[str, Any] = field(default_factory=dict) @dataclass class UnitTestNodeVersions(dbtClassMixin): include: Optional[List[NodeVersion]] = None exclude: Optional[List[NodeVersion]] = None @dataclass class UnitTestOutputFixture(dbtClassMixin): rows: Optional[Union[str, List[Dict[str, Any]]]] = None format: UnitTestFormat = UnitTestFormat.Dict fixture: Optional[str] = None @dataclass class UnitTestDefinitionMandatory: model: str given: Sequence[UnitTestInputFixture] expect: UnitTestOutputFixture @dataclass class UnitTestDefinition(GraphResource, UnitTestDefinitionMandatory): description: str = "" overrides: Optional[UnitTestOverrides] = None depends_on: DependsOn = field(default_factory=DependsOn) config: UnitTestConfig = field(default_factory=UnitTestConfig) checksum: Optional[str] = None schema: Optional[str] = None created_at: float = field(default_factory=lambda: time.time()) versions: Optional[UnitTestNodeVersions] = None version: Optional[NodeVersion] = None ================================================ FILE: core/dbt/artifacts/schemas/__init__.py ================================================ ================================================ FILE: core/dbt/artifacts/schemas/base.py ================================================ import dataclasses import functools from datetime import datetime, timezone from typing import Any, ClassVar, Dict, Optional, Type, TypeVar from mashumaro.jsonschema import build_json_schema from mashumaro.jsonschema.dialects import DRAFT_2020_12 from dbt.artifacts.exceptions import IncompatibleSchemaError from dbt.version import __version__ from dbt_common.clients.system import read_json, write_json from dbt_common.dataclass_schema import dbtClassMixin from dbt_common.events.functions import get_metadata_vars from dbt_common.exceptions import DbtInternalError, DbtRuntimeError from dbt_common.invocation import get_invocation_id, get_invocation_started_at BASE_SCHEMAS_URL = "https://schemas.getdbt.com/" SCHEMA_PATH = "dbt/{name}/v{version}.json" @dataclasses.dataclass class SchemaVersion: name: str version: int @property def path(self) -> str: return SCHEMA_PATH.format(name=self.name, version=self.version) def __str__(self) -> str: return BASE_SCHEMAS_URL + self.path class Writable: def write(self, path: str): write_json(path, self.to_dict(omit_none=False, context={"artifact": True})) # type: ignore class Readable: @classmethod def read(cls, path: str): try: data = read_json(path) except (EnvironmentError, ValueError) as exc: raise DbtRuntimeError( f'Could not read {cls.__name__} at "{path}" as JSON: {exc}' ) from exc return cls.from_dict(data) # type: ignore # This is used in the ManifestMetadata, RunResultsMetadata, RunOperationResultMetadata, # FreshnessMetadata, and CatalogMetadata classes @dataclasses.dataclass class BaseArtifactMetadata(dbtClassMixin): dbt_schema_version: str dbt_version: str = __version__ generated_at: datetime = dataclasses.field( default_factory=lambda: datetime.now(timezone.utc).replace(tzinfo=None) ) invocation_id: Optional[str] = dataclasses.field(default_factory=get_invocation_id) invocation_started_at: Optional[datetime] = dataclasses.field( default_factory=get_invocation_started_at ) env: Dict[str, str] = dataclasses.field(default_factory=get_metadata_vars) def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None): dct = super().__post_serialize__(dct, context) if dct["generated_at"] and dct["generated_at"].endswith("+00:00"): dct["generated_at"] = dct["generated_at"].replace("+00:00", "") + "Z" return dct # This is used as a class decorator to set the schema_version in the # 'dbt_schema_version' class attribute. (It's copied into the metadata objects.) # Name attributes of SchemaVersion in classes with the 'schema_version' decorator: # manifest # run-results # run-operation-result # sources # catalog # remote-compile-result # remote-execution-result # remote-run-result S = TypeVar("S", bound="VersionedSchema") def schema_version(name: str, version: int): def inner(cls: Type[S]): cls.dbt_schema_version = SchemaVersion( name=name, version=version, ) return cls return inner # This is used in the ArtifactMixin and RemoteCompileResultMixin classes @dataclasses.dataclass class VersionedSchema(dbtClassMixin): dbt_schema_version: ClassVar[SchemaVersion] @classmethod @functools.lru_cache def json_schema(cls) -> Dict[str, Any]: json_schema_obj = build_json_schema(cls, dialect=DRAFT_2020_12, with_dialect_uri=True) json_schema = json_schema_obj.to_dict() json_schema["$id"] = str(cls.dbt_schema_version) return json_schema @classmethod def is_compatible_version(cls, schema_version): compatible_versions = [str(cls.dbt_schema_version)] if hasattr(cls, "compatible_previous_versions"): for name, version in cls.compatible_previous_versions(): compatible_versions.append(str(SchemaVersion(name, version))) return str(schema_version) in compatible_versions @classmethod def read_and_check_versions(cls, path: str): try: data = read_json(path) except (EnvironmentError, ValueError) as exc: raise DbtRuntimeError( f'Could not read {cls.__name__} at "{path}" as JSON: {exc}' ) from exc # Check metadata version. There is a class variable 'dbt_schema_version', but # that doesn't show up in artifacts, where it only exists in the 'metadata' # dictionary. if hasattr(cls, "dbt_schema_version"): if "metadata" in data and "dbt_schema_version" in data["metadata"]: previous_schema_version = data["metadata"]["dbt_schema_version"] # cls.dbt_schema_version is a SchemaVersion object if not cls.is_compatible_version(previous_schema_version): raise IncompatibleSchemaError( expected=str(cls.dbt_schema_version), found=previous_schema_version, ) return cls.upgrade_schema_version(data) @classmethod def upgrade_schema_version(cls, data): """This will modify the data (dictionary) passed in to match the current artifact schema code, if necessary. This is the default method, which just returns the instantiated object via from_dict.""" return cls.from_dict(data) T = TypeVar("T", bound="ArtifactMixin") # metadata should really be a Generic[T_M] where T_M is a TypeVar bound to # BaseArtifactMetadata. Unfortunately this isn't possible due to a mypy issue: # https://github.com/python/mypy/issues/7520 # This is used in the WritableManifest, RunResultsArtifact, RunOperationResultsArtifact, # and CatalogArtifact @dataclasses.dataclass(init=False) class ArtifactMixin(VersionedSchema, Writable, Readable): metadata: BaseArtifactMetadata @classmethod def validate(cls, data): super().validate(data) if cls.dbt_schema_version is None: raise DbtInternalError("Cannot call from_dict with no schema version!") def get_artifact_schema_version(dct: dict) -> int: schema_version = dct.get("metadata", {}).get("dbt_schema_version", None) if not schema_version: raise ValueError("Artifact is missing schema version") # schema_version is in this format: https://schemas.getdbt.com/dbt/manifest/v10.json # What the code below is doing: # 1. Split on "/" – v10.json # 2. Split on "." – v10 # 3. Skip first character – 10 # 4. Convert to int # TODO: If this gets more complicated, turn into a regex return int(schema_version.split("/")[-1].split(".")[0][1:]) def get_artifact_dbt_version(dct: dict) -> Optional[str]: dbt_version = dct.get("metadata", {}).get("dbt_version", None) if dbt_version is None: return None return str(dbt_version) ================================================ FILE: core/dbt/artifacts/schemas/batch_results.py ================================================ from __future__ import annotations from dataclasses import dataclass, field from datetime import datetime from typing import List, Tuple from dbt_common.dataclass_schema import dbtClassMixin BatchType = Tuple[datetime, datetime] @dataclass class BatchResults(dbtClassMixin): successful: List[BatchType] = field(default_factory=list) failed: List[BatchType] = field(default_factory=list) def __add__(self, other: BatchResults) -> BatchResults: return BatchResults( successful=self.successful + other.successful, failed=self.failed + other.failed, ) def __len__(self): return len(self.successful) + len(self.failed) ================================================ FILE: core/dbt/artifacts/schemas/catalog/__init__.py ================================================ # alias to latest from dbt.artifacts.schemas.catalog.v1.catalog import * # noqa from dbt_common.contracts.metadata import ( CatalogKey, CatalogTable, ColumnMap, ColumnMetadata, StatsDict, StatsItem, TableMetadata, ) ================================================ FILE: core/dbt/artifacts/schemas/catalog/v1/__init__.py ================================================ ================================================ FILE: core/dbt/artifacts/schemas/catalog/v1/catalog.py ================================================ from dataclasses import dataclass, field from datetime import datetime from typing import Any, Dict, List, Optional, Union from dbt.artifacts.schemas.base import ( ArtifactMixin, BaseArtifactMetadata, schema_version, ) from dbt_common.contracts.metadata import CatalogTable from dbt_common.dataclass_schema import dbtClassMixin Primitive = Union[bool, str, float, None] PrimitiveDict = Dict[str, Primitive] @dataclass class CatalogMetadata(BaseArtifactMetadata): dbt_schema_version: str = field( default_factory=lambda: str(CatalogArtifact.dbt_schema_version) ) @dataclass class CatalogResults(dbtClassMixin): nodes: Dict[str, CatalogTable] sources: Dict[str, CatalogTable] errors: Optional[List[str]] = None _compile_results: Optional[Any] = None def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None): dct = super().__post_serialize__(dct, context) if "_compile_results" in dct: del dct["_compile_results"] return dct @dataclass @schema_version("catalog", 1) class CatalogArtifact(CatalogResults, ArtifactMixin): metadata: CatalogMetadata @classmethod def from_results( cls, generated_at: datetime, nodes: Dict[str, CatalogTable], sources: Dict[str, CatalogTable], compile_results: Optional[Any], errors: Optional[List[str]], ) -> "CatalogArtifact": meta = CatalogMetadata(generated_at=generated_at) return cls( metadata=meta, nodes=nodes, sources=sources, errors=errors, _compile_results=compile_results, ) ================================================ FILE: core/dbt/artifacts/schemas/freshness/__init__.py ================================================ from dbt.artifacts.schemas.freshness.v3.freshness import * # noqa ================================================ FILE: core/dbt/artifacts/schemas/freshness/v3/__init__.py ================================================ ================================================ FILE: core/dbt/artifacts/schemas/freshness/v3/freshness.py ================================================ from dataclasses import dataclass, field from datetime import datetime from typing import Any, Dict, List, Optional, Sequence, Union from dbt.artifacts.resources import FreshnessThreshold from dbt.artifacts.schemas.base import ( ArtifactMixin, BaseArtifactMetadata, VersionedSchema, schema_version, ) from dbt.artifacts.schemas.results import ( ExecutionResult, FreshnessStatus, NodeResult, TimingInfo, ) from dbt.contracts.graph.nodes import SourceDefinition from dbt_common.dataclass_schema import StrEnum, dbtClassMixin from dbt_common.exceptions import DbtInternalError @dataclass class SourceFreshnessResult(NodeResult): node: SourceDefinition status: FreshnessStatus max_loaded_at: datetime snapshotted_at: datetime age: float @property def skipped(self): return False @dataclass class PartialSourceFreshnessResult(NodeResult): status: FreshnessStatus @property def skipped(self): return False FreshnessNodeResult = Union[PartialSourceFreshnessResult, SourceFreshnessResult] @dataclass class FreshnessMetadata(BaseArtifactMetadata): dbt_schema_version: str = field( default_factory=lambda: str(FreshnessExecutionResultArtifact.dbt_schema_version) ) @dataclass class FreshnessResult(ExecutionResult): metadata: FreshnessMetadata results: Sequence[FreshnessNodeResult] @classmethod def from_node_results( cls, results: List[FreshnessNodeResult], elapsed_time: float, generated_at: datetime, ): meta = FreshnessMetadata(generated_at=generated_at) return cls(metadata=meta, results=results, elapsed_time=elapsed_time) def write(self, path): FreshnessExecutionResultArtifact.from_result(self).write(path) @dataclass class SourceFreshnessOutput(dbtClassMixin): unique_id: str max_loaded_at: datetime snapshotted_at: datetime max_loaded_at_time_ago_in_s: float status: FreshnessStatus criteria: FreshnessThreshold adapter_response: Dict[str, Any] timing: List[TimingInfo] thread_id: str execution_time: float class FreshnessErrorEnum(StrEnum): runtime_error = "runtime error" @dataclass class SourceFreshnessRuntimeError(dbtClassMixin): unique_id: str error: Optional[Union[str, int]] status: FreshnessErrorEnum FreshnessNodeOutput = Union[SourceFreshnessRuntimeError, SourceFreshnessOutput] @dataclass @schema_version("sources", 3) class FreshnessExecutionResultArtifact( ArtifactMixin, VersionedSchema, ): metadata: FreshnessMetadata results: Sequence[FreshnessNodeOutput] elapsed_time: float @classmethod def from_result(cls, base: FreshnessResult): processed = [ process_freshness_result(r) for r in base.results if isinstance(r, SourceFreshnessResult) ] return cls( metadata=base.metadata, results=processed, elapsed_time=base.elapsed_time, ) def process_freshness_result(result: FreshnessNodeResult) -> FreshnessNodeOutput: unique_id = result.node.unique_id if result.status == FreshnessStatus.RuntimeErr: return SourceFreshnessRuntimeError( unique_id=unique_id, error=result.message, status=FreshnessErrorEnum.runtime_error, ) # we know that this must be a SourceFreshnessResult if not isinstance(result, SourceFreshnessResult): raise DbtInternalError( "Got {} instead of a SourceFreshnessResult for a " "non-error result in freshness execution!".format(type(result)) ) # if we're here, we must have a non-None freshness threshold criteria = result.node.freshness if criteria is None: raise DbtInternalError( "Somehow evaluated a freshness result for a source that has no freshness criteria!" ) return SourceFreshnessOutput( unique_id=unique_id, max_loaded_at=result.max_loaded_at, snapshotted_at=result.snapshotted_at, max_loaded_at_time_ago_in_s=result.age, status=result.status, criteria=criteria, adapter_response=result.adapter_response, timing=result.timing, thread_id=result.thread_id, execution_time=result.execution_time, ) ================================================ FILE: core/dbt/artifacts/schemas/manifest/__init__.py ================================================ # alias to latest from dbt.artifacts.schemas.manifest.v12.manifest import * # noqa ================================================ FILE: core/dbt/artifacts/schemas/manifest/v12/__init__.py ================================================ ================================================ FILE: core/dbt/artifacts/schemas/manifest/v12/manifest.py ================================================ from dataclasses import dataclass, field from datetime import datetime from typing import Any, Dict, Iterable, List, Mapping, Optional, Tuple, Union from uuid import UUID from dbt import tracking from dbt.artifacts.resources import ( Analysis, Documentation, Exposure, Function, GenericTest, Group, HookNode, Macro, Metric, Model, SavedQuery, Seed, SemanticModel, SingularTest, Snapshot, SourceDefinition, SqlOperation, UnitTestDefinition, ) from dbt.artifacts.resources.v1.components import Quoting from dbt.artifacts.schemas.base import ( ArtifactMixin, BaseArtifactMetadata, get_artifact_dbt_version, get_artifact_schema_version, schema_version, ) from dbt.artifacts.schemas.upgrades import ( upgrade_manifest_json, upgrade_manifest_json_dbt_version, ) from dbt.version import __version__ from dbt_common.exceptions import DbtInternalError NodeEdgeMap = Dict[str, List[str]] UniqueID = str ManifestResource = Union[ Seed, Analysis, SingularTest, HookNode, Model, SqlOperation, GenericTest, Snapshot, Function ] DisabledManifestResource = Union[ ManifestResource, SourceDefinition, Exposure, Metric, SavedQuery, SemanticModel, UnitTestDefinition, ] @dataclass class ManifestMetadata(BaseArtifactMetadata): """Metadata for the manifest.""" dbt_schema_version: str = field( default_factory=lambda: str(WritableManifest.dbt_schema_version) ) project_name: Optional[str] = field( default=None, metadata={ "description": "Name of the root project", }, ) project_id: Optional[str] = field( default=None, metadata={ "description": "A unique identifier for the project, hashed from the project name", }, ) user_id: Optional[UUID] = field( default=None, metadata={ "description": "A unique identifier for the user", }, ) send_anonymous_usage_stats: Optional[bool] = field( default=None, metadata=dict( description=("Whether dbt is configured to send anonymous usage statistics") ), ) adapter_type: Optional[str] = field( default=None, metadata=dict(description="The type name of the adapter"), ) quoting: Optional[Quoting] = field( default_factory=Quoting, metadata=dict(description="The quoting configuration for the project"), ) run_started_at: Optional[datetime] = field( default=tracking.active_user.run_started_at if tracking.active_user is not None else None, metadata=dict(description="The timestamp when the run started"), ) @classmethod def default(cls): return cls( dbt_schema_version=str(WritableManifest.dbt_schema_version), ) @dataclass @schema_version("manifest", 12) class WritableManifest(ArtifactMixin): nodes: Mapping[UniqueID, ManifestResource] = field( metadata=dict(description=("The nodes defined in the dbt project and its dependencies")) ) sources: Mapping[UniqueID, SourceDefinition] = field( metadata=dict(description=("The sources defined in the dbt project and its dependencies")) ) macros: Mapping[UniqueID, Macro] = field( metadata=dict(description=("The macros defined in the dbt project and its dependencies")) ) docs: Mapping[UniqueID, Documentation] = field( metadata=dict(description=("The docs defined in the dbt project and its dependencies")) ) exposures: Mapping[UniqueID, Exposure] = field( metadata=dict( description=("The exposures defined in the dbt project and its dependencies") ) ) metrics: Mapping[UniqueID, Metric] = field( metadata=dict(description=("The metrics defined in the dbt project and its dependencies")) ) groups: Mapping[UniqueID, Group] = field( metadata=dict(description=("The groups defined in the dbt project")) ) selectors: Mapping[UniqueID, Any] = field( metadata=dict(description=("The selectors defined in selectors.yml")) ) disabled: Optional[Mapping[UniqueID, List[DisabledManifestResource]]] = field( metadata=dict(description="A mapping of the disabled nodes in the target") ) parent_map: Optional[NodeEdgeMap] = field( metadata=dict( description="A mapping from child nodes to their dependencies", ) ) child_map: Optional[NodeEdgeMap] = field( metadata=dict( description="A mapping from parent nodes to their dependents", ) ) group_map: Optional[NodeEdgeMap] = field( metadata=dict( description="A mapping from group names to their nodes", ) ) saved_queries: Mapping[UniqueID, SavedQuery] = field( metadata=dict(description=("The saved queries defined in the dbt project")) ) semantic_models: Mapping[UniqueID, SemanticModel] = field( metadata=dict(description=("The semantic models defined in the dbt project")) ) metadata: ManifestMetadata = field( metadata=dict( description="Metadata about the manifest", ) ) unit_tests: Mapping[UniqueID, UnitTestDefinition] = field( metadata=dict( description="The unit tests defined in the project", ) ) functions: Mapping[UniqueID, Function] = field( default_factory=dict, metadata=dict(description=("The functions defined in the dbt project")), ) @classmethod def compatible_previous_versions(cls) -> Iterable[Tuple[str, int]]: return [ ("manifest", 4), ("manifest", 5), ("manifest", 6), ("manifest", 7), ("manifest", 8), ("manifest", 9), ("manifest", 10), ("manifest", 11), ] @classmethod def upgrade_schema_version(cls, data): """This overrides the "upgrade_schema_version" call in VersionedSchema (via ArtifactMixin) to modify the dictionary passed in from earlier versions of the manifest.""" manifest_schema_version = get_artifact_schema_version(data) if manifest_schema_version < cls.dbt_schema_version.version: data = upgrade_manifest_json(data, manifest_schema_version) manifest_dbt_version = get_artifact_dbt_version(data) if manifest_dbt_version and manifest_dbt_version != __version__: data = upgrade_manifest_json_dbt_version(data) return cls.from_dict(data) @classmethod def validate(cls, _): # When dbt try to load an artifact with additional optional fields # that are not present in the schema, from_dict will work fine. # As long as validate is not called, the schema will not be enforced. # This is intentional, as it allows for safer schema upgrades. raise DbtInternalError( "The WritableManifest should never be validated directly to allow for schema upgrades." ) ================================================ FILE: core/dbt/artifacts/schemas/results.py ================================================ from dataclasses import dataclass from datetime import datetime, timezone from typing import Any, Callable, Dict, List, Optional, Sequence, Union from dbt.contracts.graph.nodes import ResultNode from dbt_common.dataclass_schema import StrEnum, dbtClassMixin from dbt_common.events.helpers import datetime_to_json_string from dbt_common.utils import cast_to_int, cast_to_str @dataclass class TimingInfo(dbtClassMixin): """ Represents a step in the execution of a node. `name` should be one of: compile, execute, or other Do not call directly, use `collect_timing_info` instead. """ name: str started_at: Optional[datetime] = None completed_at: Optional[datetime] = None def begin(self): self.started_at = datetime.now(timezone.utc).replace(tzinfo=None) def end(self): self.completed_at = datetime.now(timezone.utc).replace(tzinfo=None) def to_msg_dict(self): msg_dict = {"name": str(self.name)} if self.started_at: msg_dict["started_at"] = datetime_to_json_string(self.started_at) if self.completed_at: msg_dict["completed_at"] = datetime_to_json_string(self.completed_at) return msg_dict # This is a context manager class collect_timing_info: def __init__(self, name: str, callback: Callable[[TimingInfo], None]) -> None: self.timing_info = TimingInfo(name=name) self.callback = callback def __enter__(self): self.timing_info.begin() def __exit__(self, exc_type, exc_value, traceback): self.timing_info.end() self.callback(self.timing_info) class RunningStatus(StrEnum): Started = "started" Compiling = "compiling" Executing = "executing" class NodeStatus(StrEnum): Success = "success" Error = "error" Fail = "fail" Warn = "warn" Skipped = "skipped" PartialSuccess = "partial success" Pass = "pass" RuntimeErr = "runtime error" NoOp = "no-op" class RunStatus(StrEnum): Success = NodeStatus.Success Error = NodeStatus.Error Skipped = NodeStatus.Skipped PartialSuccess = NodeStatus.PartialSuccess NoOp = NodeStatus.NoOp class TestStatus(StrEnum): __test__ = False Pass = NodeStatus.Pass Error = NodeStatus.Error Fail = NodeStatus.Fail Warn = NodeStatus.Warn Skipped = NodeStatus.Skipped class FreshnessStatus(StrEnum): Pass = NodeStatus.Pass Warn = NodeStatus.Warn Error = NodeStatus.Error RuntimeErr = NodeStatus.RuntimeErr @dataclass class BaseResult(dbtClassMixin): status: Union[RunStatus, TestStatus, FreshnessStatus] timing: List[TimingInfo] thread_id: str execution_time: float adapter_response: Dict[str, Any] message: Optional[str] failures: Optional[int] @classmethod def __pre_deserialize__(cls, data): data = super().__pre_deserialize__(data) if "message" not in data: data["message"] = None if "failures" not in data: data["failures"] = None return data def to_msg_dict(self): msg_dict = { "status": str(self.status), "message": cast_to_str(self.message), "thread": self.thread_id, "execution_time": self.execution_time, "num_failures": cast_to_int(self.failures), "timing_info": [ti.to_msg_dict() for ti in self.timing], "adapter_response": self.adapter_response, } return msg_dict @dataclass class NodeResult(BaseResult): node: ResultNode @dataclass class ExecutionResult(dbtClassMixin): results: Sequence[BaseResult] elapsed_time: float def __len__(self): return len(self.results) def __iter__(self): return iter(self.results) def __getitem__(self, idx): return self.results[idx] # due to issues with typing.Union collapsing subclasses, this can't subclass # PartialResult ================================================ FILE: core/dbt/artifacts/schemas/run/__init__.py ================================================ # alias to latest from dbt.artifacts.schemas.run.v5.run import * # noqa ================================================ FILE: core/dbt/artifacts/schemas/run/v5/__init__.py ================================================ ================================================ FILE: core/dbt/artifacts/schemas/run/v5/run.py ================================================ from __future__ import annotations import copy import threading from dataclasses import dataclass, field from datetime import datetime, timezone from typing import Any, Dict, Iterable, Optional, Sequence, Tuple # https://github.com/dbt-labs/dbt-core/issues/10098 # Needed for Mashumaro serialization of RunResult below # TODO: investigate alternative approaches to restore conditional import # if TYPE_CHECKING: import agate from dbt.artifacts.resources import CompiledResource from dbt.artifacts.schemas.base import ( ArtifactMixin, BaseArtifactMetadata, get_artifact_schema_version, schema_version, ) from dbt.artifacts.schemas.batch_results import BatchResults from dbt.artifacts.schemas.results import ( BaseResult, ExecutionResult, NodeResult, ResultNode, RunStatus, ) from dbt.exceptions import scrub_secrets from dbt_common.clients.system import write_json from dbt_common.constants import SECRET_ENV_PREFIX @dataclass class RunResult(NodeResult): agate_table: Optional["agate.Table"] = field( default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None} ) batch_results: Optional[BatchResults] = None @property def skipped(self): return self.status == RunStatus.Skipped @classmethod def from_node(cls, node: ResultNode, status: RunStatus, message: Optional[str]): thread_id = threading.current_thread().name return RunResult( status=status, thread_id=thread_id, execution_time=0, timing=[], message=message, node=node, adapter_response={}, failures=None, batch_results=None, ) @dataclass class RunResultsMetadata(BaseArtifactMetadata): dbt_schema_version: str = field( default_factory=lambda: str(RunResultsArtifact.dbt_schema_version) ) @dataclass class RunResultOutput(BaseResult): unique_id: str compiled: Optional[bool] compiled_code: Optional[str] relation_name: Optional[str] batch_results: Optional[BatchResults] = None def process_run_result(result: RunResult) -> RunResultOutput: compiled = isinstance(result.node, CompiledResource) return RunResultOutput( unique_id=result.node.unique_id, status=result.status, timing=result.timing, thread_id=result.thread_id, execution_time=result.execution_time, message=result.message, adapter_response=result.adapter_response, failures=result.failures, batch_results=result.batch_results, compiled=result.node.compiled if compiled else None, # type:ignore compiled_code=result.node.compiled_code if compiled else None, # type:ignore relation_name=result.node.relation_name if compiled else None, # type:ignore ) @dataclass class RunExecutionResult( ExecutionResult, ): results: Sequence[RunResult] args: Dict[str, Any] = field(default_factory=dict) generated_at: datetime = field( default_factory=lambda: datetime.now(timezone.utc).replace(tzinfo=None) ) def write(self, path: str): writable = RunResultsArtifact.from_execution_results( results=self.results, elapsed_time=self.elapsed_time, generated_at=self.generated_at, args=self.args, ) writable.write(path) @dataclass @schema_version("run-results", 6) class RunResultsArtifact(ExecutionResult, ArtifactMixin): results: Sequence[RunResultOutput] args: Dict[str, Any] = field(default_factory=dict) @classmethod def from_execution_results( cls, results: Sequence[RunResult], elapsed_time: float, generated_at: datetime, args: Dict, ): processed_results = [ process_run_result(result) for result in results if isinstance(result, RunResult) ] meta = RunResultsMetadata( dbt_schema_version=str(cls.dbt_schema_version), generated_at=generated_at, ) secret_vars = [ v for k, v in args["vars"].items() if k.startswith(SECRET_ENV_PREFIX) and v.strip() ] scrubbed_args = copy.deepcopy(args) # scrub secrets in invocation command scrubbed_args["invocation_command"] = scrub_secrets( scrubbed_args["invocation_command"], secret_vars ) # scrub secrets in vars dict scrubbed_args["vars"] = { k: scrub_secrets(v, secret_vars) for k, v in scrubbed_args["vars"].items() } return cls( metadata=meta, results=processed_results, elapsed_time=elapsed_time, args=scrubbed_args ) @classmethod def compatible_previous_versions(cls) -> Iterable[Tuple[str, int]]: return [ ("run-results", 4), ("run-results", 5), ] @classmethod def upgrade_schema_version(cls, data): """This overrides the "upgrade_schema_version" call in VersionedSchema (via ArtifactMixin) to modify the dictionary passed in from earlier versions of the run_results. """ run_results_schema_version = get_artifact_schema_version(data) # If less than the current version (v5), preprocess contents to match latest schema version if run_results_schema_version <= 5: # In v5, we added 'compiled' attributes to each result entry # Going forward, dbt expects these to be populated for result in data["results"]: result["compiled"] = False result["compiled_code"] = "" result["relation_name"] = "" return cls.from_dict(data) def write(self, path: str): write_json(path, self.to_dict(omit_none=False)) ================================================ FILE: core/dbt/artifacts/schemas/upgrades/__init__.py ================================================ from dbt.artifacts.schemas.upgrades.upgrade_manifest import upgrade_manifest_json from dbt.artifacts.schemas.upgrades.upgrade_manifest_dbt_version import ( upgrade_manifest_json_dbt_version, ) ================================================ FILE: core/dbt/artifacts/schemas/upgrades/upgrade_manifest.py ================================================ def rename_sql_attr(node_content: dict) -> dict: if "raw_sql" in node_content: node_content["raw_code"] = node_content.pop("raw_sql") if "compiled_sql" in node_content: node_content["compiled_code"] = node_content.pop("compiled_sql") node_content["language"] = "sql" return node_content def upgrade_ref_content(node_content: dict) -> dict: # In v1.5 we switched Node.refs from List[List[str]] to List[Dict[str, Union[NodeVersion, str]]] # Previous versions did not have a version keyword argument for ref if "refs" in node_content: upgraded_refs = [] for ref in node_content["refs"]: if isinstance(ref, list): if len(ref) == 1: upgraded_refs.append({"package": None, "name": ref[0], "version": None}) else: upgraded_refs.append({"package": ref[0], "name": ref[1], "version": None}) node_content["refs"] = upgraded_refs return node_content def upgrade_node_content(node_content): rename_sql_attr(node_content) upgrade_ref_content(node_content) if node_content["resource_type"] != "seed" and "root_path" in node_content: del node_content["root_path"] def upgrade_seed_content(node_content): # Remove compilation related attributes for attr_name in ( "language", "refs", "sources", "metrics", "compiled_path", "compiled", "compiled_code", "extra_ctes_injected", "extra_ctes", "relation_name", ): if attr_name in node_content: del node_content[attr_name] # In v1.4, we switched SeedNode.depends_on from DependsOn to MacroDependsOn node_content.get("depends_on", {}).pop("nodes", None) def drop_v9_and_prior_metrics(manifest: dict) -> None: manifest["metrics"] = {} filtered_disabled_entries = {} for entry_name, resource_list in manifest.get("disabled", {}).items(): filtered_resource_list = [] for resource in resource_list: if resource.get("resource_type") != "metric": filtered_resource_list.append(resource) filtered_disabled_entries[entry_name] = filtered_resource_list manifest["disabled"] = filtered_disabled_entries def _convert_dct_with_filter(v10_dct_with_opt_filter): """Upgrage the filter object from v10 to v11. v10 filters from a serialized manifest looked like: {..., 'filter': {'where_sql_template': '<filter_value>'}} whereas v11 filters look like: {..., 'filter': {'where_filters': [{'where_sql_template': '<filter_value>'}, ...]}} """ if v10_dct_with_opt_filter is not None and v10_dct_with_opt_filter.get("filter") is not None: v10_dct_with_opt_filter["filter"] = {"where_filters": [v10_dct_with_opt_filter["filter"]]} def _convert_metric(v10_metric_dict): """Upgrades a v10 metric object to a v11 metric object. Specifcally the following properties change 1. metric.filter 2. metric.type_params.measure.filter 3. metric.type_params.input_measures[x].filter 4. metric.type_params.numerator.filter 5. metric.type_params.denominator.filter 6. metric.type_params.metrics[x].filter" """ # handles top level metric filter _convert_dct_with_filter(v10_metric_dict) type_params = v10_metric_dict.get("type_params") if type_params is not None: _convert_dct_with_filter(type_params.get("measure")) _convert_dct_with_filter(type_params.get("numerator")) _convert_dct_with_filter(type_params.get("denominator")) # handles metric.type_params.input_measures[x].filter input_measures = type_params.get("input_measures") if input_measures is not None: for input_measure in input_measures: _convert_dct_with_filter(input_measure) # handles metric.type_params.metrics[x].filter metrics = type_params.get("metrics") if metrics is not None: for metric in metrics: _convert_dct_with_filter(metric) def upgrade_v10_metric_filters(manifest: dict): """Handles metric filters changes from v10 to v11.""" metrics = manifest.get("metrics", {}) for metric in metrics.values(): _convert_metric(metric) disabled_nodes = manifest.get("disabled", {}) for unique_id, nodes in disabled_nodes.items(): if unique_id.split(".")[0] == "metric": for node in nodes: _convert_metric(node) def upgrade_manifest_json(manifest: dict, manifest_schema_version: int) -> dict: # this should remain 9 while the check in `upgrade_schema_version` may change if manifest_schema_version <= 9: drop_v9_and_prior_metrics(manifest=manifest) elif manifest_schema_version == 10: upgrade_v10_metric_filters(manifest=manifest) for node_content in manifest.get("nodes", {}).values(): upgrade_node_content(node_content) if node_content["resource_type"] == "seed": upgrade_seed_content(node_content) for disabled in manifest.get("disabled", {}).values(): # There can be multiple disabled nodes for the same unique_id # so make sure all the nodes get the attr renamed for node_content in disabled: upgrade_node_content(node_content) if node_content["resource_type"] == "seed": upgrade_seed_content(node_content) # add group key if "groups" not in manifest: manifest["groups"] = {} if "group_map" not in manifest: manifest["group_map"] = {} # add unit_tests key if "unit_tests" not in manifest: manifest["unit_tests"] = {} for metric_content in manifest.get("metrics", {}).values(): # handle attr renames + value translation ("expression" -> "derived") metric_content = upgrade_ref_content(metric_content) if "root_path" in metric_content: del metric_content["root_path"] for exposure_content in manifest.get("exposures", {}).values(): exposure_content = upgrade_ref_content(exposure_content) if "root_path" in exposure_content: del exposure_content["root_path"] for source_content in manifest.get("sources", {}).values(): if "root_path" in source_content: del source_content["root_path"] for macro_content in manifest.get("macros", {}).values(): if "root_path" in macro_content: del macro_content["root_path"] for doc_content in manifest.get("docs", {}).values(): if "root_path" in doc_content: del doc_content["root_path"] doc_content["resource_type"] = "doc" if "semantic_models" not in manifest: manifest["semantic_models"] = {} if "saved_queries" not in manifest: manifest["saved_queries"] = {} return manifest ================================================ FILE: core/dbt/artifacts/schemas/upgrades/upgrade_manifest_dbt_version.py ================================================ def upgrade_manifest_json_dbt_version(manifest: dict) -> dict: return manifest ================================================ FILE: core/dbt/artifacts/utils/validation.py ================================================ import re HTML_COLORS = [ "aliceblue", "antiquewhite", "aqua", "aquamarine", "azure", "beige", "bisque", "black", "blanchedalmond", "blue", "blueviolet", "brown", "burlywood", "cadetblue", "chartreuse", "chocolate", "coral", "cornflowerblue", "cornsilk", "crimson", "cyan", "darkblue", "darkcyan", "darkgoldenrod", "darkgray", "darkgreen", "darkkhaki", "darkmagenta", "darkolivegreen", "darkorange", "darkorchid", "darkred", "darksalmon", "darkseagreen", "darkslateblue", "darkslategray", "darkturquoise", "darkviolet", "deeppink", "deepskyblue", "dimgray", "dodgerblue", "firebrick", "floralwhite", "forestgreen", "fuchsia", "gainsboro", "ghostwhite", "gold", "goldenrod", "gray", "green", "greenyellow", "honeydew", "hotpink", "indianred", "indigo", "ivory", "khaki", "lavender", "lavenderblush", "lawngreen", "lemonchiffon", "lightblue", "lightcoral", "lightcyan", "lightgoldenrodyellow", "lightgray", "lightgreen", "lightpink", "lightsalmon", "lightsalmon", "lightseagreen", "lightskyblue", "lightslategray", "lightsteelblue", "lightyellow", "lime", "limegreen", "linen", "magenta", "maroon", "mediumaquamarine", "mediumblue", "mediumorchid", "mediumpurple", "mediumseagreen", "mediumslateblue", "mediumslateblue", "mediumspringgreen", "mediumturquoise", "mediumvioletred", "midnightblue", "mintcream", "mistyrose", "moccasin", "navajowhite", "navy", "oldlace", "olive", "olivedrab", "orange", "orangered", "orchid", "palegoldenrod", "palegreen", "paleturquoise", "palevioletred", "papayawhip", "peachpuff", "peru", "pink", "plum", "powderblue", "purple", "rebeccapurple", "red", "rosybrown", "royalblue", "saddlebrown", "salmon", "sandybrown", "seagreen", "seashell", "sienna", "silver", "skyblue", "slateblue", "slategray", "snow", "springgreen", "steelblue", "tan", "teal", "thistle", "tomato", "turquoise", "violet", "wheat", "white", "whitesmoke", "yellow", "yellowgreen", ] def validate_color(color: str) -> bool: match_hex = re.search(r"^#(?:[0-9a-f]{3}){1,2}$", color.lower()) match_html_color_name = color.lower() in HTML_COLORS return bool(match_hex or match_html_color_name) ================================================ FILE: core/dbt/cli/__init__.py ================================================ from .main import cli as dbt_cli # noqa ================================================ FILE: core/dbt/cli/context.py ================================================ from typing import Optional import click from dbt.cli.main import cli as dbt def make_context(args, command=dbt) -> Optional[click.Context]: try: ctx = command.make_context(command.name, args) except click.exceptions.Exit: return None ctx.invoked_subcommand = ctx.protected_args[0] if ctx.protected_args else None ctx.obj = {} return ctx ================================================ FILE: core/dbt/cli/exceptions.py ================================================ from typing import IO, List, Optional, Union from click.exceptions import ClickException from dbt.artifacts.schemas.catalog import CatalogArtifact from dbt.contracts.graph.manifest import Manifest from dbt.contracts.results import RunExecutionResult from dbt.utils import ExitCodes class DbtUsageException(Exception): pass class DbtInternalException(Exception): pass class CliException(ClickException): """The base exception class for our implementation of the click CLI. The exit_code attribute is used by click to determine which exit code to produce after an invocation.""" def __init__(self, exit_code: ExitCodes) -> None: self.exit_code = exit_code.value # the typing of _file is to satisfy the signature of ClickException.show # overriding this method prevents click from printing any exceptions to stdout def show(self, _file: Optional[IO] = None) -> None: # type: ignore[type-arg] pass class ResultExit(CliException): """This class wraps any exception that contains results while invoking dbt, or the results of an invocation that did not succeed but did not throw any exceptions.""" def __init__( self, result: Union[ bool, # debug CatalogArtifact, # docs generate List[str], # list/ls Manifest, # parse None, # clean, deps, init, source RunExecutionResult, # build, compile, run, seed, snapshot, test, run-operation ] = None, ) -> None: super().__init__(ExitCodes.ModelError) self.result = result class ExceptionExit(CliException): """This class wraps any exception that does not contain results thrown while invoking dbt.""" def __init__(self, exception: Exception) -> None: super().__init__(ExitCodes.UnhandledError) self.exception = exception ================================================ FILE: core/dbt/cli/flags.py ================================================ import os import sys from dataclasses import dataclass from datetime import datetime from importlib import import_module from pathlib import Path from pprint import pformat as pf from typing import Any, Callable, Dict, List, Optional, Set, Union from click import Context, Parameter, get_current_context from click.core import Command as ClickCommand from click.core import Group, ParameterSource from dbt.cli.exceptions import DbtUsageException from dbt.cli.resolvers import default_log_path, default_project_dir from dbt.cli.types import Command as CliCommand from dbt.config.project import read_project_flags from dbt.config.utils import normalize_warn_error_options from dbt.contracts.project import ProjectFlags from dbt.deprecations import fire_buffered_deprecations, renamed_env_var, warn from dbt.events import ALL_EVENT_NAMES from dbt.events.types import SelectExcludeIgnoredWithSelectorWarning from dbt_common import ui from dbt_common.clients import jinja from dbt_common.events import functions from dbt_common.events.functions import fire_event from dbt_common.exceptions import DbtInternalError from dbt_common.helper_types import WarnErrorOptionsV2 if os.name != "nt": # https://bugs.python.org/issue41567 import multiprocessing.popen_spawn_posix # type: ignore # noqa: F401 FLAGS_DEFAULTS = { "INDIRECT_SELECTION": "eager", "TARGET_PATH": None, "DEFER_STATE": None, # necessary because of retry construction of flags "WARN_ERROR": None, # Cli args without project_flags or env var option. "FULL_REFRESH": False, "STRICT_MODE": False, "STORE_FAILURES": False, "INTROSPECT": True, "STATE_MODIFIED_COMPARE_VARS": False, } DEPRECATED_PARAMS = { "deprecated_defer": "defer", "deprecated_favor_state": "favor_state", "deprecated_print": "print", "deprecated_state": "state", } DEPRECATED_FLAGS_TO_WARNINGS = {("--models", "--model", "-m"): "model-param-usage-deprecation"} WHICH_KEY = "which" def convert_config(config_name, config_value): """Convert the values from config and original set_from_args to the correct type.""" ret = config_value if config_name.lower() == "warn_error_options" and type(config_value) == dict: normalize_warn_error_options(ret) ret = WarnErrorOptionsV2( error=config_value.get("error", []), warn=config_value.get("warn", []), silence=config_value.get("silence", []), valid_error_names=ALL_EVENT_NAMES, ) return ret def args_to_context(args: List[str]) -> Context: """Convert a list of args to a click context with proper hierarchy for dbt commands""" from dbt.cli.main import cli cli_ctx = cli.make_context(cli.name, args) # Split args if they're a comma separated string. if len(args) == 1 and "," in args[0]: args = args[0].split(",") sub_command_name, sub_command, args = cli.resolve_command(cli_ctx, args) # Handle source and docs group. if isinstance(sub_command, Group): sub_command_name, sub_command, args = sub_command.resolve_command(cli_ctx, args) assert isinstance(sub_command, ClickCommand) sub_command_ctx = sub_command.make_context(sub_command_name, args) sub_command_ctx.parent = cli_ctx return sub_command_ctx @dataclass(frozen=True) class Flags: """Primary configuration artifact for running dbt""" def __init__( self, ctx: Optional[Context] = None, project_flags: Optional[ProjectFlags] = None ) -> None: # Set the default flags. for key, value in FLAGS_DEFAULTS.items(): object.__setattr__(self, key, value) # Use to handle duplicate params in _assign_params flags_defaults_list = list(FLAGS_DEFAULTS.keys()) if ctx is None: ctx = get_current_context() def _get_params_by_source(ctx: Context, source_type: ParameterSource): """Generates all params of a given source type.""" yield from [ name for name, source in ctx._parameter_source.items() if source is source_type ] if ctx.parent: yield from _get_params_by_source(ctx.parent, source_type) # Ensure that any params sourced from the commandline are not present more than once. # Click handles this exclusivity, but only at a per-subcommand level. seen_params = [] for param in _get_params_by_source(ctx, ParameterSource.COMMANDLINE): if param in seen_params: raise DbtUsageException( f"{param.lower()} was provided both before and after the subcommand, it can only be set either before or after.", ) seen_params.append(param) def _assign_params( ctx: Context, params_assigned_from_default: set, params_assigned_from_user: set, deprecated_env_vars: Dict[str, Callable], ): """Recursively adds all click params to flag object""" for param_name, param_value in ctx.params.items(): # N.B. You have to use the base MRO method (object.__setattr__) to set attributes # when using frozen dataclasses. # https://docs.python.org/3/library/dataclasses.html#frozen-instances # Handle deprecated env vars while still respecting old values # e.g. DBT_NO_PRINT -> DBT_PRINT if DBT_NO_PRINT is set, it is # respected over DBT_PRINT or --print. new_name: Union[str, None] = None if param_name in DEPRECATED_PARAMS: # Deprecated env vars can only be set via env var. # We use the deprecated option in click to serialize the value # from the env var string. param_source = ctx.get_parameter_source(param_name) if param_source == ParameterSource.DEFAULT: continue elif param_source != ParameterSource.ENVIRONMENT: raise DbtUsageException( "Deprecated parameters can only be set via environment variables", ) # Rename for clarity. dep_name = param_name new_name = DEPRECATED_PARAMS.get(dep_name) try: assert isinstance(new_name, str) except AssertionError: raise Exception( f"No deprecated param name match in DEPRECATED_PARAMS from {dep_name} to {new_name}" ) # Find param objects for their envvar name. try: dep_param = [x for x in ctx.command.params if x.name == dep_name][0] new_param = [x for x in ctx.command.params if x.name == new_name][0] except IndexError: raise Exception( f"No deprecated param name match in context from {dep_name} to {new_name}" ) # Remove param from defaulted set since the deprecated # value is not set from default, but from an env var. if new_name in params_assigned_from_default: params_assigned_from_default.remove(new_name) # Add the deprecation warning function to the set. # Get the old and new env vars if isinstance(dep_param.envvar, list): dep_env_var_without_prefix = dep_param.envvar[1] elif isinstance(dep_param.envvar, str): dep_env_var_without_prefix = dep_param.envvar else: # Error raised as an assertion error to maintain history of how this was handled in the past assert ( False ), f"Invalid envvar type: {type(dep_param.envvar)}, expected a list or string" if isinstance(new_param.envvar, list): new_env_var_with_prefix = new_param.envvar[0] elif isinstance(new_param.envvar, str): new_env_var_with_prefix = new_param.envvar else: # Error raised as an assertion error to maintain history of how this was handled in the past assert ( False ), f"Invalid envvar type: {type(new_param.envvar)}, expected a list or string" deprecated_env_vars[new_name] = renamed_env_var( old_name=dep_env_var_without_prefix, new_name=new_env_var_with_prefix, ) # end deprecated_params # Set the flag value. is_duplicate = ( hasattr(self, param_name.upper()) and param_name.upper() not in flags_defaults_list ) # First time through, set as though FLAGS_DEFAULTS hasn't been set, so not a duplicate. # Subsequent pass (to process "parent" params) should be treated as duplicates. if param_name.upper() in flags_defaults_list: flags_defaults_list.remove(param_name.upper()) # Note: the following determines whether parameter came from click default, # not from FLAGS_DEFAULTS in __init__. is_default = ctx.get_parameter_source(param_name) == ParameterSource.DEFAULT is_envvar = ctx.get_parameter_source(param_name) == ParameterSource.ENVIRONMENT flag_name = (new_name or param_name).upper() # envvar flags are assigned in either parent or child context if there # isn't an overriding cli command flag. # If the flag has been encountered as a child cli flag, we don't # want to overwrite with parent envvar, since the commandline flag takes precedence. if (is_duplicate and not (is_default or is_envvar)) or not is_duplicate: object.__setattr__(self, flag_name, param_value) # Track default assigned params. # For flags that are accepted at both 'parent' and 'child' levels, # we need to track user-provided and default values across both, # to support detection of mutually exclusive flags later on. if not is_default: params_assigned_from_user.add(param_name) if param_name in params_assigned_from_default: params_assigned_from_default.remove(param_name) if is_default and param_name not in params_assigned_from_user: params_assigned_from_default.add(param_name) if ctx.parent: _assign_params( ctx.parent, params_assigned_from_default, params_assigned_from_user, deprecated_env_vars, ) params_assigned_from_user = set() # type: Set[str] params_assigned_from_default = set() # type: Set[str] deprecated_env_vars: Dict[str, Callable] = {} _assign_params( ctx, params_assigned_from_default, params_assigned_from_user, deprecated_env_vars ) # Set deprecated_env_var_warnings to be fired later after events have been init. object.__setattr__( self, "deprecated_env_var_warnings", [x for x in deprecated_env_vars.values()] ) # Get the invoked command flags. invoked_subcommand_name = ( ctx.invoked_subcommand if hasattr(ctx, "invoked_subcommand") else None ) if invoked_subcommand_name is not None: invoked_subcommand = getattr(import_module("dbt.cli.main"), invoked_subcommand_name) invoked_subcommand.allow_extra_args = True invoked_subcommand.ignore_unknown_options = True invoked_subcommand_ctx = invoked_subcommand.make_context(None, sys.argv) _assign_params( invoked_subcommand_ctx, params_assigned_from_default, params_assigned_from_user, deprecated_env_vars, ) if not project_flags: project_dir = getattr(self, "PROJECT_DIR", str(default_project_dir())) profiles_dir = getattr(self, "PROFILES_DIR", None) if profiles_dir and project_dir: project_flags = read_project_flags(project_dir, profiles_dir) else: project_flags = None # Add entire invocation command to flags object.__setattr__(self, "INVOCATION_COMMAND", "dbt " + " ".join(sys.argv[1:])) if project_flags: # Overwrite default assignments with project flags if available. param_assigned_from_default_copy = params_assigned_from_default.copy() for param_assigned_from_default in params_assigned_from_default: project_flags_param_value = getattr( project_flags, param_assigned_from_default, None ) if project_flags_param_value is not None: object.__setattr__( self, param_assigned_from_default.upper(), convert_config(param_assigned_from_default, project_flags_param_value), ) param_assigned_from_default_copy.remove(param_assigned_from_default) params_assigned_from_default = param_assigned_from_default_copy # Add project-level flags that are not available as CLI options / env vars for ( project_level_flag_name, project_level_flag_value, ) in project_flags.project_only_flags.items(): object.__setattr__(self, project_level_flag_name.upper(), project_level_flag_value) # Set hard coded flags. object.__setattr__(self, "WHICH", invoked_subcommand_name or ctx.info_name) # Apply the lead/follow relationship between some parameters. self._override_if_set("USE_COLORS", "USE_COLORS_FILE", params_assigned_from_default) self._override_if_set("LOG_LEVEL", "LOG_LEVEL_FILE", params_assigned_from_default) self._override_if_set("LOG_FORMAT", "LOG_FORMAT_FILE", params_assigned_from_default) # Set default LOG_PATH from PROJECT_DIR, if available. # Starting in v1.5, if `log-path` is set in `dbt_project.yml`, it will raise a deprecation warning, # with the possibility of removing it in a future release. if getattr(self, "LOG_PATH", None) is None: project_dir = getattr(self, "PROJECT_DIR", str(default_project_dir())) version_check = getattr(self, "VERSION_CHECK", True) object.__setattr__( self, "LOG_PATH", default_log_path(Path(project_dir), version_check) ) # Support console DO NOT TRACK initiative. if os.getenv("DO_NOT_TRACK", "").lower() in ("1", "t", "true", "y", "yes"): object.__setattr__(self, "SEND_ANONYMOUS_USAGE_STATS", False) # Check mutual exclusivity once all flags are set. self._assert_mutually_exclusive( params_assigned_from_default, ["WARN_ERROR", "WARN_ERROR_OPTIONS"] ) # Handle arguments mutually exclusive with INLINE self._assert_mutually_exclusive(params_assigned_from_default, ["SELECT", "INLINE"]) self._assert_mutually_exclusive(params_assigned_from_default, ["SELECTOR", "INLINE"]) # Check event_time configs for validity self._validate_event_time_configs() # Support lower cased access for legacy code. params = set( x for x in dir(self) if not callable(getattr(self, x)) and not x.startswith("__") ) for param in params: object.__setattr__(self, param.lower(), getattr(self, param)) self.set_common_global_flags() def __str__(self) -> str: return str(pf(self.__dict__)) def _override_if_set(self, lead: str, follow: str, defaulted: Set[str]) -> None: """If the value of the lead parameter was set explicitly, apply the value to follow, unless follow was also set explicitly.""" if lead.lower() not in defaulted and follow.lower() in defaulted: object.__setattr__(self, follow.upper(), getattr(self, lead.upper(), None)) def _assert_mutually_exclusive( self, params_assigned_from_default: Set[str], group: List[str] ) -> None: """ Ensure no elements from group are simultaneously provided by a user, as inferred from params_assigned_from_default. Raises click.UsageError if any two elements from group are simultaneously provided by a user. """ set_flag = None for flag in group: flag_set_by_user = ( hasattr(self, flag) and flag.lower() not in params_assigned_from_default ) if flag_set_by_user and set_flag: raise DbtUsageException( f"{flag.lower()}: not allowed with argument {set_flag.lower()}" ) elif flag_set_by_user: set_flag = flag def _validate_event_time_configs(self) -> None: event_time_start: datetime = ( getattr(self, "EVENT_TIME_START") if hasattr(self, "EVENT_TIME_START") else None ) event_time_end: datetime = ( getattr(self, "EVENT_TIME_END") if hasattr(self, "EVENT_TIME_END") else None ) # only do validations if at least one of `event_time_start` or `event_time_end` are specified if event_time_start is not None or event_time_end is not None: # These `ifs`, combined with the parent `if` make it so that `event_time_start` and # `event_time_end` are mutually required if event_time_start is None: raise DbtUsageException( "The flag `--event-time-end` was specified, but `--event-time-start` was not. " "When specifying `--event-time-end`, `--event-time-start` must also be present." ) if event_time_end is None: raise DbtUsageException( "The flag `--event-time-start` was specified, but `--event-time-end` was not. " "When specifying `--event-time-start`, `--event-time-end` must also be present." ) # This `if` just is a sanity check that `event_time_start` is before `event_time_end` if event_time_start >= event_time_end: raise DbtUsageException( "Value for `--event-time-start` must be less than `--event-time-end`" ) def fire_deprecations(self, ctx: Optional[Context] = None): """Fires events for deprecated env_var usage.""" [dep_fn() for dep_fn in self.deprecated_env_var_warnings] # Warn when --selector is used together with --select or --exclude (they are ignored) if getattr(self, "SELECTOR", None) and ( getattr(self, "SELECT", ()) or getattr(self, "EXCLUDE", ()) ): fire_event(SelectExcludeIgnoredWithSelectorWarning()) # It is necessary to remove this attr from the class so it does # not get pickled when written to disk as json. object.__delattr__(self, "deprecated_env_var_warnings") fire_buffered_deprecations() # Handle firing deprecations of CLI aliases separately using argv or dbtRunner args # because click doesn't make it possible to disambiguite which literal CLI option was used # and only preserves the 'canonical' representation. original_command_args = ( ctx.obj["dbt_runner_command_args"] if (ctx and ctx.obj and "dbt_runner_command_args" in ctx.obj) else sys.argv ) for deprecated_flags, warning in DEPRECATED_FLAGS_TO_WARNINGS.items(): for deprecated_flag in deprecated_flags: if deprecated_flag in original_command_args: warn(warning) @classmethod def from_dict(cls, command: CliCommand, args_dict: Dict[str, Any]) -> "Flags": command_arg_list = command_params(command, args_dict) ctx = args_to_context(command_arg_list) flags = cls(ctx=ctx) flags.fire_deprecations(ctx=ctx) return flags def set_common_global_flags(self): # Set globals for common.ui if getattr(self, "PRINTER_WIDTH", None) is not None: ui.PRINTER_WIDTH = getattr(self, "PRINTER_WIDTH") if getattr(self, "USE_COLORS", None) is not None: ui.USE_COLOR = getattr(self, "USE_COLORS") # Set globals for common.events.functions functions.WARN_ERROR = getattr(self, "WARN_ERROR", False) if getattr(self, "WARN_ERROR_OPTIONS", None) is not None: functions.WARN_ERROR_OPTIONS = getattr(self, "WARN_ERROR_OPTIONS") # Set globals for common.jinja if getattr(self, "MACRO_DEBUGGING", None) is not None: jinja.MACRO_DEBUGGING = getattr(self, "MACRO_DEBUGGING") # This is here to prevent mypy from complaining about all of the # attributes which we added dynamically. def __getattr__(self, name: str) -> Any: return super().__getattribute__(name) # type: ignore CommandParams = List[str] def command_params(command: CliCommand, args_dict: Dict[str, Any]) -> CommandParams: """Given a command and a dict, returns a list of strings representing the CLI params for that command. The order of this list is consistent with which flags are expected at the parent level vs the command level. e.g. fn("run", {"defer": True, "print": False}) -> ["--no-print", "run", "--defer"] The result of this function can be passed in to the args_to_context function to produce a click context to instantiate Flags with. """ cmd_args = set(command_args(command)) prnt_args = set(parent_args()) default_args = set([x.lower() for x in FLAGS_DEFAULTS.keys()]) res = command.to_list() for k, v in args_dict.items(): k = k.lower() # if a "which" value exists in the args dict, it should match the command provided if k == WHICH_KEY: if v != command.value: raise DbtInternalError( f"Command '{command.value}' does not match value of which: '{v}'" ) continue # param was assigned from defaults and should not be included if k not in (cmd_args | prnt_args) or ( k in default_args and v == FLAGS_DEFAULTS[k.upper()] ): continue # if the param is in parent args, it should come before the arg name # e.g. ["--print", "run"] vs ["run", "--print"] add_fn = res.append if k in prnt_args: def add_fn(x): res.insert(0, x) spinal_cased = k.replace("_", "-") # MultiOption flags come back as lists, but we want to pass them as space separated strings if isinstance(v, list): if len(v) > 0: v = " ".join(v) else: continue if k == "macro" and command == CliCommand.RUN_OPERATION: add_fn(v) # None is a Singleton, False is a Flyweight, only one instance of each. elif (v is None or v is False) and k not in ( # These are None by default but they do not support --no-{flag} "defer_state", "log_format", ): add_fn(f"--no-{spinal_cased}") elif v is True: add_fn(f"--{spinal_cased}") else: add_fn(f"--{spinal_cased}={v}") return res ArgsList = List[str] def parent_args() -> ArgsList: """Return a list representing the params the base click command takes.""" from dbt.cli.main import cli return format_params(cli.params) def command_args(command: CliCommand) -> ArgsList: """Given a command, return a list of strings representing the params that command takes. This function only returns params assigned to a specific command, not those of its parent command. e.g. fn("run") -> ["defer", "favor_state", "exclude", ...] """ import dbt.cli.main as cli CMD_DICT: Dict[CliCommand, ClickCommand] = { CliCommand.BUILD: cli.build, CliCommand.CLEAN: cli.clean, CliCommand.CLONE: cli.clone, CliCommand.COMPILE: cli.compile, CliCommand.DOCS_GENERATE: cli.docs_generate, CliCommand.DOCS_SERVE: cli.docs_serve, CliCommand.DEBUG: cli.debug, CliCommand.DEPS: cli.deps, CliCommand.INIT: cli.init, CliCommand.LIST: cli.list, CliCommand.PARSE: cli.parse, CliCommand.RUN: cli.run, CliCommand.RUN_OPERATION: cli.run_operation, CliCommand.SEED: cli.seed, CliCommand.SHOW: cli.show, CliCommand.SNAPSHOT: cli.snapshot, CliCommand.SOURCE_FRESHNESS: cli.freshness, CliCommand.TEST: cli.test, CliCommand.RETRY: cli.retry, } click_cmd: Optional[ClickCommand] = CMD_DICT.get(command, None) if click_cmd is None: raise DbtInternalError(f"No command found for name '{command.name}'") return format_params(click_cmd.params) def format_params(params: List[Parameter]) -> ArgsList: return [str(x.name) for x in params if not str(x.name).lower().startswith("deprecated_")] ================================================ FILE: core/dbt/cli/main.py ================================================ import functools from copy import copy from dataclasses import dataclass from typing import Callable, List, Optional, Union import click from click.exceptions import BadOptionUsage from click.exceptions import Exit as ClickExit from click.exceptions import NoSuchOption, UsageError from dbt.adapters.factory import register_adapter from dbt.artifacts.schemas.catalog import CatalogArtifact from dbt.artifacts.schemas.run import RunExecutionResult from dbt.cli import params as p from dbt.cli import requires from dbt.cli.exceptions import DbtInternalException, DbtUsageException from dbt.cli.requires import setup_manifest from dbt.contracts.graph.manifest import Manifest from dbt.mp_context import get_mp_context from dbt_common.events.base_types import EventMsg @dataclass class dbtRunnerResult: """Contains the result of an invocation of the dbtRunner""" success: bool exception: Optional[BaseException] = None result: Union[ bool, # debug CatalogArtifact, # docs generate List[str], # list/ls Manifest, # parse None, # clean, deps, init, source RunExecutionResult, # build, compile, run, seed, snapshot, test, run-operation ] = None # Programmatic invocation class dbtRunner: def __init__( self, manifest: Optional[Manifest] = None, callbacks: Optional[List[Callable[[EventMsg], None]]] = None, ) -> None: self.manifest = manifest if callbacks is None: callbacks = [] self.callbacks = callbacks def invoke(self, args: List[str], **kwargs) -> dbtRunnerResult: try: dbt_ctx = cli.make_context(cli.name, args.copy()) dbt_ctx.obj = { "manifest": self.manifest, "callbacks": self.callbacks, "dbt_runner_command_args": args, } for key, value in kwargs.items(): dbt_ctx.params[key] = value # Hack to set parameter source to custom string dbt_ctx.set_parameter_source(key, "kwargs") # type: ignore result, success = cli.invoke(dbt_ctx) return dbtRunnerResult( result=result, success=success, ) except requires.ResultExit as e: return dbtRunnerResult( result=e.result, success=False, ) except requires.ExceptionExit as e: return dbtRunnerResult( exception=e.exception, success=False, ) except (BadOptionUsage, NoSuchOption, UsageError) as e: return dbtRunnerResult( exception=DbtUsageException(e.message), success=False, ) except ClickExit as e: if e.exit_code == 0: return dbtRunnerResult(success=True) return dbtRunnerResult( exception=DbtInternalException(f"unhandled exit code {e.exit_code}"), success=False, ) except BaseException as e: return dbtRunnerResult( exception=e, success=False, ) # approach from https://github.com/pallets/click/issues/108#issuecomment-280489786 def global_flags(func): @p.cache_selected_only @p.debug @p.defer @p.deprecated_defer @p.defer_state @p.deprecated_favor_state @p.deprecated_print @p.deprecated_state @p.fail_fast @p.favor_state @p.indirect_selection @p.log_cache_events @p.log_file_max_bytes @p.log_format @p.log_format_file @p.log_level @p.log_level_file @p.log_path @p.macro_debugging @p.partial_parse @p.partial_parse_file_path @p.partial_parse_file_diff @p.populate_cache @p.print @p.printer_width @p.profile @p.quiet @p.record_timing_info @p.send_anonymous_usage_stats @p.single_threaded @p.show_all_deprecations @p.state @p.static_parser @p.target @p.use_colors @p.use_colors_file @p.use_experimental_parser @p.version @p.version_check @p.warn_error @p.warn_error_options @p.write_json @p.use_fast_test_edges @p.upload_artifacts @functools.wraps(func) def wrapper(*args, **kwargs): return func(*args, **kwargs) return wrapper # dbt @click.group( context_settings={"help_option_names": ["-h", "--help"]}, invoke_without_command=True, no_args_is_help=True, epilog="Specify one of these sub-commands and you can find more help from there.", ) @click.pass_context @global_flags @p.show_resource_report def cli(ctx, **kwargs): """An ELT tool for managing your SQL transformations and data models. For more documentation on these commands, visit: docs.getdbt.com """ # dbt build @cli.command("build") @click.pass_context @global_flags @p.empty @p.event_time_start @p.event_time_end @p.exclude @p.export_saved_queries @p.full_refresh @p.deprecated_include_saved_query @p.profiles_dir @p.project_dir @p.resource_type @p.exclude_resource_type @p.sqlparse_options @p.sample @p.select @p.selector @p.show @p.store_failures @p.target_path @p.threads @p.vars @requires.postflight @requires.preflight @requires.profile @requires.project @requires.catalogs @requires.runtime_config @requires.manifest def build(ctx, **kwargs): """Run all seeds, models, snapshots, and tests in DAG order""" from dbt.task.build import BuildTask task = BuildTask( ctx.obj["flags"], ctx.obj["runtime_config"], ctx.obj["manifest"], ) results = task.run() success = task.interpret_results(results) return results, success # dbt clean @cli.command("clean") @click.pass_context @global_flags @p.clean_project_files_only @p.profiles_dir @p.project_dir @p.target_path @p.vars @requires.postflight @requires.preflight @requires.unset_profile @requires.project def clean(ctx, **kwargs): """Delete all folders in the clean-targets list (usually the dbt_packages and target directories.)""" from dbt.task.clean import CleanTask with CleanTask(ctx.obj["flags"], ctx.obj["project"]) as task: results = task.run() success = task.interpret_results(results) return results, success # dbt docs @cli.group() @click.pass_context @global_flags def docs(ctx, **kwargs): """Generate or serve the documentation website for your project""" # dbt docs generate @docs.command("generate") @click.pass_context @global_flags @p.compile_docs @p.exclude @p.sqlparse_options @p.profiles_dir @p.project_dir @p.select @p.selector @p.empty_catalog @p.static @p.target_path @p.threads @p.vars @requires.postflight @requires.preflight @requires.profile @requires.project @requires.runtime_config @requires.manifest(write=False) def docs_generate(ctx, **kwargs): """Generate the documentation website for your project""" from dbt.task.docs.generate import GenerateTask task = GenerateTask( ctx.obj["flags"], ctx.obj["runtime_config"], ctx.obj["manifest"], ) results = task.run() success = task.interpret_results(results) return results, success # dbt docs serve @docs.command("serve") @click.pass_context @global_flags @p.browser @p.host @p.port @p.profiles_dir @p.project_dir @p.target_path @p.vars @requires.postflight @requires.preflight @requires.profile @requires.project @requires.runtime_config def docs_serve(ctx, **kwargs): """Serve the documentation website for your project""" from dbt.task.docs.serve import ServeTask task = ServeTask( ctx.obj["flags"], ctx.obj["runtime_config"], ) results = task.run() success = task.interpret_results(results) return results, success # dbt compile @cli.command("compile") @click.pass_context @global_flags @p.exclude @p.full_refresh @p.show_output_format @p.introspect @p.profiles_dir @p.project_dir @p.empty @p.select @p.selector @p.inline @p.compile_inject_ephemeral_ctes @p.sqlparse_options @p.target_path @p.threads @p.vars @requires.postflight @requires.preflight @requires.profile @requires.project @requires.catalogs @requires.runtime_config @requires.manifest def compile(ctx, **kwargs): """Generates executable SQL from source, model, test, and analysis files. Compiled SQL files are written to the target/ directory.""" from dbt.task.compile import CompileTask task = CompileTask( ctx.obj["flags"], ctx.obj["runtime_config"], ctx.obj["manifest"], ) results = task.run() success = task.interpret_results(results) return results, success # dbt show @cli.command("show") @click.pass_context @global_flags @p.exclude @p.full_refresh @p.show_output_format @p.show_limit @p.introspect @p.profiles_dir @p.project_dir @p.select @p.selector @p.inline @p.inline_direct @p.sqlparse_options @p.target_path @p.threads @p.vars @requires.postflight @requires.preflight @requires.profile @requires.project @requires.runtime_config def show(ctx, **kwargs): """Generates executable SQL for a named resource or inline query, runs that SQL, and returns a preview of the results. Does not materialize anything to the warehouse.""" from dbt.task.show import ShowTask, ShowTaskDirect if ctx.obj["flags"].inline_direct: # Issue the inline query directly, with no templating. Does not require # loading the manifest. register_adapter(ctx.obj["runtime_config"], get_mp_context()) task = ShowTaskDirect( ctx.obj["flags"], ctx.obj["runtime_config"], ) else: setup_manifest(ctx) task = ShowTask( ctx.obj["flags"], ctx.obj["runtime_config"], ctx.obj["manifest"], ) results = task.run() success = task.interpret_results(results) return results, success # dbt debug @cli.command("debug") @click.pass_context @global_flags @p.debug_connection @p.config_dir @p.profiles_dir_exists_false @p.project_dir @p.vars @requires.postflight @requires.preflight def debug(ctx, **kwargs): """Show information on the current dbt environment and check dependencies, then test the database connection. Not to be confused with the --debug option which increases verbosity.""" from dbt.task.debug import DebugTask task = DebugTask( ctx.obj["flags"], ) results = task.run() success = task.interpret_results(results) return results, success # dbt deps @cli.command("deps") @click.pass_context @global_flags @p.profiles_dir_exists_false @p.project_dir @p.vars @p.source @p.lock @p.upgrade @p.add_package @requires.postflight @requires.preflight @requires.unset_profile @requires.project def deps(ctx, **kwargs): """Install dbt packages specified. In the following case, a new `package-lock.yml` will be generated and the packages are installed: - user updated the packages.yml - user specify the flag --update, which means for packages that are specified as a range, dbt-core will try to install the newer version Otherwise, deps will use `package-lock.yml` as source of truth to install packages. There is a way to add new packages by providing an `--add-package` flag to deps command which will allow user to specify a package they want to add in the format of packagename@version. """ from dbt.task.deps import DepsTask flags = ctx.obj["flags"] if flags.ADD_PACKAGE: if not flags.ADD_PACKAGE["version"] and flags.SOURCE != "local": raise BadOptionUsage( message=f"Version is required in --add-package when a package when source is {flags.SOURCE}", option_name="--add-package", ) with DepsTask(flags, ctx.obj["project"]) as task: results = task.run() success = task.interpret_results(results) return results, success # dbt init @cli.command("init") @click.pass_context @global_flags # for backwards compatibility, accept 'project_name' as an optional positional argument @click.argument("project_name", required=False) @p.profiles_dir_exists_false @p.project_dir @p.skip_profile_setup @p.skip_debug @p.vars @requires.postflight @requires.preflight def init(ctx, **kwargs): """Initialize a new dbt project.""" from dbt.task.init import InitTask with InitTask(ctx.obj["flags"]) as task: results = task.run() success = task.interpret_results(results) return results, success # dbt list @cli.command("list") @click.pass_context @global_flags @p.exclude @p.models @p.output @p.output_keys @p.profiles_dir @p.project_dir @p.resource_type @p.exclude_resource_type @p.raw_select @p.selector @p.target_path @p.vars @requires.postflight @requires.preflight @requires.profile @requires.project @requires.runtime_config @requires.manifest def list(ctx, **kwargs): """List the resources in your project""" from dbt.task.list import ListTask task = ListTask( ctx.obj["flags"], ctx.obj["runtime_config"], ctx.obj["manifest"], ) results = task.run() success = task.interpret_results(results) return results, success # Alias "list" to "ls" ls = copy(cli.commands["list"]) ls.hidden = True cli.add_command(ls, "ls") # dbt parse @cli.command("parse") @click.pass_context @global_flags @p.profiles_dir @p.project_dir @p.target_path @p.threads @p.vars @requires.postflight @requires.preflight @requires.profile @requires.project @requires.catalogs @requires.runtime_config @requires.manifest(write_perf_info=True) def parse(ctx, **kwargs): """Parses the project and provides information on performance""" # manifest generation and writing happens in @requires.manifest return ctx.obj["manifest"], True # dbt run @cli.command("run") @click.pass_context @global_flags @p.exclude @p.full_refresh @p.profiles_dir @p.project_dir @p.empty @p.event_time_start @p.event_time_end @p.sqlparse_options @p.sample @p.select @p.selector @p.target_path @p.threads @p.vars @requires.postflight @requires.preflight @requires.profile @requires.project @requires.catalogs @requires.runtime_config @requires.manifest def run(ctx, **kwargs): """Compile SQL and execute against the current target database.""" from dbt.task.run import RunTask task = RunTask( ctx.obj["flags"], ctx.obj["runtime_config"], ctx.obj["manifest"], ) results = task.run() success = task.interpret_results(results) return results, success # dbt retry @cli.command("retry") @click.pass_context @global_flags @p.project_dir @p.profiles_dir @p.vars @p.target_path @p.threads @p.full_refresh @requires.postflight @requires.preflight @requires.profile @requires.project @requires.runtime_config def retry(ctx, **kwargs): """Retry the nodes that failed in the previous run.""" from dbt.task.retry import RetryTask # Retry will parse manifest inside the task after we consolidate the flags task = RetryTask( ctx.obj["flags"], ctx.obj["runtime_config"], ) results = task.run() success = task.interpret_results(results) return results, success # dbt clone @cli.command("clone") @click.pass_context @global_flags @p.exclude @p.full_refresh @p.profiles_dir @p.project_dir @p.resource_type @p.exclude_resource_type @p.select @p.selector @p.target_path @p.threads @p.vars @requires.preflight @requires.profile @requires.project @requires.runtime_config @requires.manifest @requires.postflight def clone(ctx, **kwargs): """Create clones of selected nodes based on their location in the manifest provided to --state.""" from dbt.task.clone import CloneTask task = CloneTask( ctx.obj["flags"], ctx.obj["runtime_config"], ctx.obj["manifest"], ) results = task.run() success = task.interpret_results(results) return results, success # dbt run operation @cli.command("run-operation") @click.pass_context @global_flags @click.argument("macro") @p.args @p.profiles_dir @p.project_dir @p.target_path @p.threads @p.vars @requires.postflight @requires.preflight @requires.profile @requires.project @requires.runtime_config @requires.manifest def run_operation(ctx, **kwargs): """Run the named macro with any supplied arguments.""" from dbt.task.run_operation import RunOperationTask task = RunOperationTask( ctx.obj["flags"], ctx.obj["runtime_config"], ctx.obj["manifest"], ) results = task.run() success = task.interpret_results(results) return results, success # dbt seed @cli.command("seed") @click.pass_context @global_flags @p.exclude @p.full_refresh @p.profiles_dir @p.project_dir @p.sqlparse_options @p.select @p.selector @p.show @p.target_path @p.threads @p.vars @requires.postflight @requires.preflight @requires.profile @requires.project @requires.catalogs @requires.runtime_config @requires.manifest def seed(ctx, **kwargs): """Load data from csv files into your data warehouse.""" from dbt.task.seed import SeedTask task = SeedTask( ctx.obj["flags"], ctx.obj["runtime_config"], ctx.obj["manifest"], ) results = task.run() success = task.interpret_results(results) return results, success # dbt snapshot @cli.command("snapshot") @click.pass_context @global_flags @p.empty @p.exclude @p.profiles_dir @p.project_dir @p.sqlparse_options @p.select @p.selector @p.target_path @p.threads @p.vars @requires.postflight @requires.preflight @requires.profile @requires.project @requires.catalogs @requires.runtime_config @requires.manifest def snapshot(ctx, **kwargs): """Execute snapshots defined in your project""" from dbt.task.snapshot import SnapshotTask task = SnapshotTask( ctx.obj["flags"], ctx.obj["runtime_config"], ctx.obj["manifest"], ) results = task.run() success = task.interpret_results(results) return results, success # dbt source @cli.group() @click.pass_context @global_flags def source(ctx, **kwargs): """Manage your project's sources""" # dbt source freshness @source.command("freshness") @click.pass_context @global_flags @p.exclude @p.output_path # TODO: Is this ok to re-use? We have three different output params, how much can we consolidate? @p.sqlparse_options @p.profiles_dir @p.project_dir @p.select @p.selector @p.target_path @p.threads @p.vars @requires.postflight @requires.preflight @requires.profile @requires.project @requires.runtime_config @requires.manifest def freshness(ctx, **kwargs): """check the current freshness of the project's sources""" from dbt.task.freshness import FreshnessTask task = FreshnessTask( ctx.obj["flags"], ctx.obj["runtime_config"], ctx.obj["manifest"], ) results = task.run() success = task.interpret_results(results) return results, success # Alias "source freshness" to "snapshot-freshness" snapshot_freshness = copy(cli.commands["source"].commands["freshness"]) # type: ignore snapshot_freshness.hidden = True cli.commands["source"].add_command(snapshot_freshness, "snapshot-freshness") # type: ignore # dbt test @cli.command("test") @click.pass_context @global_flags @p.exclude @p.resource_type @p.exclude_resource_type @p.profiles_dir @p.project_dir @p.sqlparse_options @p.select @p.selector @p.store_failures @p.target_path @p.threads @p.vars @requires.postflight @requires.preflight @requires.profile @requires.project @requires.catalogs @requires.runtime_config @requires.manifest def test(ctx, **kwargs): """Runs tests on data in deployed models. Run this after `dbt run`""" from dbt.task.test import TestTask task = TestTask( ctx.obj["flags"], ctx.obj["runtime_config"], ctx.obj["manifest"], ) results = task.run() success = task.interpret_results(results) return results, success # Support running as a module if __name__ == "__main__": cli() ================================================ FILE: core/dbt/cli/option_types.py ================================================ from typing import Optional import pytz from click import Choice, Context, Parameter, ParamType from dbt.config.utils import normalize_warn_error_options, parse_cli_yaml_string from dbt.event_time.sample_window import SampleWindow from dbt.events import ALL_EVENT_NAMES from dbt.exceptions import OptionNotYamlDictError, ValidationError from dbt_common.exceptions import DbtValidationError from dbt_common.helper_types import WarnErrorOptionsV2 class YAML(ParamType): """The Click YAML type. Converts YAML strings into objects.""" name = "YAML" def convert(self, value, param, ctx): # assume non-string values are a problem if not isinstance(value, str): self.fail(f"Cannot load YAML from type {type(value)}", param, ctx) try: param_option_name = param.opts[0] if param.opts else param.name return parse_cli_yaml_string(value, param_option_name.strip("-")) except (ValidationError, DbtValidationError, OptionNotYamlDictError): self.fail(f"String '{value}' is not valid YAML", param, ctx) class Package(ParamType): """The Click STRING type. Converts string into dict with package name and version. Example package: package-name@1.0.0 package-name """ name = "NewPackage" def convert(self, value, param, ctx): # assume non-string values are a problem if not isinstance(value, str): self.fail(f"Cannot load Package from type {type(value)}", param, ctx) try: package_name, package_version = value.split("@") return {"name": package_name, "version": package_version} except ValueError: return {"name": value, "version": None} class WarnErrorOptionsType(YAML): """The Click WarnErrorOptions type. Converts YAML strings into objects.""" name = "WarnErrorOptionsType" def convert(self, value, param, ctx): # this function is being used by param in click warn_error_options = super().convert(value, param, ctx) normalize_warn_error_options(warn_error_options) return WarnErrorOptionsV2( error=warn_error_options.get("error", []), warn=warn_error_options.get("warn", []), silence=warn_error_options.get("silence", []), valid_error_names=ALL_EVENT_NAMES, ) class SqlParseOptionsType(YAML): """The Click SqlParseOptions type. Parses YAML and applies sqlparse engine settings.""" name = "SqlParseOptionsType" VALID_KEYS = {"MAX_GROUPING_DEPTH", "MAX_GROUPING_TOKENS"} def convert(self, value, param, ctx): if value is None: return None import sqlparse options = super().convert(value, param, ctx) for key, val in options.items(): if key not in self.VALID_KEYS: self.fail( f"Unknown sqlparse option: {key}. " f"Valid options: {', '.join(sorted(self.VALID_KEYS))}", param, ctx, ) # None params may get stringified during `dbt retry` if val is None or (isinstance(val, str) and val.lower() == "none"): setattr(sqlparse.engine.grouping, key, None) else: try: int_val = int(val) except (TypeError, ValueError): self.fail(f"Value for {key} must be an integer, got: {val}", param, ctx) setattr(sqlparse.engine.grouping, key, int_val) return options class Truthy(ParamType): """The Click Truthy type. Converts strings into a "truthy" type""" name = "TRUTHY" def convert(self, value, param, ctx): # assume non-string / non-None values are a problem if not isinstance(value, (str, None)): self.fail(f"Cannot load TRUTHY from type {type(value)}", param, ctx) if value is None or value.lower() in ("0", "false", "f"): return None else: return value class ChoiceTuple(Choice): name = "CHOICE_TUPLE" def convert(self, value, param, ctx): if not isinstance(value, str): for value_item in value: super().convert(value_item, param, ctx) else: super().convert(value, param, ctx) return value class SampleType(ParamType): name = "SAMPLE" def convert( self, value, param: Optional[Parameter], ctx: Optional[Context] ) -> Optional[SampleWindow]: if value is None: return None if isinstance(value, str): try: # Try and identify if it's a "dict" or a "str" if value.lstrip()[0] == "{": param_option_name: str = param.opts[0] if param.opts else param.name # type: ignore parsed_dict = parse_cli_yaml_string(value, param_option_name.strip("-")) sample_window = SampleWindow.from_dict(parsed_dict) sample_window.start = sample_window.start.replace(tzinfo=pytz.UTC) sample_window.end = sample_window.end.replace(tzinfo=pytz.UTC) return sample_window else: return SampleWindow.from_relative_string(value) except Exception as e: self.fail(e.__str__(), param, ctx) else: self.fail(f"Cannot load SAMPLE_WINDOW from type {type(value)}", param, ctx) ================================================ FILE: core/dbt/cli/options.py ================================================ import inspect import typing as t import click from dbt.cli.option_types import ChoiceTuple if t.TYPE_CHECKING: from click import Context from click.parser import _OptionParser, _ParsingState # Implementation from: https://stackoverflow.com/a/48394004 # Note MultiOption options must be specified with type=tuple or type=ChoiceTuple (https://github.com/pallets/click/issues/2012) class MultiOption(click.Option): def __init__(self, *args, **kwargs) -> None: self.save_other_options = kwargs.pop("save_other_options", True) nargs = kwargs.pop("nargs", -1) assert nargs == -1, "nargs, if set, must be -1 not {}".format(nargs) super(MultiOption, self).__init__(*args, **kwargs) # this makes mypy happy, setting these to None causes mypy failures self._previous_parser_process = lambda *args, **kwargs: None self._eat_all_parser = lambda *args, **kwargs: None # validate that multiple=True multiple = kwargs.pop("multiple", None) msg = f"MultiOption named `{self.name}` must have multiple=True (rather than {multiple})" assert multiple, msg # validate that type=tuple or type=ChoiceTuple option_type = kwargs.pop("type", None) msg = f"MultiOption named `{self.name}` must be tuple or ChoiceTuple (rather than {option_type})" if inspect.isclass(option_type): assert issubclass(option_type, tuple), msg else: assert isinstance(option_type, ChoiceTuple), msg def add_to_parser(self, parser: "_OptionParser", ctx: "Context"): def parser_process(value: str, state: "_ParsingState"): # method to hook to the parser.process done = False value_list = str.split(value, " ") if self.save_other_options: # grab everything up to the next option while state.rargs and not done: for prefix in self._eat_all_parser.prefixes: # type: ignore[attr-defined] if state.rargs[0].startswith(prefix): done = True if not done: value_list.append(state.rargs.pop(0)) else: # grab everything remaining value_list += state.rargs state.rargs[:] = [] value_tuple = tuple(value_list) # call the actual process self._previous_parser_process(value_tuple, state) retval = super(MultiOption, self).add_to_parser(parser, ctx) for name in self.opts: our_parser = parser._long_opt.get(name) or parser._short_opt.get(name) if our_parser: self._eat_all_parser = our_parser # type: ignore[assignment] self._previous_parser_process = our_parser.process # mypy doesnt like assingment to a method see https://github.com/python/mypy/issues/708 our_parser.process = parser_process # type: ignore[method-assign] break return retval def type_cast_value(self, ctx: "Context", value: t.Any) -> t.Any: def flatten(data): if isinstance(data, tuple): for x in data: yield from flatten(x) else: yield data # there will be nested tuples to flatten when multiple=True value = super(MultiOption, self).type_cast_value(ctx, value) if value: value = tuple(flatten(value)) return value ================================================ FILE: core/dbt/cli/params.py ================================================ from dataclasses import dataclass from pathlib import Path from typing import Any, Callable, List, Optional import click from dbt.cli.option_types import ( YAML, ChoiceTuple, Package, SampleType, SqlParseOptionsType, WarnErrorOptionsType, ) from dbt.cli.options import MultiOption from dbt.cli.resolvers import default_profiles_dir, default_project_dir from dbt.version import get_version_information from dbt_common.constants import ENGINE_ENV_PREFIX from dbt_common.exceptions import DbtInternalError # --- shared option specs --- # model_decls = ("-m", "--models", "--model") select_decls = ("-s", "--select") select_attrs = { "envvar": None, "help": "Specify the nodes to include.", "cls": MultiOption, "multiple": True, "type": tuple, } @dataclass(frozen=True, init=False) class EngineEnvVar: name: str old_name: Optional[str] = None def __init__(self, envvar: str) -> None: if envvar.startswith(ENGINE_ENV_PREFIX): object.__setattr__(self, "name", envvar) object.__setattr__(self, "old_name", None) elif envvar.startswith("DBT"): object.__setattr__(self, "name", envvar.replace("DBT", f"{ENGINE_ENV_PREFIX}")) object.__setattr__(self, "old_name", envvar) else: raise DbtInternalError( f"Invalid environment variable: {envvar}, this will only happen if we add a new option to dbt that has an envvar that doesn't start with DBT_ or {ENGINE_ENV_PREFIX}" ) # Record of env vars associated with options KNOWN_ENV_VARS: List[EngineEnvVar] = [] def _create_option_and_track_env_var( *args: Any, **kwargs: Any ) -> Callable[[click.decorators.FC], click.decorators.FC]: """ Abstraction that ensures that all options with env vars are * tracked (used in validation later) * have a `DBT_ENGINE_` prefixed env var """ global KNOWN_ENV_VARS envvar = kwargs.get("envvar", None) if isinstance(envvar, str): engine_env_var = EngineEnvVar(envvar) KNOWN_ENV_VARS.append(engine_env_var) if engine_env_var.old_name is not None: # Order matters, the first envvar in the list is preferred kwargs["envvar"] = [engine_env_var.name, engine_env_var.old_name] return click.option(*args, **kwargs) # --- The actual option definitions --- # add_package = _create_option_and_track_env_var( "--add-package", help="Add a package to current package spec, specify it as package-name@version. Change the source with --source flag.", envvar=None, type=Package(), ) args = _create_option_and_track_env_var( "--args", envvar=None, help="Supply arguments to the macro. This dictionary will be mapped to the keyword arguments defined in the selected macro. This argument should be a YAML string, eg. '{my_variable: my_value}'", type=YAML(), ) browser = _create_option_and_track_env_var( "--browser/--no-browser", envvar=None, help="Whether or not to open a local web browser after starting the server", default=True, ) cache_selected_only = _create_option_and_track_env_var( "--cache-selected-only/--no-cache-selected-only", envvar="DBT_CACHE_SELECTED_ONLY", help="At start of run, populate relational cache only for schemas containing selected nodes, or for all schemas of interest.", ) clean_project_files_only = _create_option_and_track_env_var( "--clean-project-files-only / --no-clean-project-files-only", envvar="DBT_CLEAN_PROJECT_FILES_ONLY", help="If disabled, dbt clean will delete all paths specified in clean-paths, even if they're outside the dbt project.", default=True, ) compile_docs = _create_option_and_track_env_var( "--compile/--no-compile", envvar=None, help="Whether or not to run 'dbt compile' as part of docs generation", default=True, ) compile_inject_ephemeral_ctes = _create_option_and_track_env_var( "--inject-ephemeral-ctes/--no-inject-ephemeral-ctes", envvar=None, help="Internal flag controlling injection of referenced ephemeral models' CTEs during `compile`.", hidden=True, default=True, ) config_dir = _create_option_and_track_env_var( "--config-dir", envvar=None, help="Print a system-specific command to access the directory that the current dbt project is searching for a profiles.yml. Then, exit. This flag renders other debug step flags no-ops.", is_flag=True, ) debug = _create_option_and_track_env_var( "--debug/--no-debug", "-d/ ", envvar="DBT_DEBUG", help="Display debug logging during dbt execution. Useful for debugging and making bug reports.", ) debug_connection = _create_option_and_track_env_var( "--connection", envvar=None, help="Test the connection to the target database independent of dependency checks.", is_flag=True, ) # flag was previously named DEFER_MODE defer = _create_option_and_track_env_var( "--defer/--no-defer", envvar="DBT_DEFER", help="If set, resolve unselected nodes by deferring to the manifest within the --state directory.", ) defer_state = _create_option_and_track_env_var( "--defer-state", envvar="DBT_DEFER_STATE", help="Override the state directory for deferral only.", type=click.Path( dir_okay=True, file_okay=False, readable=True, resolve_path=False, path_type=Path, ), ) deprecated_defer = _create_option_and_track_env_var( "--deprecated-defer", envvar="DBT_DEFER_TO_STATE", help="Internal flag for deprecating old env var.", default=False, hidden=True, ) deprecated_favor_state = _create_option_and_track_env_var( "--deprecated-favor-state", envvar="DBT_FAVOR_STATE_MODE", help="Internal flag for deprecating old env var.", ) # Renamed to --export-saved-queries deprecated_include_saved_query = _create_option_and_track_env_var( "--include-saved-query/--no-include-saved-query", envvar="DBT_INCLUDE_SAVED_QUERY", help="Include saved queries in the list of resources to be selected for build command", is_flag=True, hidden=True, ) deprecated_print = _create_option_and_track_env_var( "--deprecated-print/--deprecated-no-print", envvar="DBT_NO_PRINT", help="Internal flag for deprecating old env var.", default=True, hidden=True, callback=lambda ctx, param, value: not value, ) deprecated_state = _create_option_and_track_env_var( "--deprecated-state", envvar="DBT_ARTIFACT_STATE_PATH", help="Internal flag for deprecating old env var.", hidden=True, type=click.Path( dir_okay=True, file_okay=False, readable=True, resolve_path=True, path_type=Path, ), ) empty = _create_option_and_track_env_var( "--empty/--no-empty", envvar="DBT_EMPTY", help="If specified, limit input refs and sources to zero rows.", is_flag=True, ) empty_catalog = _create_option_and_track_env_var( "--empty-catalog", help="If specified, generate empty catalog.json file during the `dbt docs generate` command.", default=False, is_flag=True, ) event_time_end = _create_option_and_track_env_var( "--event-time-end", envvar="DBT_EVENT_TIME_END", help="If specified, the end datetime dbt uses to filter microbatch model inputs (exclusive).", type=click.DateTime(), default=None, ) event_time_start = _create_option_and_track_env_var( "--event-time-start", envvar="DBT_EVENT_TIME_START", help="If specified, the start datetime dbt uses to filter microbatch model inputs (inclusive).", type=click.DateTime(), default=None, ) exclude = _create_option_and_track_env_var( "--exclude", envvar=None, type=tuple, cls=MultiOption, multiple=True, help="Specify the nodes to exclude.", ) exclude_resource_type = _create_option_and_track_env_var( "--exclude-resource-types", "--exclude-resource-type", envvar="DBT_EXCLUDE_RESOURCE_TYPES", help="Specify the types of resources that dbt will exclude", type=ChoiceTuple( [ "metric", "semantic_model", "saved_query", "source", "analysis", "model", "test", "unit_test", "exposure", "snapshot", "seed", "function", "default", ], case_sensitive=False, ), cls=MultiOption, multiple=True, default=(), ) export_saved_queries = _create_option_and_track_env_var( "--export-saved-queries/--no-export-saved-queries", envvar="DBT_EXPORT_SAVED_QUERIES", help="Export saved queries within the 'build' command, otherwise no-op", is_flag=True, hidden=True, ) fail_fast = _create_option_and_track_env_var( "--fail-fast/--no-fail-fast", "-x/ ", envvar="DBT_FAIL_FAST", help="Stop execution on first failure.", ) favor_state = _create_option_and_track_env_var( "--favor-state/--no-favor-state", envvar="DBT_FAVOR_STATE", help="If set, defer to the argument provided to the state flag for resolving unselected nodes, even if the node(s) exist as a database object in the current environment.", ) full_refresh = _create_option_and_track_env_var( "--full-refresh", "-f", envvar="DBT_FULL_REFRESH", help="If specified, dbt will drop incremental models and fully-recalculate the incremental table from the model definition.", is_flag=True, ) host = _create_option_and_track_env_var( "--host", envvar="DBT_HOST", help="host to serve dbt docs on", type=click.STRING, default="127.0.0.1", ) indirect_selection = _create_option_and_track_env_var( "--indirect-selection", envvar="DBT_INDIRECT_SELECTION", help="Choose which tests to select that are adjacent to selected resources. Eager is most inclusive, cautious is most exclusive, and buildable is in between. Empty includes no tests at all.", type=click.Choice(["eager", "cautious", "buildable", "empty"], case_sensitive=False), default="eager", ) inline = _create_option_and_track_env_var( "--inline", envvar=None, help="Pass SQL inline to dbt compile and show", ) inline_direct = _create_option_and_track_env_var( "--inline-direct", envvar=None, help="Internal flag to pass SQL inline to dbt show. Do not load the entire project or apply templating.", hidden=True, ) introspect = _create_option_and_track_env_var( "--introspect/--no-introspect", envvar="DBT_INTROSPECT", help="Whether to scaffold introspective queries as part of compilation", default=True, ) lock = _create_option_and_track_env_var( "--lock", envvar=None, help="Generate the package-lock.yml file without install the packages.", is_flag=True, ) log_cache_events = _create_option_and_track_env_var( "--log-cache-events/--no-log-cache-events", help="Enable verbose logging for relational cache events to help when debugging.", envvar="DBT_LOG_CACHE_EVENTS", ) log_format = _create_option_and_track_env_var( "--log-format", envvar="DBT_LOG_FORMAT", help="Specify the format of logging to the console and the log file. Use --log-format-file to configure the format for the log file differently than the console.", type=click.Choice(["text", "debug", "json", "default"], case_sensitive=False), default="default", ) log_format_file = _create_option_and_track_env_var( "--log-format-file", envvar="DBT_LOG_FORMAT_FILE", help="Specify the format of logging to the log file by overriding the default value and the general --log-format setting.", type=click.Choice(["text", "debug", "json", "default"], case_sensitive=False), default="debug", ) log_level = _create_option_and_track_env_var( "--log-level", envvar="DBT_LOG_LEVEL", help="Specify the minimum severity of events that are logged to the console and the log file. Use --log-level-file to configure the severity for the log file differently than the console.", type=click.Choice(["debug", "info", "warn", "error", "none"], case_sensitive=False), default="info", ) log_level_file = _create_option_and_track_env_var( "--log-level-file", envvar="DBT_LOG_LEVEL_FILE", help="Specify the minimum severity of events that are logged to the log file by overriding the default value and the general --log-level setting.", type=click.Choice(["debug", "info", "warn", "error", "none"], case_sensitive=False), default="debug", ) log_file_max_bytes = _create_option_and_track_env_var( "--log-file-max-bytes", envvar="DBT_LOG_FILE_MAX_BYTES", help="Configure the max file size in bytes for a single dbt.log file, before rolling over. 0 means no limit.", default=10 * 1024 * 1024, # 10mb type=click.INT, hidden=True, ) log_path = _create_option_and_track_env_var( "--log-path", envvar="DBT_LOG_PATH", help="Configure the 'log-path'. Only applies this setting for the current run. Overrides the 'DBT_LOG_PATH' if it is set.", default=None, type=click.Path(resolve_path=True, path_type=Path), ) macro_debugging = _create_option_and_track_env_var( "--macro-debugging/--no-macro-debugging", envvar="DBT_MACRO_DEBUGGING", hidden=True, ) sqlparse_options = _create_option_and_track_env_var( "--sqlparse", envvar="DBT_ENGINE_SQLPARSE", hidden=True, help="Set sqlparse options MAX_GROUPING_DEPTH and MAX_GROUPING_TOKENS as YAML.", type=SqlParseOptionsType(), default='{"MAX_GROUPING_DEPTH": null, "MAX_GROUPING_TOKENS": null}', is_eager=True, ) models = _create_option_and_track_env_var(*model_decls, **select_attrs) # type: ignore[arg-type] # This less standard usage of --output where output_path below is more standard output = _create_option_and_track_env_var( "--output", envvar=None, help="Specify the output format: either JSON or a newline-delimited list of selectors, paths, or names", type=click.Choice(["json", "name", "path", "selector"], case_sensitive=False), default="selector", ) output_keys = _create_option_and_track_env_var( "--output-keys", envvar=None, help=( "Space-delimited listing of node properties to include as custom keys for JSON output. " "Supports nested keys using dot notation " "(e.g. `--output json --output-keys name config.materialized resource_type`)" ), type=tuple, cls=MultiOption, multiple=True, default=[], ) output_path = _create_option_and_track_env_var( "--output", "-o", envvar=None, help="Specify the output path for the JSON report. By default, outputs to 'target/sources.json'", type=click.Path(file_okay=True, dir_okay=False, writable=True), default=None, ) partial_parse = _create_option_and_track_env_var( "--partial-parse/--no-partial-parse", envvar="DBT_PARTIAL_PARSE", help="Allow for partial parsing by looking for and writing to a pickle file in the target directory. This overrides the user configuration file.", default=True, ) partial_parse_file_diff = _create_option_and_track_env_var( "--partial-parse-file-diff/--no-partial-parse-file-diff", envvar="DBT_PARTIAL_PARSE_FILE_DIFF", help="Internal flag for whether to compute a file diff during partial parsing.", hidden=True, default=True, ) partial_parse_file_path = _create_option_and_track_env_var( "--partial-parse-file-path", envvar="DBT_PARTIAL_PARSE_FILE_PATH", help="Internal flag for path to partial_parse.manifest file.", default=None, hidden=True, type=click.Path(exists=True, dir_okay=False, resolve_path=True), ) print = _create_option_and_track_env_var( "--print/--no-print", envvar="DBT_PRINT", help="Output all {{ print() }} macro calls.", default=True, ) populate_cache = _create_option_and_track_env_var( "--populate-cache/--no-populate-cache", envvar="DBT_POPULATE_CACHE", help="At start of run, use `show` or `information_schema` queries to populate a relational cache, which can speed up subsequent materializations.", default=True, ) port = _create_option_and_track_env_var( "--port", envvar=None, help="Specify the port number for the docs server", default=8080, type=click.INT, ) printer_width = _create_option_and_track_env_var( "--printer-width", envvar="DBT_PRINTER_WIDTH", help="Sets the width of terminal output", type=click.INT, default=80, ) profile = _create_option_and_track_env_var( "--profile", envvar="DBT_PROFILE", help="Which existing profile to load. Overrides setting in dbt_project.yml.", ) profiles_dir = _create_option_and_track_env_var( "--profiles-dir", envvar="DBT_PROFILES_DIR", help="Which directory to look in for the profiles.yml file. If not set, dbt will look in the current working directory first, then HOME/.dbt/", default=default_profiles_dir, type=click.Path(exists=True), ) # `dbt debug` uses this because it implements custom behaviour for non-existent profiles.yml directories # `dbt deps` does not load a profile at all # `dbt init` will write profiles.yml if it doesn't yet exist profiles_dir_exists_false = _create_option_and_track_env_var( "--profiles-dir", envvar="DBT_PROFILES_DIR", help="Which directory to look in for the profiles.yml file. If not set, dbt will look in the current working directory first, then HOME/.dbt/", default=default_profiles_dir, type=click.Path(exists=False), ) project_dir = _create_option_and_track_env_var( "--project-dir", envvar="DBT_PROJECT_DIR", help="Which directory to look in for the dbt_project.yml file. Default is the current working directory and its parents.", default=default_project_dir, type=click.Path(exists=True), ) quiet = _create_option_and_track_env_var( "--quiet/--no-quiet", "-q", envvar="DBT_QUIET", help="Suppress all non-error logging to stdout. Does not affect {{ print() }} macro calls.", ) raw_select = _create_option_and_track_env_var(*select_decls, **select_attrs) # type: ignore[arg-type] record_timing_info = _create_option_and_track_env_var( "--record-timing-info", "-r", envvar=None, help="When this option is passed, dbt will output low-level timing stats to the specified file. Example: `--record-timing-info output.profile`", type=click.Path(exists=False), ) resource_type = _create_option_and_track_env_var( "--resource-types", "--resource-type", envvar="DBT_RESOURCE_TYPES", help="Restricts the types of resources that dbt will include", type=ChoiceTuple( [ "metric", "semantic_model", "saved_query", "source", "analysis", "function", "model", "test", "unit_test", "exposure", "snapshot", "seed", "default", "all", ], case_sensitive=False, ), cls=MultiOption, multiple=True, default=(), ) sample = _create_option_and_track_env_var( "--sample", envvar="DBT_SAMPLE", help="Run in sample mode with given SAMPLE_WINDOW spec, such that ref/source calls are sampled by the sample window.", default=None, type=SampleType(), ) # `--select` and `--models` are analogous for most commands except `dbt list` for legacy reasons. # Most CLI arguments should use the combined `select` option that aliases `--models` to `--select`. # However, if you need to split out these separators (like `dbt ls`), use the `models` and `raw_select` options instead. # See https://github.com/dbt-labs/dbt-core/pull/6774#issuecomment-1408476095 for more info. select = _create_option_and_track_env_var(*select_decls, *model_decls, **select_attrs) # type: ignore[arg-type] selector = _create_option_and_track_env_var( "--selector", envvar=None, help="The selector name to use, as defined in selectors.yml", ) send_anonymous_usage_stats = _create_option_and_track_env_var( "--send-anonymous-usage-stats/--no-send-anonymous-usage-stats", envvar="DBT_SEND_ANONYMOUS_USAGE_STATS", help="Send anonymous usage stats to dbt Labs.", default=True, ) show = _create_option_and_track_env_var( "--show", envvar=None, help="Show a sample of the loaded data in the terminal", is_flag=True, ) show_limit = _create_option_and_track_env_var( "--limit", envvar=None, help="Limit the number of results returned by dbt show", type=click.INT, default=5, ) show_output_format = _create_option_and_track_env_var( "--output", envvar=None, help="Output format for dbt compile and dbt show", type=click.Choice(["json", "text"], case_sensitive=False), default="text", ) show_resource_report = _create_option_and_track_env_var( "--show-resource-report/--no-show-resource-report", default=False, envvar="DBT_SHOW_RESOURCE_REPORT", hidden=True, ) # TODO: The env var is a correction! # The original env var was `DBT_TEST_SINGLE_THREADED`. # This broke the existing naming convention. # This will need to be communicated as a change to the community! # # N.B. This flag is only used for testing, hence it's hidden from help text. single_threaded = _create_option_and_track_env_var( "--single-threaded/--no-single-threaded", envvar="DBT_SINGLE_THREADED", default=False, hidden=True, ) show_all_deprecations = _create_option_and_track_env_var( "--show-all-deprecations/--no-show-all-deprecations", envvar=None, help="By default, each type of a deprecation warning is only shown once. Use this flag to show all deprecation warning instances.", is_flag=True, default=False, ) skip_profile_setup = _create_option_and_track_env_var( "--skip-profile-setup", "-s", envvar=None, help="Skip interactive profile setup.", is_flag=True, ) skip_debug = _create_option_and_track_env_var( "--skip-debug", envvar=None, help="Skip running dbt debug after project initialization.", is_flag=True, default=False, ) source = _create_option_and_track_env_var( "--source", envvar=None, help="Source to download page from, must be one of hub, git, or local. Defaults to hub.", type=click.Choice(["hub", "git", "local"], case_sensitive=True), default="hub", ) state = _create_option_and_track_env_var( "--state", envvar="DBT_STATE", help="Unless overridden, use this state directory for both state comparison and deferral.", type=click.Path( dir_okay=True, file_okay=False, readable=True, resolve_path=False, path_type=Path, ), ) static = _create_option_and_track_env_var( "--static", help="Generate an additional static_index.html with manifest and catalog built-in.", default=False, is_flag=True, ) static_parser = _create_option_and_track_env_var( "--static-parser/--no-static-parser", envvar="DBT_STATIC_PARSER", help="Use the static parser.", default=True, ) store_failures = _create_option_and_track_env_var( "--store-failures", envvar="DBT_STORE_FAILURES", help="Store test results (failing rows) in the database", is_flag=True, ) target = _create_option_and_track_env_var( "--target", "-t", envvar="DBT_TARGET", help="Which target to load for the given profile", ) target_path = _create_option_and_track_env_var( "--target-path", envvar="DBT_TARGET_PATH", help="Configure the 'target-path'. Only applies this setting for the current run. Overrides the 'DBT_TARGET_PATH' if it is set.", type=click.Path(), ) threads = _create_option_and_track_env_var( "--threads", envvar=None, help="Specify number of threads to use while executing models. Overrides settings in profiles.yml.", default=None, type=click.INT, ) upgrade = _create_option_and_track_env_var( "--upgrade", envvar=None, help="Upgrade packages to the latest version.", is_flag=True, ) use_colors = _create_option_and_track_env_var( "--use-colors/--no-use-colors", envvar="DBT_USE_COLORS", help="Specify whether log output is colorized in the console and the log file. Use --use-colors-file/--no-use-colors-file to colorize the log file differently than the console.", default=True, ) use_colors_file = _create_option_and_track_env_var( "--use-colors-file/--no-use-colors-file", envvar="DBT_USE_COLORS_FILE", help="Specify whether log file output is colorized by overriding the default value and the general --use-colors/--no-use-colors setting.", default=True, ) use_experimental_parser = _create_option_and_track_env_var( "--use-experimental-parser/--no-use-experimental-parser", envvar="DBT_USE_EXPERIMENTAL_PARSER", help="Enable experimental parsing features.", ) use_fast_test_edges = _create_option_and_track_env_var( "--use-fast-test-edges/--no-use-fast-test-edges", envvar="DBT_USE_FAST_TEST_EDGES", default=False, hidden=True, ) vars = _create_option_and_track_env_var( "--vars", envvar=None, help="Supply variables to the project. This argument overrides variables defined in your dbt_project.yml file. This argument should be a YAML string, eg. '{my_variable: my_value}'", type=YAML(), default="{}", ) # TODO: when legacy flags are deprecated use # click.version_option instead of a callback def _version_callback(ctx, _param, value): if not value or ctx.resilient_parsing: return click.echo(get_version_information()) ctx.exit() version = _create_option_and_track_env_var( "--version", "-V", "-v", callback=_version_callback, envvar=None, expose_value=False, help="Show version information and exit", is_eager=True, is_flag=True, ) version_check = _create_option_and_track_env_var( "--version-check/--no-version-check", envvar="DBT_VERSION_CHECK", help="If set, ensure the installed dbt version matches the require-dbt-version specified in the dbt_project.yml file (if any). Otherwise, allow them to differ.", default=True, ) warn_error = _create_option_and_track_env_var( "--warn-error", envvar="DBT_WARN_ERROR", help="If dbt would normally warn, instead raise an exception. Examples include --select that selects nothing, deprecations, configurations with no associated models, invalid test configurations, and missing sources/refs in tests.", default=None, is_flag=True, ) warn_error_options = _create_option_and_track_env_var( "--warn-error-options", envvar="DBT_WARN_ERROR_OPTIONS", default="{}", help="""If dbt would normally warn, instead raise an exception based on error/warn configuration. Examples include --select that selects nothing, deprecations, configurations with no associated models, invalid test configurations, and missing sources/refs in tests. This argument should be a YAML string, with keys 'error' or 'warn'. eg. '{"error": "all", "warn": ["NoNodesForSelectionCriteria"]}'""", type=WarnErrorOptionsType(), ) write_json = _create_option_and_track_env_var( "--write-json/--no-write-json", envvar="DBT_WRITE_JSON", help="Whether or not to write the manifest.json and run_results.json files to the target directory", default=True, ) upload_artifacts = _create_option_and_track_env_var( "--upload-to-artifacts-ingest-api/--no-upload-to-artifacts-ingest-api", envvar="DBT_UPLOAD_TO_ARTIFACTS_INGEST_API", help="Whether or not to upload the artifacts to the dbt Cloud API", default=False, ) ================================================ FILE: core/dbt/cli/requires.py ================================================ import importlib.util import os import time import traceback from functools import update_wrapper from typing import Dict, Optional from click import Context import dbt.tracking from dbt.adapters.factory import adapter_management, get_adapter, register_adapter from dbt.cli.exceptions import ExceptionExit, ResultExit from dbt.cli.flags import Flags from dbt.config import RuntimeConfig from dbt.config.catalogs import get_active_write_integration, load_catalogs from dbt.config.runtime import UnsetProfile, load_profile, load_project from dbt.context.providers import generate_runtime_macro_context from dbt.context.query_header import generate_query_header_context from dbt.deprecations import show_deprecations_summary from dbt.env_vars import KNOWN_ENGINE_ENV_VARS, validate_engine_env_vars from dbt.events.logging import setup_event_logger from dbt.events.types import ( ArtifactUploadError, CommandCompleted, MainEncounteredError, MainReportArgs, MainReportVersion, MainStackTrace, MainTrackingUserState, ResourceReport, ) from dbt.exceptions import DbtProjectError, FailFastError from dbt.flags import get_flag_dict, get_flags, set_flags from dbt.mp_context import get_mp_context from dbt.parser.manifest import parse_manifest from dbt.plugins import set_up_plugin_manager from dbt.profiler import profiler from dbt.tracking import active_user, initialize_from_flags, track_run from dbt.utils import try_get_max_rss_kb from dbt.utils.artifact_upload import upload_artifacts from dbt.version import installed as installed_version from dbt_common.clients.system import get_env from dbt_common.context import get_invocation_context, set_invocation_context from dbt_common.events.base_types import EventLevel from dbt_common.events.event_manager_client import get_event_manager from dbt_common.events.functions import LOG_VERSION, fire_event from dbt_common.events.helpers import get_json_string_utcnow from dbt_common.exceptions import DbtBaseException as DbtException from dbt_common.invocation import reset_invocation_id from dbt_common.record import ( Recorder, RecorderMode, get_record_mode_from_env, get_record_types_from_dict, get_record_types_from_env, ) from dbt_common.utils import cast_dict_to_dict_of_strings def _cross_propagate_engine_env_vars(env_dict: Dict[str, str]) -> None: for env_var in KNOWN_ENGINE_ENV_VARS: if env_var.old_name is not None: # If the old name is in the env dict, and not the new name, set the new name based on the old name if env_var.old_name in env_dict and env_var.name not in env_dict: env_dict[env_var.name] = env_dict[env_var.old_name] # If the new name is in the env dict, override the old name with it elif env_var.name in env_dict: env_dict[env_var.old_name] = env_dict[env_var.name] def preflight(func): def wrapper(*args, **kwargs): ctx = args[0] assert isinstance(ctx, Context) ctx.obj = ctx.obj or {} set_invocation_context({}) # Record/Replay setup_record_replay() # Must be set after record/replay is set up so that the env can be # recorded or replayed if needed. env_dict = get_env() _cross_propagate_engine_env_vars(env_dict) get_invocation_context()._env = env_dict # Flags flags = Flags(ctx) ctx.obj["flags"] = flags set_flags(flags) get_event_manager().require_warn_or_error_handling = ( flags.require_all_warnings_handled_by_warn_error ) # Reset invocation_id for each 'invocation' of a dbt command (can happen multiple times in a single process) reset_invocation_id() # Logging callbacks = ctx.obj.get("callbacks", []) setup_event_logger(flags=flags, callbacks=callbacks) # Tracking initialize_from_flags(flags.SEND_ANONYMOUS_USAGE_STATS, flags.PROFILES_DIR) ctx.with_resource(track_run(run_command=flags.WHICH)) # Now that we have our logger, fire away! fire_event(MainReportVersion(version=str(installed_version), log_version=LOG_VERSION)) flags_dict_str = cast_dict_to_dict_of_strings(get_flag_dict()) fire_event(MainReportArgs(args=flags_dict_str)) # Deprecation warnings flags.fire_deprecations(ctx) if active_user is not None: # mypy appeasement, always true fire_event(MainTrackingUserState(user_state=active_user.state())) # Profiling if flags.RECORD_TIMING_INFO: ctx.with_resource(profiler(enable=True, outfile=flags.RECORD_TIMING_INFO)) # Adapter management ctx.with_resource(adapter_management()) # Validate engine env var restricted name space validate_engine_env_vars() return func(*args, **kwargs) return update_wrapper(wrapper, func) def setup_record_replay(): rec_mode = get_record_mode_from_env() rec_types = get_record_types_from_env() recorder: Optional[Recorder] = None if rec_mode == RecorderMode.REPLAY: previous_recording_path = os.environ.get( "DBT_ENGINE_RECORDER_FILE_PATH" ) or os.environ.get("DBT_RECORDER_FILE_PATH") recorder = Recorder( RecorderMode.REPLAY, types=rec_types, previous_recording_path=previous_recording_path ) elif rec_mode == RecorderMode.DIFF: previous_recording_path = os.environ.get( "DBT_ENGINE_RECORDER_FILE_PATH" ) or os.environ.get("DBT_RECORDER_FILE_PATH") # ensure types match the previous recording types = get_record_types_from_dict(previous_recording_path) recorder = Recorder( RecorderMode.DIFF, types=types, previous_recording_path=previous_recording_path ) elif rec_mode == RecorderMode.RECORD: recorder = Recorder(RecorderMode.RECORD, types=rec_types) get_invocation_context().recorder = recorder def tear_down_record_replay(): recorder = get_invocation_context().recorder if recorder is not None: if recorder.mode == RecorderMode.RECORD: recorder.write() if recorder.mode == RecorderMode.DIFF: recorder.write() recorder.write_diffs(diff_file_name="recording_diffs.json") elif recorder.mode == RecorderMode.REPLAY: recorder.write_diffs("replay_diffs.json") def postflight(func): """The decorator that handles all exception handling for the click commands. This decorator must be used before any other decorators that may throw an exception.""" def wrapper(*args, **kwargs): ctx = args[0] start_func = time.perf_counter() success = False try: result, success = func(*args, **kwargs) except FailFastError as e: fire_event(MainEncounteredError(exc=str(e))) raise ResultExit(e.result) except DbtException as e: fire_event(MainEncounteredError(exc=str(e))) raise ExceptionExit(e) except BaseException as e: fire_event(MainEncounteredError(exc=str(e))) fire_event(MainStackTrace(stack_trace=traceback.format_exc())) raise ExceptionExit(e) finally: # Fire ResourceReport, but only on systems which support the resource # module. (Skip it on Windows). try: if get_flags().upload_to_artifacts_ingest_api: upload_artifacts( get_flags().project_dir, get_flags().target_path, ctx.command.name ) except Exception as e: fire_event(ArtifactUploadError(msg=str(e))) show_deprecations_summary() if importlib.util.find_spec("resource") is not None: import resource rusage = resource.getrusage(resource.RUSAGE_SELF) fire_event( ResourceReport( command_name=ctx.command.name, command_success=success, command_wall_clock_time=time.perf_counter() - start_func, process_user_time=rusage.ru_utime, process_kernel_time=rusage.ru_stime, process_mem_max_rss=try_get_max_rss_kb() or rusage.ru_maxrss, process_in_blocks=rusage.ru_inblock, process_out_blocks=rusage.ru_oublock, ), ( EventLevel.INFO if "flags" in ctx.obj and ctx.obj["flags"].SHOW_RESOURCE_REPORT else None ), ) fire_event( CommandCompleted( command=ctx.command_path, success=success, completed_at=get_json_string_utcnow(), elapsed=time.perf_counter() - start_func, ) ) tear_down_record_replay() if not success: raise ResultExit(result) return (result, success) return update_wrapper(wrapper, func) # TODO: UnsetProfile is necessary for deps and clean to load a project. # This decorator and its usage can be removed once https://github.com/dbt-labs/dbt-core/issues/6257 is closed. def unset_profile(func): def wrapper(*args, **kwargs): ctx = args[0] assert isinstance(ctx, Context) profile = UnsetProfile() ctx.obj["profile"] = profile return func(*args, **kwargs) return update_wrapper(wrapper, func) def profile(func): def wrapper(*args, **kwargs): ctx = args[0] assert isinstance(ctx, Context) flags = ctx.obj["flags"] # TODO: Generalize safe access to flags.THREADS: # https://github.com/dbt-labs/dbt-core/issues/6259 threads = getattr(flags, "THREADS", None) profile = load_profile(flags.PROJECT_DIR, flags.VARS, flags.PROFILE, flags.TARGET, threads) ctx.obj["profile"] = profile get_invocation_context().uses_adapter(profile.credentials.type) return func(*args, **kwargs) return update_wrapper(wrapper, func) def project(func): def wrapper(*args, **kwargs): ctx = args[0] assert isinstance(ctx, Context) # TODO: Decouple target from profile, and remove the need for profile here: # https://github.com/dbt-labs/dbt-core/issues/6257 if not ctx.obj.get("profile"): raise DbtProjectError("profile required for project") flags = ctx.obj["flags"] # TODO deprecations warnings fired from loading the project will lack # the project_id in the snowplow event. # Determine if vars should be required during project loading. # Commands that don't need vars evaluated (like 'deps', 'clean') # should use lenient mode (require_vars=False) to allow missing vars. # Commands that validate or execute (like 'run', 'compile', 'build', 'debug') should use # strict mode (require_vars=True) to show helpful "Required var X not found" errors. # If adding more commands to lenient mode, update this condition. require_vars = flags.WHICH != "deps" project = load_project( flags.PROJECT_DIR, flags.VERSION_CHECK, ctx.obj["profile"], flags.VARS, validate=True, require_vars=require_vars, ) ctx.obj["project"] = project # Plugins set_up_plugin_manager(project_name=project.project_name) if dbt.tracking.active_user is not None: project_id = None if project is None else project.hashed_name() dbt.tracking.track_project_id({"project_id": project_id}) return func(*args, **kwargs) return update_wrapper(wrapper, func) def runtime_config(func): """A decorator used by click command functions for generating a runtime config given a profile and project. """ def wrapper(*args, **kwargs): ctx = args[0] assert isinstance(ctx, Context) req_strs = ["profile", "project"] reqs = [ctx.obj.get(req_str) for req_str in req_strs] if None in reqs: raise DbtProjectError("profile and project required for runtime_config") config = RuntimeConfig.from_parts( ctx.obj["project"], ctx.obj["profile"], ctx.obj["flags"], ) ctx.obj["runtime_config"] = config if dbt.tracking.active_user is not None: adapter_type = ( getattr(config.credentials, "type", None) if hasattr(config, "credentials") else None ) adapter_unique_id = ( config.credentials.hashed_unique_field() if hasattr(config, "credentials") else None ) dbt.tracking.track_adapter_info( { "adapter_type": adapter_type, "adapter_unique_id": adapter_unique_id, } ) return func(*args, **kwargs) return update_wrapper(wrapper, func) def catalogs(func): """A decorator used by click command functions for loading catalogs""" def wrapper(*args, **kwargs): ctx = args[0] assert isinstance(ctx, Context) req_strs = ["flags", "profile", "project"] reqs = [ctx.obj.get(req_str) for req_str in req_strs] if None in reqs: raise DbtProjectError("profile and flags required to load catalogs") flags = ctx.obj["flags"] ctx_project = ctx.obj["project"] _catalogs = load_catalogs(flags.PROJECT_DIR, ctx_project.project_name, flags.VARS) ctx.obj["catalogs"] = _catalogs return func(*args, **kwargs) return update_wrapper(wrapper, func) def manifest(*args0, write=True, write_perf_info=False): """A decorator used by click command functions for generating a manifest given a profile, project, and runtime config. This also registers the adapter from the runtime config and conditionally writes the manifest to disk. """ def outer_wrapper(func): def wrapper(*args, **kwargs): ctx = args[0] assert isinstance(ctx, Context) setup_manifest(ctx, write=write, write_perf_info=write_perf_info) return func(*args, **kwargs) return update_wrapper(wrapper, func) # if there are no args, the decorator was used without params @decorator # otherwise, the decorator was called with params @decorator(arg) if len(args0) == 0: return outer_wrapper return outer_wrapper(args0[0]) def setup_manifest(ctx: Context, write: bool = True, write_perf_info: bool = False): """Load the manifest and add it to the context.""" req_strs = ["profile", "project", "runtime_config"] reqs = [ctx.obj.get(dep) for dep in req_strs] if None in reqs: raise DbtProjectError("profile, project, and runtime_config required for manifest") runtime_config = ctx.obj["runtime_config"] catalogs = ctx.obj["catalogs"] if "catalogs" in ctx.obj else [] active_integrations = [get_active_write_integration(catalog) for catalog in catalogs] # if a manifest has already been set on the context, don't overwrite it if ctx.obj.get("manifest") is None: ctx.obj["manifest"] = parse_manifest( runtime_config, write_perf_info, write, ctx.obj["flags"].write_json, active_integrations, ) adapter = get_adapter(runtime_config) else: register_adapter(runtime_config, get_mp_context()) adapter = get_adapter(runtime_config) adapter.set_macro_context_generator(generate_runtime_macro_context) # type: ignore[arg-type] adapter.set_macro_resolver(ctx.obj["manifest"]) query_header_context = generate_query_header_context(adapter.config, ctx.obj["manifest"]) # type: ignore[attr-defined] adapter.connections.set_query_header(query_header_context) for integration in active_integrations: adapter.add_catalog_integration(integration) ================================================ FILE: core/dbt/cli/resolvers.py ================================================ from pathlib import Path from dbt.config.project import PartialProject from dbt.exceptions import DbtProjectError def default_project_dir() -> Path: paths = list(Path.cwd().parents) paths.insert(0, Path.cwd()) return next((x for x in paths if (x / "dbt_project.yml").exists()), Path.cwd()) def default_profiles_dir() -> Path: return Path.cwd() if (Path.cwd() / "profiles.yml").exists() else Path.home() / ".dbt" def default_log_path(project_dir: Path, verify_version: bool = False) -> Path: """If available, derive a default log path from dbt_project.yml. Otherwise, default to "logs". Known limitations: 1. Using PartialProject here, so no jinja rendering of log-path. 2. Programmatic invocations of the cli via dbtRunner may pass a Project object directly, which is not being taken into consideration here to extract a log-path. """ default_log_path = Path("logs") try: partial = PartialProject.from_project_root(str(project_dir), verify_version=verify_version) partial_log_path = partial.project_dict.get("log-path") or default_log_path default_log_path = Path(project_dir) / partial_log_path except DbtProjectError: pass return default_log_path ================================================ FILE: core/dbt/cli/types.py ================================================ from enum import Enum from typing import List from dbt_common.exceptions import DbtInternalError class Command(Enum): BUILD = "build" CLEAN = "clean" COMPILE = "compile" CLONE = "clone" DOCS_GENERATE = "generate" DOCS_SERVE = "serve" DEBUG = "debug" DEPS = "deps" INIT = "init" LIST = "list" PARSE = "parse" RUN = "run" RUN_OPERATION = "run-operation" SEED = "seed" SHOW = "show" SNAPSHOT = "snapshot" SOURCE_FRESHNESS = "freshness" TEST = "test" RETRY = "retry" @classmethod def from_str(cls, s: str) -> "Command": try: return cls(s) except ValueError: raise DbtInternalError(f"No value '{s}' exists in Command enum") def to_list(self) -> List[str]: return { Command.DOCS_GENERATE: ["docs", "generate"], Command.DOCS_SERVE: ["docs", "serve"], Command.SOURCE_FRESHNESS: ["source", "freshness"], }.get(self, [self.value]) ================================================ FILE: core/dbt/clients/README.md ================================================ # Clients README ### Jinja #### How are materializations defined Model materializations are defined by adapters. See the [dbt-adapters](https://github.com/dbt-labs/dbt-adapters) project for [base implementations](https://github.com/dbt-labs/dbt-adapters/tree/main/dbt-adapters/src/dbt/include/global_project/macros/materializations/models). Materializations are defined using syntax that isn't part of the Jinja standard library. These tags are referenced internally, and materializations can be overridden in user projects when users have specific needs. ``` -- Pseudocode for arguments {% materialization <name>, <target name := one_of{default, adapter}> %}' … {% endmaterialization %} ``` These blocks are referred to Jinja extensions. Extensions are defined as part of the accepted Jinja code encapsulated within a dbt project. This includes system code used internally by dbt and user space (i.e. user-defined) macros. Extensions exist to help Jinja users create reusable code blocks or abstract objects--for us, materializations are a great use-case since we pass these around as arguments within dbt system code. The code that defines this extension is a class `MaterializationExtension` and a `parse` routine. That code lives in [clients/jinja.py](https://github.com/dbt-labs/dbt-core/blob/main/core/dbt/clients/jinja.py). The routine enables Jinja to parse (i.e. recognize) the unique comma separated arg structure our `materialization` tags exhibit (the `table, default` as seen above). ================================================ FILE: core/dbt/clients/__init__.py ================================================ ================================================ FILE: core/dbt/clients/checked_load.py ================================================ import collections import dataclasses from typing import Any, Dict, List, Optional, Tuple import yaml from dbt import deprecations from dbt.clients.yaml_helper import load_yaml_text # the C version is faster, but it doesn't always exist try: from yaml import CSafeLoader as SafeLoader except ImportError: from yaml import SafeLoader # type: ignore # noqa: F401 @dataclasses.dataclass class YamlCheckFailure: failure_type: str message: str def checked_load(contents) -> Tuple[Optional[Dict[str, Any]], List[YamlCheckFailure]]: # A hacky (but sadly justified) method for modifying a bit of PyYAML. We create # a new local subclass of SafeLoader, since we need to associate state with # the static class, but static classes do not have non-static state. This allows # us to be sure we have exclusive access to the class. class CheckedLoader(SafeLoader): check_failures: List[YamlCheckFailure] = [] def construct_mapping(self, node, deep=False): if not isinstance(node, yaml.MappingNode): raise yaml.constructor.ConstructorError( None, None, "expected a mapping node, but found %s" % node.id, node.start_mark ) is_override = ( len(node.value) > 0 and len(node.value[0]) > 0 and getattr(node.value[0][0], "value") == "<<" ) self.flatten_mapping(node) mapping = {} for key_node, value_node in node.value: key = self.construct_object(key_node, deep=deep) if not isinstance(key, collections.abc.Hashable): raise yaml.constructor.ConstructorError( "while constructing a mapping", node.start_mark, "found unhashable key", key_node.start_mark, ) value = self.construct_object(value_node, deep=deep) if not is_override and key in mapping: start_mark = str(key_node.start_mark) if start_mark.startswith(" in"): # this means it was at the top level message = f"Duplicate key '{key}' {start_mark.lstrip()}" else: message = f"Duplicate key '{key}' at {key_node.start_mark}" self.check_failures.append(YamlCheckFailure("duplicate_key", message)) mapping[key] = value return mapping CheckedLoader.add_constructor( yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, CheckedLoader.construct_mapping ) dct = load_yaml_text(contents, loader=CheckedLoader) check_failures = CheckedLoader.check_failures return (dct, check_failures) def issue_deprecation_warnings_for_failures(failures: List[YamlCheckFailure], file: str): for failure in failures: if failure.failure_type == "duplicate_key": deprecations.warn( "duplicate-yaml-keys-deprecation", duplicate_description=failure.message, file=file, ) ================================================ FILE: core/dbt/clients/git.py ================================================ import os.path import re from packaging import version from dbt.events.types import ( GitNothingToDo, GitProgressCheckedOutAt, GitProgressCheckoutRevision, GitProgressPullingNewDependency, GitProgressUpdatedCheckoutRange, GitProgressUpdatingExistingDependency, GitSparseCheckoutSubdirectory, ) from dbt.exceptions import ( CommandResultError, DbtRuntimeError, GitCheckoutError, GitCloningError, UnknownGitCloningProblemError, ) from dbt_common.clients.system import rmdir, run_cmd from dbt_common.events.functions import fire_event def _is_commit(revision: str) -> bool: # match SHA-1 git commit return bool(re.match(r"\b[0-9a-f]{40}\b", revision)) def clone(repo, cwd, dirname=None, remove_git_dir=False, revision=None, subdirectory=None): has_revision = revision is not None is_commit = _is_commit(revision or "") clone_cmd = ["git", "clone", "--depth", "1"] if subdirectory: fire_event(GitSparseCheckoutSubdirectory(subdir=subdirectory)) out, _ = run_cmd(cwd, ["git", "--version"], env={"LC_ALL": "C"}) git_version = version.parse(re.search(r"\d+\.\d+\.\d+", out.decode("utf-8")).group(0)) if not git_version >= version.parse("2.25.0"): # 2.25.0 introduces --sparse raise RuntimeError( "Please update your git version to pull a dbt package " "from a subdirectory: your version is {}, >= 2.25.0 needed".format(git_version) ) clone_cmd.extend(["--filter=blob:none", "--sparse"]) if has_revision and not is_commit: clone_cmd.extend(["--branch", revision]) clone_cmd.append(repo) if dirname is not None: clone_cmd.append(dirname) try: result = run_cmd(cwd, clone_cmd, env={"LC_ALL": "C"}) except CommandResultError as exc: raise GitCloningError(repo, revision, exc) if subdirectory: cwd_subdir = os.path.join(cwd, dirname or "") clone_cmd_subdir = ["git", "sparse-checkout", "set", subdirectory] try: run_cmd(cwd_subdir, clone_cmd_subdir) except CommandResultError as exc: raise GitCloningError(repo, revision, exc) if remove_git_dir: rmdir(os.path.join(dirname, ".git")) return result def list_tags(cwd): out, err = run_cmd(cwd, ["git", "tag", "--list"], env={"LC_ALL": "C"}) tags = out.decode("utf-8").strip().split("\n") return tags def _checkout(cwd, repo, revision): fire_event(GitProgressCheckoutRevision(revision=revision)) fetch_cmd = ["git", "fetch", "origin", "--depth", "1"] if _is_commit(revision): run_cmd(cwd, fetch_cmd + [revision]) else: run_cmd(cwd, ["git", "remote", "set-branches", "origin", revision]) run_cmd(cwd, fetch_cmd + ["--tags", revision]) if _is_commit(revision): spec = revision # Prefer tags to branches if one exists elif revision in list_tags(cwd): spec = "tags/{}".format(revision) else: spec = "origin/{}".format(revision) out, err = run_cmd(cwd, ["git", "reset", "--hard", spec], env={"LC_ALL": "C"}) return out, err def checkout(cwd, repo, revision=None): if revision is None: revision = "HEAD" try: return _checkout(cwd, repo, revision) except CommandResultError as exc: raise GitCheckoutError(repo=repo, revision=revision, error=exc) def get_current_sha(cwd): out, err = run_cmd(cwd, ["git", "rev-parse", "HEAD"], env={"LC_ALL": "C"}) return out.decode("utf-8").strip() def remove_remote(cwd): return run_cmd(cwd, ["git", "remote", "rm", "origin"], env={"LC_ALL": "C"}) def clone_and_checkout( repo, cwd, dirname=None, remove_git_dir=False, revision=None, subdirectory=None ): exists = None try: _, err = clone( repo, cwd, dirname=dirname, remove_git_dir=remove_git_dir, subdirectory=subdirectory, ) except CommandResultError as exc: err = exc.stderr exists = re.match("fatal: destination path '(.+)' already exists", err) if not exists: raise UnknownGitCloningProblemError(repo) directory = None start_sha = None if exists: directory = exists.group(1) fire_event(GitProgressUpdatingExistingDependency(dir=directory)) else: matches = re.match("Cloning into '(.+)'", err.decode("utf-8")) if matches is None: raise DbtRuntimeError(f'Error cloning {repo} - never saw "Cloning into ..." from git') directory = matches.group(1) fire_event(GitProgressPullingNewDependency(dir=directory)) full_path = os.path.join(cwd, directory) start_sha = get_current_sha(full_path) checkout(full_path, repo, revision) end_sha = get_current_sha(full_path) if exists: if start_sha == end_sha: fire_event(GitNothingToDo(sha=start_sha[:7])) else: fire_event( GitProgressUpdatedCheckoutRange(start_sha=start_sha[:7], end_sha=end_sha[:7]) ) else: fire_event(GitProgressCheckedOutAt(end_sha=end_sha[:7])) return os.path.join(directory, subdirectory or "") ================================================ FILE: core/dbt/clients/jinja.py ================================================ import re import threading from contextlib import contextmanager from typing import Any, Dict, List, NoReturn, Optional, Tuple, Union import jinja2 import jinja2.ext import jinja2.nativetypes # type: ignore import jinja2.nodes import jinja2.parser import jinja2.sandbox from dbt.contracts.graph.nodes import GenericTestNode from dbt.exceptions import ( DbtInternalError, MaterializtionMacroNotUsedError, NoSupportedLanguagesFoundError, ) from dbt.node_types import ModelLanguage from dbt_common.clients.jinja import ( CallableMacroGenerator, MacroProtocol, get_template, render_template, ) from dbt_common.utils import deep_map_render SUPPORTED_LANG_ARG = jinja2.nodes.Name("supported_languages", "param") class MacroStack(threading.local): def __init__(self): super().__init__() self.call_stack = [] @property def depth(self) -> int: return len(self.call_stack) def push(self, name): self.call_stack.append(name) def pop(self, name): got = self.call_stack.pop() if got != name: raise DbtInternalError(f"popped {got}, expected {name}") class MacroGenerator(CallableMacroGenerator): def __init__( self, macro: MacroProtocol, context: Optional[Dict[str, Any]] = None, node: Optional[Any] = None, stack: Optional[MacroStack] = None, ) -> None: super().__init__(macro, context) self.node = node self.stack = stack # This adds the macro's unique id to the node's 'depends_on' @contextmanager def track_call(self): # This is only called from __call__ if self.stack is None: yield else: unique_id = self.macro.unique_id depth = self.stack.depth # only mark depth=0 as a dependency, when creating this dependency we don't pass in stack if depth == 0 and self.node: self.node.depends_on.add_macro(unique_id) self.stack.push(unique_id) try: yield finally: self.stack.pop(unique_id) # this makes MacroGenerator objects callable like functions def __call__(self, *args, **kwargs): with self.track_call(): return self.call_macro(*args, **kwargs) class UnitTestMacroGenerator(MacroGenerator): # this makes UnitTestMacroGenerator objects callable like functions def __init__( self, macro_generator: MacroGenerator, call_return_value: Any, ) -> None: super().__init__( macro_generator.macro, macro_generator.context, macro_generator.node, macro_generator.stack, ) self.call_return_value = call_return_value def __call__(self, *args, **kwargs): with self.track_call(): return self.call_return_value # performance note: Local benmcharking (so take it with a big grain of salt!) # on this indicates that it is is on average slightly slower than # checking two separate patterns, but the standard deviation is smaller with # one pattern. The time difference between the two was ~2 std deviations, which # is small enough that I've just chosen the more readable option. _HAS_RENDER_CHARS_PAT = re.compile(r"({[{%#]|[#}%]})") _render_cache: Dict[str, Any] = dict() def get_rendered( string: str, ctx: Dict[str, Any], node=None, capture_macros: bool = False, native: bool = False, ) -> Any: # performance optimization: if there are no jinja control characters in the # string, we can just return the input. Fall back to jinja if the type is # not a string or if native rendering is enabled (so '1' -> 1, etc...) # If this is desirable in the native env as well, we could handle the # native=True case by passing the input string to ast.literal_eval, like # the native renderer does. has_render_chars = not isinstance(string, str) or _HAS_RENDER_CHARS_PAT.search(string) if not has_render_chars: if not native: return string elif string in _render_cache: return _render_cache[string] template = get_template( string, ctx, node, capture_macros=capture_macros, native=native, ) rendered = render_template(template, ctx, node) if not has_render_chars and native: _render_cache[string] = rendered return rendered def undefined_error(msg) -> NoReturn: raise jinja2.exceptions.UndefinedError(msg) GENERIC_TEST_KWARGS_NAME = "_dbt_generic_test_kwargs" def add_rendered_test_kwargs( context: Dict[str, Any], node: GenericTestNode, capture_macros: bool = False, ) -> None: """Render each of the test kwargs in the given context using the native renderer, then insert that value into the given context as the special test keyword arguments member. """ looks_like_func = r"^\s*(env_var|ref|var|source|doc)\s*\(.+\)\s*$" def _convert_function(value: Any, keypath: Tuple[Union[str, int], ...]) -> Any: if isinstance(value, str): if keypath == ("column_name",): # special case: Don't render column names as native, make them # be strings return value if re.match(looks_like_func, value) is not None: # curly braces to make rendering happy value = f"{{{{ {value} }}}}" value = get_rendered(value, context, node, capture_macros=capture_macros, native=True) return value # The test_metadata.kwargs come from the test builder, and were set # when the test node was created in _parse_generic_test. kwargs = deep_map_render(_convert_function, node.test_metadata.kwargs) context[GENERIC_TEST_KWARGS_NAME] = kwargs def get_supported_languages(node: jinja2.nodes.Macro) -> List[ModelLanguage]: if "materialization" not in node.name: raise MaterializtionMacroNotUsedError(node=node) no_kwargs = not node.defaults no_langs_found = SUPPORTED_LANG_ARG not in node.args if no_kwargs or no_langs_found: raise NoSupportedLanguagesFoundError(node=node) lang_idx = node.args.index(SUPPORTED_LANG_ARG) # indexing defaults from the end # since supported_languages is a kwarg, and kwargs are at always after args return [ ModelLanguage[item.value] for item in node.defaults[-(len(node.args) - lang_idx)].items ] ================================================ FILE: core/dbt/clients/jinja_static.py ================================================ import typing from typing import Any, Dict, List, Optional, Union import jinja2 from dbt.artifacts.resources import RefArgs from dbt.exceptions import MacroNamespaceNotStringError, ParsingError from dbt_common.clients.jinja import get_environment from dbt_common.exceptions.macros import MacroNameNotStringError from dbt_common.tests import test_caching_enabled from dbt_extractor import ExtractionError, py_extract_from_source # type: ignore if typing.TYPE_CHECKING: from dbt.context.providers import ParseDatabaseWrapper _TESTING_MACRO_CACHE: Dict[str, Any] = {} def statically_extract_has_name_this(source: str) -> bool: """Checks whether the raw jinja has any references to `this`""" env = get_environment(None, capture_macros=True) parsed = env.parse(source) names = tuple(parsed.find_all(jinja2.nodes.Name)) for name in names: if hasattr(name, "name") and name.name == "this": return True return False def statically_extract_macro_calls( source: str, ctx: Dict[str, Any], db_wrapper: Optional["ParseDatabaseWrapper"] = None ) -> List[str]: # set 'capture_macros' to capture undefined env = get_environment(None, capture_macros=True) global _TESTING_MACRO_CACHE if test_caching_enabled() and source in _TESTING_MACRO_CACHE: parsed = _TESTING_MACRO_CACHE.get(source, None) func_calls = getattr(parsed, "_dbt_cached_calls") else: parsed = env.parse(source) func_calls = tuple(parsed.find_all(jinja2.nodes.Call)) if test_caching_enabled(): _TESTING_MACRO_CACHE[source] = parsed setattr(parsed, "_dbt_cached_calls", func_calls) standard_calls = ["source", "ref", "config"] possible_macro_calls = [] for func_call in func_calls: func_name = None if hasattr(func_call, "node") and hasattr(func_call.node, "name"): func_name = func_call.node.name else: if ( hasattr(func_call, "node") and hasattr(func_call.node, "node") and type(func_call.node.node).__name__ == "Name" and hasattr(func_call.node, "attr") ): package_name = func_call.node.node.name macro_name = func_call.node.attr if package_name == "adapter": if macro_name == "dispatch": ad_macro_calls = statically_parse_adapter_dispatch( func_call, ctx, db_wrapper ) possible_macro_calls.extend(ad_macro_calls) else: # This skips calls such as adapter.parse_index continue else: func_name = f"{package_name}.{macro_name}" else: continue if not func_name: continue if func_name in standard_calls: continue elif ctx.get(func_name): continue else: if func_name not in possible_macro_calls: possible_macro_calls.append(func_name) return possible_macro_calls def statically_parse_adapter_dispatch( func_call, ctx: Dict[str, Any], db_wrapper: Optional["ParseDatabaseWrapper"] ) -> List[str]: possible_macro_calls = [] # This captures an adapter.dispatch('<macro_name>') call. func_name = None # macro_name positional argument if len(func_call.args) > 0: func_name = func_call.args[0].value if func_name: possible_macro_calls.append(func_name) # packages positional argument macro_namespace = None packages_arg = None packages_arg_type = None if len(func_call.args) > 1: packages_arg = func_call.args[1] # This can be a List or a Call packages_arg_type = type(func_call.args[1]).__name__ # keyword arguments if func_call.kwargs: for kwarg in func_call.kwargs: if kwarg.key == "macro_name": # This will remain to enable static resolution if type(kwarg.value).__name__ == "Const": func_name = kwarg.value.value possible_macro_calls.append(func_name) else: raise MacroNameNotStringError(kwarg_value=kwarg.value.value) elif kwarg.key == "macro_namespace": # This will remain to enable static resolution kwarg_type = type(kwarg.value).__name__ if kwarg_type == "Const": macro_namespace = kwarg.value.value else: raise MacroNamespaceNotStringError(kwarg_type) # positional arguments if packages_arg: if packages_arg_type == "List": # This will remain to enable static resolution packages = [] for item in packages_arg.items: packages.append(item.value) elif packages_arg_type == "Const": # This will remain to enable static resolution macro_namespace = packages_arg.value if db_wrapper: macro = db_wrapper.dispatch(func_name, macro_namespace=macro_namespace).macro func_name = f"{macro.package_name}.{macro.name}" # type: ignore[attr-defined] possible_macro_calls.append(func_name) else: # this is only for tests/unit/test_macro_calls.py if macro_namespace: packages = [macro_namespace] else: packages = [] for package_name in packages: possible_macro_calls.append(f"{package_name}.{func_name}") return possible_macro_calls def statically_parse_ref_or_source(expression: str) -> Union[RefArgs, List[str]]: """ Returns a RefArgs or List[str] object, corresponding to ref or source respectively, given an input jinja expression. input: str representing how input node is referenced in tested model sql * examples: - "ref('my_model_a')" - "ref('my_model_a', version=3)" - "ref('package', 'my_model_a', version=3)" - "source('my_source_schema', 'my_source_name')" If input is not a well-formed jinja ref or source expression, a ParsingError is raised. """ ref_or_source: Union[RefArgs, List[str]] try: statically_parsed = py_extract_from_source(f"{{{{ {expression} }}}}") except ExtractionError: raise ParsingError(f"Invalid jinja expression: {expression}") if statically_parsed.get("refs"): raw_ref = list(statically_parsed["refs"])[0] ref_or_source = RefArgs( package=raw_ref.get("package"), name=raw_ref.get("name"), version=raw_ref.get("version"), ) elif statically_parsed.get("sources"): source_name, source_table_name = list(statically_parsed["sources"])[0] ref_or_source = [source_name, source_table_name] else: raise ParsingError(f"Invalid ref or source expression: {expression}") return ref_or_source def statically_parse_unrendered_config(string: str) -> Optional[Dict[str, Any]]: """ Given a string with jinja, extract an unrendered config call. If no config call is present, returns None. For example, given: "{{ config(materialized=env_var('DBT_TEST_STATE_MODIFIED')) }}\nselect 1 as id" returns: {'materialized': "Keyword(key='materialized', value=Call(node=Name(name='env_var', ctx='load'), args=[Const(value='DBT_TEST_STATE_MODIFIED')], kwargs=[], dyn_args=None, dyn_kwargs=None))"} No config call: "select 1 as id" returns: None """ # Return early to avoid creating jinja environemt if no config call in input string if "config(" not in string: return None # set 'capture_macros' to capture undefined env = get_environment(None, capture_macros=True) global _TESTING_MACRO_CACHE if test_caching_enabled() and _TESTING_MACRO_CACHE and string in _TESTING_MACRO_CACHE: parsed = _TESTING_MACRO_CACHE.get(string, None) func_calls = getattr(parsed, "_dbt_cached_calls") else: parsed = env.parse(string) func_calls = tuple(parsed.find_all(jinja2.nodes.Call)) config_func_calls = list( filter( lambda f: hasattr(f, "node") and hasattr(f.node, "name") and f.node.name == "config", func_calls, ) ) # There should only be one {{ config(...) }} call per input config_func_call = config_func_calls[0] if config_func_calls else None if not config_func_call: return None unrendered_config = {} for kwarg in config_func_call.kwargs: unrendered_config[kwarg.key] = construct_static_kwarg_value(kwarg) return unrendered_config def construct_static_kwarg_value(kwarg) -> str: # Instead of trying to re-assemble complex kwarg value, simply stringify the value. # This is still useful to be able to detect changes in unrendered configs, even if it is # not an exact representation of the user input. return str(kwarg) ================================================ FILE: core/dbt/clients/registry.py ================================================ import functools import os from typing import Any, Dict, List, Optional import requests from dbt import deprecations from dbt.events.types import ( RegistryIndexProgressGETRequest, RegistryIndexProgressGETResponse, RegistryProgressGETRequest, RegistryProgressGETResponse, RegistryResponseExtraNestedKeys, RegistryResponseMissingNestedKeys, RegistryResponseMissingTopKeys, RegistryResponseUnexpectedType, ) from dbt.utils import memoized from dbt_common import semver from dbt_common.events.functions import fire_event from dbt_common.utils.connection import connection_exception_retry if os.getenv("DBT_PACKAGE_HUB_URL"): DEFAULT_REGISTRY_BASE_URL = os.getenv("DBT_PACKAGE_HUB_URL") else: DEFAULT_REGISTRY_BASE_URL = "https://hub.getdbt.com/" def _get_url(name, registry_base_url=None): if registry_base_url is None: registry_base_url = DEFAULT_REGISTRY_BASE_URL url = "api/v1/{}.json".format(name) return "{}{}".format(registry_base_url, url) def _check_package_redirect(package_name: str, response: Dict[str, Any]) -> None: # Either redirectnamespace or redirectname in the JSON response indicate a redirect # redirectnamespace redirects based on package ownership # redirectname redirects based on package name # Both can be present at the same time, or neither. Fails gracefully to old name if ("redirectnamespace" in response) or ("redirectname" in response): use_namespace = response.get("redirectnamespace") or response["namespace"] use_name = response.get("redirectname") or response["name"] new_nwo = f"{use_namespace}/{use_name}" deprecations.warn("package-redirect", old_name=package_name, new_name=new_nwo) def _get_package_with_retries( package_name: str, registry_base_url: Optional[str] = None ) -> Dict[str, Any]: get_fn = functools.partial(_get, package_name, registry_base_url) response: Dict[str, Any] = connection_exception_retry(get_fn, 5) _check_package_redirect(package_name, response) return response def _get(package_name, registry_base_url=None): url = _get_url(package_name, registry_base_url) fire_event(RegistryProgressGETRequest(url=url)) # all exceptions from requests get caught in the retry logic so no need to wrap this here resp = requests.get(url, timeout=30) fire_event(RegistryProgressGETResponse(url=url, resp_code=resp.status_code)) resp.raise_for_status() # The response should always be a dictionary. Anything else is unexpected, raise error. # Raising this error will cause this function to retry (if called within _get_package_with_retries) # and hopefully get a valid response. This seems to happen when there's an issue with the Hub. # Since we control what we expect the HUB to return, this is safe. # See https://github.com/dbt-labs/dbt-core/issues/4577 # and https://github.com/dbt-labs/dbt-core/issues/4849 response = resp.json() if not isinstance(response, dict): # This will also catch Nonetype error_msg = ( f"Request error: Expected a response type of <dict> but got {type(response)} instead" ) fire_event(RegistryResponseUnexpectedType(response=response)) raise requests.exceptions.ContentDecodingError(error_msg, response=resp) # check for expected top level keys expected_keys = {"name", "versions"} if not expected_keys.issubset(response): error_msg = ( f"Request error: Expected the response to contain keys {expected_keys} " f"but is missing {expected_keys.difference(set(response))}" ) fire_event(RegistryResponseMissingTopKeys(response=response)) raise requests.exceptions.ContentDecodingError(error_msg, response=resp) # check for the keys we need nested under each version expected_version_keys = {"name", "packages", "downloads"} all_keys = set().union(*(response["versions"][d] for d in response["versions"])) if not expected_version_keys.issubset(all_keys): error_msg = ( "Request error: Expected the response for the version to contain keys " f"{expected_version_keys} but is missing {expected_version_keys.difference(all_keys)}" ) fire_event(RegistryResponseMissingNestedKeys(response=response)) raise requests.exceptions.ContentDecodingError(error_msg, response=resp) # all version responses should contain identical keys. has_extra_keys = set().difference(*(response["versions"][d] for d in response["versions"])) if has_extra_keys: error_msg = ( "Request error: Keys for all versions do not match. Found extra key(s) " f"of {has_extra_keys}." ) fire_event(RegistryResponseExtraNestedKeys(response=response)) raise requests.exceptions.ContentDecodingError(error_msg, response=resp) return response _get_cached = memoized(_get_package_with_retries) def package(package_name, registry_base_url=None) -> Dict[str, Any]: # returns a dictionary of metadata for all versions of a package response = _get_cached(package_name, registry_base_url) return response["versions"] def package_version(package_name, version, registry_base_url=None) -> Dict[str, Any]: # returns the metadata of a specific version of a package response = package(package_name, registry_base_url) return response[version] def is_compatible_version(package_spec, dbt_version) -> bool: require_dbt_version = package_spec.get("require_dbt_version") if not require_dbt_version: # if version requirements are missing or empty, assume any version is compatible return True else: # determine whether dbt_version satisfies this package's require-dbt-version config if not isinstance(require_dbt_version, list): require_dbt_version = [require_dbt_version] supported_versions = [ semver.VersionSpecifier.from_version_string(v) for v in require_dbt_version ] return semver.versions_compatible(dbt_version, *supported_versions) def get_compatible_versions(package_name, dbt_version, should_version_check) -> List["str"]: # returns a list of all available versions of a package response = package(package_name) # if the user doesn't care about installing compatible versions, just return them all if not should_version_check: return list(response) # otherwise, only return versions that are compatible with the installed version of dbt-core else: compatible_versions = [ pkg_version for pkg_version, info in response.items() if is_compatible_version(info, dbt_version) ] return compatible_versions def _get_index(registry_base_url=None): url = _get_url("index", registry_base_url) fire_event(RegistryIndexProgressGETRequest(url=url)) # all exceptions from requests get caught in the retry logic so no need to wrap this here resp = requests.get(url, timeout=30) fire_event(RegistryIndexProgressGETResponse(url=url, resp_code=resp.status_code)) resp.raise_for_status() # The response should be a list. Anything else is unexpected, raise an error. # Raising this error will cause this function to retry and hopefully get a valid response. response = resp.json() if not isinstance(response, list): # This will also catch Nonetype error_msg = ( f"Request error: The response type of {type(response)} is not valid: {resp.text}" ) raise requests.exceptions.ContentDecodingError(error_msg, response=resp) return response def index(registry_base_url=None) -> List[str]: # this returns a list of all packages on the Hub get_index_fn = functools.partial(_get_index, registry_base_url) return connection_exception_retry(get_index_fn, 5) index_cached = memoized(index) ================================================ FILE: core/dbt/clients/yaml_helper.py ================================================ from typing import Any, Dict, Optional import yaml import dbt_common.exceptions import dbt_common.exceptions.base # the C version is faster, but it doesn't always exist try: from yaml import CDumper as Dumper from yaml import CLoader as Loader from yaml import CSafeLoader as SafeLoader except ImportError: from yaml import Dumper, Loader, SafeLoader # type: ignore # noqa: F401 YAML_ERROR_MESSAGE = """ Syntax error near line {line_number} ------------------------------ {nice_error} Raw Error: ------------------------------ {raw_error} """.strip() def line_no(i, line, width=3): line_number = str(i).ljust(width) return "{}| {}".format(line_number, line) def prefix_with_line_numbers(string, no_start, no_end): line_list = string.split("\n") numbers = range(no_start, no_end) relevant_lines = line_list[no_start:no_end] return "\n".join([line_no(i + 1, line) for (i, line) in zip(numbers, relevant_lines)]) def contextualized_yaml_error(raw_contents, error): mark = error.problem_mark min_line = max(mark.line - 3, 0) max_line = mark.line + 4 nice_error = prefix_with_line_numbers(raw_contents, min_line, max_line) return YAML_ERROR_MESSAGE.format( line_number=mark.line + 1, nice_error=nice_error, raw_error=error ) def safe_load(contents) -> Optional[Dict[str, Any]]: return yaml.load(contents, Loader=SafeLoader) def load_yaml_text(contents, path=None, loader=SafeLoader) -> Optional[Dict[str, Any]]: try: return yaml.load(contents, loader) except (yaml.scanner.ScannerError, yaml.YAMLError) as e: if hasattr(e, "problem_mark"): error = contextualized_yaml_error(contents, e) else: error = str(e) raise dbt_common.exceptions.base.DbtValidationError(error) ================================================ FILE: core/dbt/compilation.py ================================================ import dataclasses import json import os import pickle from collections import defaultdict, deque from typing import Any, Dict, Iterable, List, Optional, Set, Tuple import networkx as nx # type: ignore import sqlparse import dbt.tracking from dbt.adapters.factory import get_adapter from dbt.clients import jinja from dbt.context.providers import ( generate_runtime_model_context, generate_runtime_unit_test_context, ) from dbt.contracts.graph.manifest import Manifest, UniqueID from dbt.contracts.graph.nodes import ( GenericTestNode, GraphMemberNode, InjectedCTE, ManifestNode, ManifestSQLNode, ModelNode, SeedNode, UnitTestDefinition, UnitTestNode, UnitTestSourceDefinition, ) from dbt.events.types import FoundStats, WritingInjectedSQLForNode from dbt.exceptions import ( DbtInternalError, DbtRuntimeError, ForeignKeyConstraintToSyntaxError, GraphDependencyNotFoundError, ParsingError, ) from dbt.flags import get_flags from dbt.graph import Graph from dbt.node_types import ModelLanguage, NodeType from dbt_common.clients.system import make_directory from dbt_common.contracts.constraints import ConstraintType from dbt_common.events.contextvars import get_node_info from dbt_common.events.format import pluralize from dbt_common.events.functions import fire_event from dbt_common.events.types import Note from dbt_common.exceptions import CompilationError from dbt_common.invocation import get_invocation_id graph_file_name = "graph.gpickle" def print_compile_stats(stats: Dict[NodeType, int]): # create tracking event for resource_counts if dbt.tracking.active_user is not None: resource_counts = {k.pluralize(): v for k, v in stats.items()} dbt.tracking.track_resource_counts(resource_counts) # do not include resource types that are not actually defined in the project stat_line = ", ".join( [pluralize(ct, t).replace("_", " ") for t, ct in stats.items() if ct != 0] ) fire_event(FoundStats(stat_line=stat_line)) def _node_enabled(node: ManifestNode): # Disabled models are already excluded from the manifest if node.resource_type == NodeType.Test and not node.config.enabled: return False else: return True def _generate_stats(manifest: Manifest) -> Dict[NodeType, int]: stats: Dict[NodeType, int] = defaultdict(int) for node in manifest.nodes.values(): if _node_enabled(node): stats[node.resource_type] += 1 # Disabled nodes don't appear in the following collections, so we don't check. stats[NodeType.Source] += len(manifest.sources) stats[NodeType.Exposure] += len(manifest.exposures) stats[NodeType.Metric] += len(manifest.metrics) stats[NodeType.Macro] += len(manifest.macros) stats[NodeType.Group] += len(manifest.groups) stats[NodeType.SemanticModel] += len(manifest.semantic_models) stats[NodeType.SavedQuery] += len(manifest.saved_queries) stats[NodeType.Unit] += len(manifest.unit_tests) # TODO: should we be counting dimensions + entities? return stats def _add_prepended_cte(prepended_ctes, new_cte): for cte in prepended_ctes: if cte.id == new_cte.id and new_cte.sql: cte.sql = new_cte.sql return if new_cte.sql: prepended_ctes.append(new_cte) def _extend_prepended_ctes(prepended_ctes, new_prepended_ctes): for new_cte in new_prepended_ctes: _add_prepended_cte(prepended_ctes, new_cte) def _get_tests_for_node(manifest: Manifest, unique_id: UniqueID) -> List[UniqueID]: """Get a list of tests that depend on the node with the provided unique id""" tests = [] if unique_id in manifest.child_map: for child_unique_id in manifest.child_map[unique_id]: if child_unique_id.startswith("test."): tests.append(child_unique_id) return tests @dataclasses.dataclass class SeenDetails: node_id: UniqueID visits: int = 0 ancestors: Set[UniqueID] = dataclasses.field(default_factory=set) awaits_tests: Set[Tuple[UniqueID, Tuple[UniqueID, ...]]] = dataclasses.field( default_factory=set ) class Linker: def __init__(self, data=None) -> None: if data is None: data = {} self.graph: nx.DiGraph = nx.DiGraph(**data) def edges(self): return self.graph.edges() def nodes(self): return self.graph.nodes() def find_cycles(self): try: cycle = nx.find_cycle(self.graph) except nx.NetworkXNoCycle: return None else: # cycles is a List[Tuple[str, ...]] return " --> ".join(c[0] for c in cycle) def dependency(self, node1, node2): "indicate that node1 depends on node2" self.graph.add_node(node1) self.graph.add_node(node2) self.graph.add_edge(node2, node1) def add_node(self, node): self.graph.add_node(node) def write_graph(self, outfile: str, manifest: Manifest): """Write the graph to a gpickle file. Before doing so, serialize and include all nodes in their corresponding graph entries. """ out_graph = self.graph.copy() for node_id in self.graph: data = manifest.expect(node_id).to_dict(omit_none=True) out_graph.add_node(node_id, **data) with open(outfile, "wb") as outfh: pickle.dump(out_graph, outfh, protocol=pickle.HIGHEST_PROTOCOL) def link_node(self, node: GraphMemberNode, manifest: Manifest): self.add_node(node.unique_id) for dependency in node.depends_on_nodes: if dependency in manifest.nodes: self.dependency(node.unique_id, (manifest.nodes[dependency].unique_id)) elif dependency in manifest.sources: self.dependency(node.unique_id, (manifest.sources[dependency].unique_id)) elif dependency in manifest.metrics: self.dependency(node.unique_id, (manifest.metrics[dependency].unique_id)) elif dependency in manifest.semantic_models: self.dependency(node.unique_id, (manifest.semantic_models[dependency].unique_id)) elif dependency in manifest.functions: self.dependency(node.unique_id, (manifest.functions[dependency].unique_id)) else: raise GraphDependencyNotFoundError(node, dependency) def link_graph(self, manifest: Manifest): for source in manifest.sources.values(): self.add_node(source.unique_id) for node in manifest.nodes.values(): self.link_node(node, manifest) for semantic_model in manifest.semantic_models.values(): self.link_node(semantic_model, manifest) for exposure in manifest.exposures.values(): self.link_node(exposure, manifest) for function in manifest.functions.values(): self.link_node(function, manifest) for metric in manifest.metrics.values(): self.link_node(metric, manifest) for unit_test in manifest.unit_tests.values(): self.link_node(unit_test, manifest) for saved_query in manifest.saved_queries.values(): self.link_node(saved_query, manifest) cycle = self.find_cycles() if cycle: raise RuntimeError("Found a cycle: {}".format(cycle)) def add_test_edges(self, manifest: Manifest) -> None: if not get_flags().USE_FAST_TEST_EDGES: self.add_test_edges_1(manifest) else: self.add_test_edges_2(manifest) def add_test_edges_1(self, manifest: Manifest) -> None: """This method adds additional edges to the DAG. For a given non-test executable node, add an edge from an upstream test to the given node if the set of nodes the test depends on is a subset of the upstream nodes for the given node.""" # HISTORICAL NOTE: To understand the motivation behind this function, # consider a node A with tests and a node B which depends (either directly # or indirectly) on A. It would be nice if B were not executed until # all of the tests on A are finished. After all, we don't want to # propagate bad data. We can enforce that behavior by adding new # dependencies (edges) from tests to nodes that should wait on them. # # This function implements a rough approximation of the behavior just # described. In fact, for tests that only depend on a single node, it # always works. # # Things get trickier for tests that depend on multiple nodes. In that # case, if we are not careful, we will introduce cycles. That seems to # be the reason this function adds dependencies from a downstream node to # an upstream test if and only if the downstream node is already a # descendant of all the nodes the upstream test depends on. By following # that rule, it never makes the node dependent on new upstream nodes other # than the tests themselves, and no cycles will be created. # # One drawback (Drawback 1) of the approach taken in this function is # that it could still allow a downstream node to proceed before all # testing is done on its ancestors, if it happens to have ancestors that # are not also ancestors of a test with multiple dependencies. # # Another drawback (Drawback 2) is that the approach below adds far more # edges than are strictly needed. After all, if we have A -> B -> C, # there is no need to add a new edge A -> C. But this function often does. # # Drawback 2 is resolved in the new add_test_edges_2() implementation # below, which is also typically much faster. Drawback 1 has been left in # place in order to conservatively retain existing behavior, and so that # the new implementation can be verified against this existing # implementation by ensuring both resulting graphs have the same transitive # reduction. # MOTIVATING IDEA: Given a graph... # # model1 --> model2 --> model3 # | | # | \/ # \/ test 2 # test1 # # ...produce the following... # # model1 --> model2 --> model3 # | /\ | /\ /\ # | | \/ | | # \/ | test2 ----| | # test1 ----|---------------| for node_id in self.graph: # If node is executable (in manifest.nodes) and does _not_ # represent a test, continue. if ( node_id in manifest.nodes and manifest.nodes[node_id].resource_type != NodeType.Test ): # Get *everything* upstream of the node all_upstream_nodes = nx.traversal.bfs_tree(self.graph, node_id, reverse=True) # Get the set of upstream nodes not including the current node. upstream_nodes = set([n for n in all_upstream_nodes if n != node_id]) # Get all tests that depend on any upstream nodes. upstream_tests = [] for upstream_node in upstream_nodes: # This gets tests with unique_ids starting with "test." upstream_tests += _get_tests_for_node(manifest, upstream_node) for upstream_test in upstream_tests: # Get the set of all nodes that the test depends on # including the upstream_node itself. This is necessary # because tests can depend on multiple nodes (ex: # relationship tests). Test nodes do not distinguish # between what node the test is "testing" and what # node(s) it depends on. test_depends_on = set(manifest.nodes[upstream_test].depends_on_nodes) # If the set of nodes that an upstream test depends on # is a subset of all upstream nodes of the current node, # add an edge from the upstream test to the current node. if test_depends_on.issubset(upstream_nodes): self.graph.add_edge(upstream_test, node_id, edge_type="parent_test") def add_test_edges_2(self, manifest: Manifest): graph = self.graph new_edges = self._get_test_edges_2(graph, manifest) for e in new_edges: graph.add_edge(e[0], e[1], edge_type="parent_test") @staticmethod def _get_test_edges_2( graph: nx.DiGraph, manifest: Manifest ) -> Iterable[Tuple[UniqueID, UniqueID]]: # This function enforces the same execution behavior as add_test_edges, # but executes far more quickly and adds far fewer edges. See the # HISTORICAL NOTE above. # # The idea is to first scan for "single-tested" nodes (which have tests # that depend only upon on that node) and "multi-tested" nodes (which # have tests that depend on multiple nodes). Single-tested nodes are # handled quickly and easily. # # The less common but more complex case of multi-tested nodes is handled # by a specialized function. new_edges: List[Tuple[UniqueID, UniqueID]] = [] source_nodes: List[UniqueID] = [] executable_nodes: Set[UniqueID] = set() multi_tested_nodes = set() # Dictionary mapping nodes with single-dep tests to a list of those tests. single_tested_nodes: dict[UniqueID, List[UniqueID]] = defaultdict(list) for node_id in graph.nodes: manifest_node = manifest.nodes.get(node_id, None) if manifest_node is None: continue if next(graph.predecessors(node_id), None) is None: source_nodes.append(node_id) if manifest_node.resource_type != NodeType.Test: executable_nodes.add(node_id) else: test_deps = manifest_node.depends_on_nodes if len(test_deps) == 1: single_tested_nodes[test_deps[0]].append(node_id) elif len(test_deps) > 1: multi_tested_nodes.update(manifest_node.depends_on_nodes) # Now that we have all the necessary information conveniently organized, # add new edges for single-tested nodes. for node_id, test_ids in single_tested_nodes.items(): succs = [s for s in graph.successors(node_id) if s in executable_nodes] for succ_id in succs: for test_id in test_ids: new_edges.append((test_id, succ_id)) # Get the edges for multi-tested nodes separately, if needed. if len(multi_tested_nodes) > 0: multi_test_edges = Linker._get_multi_test_edges( graph, manifest, source_nodes, executable_nodes, multi_tested_nodes ) new_edges += multi_test_edges return new_edges @staticmethod def _get_multi_test_edges( graph: nx.DiGraph, manifest: Manifest, source_nodes: Iterable[UniqueID], executable_nodes: Set[UniqueID], multi_tested_nodes, ) -> List[Tuple[UniqueID, UniqueID]]: # Works through the graph in a breadth-first style, processing nodes from # a ready queue which initially consists of nodes with no ancestors, # and adding more nodes to the ready queue after all their ancestors # have been processed. All the while, the relevant details of all nodes # "seen" by the search so far are maintained in a SeenDetails record, # including the ancestor set which tests it is "awaiting" (i.e. tests of # its ancestors). The processing step adds test edges when every dependency # of an awaited test is an ancestor of a node that is being processed. # Downstream nodes are then exempted from awaiting the test. # # Memory consumption is potentially O(n^2) with n the number of nodes in # the graph, since the average number of ancestors and tests being awaited # for each of the n nodes could itself be O(n) but we only track ancestors # that are multi-tested, which should keep things closer to O(n) in real- # world scenarios. new_edges: List[Tuple[UniqueID, UniqueID]] = [] ready: deque = deque(source_nodes) details = {node_id: SeenDetails(node_id) for node_id in source_nodes} while len(ready) > 0: curr_details: SeenDetails = details[ready.pop()] test_ids = _get_tests_for_node(manifest, curr_details.node_id) new_awaits_for_succs = curr_details.awaits_tests.copy() for test_id in test_ids: deps: List[UniqueID] = sorted(manifest.nodes[test_id].depends_on_nodes) if len(deps) > 1: # Tests with only one dep were already handled. new_awaits_for_succs.add((test_id, tuple(deps))) for succ_id in [ s for s in graph.successors(curr_details.node_id) if s in executable_nodes ]: suc_details = details.get(succ_id, None) if suc_details is None: suc_details = SeenDetails(succ_id) details[succ_id] = suc_details suc_details.visits += 1 suc_details.awaits_tests.update(new_awaits_for_succs) suc_details.ancestors.update(curr_details.ancestors) if curr_details.node_id in multi_tested_nodes: # Only track ancestry information for the set of nodes # we will actually check against later. suc_details.ancestors.add(curr_details.node_id) if suc_details.visits == graph.in_degree(succ_id): if len(suc_details.awaits_tests) > 0: removes = set() for awt in suc_details.awaits_tests: if not any(True for a in awt[1] if a not in suc_details.ancestors): removes.add(awt) new_edges.append((awt[0], succ_id)) suc_details.awaits_tests.difference_update(removes) ready.appendleft(succ_id) # We are now done with the current node and all of its ancestors. # Discard its details to save memory. del details[curr_details.node_id] return new_edges def get_graph(self, manifest: Manifest) -> Graph: self.link_graph(manifest) return Graph(self.graph) def get_graph_summary(self, manifest: Manifest) -> Dict[int, Dict[str, Any]]: """Create a smaller summary of the graph, suitable for basic diagnostics and performance tuning. The summary includes only the edge structure, node types, and node names. Each of the n nodes is assigned an integer index 0, 1, 2,..., n-1 for compactness""" graph_nodes = dict() index_dict = dict() for node_index, node_name in enumerate(self.graph): index_dict[node_name] = node_index data = manifest.expect(node_name).to_dict(omit_none=True) graph_nodes[node_index] = {"name": node_name, "type": data["resource_type"]} for node_index, node in graph_nodes.items(): successors = [index_dict[n] for n in self.graph.successors(node["name"])] if successors: node["succ"] = [index_dict[n] for n in self.graph.successors(node["name"])] return graph_nodes class Compiler: def __init__(self, config) -> None: self.config = config # Set of unique_ids for nodes selected in the current run. # Used to determine whether FK constraint targets should use # deferred relations or current relations during compilation. self.selected_node_ids: Set[str] = set() def initialize(self): make_directory(self.config.project_target_path) # creates a ModelContext which is converted to # a dict for jinja rendering of SQL def _create_node_context( self, node: ManifestSQLNode, manifest: Manifest, extra_context: Dict[str, Any], ) -> Dict[str, Any]: if isinstance(node, UnitTestNode): context = generate_runtime_unit_test_context(node, self.config, manifest) else: context = generate_runtime_model_context(node, self.config, manifest) context.update(extra_context) if isinstance(node, GenericTestNode): # for test nodes, add a special keyword args value to the context jinja.add_rendered_test_kwargs(context, node) return context def add_ephemeral_prefix(self, name: str): adapter = get_adapter(self.config) relation_cls = adapter.Relation return relation_cls.add_ephemeral_prefix(name) def _recursively_prepend_ctes( self, model: ManifestSQLNode, manifest: Manifest, extra_context: Optional[Dict[str, Any]], ) -> Tuple[ManifestSQLNode, List[InjectedCTE]]: """This method is called by the 'compile_node' method. Starting from the node that it is passed in, it will recursively call itself using the 'extra_ctes'. The 'ephemeral' models do not produce SQL that is executed directly, instead they are rolled up into the models that refer to them by inserting CTEs into the SQL. """ if model.compiled_code is None: raise DbtRuntimeError("Cannot inject ctes into an uncompiled node", model) # tech debt: safe flag/arg access (#6259) if not getattr(self.config.args, "inject_ephemeral_ctes", True): return (model, []) # extra_ctes_injected flag says that we've already recursively injected the ctes if model.extra_ctes_injected: return (model, model.extra_ctes) # Just to make it plain that nothing is actually injected for this case if len(model.extra_ctes) == 0: # SeedNodes don't have compilation attributes if not isinstance(model, SeedNode): model.extra_ctes_injected = True return (model, []) # This stores the ctes which will all be recursively # gathered and then "injected" into the model. prepended_ctes: List[InjectedCTE] = [] # extra_ctes are added to the model by # RuntimeRefResolver.create_relation, which adds an # extra_cte for every model relation which is an # ephemeral model. InjectedCTEs have a unique_id and sql. # extra_ctes start out with sql set to None, and the sql is set in this loop. for cte in model.extra_ctes: if cte.id not in manifest.nodes: raise DbtInternalError( f"During compilation, found a cte reference that " f"could not be resolved: {cte.id}" ) cte_model = manifest.nodes[cte.id] assert not isinstance(cte_model, SeedNode) if not cte_model.is_ephemeral_model: raise DbtInternalError(f"{cte.id} is not ephemeral") # Lock the ephemeral cte_model to ensure only one thread compiles it. # The lock is scoped to just the compilation step; the recursive # _recursively_prepend_ctes call happens outside the lock to avoid # deadlock (it will re-acquire this node's lock at CTE injection). new_prepended_ctes = None needs_recursion = False with cte_model._lock: # This model has already been compiled and extra_ctes_injected, # so it's been through here before. We already checked above for # extra_ctes_injected, but checking again because updates may # have happened in another thread. if cte_model.compiled is True and cte_model.extra_ctes_injected is True: new_prepended_ctes = cte_model.extra_ctes elif not cte_model.compiled: # This is an ephemeral parsed model that we can compile. # Render the raw_code and set compiled to True cte_model = self._compile_code(cte_model, manifest, extra_context) needs_recursion = True else: # compiled=True but extra_ctes not yet injected; another # thread compiled it but hasn't finished recursion yet. # We still need to recurse to get the prepended CTEs. needs_recursion = True if needs_recursion: # recursively call this method, sets extra_ctes_injected to True cte_model, new_prepended_ctes = self._recursively_prepend_ctes( cte_model, manifest, extra_context ) # Write compiled SQL file self._write_node(cte_model) if new_prepended_ctes is None: raise DbtInternalError( f"Expected extra_ctes to be set for ephemeral model {cte.id}" ) _extend_prepended_ctes(prepended_ctes, new_prepended_ctes) cte_name = ( cte_model.cte_name if isinstance(cte_model, UnitTestSourceDefinition) else cte_model.identifier ) new_cte_name = self.add_ephemeral_prefix(cte_name) rendered_sql = cte_model._pre_injected_sql or cte_model.compiled_code sql = f" {new_cte_name} as (\n{rendered_sql}\n)" _add_prepended_cte(prepended_ctes, InjectedCTE(id=cte.id, sql=sql)) # Lock the consuming model to prevent duplicate CTE injection with model._lock: if not model.extra_ctes_injected: injected_sql = inject_ctes_into_sql( model.compiled_code, prepended_ctes, ) model.extra_ctes_injected = True model._pre_injected_sql = model.compiled_code model.compiled_code = injected_sql model.extra_ctes = prepended_ctes # if model.extra_ctes is not set to prepended ctes, something went wrong return model, model.extra_ctes # Sets compiled_code and compiled flag in the ManifestSQLNode passed in, # creates a "context" dictionary for jinja rendering, # and then renders the "compiled_code" using the node, the # raw_code and the context. def _compile_code( self, node: ManifestSQLNode, manifest: Manifest, extra_context: Optional[Dict[str, Any]] = None, ) -> ManifestSQLNode: if extra_context is None: extra_context = {} if node.language == ModelLanguage.python and node.resource_type == NodeType.Model: context = self._create_node_context(node, manifest, extra_context) postfix = jinja.get_rendered( "{{ py_script_postfix(model) }}", context, node, ) # we should NOT jinja render the python model's 'raw code' node.compiled_code = f"{node.raw_code}\n\n{postfix}" else: context = self._create_node_context(node, manifest, extra_context) node.compiled_code = jinja.get_rendered( node.raw_code, context, node, ) node.compiled = True # relation_name is set at parse time, except for tests without store_failures, # but cli param can turn on store_failures, so we set here. if ( node.resource_type == NodeType.Test and node.relation_name is None and node.is_relational ): adapter = get_adapter(self.config) relation_cls = adapter.Relation relation_name = str(relation_cls.create_from(self.config, node)) node.relation_name = relation_name # Compile 'ref' and 'source' expressions in foreign key constraints if isinstance(node, ModelNode): for constraint in node.all_constraints: if constraint.type == ConstraintType.foreign_key and constraint.to: constraint.to = self._compile_relation_for_foreign_key_constraint_to( manifest, node, constraint.to ) return node def _compile_relation_for_foreign_key_constraint_to( self, manifest: Manifest, node: ManifestSQLNode, to_expression: str ) -> str: try: foreign_key_node = manifest.find_node_from_ref_or_source(to_expression) except ParsingError: raise ForeignKeyConstraintToSyntaxError(node, to_expression) if not foreign_key_node: raise GraphDependencyNotFoundError(node, to_expression) adapter = get_adapter(self.config) # Use deferred relation only if: # 1. The foreign key node has a defer_relation (from previous state) # 2. The --defer flag is set # 3. The foreign key node is NOT being built in the current run # (i.e., not in selected_node_ids) # This mirrors the logic in RuntimeRefResolver.create_relation() for # model body refs, ensuring FK constraints behave consistently. if ( hasattr(foreign_key_node, "defer_relation") and foreign_key_node.defer_relation and self.config.args.defer and foreign_key_node.unique_id not in self.selected_node_ids ): return str(adapter.Relation.create_from(self.config, foreign_key_node.defer_relation)) else: return str(adapter.Relation.create_from(self.config, foreign_key_node)) # This method doesn't actually "compile" any of the nodes. That is done by the # "compile_node" method. This creates a Linker and builds the networkx graph, # writes out the graph.gpickle file, and prints the stats, returning a Graph object. def compile(self, manifest: Manifest, write=True, add_test_edges=False) -> Graph: self.initialize() linker = Linker() linker.link_graph(manifest) # Create a file containing basic information about graph structure, # supporting diagnostics and performance analysis. summaries: Dict = dict() summaries["_invocation_id"] = get_invocation_id() summaries["linked"] = linker.get_graph_summary(manifest) # This is only called for the "build" command if add_test_edges: manifest.build_parent_and_child_maps() linker.add_test_edges(manifest) # Create another diagnostic summary, just as above, but this time # including the test edges. summaries["with_test_edges"] = linker.get_graph_summary(manifest) with open( os.path.join(self.config.project_target_path, "graph_summary.json"), "w" ) as out_stream: try: out_stream.write(json.dumps(summaries)) except Exception as e: # This is non-essential information, so merely note failures. fire_event( Note( msg=f"An error was encountered writing the graph summary information: {e}" ) ) stats = _generate_stats(manifest) if write: self.write_graph_file(linker, manifest) # Do not print these for list command if self.config.args.which != "list": stats = _generate_stats(manifest) print_compile_stats(stats) return Graph(linker.graph) def write_graph_file(self, linker: Linker, manifest: Manifest): filename = graph_file_name graph_path = os.path.join(self.config.project_target_path, filename) flags = get_flags() if flags.WRITE_JSON: linker.write_graph(graph_path, manifest) # writes the "compiled_code" into the target/compiled directory def _write_node( self, node: ManifestSQLNode, split_suffix: Optional[str] = None ) -> ManifestSQLNode: if not node.extra_ctes_injected or node.resource_type in (NodeType.Seed,): return node fire_event(WritingInjectedSQLForNode(node_info=get_node_info())) if node.compiled_code: node.compiled_path = node.get_target_write_path( self.config.target_path, "compiled", split_suffix ) node.write_node(self.config.project_root, node.compiled_path, node.compiled_code) return node def compile_node( self, node: ManifestSQLNode, manifest: Manifest, extra_context: Optional[Dict[str, Any]] = None, write: bool = True, split_suffix: Optional[str] = None, ) -> ManifestSQLNode: """This is the main entry point into this code. It's called by CompileRunner.compile, GenericRPCRunner.compile, and RunTask.get_hook_sql. It calls '_compile_code' to render the node's raw_code into compiled_code, and then calls the recursive method to "prepend" the ctes. """ # REVIEW: UnitTestDefinition shouldn't be possible here because of the # type of node, and it is likewise an invalid return type. if isinstance(node, UnitTestDefinition): return node # Make sure Lexer for sqlparse 0.4.4 is initialized from sqlparse.lexer import Lexer # type: ignore if hasattr(Lexer, "get_default_instance"): Lexer.get_default_instance() node = self._compile_code(node, manifest, extra_context) node, _ = self._recursively_prepend_ctes(node, manifest, extra_context) if write: self._write_node(node, split_suffix=split_suffix) return node def inject_ctes_into_sql(sql: str, ctes: List[InjectedCTE]) -> str: """ `ctes` is a list of InjectedCTEs like: [ InjectedCTE( id="cte_id_1", sql="__dbt__cte__ephemeral as (select * from table)", ), InjectedCTE( id="cte_id_2", sql="__dbt__cte__events as (select id, type from events)", ), ] Given `sql` like: "with internal_cte as (select * from sessions) select * from internal_cte" This will spit out: "with __dbt__cte__ephemeral as (select * from table), __dbt__cte__events as (select id, type from events), internal_cte as (select * from sessions) select * from internal_cte" (Whitespace enhanced for readability.) """ if len(ctes) == 0: return sql try: parsed_stmts = sqlparse.parse(sql) except Exception as e: compilation_exception_msg = str(e) if "grouping depth exceeded" in compilation_exception_msg.lower(): compilation_exception_msg = ( f"{compilation_exception_msg} You may raise the limit via " f'--sqlparse \'{{"MAX_GROUPING_DEPTH": "<value>"}}\'.' ) if "number of tokens exceeded" in compilation_exception_msg.lower(): compilation_exception_msg = ( f"{compilation_exception_msg} You may raise the limit via " f'--sqlparse \'{{"MAX_GROUPING_TOKENS": "<value>"}}\'.' ) raise CompilationError(compilation_exception_msg) parsed = parsed_stmts[0] with_stmt = None for token in parsed.tokens: if token.is_keyword and token.normalized == "WITH": with_stmt = token elif token.is_keyword and token.normalized == "RECURSIVE" and with_stmt is not None: with_stmt = token break elif not token.is_whitespace and with_stmt is not None: break if with_stmt is None: # no with stmt, add one, and inject CTEs right at the beginning # [original_sql] first_token = parsed.token_first() with_token = sqlparse.sql.Token(sqlparse.tokens.Keyword, "with") parsed.insert_before(first_token, with_token) # [with][original_sql] injected_ctes = ", ".join(c.sql for c in ctes) + " " injected_ctes_token = sqlparse.sql.Token(sqlparse.tokens.Keyword, injected_ctes) parsed.insert_after(with_token, injected_ctes_token) # [with][joined_ctes][original_sql] else: # with stmt exists so we don't need to add one, but we do need to add a comma # between the injected ctes and the original sql # [with][original_sql] injected_ctes = ", ".join(c.sql for c in ctes) injected_ctes_token = sqlparse.sql.Token(sqlparse.tokens.Keyword, injected_ctes) parsed.insert_after(with_stmt, injected_ctes_token) # [with][joined_ctes][original_sql] comma_token = sqlparse.sql.Token(sqlparse.tokens.Punctuation, ", ") parsed.insert_after(injected_ctes_token, comma_token) # [with][joined_ctes][, ][original_sql] return str(parsed) ================================================ FILE: core/dbt/config/README.md ================================================ # Config README ================================================ FILE: core/dbt/config/__init__.py ================================================ # all these are just exports, they need "noqa" so flake8 will not complain. from .profile import Profile # noqa from .project import IsFQNResource, PartialProject, Project # noqa from .runtime import RuntimeConfig # noqa ================================================ FILE: core/dbt/config/catalogs.py ================================================ import os from copy import deepcopy from typing import Any, Dict, List, Optional from dbt.artifacts.resources import Catalog, CatalogWriteIntegrationConfig from dbt.clients.yaml_helper import load_yaml_text from dbt.config.renderer import SecretRenderer from dbt.constants import CATALOGS_FILE_NAME from dbt.exceptions import YamlLoadError from dbt_common.clients.system import load_file_contents from dbt_common.exceptions import CompilationError, DbtValidationError def load_catalogs_yml(project_dir: str, project_name: str) -> Dict[str, Any]: path = os.path.join(project_dir, CATALOGS_FILE_NAME) if os.path.isfile(path): try: contents = load_file_contents(path, strip=False) yaml_content = load_yaml_text(contents) if not yaml_content: raise DbtValidationError(f"The file at {path} is empty") return yaml_content except DbtValidationError as e: raise YamlLoadError(project_name=project_name, path=CATALOGS_FILE_NAME, exc=e) return {} def load_single_catalog(raw_catalog: Dict[str, Any], renderer: SecretRenderer) -> Catalog: try: rendered_catalog = renderer.render_data(raw_catalog) except CompilationError as exc: raise DbtValidationError(str(exc)) from exc Catalog.validate(rendered_catalog) write_integrations = [] write_integration_names = set() for raw_integration in rendered_catalog.get("write_integrations", []): if raw_integration["name"] in write_integration_names: raise DbtValidationError( f"Catalog '{rendered_catalog['name']}' cannot have multiple 'write_integrations' with the same name: '{raw_integration['name']}'." ) # We're going to let the adapter validate the integration config write_integrations.append( CatalogWriteIntegrationConfig(**raw_integration, catalog_name=raw_catalog["name"]) ) write_integration_names.add(raw_integration["name"]) # Validate + set default active_write_integration if unset active_write_integration = rendered_catalog.get("active_write_integration") valid_write_integration_names = [integration.name for integration in write_integrations] if not active_write_integration: if len(valid_write_integration_names) == 1: active_write_integration = write_integrations[0].name else: raise DbtValidationError( f"Catalog '{rendered_catalog['name']}' must specify an 'active_write_integration' when multiple 'write_integrations' are provided." ) else: if active_write_integration not in valid_write_integration_names: raise DbtValidationError( f"Catalog '{rendered_catalog['name']}' must specify an 'active_write_integration' from its set of defined 'write_integrations': {valid_write_integration_names}. Got: '{active_write_integration}'." ) return Catalog( name=raw_catalog["name"], active_write_integration=active_write_integration, write_integrations=write_integrations, ) def load_catalogs(project_dir: str, project_name: str, cli_vars: Dict[str, Any]) -> List[Catalog]: raw_catalogs = load_catalogs_yml(project_dir, project_name).get("catalogs", []) catalogs_renderer = SecretRenderer(cli_vars) return [load_single_catalog(raw_catalog, catalogs_renderer) for raw_catalog in raw_catalogs] def get_active_write_integration(catalog: Catalog) -> Optional[CatalogWriteIntegrationConfig]: for integration in catalog.write_integrations: if integration.name == catalog.active_write_integration: active_integration = deepcopy(integration) active_integration.catalog_name = active_integration.name active_integration.name = catalog.name return active_integration return None ================================================ FILE: core/dbt/config/profile.py ================================================ import os from dataclasses import dataclass from typing import Any, Dict, Optional, Tuple from dbt.adapters.contracts.connection import Credentials, HasCredentials from dbt.clients.yaml_helper import load_yaml_text from dbt.contracts.project import ProfileConfig from dbt.events.types import MissingProfileTarget from dbt.exceptions import ( CompilationError, DbtProfileError, DbtProjectError, DbtRuntimeError, ProfileConfigError, ) from dbt.flags import get_flags from dbt_common.clients.system import load_file_contents from dbt_common.dataclass_schema import ValidationError from dbt_common.events.functions import fire_event from dbt_common.exceptions import DbtValidationError from .renderer import ProfileRenderer DEFAULT_THREADS = 1 INVALID_PROFILE_MESSAGE = """ dbt encountered an error while trying to read your profiles.yml file. {error_string} """ def read_profile(profiles_dir: str) -> Dict[str, Any]: path = os.path.join(profiles_dir, "profiles.yml") contents = None if os.path.isfile(path): try: contents = load_file_contents(path, strip=False) yaml_content = load_yaml_text(contents) if not yaml_content: msg = f"The profiles.yml file at {path} is empty" raise DbtProfileError(INVALID_PROFILE_MESSAGE.format(error_string=msg)) return yaml_content except DbtValidationError as e: msg = INVALID_PROFILE_MESSAGE.format(error_string=e) raise DbtValidationError(msg) from e return {} # The Profile class is included in RuntimeConfig, so any attribute # additions must also be set where the RuntimeConfig class is created # `init=False` is a workaround for https://bugs.python.org/issue45081 @dataclass(init=False) class Profile(HasCredentials): profile_name: str target_name: str threads: int credentials: Credentials profile_env_vars: Dict[str, Any] log_cache_events: bool def __init__( self, profile_name: str, target_name: str, threads: int, credentials: Credentials, ) -> None: """ TODO: Is this no longer needed now that 3.9 is no longer supported? Explicitly defining `__init__` to work around bug in Python 3.9.7 https://bugs.python.org/issue45081 """ self.profile_name = profile_name self.target_name = target_name self.threads = threads self.credentials = credentials self.profile_env_vars = {} # never available on init self.log_cache_events = ( get_flags().LOG_CACHE_EVENTS ) # never available on init, set for adapter instantiation via AdapterRequiredConfig def to_profile_info(self, serialize_credentials: bool = False) -> Dict[str, Any]: """Unlike to_project_config, this dict is not a mirror of any existing on-disk data structure. It's used when creating a new profile from an existing one. :param serialize_credentials bool: If True, serialize the credentials. Otherwise, the Credentials object will be copied. :returns dict: The serialized profile. """ result = { "profile_name": self.profile_name, "target_name": self.target_name, "threads": self.threads, "credentials": self.credentials, } if serialize_credentials: result["credentials"] = self.credentials.to_dict(omit_none=True) return result def to_target_dict(self) -> Dict[str, Any]: target = dict(self.credentials.connection_info(with_aliases=True)) target.update( { "type": self.credentials.type, "threads": self.threads, "name": self.target_name, "target_name": self.target_name, "profile_name": self.profile_name, } ) return target def __eq__(self, other: object) -> bool: if not (isinstance(other, self.__class__) and isinstance(self, other.__class__)): return NotImplemented return self.to_profile_info() == other.to_profile_info() def validate(self): try: if self.credentials: dct = self.credentials.to_dict(omit_none=True) self.credentials.validate(dct) dct = self.to_profile_info(serialize_credentials=True) ProfileConfig.validate(dct) except ValidationError as exc: raise ProfileConfigError(exc) from exc @staticmethod def _credentials_from_profile( profile: Dict[str, Any], profile_name: str, target_name: str ) -> Credentials: # avoid an import cycle from dbt.adapters.factory import load_plugin # credentials carry their 'type' in their actual type, not their # attributes. We do want this in order to pick our Credentials class. if "type" not in profile: raise DbtProfileError( 'required field "type" not found in profile {} and target {}'.format( profile_name, target_name ) ) typename = profile.pop("type") try: cls = load_plugin(typename) data = cls.translate_aliases(profile) cls.validate(data) credentials = cls.from_dict(data) except (DbtRuntimeError, ValidationError) as e: msg = str(e) if isinstance(e, DbtRuntimeError) else e.message raise DbtProfileError( 'Credentials in profile "{}", target "{}" invalid: {}'.format( profile_name, target_name, msg ) ) from e return credentials @staticmethod def pick_profile_name( args_profile_name: Optional[str], project_profile_name: Optional[str] = None, ) -> str: # TODO: Duplicating this method as direct copy of the implementation in dbt.cli.resolvers # dbt.cli.resolvers implementation can't be used because it causes a circular dependency. # This should be removed and use a safe default access on the Flags module when # https://github.com/dbt-labs/dbt-core/issues/6259 is closed. def default_profiles_dir(): from pathlib import Path return Path.cwd() if (Path.cwd() / "profiles.yml").exists() else Path.home() / ".dbt" profile_name = project_profile_name if args_profile_name is not None: profile_name = args_profile_name if profile_name is None: NO_SUPPLIED_PROFILE_ERROR = """\ dbt cannot run because no profile was specified for this dbt project. To specify a profile for this project, add a line like the this to your dbt_project.yml file: profile: [profile name] Here, [profile name] should be replaced with a profile name defined in your profiles.yml file. You can find profiles.yml here: {profiles_file}/profiles.yml """.format( profiles_file=default_profiles_dir() ) raise DbtProjectError(NO_SUPPLIED_PROFILE_ERROR) return profile_name @staticmethod def _get_profile_data( profile: Dict[str, Any], profile_name: str, target_name: str ) -> Dict[str, Any]: if "outputs" not in profile: raise DbtProfileError("outputs not specified in profile '{}'".format(profile_name)) outputs = profile["outputs"] if target_name not in outputs: outputs = "\n".join(" - {}".format(output) for output in outputs) msg = ( "The profile '{}' does not have a target named '{}'. The " "valid target names for this profile are:\n{}".format( profile_name, target_name, outputs ) ) raise DbtProfileError(msg, result_type="invalid_target") profile_data = outputs[target_name] if not isinstance(profile_data, dict): msg = ( f"output '{target_name}' of profile '{profile_name}' is " f"misconfigured in profiles.yml" ) raise DbtProfileError(msg, result_type="invalid_target") return profile_data @classmethod def from_credentials( cls, credentials: Credentials, threads: int, profile_name: str, target_name: str, ) -> "Profile": """Create a profile from an existing set of Credentials and the remaining information. :param credentials: The credentials dict for this profile. :param threads: The number of threads to use for connections. :param profile_name: The profile name used for this profile. :param target_name: The target name used for this profile. :raises DbtProfileError: If the profile is invalid. :returns: The new Profile object. """ profile = cls( profile_name=profile_name, target_name=target_name, threads=threads, credentials=credentials, ) profile.validate() return profile @classmethod def render_profile( cls, raw_profile: Dict[str, Any], profile_name: str, target_override: Optional[str], renderer: ProfileRenderer, ) -> Tuple[str, Dict[str, Any]]: """This is a containment zone for the hateful way we're rendering profiles. """ # rendering profiles is a bit complex. Two constraints cause trouble: # 1) users should be able to use environment/cli variables to specify # the target in their profile. # 2) Missing environment/cli variables in profiles/targets that don't # end up getting selected should not cause errors. # so first we'll just render the target name, then we use that rendered # name to extract a profile that we can render. if target_override is not None: target_name = target_override elif "target" in raw_profile: # render the target if it was parsed from yaml target_name = renderer.render_value(raw_profile["target"]) else: target_name = "default" fire_event(MissingProfileTarget(profile_name=profile_name, target_name=target_name)) raw_profile_data = cls._get_profile_data(raw_profile, profile_name, target_name) try: profile_data = renderer.render_data(raw_profile_data) except CompilationError as exc: raise DbtProfileError(str(exc)) from exc return target_name, profile_data @classmethod def from_raw_profile_info( cls, raw_profile: Dict[str, Any], profile_name: str, renderer: ProfileRenderer, target_override: Optional[str] = None, threads_override: Optional[int] = None, ) -> "Profile": """Create a profile from its raw profile information. (this is an intermediate step, mostly useful for unit testing) :param raw_profile: The profile data for a single profile, from disk as yaml and its values rendered with jinja. :param profile_name: The profile name used. :param renderer: The config renderer. :param target_override: The target to use, if provided on the command line. :param threads_override: The thread count to use, if provided on the command line. :raises DbtProfileError: If the profile is invalid or missing, or the target could not be found :returns: The new Profile object. """ # TODO: should it be, and the values coerced to bool? target_name, profile_data = cls.render_profile( raw_profile, profile_name, target_override, renderer ) # valid connections never include the number of threads, but it's # stored on a per-connection level in the raw configs threads = profile_data.pop("threads", DEFAULT_THREADS) if threads_override is not None: threads = threads_override credentials: Credentials = cls._credentials_from_profile( profile_data, profile_name, target_name ) return cls.from_credentials( credentials=credentials, profile_name=profile_name, target_name=target_name, threads=threads, ) @classmethod def from_raw_profiles( cls, raw_profiles: Dict[str, Any], profile_name: str, renderer: ProfileRenderer, target_override: Optional[str] = None, threads_override: Optional[int] = None, ) -> "Profile": """ :param raw_profiles: The profile data, from disk as yaml. :param profile_name: The profile name to use. :param renderer: The config renderer. :param target_override: The target to use, if provided on the command line. :param threads_override: The thread count to use, if provided on the command line. :raises DbtProjectError: If there is no profile name specified in the project or the command line arguments :raises DbtProfileError: If the profile is invalid or missing, or the target could not be found :returns: The new Profile object. """ if profile_name not in raw_profiles: raise DbtProjectError("Could not find profile named '{}'".format(profile_name)) # First, we've already got our final decision on profile name, and we # don't render keys, so we can pluck that out raw_profile = raw_profiles[profile_name] if not raw_profile: msg = f"Profile {profile_name} in profiles.yml is empty" raise DbtProfileError(INVALID_PROFILE_MESSAGE.format(error_string=msg)) return cls.from_raw_profile_info( raw_profile=raw_profile, profile_name=profile_name, renderer=renderer, target_override=target_override, threads_override=threads_override, ) @classmethod def render( cls, renderer: ProfileRenderer, project_profile_name: Optional[str], profile_name_override: Optional[str] = None, target_override: Optional[str] = None, threads_override: Optional[int] = None, ) -> "Profile": """Given the raw profiles as read from disk and the name of the desired profile if specified, return the profile component of the runtime config. :param args argparse.Namespace: The arguments as parsed from the cli. :param project_profile_name Optional[str]: The profile name, if specified in a project. :raises DbtProjectError: If there is no profile name specified in the project or the command line arguments, or if the specified profile is not found :raises DbtProfileError: If the profile is invalid or missing, or the target could not be found. :returns Profile: The new Profile object. """ flags = get_flags() raw_profiles = read_profile(flags.PROFILES_DIR) profile_name = cls.pick_profile_name(profile_name_override, project_profile_name) return cls.from_raw_profiles( raw_profiles=raw_profiles, profile_name=profile_name, renderer=renderer, target_override=target_override, threads_override=threads_override, ) ================================================ FILE: core/dbt/config/project.py ================================================ import os from copy import deepcopy from dataclasses import dataclass, field from itertools import chain from typing import Any, Dict, List, Mapping, Optional, TypeVar, Union from typing_extensions import Protocol, runtime_checkable from dbt import deprecations from dbt.adapters.contracts.connection import QueryComment from dbt.clients.checked_load import ( checked_load, issue_deprecation_warnings_for_failures, ) from dbt.clients.yaml_helper import load_yaml_text from dbt.config.selectors import SelectorDict from dbt.config.utils import normalize_warn_error_options from dbt.constants import ( DBT_PROJECT_FILE_NAME, DEPENDENCIES_FILE_NAME, PACKAGE_LOCK_FILE_NAME, PACKAGE_LOCK_HASH_KEY, PACKAGES_FILE_NAME, VARS_FILE_NAME, ) from dbt.contracts.project import PackageConfig from dbt.contracts.project import Project as ProjectContract from dbt.contracts.project import ProjectFlags, ProjectPackageMetadata, SemverString from dbt.exceptions import ( DbtExclusivePropertyUseError, DbtProjectError, DbtRuntimeError, ProjectContractBrokenError, ProjectContractError, ) from dbt.flags import get_flags from dbt.graph import SelectionSpec from dbt.node_types import NodeType from dbt.utils import MultiDict, coerce_dict_str, md5 from dbt.version import get_installed_version from dbt_common.clients.system import load_file_contents, path_exists from dbt_common.dataclass_schema import ValidationError from dbt_common.exceptions import SemverError from dbt_common.helper_types import NoValue from dbt_common.semver import VersionSpecifier, versions_compatible from .renderer import DbtProjectYamlRenderer, PackageRenderer from .selectors import ( SelectorConfig, selector_config_from_data, selector_data_from_root, ) INVALID_VERSION_ERROR = """\ This version of dbt is not supported with the '{package}' package. Installed version of dbt: {installed} Required version of dbt for '{package}': {version_spec} Check for a different version of the '{package}' package, or run dbt again with \ --no-version-check """ IMPOSSIBLE_VERSION_ERROR = """\ The package version requirement can never be satisfied for the '{package} package. Required versions of dbt for '{package}': {version_spec} Check for a different version of the '{package}' package, or run dbt again with \ --no-version-check """ MALFORMED_PACKAGE_ERROR = """\ The packages.yml file in this project is malformed. Please double check the contents of this file and fix any errors before retrying. You can find more information on the syntax for this file here: https://docs.getdbt.com/docs/package-management Validator Error: {error} """ MISSING_DBT_PROJECT_ERROR = """\ No {DBT_PROJECT_FILE_NAME} found at expected path {path} Verify that each entry within packages.yml (and their transitive dependencies) contains a file named {DBT_PROJECT_FILE_NAME} """ @runtime_checkable class IsFQNResource(Protocol): fqn: List[str] resource_type: NodeType package_name: str def _load_yaml(path, validate: bool = False): contents = load_file_contents(path) if validate: result, failures = checked_load(contents) issue_deprecation_warnings_for_failures(failures=failures, file=path) return result else: return load_yaml_text(contents) def load_yml_dict(file_path): ret = {} if path_exists(file_path): ret = _load_yaml(file_path) or {} return ret def vars_data_from_root(project_root: str) -> Dict[str, Any]: """Load vars from vars.yml file if it exists. Returns the contents of the 'vars' key, or empty dict if file doesn't exist or has no vars key. """ vars_yml_path = os.path.join(project_root, VARS_FILE_NAME) vars_file_dict = load_yml_dict(vars_yml_path) if not vars_file_dict: return {} return vars_file_dict.get("vars", {}) def validate_vars_not_in_both( project_dict: Dict[str, Any], has_vars_file: bool, ) -> None: """Raise error if vars defined in both vars.yml and dbt_project.yml.""" has_project_vars = "vars" in project_dict and project_dict["vars"] if has_vars_file and has_project_vars: raise DbtProjectError( f"Variables cannot be defined in both {VARS_FILE_NAME} and {DBT_PROJECT_FILE_NAME}. " ) def package_and_project_data_from_root(project_root): packages_yml_dict = load_yml_dict(f"{project_root}/{PACKAGES_FILE_NAME}") dependencies_yml_dict = load_yml_dict(f"{project_root}/{DEPENDENCIES_FILE_NAME}") if "packages" in packages_yml_dict and "packages" in dependencies_yml_dict: msg = "The 'packages' key cannot be specified in both packages.yml and dependencies.yml" raise DbtProjectError(msg) if "projects" in packages_yml_dict: msg = "The 'projects' key cannot be specified in packages.yml" raise DbtProjectError(msg) packages_specified_path = PACKAGES_FILE_NAME packages_dict = {} if "packages" in dependencies_yml_dict: packages_dict["packages"] = dependencies_yml_dict["packages"] packages_specified_path = DEPENDENCIES_FILE_NAME else: # don't check for "packages" here so we capture invalid keys in packages.yml packages_dict = packages_yml_dict return packages_dict, packages_specified_path def package_config_from_data( packages_data: Dict[str, Any], unrendered_packages_data: Optional[Dict[str, Any]] = None, ) -> PackageConfig: if not packages_data: packages_data = {"packages": []} # this depends on the two lists being in the same order if unrendered_packages_data: unrendered_packages_data = deepcopy(unrendered_packages_data) for i in range(0, len(packages_data.get("packages", []))): packages_data["packages"][i]["unrendered"] = unrendered_packages_data["packages"][i] if PACKAGE_LOCK_HASH_KEY in packages_data: packages_data.pop(PACKAGE_LOCK_HASH_KEY) try: PackageConfig.validate(packages_data) packages = PackageConfig.from_dict(packages_data) except ValidationError as e: raise DbtProjectError(MALFORMED_PACKAGE_ERROR.format(error=str(e.message))) from e return packages def load_package_lock_config(project_root: str) -> PackageConfig: locked_packages = load_yml_dict(f"{project_root}/{PACKAGE_LOCK_FILE_NAME}") or {"packages": []} return PackageConfig.from_dict(locked_packages) def _parse_versions(versions: Union[List[str], str]) -> List[VersionSpecifier]: """Parse multiple versions as read from disk. The versions value may be any one of: - a single version string ('>0.12.1') - a single string specifying multiple comma-separated versions ('>0.11.1,<=0.12.2') - an array of single-version strings (['>0.11.1', '<=0.12.2']) Regardless, this will return a list of VersionSpecifiers """ if isinstance(versions, str): versions = versions.split(",") return [VersionSpecifier.from_version_string(v) for v in versions] def _all_source_paths(*args: List[str]) -> List[str]: paths = chain(*args) # Strip trailing slashes since the path is the same even though the name is not stripped_paths = map(lambda s: s.rstrip("/"), paths) return list(set(stripped_paths)) T = TypeVar("T") def flag_or(flag: Optional[T], value: Optional[T], default: T) -> T: if flag is None: return value_or(value, default) else: return flag def value_or(value: Optional[T], default: T) -> T: if value is None: return default else: return value def load_raw_project(project_root: str, validate: bool = False) -> Dict[str, Any]: project_root = os.path.normpath(project_root) project_yaml_filepath = os.path.join(project_root, DBT_PROJECT_FILE_NAME) # get the project.yml contents if not path_exists(project_yaml_filepath): raise DbtProjectError( MISSING_DBT_PROJECT_ERROR.format( path=project_yaml_filepath, DBT_PROJECT_FILE_NAME=DBT_PROJECT_FILE_NAME ) ) project_dict = _load_yaml(project_yaml_filepath, validate=validate) if validate: from dbt.jsonschemas.jsonschemas import jsonschema_validate, project_schema jsonschema_validate( schema=project_schema(), json=project_dict, file_path=project_yaml_filepath ) if not isinstance(project_dict, dict): raise DbtProjectError(f"{DBT_PROJECT_FILE_NAME} does not parse to a dictionary") if "tests" in project_dict and "data_tests" not in project_dict: project_dict["data_tests"] = project_dict.pop("tests") return project_dict def _query_comment_from_cfg( cfg_query_comment: Union[QueryComment, NoValue, str, None] ) -> QueryComment: if not cfg_query_comment: return QueryComment(comment="") if isinstance(cfg_query_comment, str): return QueryComment(comment=cfg_query_comment) if isinstance(cfg_query_comment, NoValue): return QueryComment() return cfg_query_comment def validate_version(dbt_version: List[VersionSpecifier], project_name: str): """Ensure this package works with the installed version of dbt.""" installed = get_installed_version() if not versions_compatible(*dbt_version): msg = IMPOSSIBLE_VERSION_ERROR.format( package=project_name, version_spec=[x.to_version_string() for x in dbt_version] ) raise DbtProjectError(msg) if not versions_compatible(installed, *dbt_version): msg = INVALID_VERSION_ERROR.format( package=project_name, installed=installed.to_version_string(), version_spec=[x.to_version_string() for x in dbt_version], ) raise DbtProjectError(msg) def _get_required_version( project_dict: Dict[str, Any], verify_version: bool, ) -> List[VersionSpecifier]: dbt_raw_version: Union[List[str], str] = ">=0.0.0" required = project_dict.get("require-dbt-version") if required is not None: dbt_raw_version = required try: dbt_version = _parse_versions(dbt_raw_version) except SemverError as e: raise DbtProjectError(str(e)) from e if verify_version: # no name is also an error that we want to raise if "name" not in project_dict: raise DbtProjectError( 'Required "name" field not present in project', ) validate_version(dbt_version, project_dict["name"]) return dbt_version @dataclass class RenderComponents: project_dict: Dict[str, Any] = field(metadata=dict(description="The project dictionary")) packages_dict: Dict[str, Any] = field(metadata=dict(description="The packages dictionary")) selectors_dict: Dict[str, Any] = field(metadata=dict(description="The selectors dictionary")) @dataclass class PartialProject(RenderComponents): # This class includes the project_dict, packages_dict, selectors_dict, etc from RenderComponents profile_name: Optional[str] = field( metadata=dict(description="The unrendered profile name in the project, if set") ) project_name: Optional[str] = field( metadata=dict( description=( "The name of the project. This should always be set and will not be rendered" ) ) ) project_root: str = field( metadata=dict(description="The root directory of the project"), ) verify_version: bool = field( metadata=dict(description=("If True, verify the dbt version matches the required version")) ) packages_specified_path: str = field( metadata=dict(description="The filename where packages were specified") ) def render_profile_name(self, renderer) -> Optional[str]: if self.profile_name is None: return None return renderer.render_value(self.profile_name) def get_rendered( self, renderer: DbtProjectYamlRenderer, ) -> RenderComponents: rendered_project = renderer.render_project(self.project_dict, self.project_root) rendered_packages = renderer.render_packages( self.packages_dict, self.packages_specified_path ) rendered_selectors = renderer.render_selectors(self.selectors_dict) return RenderComponents( project_dict=rendered_project, packages_dict=rendered_packages, selectors_dict=rendered_selectors, ) # Called by Project.from_project_root which first calls PartialProject.from_project_root def render( self, renderer: DbtProjectYamlRenderer, vars_from_file: Optional[Dict[str, Any]] = None, ) -> "Project": try: rendered = self.get_rendered(renderer) return self.create_project(rendered, vars_from_file=vars_from_file) except DbtProjectError as exc: if exc.path is None: exc.path = os.path.join(self.project_root, DBT_PROJECT_FILE_NAME) raise def render_package_metadata(self, renderer: PackageRenderer) -> ProjectPackageMetadata: packages_data = renderer.render_data(self.packages_dict) packages_config = package_config_from_data(packages_data, self.packages_dict) if not self.project_name: raise DbtProjectError(f"Package defined in {DBT_PROJECT_FILE_NAME} must have a name!") return ProjectPackageMetadata(self.project_name, packages_config.packages) def check_config_path( self, project_dict, deprecated_path, expected_path=None, default_value=None ): if deprecated_path in project_dict: if expected_path in project_dict: msg = ( "{deprecated_path} and {expected_path} cannot both be defined. The " "`{deprecated_path}` config has been deprecated in favor of `{expected_path}`. " f"Please update your `{DBT_PROJECT_FILE_NAME}` configuration to reflect this " "change." ) raise DbtProjectError( msg.format(deprecated_path=deprecated_path, expected_path=expected_path) ) # this field is no longer supported, but many projects may specify it with the default value # if so, let's only raise this deprecation warning if they set a custom value if not default_value or project_dict[deprecated_path] != default_value: kwargs = {"deprecated_path": deprecated_path} if expected_path: kwargs.update({"exp_path": expected_path}) deprecations.warn(f"project-config-{deprecated_path}", **kwargs) def create_project( self, rendered: RenderComponents, vars_from_file: Optional[Dict[str, Any]] = None, ) -> "Project": unrendered = RenderComponents( project_dict=self.project_dict, packages_dict=self.packages_dict, selectors_dict=self.selectors_dict, ) dbt_version = _get_required_version( rendered.project_dict, verify_version=self.verify_version, ) self.check_config_path(rendered.project_dict, "source-paths", "model-paths") self.check_config_path(rendered.project_dict, "data-paths", "seed-paths") self.check_config_path(rendered.project_dict, "log-path", default_value="logs") self.check_config_path(rendered.project_dict, "target-path", default_value="target") try: ProjectContract.validate(rendered.project_dict) cfg = ProjectContract.from_dict(rendered.project_dict) except ValidationError as e: raise ProjectContractError(e) from e # name/version are required in the Project definition, so we can assume # they are present name = cfg.name version = cfg.version # this is added at project_dict parse time and should always be here # once we see it. if cfg.project_root is None: raise DbtProjectError("cfg must have a project root!") else: project_root = cfg.project_root # this is only optional in the sense that if it's not present, it needs # to have been a cli argument. profile_name = cfg.profile # these are all the defaults # `source_paths` is deprecated but still allowed. Copy it into # `model_paths` to simlify logic throughout the rest of the system. model_paths: List[str] = value_or( cfg.model_paths if "model-paths" in rendered.project_dict else cfg.source_paths, ["models"], ) macro_paths: List[str] = value_or(cfg.macro_paths, ["macros"]) # `data_paths` is deprecated but still allowed. Copy it into # `seed_paths` to simlify logic throughout the rest of the system. seed_paths: List[str] = value_or( cfg.seed_paths if "seed-paths" in rendered.project_dict else cfg.data_paths, ["seeds"] ) test_paths: List[str] = value_or(cfg.test_paths, ["tests"]) analysis_paths: List[str] = value_or(cfg.analysis_paths, ["analyses"]) snapshot_paths: List[str] = value_or(cfg.snapshot_paths, ["snapshots"]) function_paths: List[str] = value_or(cfg.function_paths, ["functions"]) all_source_paths: List[str] = _all_source_paths( model_paths, seed_paths, snapshot_paths, analysis_paths, macro_paths, test_paths, function_paths, ) docs_paths: List[str] = value_or(cfg.docs_paths, all_source_paths) asset_paths: List[str] = value_or(cfg.asset_paths, []) global_flags = get_flags() flag_target_path = str(global_flags.TARGET_PATH) if global_flags.TARGET_PATH else None target_path: str = flag_or(flag_target_path, cfg.target_path, "target") log_path: str = str(global_flags.LOG_PATH) clean_targets: List[str] = value_or(cfg.clean_targets, [target_path]) packages_install_path: str = value_or(cfg.packages_install_path, "dbt_packages") # in the default case we'll populate this once we know the adapter type # It would be nice to just pass along a Quoting here, but that would # break many things quoting: Dict[str, Any] = {} if cfg.quoting is not None: quoting = cfg.quoting.to_dict(omit_none=True) dispatch: List[Dict[str, Any]] models: Dict[str, Any] seeds: Dict[str, Any] snapshots: Dict[str, Any] sources: Dict[str, Any] data_tests: Dict[str, Any] unit_tests: Dict[str, Any] metrics: Dict[str, Any] semantic_models: Dict[str, Any] saved_queries: Dict[str, Any] exposures: Dict[str, Any] functions: Dict[str, Any] vars_value: VarProvider dbt_cloud: Dict[str, Any] dispatch = cfg.dispatch models = cfg.models seeds = cfg.seeds snapshots = cfg.snapshots sources = cfg.sources # the `tests` config is deprecated but still allowed. Copy it into # `data_tests` to simplify logic throughout the rest of the system. data_tests = cfg.data_tests if "data_tests" in rendered.project_dict else cfg.tests unit_tests = cfg.unit_tests metrics = cfg.metrics semantic_models = cfg.semantic_models saved_queries = cfg.saved_queries exposures = cfg.exposures functions = cfg.functions # Use vars from vars.yml if provided, otherwise use vars from dbt_project.yml # Mutual exclusivity ensures only one source is populated if vars_from_file: vars_dict = vars_from_file else: vars_dict = cfg.vars or {} vars_value = VarProvider(vars_dict) # There will never be any project_env_vars when it's first created project_env_vars: Dict[str, Any] = {} on_run_start: List[str] = value_or(cfg.on_run_start, []) on_run_end: List[str] = value_or(cfg.on_run_end, []) query_comment = _query_comment_from_cfg(cfg.query_comment) packages: PackageConfig = package_config_from_data( rendered.packages_dict, unrendered.packages_dict ) selectors = selector_config_from_data(rendered.selectors_dict) manifest_selectors: Dict[str, Any] = {} if rendered.selectors_dict and rendered.selectors_dict["selectors"]: # this is a dict with a single key 'selectors' pointing to a list # of dicts. manifest_selectors = SelectorDict.parse_from_selectors_list( rendered.selectors_dict["selectors"] ) dbt_cloud = cfg.dbt_cloud flags: Dict[str, Any] = cfg.flags project = Project( project_name=name, version=version, project_root=project_root, profile_name=profile_name, model_paths=model_paths, macro_paths=macro_paths, seed_paths=seed_paths, test_paths=test_paths, analysis_paths=analysis_paths, docs_paths=docs_paths, asset_paths=asset_paths, target_path=target_path, snapshot_paths=snapshot_paths, function_paths=function_paths, clean_targets=clean_targets, log_path=log_path, packages_install_path=packages_install_path, packages_specified_path=self.packages_specified_path, quoting=quoting, models=models, on_run_start=on_run_start, on_run_end=on_run_end, dispatch=dispatch, seeds=seeds, snapshots=snapshots, dbt_version=dbt_version, packages=packages, manifest_selectors=manifest_selectors, selectors=selectors, query_comment=query_comment, sources=sources, data_tests=data_tests, unit_tests=unit_tests, metrics=metrics, semantic_models=semantic_models, saved_queries=saved_queries, exposures=exposures, functions=functions, vars=vars_value, config_version=cfg.config_version, unrendered=unrendered, project_env_vars=project_env_vars, restrict_access=cfg.restrict_access, dbt_cloud=dbt_cloud, flags=flags, vars_from_file=vars_from_file or {}, ) # sanity check - this means an internal issue project.validate() return project @classmethod def from_dicts( cls, project_root: str, project_dict: Dict[str, Any], packages_dict: Dict[str, Any], selectors_dict: Optional[Dict[str, Any]], *, verify_version: bool = False, packages_specified_path: str = PACKAGES_FILE_NAME, ): """Construct a partial project from its constituent dicts.""" project_name = project_dict.get("name") profile_name = project_dict.get("profile") # Create a PartialProject return cls( profile_name=profile_name, project_name=project_name, project_root=project_root, project_dict=project_dict, packages_dict=packages_dict, selectors_dict=selectors_dict, # type: ignore verify_version=verify_version, packages_specified_path=packages_specified_path, ) @classmethod def from_project_root( cls, project_root: str, *, verify_version: bool = False, validate: bool = False ) -> "PartialProject": project_root = os.path.normpath(project_root) project_dict = load_raw_project(project_root, validate=validate) ( packages_dict, packages_specified_path, ) = package_and_project_data_from_root(project_root) selectors_dict = selector_data_from_root(project_root) return cls.from_dicts( project_root=project_root, project_dict=project_dict, selectors_dict=selectors_dict, packages_dict=packages_dict, verify_version=verify_version, packages_specified_path=packages_specified_path, ) class VarProvider: """Var providers are tied to a particular Project.""" def __init__(self, vars: Dict[str, Dict[str, Any]]) -> None: self.vars = vars def vars_for(self, node: IsFQNResource, adapter_type: str) -> Mapping[str, Any]: # in v2, vars are only either project or globally scoped merged = MultiDict([self.vars]) merged.add(self.vars.get(node.package_name, {})) return merged def to_dict(self): return self.vars # The Project class is included in RuntimeConfig, so any attribute # additions must also be set where the RuntimeConfig class is created @dataclass class Project: project_name: str version: Optional[Union[SemverString, float]] project_root: str profile_name: Optional[str] model_paths: List[str] macro_paths: List[str] seed_paths: List[str] test_paths: List[str] analysis_paths: List[str] docs_paths: List[str] asset_paths: List[str] target_path: str snapshot_paths: List[str] function_paths: List[str] clean_targets: List[str] log_path: str packages_install_path: str packages_specified_path: str quoting: Dict[str, Any] models: Dict[str, Any] on_run_start: List[str] on_run_end: List[str] dispatch: List[Dict[str, Any]] seeds: Dict[str, Any] snapshots: Dict[str, Any] sources: Dict[str, Any] data_tests: Dict[str, Any] unit_tests: Dict[str, Any] metrics: Dict[str, Any] semantic_models: Dict[str, Any] saved_queries: Dict[str, Any] exposures: Dict[str, Any] functions: Dict[str, Any] vars: VarProvider dbt_version: List[VersionSpecifier] packages: PackageConfig manifest_selectors: Dict[str, Any] selectors: SelectorConfig query_comment: QueryComment config_version: int unrendered: RenderComponents project_env_vars: Dict[str, Any] restrict_access: bool dbt_cloud: Dict[str, Any] flags: Dict[str, Any] vars_from_file: Dict[str, Any] @property def all_source_paths(self) -> List[str]: return _all_source_paths( self.model_paths, self.seed_paths, self.snapshot_paths, self.analysis_paths, self.macro_paths, self.test_paths, self.function_paths, ) @property def generic_test_paths(self): generic_test_paths = [] for test_path in self.test_paths: generic_test_paths.append(os.path.join(test_path, "generic")) return generic_test_paths @property def fixture_paths(self): fixture_paths = [] for test_path in self.test_paths: fixture_paths.append(os.path.join(test_path, "fixtures")) return fixture_paths def __str__(self): cfg = self.to_project_config(with_packages=True) return str(cfg) def __eq__(self, other): if not (isinstance(other, self.__class__) and isinstance(self, other.__class__)): return False return self.to_project_config(with_packages=True) == other.to_project_config( with_packages=True ) def to_project_config(self, with_packages=False): """Return a dict representation of the config that could be written to disk with `yaml.safe_dump` to get this configuration. :param with_packages bool: If True, include the serialized packages file in the root. :returns dict: The serialized profile. """ result = deepcopy( { "name": self.project_name, "version": self.version, "project-root": self.project_root, "profile": self.profile_name, "model-paths": self.model_paths, "macro-paths": self.macro_paths, "seed-paths": self.seed_paths, "test-paths": self.test_paths, "analysis-paths": self.analysis_paths, "docs-paths": self.docs_paths, "asset-paths": self.asset_paths, "target-path": self.target_path, "snapshot-paths": self.snapshot_paths, "clean-targets": self.clean_targets, "log-path": self.log_path, "quoting": self.quoting, "models": self.models, "on-run-start": self.on_run_start, "on-run-end": self.on_run_end, "dispatch": self.dispatch, "seeds": self.seeds, "snapshots": self.snapshots, "sources": self.sources, "data_tests": self.data_tests, "unit_tests": self.unit_tests, "metrics": self.metrics, "semantic-models": self.semantic_models, "saved-queries": self.saved_queries, "exposures": self.exposures, "functions": self.functions, "vars": self.vars.to_dict(), "require-dbt-version": [v.to_version_string() for v in self.dbt_version], "restrict-access": self.restrict_access, "dbt-cloud": self.dbt_cloud, "flags": self.flags, } ) if self.query_comment: result["query-comment"] = self.query_comment.to_dict(omit_none=True) if with_packages: result.update(self.packages.to_dict(omit_none=True)) return result def validate(self): try: ProjectContract.validate(self.to_project_config()) except ValidationError as e: raise ProjectContractBrokenError(e) from e # Called by: # RtConfig.load_dependencies => RtConfig.load_projects => RtConfig.new_project => Project.from_project_root # RtConfig.from_args => RtConfig.collect_parts => load_project => Project.from_project_root @classmethod def from_project_root( cls, project_root: str, renderer: DbtProjectYamlRenderer, *, verify_version: bool = False, validate: bool = False, vars_from_file: Optional[Dict[str, Any]] = None, ) -> "Project": partial = PartialProject.from_project_root( project_root, verify_version=verify_version, validate=validate ) # Check mutual exclusivity before rendering validate_vars_not_in_both(partial.project_dict, bool(vars_from_file)) return partial.render(renderer, vars_from_file=vars_from_file) def hashed_name(self): return md5(self.project_name) def get_selector(self, name: str) -> Union[SelectionSpec, bool]: if name not in self.selectors: raise DbtRuntimeError( f"Could not find selector named {name}, expected one of {list(self.selectors)}" ) return self.selectors[name]["definition"] def get_default_selector_name(self) -> Union[str, None]: """This function fetch the default selector to use on `dbt run` (if any) :return: either a selector if default is set or None :rtype: Union[SelectionSpec, None] """ for selector_name, selector in self.selectors.items(): if selector["default"] is True: return selector_name return None def get_macro_search_order(self, macro_namespace: str): for dispatch_entry in self.dispatch: if dispatch_entry["macro_namespace"] == macro_namespace: return dispatch_entry["search_order"] return None @property def project_target_path(self): # If target_path is absolute, project_root will not be included return os.path.join(self.project_root, self.target_path) def read_project_flags(project_dir: str, profiles_dir: str) -> ProjectFlags: try: project_flags: Dict[str, Any] = {} # Read project_flags from dbt_project.yml first # Flags are instantiated before the project, so we don't # want to throw an error for non-existence of dbt_project.yml here # because it breaks things. project_root = os.path.normpath(project_dir) project_yaml_filepath = os.path.join(project_root, DBT_PROJECT_FILE_NAME) if path_exists(project_yaml_filepath): try: project_dict = load_raw_project(project_root) if "flags" in project_dict: project_flags = project_dict.pop("flags") except Exception: # This is probably a yaml load error.The error will be reported # later, when the project loads. pass from dbt.config.profile import read_profile profile = read_profile(profiles_dir) profile_project_flags: Optional[Dict[str, Any]] = {} if profile: profile_project_flags = coerce_dict_str(profile.get("config", {})) if project_flags and profile_project_flags: raise DbtProjectError( f"Do not specify both 'config' in profiles.yml and 'flags' in {DBT_PROJECT_FILE_NAME}. " "Using 'config' in profiles.yml is deprecated." ) if profile_project_flags: # This can't use WARN_ERROR or WARN_ERROR_OPTIONS because they're in # the config that we're loading. Uses special "buffer" method and fired after flags are initialized in preflight. deprecations.buffer("project-flags-moved") project_flags = profile_project_flags if project_flags is not None: # handle collapsing `include` and `error` as well as collapsing `exclude` and `warn` # for warn_error_options warn_error_options = project_flags.get("warn_error_options", {}) normalize_warn_error_options(warn_error_options) ProjectFlags.validate(project_flags) return ProjectFlags.from_dict(project_flags) except (DbtProjectError, DbtExclusivePropertyUseError) as exc: # We don't want to eat the DbtProjectError for UserConfig to ProjectFlags or # DbtConfigError for warn_error_options munging raise exc except (DbtRuntimeError, ValidationError): pass return ProjectFlags() ================================================ FILE: core/dbt/config/renderer.py ================================================ import re from datetime import date from typing import Any, Callable, Dict, Optional, Tuple, Union from dbt.adapters.contracts.connection import HasCredentials from dbt.clients.jinja import get_rendered from dbt.constants import DEPENDENCIES_FILE_NAME, SECRET_PLACEHOLDER from dbt.context.base import BaseContext from dbt.context.secret import SecretContext from dbt.context.target import TargetContext from dbt.exceptions import DbtProjectError from dbt_common.clients.jinja import catch_jinja from dbt_common.constants import SECRET_ENV_PREFIX from dbt_common.context import get_invocation_context from dbt_common.exceptions import CompilationError, RecursionError from dbt_common.utils import deep_map_render Keypath = Tuple[Union[str, int], ...] class BaseRenderer: def __init__(self, context: Dict[str, Any]) -> None: self.context = context @property def name(self): return "Rendering" def should_render_keypath(self, keypath: Keypath) -> bool: return True def render_entry(self, value: Any, keypath: Keypath) -> Any: if not self.should_render_keypath(keypath): return value return self.render_value(value, keypath) def render_value(self, value: Any, keypath: Optional[Keypath] = None) -> Any: # keypath is ignored (and someone who knows should explain why here) if not isinstance(value, str): return value if not isinstance(value, date) else value.isoformat() try: with catch_jinja(): return get_rendered(value, self.context, native=True) except CompilationError as exc: msg = f"Could not render {value}: {exc.msg}" raise CompilationError(msg) from exc def render_data(self, data: Dict[str, Any]) -> Dict[str, Any]: try: return deep_map_render(self.render_entry, data) except RecursionError: raise DbtProjectError( f"Cycle detected: {self.name} input has a reference to itself", project=data ) def _list_if_none(value): if value is None: value = [] return value def _dict_if_none(value): if value is None: value = {} return value def _list_if_none_or_string(value): value = _list_if_none(value) if isinstance(value, str): return [value] return value class ProjectPostprocessor(Dict[Keypath, Callable[[Any], Any]]): def __init__(self) -> None: super().__init__() self[("on-run-start",)] = _list_if_none_or_string self[("on-run-end",)] = _list_if_none_or_string for k in ("models", "seeds", "snapshots"): self[(k,)] = _dict_if_none self[(k, "vars")] = _dict_if_none self[(k, "pre-hook")] = _list_if_none_or_string self[(k, "post-hook")] = _list_if_none_or_string self[("seeds", "column_types")] = _dict_if_none def postprocess(self, value: Any, key: Keypath) -> Any: if key in self: handler = self[key] return handler(value) return value class DbtProjectYamlRenderer(BaseRenderer): _KEYPATH_HANDLERS = ProjectPostprocessor() def __init__( self, profile: Optional[HasCredentials] = None, cli_vars: Optional[Dict[str, Any]] = None, require_vars: bool = True, ) -> None: # Generate contexts here because we want to save the context # object in order to retrieve the env_vars. This is almost always # a TargetContext, but in the debug task we want a project # even when we don't have a profile. if cli_vars is None: cli_vars = {} # Store profile and cli_vars for creating strict context later self.profile = profile self.cli_vars = cli_vars # By default, require vars (strict mode) for proper error messages. # Commands that don't need vars (like 'deps') should explicitly pass # require_vars=False for lenient loading. if profile: self.ctx_obj = TargetContext( profile.to_target_dict(), cli_vars, require_vars=require_vars ) else: self.ctx_obj = BaseContext(cli_vars, require_vars=require_vars) # type:ignore context = self.ctx_obj.to_dict() super().__init__(context) @property def name(self): "Project config" # Uses SecretRenderer def get_package_renderer(self) -> BaseRenderer: return PackageRenderer(self.ctx_obj.cli_vars) def render_project( self, project: Dict[str, Any], project_root: str, ) -> Dict[str, Any]: """Render the project and insert the project root after rendering.""" rendered_project = self.render_data(project) rendered_project["project-root"] = project_root return rendered_project def render_packages(self, packages: Dict[str, Any], packages_specified_path: str): """Render the given packages dict""" packages = packages or {} # Sometimes this is none in tests package_renderer = self.get_package_renderer() if packages_specified_path == DEPENDENCIES_FILE_NAME: # We don't want to render the "packages" dictionary that came from dependencies.yml return packages else: return package_renderer.render_data(packages) def render_selectors(self, selectors: Dict[str, Any]): return self.render_data(selectors) def render_entry(self, value: Any, keypath: Keypath) -> Any: result = super().render_entry(value, keypath) return self._KEYPATH_HANDLERS.postprocess(result, keypath) def should_render_keypath(self, keypath: Keypath) -> bool: if not keypath: return True first = keypath[0] # run hooks are not rendered if first in {"on-run-start", "on-run-end", "query-comment"}: return False # don't render vars blocks until runtime if first == "vars": return False if first in {"seeds", "models", "snapshots", "tests", "data_tests"}: keypath_parts = {(k.lstrip("+ ") if isinstance(k, str) else k) for k in keypath} # model-level hooks late_rendered_hooks = {"pre-hook", "post-hook", "pre_hook", "post_hook"} if keypath_parts.intersection(late_rendered_hooks): return False return True class SecretRenderer(BaseRenderer): def __init__(self, cli_vars: Dict[str, Any] = {}) -> None: # Generate contexts here because we want to save the context # object in order to retrieve the env_vars. self.ctx_obj = SecretContext(cli_vars) context = self.ctx_obj.to_dict() super().__init__(context) @property def name(self): return "Secret" def render_value(self, value: Any, keypath: Optional[Keypath] = None) -> Any: # First, standard Jinja rendering, with special handling for 'secret' environment variables # "{{ env_var('DBT_SECRET_ENV_VAR') }}" -> "$$$DBT_SECRET_START$$$DBT_SECRET_ENV_{VARIABLE_NAME}$$$DBT_SECRET_END$$$" # This prevents Jinja manipulation of secrets via macros/filters that might leak partial/modified values in logs try: rendered = super().render_value(value, keypath) except Exception as ex: if keypath and "password" in keypath: # Passwords sometimes contain jinja-esque characters, but we # don't want to render them if they aren't valid jinja. rendered = value else: raise ex # Now, detect instances of the placeholder value ($$$DBT_SECRET_START...DBT_SECRET_END$$$) # and replace them with the actual secret value if SECRET_ENV_PREFIX in str(rendered): search_group = f"({SECRET_ENV_PREFIX}(.*))" pattern = SECRET_PLACEHOLDER.format(search_group).replace("$", r"\$") m = re.search( pattern, rendered, ) if m: found = m.group(1) value = get_invocation_context().env[found] replace_this = SECRET_PLACEHOLDER.format(found) return rendered.replace(replace_this, value) else: return rendered class ProfileRenderer(SecretRenderer): @property def name(self): return "Profile" class PackageRenderer(SecretRenderer): @property def name(self): return "Packages config" ================================================ FILE: core/dbt/config/runtime.py ================================================ import itertools import os from copy import deepcopy from dataclasses import dataclass from pathlib import Path from typing import ( Any, Dict, Iterable, Iterator, Mapping, MutableSet, Optional, Tuple, Type, ) from dbt import tracking from dbt.adapters.contracts.connection import ( AdapterRequiredConfig, Credentials, HasCredentials, ) from dbt.adapters.contracts.relation import ComponentName from dbt.adapters.factory import get_include_paths, get_relation_class_by_name from dbt.artifacts.resources import Quoting from dbt.config.project import ( load_package_lock_config, load_raw_project, vars_data_from_root, ) from dbt.contracts.graph.manifest import ManifestMetadata from dbt.contracts.project import Configuration from dbt.events.types import UnusedResourceConfigPath from dbt.exceptions import ( ConfigContractBrokenError, DbtProjectError, DbtRuntimeError, NonUniquePackageNameError, UninstalledPackagesFoundError, ) from dbt.flags import get_flags from dbt_common.dataclass_schema import ValidationError from dbt_common.events.functions import warn_or_error from dbt_common.helper_types import DictDefaultEmptyStr, FQNPath, PathSet from .profile import Profile from .project import Project from .renderer import DbtProjectYamlRenderer, ProfileRenderer # Called by RuntimeConfig.collect_parts class method def load_project( project_root: str, version_check: bool, profile: HasCredentials, cli_vars: Optional[Dict[str, Any]] = None, validate: bool = False, require_vars: bool = True, ) -> Project: if cli_vars is None: cli_vars = {} # Load vars.yml first (before rendering dbt_project.yml) vars_from_file = vars_data_from_root(project_root) # Merge: CLI vars take precedence over file vars merged_vars = {**vars_from_file, **cli_vars} # Renderer receives merged vars for Jinja in dbt_project.yml project_renderer = DbtProjectYamlRenderer(profile, merged_vars, require_vars=require_vars) project = Project.from_project_root( project_root, project_renderer, verify_version=version_check, validate=validate, vars_from_file=vars_from_file, ) # Save env_vars encountered in rendering for partial parsing project.project_env_vars = project_renderer.ctx_obj.env_vars return project def load_profile( project_root: str, cli_vars: Dict[str, Any], profile_name_override: Optional[str] = None, target_override: Optional[str] = None, threads_override: Optional[int] = None, ) -> Profile: raw_project = load_raw_project(project_root) raw_profile_name = raw_project.get("profile") profile_renderer = ProfileRenderer(cli_vars) profile_name = profile_renderer.render_value(raw_profile_name) profile = Profile.render( profile_renderer, profile_name, profile_name_override, target_override, threads_override ) # Save env_vars encountered in rendering for partial parsing profile.profile_env_vars = profile_renderer.ctx_obj.env_vars return profile def _project_quoting_dict(proj: Project, profile: Profile) -> Dict[ComponentName, bool]: src: Dict[str, Any] = profile.credentials.translate_aliases(proj.quoting) result: Dict[ComponentName, bool] = {} for key in ComponentName: if key in src: value = src[key] if isinstance(value, bool): result[key] = value return result @dataclass class RuntimeConfig(Project, Profile, AdapterRequiredConfig): args: Any profile_name: str cli_vars: Dict[str, Any] dependencies: Optional[Mapping[str, "RuntimeConfig"]] = None def __post_init__(self): self.validate() @classmethod def get_profile( cls, project_root: str, cli_vars: Dict[str, Any], args: Any, ) -> Profile: return load_profile( project_root, cli_vars, args.profile, args.target, args.threads, ) # Called by 'new_project' and 'from_args' @classmethod def from_parts( cls, project: Project, profile: Profile, args: Any, dependencies: Optional[Mapping[str, "RuntimeConfig"]] = None, ) -> "RuntimeConfig": """Instantiate a RuntimeConfig from its components. :param profile: A parsed dbt Profile. :param project: A parsed dbt Project. :param args: The parsed command-line arguments. :returns RuntimeConfig: The new configuration. """ quoting: Dict[str, Any] = ( get_relation_class_by_name(profile.credentials.type) .get_default_quote_policy() .replace_dict(_project_quoting_dict(project, profile)) ).to_dict(omit_none=True) cli_vars: Dict[str, Any] = getattr(args, "vars", {}) log_cache_events: bool = getattr(args, "log_cache_events", profile.log_cache_events) return cls( project_name=project.project_name, version=project.version, project_root=project.project_root, model_paths=project.model_paths, macro_paths=project.macro_paths, seed_paths=project.seed_paths, test_paths=project.test_paths, analysis_paths=project.analysis_paths, docs_paths=project.docs_paths, asset_paths=project.asset_paths, function_paths=project.function_paths, target_path=project.target_path, snapshot_paths=project.snapshot_paths, clean_targets=project.clean_targets, log_path=project.log_path, packages_install_path=project.packages_install_path, packages_specified_path=project.packages_specified_path, quoting=quoting, models=project.models, on_run_start=project.on_run_start, on_run_end=project.on_run_end, dispatch=project.dispatch, seeds=project.seeds, snapshots=project.snapshots, dbt_version=project.dbt_version, packages=project.packages, manifest_selectors=project.manifest_selectors, selectors=project.selectors, query_comment=project.query_comment, sources=project.sources, data_tests=project.data_tests, unit_tests=project.unit_tests, metrics=project.metrics, semantic_models=project.semantic_models, saved_queries=project.saved_queries, exposures=project.exposures, functions=project.functions, vars=project.vars, config_version=project.config_version, unrendered=project.unrendered, project_env_vars=project.project_env_vars, restrict_access=project.restrict_access, profile_env_vars=profile.profile_env_vars, profile_name=profile.profile_name, target_name=profile.target_name, threads=profile.threads, credentials=profile.credentials, args=args, cli_vars=cli_vars, log_cache_events=log_cache_events, dependencies=dependencies, dbt_cloud=project.dbt_cloud, flags=project.flags, vars_from_file=project.vars_from_file, ) # Called by 'load_projects' in this class def new_project(self, project_root: str) -> "RuntimeConfig": """Given a new project root, read in its project dictionary, supply the existing project's profile info, and create a new project file. :param project_root: A filepath to a dbt project. :raises DbtProfileError: If the profile is invalid. :raises DbtProjectError: If project is missing or invalid. :returns: The new configuration. """ # copy profile profile = Profile(**self.to_profile_info()) profile.validate() # load the new project and its packages. Don't pass cli variables. renderer = DbtProjectYamlRenderer(profile) project = Project.from_project_root( project_root, renderer, verify_version=bool(getattr(self.args, "VERSION_CHECK", True)), ) runtime_config = self.from_parts( project=project, profile=profile, args=deepcopy(self.args), ) # force our quoting back onto the new project. runtime_config.quoting = deepcopy(self.quoting) return runtime_config def serialize(self) -> Dict[str, Any]: """Serialize the full configuration to a single dictionary. For any instance that has passed validate() (which happens in __init__), it matches the Configuration contract. Note that args are not serialized. :returns dict: The serialized configuration. """ result = self.to_project_config(with_packages=True) result.update(self.to_profile_info(serialize_credentials=True)) result["cli_vars"] = deepcopy(self.cli_vars) return result def validate(self): """Validate the configuration against its contract. :raises DbtProjectError: If the configuration fails validation. """ try: Configuration.validate(self.serialize()) except ValidationError as e: raise ConfigContractBrokenError(e) from e # Called by RuntimeConfig.from_args @classmethod def collect_parts(cls: Type["RuntimeConfig"], args: Any) -> Tuple[Project, Profile]: # profile_name from the project project_root = args.project_dir if args.project_dir else os.getcwd() cli_vars: Dict[str, Any] = getattr(args, "vars", {}) profile = cls.get_profile( project_root, cli_vars, args, ) flags = get_flags() # For dbt deps, use lenient var validation to allow missing vars # For all other commands, use strict validation for helpful error messages # If command is not set (e.g., during test setup), default to strict mode # unless the command is explicitly "deps" require_vars = getattr(flags, "WHICH", None) != "deps" project = load_project( project_root, bool(flags.VERSION_CHECK), profile, cli_vars, require_vars=require_vars ) return project, profile # Called in task/base.py, in BaseTask.from_args @classmethod def from_args(cls, args: Any) -> "RuntimeConfig": """Given arguments, read in dbt_project.yml from the current directory, read in packages.yml if it exists, and use them to find the profile to load. :param args: The arguments as parsed from the cli. :raises DbtProjectError: If the project is invalid or missing. :raises DbtProfileError: If the profile is invalid or missing. :raises DbtValidationError: If the cli variables are invalid. """ project, profile = cls.collect_parts(args) return cls.from_parts( project=project, profile=profile, args=args, ) def get_metadata(self) -> ManifestMetadata: return ManifestMetadata( project_name=self.project_name, project_id=self.hashed_name(), user_id=tracking.active_user.id if tracking.active_user else None, send_anonymous_usage_stats=( get_flags().SEND_ANONYMOUS_USAGE_STATS if tracking.active_user else None ), adapter_type=self.credentials.type, quoting=Quoting( database=self.quoting.get("database", None), schema=self.quoting.get("schema", None), identifier=self.quoting.get("identifier", None), column=self.quoting.get("column", None), ), run_started_at=( tracking.active_user.run_started_at if tracking.active_user is not None else None ), ) def _get_v2_config_paths( self, config, path: FQNPath, paths: MutableSet[FQNPath], ) -> PathSet: for key, value in config.items(): if isinstance(value, dict) and not key.startswith("+"): self._get_config_paths(value, path + (key,), paths) else: paths.add(path) return frozenset(paths) def _get_config_paths( self, config: Dict[str, Any], path: FQNPath = (), paths: Optional[MutableSet[FQNPath]] = None, ) -> PathSet: if paths is None: paths = set() for key, value in config.items(): if isinstance(value, dict) and not key.startswith("+"): self._get_v2_config_paths(value, path + (key,), paths) else: paths.add(path) return frozenset(paths) def get_resource_config_paths(self) -> Dict[str, PathSet]: """Return a dictionary with resource type keys whose values are lists of lists of strings, where each inner list of strings represents a configured path in the resource. """ return { "models": self._get_config_paths(self.models), "seeds": self._get_config_paths(self.seeds), "snapshots": self._get_config_paths(self.snapshots), "sources": self._get_config_paths(self.sources), "data_tests": self._get_config_paths(self.data_tests), "unit_tests": self._get_config_paths(self.unit_tests), "metrics": self._get_config_paths(self.metrics), "semantic_models": self._get_config_paths(self.semantic_models), "saved_queries": self._get_config_paths(self.saved_queries), "exposures": self._get_config_paths(self.exposures), "functions": self._get_config_paths(self.functions), } def warn_for_unused_resource_config_paths( self, resource_fqns: Mapping[str, PathSet], disabled: PathSet, ) -> None: """Return a list of lists of strings, where each inner list of strings represents a type + FQN path of a resource configuration that is not used. """ disabled_fqns = frozenset(tuple(fqn) for fqn in disabled) resource_config_paths = self.get_resource_config_paths() unused_resource_config_paths = [] for resource_type, config_paths in resource_config_paths.items(): used_fqns = resource_fqns.get(resource_type, frozenset()) fqns = used_fqns | disabled_fqns for config_path in config_paths: if not _is_config_used(config_path, fqns): resource_path = ".".join(i for i in ((resource_type,) + config_path)) unused_resource_config_paths.append(resource_path) if len(unused_resource_config_paths) == 0: return warn_or_error(UnusedResourceConfigPath(unused_config_paths=unused_resource_config_paths)) def load_dependencies(self, base_only=False) -> Mapping[str, "RuntimeConfig"]: if self.dependencies is None: all_projects = {self.project_name: self} internal_packages = get_include_paths(self.credentials.type) if base_only: # Test setup -- we want to load macros without dependencies project_paths = itertools.chain(internal_packages) else: locked_packages = load_package_lock_config(self.project_root) specified_packages = locked_packages.packages if not specified_packages: specified_packages = self.packages.packages specified_package_names = { p.name for p in locked_packages.packages if p.name is not None } installed_package_names = {p.stem for p in self._get_project_directories()} count_packages_specified = len(specified_packages) count_packages_installed = len(installed_package_names) uninstalled_packages = specified_package_names - installed_package_names # we expect same number of packages specified and installed if count_packages_specified != count_packages_installed: raise UninstalledPackagesFoundError( count_packages_installed, count_packages_specified, packages_specified_path=self.packages_specified_path, packages_install_path=self.packages_install_path, uninstalled_packages=tuple(uninstalled_packages), ) project_paths = itertools.chain(internal_packages, self._get_project_directories()) for project_name, project in self.load_projects(project_paths): if project_name in all_projects: raise NonUniquePackageNameError(project_name) all_projects[project_name] = project self.dependencies = all_projects return self.dependencies def clear_dependencies(self): self.dependencies = None # Called by 'load_dependencies' in this class def load_projects(self, paths: Iterable[Path]) -> Iterator[Tuple[str, "RuntimeConfig"]]: for path in paths: try: project = self.new_project(str(path)) except DbtProjectError as e: raise DbtProjectError( f"Failed to read package: {e}", result_type="invalid_project", path=path, ) from e else: yield project.project_name, project def _get_project_directories(self) -> Iterator[Path]: root = Path(self.project_root) / self.packages_install_path if root.exists(): for path in root.iterdir(): if path.is_dir() and not path.name.startswith("__"): yield path class UnsetCredentials(Credentials): def __init__(self) -> None: super().__init__("", "") @property def type(self): return None @property def unique_field(self): return None def connection_info(self, *args, **kwargs): return {} def _connection_keys(self): return () # This is used by commands which do not require # a profile, i.e. dbt deps and clean class UnsetProfile(Profile): def __init__(self): self.credentials = UnsetCredentials() self.profile_name = "" self.target_name = "" self.threads = -1 def to_target_dict(self): return DictDefaultEmptyStr({}) def __getattribute__(self, name): if name in {"profile_name", "target_name", "threads"}: raise DbtRuntimeError(f'Error: disallowed attribute "{name}" - no profile!') return Profile.__getattribute__(self, name) UNUSED_RESOURCE_CONFIGURATION_PATH_MESSAGE = """\ Configuration paths exist in your dbt_project.yml file which do not \ apply to any resources. There are {} unused configuration paths: {} """ def _is_config_used(path, fqns): if fqns: for fqn in fqns: if len(path) <= len(fqn) and fqn[: len(path)] == path: return True return False ================================================ FILE: core/dbt/config/selectors.py ================================================ from copy import deepcopy from pathlib import Path from typing import Any, Dict, Optional, Union from dbt.clients.yaml_helper import Dumper, Loader, load_yaml_text, yaml # noqa: F401 from dbt.contracts.selection import SelectorFile from dbt.exceptions import DbtSelectorsError from dbt.graph import SelectionSpec, parse_from_selectors_definition from dbt.graph.selector_spec import SelectionCriteria from dbt_common.clients.system import ( load_file_contents, path_exists, resolve_path_from_base, ) from dbt_common.dataclass_schema import ValidationError from dbt_common.exceptions import DbtRuntimeError from .renderer import BaseRenderer MALFORMED_SELECTOR_ERROR = """\ The selectors.yml file in this project is malformed. Please double check the contents of this file and fix any errors before retrying. You can find more information on the syntax for this file here: https://docs.getdbt.com/reference/node-selection/yaml-selectors Validator Error: {error} """ class SelectorConfig(Dict[str, Dict[str, Union[SelectionSpec, bool]]]): @classmethod def selectors_from_dict(cls, data: Dict[str, Any]) -> "SelectorConfig": try: SelectorFile.validate(data) selector_file = SelectorFile.from_dict(data) validate_selector_default(selector_file) selectors = parse_from_selectors_definition(selector_file) except ValidationError as exc: yaml_sel_cfg = yaml.dump(exc.instance) raise DbtSelectorsError( f"Could not parse selector file data: \n{yaml_sel_cfg}\n" f"Valid root-level selector definitions: " f"union, intersection, string, dictionary. No lists. " f"\nhttps://docs.getdbt.com/reference/node-selection/" f"yaml-selectors", result_type="invalid_selector", ) from exc except DbtRuntimeError as exc: raise DbtSelectorsError( f"Could not read selector file data: {exc}", result_type="invalid_selector", ) from exc return cls(selectors) @classmethod def render_from_dict( cls, data: Dict[str, Any], renderer: BaseRenderer, ) -> "SelectorConfig": try: rendered = renderer.render_data(data) except (ValidationError, DbtRuntimeError) as exc: raise DbtSelectorsError( f"Could not render selector data: {exc}", result_type="invalid_selector", ) from exc return cls.selectors_from_dict(rendered) @classmethod def from_path( cls, path: Path, renderer: BaseRenderer, ) -> "SelectorConfig": try: data = load_yaml_text(load_file_contents(str(path))) if data is None: raise ValidationError("No data found in selector file at path: {path}") except (ValidationError, DbtRuntimeError) as exc: raise DbtSelectorsError( f"Could not read selector file: {exc}", result_type="invalid_selector", path=path, ) from exc try: return cls.render_from_dict(data, renderer) except DbtSelectorsError as exc: exc.path = path raise def selector_data_from_root(project_root: str) -> Optional[Dict[str, Any]]: selector_filepath = resolve_path_from_base("selectors.yml", project_root) if path_exists(selector_filepath): selectors_dict = load_yaml_text(load_file_contents(selector_filepath)) else: selectors_dict = None return selectors_dict def selector_config_from_data(selectors_data: Dict[str, Any]) -> SelectorConfig: if not selectors_data: selectors_data = {"selectors": []} try: selectors = SelectorConfig.selectors_from_dict(selectors_data) except ValidationError as e: raise DbtSelectorsError( MALFORMED_SELECTOR_ERROR.format(error=str(e.message)), result_type="invalid_selector", ) from e return selectors def validate_selector_default(selector_file: SelectorFile) -> None: """Check if a selector.yml file has more than 1 default key set to true""" default_set: bool = False default_selector_name: Union[str, None] = None for selector in selector_file.selectors: if selector.default is True and default_set is False: default_set = True default_selector_name = selector.name continue if selector.default is True and default_set is True: raise DbtSelectorsError( "Error when parsing the selector file. " "Found multiple selectors with `default: true`:" f"{default_selector_name} and {selector.name}" ) # These are utilities to clean up the dictionary created from # selectors.yml by turning the cli-string format entries into # normalized dictionary entries. It parallels the flow in # dbt/graph/cli.py. If changes are made there, it might # be necessary to make changes here. Ideally it would be # good to combine the two flows into one at some point. class SelectorDict: @classmethod def parse_dict_definition(cls, definition, selector_dict={}): key = list(definition)[0] value = definition[key] if isinstance(value, list): new_values = [] for sel_def in value: new_value = cls.parse_from_definition(sel_def, selector_dict=selector_dict) new_values.append(new_value) value = new_values if key == "exclude": definition = {key: value} elif len(definition) == 1: definition = {"method": key, "value": value} return definition @classmethod def parse_a_definition(cls, def_type, definition, selector_dict={}): # this definition must be a list new_dict = {def_type: []} for sel_def in definition[def_type]: if isinstance(sel_def, dict): sel_def = cls.parse_from_definition(sel_def, selector_dict=selector_dict) new_dict[def_type].append(sel_def) elif isinstance(sel_def, str): sel_def = SelectionCriteria.dict_from_single_spec(sel_def) new_dict[def_type].append(sel_def) else: new_dict[def_type].append(sel_def) return new_dict @classmethod def parse_from_definition(cls, definition, selector_dict={}): if isinstance(definition, str): definition = SelectionCriteria.dict_from_single_spec(definition) elif "union" in definition: definition = cls.parse_a_definition("union", definition, selector_dict=selector_dict) elif "intersection" in definition: definition = cls.parse_a_definition( "intersection", definition, selector_dict=selector_dict ) elif isinstance(definition, dict): definition = cls.parse_dict_definition(definition, selector_dict=selector_dict) return definition # This is the normal entrypoint of this code. Give it the # list of selectors generated from the selectors.yml file. @classmethod def parse_from_selectors_list(cls, selectors): selector_dict = {} for selector in selectors: sel_name = selector["name"] selector_dict[sel_name] = selector definition = cls.parse_from_definition( selector["definition"], selector_dict=deepcopy(selector_dict) ) selector_dict[sel_name]["definition"] = definition return selector_dict ================================================ FILE: core/dbt/config/utils.py ================================================ from typing import Any, Dict, Optional from dbt import deprecations from dbt.clients import yaml_helper from dbt.events.types import InvalidOptionYAML from dbt.exceptions import DbtExclusivePropertyUseError, OptionNotYamlDictError from dbt_common.events.functions import fire_event from dbt_common.exceptions import DbtValidationError def parse_cli_vars(var_string: str) -> Dict[str, Any]: return parse_cli_yaml_string(var_string, "vars") def parse_cli_yaml_string(var_string: str, cli_option_name: str) -> Dict[str, Any]: try: cli_vars = yaml_helper.load_yaml_text(var_string) var_type = type(cli_vars) if cli_vars is not None and var_type is dict: return cli_vars else: raise OptionNotYamlDictError(var_type, cli_option_name) except (DbtValidationError, OptionNotYamlDictError): fire_event(InvalidOptionYAML(option_name=cli_option_name)) raise def exclusive_primary_alt_value_setting( dictionary: Optional[Dict[str, Any]], primary: str, alt: str, parent_config: Optional[str] = None, ) -> None: """Munges in place under the primary the options for the primary and alt values Sometimes we allow setting something via TWO keys, but not at the same time. If both the primary key and alt key have values, an error gets raised. If the alt key has values, then we update the dictionary to ensure the primary key contains the values. If neither are set, nothing happens. """ if dictionary is None: return primary_options = dictionary.get(primary) alt_options = dictionary.get(alt) if primary_options and alt_options: where = f" in `{parent_config}`" if parent_config is not None else "" raise DbtExclusivePropertyUseError( f"Only `{alt}` or `{primary}` can be specified{where}, not both" ) if alt in dictionary: alt_value = dictionary.pop(alt) dictionary[primary] = alt_value def normalize_warn_error_options(warn_error_options: Dict[str, Any]) -> None: has_include = "include" in warn_error_options has_exclude = "exclude" in warn_error_options if has_include or has_exclude: deprecations.buffer( "weo-include-exclude-deprecation", found_include=has_include, found_exclude=has_exclude, ) exclusive_primary_alt_value_setting( warn_error_options, "error", "include", "warn_error_options" ) exclusive_primary_alt_value_setting( warn_error_options, "warn", "exclude", "warn_error_options" ) for key in ("error", "warn", "silence"): if key in warn_error_options and warn_error_options[key] is None: warn_error_options[key] = [] ================================================ FILE: core/dbt/constants.py ================================================ from dbt_semantic_interfaces.type_enums import TimeGranularity DEFAULT_ENV_PLACEHOLDER = "DBT_DEFAULT_PLACEHOLDER" SECRET_PLACEHOLDER = "$$$DBT_SECRET_START$$${}$$$DBT_SECRET_END$$$" MAXIMUM_SEED_SIZE = 1 * 1024 * 1024 MAXIMUM_SEED_SIZE_NAME = "1MB" PIN_PACKAGE_URL = ( "https://docs.getdbt.com/docs/package-management#section-specifying-package-versions" ) DBT_PROJECT_FILE_NAME = "dbt_project.yml" VARS_FILE_NAME = "vars.yml" PACKAGES_FILE_NAME = "packages.yml" DEPENDENCIES_FILE_NAME = "dependencies.yml" PACKAGE_LOCK_FILE_NAME = "package-lock.yml" MANIFEST_FILE_NAME = "manifest.json" SEMANTIC_MANIFEST_FILE_NAME = "semantic_manifest.json" LEGACY_TIME_SPINE_MODEL_NAME = "metricflow_time_spine" LEGACY_TIME_SPINE_GRANULARITY = TimeGranularity.DAY MINIMUM_REQUIRED_TIME_SPINE_GRANULARITY = TimeGranularity.DAY PARTIAL_PARSE_FILE_NAME = "partial_parse.msgpack" PACKAGE_LOCK_HASH_KEY = "sha1_hash" CATALOGS_FILE_NAME = "catalogs.yml" RUN_RESULTS_FILE_NAME = "run_results.json" CATALOG_FILENAME = "catalog.json" SOURCE_RESULT_FILE_NAME = "sources.json" ================================================ FILE: core/dbt/context/README.md ================================================ # Contexts and Jinja rendering Contexts are used for Jinja rendering. They include context methods, executable macros, and various settings that are available in Jinja. Refer to [YAML tips and tricks}(https://docs.getdbt.com/docs/build/dbt-tips#yaml-tips) for more information. The most common entrypoint to Jinja rendering in dbt is a method named `get_rendered`, which takes two arguments: templated code (string), and a context used to render it (dictionary). The context is the bundle of information that is in "scope" when rendering Jinja-templated code. For instance, imagine a simple Jinja template: ``` {% set new_value = some_macro(some_variable) %} ``` Both `some_macro()` and `some_variable` must be defined in that context. Otherwise, it will raise an error when rendering. Different contexts are used in different places because we allow access to different methods and data in different places. Executable SQL, for example, includes all available macros and the model being run. The variables and macros in scope for Jinja defined in yaml files is much more limited. ### Implementation The context that is passed to Jinja is always in a dictionary format, not an actual class, so a `to_dict()` is executed on a context class before it is used for rendering. Each context has a `generate_<name>_context` function to create the context. `ProviderContext` subclasses have different generate functions for parsing and for execution, so that certain functions (notably `ref`, `source`, and `config`) can return different results ### Hierarchy All contexts inherit from the `BaseContext`, which includes "pure" methods (e.g. `tojson`), `env_var()`, and `var()` (but only CLI values, passed via `--vars`). Methods available in parent contexts are also available in child contexts. ``` BaseContext -- core/dbt/context/base.py SecretContext -- core/dbt/context/secret.py TargetContext -- core/dbt/context/target.py ConfiguredContext -- core/dbt/context/configured.py SchemaYamlContext -- core/dbt/context/configured.py DocsRuntimeContext -- core/dbt/context/configured.py MacroResolvingContext -- core/dbt/context/configured.py ManifestContext -- core/dbt/context/manifest.py QueryHeaderContext -- core/dbt/context/manifest.py ProviderContext -- core/dbt/context/provider.py MacroContext -- core/dbt/context/provider.py ModelContext -- core/dbt/context/provider.py TestContext -- core/dbt/context/provider.py ``` ### Contexts for configuration Contexts for rendering "special" `.yml` (configuration) files: - `SecretContext`: Supports "secret" env vars, which are prefixed with `DBT_ENV_SECRET_`. Used for rendering in `profiles.yml` and `packages.yml` ONLY. Secrets defined elsewhere will raise explicit errors. - `TargetContext`: The same as `Base`, plus `target` (connection profile). Used most notably in `dbt_project.yml` and `selectors.yml`. Contexts for other `.yml` files in the project: - `SchemaYamlContext`: Supports `vars` declared on the CLI and in `dbt_project.yml`. Does not support custom macros, beyond `var()` + `env_var()` methods. Used for all `.yml` files, to define properties and configuration. - `DocsRuntimeContext`: Standard `.yml` file context, plus `doc()` method (with all `docs` blocks in scope). Used to resolve `description` properties. ================================================ FILE: core/dbt/context/__init__.py ================================================ ================================================ FILE: core/dbt/context/base.py ================================================ from __future__ import annotations import datetime import itertools import json import os import re import threading from typing import Any, Callable, Dict, Iterable, List, Mapping, NoReturn, Optional, Set # These modules are added to the context. Consider alternative # approaches which will extend well to potentially many modules import pytz import dbt.deprecations as deprecations import dbt.flags as flags_module from dbt import tracking, utils from dbt.clients.jinja import get_rendered from dbt.clients.yaml_helper import ( # noqa: F401 Dumper, Loader, SafeLoader, safe_load, yaml, ) from dbt.constants import DEFAULT_ENV_PLACEHOLDER, SECRET_PLACEHOLDER from dbt.contracts.graph.nodes import Resource from dbt.events.types import JinjaLogDebug, JinjaLogInfo from dbt.exceptions import ( EnvVarMissingError, RequiredVarNotFoundError, SecretEnvVarLocationError, SetStrictWrongTypeError, ZipStrictWrongTypeError, ) from dbt.flags import get_flags from dbt.version import __version__ as dbt_version from dbt_common.constants import SECRET_ENV_PREFIX from dbt_common.context import get_invocation_context from dbt_common.events.contextvars import get_node_info from dbt_common.events.functions import fire_event, get_invocation_id from dbt_common.events.types import PrintEvent from dbt_common.exceptions.macros import MacroReturn # See the `contexts` module README for more information on how contexts work def get_pytz_module_context() -> Dict[str, Any]: context_exports = pytz.__all__ # type: ignore return {name: getattr(pytz, name) for name in context_exports} def get_datetime_module_context() -> Dict[str, Any]: context_exports = ["date", "datetime", "time", "timedelta", "tzinfo"] return {name: getattr(datetime, name) for name in context_exports} def get_re_module_context() -> Dict[str, Any]: # TODO CT-211 context_exports = re.__all__ # type: ignore[attr-defined] return {name: getattr(re, name) for name in context_exports} def get_itertools_module_context() -> Dict[str, Any]: # Excluded dropwhile, filterfalse, takewhile and groupby; # first 3 illogical for Jinja and last redundant. context_exports = [ "count", "cycle", "repeat", "accumulate", "chain", "compress", "islice", "starmap", "tee", "zip_longest", "product", "permutations", "combinations", "combinations_with_replacement", ] def deprecation_wrapper(fn): def deprecation_wrapper_inner(*args, **kwargs): deprecations.warn("modules-itertools-usage-deprecation") return fn(*args, **kwargs) return deprecation_wrapper_inner return {name: deprecation_wrapper(getattr(itertools, name)) for name in context_exports} def get_context_modules() -> Dict[str, Dict[str, Any]]: return { "pytz": get_pytz_module_context(), "datetime": get_datetime_module_context(), "re": get_re_module_context(), "itertools": get_itertools_module_context(), } class ContextMember: def __init__(self, value: Any, name: Optional[str] = None) -> None: self.name = name self.inner = value def key(self, default: str) -> str: if self.name is None: return default return self.name def contextmember(value: Optional[str] = None) -> Callable: return lambda v: ContextMember(v, name=value) def contextproperty(value: Optional[str] = None) -> Callable: return lambda v: ContextMember(property(v), name=value) class ContextMeta(type): def __new__(mcls, name, bases, dct: Dict[str, Any]) -> ContextMeta: context_members: Dict[str, Any] = {} context_attrs: Dict[str, Any] = {} new_dct: Dict[str, Any] = {} for base in bases: context_members.update(getattr(base, "_context_members_", {})) context_attrs.update(getattr(base, "_context_attrs_", {})) for key, value in dct.items(): if isinstance(value, ContextMember): context_key = value.key(key) context_members[context_key] = value.inner context_attrs[context_key] = key value = value.inner new_dct[key] = value new_dct["_context_members_"] = context_members new_dct["_context_attrs_"] = context_attrs return type.__new__(mcls, name, bases, new_dct) class Var: _VAR_NOTSET = object() def __init__( self, context: Mapping[str, Any], cli_vars: Mapping[str, Any], node: Optional[Resource] = None, require_vars: bool = True, ) -> None: self._context: Mapping[str, Any] = context self._cli_vars: Mapping[str, Any] = cli_vars self._node: Optional[Resource] = node self._require_vars: bool = require_vars self._merged: Mapping[str, Any] = self._generate_merged() def _generate_merged(self) -> Mapping[str, Any]: return self._cli_vars @property def node_name(self) -> str: if self._node is not None: return self._node.name else: return "<Configuration>" def get_missing_var(self, var_name: str) -> None: # Only raise an error if vars are _required_ if self._require_vars: # TODO function name implies a non exception resolution raise RequiredVarNotFoundError(var_name, dict(self._merged), self._node) def has_var(self, var_name: str) -> bool: return var_name in self._merged def get_rendered_var(self, var_name: str) -> Any: raw = self._merged[var_name] # if bool/int/float/etc are passed in, don't compile anything if not isinstance(raw, str): return raw return get_rendered(raw, dict(self._context)) def __call__(self, var_name: str, default: Any = _VAR_NOTSET) -> Any: if self.has_var(var_name): return self.get_rendered_var(var_name) elif default is not self._VAR_NOTSET: return default else: return self.get_missing_var(var_name) class BaseContext(metaclass=ContextMeta): # Set by ContextMeta _context_members_: Dict[str, Any] _context_attrs_: Dict[str, Any] # subclass is TargetContext def __init__(self, cli_vars: Dict[str, Any], require_vars: bool = True) -> None: self._ctx: Dict[str, Any] = {} self.cli_vars: Dict[str, Any] = cli_vars self.env_vars: Dict[str, Any] = {} self.require_vars: bool = require_vars def generate_builtins(self) -> Dict[str, Any]: builtins: Dict[str, Any] = {} for key, value in self._context_members_.items(): if hasattr(value, "__get__"): # handle properties, bound methods, etc value = value.__get__(self) builtins[key] = value return builtins # no dbtClassMixin so this is not an actual override def to_dict(self) -> Dict[str, Any]: self._ctx["context"] = self._ctx builtins = self.generate_builtins() self._ctx["builtins"] = builtins self._ctx.update(builtins) return self._ctx @contextproperty() def dbt_version(self) -> str: """The `dbt_version` variable returns the installed version of dbt that is currently running. It can be used for debugging or auditing purposes. > macros/get_version.sql {% macro get_version() %} {% set msg = "The installed version of dbt is: " ~ dbt_version %} {% do log(msg, info=true) %} {% endmacro %} Example output: $ dbt run-operation get_version The installed version of dbt is 0.16.0 """ return dbt_version @contextproperty() def var(self) -> Var: """Variables can be passed from your `dbt_project.yml` file into models during compilation. These variables are useful for configuring packages for deployment in multiple environments, or defining values that should be used across multiple models within a package. To add a variable to a model, use the `var()` function: > my_model.sql: select * from events where event_type = '{{ var("event_type") }}' If you try to run this model without supplying an `event_type` variable, you'll receive a compilation error that looks like this: Encountered an error: ! Compilation error while compiling model package_name.my_model: ! Required var 'event_type' not found in config: Vars supplied to package_name.my_model = { } To supply a variable to a given model, add one or more `vars` dictionaries to the `models` config in your `dbt_project.yml` file. These `vars` are in-scope for all models at or below where they are defined, so place them where they make the most sense. Below are three different placements of the `vars` dict, all of which will make the `my_model` model compile. > dbt_project.yml: # 1) scoped at the model level models: package_name: my_model: materialized: view vars: event_type: activation # 2) scoped at the package level models: package_name: vars: event_type: activation my_model: materialized: view # 3) scoped globally models: vars: event_type: activation package_name: my_model: materialized: view ## Variable default values The `var()` function takes an optional second argument, `default`. If this argument is provided, then it will be the default value for the variable if one is not explicitly defined. > my_model.sql: -- Use 'activation' as the event_type if the variable is not -- defined. select * from events where event_type = '{{ var("event_type", "activation") }}' """ return Var(self._ctx, self.cli_vars, require_vars=self.require_vars) @contextmember() def env_var(self, var: str, default: Optional[str] = None) -> str: """The env_var() function. Return the environment variable named 'var'. If there is no such environment variable set, return the default. If the default is None, raise an exception for an undefined variable. """ return_value = None if var.startswith(SECRET_ENV_PREFIX): raise SecretEnvVarLocationError(var) env = get_invocation_context().env if var in env: return_value = env[var] elif default is not None: return_value = default if return_value is not None: # If the environment variable is set from a default, store a string indicating # that so we can skip partial parsing. Otherwise the file will be scheduled for # reparsing. If the default changes, the file will have been updated and therefore # will be scheduled for reparsing anyways. self.env_vars[var] = return_value if var in env else DEFAULT_ENV_PLACEHOLDER return return_value else: raise EnvVarMissingError(var) if os.environ.get("DBT_MACRO_DEBUGGING"): @contextmember() @staticmethod def debug(): """Enter a debugger at this line in the compiled jinja code.""" import sys import ipdb # type: ignore frame = sys._getframe(3) ipdb.set_trace(frame) return "" @contextmember("return") @staticmethod def _return(data: Any) -> NoReturn: """The `return` function can be used in macros to return data to the caller. The type of the data (`dict`, `list`, `int`, etc) will be preserved through the return call. :param data: The data to return to the caller > macros/example.sql: {% macro get_data() %} {{ return([1,2,3]) }} {% endmacro %} > models/my_model.sql: select -- getdata() returns a list! {% for i in getdata() %} {{ i }} {% if not loop.last %},{% endif %} {% endfor %} """ raise MacroReturn(data) @contextmember() @staticmethod def fromjson(string: str, default: Any = None) -> Any: """The `fromjson` context method can be used to deserialize a json string into a Python object primitive, eg. a `dict` or `list`. :param value: The json string to deserialize :param default: A default value to return if the `string` argument cannot be deserialized (optional) Usage: {% set my_json_str = '{"abc": 123}' %} {% set my_dict = fromjson(my_json_str) %} {% do log(my_dict['abc']) %} """ try: return json.loads(string) except ValueError: return default @contextmember() @staticmethod def tojson(value: Any, default: Any = None, sort_keys: bool = False) -> Any: """The `tojson` context method can be used to serialize a Python object primitive, eg. a `dict` or `list` to a json string. :param value: The value serialize to json :param default: A default value to return if the `value` argument cannot be serialized :param sort_keys: If True, sort the keys. Usage: {% set my_dict = {"abc": 123} %} {% set my_json_string = tojson(my_dict) %} {% do log(my_json_string) %} """ try: return json.dumps(value, sort_keys=sort_keys) except ValueError: return default @contextmember() @staticmethod def fromyaml(value: str, default: Any = None) -> Any: """The fromyaml context method can be used to deserialize a yaml string into a Python object primitive, eg. a `dict` or `list`. :param value: The yaml string to deserialize :param default: A default value to return if the `string` argument cannot be deserialized (optional) Usage: {% set my_yml_str -%} dogs: - good - bad {%- endset %} {% set my_dict = fromyaml(my_yml_str) %} {% do log(my_dict['dogs'], info=true) %} -- ["good", "bad"] {% do my_dict['dogs'].pop() } {% do log(my_dict['dogs'], info=true) %} -- ["good"] """ try: return safe_load(value) except (AttributeError, ValueError, yaml.YAMLError): return default # safe_dump defaults to sort_keys=True, but we act like json.dumps (the # opposite) @contextmember() @staticmethod def toyaml( value: Any, default: Optional[str] = None, sort_keys: bool = False ) -> Optional[str]: """The `tojson` context method can be used to serialize a Python object primitive, eg. a `dict` or `list` to a yaml string. :param value: The value serialize to yaml :param default: A default value to return if the `value` argument cannot be serialized :param sort_keys: If True, sort the keys. Usage: {% set my_dict = {"abc": 123} %} {% set my_yaml_string = toyaml(my_dict) %} {% do log(my_yaml_string) %} """ try: return yaml.safe_dump(data=value, sort_keys=sort_keys) except (ValueError, yaml.YAMLError): return default @contextmember("set") @staticmethod def _set(value: Iterable[Any], default: Any = None) -> Optional[Set[Any]]: """The `set` context method can be used to convert any iterable to a sequence of iterable elements that are unique (a set). :param value: The iterable :param default: A default value to return if the `value` argument is not an iterable Usage: {% set my_list = [1, 2, 2, 3] %} {% set my_set = set(my_list) %} {% do log(my_set) %} {# {1, 2, 3} #} """ try: return set(value) except TypeError: return default @contextmember() @staticmethod def set_strict(value: Iterable[Any]) -> Set[Any]: """The `set_strict` context method can be used to convert any iterable to a sequence of iterable elements that are unique (a set). The difference to the `set` context method is that the `set_strict` method will raise an exception on a TypeError. :param value: The iterable Usage: {% set my_list = [1, 2, 2, 3] %} {% set my_set = set_strict(my_list) %} {% do log(my_set) %} {# {1, 2, 3} #} """ try: return set(value) except TypeError as e: raise SetStrictWrongTypeError(e) @contextmember("zip") @staticmethod def _zip(*args: Iterable[Any], default: Any = None) -> Optional[Iterable[Any]]: """The `zip` context method can be used to used to return an iterator of tuples, where the i-th tuple contains the i-th element from each of the argument iterables. :param *args: Any number of iterables :param default: A default value to return if `*args` is not iterable Usage: {% set my_list_a = [1, 2] %} {% set my_list_b = ['alice', 'bob'] %} {% set my_zip = zip(my_list_a, my_list_b) | list %} {% do log(my_set) %} {# [(1, 'alice'), (2, 'bob')] #} """ try: return zip(*args) except TypeError: return default @contextmember() @staticmethod def zip_strict(*args: Iterable[Any]) -> Iterable[Any]: """The `zip_strict` context method can be used to used to return an iterator of tuples, where the i-th tuple contains the i-th element from each of the argument iterables. The difference to the `zip` context method is that the `zip_strict` method will raise an exception on a TypeError. :param *args: Any number of iterables Usage: {% set my_list_a = [1, 2] %} {% set my_list_b = ['alice', 'bob'] %} {% set my_zip = zip_strict(my_list_a, my_list_b) | list %} {% do log(my_set) %} {# [(1, 'alice'), (2, 'bob')] #} """ try: return zip(*args) except TypeError as e: raise ZipStrictWrongTypeError(e) @contextmember() @staticmethod def log(msg: str, info: bool = False) -> str: """Logs a line to either the log file or stdout. :param msg: The message to log :param info: If `False`, write to the log file. If `True`, write to both the log file and stdout. > macros/my_log_macro.sql {% macro some_macro(arg1, arg2) %} {{ log("Running some_macro: " ~ arg1 ~ ", " ~ arg2) }} {% endmacro %}" """ # Detect instances of the placeholder value ($$$DBT_SECRET_START...DBT_SECRET_END$$$) # and replace it with the standard mask '*****' if "DBT_SECRET_START" in str(msg): search_group = f"({SECRET_ENV_PREFIX}(.*))" pattern = SECRET_PLACEHOLDER.format(search_group).replace("$", r"\$") m = re.search( pattern, msg, ) if m: msg = re.sub(pattern, "*****", msg) if info: fire_event(JinjaLogInfo(msg=msg, node_info=get_node_info())) else: fire_event(JinjaLogDebug(msg=msg, node_info=get_node_info())) return "" @contextproperty() def run_started_at(self) -> Optional[datetime.datetime]: """`run_started_at` outputs the timestamp that this run started, e.g. `2017-04-21 01:23:45.678`. The `run_started_at` variable is a Python `datetime` object. As of 0.9.1, the timezone of this variable defaults to UTC. > run_started_at_example.sql select '{{ run_started_at.strftime("%Y-%m-%d") }}' as date_day from ... To modify the timezone of this variable, use the the `pytz` module: > run_started_at_utc.sql {% set est = modules.pytz.timezone("America/New_York") %} select '{{ run_started_at.astimezone(est) }}' as run_started_est from ... """ if tracking.active_user is not None: return tracking.active_user.run_started_at else: return None @contextproperty() def invocation_id(self) -> Optional[str]: """invocation_id outputs a UUID generated for this dbt run (useful for auditing) """ return get_invocation_id() @contextproperty() def thread_id(self) -> str: """thread_id outputs an ID for the current thread (useful for auditing)""" return threading.current_thread().name @contextproperty() def modules(self) -> Dict[str, Any]: """The `modules` variable in the Jinja context contains useful Python modules for operating on data. # datetime This variable is a pointer to the Python datetime module. Usage: {% set dt = modules.datetime.datetime.now() %} # pytz This variable is a pointer to the Python pytz module. Usage: {% set dt = modules.datetime.datetime(2002, 10, 27, 6, 0, 0) %} {% set dt_local = modules.pytz.timezone('US/Eastern').localize(dt) %} {{ dt_local }} """ # noqa return get_context_modules() @contextproperty() def flags(self) -> Any: """The `flags` variable contains true/false values for flags provided on the command line. > flags.sql: {% if flags.FULL_REFRESH %} drop table ... {% else %} -- no-op {% endif %} This supports all flags defined in flags submodule (core/dbt/flags.py) """ return flags_module.get_flag_obj() @contextmember() @staticmethod def print(msg: str) -> str: """Prints a line to stdout. :param msg: The message to print > macros/my_log_macro.sql {% macro some_macro(arg1, arg2) %} {{ print("Running some_macro: " ~ arg1 ~ ", " ~ arg2) }} {% endmacro %}" """ if get_flags().PRINT: # No formatting, still get to stdout when --quiet is used fire_event(PrintEvent(msg=msg)) return "" @contextmember() @staticmethod def diff_of_two_dicts( dict_a: Dict[str, List[str]], dict_b: Dict[str, List[str]] ) -> Dict[str, List[str]]: """ Given two dictionaries of type Dict[str, List[str]]: dict_a = {'key_x': ['value_1', 'VALUE_2'], 'KEY_Y': ['value_3']} dict_b = {'key_x': ['value_1'], 'key_z': ['value_4']} Return the same dictionary representation of dict_a MINUS dict_b, performing a case-insensitive comparison between the strings in each. All keys returned will be in the original case of dict_a. returns {'key_x': ['VALUE_2'], 'KEY_Y': ['value_3']} """ dict_diff = {} dict_b_lowered = {k.casefold(): [x.casefold() for x in v] for k, v in dict_b.items()} for k in dict_a: if k.casefold() in dict_b_lowered.keys(): diff = [] for v in dict_a[k]: if v.casefold() not in dict_b_lowered[k.casefold()]: diff.append(v) if diff: dict_diff.update({k: diff}) else: dict_diff.update({k: dict_a[k]}) return dict_diff @contextmember() @staticmethod def local_md5(value: str) -> str: """Calculates an MD5 hash of the given string. It's called "local_md5" to emphasize that it runs locally in dbt (in jinja context) and not an MD5 SQL command. :param value: The value to hash Usage: {% set value_hash = local_md5("hello world") %} """ return utils.md5(value) def generate_base_context(cli_vars: Dict[str, Any]) -> Dict[str, Any]: ctx = BaseContext(cli_vars) # This is not a Mashumaro to_dict call return ctx.to_dict() ================================================ FILE: core/dbt/context/configured.py ================================================ from typing import Any, Dict, Optional from dbt.adapters.contracts.connection import AdapterRequiredConfig from dbt.constants import DEFAULT_ENV_PLACEHOLDER from dbt.context.base import Var, contextmember, contextproperty from dbt.context.target import TargetContext from dbt.exceptions import EnvVarMissingError, SecretEnvVarLocationError from dbt.node_types import NodeType from dbt.utils import MultiDict from dbt_common.constants import SECRET_ENV_PREFIX from dbt_common.context import get_invocation_context class ConfiguredContext(TargetContext): # subclasses are SchemaYamlContext, MacroResolvingContext, ManifestContext config: AdapterRequiredConfig def __init__(self, config: AdapterRequiredConfig, require_vars: bool = True) -> None: super().__init__(config.to_target_dict(), config.cli_vars, require_vars=require_vars) self.config = config @contextproperty() def project_name(self) -> str: return self.config.project_name class FQNLookup: def __init__(self, package_name: str): self.package_name = package_name self.fqn = [package_name] self.resource_type = NodeType.Model class ConfiguredVar(Var): def __init__( self, context: Dict[str, Any], config: AdapterRequiredConfig, project_name: str, ): super().__init__(context, config.cli_vars) self._config = config self._project_name = project_name def __call__(self, var_name, default=Var._VAR_NOTSET): my_config = self._config.load_dependencies()[self._project_name] # cli vars > active project > local project if var_name in self._config.cli_vars: return self._config.cli_vars[var_name] adapter_type = self._config.credentials.type lookup = FQNLookup(self._project_name) active_vars = self._config.vars.vars_for(lookup, adapter_type) all_vars = MultiDict() if self._config.project_name != my_config.project_name: all_vars.add(my_config.vars.vars_for(lookup, adapter_type)) all_vars.add(active_vars) if var_name in all_vars: return all_vars[var_name] if default is not Var._VAR_NOTSET: return default return self.get_missing_var(var_name) class SchemaYamlVars: def __init__(self): self.env_vars = {} self.vars = {} class SchemaYamlContext(ConfiguredContext): # subclass is DocsRuntimeContext def __init__(self, config, project_name: str, schema_yaml_vars: Optional[SchemaYamlVars]): super().__init__(config) self._project_name = project_name self.schema_yaml_vars = schema_yaml_vars @contextproperty() def var(self) -> ConfiguredVar: return ConfiguredVar(self._ctx, self.config, self._project_name) @contextmember() def env_var(self, var: str, default: Optional[str] = None) -> str: return_value = None if var.startswith(SECRET_ENV_PREFIX): raise SecretEnvVarLocationError(var) env = get_invocation_context().env if var in env: return_value = env[var] elif default is not None: return_value = default if return_value is not None: if self.schema_yaml_vars: # If the environment variable is set from a default, store a string indicating # that so we can skip partial parsing. Otherwise the file will be scheduled for # reparsing. If the default changes, the file will have been updated and therefore # will be scheduled for reparsing anyways. self.schema_yaml_vars.env_vars[var] = ( return_value if var in env else DEFAULT_ENV_PLACEHOLDER ) return return_value else: raise EnvVarMissingError(var) class MacroResolvingContext(ConfiguredContext): def __init__(self, config): super().__init__(config) @contextproperty() def var(self) -> ConfiguredVar: return ConfiguredVar(self._ctx, self.config, self.config.project_name) def generate_schema_yml_context( config: AdapterRequiredConfig, project_name: str, schema_yaml_vars: Optional[SchemaYamlVars] = None, ) -> Dict[str, Any]: ctx = SchemaYamlContext(config, project_name, schema_yaml_vars) return ctx.to_dict() def generate_macro_context( config: AdapterRequiredConfig, ) -> Dict[str, Any]: ctx = MacroResolvingContext(config) return ctx.to_dict() ================================================ FILE: core/dbt/context/context_config.py ================================================ from abc import abstractmethod from copy import deepcopy from dataclasses import dataclass from typing import Any, Dict, Generic, Iterator, List, Optional, TypeVar from dbt.adapters.factory import get_config_class_by_name from dbt.config import IsFQNResource, Project, RuntimeConfig from dbt.contracts.graph.model_config import get_config_for from dbt.exceptions import SchemaConfigError from dbt.flags import get_flags from dbt.node_types import NodeType from dbt.utils import fqn_search from dbt_common.contracts.config.base import BaseConfig, merge_config_dicts from dbt_common.dataclass_schema import ValidationError from dbt_common.exceptions import DbtInternalError @dataclass class ModelParts(IsFQNResource): fqn: List[str] resource_type: NodeType package_name: str T = TypeVar("T") # any old type C = TypeVar("C", bound=BaseConfig) class ConfigSource: def __init__(self, project): self.project = project def get_config_dict(self, resource_type: NodeType): ... class UnrenderedConfig(ConfigSource): def __init__(self, project: Project): self.project = project def get_config_dict(self, resource_type: NodeType) -> Dict[str, Any]: unrendered = self.project.unrendered.project_dict if resource_type == NodeType.Seed: model_configs = unrendered.get("seeds") elif resource_type == NodeType.Snapshot: model_configs = unrendered.get("snapshots") elif resource_type == NodeType.Source: model_configs = unrendered.get("sources") elif resource_type == NodeType.Test: model_configs = unrendered.get("data_tests") elif resource_type == NodeType.Metric: model_configs = unrendered.get("metrics") elif resource_type == NodeType.SemanticModel: model_configs = unrendered.get("semantic_models") elif resource_type == NodeType.SavedQuery: model_configs = unrendered.get("saved_queries") elif resource_type == NodeType.Exposure: model_configs = unrendered.get("exposures") elif resource_type == NodeType.Unit: model_configs = unrendered.get("unit_tests") else: model_configs = unrendered.get("models") if model_configs is None: return {} else: return model_configs class RenderedConfig(ConfigSource): def __init__(self, project: Project): self.project = project def get_config_dict(self, resource_type: NodeType) -> Dict[str, Any]: if resource_type == NodeType.Seed: model_configs = self.project.seeds elif resource_type == NodeType.Snapshot: model_configs = self.project.snapshots elif resource_type == NodeType.Source: model_configs = self.project.sources elif resource_type == NodeType.Test: model_configs = self.project.data_tests elif resource_type == NodeType.Metric: model_configs = self.project.metrics elif resource_type == NodeType.SemanticModel: model_configs = self.project.semantic_models elif resource_type == NodeType.SavedQuery: model_configs = self.project.saved_queries elif resource_type == NodeType.Exposure: model_configs = self.project.exposures elif resource_type == NodeType.Unit: model_configs = self.project.unit_tests elif resource_type == NodeType.Function: model_configs = self.project.functions else: model_configs = self.project.models return model_configs class BaseContextConfigGenerator(Generic[T]): def __init__(self, active_project: RuntimeConfig): self._active_project = active_project def get_config_source(self, project: Project) -> ConfigSource: return RenderedConfig(project) def get_node_project(self, project_name: str): if project_name == self._active_project.project_name: return self._active_project dependencies = self._active_project.load_dependencies() if project_name not in dependencies: raise DbtInternalError( f"Project name {project_name} not found in dependencies " f"(found {list(dependencies)})" ) return dependencies[project_name] def _project_configs( self, project: Project, fqn: List[str], resource_type: NodeType ) -> Iterator[Dict[str, Any]]: src = self.get_config_source(project) model_configs = src.get_config_dict(resource_type) for level_config in fqn_search(model_configs, fqn): result = {} for key, value in level_config.items(): if key.startswith("+"): result[key[1:].strip()] = deepcopy(value) elif not isinstance(value, dict): result[key] = deepcopy(value) yield result def _active_project_configs( self, fqn: List[str], resource_type: NodeType ) -> Iterator[Dict[str, Any]]: return self._project_configs(self._active_project, fqn, resource_type) @abstractmethod def _update_from_config( self, result: T, partial: Dict[str, Any], validate: bool = False ) -> T: ... @abstractmethod def initial_result(self, resource_type: NodeType, base: bool) -> T: ... def calculate_node_config( self, # this is the config from the sql file config_call_dict: Dict[str, Any], fqn: List[str], resource_type: NodeType, project_name: str, base: bool, # this is the config from the schema file patch_config_dict: Optional[Dict[str, Any]] = None, ) -> BaseConfig: own_config = self.get_node_project(project_name) result = self.initial_result(resource_type=resource_type, base=base) # builds the config from what was specified in the runtime_config, which generally # comes from the project's dbt_project.yml file. project_configs = self._project_configs(own_config, fqn, resource_type) for fqn_config in project_configs: result = self._update_from_config(result, fqn_config) # When schema files patch config, it has lower precedence than # config in the models (config_call_dict), so we add the patch_config_dict # before the config_call_dict if patch_config_dict: result = self._update_from_config(result, patch_config_dict) # config_calls are created in the 'experimental' model parser and # the ParseConfigObject (via add_config_call) result = self._update_from_config(result, config_call_dict) if own_config.project_name != self._active_project.project_name: for fqn_config in self._active_project_configs(fqn, resource_type): result = self._update_from_config(result, fqn_config) # this is mostly impactful in the snapshot config case # TODO CT-211 return result # type: ignore[return-value] @abstractmethod def calculate_node_config_dict( self, config_call_dict: Dict[str, Any], fqn: List[str], resource_type: NodeType, project_name: str, base: bool, patch_config_dict: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: ... class ContextConfigGenerator(BaseContextConfigGenerator[C]): def __init__(self, active_project: RuntimeConfig): self._active_project = active_project def get_config_source(self, project: Project) -> ConfigSource: return RenderedConfig(project) def initial_result(self, resource_type: NodeType, base: bool) -> C: # defaults, own_config, config calls, active_config (if != own_config) config_cls = get_config_for(resource_type, base=base) # Calculate the defaults. We don't want to validate the defaults, # because it might be invalid in the case of required config members # (such as on snapshots!) result = config_cls.from_dict({}) return result def _update_from_config(self, result: C, partial: Dict[str, Any], validate: bool = False) -> C: translated = self._active_project.credentials.translate_aliases(partial) translated = self.translate_hook_names(translated) adapter_type = self._active_project.credentials.type adapter_config_cls = get_config_class_by_name(adapter_type) updated = result.update_from(translated, adapter_config_cls, validate=validate) return updated def translate_hook_names(self, project_dict): # This is a kind of kludge because the fix for #6411 specifically allowed misspelling # the hook field names in dbt_project.yml, which only ever worked because we didn't # run validate on the dbt_project configs. if "pre_hook" in project_dict: project_dict["pre-hook"] = project_dict.pop("pre_hook") if "post_hook" in project_dict: project_dict["post-hook"] = project_dict.pop("post_hook") return project_dict def calculate_node_config_dict( self, config_call_dict: Dict[str, Any], fqn: List[str], resource_type: NodeType, project_name: str, base: bool, patch_config_dict: Optional[dict] = None, ) -> Dict[str, Any]: config = self.calculate_node_config( config_call_dict=config_call_dict, fqn=fqn, resource_type=resource_type, project_name=project_name, base=base, patch_config_dict=patch_config_dict, ) try: finalized = config.finalize_and_validate() return finalized.to_dict(omit_none=True) except ValidationError as exc: # we got a ValidationError - probably bad types in config() raise SchemaConfigError(exc, node=config) from exc class UnrenderedConfigGenerator(BaseContextConfigGenerator[Dict[str, Any]]): def get_config_source(self, project: Project) -> ConfigSource: return UnrenderedConfig(project) def calculate_node_config_dict( self, config_call_dict: Dict[str, Any], fqn: List[str], resource_type: NodeType, project_name: str, base: bool, patch_config_dict: Optional[dict] = None, ) -> Dict[str, Any]: # TODO CT-211 return self.calculate_node_config( config_call_dict=config_call_dict, fqn=fqn, resource_type=resource_type, project_name=project_name, base=base, patch_config_dict=patch_config_dict, ) # type: ignore[return-value] def initial_result(self, resource_type: NodeType, base: bool) -> Dict[str, Any]: return {} def _update_from_config( self, result: Dict[str, Any], partial: Dict[str, Any], validate: bool = False, ) -> Dict[str, Any]: translated = self._active_project.credentials.translate_aliases(partial) result.update(translated) return result class ContextConfig: def __init__( self, active_project: RuntimeConfig, fqn: List[str], resource_type: NodeType, project_name: str, ) -> None: self._config_call_dict: Dict[str, Any] = {} self._unrendered_config_call_dict: Dict[str, Any] = {} self._active_project = active_project self._fqn = fqn self._resource_type = resource_type self._project_name = project_name def add_config_call(self, opts: Dict[str, Any]) -> None: dct = self._config_call_dict merge_config_dicts(dct, opts) def add_unrendered_config_call(self, opts: Dict[str, Any]) -> None: # Cannot perform complex merge behaviours on unrendered configs as they may not be appropriate types. self._unrendered_config_call_dict.update(opts) def build_config_dict( self, base: bool = False, *, rendered: bool = True, patch_config_dict: Optional[dict] = None, ) -> Dict[str, Any]: if rendered: # TODO CT-211 src = ContextConfigGenerator(self._active_project) # type: ignore[var-annotated] config_call_dict = self._config_call_dict else: # TODO CT-211 src = UnrenderedConfigGenerator(self._active_project) # type: ignore[assignment] # preserve legacy behaviour - using unreliable (potentially rendered) _config_call_dict if get_flags().state_modified_compare_more_unrendered_values is False: config_call_dict = self._config_call_dict else: # Prefer _config_call_dict if it is available and _unrendered_config_call_dict is not, # as _unrendered_config_call_dict is unreliable for non-sql nodes (e.g. no jinja config block rendered for python models, etc) if self._config_call_dict and not self._unrendered_config_call_dict: config_call_dict = self._config_call_dict else: config_call_dict = self._unrendered_config_call_dict return src.calculate_node_config_dict( config_call_dict=config_call_dict, fqn=self._fqn, resource_type=self._resource_type, project_name=self._project_name, base=base, patch_config_dict=patch_config_dict, ) ================================================ FILE: core/dbt/context/docs.py ================================================ from typing import Any, Dict, Union from dbt.config.runtime import RuntimeConfig from dbt.context.base import contextmember from dbt.context.configured import SchemaYamlContext from dbt.contracts.graph.manifest import Manifest from dbt.contracts.graph.nodes import Macro, ResultNode from dbt.exceptions import DocArgsError, DocTargetNotFoundError class DocsRuntimeContext(SchemaYamlContext): def __init__( self, config: RuntimeConfig, node: Union[Macro, ResultNode], manifest: Manifest, current_project: str, ) -> None: super().__init__(config, current_project, None) self.node = node self.manifest = manifest @contextmember() def doc(self, *args: str) -> str: """The `doc` function is used to reference docs blocks in schema.yml files. It is analogous to the `ref` function. For more information, consult the Documentation guide. > orders.md: {% docs orders %} # docs - go - here {% enddocs %} > schema.yml version: 2 models: - name: orders description: "{{ doc('orders') }}" """ # when you call doc(), this is what happens at runtime if len(args) == 1: doc_package_name = None doc_name = args[0] elif len(args) == 2: doc_package_name, doc_name = args else: raise DocArgsError(self.node, args) # Documentation target_doc = self.manifest.resolve_doc( doc_name, doc_package_name, self._project_name, self.node.package_name, ) if target_doc: file_id = target_doc.file_id if file_id in self.manifest.files: source_file = self.manifest.files[file_id] # TODO CT-211 source_file.add_node(self.node.unique_id) # type: ignore[union-attr] else: raise DocTargetNotFoundError( node=self.node, target_doc_name=doc_name, target_doc_package=doc_package_name ) return target_doc.block_contents def generate_runtime_docs_context( config: RuntimeConfig, target: Any, manifest: Manifest, current_project: str, ) -> Dict[str, Any]: ctx = DocsRuntimeContext(config, target, manifest, current_project) # This is not a Mashumaro to_dict call return ctx.to_dict() ================================================ FILE: core/dbt/context/exceptions_jinja.py ================================================ import functools from typing import NoReturn from dbt.adapters.exceptions import ( ColumnTypeMissingError, MissingConfigError, MissingMaterializationError, RelationWrongTypeError, ) from dbt.adapters.exceptions.cache import CacheInconsistencyError from dbt.events.types import JinjaLogWarning, SnapshotTimestampWarning from dbt.exceptions import ( AmbiguousAliasError, AmbiguousCatalogMatchError, CompilationError, ContractError, DependencyError, DependencyNotFoundError, DuplicatePatchPathError, DuplicateResourceNameError, FailFastError, MissingRelationError, PropertyYMLError, env_secrets, scrub_secrets, ) from dbt_common.events.functions import warn_or_error from dbt_common.exceptions import ( DataclassNotDictError, DbtDatabaseError, DbtRuntimeError, NotImplementedError, ) def warn(msg, node=None): warn_or_error(JinjaLogWarning(msg=msg), node=node) return "" def missing_config(model, name) -> NoReturn: raise MissingConfigError(unique_id=model.unique_id, name=name) def missing_materialization(model, adapter_type) -> NoReturn: raise MissingMaterializationError( materialization=model.config.materialized, adapter_type=adapter_type ) def missing_relation(relation, model=None) -> NoReturn: raise MissingRelationError(relation, model) def raise_ambiguous_alias(node_1, node_2, duped_name=None) -> NoReturn: raise AmbiguousAliasError(node_1, node_2, duped_name) def raise_ambiguous_catalog_match(unique_id, match_1, match_2) -> NoReturn: raise AmbiguousCatalogMatchError(unique_id, match_1, match_2) def raise_cache_inconsistent(message) -> NoReturn: raise CacheInconsistencyError(message) def raise_dataclass_not_dict(obj) -> NoReturn: raise DataclassNotDictError(obj) def raise_compiler_error(msg, node=None) -> NoReturn: raise CompilationError(msg, node) def raise_contract_error(yaml_columns, sql_columns) -> NoReturn: raise ContractError(yaml_columns, sql_columns) def raise_database_error(msg, node=None) -> NoReturn: raise DbtDatabaseError(msg, node) def raise_dep_not_found(node, node_description, required_pkg) -> NoReturn: raise DependencyNotFoundError(node, node_description, required_pkg) def raise_dependency_error(msg) -> NoReturn: raise DependencyError(scrub_secrets(msg, env_secrets())) def raise_duplicate_patch_name(patch_1, existing_patch_path) -> NoReturn: raise DuplicatePatchPathError(patch_1, existing_patch_path) def raise_duplicate_resource_name(node_1, node_2) -> NoReturn: raise DuplicateResourceNameError(node_1, node_2) def raise_invalid_property_yml_version(path, issue) -> NoReturn: raise PropertyYMLError(path, issue) def raise_not_implemented(msg) -> NoReturn: raise NotImplementedError(msg) def relation_wrong_type(relation, expected_type, model=None) -> NoReturn: raise RelationWrongTypeError(relation, expected_type, model) def column_type_missing(column_names) -> NoReturn: raise ColumnTypeMissingError(column_names) def raise_fail_fast_error(msg, node=None) -> NoReturn: raise FailFastError(msg, node=node) def warn_snapshot_timestamp_data_types( snapshot_time_data_type: str, updated_at_data_type: str ) -> None: warn_or_error( SnapshotTimestampWarning( snapshot_time_data_type=snapshot_time_data_type, updated_at_data_type=updated_at_data_type, ) ) # Update this when a new function should be added to the # dbt context's `exceptions` key! CONTEXT_EXPORTS = { fn.__name__: fn for fn in [ warn, missing_config, missing_materialization, missing_relation, raise_ambiguous_alias, raise_ambiguous_catalog_match, raise_cache_inconsistent, raise_dataclass_not_dict, raise_compiler_error, raise_database_error, raise_dep_not_found, raise_dependency_error, raise_duplicate_patch_name, raise_duplicate_resource_name, raise_invalid_property_yml_version, raise_not_implemented, relation_wrong_type, raise_contract_error, column_type_missing, raise_fail_fast_error, warn_snapshot_timestamp_data_types, ] } # wraps context based exceptions in node info def wrapper(model): def wrap(func): @functools.wraps(func) def inner(*args, **kwargs): try: return func(*args, **kwargs) except DbtRuntimeError as exc: exc.add_node(model) raise exc return inner return wrap def wrapped_exports(model): wrap = wrapper(model) return {name: wrap(export) for name, export in CONTEXT_EXPORTS.items()} ================================================ FILE: core/dbt/context/macro_resolver.py ================================================ from typing import Dict, MutableMapping, Optional from dbt.clients.jinja import MacroGenerator from dbt.contracts.graph.nodes import Macro from dbt.exceptions import DuplicateMacroNameError, PackageNotFoundForMacroError from dbt.include.global_project import PROJECT_NAME as GLOBAL_PROJECT_NAME MacroNamespace = Dict[str, Macro] # This class builds the MacroResolver by adding macros # to various categories for finding macros in the right order, # so that higher precedence macros are found first. # This functionality is also provided by the MacroNamespace, # but the intention is to eventually replace that class. # This enables us to get the macro unique_id without # processing every macro in the project. # Note: the root project macros override everything in the # dbt internal projects. External projects (dependencies) will # use their own macros first, then pull from the root project # followed by dbt internal projects. class MacroResolver: def __init__( self, macros: MutableMapping[str, Macro], root_project_name: str, internal_package_names, ) -> None: self.root_project_name = root_project_name self.macros = macros # internal packages comes from get_adapter_package_names self.internal_package_names = internal_package_names # To be filled in from macros. self.internal_packages: Dict[str, MacroNamespace] = {} self.packages: Dict[str, MacroNamespace] = {} self.root_package_macros: MacroNamespace = {} # add the macros to internal_packages, packages, and root packages self.add_macros() self._build_internal_packages_namespace() self._build_macros_by_name() def _build_internal_packages_namespace(self) -> None: # Iterate in reverse-order and overwrite: the packages that are first # in the list are the ones we want to "win". self.internal_packages_namespace: MacroNamespace = {} for pkg in reversed(self.internal_package_names): if pkg in self.internal_packages: # Turn the internal packages into a flat namespace self.internal_packages_namespace.update(self.internal_packages[pkg]) # search order: # local_namespace (package of particular node), not including # the internal packages or the root package # This means that within an extra package, it uses its own macros # root package namespace # non-internal packages (that aren't local or root) # dbt internal packages def _build_macros_by_name(self) -> None: macros_by_name = {} # all internal packages (already in the right order) for macro in self.internal_packages_namespace.values(): macros_by_name[macro.name] = macro # non-internal packages for fnamespace in self.packages.values(): for macro in fnamespace.values(): macros_by_name[macro.name] = macro # root package macros for macro in self.root_package_macros.values(): macros_by_name[macro.name] = macro self.macros_by_name = macros_by_name def _add_macro_to( self, package_namespaces: Dict[str, MacroNamespace], macro: Macro, ) -> None: if macro.package_name in package_namespaces: namespace = package_namespaces[macro.package_name] else: namespace = {} package_namespaces[macro.package_name] = namespace if macro.name in namespace: raise DuplicateMacroNameError(macro, macro, macro.package_name) package_namespaces[macro.package_name][macro.name] = macro def add_macro(self, macro: Macro) -> None: macro_name: str = macro.name # internal macros (from plugins) will be processed separately from # project macros, so store them in a different place if macro.package_name in self.internal_package_names: self._add_macro_to(self.internal_packages, macro) else: # if it's not an internal package self._add_macro_to(self.packages, macro) # add to root_package_macros if it's in the root package if macro.package_name == self.root_project_name: self.root_package_macros[macro_name] = macro def add_macros(self) -> None: for macro in self.macros.values(): self.add_macro(macro) def get_macro(self, local_package, macro_name) -> Optional[Macro]: local_package_macros = {} # If the macro is explicitly prefixed with an internal namespace # (e.g. 'dbt.some_macro'), look there first if local_package in self.internal_package_names: local_package_macros = self.internal_packages[local_package] # If the macro is explicitly prefixed with a different package name # (e.g. 'dbt_utils.some_macro'), look there first if local_package not in self.internal_package_names and local_package in self.packages: local_package_macros = self.packages[local_package] # First: search the specified package for this macro if macro_name in local_package_macros: return local_package_macros[macro_name] # Now look up in the standard search order if macro_name in self.macros_by_name: return self.macros_by_name[macro_name] return None def get_macro_id(self, local_package, macro_name) -> Optional[str]: macro = self.get_macro(local_package, macro_name) if macro is None: return None else: return macro.unique_id # Currently this is just used by test processing in the schema # parser (in connection with the MacroResolver). Future work # will extend the use of these classes to other parsing areas. # One of the features of this class compared to the MacroNamespace # is that you can limit the number of macros provided to the # context dictionary in the 'to_dict' manifest method. class TestMacroNamespace: def __init__(self, macro_resolver, ctx, node, thread_ctx, depends_on_macros): self.macro_resolver = macro_resolver self.ctx = ctx self.node = node # can be none self.thread_ctx = thread_ctx self.local_namespace = {} self.project_namespace = {} if depends_on_macros: dep_macros = [] self.recursively_get_depends_on_macros(depends_on_macros, dep_macros) for macro_unique_id in dep_macros: if macro_unique_id in self.macro_resolver.macros: # Split up the macro unique_id to get the project_name (_, project_name, macro_name) = macro_unique_id.split(".") # Save the plain macro_name in the local_namespace macro = self.macro_resolver.macros[macro_unique_id] macro_gen = MacroGenerator( macro, self.ctx, self.node, self.thread_ctx, ) self.local_namespace[macro_name] = macro_gen # We also need the two part macro name if project_name not in self.project_namespace: self.project_namespace[project_name] = {} self.project_namespace[project_name][macro_name] = macro_gen def recursively_get_depends_on_macros(self, depends_on_macros, dep_macros): for macro_unique_id in depends_on_macros: if macro_unique_id in dep_macros: continue dep_macros.append(macro_unique_id) if macro_unique_id in self.macro_resolver.macros: macro = self.macro_resolver.macros[macro_unique_id] if macro.depends_on.macros: self.recursively_get_depends_on_macros(macro.depends_on.macros, dep_macros) def get_from_package(self, package_name: Optional[str], name: str) -> Optional[MacroGenerator]: macro = None if package_name is None: macro = self.macro_resolver.macros_by_name.get(name) elif package_name == GLOBAL_PROJECT_NAME: macro = self.macro_resolver.internal_packages_namespace.get(name) elif package_name in self.macro_resolver.packages: macro = self.macro_resolver.packages[package_name].get(name) else: raise PackageNotFoundForMacroError(package_name) if not macro: return None macro_func = MacroGenerator(macro, self.ctx, self.node, self.thread_ctx) return macro_func ================================================ FILE: core/dbt/context/macros.py ================================================ from typing import Any, Dict, Iterable, Iterator, List, Mapping, Optional, Set, Union from dbt.clients.jinja import MacroGenerator, MacroStack from dbt.contracts.graph.nodes import Macro from dbt.exceptions import DuplicateMacroNameError, PackageNotFoundForMacroError from dbt.include.global_project import PROJECT_NAME as GLOBAL_PROJECT_NAME FlatNamespace = Dict[str, MacroGenerator] NamespaceMember = Union[FlatNamespace, MacroGenerator] FullNamespace = Dict[str, NamespaceMember] # The point of this class is to collect the various macros # and provide the ability to flatten them into the ManifestContexts # that are created for jinja, so that macro calls can be resolved. # Creates special iterators and _keys methods to flatten the lists. # When this class is created it has a static 'local_namespace' which # depends on the package of the node, so it only works for one # particular local package at a time for "flattening" into a context. # 'get_by_package' should work for any macro. class MacroNamespace(Mapping): def __init__( self, global_namespace: FlatNamespace, # root package macros local_namespace: FlatNamespace, # packages for *this* node global_project_namespace: FlatNamespace, # internal packages packages: Dict[str, FlatNamespace], # non-internal packages ): self.global_namespace: FlatNamespace = global_namespace self.local_namespace: FlatNamespace = local_namespace self.packages: Dict[str, FlatNamespace] = packages self.global_project_namespace: FlatNamespace = global_project_namespace def _search_order(self) -> Iterable[Union[FullNamespace, FlatNamespace]]: yield self.local_namespace # local package yield self.global_namespace # root package # TODO CT-211 yield self.packages # type: ignore[misc] # non-internal packages yield { # TODO CT-211 GLOBAL_PROJECT_NAME: self.global_project_namespace, # type: ignore[misc] # dbt } yield self.global_project_namespace # other internal project besides dbt # provides special keys method for MacroNamespace iterator # returns keys from local_namespace, global_namespace, packages, # global_project_namespace def _keys(self) -> Set[str]: keys: Set[str] = set() for search in self._search_order(): keys.update(search) return keys # special iterator using special keys def __iter__(self) -> Iterator[str]: for key in self._keys(): yield key def __len__(self): return len(self._keys()) def __getitem__(self, key: str) -> NamespaceMember: for dct in self._search_order(): if key in dct: return dct[key] raise KeyError(key) def get_from_package(self, package_name: Optional[str], name: str) -> Optional[MacroGenerator]: if package_name is None: return self.get(name) elif package_name == GLOBAL_PROJECT_NAME: return self.global_project_namespace.get(name) elif package_name in self.packages: return self.packages[package_name].get(name) else: raise PackageNotFoundForMacroError(package_name) # This class builds the MacroNamespace by adding macros to # internal_packages or packages, and locals/globals. # Call 'build_namespace' to return a MacroNamespace. # This is used by ManifestContext (and subclasses) class MacroNamespaceBuilder: def __init__( self, root_package: str, search_package: str, thread_ctx: MacroStack, internal_packages: List[str], node: Optional[Any] = None, ) -> None: self.root_package = root_package self.search_package = search_package # internal packages comes from get_adapter_package_names self.internal_package_names = set(internal_packages) self.internal_package_names_order = internal_packages # macro_func is added here if in root package, since # the root package acts as a "global" namespace, overriding # everything else except local external package macro calls self.globals: FlatNamespace = {} # macro_func is added here if it's the package for this node self.locals: FlatNamespace = {} # Create a dictionary of [package name][macro name] = # MacroGenerator object which acts like a function self.internal_packages: Dict[str, FlatNamespace] = {} self.packages: Dict[str, FlatNamespace] = {} self.thread_ctx = thread_ctx self.node = node def _add_macro_to( self, hierarchy: Dict[str, FlatNamespace], macro: Macro, macro_func: MacroGenerator, ): if macro.package_name in hierarchy: namespace = hierarchy[macro.package_name] else: namespace = {} hierarchy[macro.package_name] = namespace if macro.name in namespace: raise DuplicateMacroNameError(macro_func.macro, macro, macro.package_name) hierarchy[macro.package_name][macro.name] = macro_func def add_macro(self, macro: Macro, ctx: Dict[str, Any]) -> None: macro_name: str = macro.name # MacroGenerator is in clients/jinja.py # a MacroGenerator object is a callable object that will # execute the MacroGenerator.__call__ function macro_func: MacroGenerator = MacroGenerator(macro, ctx, self.node, self.thread_ctx) # internal macros (from plugins) will be processed separately from # project macros, so store them in a different place if macro.package_name in self.internal_package_names: self._add_macro_to(self.internal_packages, macro, macro_func) else: # if it's not an internal package self._add_macro_to(self.packages, macro, macro_func) # add to locals if it's the package this node is in if macro.package_name == self.search_package: self.locals[macro_name] = macro_func # add to globals if it's in the root package elif macro.package_name == self.root_package: self.globals[macro_name] = macro_func def add_macros(self, macros: Iterable[Macro], ctx: Dict[str, Any]) -> None: for macro in macros: self.add_macro(macro, ctx) def build_namespace( self, macros_by_package: Dict[str, Dict[str, Macro]], ctx: Dict[str, Any] ) -> MacroNamespace: for package in macros_by_package.values(): self.add_macros(package.values(), ctx) # Iterate in reverse-order and overwrite: the packages that are first # in the list are the ones we want to "win". global_project_namespace: FlatNamespace = {} for pkg in reversed(self.internal_package_names_order): if pkg in self.internal_packages: # add the macros pointed to by this package name global_project_namespace.update(self.internal_packages[pkg]) return MacroNamespace( global_namespace=self.globals, # root package macros local_namespace=self.locals, # packages for *this* node global_project_namespace=global_project_namespace, # internal packages packages=self.packages, # non internal_packages ) ================================================ FILE: core/dbt/context/manifest.py ================================================ from typing import List from dbt.adapters.contracts.connection import AdapterRequiredConfig from dbt.clients.jinja import MacroStack from dbt.context.macro_resolver import TestMacroNamespace from dbt.contracts.graph.manifest import Manifest from .base import contextproperty from .configured import ConfiguredContext from .macros import MacroNamespace, MacroNamespaceBuilder class ManifestContext(ConfiguredContext): """The Macro context has everything in the target context, plus the macros in the manifest. The given macros can override any previous context values, which will be available as if they were accessed relative to the package name. """ # subclasses are QueryHeaderContext and ProviderContext def __init__( self, config: AdapterRequiredConfig, manifest: Manifest, search_package: str, ) -> None: super().__init__(config) self.manifest = manifest # this is the package of the node for which this context was built self.search_package = search_package self.macro_stack = MacroStack() # This namespace is used by the BaseDatabaseWrapper in jinja rendering. # The namespace is passed to it when it's constructed. It expects # to be able to do: namespace.get_from_package(..) self.namespace = self._build_namespace() def _build_namespace(self) -> MacroNamespace: # this takes all the macros in the manifest and adds them # to the MacroNamespaceBuilder stored in self.namespace builder = self._get_namespace_builder() return builder.build_namespace(self.manifest.get_macros_by_package(), self._ctx) def _get_namespace_builder(self) -> MacroNamespaceBuilder: # avoid an import loop from dbt.adapters.factory import get_adapter_package_names internal_packages: List[str] = get_adapter_package_names(self.config.credentials.type) return MacroNamespaceBuilder( self.config.project_name, self.search_package, self.macro_stack, internal_packages, None, ) # This does not use the Mashumaro code def to_dict(self): dct = super().to_dict() # This moves all of the macros in the 'namespace' into top level # keys in the manifest dictionary if isinstance(self.namespace, TestMacroNamespace): dct.update(self.namespace.local_namespace) dct.update(self.namespace.project_namespace) else: dct.update(self.namespace) return dct @contextproperty() def context_macro_stack(self): return self.macro_stack ================================================ FILE: core/dbt/context/providers.py ================================================ import abc import csv import os from copy import deepcopy from typing import ( TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Mapping, Optional, Set, Tuple, Type, TypeVar, Union, ) from typing_extensions import Protocol from dbt import selected_resources from dbt.adapters.base.column import Column from dbt.adapters.base.relation import EventTimeFilter, RelationType from dbt.adapters.contracts.connection import AdapterResponse from dbt.adapters.exceptions import MissingConfigError from dbt.adapters.factory import ( get_adapter, get_adapter_package_names, get_adapter_type_names, ) from dbt.artifacts.resources import ( NodeConfig, NodeVersion, RefArgs, SeedConfig, SourceConfig, ) from dbt.clients.jinja import ( MacroGenerator, MacroStack, UnitTestMacroGenerator, get_rendered, ) from dbt.clients.jinja_static import statically_parse_unrendered_config from dbt.config import IsFQNResource, Project, RuntimeConfig from dbt.constants import DEFAULT_ENV_PLACEHOLDER from dbt.context.base import Var, contextmember, contextproperty from dbt.context.configured import FQNLookup from dbt.context.context_config import ContextConfig from dbt.context.exceptions_jinja import wrapped_exports from dbt.context.macro_resolver import MacroResolver, TestMacroNamespace from dbt.context.macros import MacroNamespace, MacroNamespaceBuilder from dbt.context.manifest import ManifestContext from dbt.contracts.graph.manifest import Disabled, Manifest from dbt.contracts.graph.metrics import MetricReference, ResolvedMetricReference from dbt.contracts.graph.nodes import ( AccessType, Exposure, FunctionNode, Macro, ManifestNode, ModelNode, Resource, SeedNode, SemanticModel, SnapshotNode, SourceDefinition, UnitTestNode, ) from dbt.events.types import JinjaLogWarning from dbt.exceptions import ( CompilationError, ConflictingConfigKeysError, DbtReferenceError, EnvVarMissingError, InlineModelConfigError, LoadAgateTableNotSeedError, LoadAgateTableValueError, MacroDispatchArgError, MacroResultAlreadyLoadedError, MetricArgsError, NumberSourceArgsError, OperationsCannotRefEphemeralNodesError, ParsingError, PersistDocsValueTypeError, RefArgsError, RefBadContextError, SecretEnvVarLocationError, TargetNotFoundError, ) from dbt.flags import get_flags from dbt.materializations.incremental.microbatch import MicrobatchBuilder from dbt.node_types import ModelLanguage, NodeType from dbt.utils import MultiDict, args_to_dict from dbt_common.clients.jinja import MacroProtocol from dbt_common.constants import SECRET_ENV_PREFIX from dbt_common.context import get_invocation_context from dbt_common.events.functions import fire_event, get_metadata_vars from dbt_common.exceptions import ( DbtInternalError, DbtRuntimeError, DbtValidationError, MacrosSourcesUnWriteableError, ) from dbt_common.utils import AttrDict, cast_to_str, merge if TYPE_CHECKING: import agate _MISSING = object() # base classes class RelationProxy: def __init__(self, adapter): self._quoting_config = adapter.config.quoting self._relation_type = adapter.Relation def __getattr__(self, key): return getattr(self._relation_type, key) def create(self, *args, **kwargs): kwargs["quote_policy"] = merge(self._quoting_config, kwargs.pop("quote_policy", {})) return self._relation_type.create(*args, **kwargs) class BaseDatabaseWrapper: """ Wrapper for runtime database interaction. Applies the runtime quote policy via a relation proxy. """ def __init__(self, adapter, namespace: MacroNamespace): self._adapter = adapter self.Relation = RelationProxy(adapter) self._namespace = namespace def __getattr__(self, name): raise NotImplementedError("subclasses need to implement this") @property def config(self): return self._adapter.config def type(self): return self._adapter.type() def commit(self): return self._adapter.commit_if_has_connection() def _get_adapter_macro_prefixes(self) -> List[str]: # order matters for dispatch: # 1. current adapter # 2. any parent adapters (dependencies) # 3. 'default' search_prefixes = get_adapter_type_names(self._adapter.type()) + ["default"] return search_prefixes def _get_search_packages(self, namespace: Optional[str] = None) -> List[Optional[str]]: search_packages: List[Optional[str]] = [None] if namespace is None: search_packages = [None] elif isinstance(namespace, str): macro_search_order = self._adapter.config.get_macro_search_order(namespace) if macro_search_order: search_packages = macro_search_order elif not macro_search_order and namespace in self._adapter.config.dependencies: search_packages = [self.config.project_name, namespace] else: raise CompilationError( f"In adapter.dispatch, got a {type(namespace)} macro_namespace argument " f'("{namespace}"), but macro_namespace should be None or a string.' ) return search_packages def dispatch( self, macro_name: str, macro_namespace: Optional[str] = None, packages: Optional[List[str]] = None, # eventually remove since it's fully deprecated ) -> MacroGenerator: search_packages: List[Optional[str]] if "." in macro_name: suggest_macro_namespace, suggest_macro_name = macro_name.split(".", 1) msg = ( f'In adapter.dispatch, got a macro name of "{macro_name}", ' f'but "." is not a valid macro name component. Did you mean ' f'`adapter.dispatch("{suggest_macro_name}", ' f'macro_namespace="{suggest_macro_namespace}")`?' ) raise CompilationError(msg) if packages is not None: raise MacroDispatchArgError(macro_name) search_packages = self._get_search_packages(macro_namespace) attempts = [] for package_name in search_packages: for prefix in self._get_adapter_macro_prefixes(): search_name = f"{prefix}__{macro_name}" try: # this uses the namespace from the context macro = self._namespace.get_from_package(package_name, search_name) except CompilationError: # Only raise CompilationError if macro is not found in # any package macro = None if package_name is None: attempts.append(search_name) else: attempts.append(f"{package_name}.{search_name}") if macro is not None: return macro searched = ", ".join(repr(a) for a in attempts) msg = f"In dispatch: No macro named '{macro_name}' found within namespace: '{macro_namespace}'\n Searched for: {searched}" raise CompilationError(msg) class BaseResolver(metaclass=abc.ABCMeta): def __init__(self, db_wrapper, model, config, manifest): self.db_wrapper = db_wrapper self.model = model self.config = config self.manifest = manifest @property def current_project(self): return self.config.project_name @property def Relation(self): return self.db_wrapper.Relation @property def resolve_limit(self) -> Optional[int]: return 0 if getattr(self.config.args, "EMPTY", False) else None def _resolve_event_time_field_name(self, target: ManifestNode) -> str: """Get the event time field name with proper quoting based on configuration.""" # Default to False for quoting should_quote = False column_found = False column = None # Check if config has event_time attribute if not hasattr(target.config, "event_time") or target.config.event_time is None: return "" # Check column-level quote configuration first (overrides source-level) if hasattr(target, "columns") and target.columns and isinstance(target.columns, dict): for _, column_info in target.columns.items(): if column_info.name == target.config.event_time: column_found = True # Create the column object column = Column.create( column_info.name, column_info.data_type if column_info.data_type else "" ) # Column-level quote setting takes precedence if hasattr(column_info, "quote") and column_info.quote is not None: should_quote = column_info.quote # Fallback to source-level quote setting elif ( hasattr(target, "quoting") and hasattr(target.quoting, "column") and target.quoting.column is not None ): should_quote = target.quoting.column break # If column not found, fall back to source-level quote setting if not column_found: if ( hasattr(target, "quoting") and hasattr(target.quoting, "column") and target.quoting.column is not None ): should_quote = target.quoting.column # Create column object for quoting column = Column.create(target.config.event_time, "") # Apply quoting logic if should_quote and column is not None: return column.quoted else: return target.config.event_time def resolve_event_time_filter(self, target: ManifestNode) -> Optional[EventTimeFilter]: event_time_filter = None sample_mode = getattr(self.config.args, "sample", None) is not None field_name = self._resolve_event_time_field_name(target) # TODO The number of branches here is getting rough. We should consider ways to simplify # what is going on to make it easier to maintain # Only do event time filtering if the base node has the necessary event time configs if ( isinstance(target.config, (NodeConfig, SeedConfig, SourceConfig)) and target.config.event_time and isinstance(self.model, (ModelNode, SnapshotNode)) ): # Handling of microbatch models if ( isinstance(self.model, ModelNode) and self.model.config.materialized == "incremental" and self.model.config.incremental_strategy == "microbatch" and self.manifest.use_microbatch_batches(project_name=self.config.project_name) and self.model.batch is not None ): # Sample mode microbatch models if sample_mode: start = ( self.config.args.sample.start if self.config.args.sample.start > self.model.batch.event_time_start else self.model.batch.event_time_start ) end = ( self.config.args.sample.end if self.config.args.sample.end < self.model.batch.event_time_end else self.model.batch.event_time_end ) event_time_filter = EventTimeFilter( field_name=field_name, start=start, end=end, ) # Regular microbatch models else: event_time_filter = EventTimeFilter( field_name=field_name, start=self.model.batch.event_time_start, end=self.model.batch.event_time_end, ) # Sample mode _non_ microbatch models elif sample_mode: event_time_filter = EventTimeFilter( field_name=field_name, start=self.config.args.sample.start, end=self.config.args.sample.end, ) return event_time_filter @abc.abstractmethod def __call__(self, *args: str) -> Union[str, RelationProxy, MetricReference]: pass class BaseRefResolver(BaseResolver): @abc.abstractmethod def resolve( self, name: str, package: Optional[str] = None, version: Optional[NodeVersion] = None ) -> RelationProxy: ... def _repack_args( self, name: str, package: Optional[str], version: Optional[NodeVersion] ) -> RefArgs: return RefArgs(package=package, name=name, version=version) def validate_args(self, name: str, package: Optional[str], version: Optional[NodeVersion]): if not isinstance(name, str): raise CompilationError( f"The name argument to ref() must be a string, got {type(name)}" ) if package is not None and not isinstance(package, str): raise CompilationError( f"The package argument to ref() must be a string or None, got {type(package)}" ) if version is not None and not isinstance(version, (str, int, float)): raise CompilationError( f"The version argument to ref() must be a string, int, float, or None - got {type(version)}" ) def __call__(self, *args: str, **kwargs) -> RelationProxy: name: str package: Optional[str] = None version: Optional[NodeVersion] = None if len(args) == 1: name = args[0] elif len(args) == 2: package, name = args else: raise RefArgsError(node=self.model, args=args) version = kwargs.get("version") or kwargs.get("v") self.validate_args(name, package, version) return self.resolve(name, package, version) class BaseSourceResolver(BaseResolver): @abc.abstractmethod def resolve(self, source_name: str, table_name: str): pass def validate_args(self, source_name: str, table_name: str): if not isinstance(source_name, str): raise CompilationError( f"The source name (first) argument to source() must be a " f"string, got {type(source_name)}" ) if not isinstance(table_name, str): raise CompilationError( f"The table name (second) argument to source() must be a " f"string, got {type(table_name)}" ) def __call__(self, *args: str) -> RelationProxy: if len(args) != 2: raise NumberSourceArgsError(args, node=self.model) self.validate_args(args[0], args[1]) return self.resolve(args[0], args[1]) class BaseMetricResolver(BaseResolver): @abc.abstractmethod def resolve(self, name: str, package: Optional[str] = None) -> MetricReference: ... def _repack_args(self, name: str, package: Optional[str]) -> List[str]: if package is None: return [name] else: return [package, name] def validate_args(self, name: str, package: Optional[str]): if not isinstance(name, str): raise CompilationError( f"The name argument to metric() must be a string, got {type(name)}" ) if package is not None and not isinstance(package, str): raise CompilationError( f"The package argument to metric() must be a string or None, got {type(package)}" ) def __call__(self, *args: str) -> MetricReference: name: str package: Optional[str] = None if len(args) == 1: name = args[0] elif len(args) == 2: package, name = args else: raise MetricArgsError(node=self.model, args=args) self.validate_args(name, package) return self.resolve(name, package) class BaseFunctionResolver(BaseResolver): @abc.abstractmethod def resolve(self, name: str, package: Optional[str] = None): ... def _repack_args(self, name: str, package: Optional[str]) -> List[str]: if package is None: return [name] else: return [package, name] def validate_args(self, name: str, package: Optional[str]): if not isinstance(name, str): raise CompilationError( f"The name argument to function() must be a string, got {type(name)}" ) if package is not None and not isinstance(package, str): raise CompilationError( f"The package argument to function() must be a string or None, got {type(package)}" ) def __call__(self, *args: str): name: str package: Optional[str] = None if len(args) == 1: name = args[0] elif len(args) == 2: package, name = args else: raise RefArgsError(node=self.model, args=args) self.validate_args(name, package) return self.resolve(name, package) class Config(Protocol): def __init__(self, model, context_config: Optional[ContextConfig]): ... # Implementation of "config(..)" calls in models class ParseConfigObject(Config): def __init__(self, model, context_config: Optional[ContextConfig]): self.model = model self.context_config = context_config def _transform_config(self, config): for oldkey in ("pre_hook", "post_hook"): if oldkey in config: newkey = oldkey.replace("_", "-") if newkey in config: raise ConflictingConfigKeysError(oldkey, newkey, node=self.model) config[newkey] = config.pop(oldkey) return config def __call__(self, *args, **kwargs): if len(args) == 1 and len(kwargs) == 0: opts = args[0] elif len(args) == 0 and len(kwargs) > 0: opts = kwargs else: raise InlineModelConfigError(node=self.model) opts = self._transform_config(opts) # it's ok to have a parse context with no context config, but you must # not call it! if self.context_config is None: raise DbtRuntimeError("At parse time, did not receive a context config") # Track unrendered opts to build parsed node unrendered_config later on if get_flags().state_modified_compare_more_unrendered_values: unrendered_config = statically_parse_unrendered_config(self.model.raw_code) if unrendered_config: self.context_config.add_unrendered_config_call(unrendered_config) # Use rendered opts to populate context_config self.context_config.add_config_call(opts) return "" def set(self, name, value): return self.__call__({name: value}) def require(self, name, validator=None): return "" def meta_require(self, name, validator=None): return "" def get(self, name, default=None, validator=None): return "" def meta_get(self, name, default=None, validator=None): return "" def persist_relation_docs(self) -> bool: return False def persist_column_docs(self) -> bool: return False class RuntimeConfigObject(Config): def __init__(self, model, context_config: Optional[ContextConfig] = None): self.model = model # we never use or get a config, only the parser cares def __call__(self, *args, **kwargs): return "" def set(self, name, value): return self.__call__({name: value}) def _validate(self, validator, value): validator(value) def _lookup(self, name, default=_MISSING): # if this is a macro, there might be no `model.config`. if not hasattr(self.model, "config"): result = default else: result = self.model.config.get(name, default) if result is _MISSING: raise MissingConfigError(unique_id=self.model.unique_id, name=name) return result def _lookup_meta(self, name, default=_MISSING): # if this is a macro, there might be no `model.config`. if not hasattr(self.model, "config"): result = default else: result = self.model.config.meta_get(name, default) if result is _MISSING: raise MissingConfigError(unique_id=self.model.unique_id, name=name) return result def require(self, name, validator=None): to_return = self._lookup(name) if validator is not None: self._validate(validator, to_return) return to_return def meta_require(self, name, validator=None): to_return = self._lookup_meta(name) if validator is not None: self._validate(validator, to_return) return to_return def get(self, name, default=None, validator=None): to_return = self._lookup(name, default) if validator is not None and default is not None: self._validate(validator, to_return) return to_return def meta_get(self, name, default=None, validator=None): to_return = self._lookup_meta(name, default) if validator is not None and default is not None: self._validate(validator, to_return) return to_return def persist_relation_docs(self) -> bool: persist_docs = self.get("persist_docs", default={}) if not isinstance(persist_docs, dict): raise PersistDocsValueTypeError(persist_docs) return persist_docs.get("relation", False) def persist_column_docs(self) -> bool: persist_docs = self.get("persist_docs", default={}) if not isinstance(persist_docs, dict): raise PersistDocsValueTypeError(persist_docs) return persist_docs.get("columns", False) # `adapter` implementations class ParseDatabaseWrapper(BaseDatabaseWrapper): """The parser subclass of the database wrapper applies any explicit parse-time overrides. """ def __getattr__(self, name): override = name in self._adapter._available_ and name in self._adapter._parse_replacements_ if override: return self._adapter._parse_replacements_[name] elif name in self._adapter._available_: return getattr(self._adapter, name) else: raise AttributeError( "'{}' object has no attribute '{}'".format(self.__class__.__name__, name) ) class RuntimeDatabaseWrapper(BaseDatabaseWrapper): """The runtime database wrapper exposes everything the adapter marks available. """ def __getattr__(self, name): if name in self._adapter._available_: return getattr(self._adapter, name) else: raise AttributeError( "'{}' object has no attribute '{}'".format(self.__class__.__name__, name) ) # `ref` implementations class ParseRefResolver(BaseRefResolver): def resolve( self, name: str, package: Optional[str] = None, version: Optional[NodeVersion] = None ) -> RelationProxy: self.model.refs.append(self._repack_args(name, package, version)) # This is not the ref for the "name" passed in, but for the current model. return self.Relation.create_from(self.config, self.model) ResolveRef = Union[Disabled, ManifestNode] class RuntimeRefResolver(BaseRefResolver): def resolve( self, target_name: str, target_package: Optional[str] = None, target_version: Optional[NodeVersion] = None, ) -> RelationProxy: target_model = self.manifest.resolve_ref( self.model, target_name, target_package, target_version, self.current_project, self.model.package_name, ) # Raise an error if the reference target is missing if target_model is None or isinstance(target_model, Disabled): raise TargetNotFoundError( node=self.model, target_name=target_name, target_kind="node", target_package=target_package, target_version=target_version, disabled=isinstance(target_model, Disabled), ) # Raise error if trying to reference a 'private' resource outside its 'group' elif self.manifest.is_invalid_private_ref( self.model, target_model, self.config.dependencies ): raise DbtReferenceError( unique_id=self.model.unique_id, ref_unique_id=target_model.unique_id, access=AccessType.Private, scope=cast_to_str(target_model.group), ) # Or a 'protected' resource outside its project/package namespace elif self.manifest.is_invalid_protected_ref( self.model, target_model, self.config.dependencies ): raise DbtReferenceError( unique_id=self.model.unique_id, ref_unique_id=target_model.unique_id, access=AccessType.Protected, scope=target_model.package_name, ) self.validate(target_model, target_name, target_package, target_version) return self.create_relation(target_model) def create_relation(self, target_model: ManifestNode) -> RelationProxy: if target_model.is_ephemeral_model: self.model.set_cte(target_model.unique_id, None) return self.Relation.create_ephemeral_from( target_model, limit=self.resolve_limit, event_time_filter=self.resolve_event_time_filter(target_model), ) elif ( hasattr(target_model, "defer_relation") and target_model.defer_relation and self.config.args.defer and ( # User has explicitly opted to prefer defer_relation for unselected resources ( self.config.args.favor_state and target_model.unique_id not in selected_resources.SELECTED_RESOURCES ) # Or, this node's relation does not exist in the expected target location (cache lookup) or not get_adapter(self.config).get_relation( target_model.database, target_model.schema, target_model.identifier ) ) ): return self.Relation.create_from( self.config, target_model.defer_relation, limit=self.resolve_limit, event_time_filter=self.resolve_event_time_filter(target_model), ) else: return self.Relation.create_from( self.config, target_model, limit=self.resolve_limit, event_time_filter=self.resolve_event_time_filter(target_model), ) def validate( self, resolved: ManifestNode, target_name: str, target_package: Optional[str], target_version: Optional[NodeVersion], ) -> None: if resolved.unique_id not in self.model.depends_on.nodes: args = self._repack_args(target_name, target_package, target_version) raise RefBadContextError(node=self.model, args=args) class OperationRefResolver(RuntimeRefResolver): def validate( self, resolved: ManifestNode, target_name: str, target_package: Optional[str], target_version: Optional[NodeVersion], ) -> None: pass def create_relation(self, target_model: ManifestNode) -> RelationProxy: if target_model.is_ephemeral_model: # In operations, we can't ref() ephemeral nodes, because # Macros do not support set_cte raise OperationsCannotRefEphemeralNodesError(target_model.name, node=self.model) else: return super().create_relation(target_model) class RuntimeUnitTestRefResolver(RuntimeRefResolver): @property def resolve_limit(self) -> Optional[int]: # Unit tests should never respect --empty flag or provide a limit since they are based on fake data. return None def resolve( self, target_name: str, target_package: Optional[str] = None, target_version: Optional[NodeVersion] = None, ) -> RelationProxy: return super().resolve(target_name, target_package, target_version) # `source` implementations class ParseSourceResolver(BaseSourceResolver): def resolve(self, source_name: str, table_name: str): # When you call source(), this is what happens at parse time self.model.sources.append([source_name, table_name]) return self.Relation.create_from(self.config, self.model) class RuntimeSourceResolver(BaseSourceResolver): def resolve(self, source_name: str, table_name: str): target_source = self.manifest.resolve_source( source_name, table_name, self.current_project, self.model.package_name, ) if target_source is None or isinstance(target_source, Disabled): raise TargetNotFoundError( node=self.model, target_name=f"{source_name}.{table_name}", target_kind="source", disabled=(isinstance(target_source, Disabled)), ) # Source quoting does _not_ respect global configs in dbt_project.yml, as documented here: # https://docs.getdbt.com/reference/project-configs/quoting # Use an object with an empty quoting field to bypass any settings in self. class SourceQuotingBaseConfig: quoting: Dict[str, Any] = {} return self.Relation.create_from( SourceQuotingBaseConfig(), target_source, limit=self.resolve_limit, event_time_filter=self.resolve_event_time_filter(target_source), ) class RuntimeUnitTestSourceResolver(BaseSourceResolver): @property def resolve_limit(self) -> Optional[int]: # Unit tests should never respect --empty flag or provide a limit since they are based on fake data. return None def resolve(self, source_name: str, table_name: str): target_source = self.manifest.resolve_source( source_name, table_name, self.current_project, self.model.package_name, ) if target_source is None or isinstance(target_source, Disabled): raise TargetNotFoundError( node=self.model, target_name=f"{source_name}.{table_name}", target_kind="source", disabled=(isinstance(target_source, Disabled)), ) # For unit tests, this isn't a "real" source, it's a ModelNode taking # the place of a source. We don't really need to return the relation here, # we just need to set_cte, but skipping it confuses typing. We *do* need # the relation in the "this" property. self.model.set_cte(target_source.unique_id, None) identifier = self.Relation.add_ephemeral_prefix(target_source.cte_name) return self.Relation.create( type=self.Relation.CTE, identifier=identifier, ).quote(identifier=False) # metric` implementations class ParseMetricResolver(BaseMetricResolver): def resolve(self, name: str, package: Optional[str] = None) -> MetricReference: self.model.metrics.append(self._repack_args(name, package)) return MetricReference(name, package) class RuntimeMetricResolver(BaseMetricResolver): def resolve(self, target_name: str, target_package: Optional[str] = None) -> MetricReference: target_metric = self.manifest.resolve_metric( target_name, target_package, self.current_project, self.model.package_name, ) if target_metric is None or isinstance(target_metric, Disabled): raise TargetNotFoundError( node=self.model, target_name=target_name, target_kind="metric", target_package=target_package, ) return ResolvedMetricReference(target_metric, self.manifest) # `var` implementations. class ModelConfiguredVar(Var): def __init__( self, context: Dict[str, Any], config: RuntimeConfig, node: Resource, ) -> None: self._node: Resource self._config: RuntimeConfig = config super().__init__(context, config.cli_vars, node=node) def packages_for_node(self) -> Iterable[Project]: dependencies = self._config.load_dependencies() package_name = self._node.package_name if package_name != self._config.project_name: if package_name in dependencies: yield dependencies[package_name] yield self._config def _generate_merged(self) -> Mapping[str, Any]: search_node: IsFQNResource if isinstance(self._node, IsFQNResource): search_node = self._node else: search_node = FQNLookup(self._node.package_name) adapter_type = self._config.credentials.type merged = MultiDict() for project in self.packages_for_node(): merged.add(project.vars.vars_for(search_node, adapter_type)) merged.add(self._cli_vars) return merged class ParseVar(ModelConfiguredVar): def get_missing_var(self, var_name): # in the parser, just always return None. return None class RuntimeVar(ModelConfiguredVar): pass class UnitTestVar(RuntimeVar): def __init__( self, context: Dict[str, Any], config: RuntimeConfig, node: Resource, ) -> None: config_copy = None assert isinstance(node, UnitTestNode) if node.overrides and node.overrides.vars: config_copy = deepcopy(config) config_copy.cli_vars.update(node.overrides.vars) super().__init__(context, config_copy or config, node=node) # `function` implementations. class ParseFunctionResolver(BaseFunctionResolver): def resolve(self, name: str, package: Optional[str] = None): # When you call function(), this is what happens at parse time self.model.functions.append(self._repack_args(name, package)) return self.Relation.create_from(self.config, self.model, type=RelationType.Function) class RuntimeFunctionResolver(BaseFunctionResolver): def resolve(self, name: str, package: Optional[str] = None): target_function = self.manifest.resolve_function( name, package, self.current_project, self.model.package_name, ) if target_function is None or isinstance(target_function, Disabled): raise TargetNotFoundError( node=self.model, target_name=name, target_kind="function", disabled=(isinstance(target_function, Disabled)), ) function_node = target_function if ( hasattr(target_function, "defer_function") and target_function.defer_function and self.config.args.defer and ( ( self.config.args.favor_state and target_function.unique_id not in selected_resources.SELECTED_RESOURCES ) or not get_adapter(self.config).get_relation( target_function.database, target_function.schema, target_function.identifier, ) ) ): function_node = target_function.defer_function # Source quoting does _not_ respect global configs in dbt_project.yml, as documented here: # https://docs.getdbt.com/reference/project-configs/quoting # Use an object with an empty quoting field to bypass any settings in self. class SourceQuotingBaseConfig: quoting: Dict[str, Any] = {} return self.Relation.create_from( SourceQuotingBaseConfig(), function_node, limit=self.resolve_limit, event_time_filter=self.resolve_event_time_filter(target_function), type=RelationType.Function, ) # TODO: Right now the RuntimeUnitTestProvider uses the RuntimeFunctionResolver for functions, # but for CT-12025 we'll likely need to create a separate RuntimeUnitTestFunctionResolver to # handle function overrides (mocking functions) # Providers class Provider(Protocol): execute: bool Config: Type[Config] DatabaseWrapper: Type[BaseDatabaseWrapper] Var: Type[ModelConfiguredVar] ref: Type[BaseRefResolver] source: Type[BaseSourceResolver] metric: Type[BaseMetricResolver] function: Type[BaseFunctionResolver] class ParseProvider(Provider): execute = False Config = ParseConfigObject DatabaseWrapper = ParseDatabaseWrapper Var = ParseVar ref = ParseRefResolver source = ParseSourceResolver metric = ParseMetricResolver function = ParseFunctionResolver class GenerateNameProvider(Provider): execute = False Config = RuntimeConfigObject DatabaseWrapper = ParseDatabaseWrapper Var = RuntimeVar ref = ParseRefResolver source = ParseSourceResolver metric = ParseMetricResolver function = ParseFunctionResolver class RuntimeProvider(Provider): execute = True Config = RuntimeConfigObject DatabaseWrapper = RuntimeDatabaseWrapper Var = RuntimeVar ref = RuntimeRefResolver source = RuntimeSourceResolver metric = RuntimeMetricResolver function = RuntimeFunctionResolver class RuntimeUnitTestProvider(Provider): execute = True Config = RuntimeConfigObject DatabaseWrapper = RuntimeDatabaseWrapper Var = UnitTestVar ref = RuntimeUnitTestRefResolver source = RuntimeUnitTestSourceResolver metric = RuntimeMetricResolver function = RuntimeFunctionResolver class OperationProvider(RuntimeProvider): ref = OperationRefResolver T = TypeVar("T") # Base context collection, used for parsing configs. class ProviderContext(ManifestContext): # subclasses are MacroContext, ModelContext, TestContext, SourceContext def __init__( self, model, config: RuntimeConfig, manifest: Manifest, provider: Provider, context_config: Optional[ContextConfig], ) -> None: if provider is None: raise DbtInternalError(f"Invalid provider given to context: {provider}") # mypy appeasement - we know it'll be a RuntimeConfig self.config: RuntimeConfig self.model: Union[Macro, ManifestNode, SourceDefinition] = model super().__init__(config, manifest, model.package_name) self.sql_results: Dict[str, Optional[AttrDict]] = {} self.context_config: Optional[ContextConfig] = context_config self.provider: Provider = provider self.adapter = get_adapter(self.config) # The macro namespace is used in creating the DatabaseWrapper self.db_wrapper = self.provider.DatabaseWrapper(self.adapter, self.namespace) # This overrides the method in ManifestContext, and provides # a model, which the ManifestContext builder does not def _get_namespace_builder(self): internal_packages = get_adapter_package_names(self.config.credentials.type) return MacroNamespaceBuilder( self.config.project_name, self.search_package, self.macro_stack, internal_packages, self.model, ) @contextproperty() def dbt_metadata_envs(self) -> Dict[str, str]: return get_metadata_vars() @contextproperty() def invocation_args_dict(self): return args_to_dict(self.config.args) @contextproperty() def _sql_results(self) -> Dict[str, Optional[AttrDict]]: return self.sql_results @contextmember() def load_result(self, name: str) -> Optional[AttrDict]: if name in self.sql_results: # handle the special case of "main" macro # See: https://github.com/dbt-labs/dbt-core/blob/ada8860e48b32ac712d92e8b0977b2c3c9749981/core/dbt/task/run.py#L228 if name == "main": return self.sql_results["main"] # handle a None, which indicates this name was populated but has since been loaded elif self.sql_results[name] is None: raise MacroResultAlreadyLoadedError(name) # Handle the regular use case else: ret_val = self.sql_results[name] self.sql_results[name] = None return ret_val else: # Handle trying to load a result that was never stored return None @contextmember() def store_result( self, name: str, response: Any, agate_table: Optional["agate.Table"] = None ) -> str: from dbt_common.clients import agate_helper if agate_table is None: agate_table = agate_helper.empty_table() self.sql_results[name] = AttrDict( { "response": response, "data": agate_helper.as_matrix(agate_table), "table": agate_table, } ) return "" @contextmember() def store_raw_result( self, name: str, message=Optional[str], code=Optional[str], rows_affected=Optional[str], agate_table: Optional["agate.Table"] = None, ) -> str: response = AdapterResponse(_message=message, code=code, rows_affected=rows_affected) return self.store_result(name, response, agate_table) @contextproperty() def validation(self): def validate_any(*args) -> Callable[[T], None]: def inner(value: T) -> None: for arg in args: if isinstance(arg, type) and isinstance(value, arg): return elif value == arg: return raise DbtValidationError( 'Expected value "{}" to be one of {}'.format(value, ",".join(map(str, args))) ) return inner return AttrDict( { "any": validate_any, } ) @contextmember() def write(self, payload: str) -> str: # macros/source defs aren't 'writeable'. if isinstance(self.model, (Macro, SourceDefinition)): raise MacrosSourcesUnWriteableError(node=self.model) split_suffix = None if ( isinstance(self.model, ModelNode) and self.model.config.get("incremental_strategy") == "microbatch" and self.model.batch is not None ): split_suffix = MicrobatchBuilder.format_batch_start( self.model.batch.event_time_start, self.model.config.batch_size, ) self.model.build_path = self.model.get_target_write_path( self.config.target_path, "run", split_suffix=split_suffix ) self.model.write_node(self.config.project_root, self.model.build_path, payload) return "" @contextmember() def render(self, string: str) -> str: return get_rendered(string, self._ctx, self.model) @contextmember() def try_or_compiler_error( self, message_if_exception: str, func: Callable, *args, **kwargs ) -> Any: try: return func(*args, **kwargs) except Exception: raise CompilationError(message_if_exception, self.model) @staticmethod def _read_csv_header(path: str, delimiter: str) -> Optional[Set[str]]: try: # Use utf-8-sig to handle BOM if present with open(path, "r", encoding="utf-8-sig", newline="") as f: reader = csv.reader(f, delimiter=delimiter) header = next(reader) return set(header) except (IOError, StopIteration, TypeError): return None @contextmember() def load_agate_table(self) -> "agate.Table": from dbt_common.clients import agate_helper if not isinstance(self.model, SeedNode): raise LoadAgateTableNotSeedError(self.model.resource_type, node=self.model) # include package_path for seeds defined in packages package_path = ( os.path.join(self.config.packages_install_path, self.model.package_name) if self.model.package_name != self.config.project_name else "." ) path = os.path.join(self.config.project_root, package_path, self.model.original_file_path) if not os.path.exists(path): assert self.model.root_path path = os.path.join(self.model.root_path, self.model.original_file_path) column_types = self.model.config.column_types or {} delimiter = self.model.config.delimiter # Validate that column_types keys exist in the file header filtered_column_types = column_types if column_types: header_set = self._read_csv_header(path, delimiter) if header_set is not None: valid_column_types = set(column_types) & header_set invalid_column_names = set(column_types) - header_set if invalid_column_names: invalid_column_names_str = ", ".join(sorted(invalid_column_names)) msg = ( f"Column types specified for non-existent columns in seed '{self.model.name}' (file: {path}): " f"{invalid_column_names_str}. These column type overrides will be ignored." ) fire_event(JinjaLogWarning(msg=msg)) filtered_column_types = { k: v for k, v in column_types.items() if k in valid_column_types } try: table = agate_helper.from_csv( path, text_columns=filtered_column_types, delimiter=delimiter ) except ValueError as e: raise LoadAgateTableValueError(e, node=self.model) # this is used by some adapters table.original_abspath = os.path.abspath(path) # type: ignore return table @contextproperty() def ref(self) -> Callable: """The most important function in dbt is `ref()`; it's impossible to build even moderately complex models without it. `ref()` is how you reference one model within another. This is a very common behavior, as typically models are built to be "stacked" on top of one another. Here is how this looks in practice: > model_a.sql: select * from public.raw_data > model_b.sql: select * from {{ref('model_a')}} `ref()` is, under the hood, actually doing two important things. First, it is interpolating the schema into your model file to allow you to change your deployment schema via configuration. Second, it is using these references between models to automatically build the dependency graph. This will enable dbt to deploy models in the correct order when using dbt run. The `ref` function returns a Relation object. ## Advanced ref usage There is also a two-argument variant of the `ref` function. With this variant, you can pass both a package name and model name to `ref` to avoid ambiguity. This functionality is not commonly required for typical dbt usage. > model.sql: select * from {{ ref('package_name', 'model_name') }}" """ return self.provider.ref(self.db_wrapper, self.model, self.config, self.manifest) @contextproperty() def source(self) -> Callable: return self.provider.source(self.db_wrapper, self.model, self.config, self.manifest) @contextproperty() def metric(self) -> Callable: return self.provider.metric(self.db_wrapper, self.model, self.config, self.manifest) @contextproperty() def function(self) -> Callable: return self.provider.function(self.db_wrapper, self.model, self.config, self.manifest) @contextproperty("config") def ctx_config(self) -> Config: """The `config` variable exists to handle end-user configuration for custom materializations. Configs like `unique_key` can be implemented using the `config` variable in your own materializations. For example, code in the `incremental` materialization like this: {% materialization incremental, default -%} {%- set unique_key = config.get('unique_key') -%} ... is responsible for handling model code that looks like this: {{ config( materialized='incremental', unique_key='id' ) }} ## config.get name: The name of the configuration variable (required) default: The default value to use if this configuration is not provided (optional) The `config.get` function is used to get configurations for a model from the end-user. Configs defined in this way are optional, and a default value can be provided. Example usage: {% materialization incremental, default -%} -- Example w/ no default. unique_key will be None if the user does not provide this configuration {%- set unique_key = config.get('unique_key') -%} -- Example w/ default value. Default to 'id' if 'unique_key' not provided {%- set unique_key = config.get('unique_key', default='id') -%} ... ## config.require name: The name of the configuration variable (required) The `config.require` function is used to get configurations for a model from the end-user. Configs defined using this function are required, and failure to provide them will result in a compilation error. Example usage: {% materialization incremental, default -%} {%- set unique_key = config.require('unique_key') -%} ... """ # noqa return self.provider.Config(self.model, self.context_config) @contextproperty() def execute(self) -> bool: """`execute` is a Jinja variable that returns True when dbt is in "execute" mode. When you execute a dbt compile or dbt run command, dbt: - Reads all of the files in your project and generates a "manifest" comprised of models, tests, and other graph nodes present in your project. During this phase, dbt uses the `ref` statements it finds to generate the DAG for your project. *No SQL is run during this phase*, and `execute == False`. - Compiles (and runs) each node (eg. building models, or running tests). SQL is run during this phase, and `execute == True`. Any Jinja that relies on a result being returned from the database will error during the parse phase. For example, this SQL will return an error: > models/order_payment_methods.sql: {% set payment_method_query %} select distinct payment_method from {{ ref('raw_payments') }} order by 1 {% endset %} {% set results = run_query(relation_query) %} {# Return the first column #} {% set payment_methods = results.columns[0].values() %} The error returned by dbt will look as follows: Encountered an error: Compilation Error in model order_payment_methods (models/order_payment_methods.sql) 'None' has no attribute 'table' This is because Line #11 assumes that a table has been returned, when, during the parse phase, this query hasn't been run. To work around this, wrap any problematic Jinja in an `{% if execute %}` statement: > models/order_payment_methods.sql: {% set payment_method_query %} select distinct payment_method from {{ ref('raw_payments') }} order by 1 {% endset %} {% set results = run_query(relation_query) %} {% if execute %} {# Return the first column #} {% set payment_methods = results.columns[0].values() %} {% else %} {% set payment_methods = [] %} {% endif %} """ # noqa return self.provider.execute @contextproperty() def exceptions(self) -> Dict[str, Any]: """The exceptions namespace can be used to raise warnings and errors in dbt userspace. ## raise_compiler_error The `exceptions.raise_compiler_error` method will raise a compiler error with the provided message. This is typically only useful in macros or materializations when invalid arguments are provided by the calling model. Note that throwing an exception will cause a model to fail, so please use this variable with care! Example usage: > exceptions.sql: {% if number < 0 or number > 100 %} {{ exceptions.raise_compiler_error("Invalid `number`. Got: " ~ number) }} {% endif %} ## warn The `exceptions.warn` method will raise a compiler warning with the provided message. If the `--warn-error` flag is provided to dbt, then this warning will be elevated to an exception, which is raised. Example usage: > warn.sql: {% if number < 0 or number > 100 %} {% do exceptions.warn("Invalid `number`. Got: " ~ number) %} {% endif %} """ # noqa return wrapped_exports(self.model) @contextproperty() def database(self) -> str: return self.config.credentials.database @contextproperty() def schema(self) -> str: return self.config.credentials.schema @contextproperty() def var(self) -> ModelConfiguredVar: return self.provider.Var( context=self._ctx, config=self.config, node=self.model, ) @contextproperty("adapter") def ctx_adapter(self) -> BaseDatabaseWrapper: """`adapter` is a wrapper around the internal database adapter used by dbt. It allows users to make calls to the database in their dbt models. The adapter methods will be translated into specific SQL statements depending on the type of adapter your project is using. """ return self.db_wrapper @contextproperty() def api(self) -> Dict[str, Any]: return { "Relation": self.db_wrapper.Relation, "Column": self.adapter.Column, } @contextproperty() def column(self) -> Type[Column]: return self.adapter.Column @contextproperty() def env(self) -> Dict[str, Any]: return self.target @contextproperty() def graph(self) -> Dict[str, Any]: """The `graph` context variable contains information about the nodes in your dbt project. Models, sources, tests, and snapshots are all examples of nodes in dbt projects. ## The graph context variable The graph context variable is a dictionary which maps node ids onto dictionary representations of those nodes. A simplified example might look like: { "model.project_name.model_name": { "config": {"materialzed": "table", "sort": "id"}, "tags": ["abc", "123"], "path": "models/path/to/model_name.sql", ... }, "source.project_name.source_name": { "path": "models/path/to/schema.yml", "columns": { "id": { .... }, "first_name": { .... }, }, ... } } The exact contract for these model and source nodes is not currently documented, but that will change in the future. ## Accessing models The `model` entries in the `graph` dictionary will be incomplete or incorrect during parsing. If accessing the models in your project via the `graph` variable, be sure to use the `execute` flag to ensure that this code only executes at run-time and not at parse-time. Do not use the `graph` variable to build you DAG, as the resulting dbt behavior will be undefined and likely incorrect. Example usage: > graph-usage.sql: /* Print information about all of the models in the Snowplow package */ {% if execute %} {% for node in graph.nodes.values() | selectattr("resource_type", "equalto", "model") | selectattr("package_name", "equalto", "snowplow") %} {% do log(node.unique_id ~ ", materialized: " ~ node.config.materialized, info=true) %} {% endfor %} {% endif %} /* Example output --------------------------------------------------------------- model.snowplow.snowplow_id_map, materialized: incremental model.snowplow.snowplow_page_views, materialized: incremental model.snowplow.snowplow_web_events, materialized: incremental model.snowplow.snowplow_web_page_context, materialized: table model.snowplow.snowplow_web_events_scroll_depth, materialized: incremental model.snowplow.snowplow_web_events_time, materialized: incremental model.snowplow.snowplow_web_events_internal_fixed, materialized: ephemeral model.snowplow.snowplow_base_web_page_context, materialized: ephemeral model.snowplow.snowplow_base_events, materialized: ephemeral model.snowplow.snowplow_sessions_tmp, materialized: incremental model.snowplow.snowplow_sessions, materialized: table */ ## Accessing sources To access the sources in your dbt project programatically, use the "sources" attribute. Example usage: > models/events_unioned.sql /* Union all of the Snowplow sources defined in the project which begin with the string "event_" */ {% set sources = [] -%} {% for node in graph.sources.values() -%} {%- if node.name.startswith('event_') and node.source_name == 'snowplow' -%} {%- do sources.append(source(node.source_name, node.name)) -%} {%- endif -%} {%- endfor %} select * from ( {%- for source in sources %} {{ source }} {% if not loop.last %} union all {% endif %} {% endfor %} ) /* Example compiled SQL --------------------------------------------------------------- select * from ( select * from raw.snowplow.event_add_to_cart union all select * from raw.snowplow.event_remove_from_cart union all select * from raw.snowplow.event_checkout ) */ """ # noqa return self.manifest.flat_graph @contextproperty("model") def ctx_model(self) -> Dict[str, Any]: model_dct = self.model.to_dict(omit_none=True) # Maintain direct use of compiled_sql # TODO add depreciation logic[CT-934] if "compiled_code" in model_dct: model_dct["compiled_sql"] = model_dct["compiled_code"] if ( hasattr(self.model, "contract") and self.model.contract.alias_types is True and "columns" in model_dct ): for column in model_dct["columns"].values(): if "data_type" in column: orig_data_type = column["data_type"] # translate data_type to value in Column.TYPE_LABELS new_data_type = self.adapter.Column.translate_type(orig_data_type) column["data_type"] = new_data_type return model_dct @contextproperty() def pre_hooks(self) -> Optional[List[Dict[str, Any]]]: return None @contextproperty() def post_hooks(self) -> Optional[List[Dict[str, Any]]]: return None @contextproperty() def sql(self) -> Optional[str]: return None @contextproperty() def sql_now(self) -> str: return self.adapter.date_function() @contextmember() def adapter_macro(self, name: str, *args, **kwargs): """This was deprecated in v0.18 in favor of adapter.dispatch""" msg = ( 'The "adapter_macro" macro has been deprecated. Instead, use ' "the `adapter.dispatch` method to find a macro and call the " "result. For more information, see: " "https://docs.getdbt.com/reference/dbt-jinja-functions/dispatch)" " adapter_macro was called for: {macro_name}".format(macro_name=name) ) raise CompilationError(msg) @contextmember() def env_var(self, var: str, default: Optional[str] = None) -> str: """The env_var() function. Return the environment variable named 'var'. If there is no such environment variable set, return the default. If the default is None, raise an exception for an undefined variable. """ return_value = None if var.startswith(SECRET_ENV_PREFIX): raise SecretEnvVarLocationError(var) env = get_invocation_context().env if var in env: return_value = env[var] elif default is not None: return_value = default if return_value is not None: # Save the env_var value in the manifest and the var name in the source_file. # If this is compiling, do not save because it's irrelevant to parsing. compiling = ( True if hasattr(self.model, "compiled") and getattr(self.model, "compiled", False) is True else False ) if self.model and not compiling: # If the environment variable is set from a default, store a string indicating # that so we can skip partial parsing. Otherwise the file will be scheduled for # reparsing. If the default changes, the file will have been updated and therefore # will be scheduled for reparsing anyways. self.manifest.env_vars[var] = ( return_value if var in env else DEFAULT_ENV_PLACEHOLDER ) # hooks come from dbt_project.yml which doesn't have a real file_id if self.model.file_id in self.manifest.files: source_file = self.manifest.files[self.model.file_id] # Schema files should never get here if source_file.parse_file_type != "schema": # TODO CT-211 source_file.env_vars.append(var) # type: ignore[union-attr] return return_value else: raise EnvVarMissingError(var) @contextproperty() def selected_resources(self) -> List[str]: """The `selected_resources` variable contains a list of the resources selected based on the parameters provided to the dbt command. Currently, is not populated for the command `run-operation` that doesn't support `--select`. """ return selected_resources.SELECTED_RESOURCES @contextmember() def submit_python_job(self, parsed_model: Dict, compiled_code: str) -> AdapterResponse: # Check macro_stack and that the unique id is for a materialization macro if not ( self.context_macro_stack.depth == 2 and self.context_macro_stack.call_stack[1] == "macro.dbt.statement" and "materialization" in self.context_macro_stack.call_stack[0] ): raise DbtRuntimeError( f"submit_python_job is not intended to be called here, at model {parsed_model['alias']}, with macro call_stack {self.context_macro_stack.call_stack}." ) return self.adapter.submit_python_job(parsed_model, compiled_code) class MacroContext(ProviderContext): """Internally, macros can be executed like nodes, with some restrictions: - they don't have all values available that nodes do: - 'this', 'pre_hooks', 'post_hooks', and 'sql' are missing - 'schema' does not use any 'model' information - they can't be configured with config() directives """ def __init__( self, model: MacroProtocol, config: RuntimeConfig, manifest: Manifest, provider: Provider, search_package: Optional[str], ) -> None: super().__init__(model, config, manifest, provider, None) # override the model-based package with the given one if search_package is None: # if the search package name isn't specified, use the root project self._search_package = config.project_name else: self._search_package = search_package class SourceContext(ProviderContext): # SourceContext is being used to render jinja SQL during execution of # custom SQL in source freshness. It is not used for parsing. model: SourceDefinition @contextproperty() def this(self) -> Optional[RelationProxy]: return self.db_wrapper.Relation.create_from(self.config, self.model) @contextproperty() def source_node(self) -> SourceDefinition: return self.model class ModelContext(ProviderContext): model: ManifestNode @contextproperty() def pre_hooks(self) -> List[Dict[str, Any]]: if self.model.resource_type in [NodeType.Source, NodeType.Test, NodeType.Unit]: return [] # TODO CT-211 return [ h.to_dict(omit_none=True) for h in self.model.config.pre_hook # type: ignore[union-attr] # noqa ] @contextproperty() def post_hooks(self) -> List[Dict[str, Any]]: if self.model.resource_type in [NodeType.Source, NodeType.Test, NodeType.Unit]: return [] # TODO CT-211 return [ h.to_dict(omit_none=True) for h in self.model.config.post_hook # type: ignore[union-attr] # noqa ] @contextproperty() def compiled_code(self) -> Optional[str]: # TODO: avoid routing on args.which if possible if getattr(self.model, "defer_relation", None) and self.config.args.which == "clone": # TODO https://github.com/dbt-labs/dbt-core/issues/7976 return f"select * from {self.model.defer_relation.relation_name or str(self.defer_relation)}" # type: ignore[union-attr] elif getattr(self.model, "extra_ctes_injected", None): # TODO CT-211 return self.model.compiled_code # type: ignore[union-attr] else: return None @contextproperty() def sql(self) -> Optional[str]: # only set this for sql models, for backward compatibility if self.model.language == ModelLanguage.sql: # type: ignore[union-attr] return self.compiled_code else: return None @contextproperty() def database(self) -> str: return getattr(self.model, "database", self.config.credentials.database) @contextproperty() def schema(self) -> str: return getattr(self.model, "schema", self.config.credentials.schema) @contextproperty() def this(self) -> Optional[RelationProxy]: """`this` makes available schema information about the currently executing model. It's is useful in any context in which you need to write code that references the current model, for example when defining a `sql_where` clause for an incremental model and for writing pre- and post-model hooks that operate on the model in some way. Developers have options for how to use `this`: |------------------|------------------| | dbt Model Syntax | Output | |------------------|------------------| | {{this}} | "schema"."table" | |------------------|------------------| | {{this.schema}} | schema | |------------------|------------------| | {{this.table}} | table | |------------------|------------------| | {{this.name}} | table | |------------------|------------------| Here's an example of how to use `this` in `dbt_project.yml` to grant select rights on a table to a different db user. > example.yml: models: project-name: post-hook: - "grant select on {{ this }} to db_reader" """ if self.model.resource_type == NodeType.Operation: return None return self.db_wrapper.Relation.create_from(self.config, self.model) @contextproperty() def defer_relation(self) -> Optional[RelationProxy]: """ For commands which add information about this node's corresponding production version (via a --state artifact), access the Relation object for that stateful other """ if getattr(self.model, "defer_relation", None): return self.db_wrapper.Relation.create_from( self.config, self.model.defer_relation # type: ignore ) else: return None class UnitTestContext(ModelContext): model: UnitTestNode @contextmember() def env_var(self, var: str, default: Optional[str] = None) -> str: """The env_var() function. Return the overriden unit test environment variable named 'var'. If there is no unit test override, return the environment variable named 'var'. If there is no such environment variable set, return the default. If the default is None, raise an exception for an undefined variable. """ if self.model.overrides and var in self.model.overrides.env_vars: return self.model.overrides.env_vars[var] else: return super().env_var(var, default) @contextproperty() def this(self) -> Optional[str]: if self.model.this_input_node_unique_id: this_node = self.manifest.expect(self.model.this_input_node_unique_id) self.model.set_cte(this_node.unique_id, None) # type: ignore return self.adapter.Relation.add_ephemeral_prefix(this_node.identifier) # type: ignore return None class FunctionContext(ModelContext): model: FunctionNode @contextproperty() def this(self) -> Optional[RelationProxy]: return self.db_wrapper.Relation.create_from(self.config, self.model) # This is called by '_context_for', used in 'render_with_context' def generate_parser_model_context( model: ManifestNode, config: RuntimeConfig, manifest: Manifest, context_config: ContextConfig, ) -> Dict[str, Any]: # The __init__ method of ModelContext also initializes # a ManifestContext object which creates a MacroNamespaceBuilder # which adds every macro in the Manifest. ctx = ModelContext(model, config, manifest, ParseProvider(), context_config) # The 'to_dict' method in ManifestContext moves all of the macro names # in the macro 'namespace' up to top level keys return ctx.to_dict() def generate_parser_unit_test_context( unit_test: UnitTestNode, config: RuntimeConfig, manifest: Manifest ) -> Dict[str, Any]: context_config = ContextConfig( config, unit_test.fqn, NodeType.Unit, config.project_name, ) ctx = UnitTestContext(unit_test, config, manifest, ParseProvider(), context_config) return ctx.to_dict() def generate_generate_name_macro_context( macro: Macro, config: RuntimeConfig, manifest: Manifest, ) -> Dict[str, Any]: ctx = MacroContext(macro, config, manifest, GenerateNameProvider(), None) return ctx.to_dict() def generate_runtime_model_context( model: ManifestNode, config: RuntimeConfig, manifest: Manifest, ) -> Dict[str, Any]: ctx = ModelContext(model, config, manifest, RuntimeProvider(), None) return ctx.to_dict() def generate_runtime_macro_context( macro: MacroProtocol, config: RuntimeConfig, manifest: Manifest, package_name: Optional[str], ) -> Dict[str, Any]: ctx = MacroContext(macro, config, manifest, OperationProvider(), package_name) return ctx.to_dict() def generate_runtime_unit_test_context( unit_test: UnitTestNode, config: RuntimeConfig, manifest: Manifest, ) -> Dict[str, Any]: ctx = UnitTestContext(unit_test, config, manifest, RuntimeUnitTestProvider(), None) ctx_dict = ctx.to_dict() if unit_test.overrides and unit_test.overrides.macros: global_macro_overrides: Dict[str, Any] = {} package_macro_overrides: Dict[Tuple[str, str], Any] = {} # split macro overrides into global and package-namespaced collections for macro_name, macro_value in unit_test.overrides.macros.items(): macro_name_split = macro_name.split(".") macro_package = macro_name_split[0] if len(macro_name_split) == 2 else None macro_name = macro_name_split[-1] # macro overrides of global macros if macro_package is None and macro_name in ctx_dict: original_context_value = ctx_dict[macro_name] if isinstance(original_context_value, MacroGenerator): macro_value = UnitTestMacroGenerator(original_context_value, macro_value) global_macro_overrides[macro_name] = macro_value # macro overrides of package-namespaced macros elif ( macro_package and macro_package in ctx_dict and macro_name in ctx_dict[macro_package] ): original_context_value = ctx_dict[macro_package][macro_name] if isinstance(original_context_value, MacroGenerator): macro_value = UnitTestMacroGenerator(original_context_value, macro_value) package_macro_overrides[(macro_package, macro_name)] = macro_value # macro overrides of package-namespaced macros for (macro_package, macro_name), macro_override_value in package_macro_overrides.items(): ctx_dict[macro_package][macro_name] = macro_override_value # propgate override of namespaced dbt macro to global namespace if macro_package == "dbt": ctx_dict[macro_name] = macro_value # macro overrides of global macros, which should take precedence over equivalent package-namespaced overrides for macro_name, macro_override_value in global_macro_overrides.items(): ctx_dict[macro_name] = macro_override_value # propgate override of global dbt macro to dbt namespace if ctx_dict["dbt"].get(macro_name): ctx_dict["dbt"][macro_name] = macro_override_value return ctx_dict def generate_runtime_function_context( function: FunctionNode, config: RuntimeConfig, manifest: Manifest, ) -> Dict[str, Any]: ctx = FunctionContext(function, config, manifest, OperationProvider(), None) return ctx.to_dict() class ExposureRefResolver(BaseResolver): def __call__(self, *args, **kwargs) -> str: package = None if len(args) == 1: name = args[0] elif len(args) == 2: package, name = args else: raise RefArgsError(node=self.model, args=args) version = kwargs.get("version") or kwargs.get("v") self.model.refs.append(RefArgs(package=package, name=name, version=version)) return "" class ExposureSourceResolver(BaseResolver): def __call__(self, *args) -> str: if len(args) != 2: raise NumberSourceArgsError(args, node=self.model) self.model.sources.append(list(args)) return "" class ExposureMetricResolver(BaseResolver): def __call__(self, *args) -> str: if len(args) not in (1, 2): raise MetricArgsError(node=self.model, args=args) self.model.metrics.append(list(args)) return "" def generate_parse_exposure( exposure: Exposure, config: RuntimeConfig, manifest: Manifest, package_name: str, ) -> Dict[str, Any]: project = config.load_dependencies()[package_name] return { "ref": ExposureRefResolver( None, exposure, project, manifest, ), "source": ExposureSourceResolver( None, exposure, project, manifest, ), "metric": ExposureMetricResolver( None, exposure, project, manifest, ), } # applies to SemanticModels class SemanticModelRefResolver(BaseResolver): def __call__(self, *args, **kwargs) -> str: package = None if len(args) == 1: name = args[0] elif len(args) == 2: package, name = args else: raise RefArgsError(node=self.model, args=args) version = kwargs.get("version") or kwargs.get("v") self.validate_args(name, package, version) # "model" here is any node self.model.refs.append(RefArgs(package=package, name=name, version=version)) return "" def validate_args(self, name, package, version): if not isinstance(name, str): raise ParsingError( f"In a semantic model or metrics section in {self.model.original_file_path} " "the name argument to ref() must be a string" ) # used for semantic models def generate_parse_semantic_models( semantic_model: SemanticModel, config: RuntimeConfig, manifest: Manifest, package_name: str, ) -> Dict[str, Any]: project = config.load_dependencies()[package_name] return { "ref": SemanticModelRefResolver( None, semantic_model, project, manifest, ), } # This class is currently used by the schema parser in order # to limit the number of macros in the context by using # the TestMacroNamespace class TestContext(ProviderContext): def __init__( self, model, config: RuntimeConfig, manifest: Manifest, provider: Provider, context_config: Optional[ContextConfig], macro_resolver: MacroResolver, ) -> None: # this must be before super init so that macro_resolver exists for # build_namespace self.macro_resolver = macro_resolver self.thread_ctx = MacroStack() super().__init__(model, config, manifest, provider, context_config) self._build_test_namespace() # We need to rebuild this because it's already been built by # the ProviderContext with the wrong namespace. self.db_wrapper = self.provider.DatabaseWrapper(self.adapter, self.namespace) def _build_namespace(self): return {} # this overrides _build_namespace in ManifestContext which provides a # complete namespace of all macros to only specify macros in the depends_on # This only provides a namespace with macros in the test node # 'depends_on.macros' by using the TestMacroNamespace def _build_test_namespace(self): depends_on_macros = [] # all generic tests use a macro named 'get_where_subquery' to wrap 'model' arg # see generic_test_builders.build_model_str get_where_subquery = self.macro_resolver.macros_by_name.get("get_where_subquery") if get_where_subquery: depends_on_macros.append(get_where_subquery.unique_id) if self.model.depends_on and self.model.depends_on.macros: depends_on_macros.extend(self.model.depends_on.macros) # When support_custom_ref_kwargs is enabled, include custom ref/source # macro overrides in the test namespace so that test arguments like # ref('model', custom_kwarg=value) are properly resolved. if getattr(get_flags(), "SUPPORT_CUSTOM_REF_KWARGS", False): for macro_name in ("ref", "source"): macro = self.macro_resolver.macros_by_name.get(macro_name) if macro and macro.unique_id not in depends_on_macros: # Only include user-defined overrides, not built-in macros if macro.package_name != "dbt": depends_on_macros.append(macro.unique_id) lookup_macros = depends_on_macros.copy() for macro_unique_id in lookup_macros: lookup_macro = self.macro_resolver.macros.get(macro_unique_id) if lookup_macro: depends_on_macros.extend(lookup_macro.depends_on.macros) macro_namespace = TestMacroNamespace( self.macro_resolver, self._ctx, self.model, self.thread_ctx, depends_on_macros ) self.namespace = macro_namespace @contextmember() def env_var(self, var: str, default: Optional[str] = None) -> str: return_value = None if var.startswith(SECRET_ENV_PREFIX): raise SecretEnvVarLocationError(var) env = get_invocation_context().env if var in env: return_value = env[var] elif default is not None: return_value = default if return_value is not None: # Save the env_var value in the manifest and the var name in the source_file if self.model: # If the environment variable is set from a default, store a string indicating # that so we can skip partial parsing. Otherwise the file will be scheduled for # reparsing. If the default changes, the file will have been updated and therefore # will be scheduled for reparsing anyways. self.manifest.env_vars[var] = ( return_value if var in env else DEFAULT_ENV_PLACEHOLDER ) # the "model" should only be test nodes, but just in case, check # TODO CT-211 if self.model.resource_type == NodeType.Test and self.model.file_key_name: # type: ignore[union-attr] # noqa source_file = self.manifest.files[self.model.file_id] # TODO CT-211 (yaml_key, name) = self.model.file_key_name.split(".") # type: ignore[union-attr] # noqa # TODO CT-211 source_file.add_env_var(var, yaml_key, name) # type: ignore[union-attr] return return_value else: raise EnvVarMissingError(var) def generate_test_context( model: ManifestNode, config: RuntimeConfig, manifest: Manifest, context_config: ContextConfig, macro_resolver: MacroResolver, ) -> Dict[str, Any]: ctx = TestContext(model, config, manifest, ParseProvider(), context_config, macro_resolver) # The 'to_dict' method in ManifestContext moves all of the macro names # in the macro 'namespace' up to top level keys return ctx.to_dict() ================================================ FILE: core/dbt/context/query_header.py ================================================ from dbt.adapters.contracts.connection import AdapterRequiredConfig from dbt.context.manifest import ManifestContext from dbt.contracts.graph.manifest import Manifest class QueryHeaderContext(ManifestContext): def __init__(self, config: AdapterRequiredConfig, manifest: Manifest) -> None: super().__init__(config, manifest, config.project_name) def generate_query_header_context(config: AdapterRequiredConfig, manifest: Manifest): ctx = QueryHeaderContext(config, manifest) return ctx.to_dict() ================================================ FILE: core/dbt/context/secret.py ================================================ from typing import Any, Dict, Optional from dbt.constants import DEFAULT_ENV_PLACEHOLDER, SECRET_PLACEHOLDER from dbt.exceptions import EnvVarMissingError from dbt_common.constants import SECRET_ENV_PREFIX from dbt_common.context import get_invocation_context from .base import BaseContext, contextmember class SecretContext(BaseContext): """This context is used in profiles.yml + packages.yml. It can render secret env vars that aren't usable elsewhere""" @contextmember() def env_var(self, var: str, default: Optional[str] = None) -> str: """The env_var() function. Return the environment variable named 'var'. If there is no such environment variable set, return the default. If the default is None, raise an exception for an undefined variable. In this context *only*, env_var will accept env vars prefixed with DBT_ENV_SECRET_. It will return the name of the secret env var, wrapped in 'start' and 'end' identifiers. The actual value will be subbed in later in SecretRenderer.render_value() """ return_value = None # if this is a 'secret' env var, just return the name of the env var # instead of rendering the actual value here, to avoid any risk of # Jinja manipulation. it will be subbed out later, in SecretRenderer.render_value env = get_invocation_context().env if var in env and var.startswith(SECRET_ENV_PREFIX): return SECRET_PLACEHOLDER.format(var) if var in env: return_value = env[var] elif default is not None: return_value = default if return_value is not None: # store env vars in the internal manifest to power partial parsing # if it's a 'secret' env var, we shouldn't even get here # but just to be safe, don't save secrets if not var.startswith(SECRET_ENV_PREFIX): # If the environment variable is set from a default, store a string indicating # that so we can skip partial parsing. Otherwise the file will be scheduled for # reparsing. If the default changes, the file will have been updated and therefore # will be scheduled for reparsing anyways. self.env_vars[var] = return_value if var in env else DEFAULT_ENV_PLACEHOLDER return return_value else: raise EnvVarMissingError(var) def generate_secret_context(cli_vars: Dict[str, Any]) -> Dict[str, Any]: ctx = SecretContext(cli_vars) # This is not a Mashumaro to_dict call return ctx.to_dict() ================================================ FILE: core/dbt/context/target.py ================================================ from typing import Any, Dict from dbt.context.base import BaseContext, contextproperty class TargetContext(BaseContext): # subclass is ConfiguredContext def __init__( self, target_dict: Dict[str, Any], cli_vars: Dict[str, Any], require_vars: bool = True ): super().__init__(cli_vars=cli_vars, require_vars=require_vars) self.target_dict = target_dict @contextproperty() def target(self) -> Dict[str, Any]: """`target` contains information about your connection to the warehouse (specified in profiles.yml). Some configs are shared between all adapters, while others are adapter-specific. Common: |----------|-----------|------------------------------------------| | Variable | Example | Description | |----------|-----------|------------------------------------------| | name | dev | Name of the active target | |----------|-----------|------------------------------------------| | schema | dbt_alice | Name of the dbt schema (or, dataset on | | | | BigQuery) | |----------|-----------|------------------------------------------| | type | postgres | The active adapter being used. | |----------|-----------|------------------------------------------| | threads | 4 | The number of threads in use by dbt | |----------|-----------|------------------------------------------| Snowflake: |----------|-----------|------------------------------------------| | Variable | Example | Description | |----------|-----------|------------------------------------------| | database | RAW | The active target's database. | |----------|-----------|------------------------------------------| | warehouse| TRANSFORM | The active target's warehouse. | |----------|-----------|------------------------------------------| | user | USERNAME | The active target's user | |----------|-----------|------------------------------------------| | role | ROLENAME | The active target's role | |----------|-----------|------------------------------------------| | account | abc123 | The active target's account | |----------|-----------|------------------------------------------| Postgres/Redshift: |----------|-------------------|----------------------------------| | Variable | Example | Description | |----------|-------------------|----------------------------------| | dbname | analytics | The active target's database. | |----------|-------------------|----------------------------------| | host | abc123.us-west-2. | The active target's host. | | | redshift.amazonaws| | | | .com | | |----------|-------------------|----------------------------------| | user | dbt_user | The active target's user | |----------|-------------------|----------------------------------| | port | 5439 | The active target's port | |----------|-------------------|----------------------------------| BigQuery: |----------|-----------|------------------------------------------| | Variable | Example | Description | |----------|-----------|------------------------------------------| | project | abc-123 | The active target's project. | |----------|-----------|------------------------------------------| """ return self.target_dict ================================================ FILE: core/dbt/contracts/README.md ================================================ # Contracts README ## Artifacts ### Generating JSON schemas A helper script, `scripts/collect-artifact-schema.py` is available to generate json schemas corresponding to versioned artifacts (`ArtifactMixin`s). This script is necessary to run when a new artifact schema version is created, or when changes are made to existing artifact versions, and writes json schema to `schema/dbt/<artifact>/v<version>.json`. Schemas in `schema/dbt` power the rendering in https://schemas.getdbt.com/ via https://github.com/dbt-labs/schemas.getdbt.com/ #### Example Usage Available arguments: ```sh ❯ scripts/collect-artifact-schema.py --help usage: Collect and write dbt arfifact schema [-h] [--path PATH] [--artifact {manifest,sources,run-results,catalog}] options: -h, --help show this help message and exit --path PATH The dir to write artifact schema --artifact {manifest,sources,run-results,catalog} The name of the artifact to update ``` Generate latest version of schemas of all artifacts to `schema/dbt/<artifact>/v<version>.json` ```sh > sripts/collect-artifact-schema.py --path schemas ``` Generate latest version of schemas of manifest to `schema/dbt/manifest/v<version>.json` ```sh > sripts/collect-artifact-schema.py --path schemas --artifact manifest ``` ================================================ FILE: core/dbt/contracts/__init__.py ================================================ ================================================ FILE: core/dbt/contracts/files.py ================================================ import os from dataclasses import dataclass, field from typing import Any, Dict, List, Optional, Union from mashumaro.types import SerializableType from dbt.artifacts.resources.base import FileHash from dbt.constants import MAXIMUM_SEED_SIZE from dbt_common.dataclass_schema import StrEnum, dbtClassMixin from .util import SourceKey class ParseFileType(StrEnum): Macro = "macro" Model = "model" Snapshot = "snapshot" Analysis = "analysis" SingularTest = "singular_test" GenericTest = "generic_test" Seed = "seed" Documentation = "docs" Schema = "schema" Hook = "hook" # not a real filetype, from dbt_project.yml Fixture = "fixture" Function = "function" parse_file_type_to_parser = { ParseFileType.Macro: "MacroParser", ParseFileType.Model: "ModelParser", ParseFileType.Snapshot: "SnapshotParser", ParseFileType.Analysis: "AnalysisParser", ParseFileType.SingularTest: "SingularTestParser", ParseFileType.GenericTest: "GenericTestParser", ParseFileType.Seed: "SeedParser", ParseFileType.Documentation: "DocumentationParser", ParseFileType.Schema: "SchemaParser", ParseFileType.Hook: "HookParser", ParseFileType.Fixture: "FixtureParser", ParseFileType.Function: "FunctionParser", } @dataclass class FilePath(dbtClassMixin): searched_path: str relative_path: str modification_time: float project_root: str @property def search_key(self) -> str: # TODO: should this be project name + path relative to project root? return self.absolute_path @property def full_path(self) -> str: # useful for symlink preservation return os.path.join(self.project_root, self.searched_path, self.relative_path) @property def absolute_path(self) -> str: return os.path.abspath(self.full_path) @property def original_file_path(self) -> str: return os.path.join(self.searched_path, self.relative_path) def seed_too_large(self) -> bool: """Return whether the file this represents is over the seed size limit""" return os.stat(self.full_path).st_size > MAXIMUM_SEED_SIZE @dataclass class RemoteFile(dbtClassMixin): def __init__(self, language) -> None: if language == "sql": self.path_end = ".sql" elif language == "python": self.path_end = ".py" else: raise RuntimeError(f"Invalid language for remote File {language}") self.path = f"from remote system{self.path_end}" @property def searched_path(self) -> str: return self.path @property def relative_path(self) -> str: return self.path @property def absolute_path(self) -> str: return self.path @property def original_file_path(self): return self.path @property def modification_time(self): return self.path @dataclass class BaseSourceFile(dbtClassMixin, SerializableType): """Define a source file in dbt""" path: Union[FilePath, RemoteFile] # the path information checksum: FileHash # Seems like knowing which project the file came from would be useful project_name: Optional[str] = None # Parse file type: i.e. which parser will process this file parse_file_type: Optional[ParseFileType] = None # we don't want to serialize this contents: Optional[str] = None @property def file_id(self): if isinstance(self.path, RemoteFile): return None return f"{self.project_name}://{self.path.original_file_path}" @property def original_file_path(self): return self.path.original_file_path def _serialize(self): dct = self.to_dict() return dct @classmethod def _deserialize(cls, dct: Dict[str, int]): if dct["parse_file_type"] == "schema": sf = SchemaSourceFile.from_dict(dct) elif dct["parse_file_type"] == "fixture": sf = FixtureSourceFile.from_dict(dct) else: sf = SourceFile.from_dict(dct) return sf def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None): dct = super().__post_serialize__(dct, context) # remove empty lists to save space dct_keys = list(dct.keys()) for key in dct_keys: if isinstance(dct[key], list) and not dct[key]: del dct[key] # remove contents. Schema files will still have 'dict_from_yaml' # from the contents if "contents" in dct: del dct["contents"] return dct @dataclass class SourceFile(BaseSourceFile): nodes: List[str] = field(default_factory=list) docs: List[str] = field(default_factory=list) macros: List[str] = field(default_factory=list) env_vars: List[str] = field(default_factory=list) functions: List[str] = field(default_factory=list) @classmethod def big_seed(cls, path: FilePath) -> "SourceFile": """Parse seeds over the size limit with just the path""" self = cls(path=path, checksum=FileHash.path(path.original_file_path)) self.contents = "" return self def add_node(self, value): if value not in self.nodes: self.nodes.append(value) # TODO: do this a different way. This remote file kludge isn't going # to work long term @classmethod def remote(cls, contents: str, project_name: str, language: str) -> "SourceFile": self = cls( path=RemoteFile(language), checksum=FileHash.from_contents(contents), project_name=project_name, contents=contents, ) return self @dataclass class SchemaSourceFile(BaseSourceFile): dfy: Dict[str, Any] = field(default_factory=dict) # these are in the manifest.nodes dictionary data_tests: Dict[str, Any] = field(default_factory=dict) sources: List[str] = field(default_factory=list) exposures: List[str] = field(default_factory=list) functions: List[str] = field(default_factory=list) metrics: List[str] = field(default_factory=list) snapshots: List[str] = field(default_factory=list) # The following field will no longer be used. Leaving # here to avoid breaking existing projects. To be removed # later if possible. generated_metrics: List[str] = field(default_factory=list) # metrics generated from semantic_model measures. The key is # the name of the semantic_model, so that we can find it later. metrics_from_measures: Dict[str, Any] = field(default_factory=dict) groups: List[str] = field(default_factory=list) # node patches contain models, seeds, snapshots, analyses ndp: List[str] = field(default_factory=list) semantic_models: List[str] = field(default_factory=list) unit_tests: List[str] = field(default_factory=list) saved_queries: List[str] = field(default_factory=list) # any macro patches in this file by macro unique_id. mcp: Dict[str, str] = field(default_factory=dict) # any source patches in this file. The entries are package, name pairs # Patches are only against external sources. Sources can be # created too, but those are in 'sources' sop: List[SourceKey] = field(default_factory=list) env_vars: Dict[str, Any] = field(default_factory=dict) unrendered_configs: Dict[str, Any] = field(default_factory=dict) unrendered_databases: Dict[str, Any] = field(default_factory=dict) unrendered_schemas: Dict[str, Any] = field(default_factory=dict) pp_dict: Optional[Dict[str, Any]] = None pp_test_index: Optional[Dict[str, Any]] = None @property def dict_from_yaml(self): return self.dfy @property def node_patches(self): return self.ndp @property def macro_patches(self): return self.mcp @property def source_patches(self): return self.sop def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None): dct = super().__post_serialize__(dct, context) # Remove partial parsing specific data for key in ("pp_test_index", "pp_dict"): if key in dct: del dct[key] return dct def append_patch(self, yaml_key, unique_id): self.node_patches.append(unique_id) def add_test(self, node_unique_id, test_from): name = test_from["name"] key = test_from["key"] if key not in self.data_tests: self.data_tests[key] = {} if name not in self.data_tests[key]: self.data_tests[key][name] = [] self.data_tests[key][name].append(node_unique_id) # this is only used in tests/unit def remove_tests(self, yaml_key, name): if yaml_key in self.data_tests: if name in self.data_tests[yaml_key]: del self.data_tests[yaml_key][name] # this is only used in the tests directory (unit + functional) def get_tests(self, yaml_key, name): if yaml_key in self.data_tests: if name in self.data_tests[yaml_key]: return self.data_tests[yaml_key][name] return [] def add_metrics_from_measures(self, semantic_model_name: str, metric_unique_id: str): if self.generated_metrics: # Probably not needed, but for safety sake, convert the # old generated_metrics to metrics_from_measures. self.fix_metrics_from_measures() if semantic_model_name not in self.metrics_from_measures: self.metrics_from_measures[semantic_model_name] = [] self.metrics_from_measures[semantic_model_name].append(metric_unique_id) def fix_metrics_from_measures(self): # Temporary method to fix up existing projects with a partial parse file. # This should only be called if SchemaSourceFile in a msgpack # pack manifest has an existing "generated_metrics" list, to turn it # it into a "metrics_from_measures" dictionary, so that we can # correctly partially parse. # This code can be removed when "generated_metrics" is removed. generated_metrics = self.generated_metrics self.generated_metrics = [] # Should never be needed again # For each metric_unique_id we loop through the semantic models # looking for the name of the "measure" which generated the metric. # When it's found, add it to "metrics_from_measures", with a key # of the semantic_model name, and a list of metrics. for metric_unique_id in generated_metrics: parts = metric_unique_id.split(".") # get the metric_name metric_name = parts[-1] if "semantic_models" in self.dict_from_yaml: for sem_model in self.dict_from_yaml["semantic_models"]: if "measures" in sem_model: for measure in sem_model["measures"]: if measure["name"] == metric_name: self.add_metrics_from_measures(sem_model["name"], metric_unique_id) break def get_key_and_name_for_test(self, test_unique_id): yaml_key = None block_name = None for key in self.data_tests.keys(): for name in self.data_tests[key]: for unique_id in self.data_tests[key][name]: if unique_id == test_unique_id: yaml_key = key block_name = name break return (yaml_key, block_name) def get_all_test_ids(self): test_ids = [] for key in self.data_tests.keys(): for name in self.data_tests[key]: test_ids.extend(self.data_tests[key][name]) return test_ids def add_unrendered_config(self, unrendered_config, yaml_key, name, version=None): versioned_name = f"{name}_v{version}" if version is not None else name if yaml_key not in self.unrendered_configs: self.unrendered_configs[yaml_key] = {} if versioned_name not in self.unrendered_configs[yaml_key]: self.unrendered_configs[yaml_key][versioned_name] = unrendered_config def get_unrendered_config(self, yaml_key, name, version=None) -> Optional[Dict[str, Any]]: versioned_name = f"{name}_v{version}" if version is not None else name if yaml_key not in self.unrendered_configs: return None if versioned_name not in self.unrendered_configs[yaml_key]: return None return self.unrendered_configs[yaml_key][versioned_name] def delete_from_unrendered_configs(self, yaml_key, name): # We delete all unrendered_configs for this yaml_key/name because the # entry has been scheduled for reparsing. if self.get_unrendered_config(yaml_key, name): del self.unrendered_configs[yaml_key][name] # Delete all versioned keys associated with name version_names_to_delete = [] for potential_version_name in self.unrendered_configs[yaml_key]: if potential_version_name.startswith(f"{name}_v"): version_names_to_delete.append(potential_version_name) for version_name in version_names_to_delete: del self.unrendered_configs[yaml_key][version_name] if not self.unrendered_configs[yaml_key]: del self.unrendered_configs[yaml_key] def add_env_var(self, var, yaml_key, name): if yaml_key not in self.env_vars: self.env_vars[yaml_key] = {} if name not in self.env_vars[yaml_key]: self.env_vars[yaml_key][name] = [] if var not in self.env_vars[yaml_key][name]: self.env_vars[yaml_key][name].append(var) def delete_from_env_vars(self, yaml_key, name): # We delete all vars for this yaml_key/name because the # entry has been scheduled for reparsing. if yaml_key in self.env_vars and name in self.env_vars[yaml_key]: del self.env_vars[yaml_key][name] if not self.env_vars[yaml_key]: del self.env_vars[yaml_key] def add_unrendered_database(self, yaml_key: str, name: str, unrendered_database: str) -> None: if yaml_key not in self.unrendered_databases: self.unrendered_databases[yaml_key] = {} self.unrendered_databases[yaml_key][name] = unrendered_database def get_unrendered_database(self, yaml_key: str, name: str) -> Optional[str]: if yaml_key not in self.unrendered_databases: return None return self.unrendered_databases[yaml_key].get(name) def add_unrendered_schema(self, yaml_key: str, name: str, unrendered_schema: str) -> None: if yaml_key not in self.unrendered_schemas: self.unrendered_schemas[yaml_key] = {} self.unrendered_schemas[yaml_key][name] = unrendered_schema def get_unrendered_schema(self, yaml_key: str, name: str) -> Optional[str]: if yaml_key not in self.unrendered_schemas: return None return self.unrendered_schemas[yaml_key].get(name) @dataclass class FixtureSourceFile(BaseSourceFile): fixture: Optional[str] = None unit_tests: List[str] = field(default_factory=list) def add_unit_test(self, value): if value not in self.unit_tests: self.unit_tests.append(value) AnySourceFile = Union[SchemaSourceFile, SourceFile, FixtureSourceFile] ================================================ FILE: core/dbt/contracts/graph/__init__.py ================================================ ================================================ FILE: core/dbt/contracts/graph/manifest.py ================================================ import enum from collections import defaultdict from dataclasses import dataclass, field, replace from itertools import chain from multiprocessing.synchronize import Lock from typing import ( Any, Callable, ClassVar, DefaultDict, Dict, Generic, List, Mapping, MutableMapping, Optional, Set, Tuple, TypeVar, Union, ) from typing_extensions import Protocol import dbt_common.exceptions import dbt_common.utils from dbt import deprecations, tracking from dbt.adapters.exceptions import ( DuplicateMacroInPackageError, DuplicateMaterializationNameError, ) from dbt.adapters.factory import get_adapter_package_names # to preserve import paths from dbt.artifacts.resources import ( BaseResource, DeferFunction, DeferRelation, NodeConfig, NodeVersion, RefArgs, ) from dbt.artifacts.schemas.manifest import ManifestMetadata, UniqueID, WritableManifest from dbt.clients.jinja_static import statically_parse_ref_or_source from dbt.contracts.files import ( AnySourceFile, FileHash, FixtureSourceFile, SchemaSourceFile, SourceFile, ) from dbt.contracts.graph.nodes import ( RESOURCE_CLASS_TO_NODE_CLASS, BaseNode, Documentation, Exposure, FunctionNode, GenericTestNode, GraphMemberNode, Group, Macro, ManifestNode, Metric, ModelNode, SavedQuery, SeedNode, SemanticModel, SingularTestNode, SnapshotNode, SourceDefinition, UnitTestDefinition, UnitTestFileFixture, UnpatchedSourceDefinition, ) from dbt.contracts.graph.unparsed import SourcePatch, UnparsedVersion from dbt.contracts.util import SourceKey from dbt.events.types import ArtifactWritten, UnpinnedRefNewVersionAvailable from dbt.exceptions import ( AmbiguousResourceNameRefError, CompilationError, DuplicateResourceNameError, ) from dbt.flags import get_flags from dbt.mp_context import get_mp_context from dbt.node_types import ( REFABLE_NODE_TYPES, VERSIONED_NODE_TYPES, AccessType, NodeType, ) from dbt_common.dataclass_schema import dbtClassMixin from dbt_common.events.contextvars import get_node_info from dbt_common.events.functions import fire_event from dbt_common.helper_types import PathSet PackageName = str DocName = str RefName = str def find_unique_id_for_package(storage, key, package: Optional[PackageName]) -> Optional[UniqueID]: if key not in storage: return None pkg_dct: Mapping[PackageName, UniqueID] = storage[key] if package is None: if not pkg_dct: return None else: return next(iter(pkg_dct.values())) elif package in pkg_dct: return pkg_dct[package] else: return None class DocLookup(dbtClassMixin): def __init__(self, manifest: "Manifest") -> None: self.storage: Dict[str, Dict[PackageName, UniqueID]] = {} self.populate(manifest) def get_unique_id(self, key, package: Optional[PackageName]): return find_unique_id_for_package(self.storage, key, package) def find(self, key, package: Optional[PackageName], manifest: "Manifest"): unique_id = self.get_unique_id(key, package) if unique_id is not None: return self.perform_lookup(unique_id, manifest) return None def add_doc(self, doc: Documentation): if doc.name not in self.storage: self.storage[doc.name] = {} self.storage[doc.name][doc.package_name] = doc.unique_id def populate(self, manifest): for doc in manifest.docs.values(): self.add_doc(doc) def perform_lookup(self, unique_id: UniqueID, manifest) -> Documentation: if unique_id not in manifest.docs: raise dbt_common.exceptions.DbtInternalError( f"Doc {unique_id} found in cache but not found in manifest" ) return manifest.docs[unique_id] class SourceLookup(dbtClassMixin): def __init__(self, manifest: "Manifest") -> None: self.storage: Dict[str, Dict[PackageName, UniqueID]] = {} self.populate(manifest) def get_unique_id(self, search_name, package: Optional[PackageName]): return find_unique_id_for_package(self.storage, search_name, package) def find(self, search_name, package: Optional[PackageName], manifest: "Manifest"): unique_id = self.get_unique_id(search_name, package) if unique_id is not None: return self.perform_lookup(unique_id, manifest) return None def add_source(self, source: SourceDefinition): if source.search_name not in self.storage: self.storage[source.search_name] = {} self.storage[source.search_name][source.package_name] = source.unique_id def populate(self, manifest): for source in manifest.sources.values(): if hasattr(source, "source_name"): self.add_source(source) def perform_lookup(self, unique_id: UniqueID, manifest: "Manifest") -> SourceDefinition: if unique_id not in manifest.sources: raise dbt_common.exceptions.DbtInternalError( f"Source {unique_id} found in cache but not found in manifest" ) return manifest.sources[unique_id] class FunctionLookup(dbtClassMixin): def __init__(self, manifest: "Manifest") -> None: self.storage: Dict[str, Dict[PackageName, UniqueID]] = {} self.populate(manifest) def get_unique_id(self, search_name, package: Optional[PackageName]): return find_unique_id_for_package(self.storage, search_name, package) def find(self, search_name, package: Optional[PackageName], manifest: "Manifest"): unique_id = self.get_unique_id(search_name, package) if unique_id is not None: return self.perform_lookup(unique_id, manifest) return None def add_function(self, function: FunctionNode): if function.search_name not in self.storage: self.storage[function.search_name] = {} self.storage[function.search_name][function.package_name] = function.unique_id def populate(self, manifest): for function in manifest.functions.values(): if hasattr(function, "name"): self.add_function(function) def perform_lookup(self, unique_id: UniqueID, manifest: "Manifest") -> FunctionNode: if unique_id not in manifest.functions: raise dbt_common.exceptions.DbtInternalError( f"Function {unique_id} found in cache but not found in manifest" ) return manifest.functions[unique_id] class RefableLookup(dbtClassMixin): # model, seed, snapshot, function _lookup_types: ClassVar[set] = set(REFABLE_NODE_TYPES) _versioned_types: ClassVar[set] = set(VERSIONED_NODE_TYPES) def __init__(self, manifest: "Manifest") -> None: self.storage: Dict[str, Dict[PackageName, UniqueID]] = {} self.populate(manifest) def get_unique_id( self, key: str, package: Optional[PackageName], version: Optional[NodeVersion], node: Optional[GraphMemberNode] = None, ): if version: key = f"{key}.v{version}" unique_ids = self._find_unique_ids_for_package(key, package) if len(unique_ids) > 1: raise AmbiguousResourceNameRefError(key, unique_ids, node) else: return unique_ids[0] if unique_ids else None def find( self, key: str, package: Optional[PackageName], version: Optional[NodeVersion], manifest: "Manifest", source_node: Optional[GraphMemberNode] = None, ): unique_id = self.get_unique_id(key, package, version, source_node) if unique_id is not None: node = self.perform_lookup(unique_id, manifest) # If this is an unpinned ref (no 'version' arg was passed), # AND this is a versioned node, # AND this ref is being resolved at runtime -- get_node_info != {} # Only ModelNodes can be versioned. if ( isinstance(node, ModelNode) and version is None and node.is_versioned and get_node_info() ): # Check to see if newer versions are available, and log an "FYI" if so max_version: UnparsedVersion = max( [ UnparsedVersion(v.version) for v in manifest.nodes.values() if isinstance(v, ModelNode) and v.name == node.name and v.version is not None ] ) assert node.latest_version is not None # for mypy, whenever i may find it if max_version > UnparsedVersion(node.latest_version): fire_event( UnpinnedRefNewVersionAvailable( node_info=get_node_info(), ref_node_name=node.name, ref_node_package=node.package_name, ref_node_version=str(node.version), ref_max_version=str(max_version.v), ) ) return node return None def add_node(self, node: ManifestNode): if node.resource_type in self._lookup_types: if node.name not in self.storage: self.storage[node.name] = {} if node.is_versioned: if node.search_name not in self.storage: self.storage[node.search_name] = {} self.storage[node.search_name][node.package_name] = node.unique_id if node.is_latest_version: # type: ignore self.storage[node.name][node.package_name] = node.unique_id else: self.storage[node.name][node.package_name] = node.unique_id def populate(self, manifest): for node in manifest.nodes.values(): self.add_node(node) def perform_lookup(self, unique_id: UniqueID, manifest) -> ManifestNode: if unique_id in manifest.nodes: node = manifest.nodes[unique_id] else: raise dbt_common.exceptions.DbtInternalError( f"Node {unique_id} found in cache but not found in manifest" ) return node def _find_unique_ids_for_package(self, key, package: Optional[PackageName]) -> List[str]: if key not in self.storage: return [] pkg_dct: Mapping[PackageName, UniqueID] = self.storage[key] if package is None: if not pkg_dct: return [] else: return list(pkg_dct.values()) elif package in pkg_dct: return [pkg_dct[package]] else: return [] class MetricLookup(dbtClassMixin): def __init__(self, manifest: "Manifest") -> None: self.storage: Dict[str, Dict[PackageName, UniqueID]] = {} self.populate(manifest) def get_unique_id(self, search_name, package: Optional[PackageName]): return find_unique_id_for_package(self.storage, search_name, package) def find(self, search_name, package: Optional[PackageName], manifest: "Manifest"): unique_id = self.get_unique_id(search_name, package) if unique_id is not None: return self.perform_lookup(unique_id, manifest) return None def add_metric(self, metric: Metric): if metric.search_name not in self.storage: self.storage[metric.search_name] = {} self.storage[metric.search_name][metric.package_name] = metric.unique_id def populate(self, manifest): for metric in manifest.metrics.values(): if hasattr(metric, "name"): self.add_metric(metric) def perform_lookup(self, unique_id: UniqueID, manifest: "Manifest") -> Metric: if unique_id not in manifest.metrics: raise dbt_common.exceptions.DbtInternalError( f"Metric {unique_id} found in cache but not found in manifest" ) return manifest.metrics[unique_id] class SavedQueryLookup(dbtClassMixin): """Lookup utility for finding SavedQuery nodes""" def __init__(self, manifest: "Manifest") -> None: self.storage: Dict[str, Dict[PackageName, UniqueID]] = {} self.populate(manifest) def get_unique_id(self, search_name, package: Optional[PackageName]): return find_unique_id_for_package(self.storage, search_name, package) def find(self, search_name, package: Optional[PackageName], manifest: "Manifest"): unique_id = self.get_unique_id(search_name, package) if unique_id is not None: return self.perform_lookup(unique_id, manifest) return None def add_saved_query(self, saved_query: SavedQuery): if saved_query.search_name not in self.storage: self.storage[saved_query.search_name] = {} self.storage[saved_query.search_name][saved_query.package_name] = saved_query.unique_id def populate(self, manifest): for saved_query in manifest.saved_queries.values(): if hasattr(saved_query, "name"): self.add_saved_query(saved_query) def perform_lookup(self, unique_id: UniqueID, manifest: "Manifest") -> SavedQuery: if unique_id not in manifest.saved_queries: raise dbt_common.exceptions.DbtInternalError( f"SavedQUery {unique_id} found in cache but not found in manifest" ) return manifest.saved_queries[unique_id] class SemanticModelByMeasureLookup(dbtClassMixin): """Lookup utility for finding SemanticModel by measure This is possible because measure names are supposed to be unique across the semantic models in a manifest. """ def __init__(self, manifest: "Manifest") -> None: self.storage: DefaultDict[str, Dict[PackageName, UniqueID]] = defaultdict(dict) self.populate(manifest) def get_unique_id(self, search_name: str, package: Optional[PackageName]): return find_unique_id_for_package(self.storage, search_name, package) def find( self, search_name: str, package: Optional[PackageName], manifest: "Manifest" ) -> Optional[SemanticModel]: """Tries to find a SemanticModel based on a measure name""" unique_id = self.get_unique_id(search_name, package) if unique_id is not None: return self.perform_lookup(unique_id, manifest) return None def add(self, semantic_model: SemanticModel): """Sets all measures for a SemanticModel as paths to the SemanticModel's `unique_id`""" for measure in semantic_model.measures: self.storage[measure.name][semantic_model.package_name] = semantic_model.unique_id def populate(self, manifest: "Manifest"): """Populate storage with all the measure + package paths to the Manifest's SemanticModels""" for semantic_model in manifest.semantic_models.values(): self.add(semantic_model=semantic_model) for disabled in manifest.disabled.values(): for node in disabled: if isinstance(node, SemanticModel): self.add(semantic_model=node) def perform_lookup(self, unique_id: UniqueID, manifest: "Manifest") -> SemanticModel: """Tries to get a SemanticModel from the Manifest""" enabled_semantic_model: Optional[SemanticModel] = manifest.semantic_models.get(unique_id) disabled_semantic_model: Optional[List] = manifest.disabled.get(unique_id) if isinstance(enabled_semantic_model, SemanticModel): return enabled_semantic_model elif disabled_semantic_model is not None and isinstance( disabled_semantic_model[0], SemanticModel ): return disabled_semantic_model[0] else: raise dbt_common.exceptions.DbtInternalError( f"Semantic model `{unique_id}` found in cache but not found in manifest" ) # This handles both models/seeds/snapshots and sources/metrics/exposures/semantic_models class DisabledLookup(dbtClassMixin): def __init__(self, manifest: "Manifest") -> None: self.storage: Dict[str, Dict[PackageName, List[Any]]] = {} self.populate(manifest) def populate(self, manifest: "Manifest"): for node in list(chain.from_iterable(manifest.disabled.values())): self.add_node(node) def add_node(self, node: GraphMemberNode) -> None: if node.search_name not in self.storage: self.storage[node.search_name] = {} if node.package_name not in self.storage[node.search_name]: self.storage[node.search_name][node.package_name] = [] self.storage[node.search_name][node.package_name].append(node) # This should return a list of disabled nodes. It's different from # the other Lookup functions in that it returns full nodes, not just unique_ids def find( self, search_name, package: Optional[PackageName], version: Optional[NodeVersion] = None, resource_types: Optional[List[NodeType]] = None, ) -> Optional[List[Any]]: if version: search_name = f"{search_name}.v{version}" if search_name not in self.storage: return None pkg_dct: Mapping[PackageName, List[Any]] = self.storage[search_name] nodes = [] if package is None: if not pkg_dct: return None else: nodes = next(iter(pkg_dct.values())) elif package in pkg_dct: nodes = pkg_dct[package] else: return None if resource_types is None: return nodes else: new_nodes = [] for node in nodes: if node.resource_type in resource_types: new_nodes.append(node) if not new_nodes: return None else: return new_nodes class AnalysisLookup(RefableLookup): _lookup_types: ClassVar[set] = set([NodeType.Analysis]) _versioned_types: ClassVar[set] = set() class SingularTestLookup(dbtClassMixin): def __init__(self, manifest: "Manifest") -> None: self.storage: Dict[str, Dict[PackageName, UniqueID]] = {} self.populate(manifest) def get_unique_id(self, search_name, package: Optional[PackageName]) -> Optional[UniqueID]: return find_unique_id_for_package(self.storage, search_name, package) def find( self, search_name, package: Optional[PackageName], manifest: "Manifest" ) -> Optional[SingularTestNode]: unique_id = self.get_unique_id(search_name, package) if unique_id is not None: return self.perform_lookup(unique_id, manifest) return None def add_singular_test(self, source: SingularTestNode) -> None: if source.search_name not in self.storage: self.storage[source.search_name] = {} self.storage[source.search_name][source.package_name] = source.unique_id def populate(self, manifest: "Manifest") -> None: for node in manifest.nodes.values(): if isinstance(node, SingularTestNode): self.add_singular_test(node) def perform_lookup(self, unique_id: UniqueID, manifest: "Manifest") -> SingularTestNode: if unique_id not in manifest.nodes: raise dbt_common.exceptions.DbtInternalError( f"Singular test {unique_id} found in cache but not found in manifest" ) node = manifest.nodes[unique_id] assert isinstance(node, SingularTestNode) return node def _packages_to_search( current_project: str, node_package: str, target_package: Optional[str] = None, ) -> List[Optional[str]]: if target_package is not None: return [target_package] elif current_project == node_package: return [current_project, None] else: if get_flags().require_ref_searches_node_package_before_root: return [node_package, current_project, None] else: return [current_project, node_package, None] def _sort_values(dct): """Given a dictionary, sort each value. This makes output deterministic, which helps for tests. """ return {k: sorted(v) for k, v in dct.items()} def build_node_edges(nodes: List[ManifestNode]): """Build the forward and backward edges on the given list of ManifestNodes and return them as two separate dictionaries, each mapping unique IDs to lists of edges. """ backward_edges: Dict[str, List[str]] = {} # pre-populate the forward edge dict for simplicity forward_edges: Dict[str, List[str]] = {n.unique_id: [] for n in nodes} for node in nodes: backward_edges[node.unique_id] = node.depends_on_nodes[:] for unique_id in backward_edges[node.unique_id]: if unique_id in forward_edges.keys(): forward_edges[unique_id].append(node.unique_id) return _sort_values(forward_edges), _sort_values(backward_edges) # Build a map of children of macros and generic tests def build_macro_edges(nodes: List[Any]): forward_edges: Dict[str, List[str]] = { n.unique_id: [] for n in nodes if n.unique_id.startswith("macro") or n.depends_on_macros } for node in nodes: for unique_id in node.depends_on_macros: if unique_id in forward_edges.keys(): forward_edges[unique_id].append(node.unique_id) return _sort_values(forward_edges) def _deepcopy(value): return value.from_dict(value.to_dict(omit_none=True)) class Locality(enum.IntEnum): Core = 1 Imported = 2 Root = 3 @dataclass class MacroCandidate: locality: Locality macro: Macro def __eq__(self, other: object) -> bool: if not isinstance(other, MacroCandidate): return NotImplemented return self.locality == other.locality def __lt__(self, other: object) -> bool: if not isinstance(other, MacroCandidate): return NotImplemented if self.locality < other.locality: return True if self.locality > other.locality: return False return False @dataclass class MaterializationCandidate(MacroCandidate): # specificity describes where in the inheritance chain this materialization candidate is # a specificity of 0 means a materialization defined by the current adapter # the highest the specificity describes a default materialization. the value itself depends on # how many adapters there are in the inheritance chain specificity: int @classmethod def from_macro(cls, candidate: MacroCandidate, specificity: int) -> "MaterializationCandidate": return cls( locality=candidate.locality, macro=candidate.macro, specificity=specificity, ) def __eq__(self, other: object) -> bool: if not isinstance(other, MaterializationCandidate): return NotImplemented equal = self.specificity == other.specificity and self.locality == other.locality if equal: raise DuplicateMaterializationNameError(self.macro, other) return equal def __lt__(self, other: object) -> bool: if not isinstance(other, MaterializationCandidate): return NotImplemented if self.specificity > other.specificity: return True if self.specificity < other.specificity: return False if self.locality < other.locality: return True if self.locality > other.locality: return False return False M = TypeVar("M", bound=MacroCandidate) class CandidateList(List[M]): def last_candidate( self, valid_localities: Optional[List[Locality]] = None ) -> Optional[MacroCandidate]: """ Obtain the last (highest precedence) MacroCandidate from the CandidateList of any locality in valid_localities. If valid_localities is not specified, return the last MacroCandidate of any locality. """ if not self: return None self.sort() if valid_localities is None: return self[-1] for candidate in reversed(self): if candidate.locality in valid_localities: return candidate return None def last(self) -> Optional[Macro]: last_candidate = self.last_candidate() return last_candidate.macro if last_candidate is not None else None def _get_locality(macro: Macro, root_project_name: str, internal_packages: Set[str]) -> Locality: if macro.package_name == root_project_name: return Locality.Root elif macro.package_name in internal_packages: return Locality.Core else: return Locality.Imported class Searchable(Protocol): resource_type: NodeType package_name: str @property def search_name(self) -> str: raise NotImplementedError("search_name not implemented") D = TypeVar("D") @dataclass class Disabled(Generic[D]): target: D MaybeFunctionNode = Optional[Union[FunctionNode, Disabled[FunctionNode]]] MaybeMetricNode = Optional[Union[Metric, Disabled[Metric]]] MaybeSavedQueryNode = Optional[Union[SavedQuery, Disabled[SavedQuery]]] MaybeDocumentation = Optional[Documentation] MaybeParsedSource = Optional[ Union[ SourceDefinition, Disabled[SourceDefinition], ] ] MaybeNonSource = Optional[Union[ManifestNode, Disabled[ManifestNode]]] T = TypeVar("T", bound=GraphMemberNode) # This contains macro methods that are in both the Manifest # and the MacroManifest class MacroMethods: # Just to make mypy happy. There must be a better way. def __init__(self): self.macros = [] self.metadata = {} self._macros_by_name = {} self._macros_by_package = {} def find_macro_candidate_by_name( self, name: str, root_project_name: str, package: Optional[str] ) -> Optional[MacroCandidate]: """Find a MacroCandidate in the graph by its name and package name, or None for any package. The root project name is used to determine priority: - locally defined macros come first - then imported macros - then macros defined in the root project """ filter: Optional[Callable[[MacroCandidate], bool]] = None if package is not None: def filter(candidate: MacroCandidate) -> bool: return package == candidate.macro.package_name candidates: CandidateList = self._find_macros_by_name( name=name, root_project_name=root_project_name, filter=filter, ) return candidates.last_candidate() def find_macro_by_name( self, name: str, root_project_name: str, package: Optional[str] ) -> Optional[Macro]: macro_candidate = self.find_macro_candidate_by_name( name=name, root_project_name=root_project_name, package=package ) return macro_candidate.macro if macro_candidate else None def find_generate_macro_by_name( self, component: str, root_project_name: str, imported_package: Optional[str] = None ) -> Optional[Macro]: """ The default `generate_X_name` macros are similar to regular ones, but only includes imported packages when searching for a package. - if package is not provided: - if there is a `generate_{component}_name` macro in the root project, return it - return the `generate_{component}_name` macro from the 'dbt' internal project - if package is provided - return the `generate_{component}_name` macro from the imported package, if one exists """ def filter(candidate: MacroCandidate) -> bool: if imported_package: return ( candidate.locality == Locality.Imported and imported_package == candidate.macro.package_name ) else: return candidate.locality != Locality.Imported candidates: CandidateList = self._find_macros_by_name( name=f"generate_{component}_name", root_project_name=root_project_name, filter=filter, ) return candidates.last() def _find_macros_by_name( self, name: str, root_project_name: str, filter: Optional[Callable[[MacroCandidate], bool]] = None, ) -> CandidateList: """Find macros by their name.""" candidates: CandidateList = CandidateList() macros_by_name = self.get_macros_by_name() if name not in macros_by_name: return candidates packages = set(get_adapter_package_names(self.metadata.adapter_type)) for macro in macros_by_name[name]: candidate = MacroCandidate( locality=_get_locality(macro, root_project_name, packages), macro=macro, ) if filter is None or filter(candidate): candidates.append(candidate) return candidates def get_macros_by_name(self) -> Dict[str, List[Macro]]: if self._macros_by_name is None: # The by-name mapping doesn't exist yet (perhaps because the manifest # was deserialized), so we build it. self._macros_by_name = self._build_macros_by_name(self.macros) return self._macros_by_name @staticmethod def _build_macros_by_name(macros: Mapping[str, Macro]) -> Dict[str, List[Macro]]: # Convert a macro dictionary keyed on unique id to a flattened version # keyed on macro name for faster lookup by name. Since macro names are # not necessarily unique, the dict value is a list. macros_by_name: Dict[str, List[Macro]] = {} for macro in macros.values(): if macro.name not in macros_by_name: macros_by_name[macro.name] = [] macros_by_name[macro.name].append(macro) return macros_by_name def get_macros_by_package(self) -> Dict[str, Dict[str, Macro]]: if self._macros_by_package is None: # The by-package mapping doesn't exist yet (perhaps because the manifest # was deserialized), so we build it. self._macros_by_package = self._build_macros_by_package(self.macros) return self._macros_by_package @staticmethod def _build_macros_by_package(macros: Mapping[str, Macro]) -> Dict[str, Dict[str, Macro]]: # Convert a macro dictionary keyed on unique id to a flattened version # keyed on package name for faster lookup by name. macros_by_package: Dict[str, Dict[str, Macro]] = {} for macro in macros.values(): if macro.package_name not in macros_by_package: macros_by_package[macro.package_name] = {} macros_by_name = macros_by_package[macro.package_name] macros_by_name[macro.name] = macro return macros_by_package @dataclass class ParsingInfo: static_analysis_parsed_path_count: int = 0 static_analysis_path_count: int = 0 @dataclass class ManifestStateCheck(dbtClassMixin): vars_hash: FileHash = field(default_factory=FileHash.empty) project_env_vars_hash: FileHash = field(default_factory=FileHash.empty) profile_env_vars_hash: FileHash = field(default_factory=FileHash.empty) profile_hash: FileHash = field(default_factory=FileHash.empty) project_hashes: MutableMapping[str, FileHash] = field(default_factory=dict) NodeClassT = TypeVar("NodeClassT", bound="BaseNode") ResourceClassT = TypeVar("ResourceClassT", bound="BaseResource") @dataclass class Manifest(MacroMethods, dbtClassMixin): """The manifest for the full graph, after parsing and during compilation.""" # These attributes are both positional and by keyword. If an attribute # is added it must all be added in the __reduce_ex__ method in the # args tuple in the right position. nodes: MutableMapping[str, ManifestNode] = field(default_factory=dict) sources: MutableMapping[str, SourceDefinition] = field(default_factory=dict) macros: MutableMapping[str, Macro] = field(default_factory=dict) docs: MutableMapping[str, Documentation] = field(default_factory=dict) exposures: MutableMapping[str, Exposure] = field(default_factory=dict) functions: MutableMapping[str, FunctionNode] = field(default_factory=dict) metrics: MutableMapping[str, Metric] = field(default_factory=dict) groups: MutableMapping[str, Group] = field(default_factory=dict) selectors: MutableMapping[str, Any] = field(default_factory=dict) files: MutableMapping[str, AnySourceFile] = field(default_factory=dict) metadata: ManifestMetadata = field(default_factory=ManifestMetadata) flat_graph: Dict[str, Any] = field(default_factory=dict) state_check: ManifestStateCheck = field(default_factory=ManifestStateCheck) source_patches: MutableMapping[SourceKey, SourcePatch] = field(default_factory=dict) disabled: MutableMapping[str, List[GraphMemberNode]] = field(default_factory=dict) env_vars: MutableMapping[str, str] = field(default_factory=dict) semantic_models: MutableMapping[str, SemanticModel] = field(default_factory=dict) unit_tests: MutableMapping[str, UnitTestDefinition] = field(default_factory=dict) saved_queries: MutableMapping[str, SavedQuery] = field(default_factory=dict) fixtures: MutableMapping[str, UnitTestFileFixture] = field(default_factory=dict) _doc_lookup: Optional[DocLookup] = field( default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None} ) _source_lookup: Optional[SourceLookup] = field( default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None} ) _ref_lookup: Optional[RefableLookup] = field( default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None} ) _metric_lookup: Optional[MetricLookup] = field( default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None} ) _saved_query_lookup: Optional[SavedQueryLookup] = field( default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None} ) _semantic_model_by_measure_lookup: Optional[SemanticModelByMeasureLookup] = field( default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None} ) _disabled_lookup: Optional[DisabledLookup] = field( default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None} ) _analysis_lookup: Optional[AnalysisLookup] = field( default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None} ) _singular_test_lookup: Optional[SingularTestLookup] = field( default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None} ) _function_lookup: Optional[FunctionLookup] = field( default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None} ) _parsing_info: ParsingInfo = field( default_factory=ParsingInfo, metadata={"serialize": lambda x: None, "deserialize": lambda x: None}, ) _lock: Lock = field( default_factory=get_mp_context().Lock, metadata={"serialize": lambda x: None, "deserialize": lambda x: None}, ) _macros_by_name: Optional[Dict[str, List[Macro]]] = field( default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None}, ) _macros_by_package: Optional[Dict[str, Dict[str, Macro]]] = field( default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None}, ) def __pre_serialize__(self, context: Optional[Dict] = None): # serialization won't work with anything except an empty source_patches because # tuple keys are not supported, so ensure it's empty self.source_patches = {} return self @classmethod def __post_deserialize__(cls, obj): obj._lock = get_mp_context().Lock() return obj def build_flat_graph(self): """This attribute is used in context.common by each node, so we want to only build it once and avoid any concurrency issues around it. Make sure you don't call this until you're done with building your manifest! """ self.flat_graph = { "exposures": {k: v.to_dict(omit_none=False) for k, v in self.exposures.items()}, "functions": {k: v.to_dict(omit_none=False) for k, v in self.functions.items()}, "groups": {k: v.to_dict(omit_none=False) for k, v in self.groups.items()}, "metrics": {k: v.to_dict(omit_none=False) for k, v in self.metrics.items()}, "nodes": {k: v.to_dict(omit_none=False) for k, v in self.nodes.items()}, "sources": {k: v.to_dict(omit_none=False) for k, v in self.sources.items()}, "semantic_models": { k: v.to_dict(omit_none=False) for k, v in self.semantic_models.items() }, "saved_queries": { k: v.to_dict(omit_none=False) for k, v in self.saved_queries.items() }, "unit_tests": {k: v.to_dict(omit_none=False) for k, v in self.unit_tests.items()}, } def build_disabled_by_file_id(self): disabled_by_file_id = {} for node_list in self.disabled.values(): for node in node_list: disabled_by_file_id[node.file_id] = node return disabled_by_file_id def _get_parent_adapter_types(self, adapter_type: str) -> List[str]: # This is duplicated logic from core/dbt/context/providers.py # Ideally this would instead be incorporating actual dispatch logic from dbt.adapters.factory import get_adapter_type_names # order matters for dispatch: # 1. current adapter # 2. any parent adapters (dependencies) # 3. 'default' return get_adapter_type_names(adapter_type) + ["default"] def _materialization_candidates_for( self, project_name: str, materialization_name: str, adapter_type: str, specificity: int, ) -> CandidateList: full_name = dbt_common.utils.get_materialization_macro_name( materialization_name=materialization_name, adapter_type=adapter_type, with_prefix=False, ) return CandidateList( MaterializationCandidate.from_macro(m, specificity) for m in self._find_macros_by_name(full_name, project_name) ) def find_materialization_macro_by_name( self, project_name: str, materialization_name: str, adapter_type: str ) -> Optional[Macro]: candidates: CandidateList = CandidateList( chain.from_iterable( self._materialization_candidates_for( project_name=project_name, materialization_name=materialization_name, adapter_type=atype, specificity=specificity, # where in the inheritance chain this candidate is ) for specificity, atype in enumerate(self._get_parent_adapter_types(adapter_type)) ) ) core_candidates = [ candidate for candidate in candidates if candidate.locality == Locality.Core ] materialization_candidate = candidates.last_candidate() # If an imported materialization macro was found that also had a core candidate, fire a deprecation if ( materialization_candidate is not None and materialization_candidate.locality == Locality.Imported and core_candidates ): # preserve legacy behaviour - allow materialization override if ( get_flags().require_explicit_package_overrides_for_builtin_materializations is False ): deprecations.warn( "package-materialization-override", package_name=materialization_candidate.macro.package_name, materialization_name=materialization_name, ) else: materialization_candidate = candidates.last_candidate( valid_localities=[Locality.Core, Locality.Root] ) return materialization_candidate.macro if materialization_candidate else None def get_resource_fqns(self) -> Mapping[str, PathSet]: resource_fqns: Dict[str, Set[Tuple[str, ...]]] = {} all_resources = chain( self.exposures.values(), self.functions.values(), self.nodes.values(), self.sources.values(), self.metrics.values(), self.semantic_models.values(), self.saved_queries.values(), self.unit_tests.values(), ) for resource in all_resources: resource_type_plural = resource.resource_type.pluralize() if resource_type_plural not in resource_fqns: resource_fqns[resource_type_plural] = set() resource_fqns[resource_type_plural].add(tuple(resource.fqn)) return resource_fqns def get_used_schemas(self, resource_types=None): return frozenset( { (node.database, node.schema) for node in chain(self.nodes.values(), self.sources.values()) if not resource_types or node.resource_type in resource_types } ) def get_used_databases(self): return frozenset(x.database for x in chain(self.nodes.values(), self.sources.values())) def deepcopy(self): copy = Manifest( nodes={k: _deepcopy(v) for k, v in self.nodes.items()}, sources={k: _deepcopy(v) for k, v in self.sources.items()}, macros={k: _deepcopy(v) for k, v in self.macros.items()}, docs={k: _deepcopy(v) for k, v in self.docs.items()}, exposures={k: _deepcopy(v) for k, v in self.exposures.items()}, functions={k: _deepcopy(v) for k, v in self.functions.items()}, metrics={k: _deepcopy(v) for k, v in self.metrics.items()}, groups={k: _deepcopy(v) for k, v in self.groups.items()}, selectors={k: _deepcopy(v) for k, v in self.selectors.items()}, metadata=self.metadata, disabled={k: _deepcopy(v) for k, v in self.disabled.items()}, files={k: _deepcopy(v) for k, v in self.files.items()}, state_check=_deepcopy(self.state_check), semantic_models={k: _deepcopy(v) for k, v in self.semantic_models.items()}, unit_tests={k: _deepcopy(v) for k, v in self.unit_tests.items()}, saved_queries={k: _deepcopy(v) for k, v in self.saved_queries.items()}, ) copy.build_flat_graph() return copy def build_parent_and_child_maps(self): edge_members = list( chain( self.nodes.values(), self.sources.values(), self.exposures.values(), self.functions.values(), self.metrics.values(), self.semantic_models.values(), self.saved_queries.values(), self.unit_tests.values(), ) ) forward_edges, backward_edges = build_node_edges(edge_members) self.child_map = forward_edges self.parent_map = backward_edges def build_macro_child_map(self): edge_members = list( chain( self.nodes.values(), self.macros.values(), ) ) forward_edges = build_macro_edges(edge_members) return forward_edges def build_group_map(self): groupable_nodes = list( chain( self.nodes.values(), self.saved_queries.values(), self.semantic_models.values(), self.metrics.values(), ) ) group_map = {group.name: [] for group in self.groups.values()} for node in groupable_nodes: if node.group is not None: # group updates are not included with state:modified and # by ignoring the groups that aren't in the group map we # can avoid hitting errors for groups that are not getting # updated. This is a hack but any groups that are not # valid will be caught in # parser.manifest.ManifestLoader.check_valid_group_config_node if node.group in group_map: group_map[node.group].append(node.unique_id) self.group_map = group_map def fill_tracking_metadata(self): self.metadata.user_id = tracking.active_user.id if tracking.active_user else None self.metadata.send_anonymous_usage_stats = get_flags().SEND_ANONYMOUS_USAGE_STATS @classmethod def from_writable_manifest(cls, writable_manifest: WritableManifest) -> "Manifest": manifest = Manifest( nodes=cls._map_resources_to_map_nodes(writable_manifest.nodes), disabled=cls._map_list_resources_to_map_list_nodes(writable_manifest.disabled), unit_tests=cls._map_resources_to_map_nodes(writable_manifest.unit_tests), sources=cls._map_resources_to_map_nodes(writable_manifest.sources), macros=cls._map_resources_to_map_nodes(writable_manifest.macros), docs=cls._map_resources_to_map_nodes(writable_manifest.docs), exposures=cls._map_resources_to_map_nodes(writable_manifest.exposures), functions=cls._map_resources_to_map_nodes(writable_manifest.functions), metrics=cls._map_resources_to_map_nodes(writable_manifest.metrics), groups=cls._map_resources_to_map_nodes(writable_manifest.groups), semantic_models=cls._map_resources_to_map_nodes(writable_manifest.semantic_models), saved_queries=cls._map_resources_to_map_nodes(writable_manifest.saved_queries), selectors={ selector_id: selector for selector_id, selector in writable_manifest.selectors.items() }, metadata=writable_manifest.metadata, ) return manifest def _map_nodes_to_map_resources(cls, nodes_map: MutableMapping[str, NodeClassT]): return {node_id: node.to_resource() for node_id, node in nodes_map.items()} def _map_list_nodes_to_map_list_resources( cls, nodes_map: MutableMapping[str, List[NodeClassT]] ): return { node_id: [node.to_resource() for node in node_list] for node_id, node_list in nodes_map.items() } @classmethod def _map_resources_to_map_nodes(cls, resources_map: Mapping[str, ResourceClassT]): return { node_id: RESOURCE_CLASS_TO_NODE_CLASS[type(resource)].from_resource(resource) for node_id, resource in resources_map.items() } @classmethod def _map_list_resources_to_map_list_nodes( cls, resources_map: Optional[Mapping[str, List[ResourceClassT]]] ): if resources_map is None: return {} return { node_id: [ RESOURCE_CLASS_TO_NODE_CLASS[type(resource)].from_resource(resource) for resource in resource_list ] for node_id, resource_list in resources_map.items() } def writable_manifest(self) -> "WritableManifest": self.build_parent_and_child_maps() self.build_group_map() self.fill_tracking_metadata() return WritableManifest( nodes=self._map_nodes_to_map_resources(self.nodes), sources=self._map_nodes_to_map_resources(self.sources), macros=self._map_nodes_to_map_resources(self.macros), docs=self._map_nodes_to_map_resources(self.docs), exposures=self._map_nodes_to_map_resources(self.exposures), functions=self._map_nodes_to_map_resources(self.functions), metrics=self._map_nodes_to_map_resources(self.metrics), groups=self._map_nodes_to_map_resources(self.groups), selectors=self.selectors, metadata=self.metadata, disabled=self._map_list_nodes_to_map_list_resources(self.disabled), child_map=self.child_map, parent_map=self.parent_map, group_map=self.group_map, semantic_models=self._map_nodes_to_map_resources(self.semantic_models), unit_tests=self._map_nodes_to_map_resources(self.unit_tests), saved_queries=self._map_nodes_to_map_resources(self.saved_queries), ) def write(self, path): writable = self.writable_manifest() writable.write(path) fire_event(ArtifactWritten(artifact_type=writable.__class__.__name__, artifact_path=path)) # Called in dbt.compilation.Linker.write_graph and # dbt.graph.queue.get and ._include_in_cost def expect(self, unique_id: str) -> GraphMemberNode: if unique_id in self.nodes: return self.nodes[unique_id] elif unique_id in self.sources: return self.sources[unique_id] elif unique_id in self.exposures: return self.exposures[unique_id] elif unique_id in self.functions: return self.functions[unique_id] elif unique_id in self.metrics: return self.metrics[unique_id] elif unique_id in self.semantic_models: return self.semantic_models[unique_id] elif unique_id in self.unit_tests: return self.unit_tests[unique_id] elif unique_id in self.saved_queries: return self.saved_queries[unique_id] else: # something terrible has happened raise dbt_common.exceptions.DbtInternalError( "Expected node {} not found in manifest".format(unique_id) ) @property def doc_lookup(self) -> DocLookup: if self._doc_lookup is None: self._doc_lookup = DocLookup(self) return self._doc_lookup def rebuild_doc_lookup(self): self._doc_lookup = DocLookup(self) @property def source_lookup(self) -> SourceLookup: if self._source_lookup is None: self._source_lookup = SourceLookup(self) return self._source_lookup def rebuild_source_lookup(self): self._source_lookup = SourceLookup(self) @property def ref_lookup(self) -> RefableLookup: if self._ref_lookup is None: self._ref_lookup = RefableLookup(self) return self._ref_lookup @property def metric_lookup(self) -> MetricLookup: if self._metric_lookup is None: self._metric_lookup = MetricLookup(self) return self._metric_lookup @property def saved_query_lookup(self) -> SavedQueryLookup: """Retuns a SavedQueryLookup, instantiating it first if necessary.""" if self._saved_query_lookup is None: self._saved_query_lookup = SavedQueryLookup(self) return self._saved_query_lookup @property def semantic_model_by_measure_lookup(self) -> SemanticModelByMeasureLookup: """Gets (and creates if necessary) the lookup utility for getting SemanticModels by measures""" if self._semantic_model_by_measure_lookup is None: self._semantic_model_by_measure_lookup = SemanticModelByMeasureLookup(self) return self._semantic_model_by_measure_lookup def rebuild_ref_lookup(self): self._ref_lookup = RefableLookup(self) @property def disabled_lookup(self) -> DisabledLookup: if self._disabled_lookup is None: self._disabled_lookup = DisabledLookup(self) return self._disabled_lookup def rebuild_disabled_lookup(self): self._disabled_lookup = DisabledLookup(self) @property def analysis_lookup(self) -> AnalysisLookup: if self._analysis_lookup is None: self._analysis_lookup = AnalysisLookup(self) return self._analysis_lookup @property def singular_test_lookup(self) -> SingularTestLookup: if self._singular_test_lookup is None: self._singular_test_lookup = SingularTestLookup(self) return self._singular_test_lookup @property def function_lookup(self) -> FunctionLookup: if self._function_lookup is None: self._function_lookup = FunctionLookup(self) return self._function_lookup @property def external_node_unique_ids(self): return [node.unique_id for node in self.nodes.values() if node.is_external_node] # Called by dbt.parser.manifest._process_refs & ManifestLoader.check_for_model_deprecations def resolve_ref( self, source_node: GraphMemberNode, target_model_name: str, target_model_package: Optional[str], target_model_version: Optional[NodeVersion], current_project: str, node_package: str, ) -> MaybeNonSource: node: Optional[ManifestNode] = None disabled: Optional[List[ManifestNode]] = None candidates = _packages_to_search(current_project, node_package, target_model_package) for pkg in candidates: node = self.ref_lookup.find( target_model_name, pkg, target_model_version, self, source_node ) if node is not None and hasattr(node, "config") and node.config.enabled: return node # it's possible that the node is disabled if disabled is None: disabled = self.disabled_lookup.find( target_model_name, pkg, version=target_model_version, resource_types=REFABLE_NODE_TYPES, ) if disabled: return Disabled(disabled[0]) return None # Called by dbt.parser.manifest._resolve_sources_for_exposure # and dbt.parser.manifest._process_source_for_node def resolve_source( self, target_source_name: str, target_table_name: str, current_project: str, node_package: str, ) -> MaybeParsedSource: search_name = f"{target_source_name}.{target_table_name}" candidates = _packages_to_search(current_project, node_package) source: Optional[SourceDefinition] = None disabled: Optional[List[SourceDefinition]] = None for pkg in candidates: source = self.source_lookup.find(search_name, pkg, self) if source is not None and source.config.enabled: return source if disabled is None: disabled = self.disabled_lookup.find( f"{target_source_name}.{target_table_name}", pkg ) if disabled: return Disabled(disabled[0]) return None def resolve_function( self, target_function_name: str, target_function_package: Optional[str], current_project: str, node_package: str, ) -> MaybeFunctionNode: package_candidates = _packages_to_search( current_project, node_package, target_function_package ) disabled: Optional[List[FunctionNode]] = None for package in package_candidates: function = self.function_lookup.find(target_function_name, package, self) if function is not None and function.config.enabled: return function # it's possible that the function is disabled if disabled is None: disabled = self.disabled_lookup.find(target_function_name, package) if disabled: return Disabled(disabled[0]) return None def resolve_metric( self, target_metric_name: str, target_metric_package: Optional[str], current_project: str, node_package: str, ) -> MaybeMetricNode: metric: Optional[Metric] = None disabled: Optional[List[Metric]] = None candidates = _packages_to_search(current_project, node_package, target_metric_package) for pkg in candidates: metric = self.metric_lookup.find(target_metric_name, pkg, self) if metric is not None and metric.config.enabled: return metric # it's possible that the node is disabled if disabled is None: disabled = self.disabled_lookup.find(f"{target_metric_name}", pkg) if disabled: return Disabled(disabled[0]) return None def resolve_saved_query( self, target_saved_query_name: str, target_saved_query_package: Optional[str], current_project: str, node_package: str, ) -> MaybeSavedQueryNode: """Tries to find the SavedQuery by name within the available project and packages. Will return the first enabled SavedQuery matching the name found while iterating over the scoped packages. If no enabled SavedQuery node match is found, returns the last disabled SavedQuery node. Otherwise it returns None. """ disabled: Optional[List[SavedQuery]] = None candidates = _packages_to_search(current_project, node_package, target_saved_query_package) for pkg in candidates: saved_query = self.saved_query_lookup.find(target_saved_query_name, pkg, self) if saved_query is not None and saved_query.config.enabled: return saved_query # it's possible that the node is disabled if disabled is None: disabled = self.disabled_lookup.find(f"{target_saved_query_name}", pkg) if disabled: return Disabled(disabled[0]) return None def resolve_semantic_model_for_measure( self, target_measure_name: str, current_project: str, node_package: str, target_package: Optional[str] = None, ) -> Optional[SemanticModel]: """Tries to find the SemanticModel that a measure belongs to""" candidates = _packages_to_search(current_project, node_package, target_package) for pkg in candidates: semantic_model = self.semantic_model_by_measure_lookup.find( target_measure_name, pkg, self ) # need to return it even if it's disabled so know it's not fully missing if semantic_model is not None: return semantic_model return None # Called by DocsRuntimeContext.doc def resolve_doc( self, name: str, package: Optional[str], current_project: str, node_package: str, ) -> Optional[Documentation]: """Resolve the given documentation. This follows the same algorithm as resolve_ref except the is_enabled checks are unnecessary as docs are always enabled. """ candidates = _packages_to_search(current_project, node_package, package) for pkg in candidates: result = self.doc_lookup.find(name, pkg, self) if result is not None: return result return None def is_invalid_private_ref( self, node: GraphMemberNode, target_model: MaybeNonSource, dependencies: Optional[Mapping] ) -> bool: dependencies = dependencies or {} if not isinstance(target_model, ModelNode): return False is_private_ref = ( target_model.access == AccessType.Private # don't raise this reference error for ad hoc 'preview' queries and node.resource_type != NodeType.SqlOperation and node.resource_type != NodeType.RPCCall # TODO: rm # macros are outside the group/access system (e.g. run-operation) and node.resource_type != NodeType.Macro ) target_dependency = dependencies.get(target_model.package_name) restrict_package_access = target_dependency.restrict_access if target_dependency else False # TODO: SemanticModel and SourceDefinition do not have group, and so should not be able to make _any_ private ref. return is_private_ref and ( not hasattr(node, "group") or not node.group # Invalid reference because group does not match or node.group != target_model.group # Or, invalid because these are different namespaces (project/package) and restrict-access is enforced or (node.package_name != target_model.package_name and restrict_package_access) ) def is_invalid_protected_ref( self, node: GraphMemberNode, target_model: MaybeNonSource, dependencies: Optional[Mapping] ) -> bool: dependencies = dependencies or {} if not isinstance(target_model, ModelNode): return False is_protected_ref = ( target_model.access == AccessType.Protected # don't raise this reference error for ad hoc 'preview' queries and node.resource_type != NodeType.SqlOperation and node.resource_type != NodeType.RPCCall # TODO: rm # macros are outside the group/access system (e.g. run-operation) and node.resource_type != NodeType.Macro ) target_dependency = dependencies.get(target_model.package_name) restrict_package_access = target_dependency.restrict_access if target_dependency else False return is_protected_ref and ( node.package_name != target_model.package_name and restrict_package_access ) # Called in GraphRunnableTask.before_run, RunTask.before_run, CloneTask.before_run def merge_from_artifact(self, other: "Manifest") -> None: """Update this manifest by adding the 'defer_relation' attribute to all nodes with a counterpart in the stateful manifest used for deferral. Only non-ephemeral refable nodes are examined. Function nodes from the deferred manifest are merged so that models depending on functions can resolve them via deferral. """ refables = set(REFABLE_NODE_TYPES) for unique_id, node in other.nodes.items(): current = self.nodes.get(unique_id) if current and node.resource_type in refables and not node.is_ephemeral: assert isinstance(node.config, NodeConfig) # this makes mypy happy defer_relation = DeferRelation( database=node.database, schema=node.schema, alias=node.alias, relation_name=node.relation_name, resource_type=node.resource_type, name=node.name, description=node.description, compiled_code=(node.compiled_code if not isinstance(node, SeedNode) else None), meta=node.meta, tags=node.tags, config=node.config, ) self.nodes[unique_id] = replace(current, defer_relation=defer_relation) for unique_id, function in other.functions.items(): current = self.functions.get(unique_id) if current: defer_fn = DeferFunction( database=function.database, schema=function.schema, alias=function.alias, arguments=function.arguments, returns=function.returns, resource_type=function.resource_type, name=function.name, description=function.description, compiled_code=function.compiled_code, meta=function.meta, tags=function.tags, config=function.config, ) self.functions[unique_id] = replace(current, defer_function=defer_fn) else: self.functions[unique_id] = function # invalidate function lookup cache self._function_lookup = None # Rebuild the flat_graph, which powers the 'graph' context variable self.build_flat_graph() # Methods that were formerly in ParseResult def add_macro(self, source_file: SourceFile, macro: Macro): if macro.unique_id in self.macros: # detect that the macro exists and emit an error raise DuplicateMacroInPackageError(macro=macro, macro_mapping=self.macros) self.macros[macro.unique_id] = macro if self._macros_by_name is None: self._macros_by_name = self._build_macros_by_name(self.macros) if macro.name not in self._macros_by_name: self._macros_by_name[macro.name] = [] self._macros_by_name[macro.name].append(macro) if self._macros_by_package is None: self._macros_by_package = self._build_macros_by_package(self.macros) if macro.package_name not in self._macros_by_package: self._macros_by_package[macro.package_name] = {} self._macros_by_package[macro.package_name][macro.name] = macro source_file.macros.append(macro.unique_id) def has_file(self, source_file: SourceFile) -> bool: key = source_file.file_id if key is None: return False if key not in self.files: return False my_checksum = self.files[key].checksum return my_checksum == source_file.checksum def add_source(self, source_file: SchemaSourceFile, source: UnpatchedSourceDefinition): # sources can't be overwritten! _check_duplicates(source, self.sources) self.sources[source.unique_id] = source # type: ignore source_file.sources.append(source.unique_id) def add_node_nofile(self, node: ManifestNode): # nodes can't be overwritten! _check_duplicates(node, self.nodes) self.nodes[node.unique_id] = node def add_node(self, source_file: AnySourceFile, node: ManifestNode, test_from=None): self.add_node_nofile(node) if isinstance(source_file, SchemaSourceFile): if isinstance(node, GenericTestNode): assert test_from source_file.add_test(node.unique_id, test_from) elif isinstance(node, Metric): source_file.metrics.append(node.unique_id) elif isinstance(node, Exposure): source_file.exposures.append(node.unique_id) elif isinstance(node, Group): source_file.groups.append(node.unique_id) elif isinstance(node, SnapshotNode): source_file.snapshots.append(node.unique_id) elif isinstance(source_file, FixtureSourceFile): pass else: source_file.nodes.append(node.unique_id) def add_exposure(self, source_file: SchemaSourceFile, exposure: Exposure): _check_duplicates(exposure, self.exposures) self.exposures[exposure.unique_id] = exposure source_file.exposures.append(exposure.unique_id) def add_function(self, source_file: SourceFile, function: FunctionNode): _check_duplicates(function, self.functions) self.functions[function.unique_id] = function source_file.functions.append(function.unique_id) def add_metric( self, source_file: SchemaSourceFile, metric: Metric, generated_from: Optional[str] = None ): _check_duplicates(metric, self.metrics) self.metrics[metric.unique_id] = metric if not generated_from: source_file.metrics.append(metric.unique_id) else: source_file.add_metrics_from_measures(generated_from, metric.unique_id) def add_group(self, source_file: SchemaSourceFile, group: Group): _check_duplicates(group, self.groups) self.groups[group.unique_id] = group source_file.groups.append(group.unique_id) def add_disabled_nofile(self, node: GraphMemberNode): # There can be multiple disabled nodes for the same unique_id if node.unique_id in self.disabled: self.disabled[node.unique_id].append(node) else: self.disabled[node.unique_id] = [node] def add_disabled(self, source_file: AnySourceFile, node: GraphMemberNode, test_from=None): self.add_disabled_nofile(node) if isinstance(source_file, SchemaSourceFile): if isinstance(node, GenericTestNode): assert test_from source_file.add_test(node.unique_id, test_from) if isinstance(node, Metric): source_file.metrics.append(node.unique_id) if isinstance(node, SavedQuery): source_file.saved_queries.append(node.unique_id) if isinstance(node, SemanticModel): source_file.semantic_models.append(node.unique_id) if isinstance(node, Exposure): source_file.exposures.append(node.unique_id) if isinstance(node, FunctionNode): source_file.functions.append(node.unique_id) if isinstance(node, UnitTestDefinition): source_file.unit_tests.append(node.unique_id) elif isinstance(source_file, FixtureSourceFile): pass else: source_file.nodes.append(node.unique_id) def add_doc(self, source_file: SourceFile, doc: Documentation): _check_duplicates(doc, self.docs) self.docs[doc.unique_id] = doc source_file.docs.append(doc.unique_id) def add_semantic_model(self, source_file: SchemaSourceFile, semantic_model: SemanticModel): _check_duplicates(semantic_model, self.semantic_models) self.semantic_models[semantic_model.unique_id] = semantic_model source_file.semantic_models.append(semantic_model.unique_id) def add_unit_test(self, source_file: SchemaSourceFile, unit_test: UnitTestDefinition): if unit_test.unique_id in self.unit_tests: raise DuplicateResourceNameError(unit_test, self.unit_tests[unit_test.unique_id]) self.unit_tests[unit_test.unique_id] = unit_test source_file.unit_tests.append(unit_test.unique_id) def add_fixture(self, source_file: FixtureSourceFile, fixture: UnitTestFileFixture): if fixture.unique_id in self.fixtures: raise DuplicateResourceNameError(fixture, self.fixtures[fixture.unique_id]) self.fixtures[fixture.unique_id] = fixture source_file.fixture = fixture.unique_id def add_saved_query(self, source_file: SchemaSourceFile, saved_query: SavedQuery) -> None: _check_duplicates(saved_query, self.saved_queries) self.saved_queries[saved_query.unique_id] = saved_query source_file.saved_queries.append(saved_query.unique_id) # end of methods formerly in ParseResult def find_node_from_ref_or_source( self, expression: str ) -> Optional[Union[ModelNode, SourceDefinition]]: ref_or_source = statically_parse_ref_or_source(expression) node = None if isinstance(ref_or_source, RefArgs): node = self.ref_lookup.find( ref_or_source.name, ref_or_source.package, ref_or_source.version, self ) else: source_name, source_table_name = ref_or_source[0], ref_or_source[1] node = self.source_lookup.find(f"{source_name}.{source_table_name}", None, self) return node # Provide support for copy.deepcopy() - we just need to avoid the lock! # pickle and deepcopy use this. It returns a callable object used to # create the initial version of the object and a tuple of arguments # for the object, i.e. the Manifest. # The order of the arguments must match the order of the attributes # in the Manifest class declaration, because they are used as # positional arguments to construct a Manifest. def __reduce_ex__(self, protocol): args = ( self.nodes, self.sources, self.macros, self.docs, self.exposures, self.functions, self.metrics, self.groups, self.selectors, self.files, self.metadata, self.flat_graph, self.state_check, self.source_patches, self.disabled, self.env_vars, self.semantic_models, self.unit_tests, self.saved_queries, self._doc_lookup, self._source_lookup, self._ref_lookup, self._metric_lookup, self._semantic_model_by_measure_lookup, self._disabled_lookup, self._analysis_lookup, self._singular_test_lookup, ) return self.__class__, args def _microbatch_macro_is_core(self, project_name: str) -> bool: microbatch_is_core = False candidate = self.find_macro_candidate_by_name( name="get_incremental_microbatch_sql", root_project_name=project_name, package=None ) # We want to check for "Core", because "Core" basically means "builtin" if candidate is not None and candidate.locality == Locality.Core: microbatch_is_core = True return microbatch_is_core def use_microbatch_batches(self, project_name: str) -> bool: return ( get_flags().require_batched_execution_for_custom_microbatch_strategy or self._microbatch_macro_is_core(project_name=project_name) ) class MacroManifest(MacroMethods): def __init__(self, macros) -> None: self.macros = macros self.metadata = ManifestMetadata( user_id=tracking.active_user.id if tracking.active_user else None, send_anonymous_usage_stats=( get_flags().SEND_ANONYMOUS_USAGE_STATS if tracking.active_user else None ), ) # This is returned by the 'graph' context property # in the ProviderContext class. self.flat_graph: Dict[str, Any] = {} self._macros_by_name: Optional[Dict[str, List[Macro]]] = None self._macros_by_package: Optional[Dict[str, Dict[str, Macro]]] = None AnyManifest = Union[Manifest, MacroManifest] def _check_duplicates(value: BaseNode, src: Mapping[str, BaseNode]): if value.unique_id in src: raise DuplicateResourceNameError(value, src[value.unique_id]) K_T = TypeVar("K_T") V_T = TypeVar("V_T") def _expect_value(key: K_T, src: Mapping[K_T, V_T], old_file: SourceFile, name: str) -> V_T: if key not in src: raise CompilationError( 'Expected to find "{}" in cached "result.{}" based ' "on cached file information: {}!".format(key, name, old_file) ) return src[key] ================================================ FILE: core/dbt/contracts/graph/metrics.py ================================================ from typing import Any, Dict, Iterator, List from dbt.contracts.graph.manifest import Manifest, Metric from dbt_semantic_interfaces.type_enums import MetricType DERIVED_METRICS = [MetricType.DERIVED, MetricType.RATIO] BASE_METRICS = [MetricType.SIMPLE, MetricType.CUMULATIVE, MetricType.CONVERSION] class MetricReference(object): def __init__(self, metric_name, package_name=None) -> None: self.metric_name = metric_name self.package_name = package_name def __str__(self): return f"{self.metric_name}" class ResolvedMetricReference(MetricReference): """ Simple proxy over a Metric which delegates property lookups to the underlying node. Also adds helper functions for working with metrics (ie. __str__ and templating functions) """ def __init__(self, node: Metric, manifest: Manifest) -> None: super().__init__(node.name, node.package_name) self.node = node self.manifest = manifest def __getattr__(self, key) -> Any: return getattr(self.node, key) def __str__(self) -> str: return f"{self.node.name}" @classmethod def parent_metrics(cls, metric_node: Metric, manifest: Manifest) -> Iterator[Metric]: """For a given metric, yeilds all upstream metrics.""" yield metric_node for parent_unique_id in metric_node.depends_on.nodes: node = manifest.expect(parent_unique_id) if isinstance(node, Metric): yield from cls.parent_metrics(node, manifest) @classmethod def parent_metrics_names(cls, metric_node: Metric, manifest: Manifest) -> Iterator[str]: """For a given metric, yeilds all upstream metric names""" for metric in cls.parent_metrics(metric_node, manifest): yield metric.name @classmethod def reverse_dag_parsing( cls, metric_node: Metric, manifest: Manifest, metric_depth_count: int ) -> Iterator[Dict[str, int]]: """For the given metric, yeilds dictionaries having {<metric_name>: <depth_from_initial_metric} of upstream derived metrics. This function is intended as a helper function for other metric helper functions. """ if metric_node.type in DERIVED_METRICS: yield {metric_node.name: metric_depth_count} for parent_unique_id in metric_node.depends_on.nodes: node = manifest.expect(parent_unique_id) if isinstance(node, Metric): yield from cls.reverse_dag_parsing(node, manifest, metric_depth_count + 1) def full_metric_dependency(self): """Returns a unique list of all upstream metric names.""" to_return = list(set(self.parent_metrics_names(self.node, self.manifest))) return to_return def base_metric_dependency(self) -> List[str]: """Returns a unique list of names for all upstream non-derived metrics.""" in_scope_metrics = list(self.parent_metrics(self.node, self.manifest)) base_metrics = { metric.name for metric in in_scope_metrics if metric.type not in DERIVED_METRICS } return list(base_metrics) def derived_metric_dependency(self) -> List[str]: """Returns a unique list of names for all upstream derived metrics.""" in_scope_metrics = list(self.parent_metrics(self.node, self.manifest)) derived_metrics = { metric.name for metric in in_scope_metrics if metric.type in DERIVED_METRICS } return list(derived_metrics) def derived_metric_dependency_depth(self) -> List[Dict[str, int]]: """Returns a list of {<metric_name>: <depth_from_initial_metric>} for all upstream metrics.""" metric_depth_count = 1 to_return = list(self.reverse_dag_parsing(self.node, self.manifest, metric_depth_count)) return to_return ================================================ FILE: core/dbt/contracts/graph/model_config.py ================================================ from dataclasses import dataclass, field from typing import Any, Dict, List, Optional, Type from dbt.artifacts.resources import ( ExposureConfig, FunctionConfig, GroupConfig, MetricConfig, ModelConfig, NodeConfig, SavedQueryConfig, SeedConfig, SemanticModelConfig, SnapshotConfig, SourceConfig, TestConfig, UnitTestConfig, ) from dbt.node_types import NodeType from dbt_common.contracts.config.base import BaseConfig from dbt_common.contracts.config.metadata import Metadata def metas(*metas: Metadata) -> Dict[str, Any]: existing: Dict[str, Any] = {} for m in metas: existing = m.meta(existing) return existing def insensitive_patterns(*patterns: str): lowercased = [] for pattern in patterns: lowercased.append("".join("[{}{}]".format(s.upper(), s.lower()) for s in pattern)) return "^({})$".format("|".join(lowercased)) @dataclass class UnitTestNodeConfig(NodeConfig): expected_rows: List[Dict[str, Any]] = field(default_factory=list) expected_sql: Optional[str] = None RESOURCE_TYPES: Dict[NodeType, Type[BaseConfig]] = { NodeType.Metric: MetricConfig, NodeType.SemanticModel: SemanticModelConfig, NodeType.SavedQuery: SavedQueryConfig, NodeType.Exposure: ExposureConfig, NodeType.Source: SourceConfig, NodeType.Seed: SeedConfig, NodeType.Test: TestConfig, NodeType.Model: ModelConfig, NodeType.Snapshot: SnapshotConfig, NodeType.Unit: UnitTestConfig, NodeType.Group: GroupConfig, NodeType.Function: FunctionConfig, } # base resource types are like resource types, except nothing has mandatory # configs. BASE_RESOURCE_TYPES: Dict[NodeType, Type[BaseConfig]] = RESOURCE_TYPES.copy() def get_config_for(resource_type: NodeType, base=False) -> Type[BaseConfig]: if base: lookup = BASE_RESOURCE_TYPES else: lookup = RESOURCE_TYPES return lookup.get(resource_type, NodeConfig) ================================================ FILE: core/dbt/contracts/graph/node_args.py ================================================ from dataclasses import dataclass, field from datetime import datetime, timezone from typing import List, Optional from dbt.artifacts.resources import NodeVersion from dbt.node_types import AccessType, NodeType @dataclass class ModelNodeArgs: name: str package_name: str identifier: str schema: str database: Optional[str] = None relation_name: Optional[str] = None version: Optional[NodeVersion] = None latest_version: Optional[NodeVersion] = None deprecation_date: Optional[datetime] = None access: Optional[str] = AccessType.Protected.value generated_at: datetime = field( default_factory=lambda: datetime.now(timezone.utc).replace(tzinfo=None) ) depends_on_nodes: List[str] = field(default_factory=list) enabled: bool = True @property def unique_id(self) -> str: unique_id = f"{NodeType.Model}.{self.package_name}.{self.name}" if self.version: unique_id = f"{unique_id}.v{self.version}" return unique_id @property def fqn(self) -> List[str]: fqn = [self.package_name, self.name] # Test for None explicitly because version can be 0 if self.version is not None: fqn.append(f"v{self.version}") return fqn ================================================ FILE: core/dbt/contracts/graph/nodes.py ================================================ import hashlib import os import threading from dataclasses import dataclass, field from datetime import datetime from pathlib import Path from typing import ( Any, Dict, Iterator, List, Literal, Optional, Sequence, Tuple, Type, Union, get_args, ) from mashumaro.types import SerializableType from dbt.adapters.base import ConstraintSupport from dbt.adapters.factory import get_adapter_constraint_support from dbt.artifacts.resources import Analysis as AnalysisResource from dbt.artifacts.resources import ( BaseResource, ColumnInfo, CompiledResource, DependsOn, Docs, ) from dbt.artifacts.resources import Documentation as DocumentationResource from dbt.artifacts.resources import Exposure as ExposureResource from dbt.artifacts.resources import FileHash from dbt.artifacts.resources import Function as FunctionResource from dbt.artifacts.resources import FunctionArgument, FunctionReturns from dbt.artifacts.resources import GenericTest as GenericTestResource from dbt.artifacts.resources import GraphResource from dbt.artifacts.resources import Group as GroupResource from dbt.artifacts.resources import HasRelationMetadata as HasRelationMetadataResource from dbt.artifacts.resources import HookNode as HookNodeResource from dbt.artifacts.resources import InjectedCTE from dbt.artifacts.resources import Macro as MacroResource from dbt.artifacts.resources import MacroArgument from dbt.artifacts.resources import Metric as MetricResource from dbt.artifacts.resources import MetricInputMeasure from dbt.artifacts.resources import Model as ModelResource from dbt.artifacts.resources import ( ModelConfig, ModelFreshness, NodeConfig, NodeVersion, ParsedResource, ParsedResourceMandatory, ) from dbt.artifacts.resources import Quoting as QuotingResource from dbt.artifacts.resources import SavedQuery as SavedQueryResource from dbt.artifacts.resources import Seed as SeedResource from dbt.artifacts.resources import SemanticModel as SemanticModelResource from dbt.artifacts.resources import SingularTest as SingularTestResource from dbt.artifacts.resources import Snapshot as SnapshotResource from dbt.artifacts.resources import SourceDefinition as SourceDefinitionResource from dbt.artifacts.resources import SqlOperation as SqlOperationResource from dbt.artifacts.resources import TimeSpine from dbt.artifacts.resources import UnitTestDefinition as UnitTestDefinitionResource from dbt.artifacts.schemas.batch_results import BatchResults from dbt.clients.jinja_static import statically_extract_has_name_this from dbt.contracts.graph.model_config import UnitTestNodeConfig from dbt.contracts.graph.node_args import ModelNodeArgs from dbt.contracts.graph.unparsed import ( HasYamlMetadata, TestDef, UnitTestOverrides, UnparsedColumn, UnparsedDerivedSemantics, UnparsedMetricV2, UnparsedSemanticModelConfig, UnparsedSourceDefinition, UnparsedSourceTableDefinition, ) from dbt.events.types import ( SeedExceedsLimitAndPathChanged, SeedExceedsLimitChecksumChanged, SeedExceedsLimitSamePath, SeedIncreased, UnversionedBreakingChange, ) from dbt.exceptions import ContractBreakingChangeError, ParsingError, ValidationError from dbt.flags import get_flags from dbt.node_types import ( REFABLE_NODE_TYPES, VERSIONED_NODE_TYPES, AccessType, NodeType, ) from dbt_common.clients.system import write_file from dbt_common.contracts.constraints import ( ColumnLevelConstraint, ConstraintType, ModelLevelConstraint, ) from dbt_common.dataclass_schema import dbtClassMixin from dbt_common.events.contextvars import set_log_contextvars from dbt_common.events.functions import warn_or_error # ===================================================================== # This contains the classes for all of the nodes and node-like objects # in the manifest. In the "nodes" dictionary of the manifest we find # all of the objects in the ManifestNode union below. In addition the # manifest contains "macros", "sources", "metrics", "exposures", "docs", # and "disabled" dictionaries. # # The SeedNode is a ManifestNode, but can't be compiled because it has # no SQL. # # All objects defined in this file should have BaseNode as a parent # class. # # The two objects which do not show up in the DAG are Macro and # Documentation. # ===================================================================== # ================================================== # Various parent classes and node attribute classes # ================================================== @dataclass class BaseNode(BaseResource): """All nodes or node-like objects in this file should have this as a base class""" # In an ideal world this would be a class property. However, chaining @classmethod and # @property was deprecated in python 3.11 and removed in 3.13. There are more # complicated ways of making a class property, however a class method suits our # purposes well enough @classmethod def resource_class(cls) -> Type[BaseResource]: """Should be overriden by any class inheriting BaseNode""" raise NotImplementedError @property def search_name(self): return self.name @property def file_id(self): return f"{self.package_name}://{self.original_file_path}" @property def is_refable(self): return self.resource_type in REFABLE_NODE_TYPES @property def should_store_failures(self): return False # will this node map to an object in the database? @property def is_relational(self): return self.resource_type in REFABLE_NODE_TYPES @property def is_versioned(self): return self.resource_type in VERSIONED_NODE_TYPES and self.version is not None @property def is_ephemeral(self): return self.config.materialized == "ephemeral" @property def is_ephemeral_model(self): return self.is_refable and self.is_ephemeral def get_materialization(self): return self.config.materialized @classmethod def from_resource(cls, resource_instance: BaseResource): assert isinstance(resource_instance, cls.resource_class()) return cls.from_dict(resource_instance.to_dict()) def to_resource(self): return self.resource_class().from_dict(self.to_dict()) @dataclass class GraphNode(GraphResource, BaseNode): """Nodes in the DAG. Macro and Documentation don't have fqn.""" def same_fqn(self, other) -> bool: return self.fqn == other.fqn @dataclass class HasRelationMetadata(HasRelationMetadataResource): @classmethod def __pre_deserialize__(cls, data): data = super().__pre_deserialize__(data) if "database" not in data: data["database"] = None return data @property def quoting_dict(self) -> Dict[str, bool]: if hasattr(self, "quoting"): return self.quoting.to_dict(omit_none=True) else: return {} @dataclass class ParsedNodeMandatory(ParsedResourceMandatory, GraphNode, HasRelationMetadata): pass # This needs to be in all ManifestNodes and also in SourceDefinition, # because of "source freshness". Should not be in artifacts, because we # don't write out _event_status. @dataclass class NodeInfoMixin: _event_status: Dict[str, Any] = field(default_factory=dict) @property def node_info(self): node_info = { "node_path": getattr(self, "path", None), "node_name": getattr(self, "name", None), "unique_id": getattr(self, "unique_id", None), "resource_type": str(getattr(self, "resource_type", "")), "materialized": self.config.get("materialized"), "node_status": str(self._event_status.get("node_status")), "node_started_at": self._event_status.get("started_at"), "node_finished_at": self._event_status.get("finished_at"), "meta": getattr(self, "meta", {}), "node_relation": { "database": getattr(self, "database", None), "schema": getattr(self, "schema", None), "alias": getattr(self, "alias", None), "relation_name": getattr(self, "relation_name", None), }, "node_checksum": getattr(getattr(self, "checksum", None), "checksum", None), } return node_info def update_event_status(self, **kwargs): for k, v in kwargs.items(): self._event_status[k] = v set_log_contextvars(node_info=self.node_info) def clear_event_status(self): self._event_status = dict() @dataclass class ParsedNode(ParsedResource, NodeInfoMixin, ParsedNodeMandatory, SerializableType): def get_target_write_path( self, target_path: str, subdirectory: str, split_suffix: Optional[str] = None ): # This is called for both the "compiled" subdirectory of "target" and the "run" subdirectory if os.path.basename(self.path) == os.path.basename(self.original_file_path): # One-to-one relationship of nodes to files. path = self.original_file_path else: # Many-to-one relationship of nodes to files. path = os.path.join(self.original_file_path, self.path) if split_suffix: pathlib_path = Path(path) path = str( pathlib_path.parent / pathlib_path.stem / (pathlib_path.stem + f"_{split_suffix}" + pathlib_path.suffix) ) target_write_path = os.path.join(target_path, subdirectory, self.package_name, path) return target_write_path def write_node(self, project_root: str, compiled_path, compiled_code: str): if os.path.isabs(compiled_path): full_path = compiled_path else: full_path = os.path.join(project_root, compiled_path) write_file(full_path, compiled_code) def _serialize(self): return self.to_dict() def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None): dct = super().__post_serialize__(dct, context) if "_event_status" in dct: del dct["_event_status"] return dct @classmethod def _deserialize(cls, dct: Dict[str, int]): # The serialized ParsedNodes do not differ from each other # in fields that would allow 'from_dict' to distinguis # between them. resource_type = dct["resource_type"] if resource_type == "model": return ModelNode.from_dict(dct) elif resource_type == "analysis": return AnalysisNode.from_dict(dct) elif resource_type == "seed": return SeedNode.from_dict(dct) elif resource_type == "sql": return SqlNode.from_dict(dct) elif resource_type == "test": if "test_metadata" in dct: return GenericTestNode.from_dict(dct) else: return SingularTestNode.from_dict(dct) elif resource_type == "operation": return HookNode.from_dict(dct) elif resource_type == "seed": return SeedNode.from_dict(dct) elif resource_type == "snapshot": return SnapshotNode.from_dict(dct) else: return cls.from_dict(dct) def _persist_column_docs(self) -> bool: if hasattr(self.config, "persist_docs"): assert isinstance(self.config, NodeConfig) return bool(self.config.persist_docs.get("columns")) return False def _persist_relation_docs(self) -> bool: if hasattr(self.config, "persist_docs"): assert isinstance(self.config, NodeConfig) return bool(self.config.persist_docs.get("relation")) return False def same_persisted_description(self, other) -> bool: # the check on configs will handle the case where we have different # persist settings, so we only have to care about the cases where they # are the same.. if self._persist_relation_docs(): if self.description != other.description: return False if self._persist_column_docs(): # assert other._persist_column_docs() column_descriptions = {k: v.description for k, v in self.columns.items()} other_column_descriptions = {k: v.description for k, v in other.columns.items()} if column_descriptions != other_column_descriptions: return False return True def same_body(self, other) -> bool: return self.raw_code == other.raw_code def same_database_representation(self, other) -> bool: # compare the config representation, not the node's config value. This # compares the configured value, rather than the ultimate value (so # generate_*_name and unset values derived from the target are # ignored) keys = ("database", "schema", "alias") for key in keys: mine = self.unrendered_config.get(key) others = other.unrendered_config.get(key) if mine != others: return False return True def same_config(self, old) -> bool: return self.config.same_contents( self.unrendered_config, old.unrendered_config, ) def build_contract_checksum(self): pass def same_contract(self, old, adapter_type=None) -> bool: # This would only apply to seeds return True def same_contents(self, old, adapter_type) -> bool: if old is None: return False # Need to ensure that same_contract is called because it # could throw an error same_contract = self.same_contract(old, adapter_type) return ( self.same_body(old) and self.same_config(old) and self.same_persisted_description(old) and self.same_fqn(old) and self.same_database_representation(old) and same_contract and True ) @property def is_external_node(self): return False @dataclass class CompiledNode(CompiledResource, ParsedNode): """Contains attributes necessary for SQL files and nodes with refs, sources, etc, so all ManifestNodes except SeedNode.""" def __post_init__(self): self._lock = threading.Lock() def __getstate__(self): state = self.__dict__.copy() state.pop("_lock", None) return state def __setstate__(self, state): self.__dict__.update(state) self._lock = threading.Lock() @property def empty(self): return not self.raw_code.strip() def set_cte(self, cte_id: str, sql: str): """This is the equivalent of what self.extra_ctes[cte_id] = sql would do if extra_ctes were an OrderedDict """ for cte in self.extra_ctes: # Because it's possible that multiple threads are compiling the # node at the same time, we don't want to overwrite already compiled # sql in the extra_ctes with empty sql. if cte.id == cte_id: break else: self.extra_ctes.append(InjectedCTE(id=cte_id, sql=sql)) @property def depends_on_nodes(self): return self.depends_on.nodes @property def depends_on_macros(self): return self.depends_on.macros # ==================================== # CompiledNode subclasses # ==================================== @dataclass class AnalysisNode(AnalysisResource, CompiledNode): @classmethod def resource_class(cls) -> Type[AnalysisResource]: return AnalysisResource @dataclass class HookNode(HookNodeResource, CompiledNode): @classmethod def resource_class(cls) -> Type[HookNodeResource]: return HookNodeResource @dataclass class BatchContext(dbtClassMixin): id: str event_time_start: datetime event_time_end: datetime def __post_serialize__(self, data, context): # This is insane, but necessary, I apologize. Mashumaro handles the # dictification of this class via a compile time generated `to_dict` # method based off of the _typing_ of th class. By default `datetime` # types are converted to strings. We don't want that, we want them to # stay datetimes. # Note: This is safe because the `BatchContext` isn't part of the artifact # and thus doesn't get written out. new_data = super().__post_serialize__(data, context) new_data["event_time_start"] = self.event_time_start new_data["event_time_end"] = self.event_time_end return new_data @dataclass class ModelNode(ModelResource, CompiledNode): previous_batch_results: Optional[BatchResults] = None batch: Optional[BatchContext] = None _has_this: Optional[bool] = None def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None): dct = super().__post_serialize__(dct, context) if "_has_this" in dct: del dct["_has_this"] if "previous_batch_results" in dct: del dct["previous_batch_results"] return dct @classmethod def resource_class(cls) -> Type[ModelResource]: return ModelResource @classmethod def from_args(cls, args: ModelNodeArgs) -> "ModelNode": unique_id = args.unique_id # build unrendered config -- for usage in ParsedNode.same_contents unrendered_config = {} unrendered_config["alias"] = args.identifier unrendered_config["schema"] = args.schema if args.database: unrendered_config["database"] = args.database return cls( resource_type=NodeType.Model, name=args.name, package_name=args.package_name, unique_id=unique_id, fqn=args.fqn, version=args.version, latest_version=args.latest_version, relation_name=args.relation_name, database=args.database, schema=args.schema, alias=args.identifier, deprecation_date=args.deprecation_date, checksum=FileHash.from_contents(f"{unique_id},{args.generated_at}"), access=AccessType(args.access), original_file_path="", path="", unrendered_config=unrendered_config, depends_on=DependsOn(nodes=args.depends_on_nodes), config=ModelConfig(enabled=args.enabled), ) @property def is_external_node(self) -> bool: return not self.original_file_path and not self.path @property def is_latest_version(self) -> bool: return self.version is not None and self.version == self.latest_version @property def is_past_deprecation_date(self) -> bool: return ( self.deprecation_date is not None and self.deprecation_date < datetime.now().astimezone() ) @property def search_name(self): if self.version is None: return self.name else: return f"{self.name}.v{self.version}" @property def materialization_enforces_constraints(self) -> bool: return self.config.materialized in ["table", "incremental"] @property def all_constraints(self) -> List[Union[ModelLevelConstraint, ColumnLevelConstraint]]: constraints: List[Union[ModelLevelConstraint, ColumnLevelConstraint]] = [] for model_level_constraint in self.constraints: constraints.append(model_level_constraint) for column in self.columns.values(): for column_level_constraint in column.constraints: constraints.append(column_level_constraint) return constraints @property def has_this(self) -> bool: if self._has_this is None: self._has_this = statically_extract_has_name_this(self.raw_code) return self._has_this def infer_primary_key(self, data_tests: List["GenericTestNode"]) -> List[str]: """ Infers the columns that can be used as primary key of a model in the following order: 1. Columns with primary key constraints 2. Columns with unique and not_null data tests 3. Columns with enabled unique or dbt_utils.unique_combination_of_columns data tests 4. Columns with disabled unique or dbt_utils.unique_combination_of_columns data tests """ for constraint in self.constraints: if constraint.type == ConstraintType.primary_key: return constraint.columns for column, column_info in self.columns.items(): for column_constraint in column_info.constraints: if column_constraint.type == ConstraintType.primary_key: return [column] columns_with_enabled_unique_tests = set() columns_with_disabled_unique_tests = set() columns_with_not_null_tests = set() for test in data_tests: columns: List[str] = [] # extract columns from test kwargs, ensuring columns is a List[str] given tests can have custom (user or pacakge-defined) kwarg types if "column_name" in test.test_metadata.kwargs and isinstance( test.test_metadata.kwargs["column_name"], str ): columns = [test.test_metadata.kwargs["column_name"]] elif "combination_of_columns" in test.test_metadata.kwargs and isinstance( test.test_metadata.kwargs["combination_of_columns"], list ): columns = [ column for column in test.test_metadata.kwargs["combination_of_columns"] if isinstance(column, str) ] for column in columns: if test.test_metadata.name in ["unique", "unique_combination_of_columns"]: if test.config.enabled: columns_with_enabled_unique_tests.add(column) else: columns_with_disabled_unique_tests.add(column) elif test.test_metadata.name == "not_null": columns_with_not_null_tests.add(column) columns_with_unique_and_not_null_tests = [] for column in columns_with_not_null_tests: if ( column in columns_with_enabled_unique_tests or column in columns_with_disabled_unique_tests ): columns_with_unique_and_not_null_tests.append(column) if columns_with_unique_and_not_null_tests: return columns_with_unique_and_not_null_tests if columns_with_enabled_unique_tests: return list(columns_with_enabled_unique_tests) if columns_with_disabled_unique_tests: return list(columns_with_disabled_unique_tests) return [] def same_contents(self, old, adapter_type) -> bool: return super().same_contents(old, adapter_type) and self.same_ref_representation(old) def same_ref_representation(self, old) -> bool: return ( # Changing the latest_version may break downstream unpinned refs self.latest_version == old.latest_version # Changes to access or deprecation_date may lead to ref-related parsing errors and self.access == old.access and self.deprecation_date == old.deprecation_date ) def build_contract_checksum(self): # We don't need to construct the checksum if the model does not # have contract enforced, because it won't be used. # This needs to be executed after contract config is set # Avoid rebuilding the checksum if it has already been set. if self.contract.checksum is not None: return if self.contract.enforced is True: contract_state = "" # We need to sort the columns so that order doesn't matter # columns is a str: ColumnInfo dictionary sorted_columns = sorted(self.columns.values(), key=lambda col: col.name) for column in sorted_columns: contract_state += f"|{column.name}" contract_state += str(column.data_type) contract_state += str(column.constraints) if self.materialization_enforces_constraints: contract_state += self.config.materialized contract_state += str(self.constraints) data = contract_state.encode("utf-8") self.contract.checksum = hashlib.new("sha256", data).hexdigest() def same_contract_removed(self) -> bool: """ self: the removed (deleted, renamed, or disabled) model node """ # If the contract wasn't previously enforced, no contract change has occurred if self.contract.enforced is False: return True # Removed node is past its deprecation_date, so deletion does not constitute a contract change if self.is_past_deprecation_date: return True # Disabled, deleted, or renamed node with previously enforced contract. if not self.config.enabled: breaking_change = f"Contracted model '{self.unique_id}' was disabled." else: breaking_change = f"Contracted model '{self.unique_id}' was deleted or renamed." if self.version is None: warn_or_error( UnversionedBreakingChange( breaking_changes=[breaking_change], model_name=self.name, model_file_path=self.original_file_path, ), node=self, ) return False else: raise ( ContractBreakingChangeError( breaking_changes=[breaking_change], node=self, ) ) @staticmethod def _normalize_data_type_for_comparison(data_type: Optional[str]) -> Optional[str]: """ Normalize a data type string by removing size, precision, and scale parameters. This allows comparison of base types while ignoring non-breaking parameter changes. Examples: varchar(10) -> varchar VARCHAR(5) -> varchar numeric(10,2) -> numeric text -> text decimal(5) -> decimal None -> None Per dbt documentation, changes to size/precision/scale should not be considered breaking changes for contracts. See: https://docs.getdbt.com/reference/resource-configs/contract#size-precision-and-scale Note: Comparison is case-insensitive. Type aliases (e.g., 'varchar' vs 'character varying') are not automatically resolved - users should use consistent type names in their contracts to avoid false positives. """ if not data_type: return data_type # Split on the first '(' to get the base type without parameters # Convert to lowercase for case-insensitive comparison base_type, _, _ = data_type.partition("(") return base_type.strip().lower() def same_contract(self, old, adapter_type=None) -> bool: # If the contract wasn't previously enforced: if old.contract.enforced is False and self.contract.enforced is False: # No change -- same_contract: True return True if old.contract.enforced is False and self.contract.enforced is True: # Now it's enforced. This is a change, but not a breaking change -- same_contract: False return False # Otherwise: The contract was previously enforced, and we need to check for changes. # Happy path: The contract is still being enforced, and the checksums are identical. if self.contract.enforced is True and self.contract.checksum == old.contract.checksum: # No change -- same_contract: True return True # Otherwise: There has been a change. # We need to determine if it is a **breaking** change. # These are the categories of breaking changes: contract_enforced_disabled: bool = False columns_removed: List[str] = [] column_type_changes: List[Dict[str, str]] = [] enforced_column_constraint_removed: List[Dict[str, str]] = ( [] ) # column_name, constraint_type enforced_model_constraint_removed: List[Dict[str, Any]] = [] # constraint_type, columns materialization_changed: List[str] = [] if old.contract.enforced is True and self.contract.enforced is False: # Breaking change: the contract was previously enforced, and it no longer is contract_enforced_disabled = True constraint_support = get_adapter_constraint_support(adapter_type) column_constraints_exist = False # Next, compare each column from the previous contract (old.columns) for old_key, old_value in sorted(old.columns.items()): # Has this column been removed? if old_key not in self.columns.keys(): columns_removed.append(old_value.name) # Has this column's data type changed? elif old_value.data_type != self.columns[old_key].data_type: # Compare normalized data types (without size/precision/scale) # to determine if this is a breaking change old_normalized = self._normalize_data_type_for_comparison(old_value.data_type) new_normalized = self._normalize_data_type_for_comparison( self.columns[old_key].data_type ) # Only consider it a breaking change if the base types differ # Changes like varchar(3) -> varchar(10) are not breaking if old_normalized != new_normalized: column_type_changes.append( { "column_name": str(old_value.name), "previous_column_type": str(old_value.data_type), "current_column_type": str(self.columns[old_key].data_type), } ) # track if there are any column level constraints for the materialization check late if old_value.constraints: column_constraints_exist = True # Have enforced columns level constraints changed? # Constraints are only enforced for table and incremental materializations. # We only really care if the old node was one of those materializations for breaking changes if ( old_key in self.columns.keys() and old_value.constraints != self.columns[old_key].constraints and old.materialization_enforces_constraints ): for old_constraint in old_value.constraints: if ( old_constraint not in self.columns[old_key].constraints and constraint_support[old_constraint.type] == ConstraintSupport.ENFORCED ): enforced_column_constraint_removed.append( { "column_name": old_key, "constraint_name": old_constraint.name, "constraint_type": ConstraintType(old_constraint.type), } ) # Now compare the model level constraints if old.constraints != self.constraints and old.materialization_enforces_constraints: for old_constraint in old.constraints: if ( old_constraint not in self.constraints and constraint_support[old_constraint.type] == ConstraintSupport.ENFORCED ): enforced_model_constraint_removed.append( { "constraint_name": old_constraint.name, "constraint_type": ConstraintType(old_constraint.type), "columns": old_constraint.columns, } ) # Check for relevant materialization changes. if ( old.materialization_enforces_constraints and not self.materialization_enforces_constraints and (old.constraints or column_constraints_exist) ): materialization_changed = [old.config.materialized, self.config.materialized] # If a column has been added, it will be missing in the old.columns, and present in self.columns # That's a change (caught by the different checksums), but not a breaking change # Did we find any changes that we consider breaking? If there's an enforced contract, that's # a warning unless the model is versioned, then it's an error. if ( contract_enforced_disabled or columns_removed or column_type_changes or enforced_model_constraint_removed or enforced_column_constraint_removed or materialization_changed ): breaking_changes = [] if contract_enforced_disabled: breaking_changes.append( "Contract enforcement was removed: Previously, this model had an enforced contract. It is no longer configured to enforce its contract, and this is a breaking change." ) if columns_removed: columns_removed_str = "\n - ".join(columns_removed) breaking_changes.append(f"Columns were removed: \n - {columns_removed_str}") if column_type_changes: column_type_changes_str = "\n - ".join( [ f"{c['column_name']} ({c['previous_column_type']} -> {c['current_column_type']})" for c in column_type_changes ] ) breaking_changes.append( f"Columns with data_type changes: \n - {column_type_changes_str}" ) if enforced_column_constraint_removed: column_constraint_changes_str = "\n - ".join( [ f"'{c['constraint_name'] if c['constraint_name'] is not None else c['constraint_type']}' constraint on column {c['column_name']}" for c in enforced_column_constraint_removed ] ) breaking_changes.append( f"Enforced column level constraints were removed: \n - {column_constraint_changes_str}" ) if enforced_model_constraint_removed: model_constraint_changes_str = "\n - ".join( [ f"'{c['constraint_name'] if c['constraint_name'] is not None else c['constraint_type']}' constraint on columns {c['columns']}" for c in enforced_model_constraint_removed ] ) breaking_changes.append( f"Enforced model level constraints were removed: \n - {model_constraint_changes_str}" ) if materialization_changed: materialization_changes_str = ( f"{materialization_changed[0]} -> {materialization_changed[1]}" ) breaking_changes.append( f"Materialization changed with enforced constraints: \n - {materialization_changes_str}" ) if self.version is None: warn_or_error( UnversionedBreakingChange( contract_enforced_disabled=contract_enforced_disabled, columns_removed=columns_removed, column_type_changes=column_type_changes, enforced_column_constraint_removed=enforced_column_constraint_removed, enforced_model_constraint_removed=enforced_model_constraint_removed, breaking_changes=breaking_changes, model_name=self.name, model_file_path=self.original_file_path, ), node=self, ) else: raise ( ContractBreakingChangeError( breaking_changes=breaking_changes, node=self, ) ) # Otherwise, the contract has changed -- same_contract: False return False @dataclass class SqlNode(SqlOperationResource, CompiledNode): @classmethod def resource_class(cls) -> Type[SqlOperationResource]: return SqlOperationResource # ==================================== # Seed node # ==================================== @dataclass class SeedNode(SeedResource, ParsedNode): # No SQLDefaults! @classmethod def resource_class(cls) -> Type[SeedResource]: return SeedResource def same_seeds(self, other: "SeedNode") -> bool: # for seeds, we check the hashes. If the hashes are different types, # no match. If the hashes are both the same 'path', log a warning and # assume they are the same # if the current checksum is a path, we want to log a warning. result = self.checksum == other.checksum if self.checksum.name == "path": msg: str if other.checksum.name != "path": warn_or_error( SeedIncreased(package_name=self.package_name, name=self.name), node=self ) elif result: warn_or_error( SeedExceedsLimitSamePath(package_name=self.package_name, name=self.name), node=self, ) elif not result: warn_or_error( SeedExceedsLimitAndPathChanged(package_name=self.package_name, name=self.name), node=self, ) else: warn_or_error( SeedExceedsLimitChecksumChanged( package_name=self.package_name, name=self.name, checksum_name=other.checksum.name, ), node=self, ) return result @property def empty(self): """Seeds are never empty""" return False def _disallow_implicit_dependencies(self): """Disallow seeds to take implicit upstream dependencies via pre/post hooks""" # Seeds are root nodes in the DAG. They cannot depend on other nodes. # However, it's possible to define pre- and post-hooks on seeds, and for those # hooks to include {{ ref(...) }}. This worked in previous versions, but it # was never officially documented or supported behavior. Let's raise an explicit error, # which will surface during parsing if the user has written code such that we attempt # to capture & record a ref/source/metric call on the SeedNode. # For more details: https://github.com/dbt-labs/dbt-core/issues/6806 hooks = [f'- pre_hook: "{hook.sql}"' for hook in self.config.pre_hook] + [ f'- post_hook: "{hook.sql}"' for hook in self.config.post_hook ] hook_list = "\n".join(hooks) message = f""" Seeds cannot depend on other nodes. dbt detected a seed with a pre- or post-hook that calls 'ref', 'source', or 'metric', either directly or indirectly via other macros. Error raised for '{self.unique_id}', which has these hooks defined: \n{hook_list} """ raise ParsingError(message) @property def refs(self): self._disallow_implicit_dependencies() @property def sources(self): self._disallow_implicit_dependencies() @property def metrics(self): self._disallow_implicit_dependencies() def same_body(self, other) -> bool: return self.same_seeds(other) @property def depends_on_nodes(self): return [] @property def depends_on_macros(self) -> List[str]: return self.depends_on.macros @property def extra_ctes(self): return [] @property def extra_ctes_injected(self): return False @property def language(self): return "sql" # @property # def compiled_code(self): # return None # ==================================== # Singular Test node # ==================================== class TestShouldStoreFailures: @property def should_store_failures(self): if self.config.store_failures: return self.config.store_failures return get_flags().STORE_FAILURES @property def is_relational(self): if self.should_store_failures: return True return False @dataclass class SingularTestNode(SingularTestResource, TestShouldStoreFailures, CompiledNode): @classmethod def resource_class(cls) -> Type[SingularTestResource]: return SingularTestResource @property def test_node_type(self): return "singular" # ==================================== # Generic Test node # ==================================== @dataclass class GenericTestNode(GenericTestResource, TestShouldStoreFailures, CompiledNode): @classmethod def resource_class(cls) -> Type[GenericTestResource]: return GenericTestResource def same_contents(self, other, adapter_type: Optional[str]) -> bool: if other is None: return False return self.same_config(other) and self.same_fqn(other) and True @property def test_node_type(self): return "generic" @dataclass class UnitTestSourceDefinition(ModelNode): source_name: str = "undefined" quoting: QuotingResource = field(default_factory=QuotingResource) @property def cte_name(self): return self.unique_id.split(".")[-1] @property def search_name(self): return f"{self.source_name}.{self.name}" @dataclass class UnitTestNode(CompiledNode): resource_type: Literal[NodeType.Unit] tested_node_unique_id: Optional[str] = None this_input_node_unique_id: Optional[str] = None overrides: Optional[UnitTestOverrides] = None config: UnitTestNodeConfig = field(default_factory=UnitTestNodeConfig) @dataclass class UnitTestDefinition(NodeInfoMixin, GraphNode, UnitTestDefinitionResource): @classmethod def resource_class(cls) -> Type[UnitTestDefinitionResource]: return UnitTestDefinitionResource @property def depends_on_nodes(self): return self.depends_on.nodes @property def tags(self) -> List[str]: tags = self.config.tags return [tags] if isinstance(tags, str) else tags @property def versioned_name(self) -> str: versioned_name = self.name if self.version is not None: versioned_name += f"_v{self.version}" return versioned_name def build_unit_test_checksum(self): # everything except 'description' data = f"{self.model}-{self.versions}-{self.given}-{self.expect}-{self.overrides}" # include underlying fixture data for input in self.given: if input.fixture: data += f"-{input.rows}" self.checksum = hashlib.new("sha256", data.encode("utf-8")).hexdigest() def same_contents(self, other: Optional["UnitTestDefinition"]) -> bool: if other is None: return False return self.checksum == other.checksum @dataclass class UnitTestFileFixture(BaseNode): resource_type: Literal[NodeType.Fixture] rows: Optional[Union[List[Dict[str, Any]], str]] = None # ==================================== # Snapshot node # ==================================== @dataclass class SnapshotNode(SnapshotResource, CompiledNode): @classmethod def resource_class(cls) -> Type[SnapshotResource]: return SnapshotResource # ==================================== # Macro # ==================================== @dataclass class Macro(MacroResource, BaseNode): @classmethod def resource_class(cls) -> Type[MacroResource]: return MacroResource def same_contents(self, other: Optional["Macro"]) -> bool: if other is None: return False # the only thing that makes one macro different from another with the # same name/package is its content return self.macro_sql == other.macro_sql @property def depends_on_macros(self): return self.depends_on.macros # ==================================== # Documentation node # ==================================== @dataclass class Documentation(DocumentationResource, BaseNode): @classmethod def resource_class(cls) -> Type[DocumentationResource]: return DocumentationResource @property def search_name(self): return self.name def same_contents(self, other: Optional["Documentation"]) -> bool: if other is None: return False # the only thing that makes one doc different from another with the # same name/package is its content return self.block_contents == other.block_contents # ==================================== # Source node # ==================================== def normalize_test(testdef: TestDef) -> Dict[str, Any]: if isinstance(testdef, str): return {testdef: {}} else: return testdef @dataclass class UnpatchedSourceDefinition(BaseNode): source: UnparsedSourceDefinition table: UnparsedSourceTableDefinition fqn: List[str] resource_type: Literal[NodeType.Source] patch_path: Optional[str] = None def get_full_source_name(self): return f"{self.source.name}_{self.table.name}" def get_source_representation(self): return f'source("{self.source.name}", "{self.table.name}")' def validate_data_tests(self, is_root_project: bool): """ sources parse tests differently than models, so we need to do some validation here where it's done in the PatchParser for other nodes """ # source table-level tests if self.tests and self.data_tests: raise ValidationError( "Invalid test config: cannot have both 'tests' and 'data_tests' defined" ) if self.tests: self.data_tests.extend(self.tests) self.tests.clear() # column-level tests for column in self.columns: if column.tests and column.data_tests: raise ValidationError( "Invalid test config: cannot have both 'tests' and 'data_tests' defined" ) if column.tests: column.data_tests.extend(column.tests) column.tests.clear() @property def quote_columns(self) -> Optional[bool]: result = None if self.source.quoting.column is not None: result = self.source.quoting.column if self.table.quoting.column is not None: result = self.table.quoting.column return result @property def columns(self) -> Sequence[UnparsedColumn]: return [] if self.table.columns is None else self.table.columns def get_tests(self) -> Iterator[Tuple[Dict[str, Any], Optional[UnparsedColumn]]]: for data_test in self.data_tests: yield normalize_test(data_test), None for column in self.columns: if column.data_tests is not None: for data_test in column.data_tests: yield normalize_test(data_test), column @property def data_tests(self) -> List[TestDef]: if self.table.data_tests is None: return [] else: return self.table.data_tests # deprecated @property def tests(self) -> List[TestDef]: if self.table.tests is None: return [] else: return self.table.tests @dataclass class SourceDefinition( NodeInfoMixin, GraphNode, SourceDefinitionResource, HasRelationMetadata, ): @classmethod def resource_class(cls) -> Type[SourceDefinitionResource]: return SourceDefinitionResource def same_database_representation(self, other: "SourceDefinition") -> bool: # preserve legacy behaviour -- use potentially rendered database if get_flags().state_modified_compare_more_unrendered_values is False: same_database = self.database == other.database same_schema = self.schema == other.schema else: same_database = self.unrendered_database == other.unrendered_database same_schema = self.unrendered_schema == other.unrendered_schema return same_database and same_schema and self.identifier == other.identifier and True def same_quoting(self, other: "SourceDefinition") -> bool: return self.quoting == other.quoting def same_freshness(self, other: "SourceDefinition") -> bool: return ( self.freshness == other.freshness and self.loaded_at_field == other.loaded_at_field and True ) def same_external(self, other: "SourceDefinition") -> bool: return self.external == other.external def same_config(self, old: "SourceDefinition") -> bool: return self.config.same_contents( self.unrendered_config, old.unrendered_config, ) def same_contents(self, old: Optional["SourceDefinition"]) -> bool: # existing when it didn't before is a change! if old is None: return True # config changes are changes (because the only config is "enforced", and # enabling a source is a change!) # changing the database/schema/identifier is a change # messing around with external stuff is a change (uh, right?) # quoting changes are changes # freshness changes are changes, I guess # metadata/tags changes are not "changes" # patching/description changes are not "changes" return ( self.same_database_representation(old) and self.same_fqn(old) and self.same_config(old) and self.same_quoting(old) and self.same_freshness(old) and self.same_external(old) and True ) def get_full_source_name(self): return f"{self.source_name}_{self.name}" def get_source_representation(self): return f'source("{self.source.name}", "{self.table.name}")' @property def is_refable(self): return False @property def is_ephemeral(self): return False @property def is_ephemeral_model(self): return False @property def depends_on_nodes(self): return [] @property def depends_on(self): return DependsOn(macros=[], nodes=[]) @property def refs(self): return [] @property def sources(self): return [] @property def has_freshness(self) -> bool: return bool(self.freshness) @property def search_name(self): return f"{self.source_name}.{self.name}" @property def group(self): return None # ==================================== # Exposure node # ==================================== @dataclass class Exposure(NodeInfoMixin, GraphNode, ExposureResource): @property def depends_on_nodes(self): return self.depends_on.nodes @property def search_name(self): return self.name @classmethod def resource_class(cls) -> Type[ExposureResource]: return ExposureResource def same_depends_on(self, old: "Exposure") -> bool: return set(self.depends_on.nodes) == set(old.depends_on.nodes) def same_description(self, old: "Exposure") -> bool: return self.description == old.description def same_label(self, old: "Exposure") -> bool: return self.label == old.label def same_maturity(self, old: "Exposure") -> bool: return self.maturity == old.maturity def same_owner(self, old: "Exposure") -> bool: return self.owner == old.owner def same_exposure_type(self, old: "Exposure") -> bool: return self.type == old.type def same_url(self, old: "Exposure") -> bool: return self.url == old.url def same_config(self, old: "Exposure") -> bool: return self.config.same_contents( self.unrendered_config, old.unrendered_config, ) def same_contents(self, old: Optional["Exposure"]) -> bool: # existing when it didn't before is a change! # metadata/tags changes are not "changes" if old is None: return True return ( self.same_fqn(old) and self.same_exposure_type(old) and self.same_owner(old) and self.same_maturity(old) and self.same_url(old) and self.same_description(old) and self.same_label(old) and self.same_depends_on(old) and self.same_config(old) and True ) @property def group(self): return None def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None): dct = super().__post_serialize__(dct, context) if "_event_status" in dct: del dct["_event_status"] return dct # ==================================== # Metric node # ==================================== @dataclass class Metric(GraphNode, MetricResource): @property def depends_on_nodes(self): return self.depends_on.nodes @property def search_name(self): return self.name @classmethod def resource_class(cls) -> Type[MetricResource]: return MetricResource def same_description(self, old: "Metric") -> bool: return self.description == old.description def same_label(self, old: "Metric") -> bool: return self.label == old.label def same_config(self, old: "Metric") -> bool: return self.config.same_contents( self.unrendered_config, old.unrendered_config, ) def same_filter(self, old: "Metric") -> bool: return True # TODO def same_metadata(self, old: "Metric") -> bool: return True # TODO def same_type(self, old: "Metric") -> bool: return self.type == old.type def same_type_params(self, old: "Metric") -> bool: return True # TODO def same_contents(self, old: Optional["Metric"]) -> bool: # existing when it didn't before is a change! # metadata/tags changes are not "changes" if old is None: return True return ( self.same_filter(old) and self.same_metadata(old) and self.same_type(old) and self.same_type_params(old) and self.same_description(old) and self.same_label(old) and self.same_config(old) and True ) def add_input_measure(self, input_measure: MetricInputMeasure) -> None: for existing_input_measure in self.type_params.input_measures: if input_measure == existing_input_measure: return self.type_params.input_measures.append(input_measure) # ==================================== # Group node # ==================================== @dataclass class Group(GroupResource, BaseNode): @classmethod def resource_class(cls) -> Type[GroupResource]: return GroupResource def to_logging_dict(self) -> Dict[str, Union[str, Dict[str, str]]]: return { "name": self.name, "package_name": self.package_name, "owner": {k: str(v) for k, v in self.owner.to_dict(omit_none=True).items()}, } # ==================================== # Function node # ==================================== @dataclass class FunctionNode(CompiledNode, FunctionResource): @classmethod def resource_class(cls) -> Type[FunctionResource]: return FunctionResource # ==================================== # SemanticModel node # ==================================== @dataclass class SemanticModel(GraphNode, SemanticModelResource): @property def depends_on_nodes(self): return self.depends_on.nodes @property def depends_on_macros(self): return self.depends_on.macros @classmethod def resource_class(cls) -> Type[SemanticModelResource]: return SemanticModelResource def same_model(self, old: "SemanticModel") -> bool: return self.model == old.model def same_description(self, old: "SemanticModel") -> bool: return self.description == old.description def same_defaults(self, old: "SemanticModel") -> bool: return self.defaults == old.defaults def same_entities(self, old: "SemanticModel") -> bool: return self.entities == old.entities def same_dimensions(self, old: "SemanticModel") -> bool: return self.dimensions == old.dimensions def same_measures(self, old: "SemanticModel") -> bool: return self.measures == old.measures def same_config(self, old: "SemanticModel") -> bool: return self.config == old.config def same_primary_entity(self, old: "SemanticModel") -> bool: return self.primary_entity == old.primary_entity def same_group(self, old: "SemanticModel") -> bool: return self.group == old.group def same_contents(self, old: Optional["SemanticModel"]) -> bool: # existing when it didn't before is a change! # metadata/tags changes are not "changes" if old is None: return True return ( self.same_model(old) and self.same_description(old) and self.same_defaults(old) and self.same_entities(old) and self.same_dimensions(old) and self.same_measures(old) and self.same_config(old) and self.same_primary_entity(old) and self.same_group(old) and True ) # ==================================== # SavedQuery # ==================================== @dataclass class SavedQuery(NodeInfoMixin, GraphNode, SavedQueryResource): @classmethod def resource_class(cls) -> Type[SavedQueryResource]: return SavedQueryResource def same_metrics(self, old: "SavedQuery") -> bool: return self.query_params.metrics == old.query_params.metrics def same_group_by(self, old: "SavedQuery") -> bool: return self.query_params.group_by == old.query_params.group_by def same_description(self, old: "SavedQuery") -> bool: return self.description == old.description def same_where(self, old: "SavedQuery") -> bool: return self.query_params.where == old.query_params.where def same_label(self, old: "SavedQuery") -> bool: return self.label == old.label def same_config(self, old: "SavedQuery") -> bool: return self.config == old.config def same_group(self, old: "SavedQuery") -> bool: return self.group == old.group def same_exports(self, old: "SavedQuery") -> bool: if len(self.exports) != len(old.exports): return False # exports should be in the same order, so we zip them for easy iteration for old_export, new_export in zip(old.exports, self.exports): if not (old_export.name == new_export.name): return False keys = ["export_as", "schema", "alias"] for key in keys: if old_export.unrendered_config.get(key) != new_export.unrendered_config.get(key): return False return True def same_tags(self, old: "SavedQuery") -> bool: return self.tags == old.tags def same_contents(self, old: Optional["SavedQuery"]) -> bool: # existing when it didn't before is a change! # metadata/tags changes are not "changes" if old is None: return True return ( self.same_metrics(old) and self.same_group_by(old) and self.same_description(old) and self.same_where(old) and self.same_label(old) and self.same_config(old) and self.same_group(old) and self.same_exports(old) and self.same_tags(old) and True ) def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None): dct = super().__post_serialize__(dct, context) if "_event_status" in dct: del dct["_event_status"] return dct # ==================================== # Patches # ==================================== @dataclass class ParsedPatch(HasYamlMetadata): name: str description: str meta: Dict[str, Any] docs: Docs config: Dict[str, Any] # The parsed node update is only the 'patch', not the test. The test became a # regular parsed node. Note that description and columns must be present, but # may be empty. @dataclass class ParsedNodePatch(ParsedPatch): columns: Dict[str, ColumnInfo] access: Optional[str] version: Optional[NodeVersion] latest_version: Optional[NodeVersion] constraints: List[Dict[str, Any]] deprecation_date: Optional[datetime] time_spine: Optional[TimeSpine] = None semantic_model: Union[UnparsedSemanticModelConfig, bool, None] = None metrics: Optional[List[UnparsedMetricV2]] = None derived_semantics: Optional[UnparsedDerivedSemantics] = None agg_time_dimension: Optional[str] = None primary_entity: Optional[str] = None freshness: Optional[ModelFreshness] = None @dataclass class ParsedFunctionPatchRequired: returns: FunctionReturns # TODO: Maybe this shouldn't be a subclass of ParsedNodePatch, but ParsedPatch instead # Currently, `functions` have the fields like `columns`, `access`, `version`, and etc, # but they don't actually do anything. If we remove those properties from FunctionNode, # we can remove this class and use ParsedPatch instead. @dataclass class ParsedFunctionPatch(ParsedNodePatch, ParsedFunctionPatchRequired): arguments: List[FunctionArgument] = field(default_factory=list) @dataclass class ParsedMacroPatch(ParsedPatch): arguments: List[MacroArgument] = field(default_factory=list) @dataclass class ParsedSingularTestPatch(ParsedPatch): pass # ==================================== # Node unions/categories # ==================================== # ManifestNode without SeedNode, which doesn't have the # SQL related attributes ManifestSQLNode = Union[ AnalysisNode, FunctionNode, SingularTestNode, HookNode, ModelNode, SqlNode, GenericTestNode, SnapshotNode, UnitTestNode, ] # All SQL nodes plus SeedNode (csv files) ManifestNode = Union[ ManifestSQLNode, SeedNode, ] ResultNode = Union[ ManifestNode, SourceDefinition, HookNode, ] # All nodes that can be in the DAG GraphMemberNode = Union[ ResultNode, Exposure, Metric, SavedQuery, SemanticModel, UnitTestDefinition, ] # All "nodes" (or node-like objects) in this file Resource = Union[ GraphMemberNode, Documentation, Macro, Group, ] TestNode = Union[SingularTestNode, GenericTestNode] SemanticManifestNode = Union[SavedQuery, SemanticModel, Metric] RESOURCE_CLASS_TO_NODE_CLASS: Dict[Type[BaseResource], Type[BaseNode]] = { node_class.resource_class(): node_class for node_class in get_args(Resource) if node_class is not UnitTestNode } ================================================ FILE: core/dbt/contracts/graph/semantic_manifest.py ================================================ from typing import List, Optional, Set from dbt import deprecations from dbt.constants import ( LEGACY_TIME_SPINE_GRANULARITY, LEGACY_TIME_SPINE_MODEL_NAME, MINIMUM_REQUIRED_TIME_SPINE_GRANULARITY, ) from dbt.contracts.graph.manifest import Manifest from dbt.contracts.graph.nodes import ModelNode from dbt.events.types import ArtifactWritten, SemanticValidationFailure from dbt.exceptions import ParsingError from dbt.flags import get_flags from dbt_common.clients.system import write_file from dbt_common.events.base_types import EventLevel from dbt_common.events.functions import fire_event from dbt_semantic_interfaces.implementations.metric import PydanticMetric from dbt_semantic_interfaces.implementations.node_relation import PydanticNodeRelation from dbt_semantic_interfaces.implementations.project_configuration import ( PydanticProjectConfiguration, ) from dbt_semantic_interfaces.implementations.saved_query import PydanticSavedQuery from dbt_semantic_interfaces.implementations.semantic_manifest import ( PydanticSemanticManifest, ) from dbt_semantic_interfaces.implementations.semantic_model import PydanticSemanticModel from dbt_semantic_interfaces.implementations.time_spine import ( PydanticTimeSpine, PydanticTimeSpineCustomGranularityColumn, PydanticTimeSpinePrimaryColumn, ) from dbt_semantic_interfaces.implementations.time_spine_table_configuration import ( PydanticTimeSpineTableConfiguration as LegacyTimeSpine, ) from dbt_semantic_interfaces.type_enums import TimeGranularity from dbt_semantic_interfaces.validations.semantic_manifest_validator import ( SemanticManifestValidator, ) from dbt_semantic_interfaces.validations.validator_helpers import ( FileContext, ValidationError, ValidationIssueContext, ) class SemanticManifest: def __init__(self, manifest: Manifest) -> None: self.manifest = manifest def validate(self) -> bool: # TODO: Enforce this check. # if self.manifest.metrics and not self.manifest.semantic_models: # fire_event( # SemanticValidationFailure( # msg="Metrics require semantic models, but none were found." # ), # EventLevel.ERROR, # ) # return False if not self.manifest.metrics or not self.manifest.semantic_models: return True semantic_manifest = self._get_pydantic_semantic_manifest() validator = SemanticManifestValidator[PydanticSemanticManifest]() validation_results = validator.validate_semantic_manifest(semantic_manifest) validation_result_errors = list(validation_results.errors) metrics_using_old_params: Set[str] = set() for metric in semantic_manifest.metrics or []: for field in ("window", "grain_to_date"): type_params_field_value = getattr(metric.type_params, field) # Warn that the old type_params structure has been deprecated. if type_params_field_value: metrics_using_old_params.add(metric.name) if metrics_using_old_params: if get_flags().require_nested_cumulative_type_params is False: deprecations.warn( "mf-cumulative-type-params-deprecation", ) else: names = ", ".join(metrics_using_old_params) validation_result_errors.append( ValidationError( context=ValidationIssueContext( # We don't have the file context at this point. file_context=FileContext(), object_name=names, object_type="metric", ), message=f"Cumulative fields `type_params.window` and `type_params.grain_to_date` should be nested under `type_params.cumulative_type_params.window` and `type_params.cumulative_type_params.grain_to_date`. Invalid metrics: {names}. See documentation on behavior changes: https://docs.getdbt.com/reference/global-configs/behavior-changes.", ) ) time_spines = semantic_manifest.project_configuration.time_spines legacy_time_spines = ( semantic_manifest.project_configuration.time_spine_table_configurations ) # If the time spine contains a day grain then it is functionally equivalent to the legacy time spine. time_spines_contain_day = any( c for c in time_spines if c.primary_column.time_granularity == TimeGranularity.DAY ) if ( get_flags().require_yaml_configuration_for_mf_time_spines is False and legacy_time_spines and not time_spines_contain_day ): deprecations.warn( "mf-timespine-without-yaml-configuration", ) for warning in validation_results.warnings: fire_event(SemanticValidationFailure(msg=warning.message)) for deprecation in validation_results.future_errors: if ( "time dimension" in deprecation.message and "must have a time granularity set" in deprecation.message ): deprecations.warn( "time-dimensions-require-granularity-deprecation", deprecation.message ) else: # We have three options for this case: # 1. Swallow it (don't send any event) # 2. Raise a generic deprecation # 3. Raise an error about an unknown SL future error # # (1) is the easiest, but useless to the user and also to us for debugging. # (3) is not safe because if a new deprecation is added in a DSI patch, then # suddenly dbt core would be broken. # This leaves (2) only remaining option, which is what we do. deprecations.warn("generic-semantic-layer-deprecation", deprecation.message) for error in validation_result_errors: fire_event(SemanticValidationFailure(msg=error.message), EventLevel.ERROR) return not validation_result_errors def write_json_to_file(self, file_path: str): semantic_manifest = self._get_pydantic_semantic_manifest() json = semantic_manifest.json() write_file(file_path, json) fire_event(ArtifactWritten(artifact_type=self.__class__.__name__, artifact_path=file_path)) def _get_pydantic_semantic_manifest(self) -> PydanticSemanticManifest: pydantic_time_spines: List[PydanticTimeSpine] = [] minimum_time_spine_granularity: Optional[TimeGranularity] = None for node in self.manifest.nodes.values(): if not (isinstance(node, ModelNode) and node.time_spine): continue time_spine = node.time_spine standard_granularity_column = None for column in node.columns.values(): if column.name == time_spine.standard_granularity_column: standard_granularity_column = column break # Assertions needed for type checking if not standard_granularity_column: raise ParsingError( "Expected to find time spine standard granularity column in model columns, but did not. " "This should have been caught in YAML parsing." ) if not standard_granularity_column.granularity: raise ParsingError( "Expected to find granularity set for time spine standard granularity column, but did not. " "This should have been caught in YAML parsing." ) pydantic_time_spine = PydanticTimeSpine( node_relation=PydanticNodeRelation( alias=node.alias, schema_name=node.schema, database=node.database, relation_name=node.relation_name, ), primary_column=PydanticTimeSpinePrimaryColumn( name=time_spine.standard_granularity_column, time_granularity=standard_granularity_column.granularity, ), custom_granularities=[ PydanticTimeSpineCustomGranularityColumn( name=custom_granularity.name, column_name=custom_granularity.column_name ) for custom_granularity in time_spine.custom_granularities ], ) pydantic_time_spines.append(pydantic_time_spine) if ( not minimum_time_spine_granularity or standard_granularity_column.granularity.to_int() < minimum_time_spine_granularity.to_int() ): minimum_time_spine_granularity = standard_granularity_column.granularity project_config = PydanticProjectConfiguration( time_spine_table_configurations=[], time_spines=pydantic_time_spines ) pydantic_semantic_manifest = PydanticSemanticManifest( metrics=[], semantic_models=[], project_configuration=project_config ) for semantic_model in self.manifest.semantic_models.values(): pydantic_semantic_manifest.semantic_models.append( PydanticSemanticModel.parse_obj(semantic_model.to_dict()) ) for metric in self.manifest.metrics.values(): pydantic_semantic_manifest.metrics.append(PydanticMetric.parse_obj(metric.to_dict())) for saved_query in self.manifest.saved_queries.values(): pydantic_semantic_manifest.saved_queries.append( PydanticSavedQuery.parse_obj(saved_query.to_dict()) ) legacy_time_spine_model: Optional[ModelNode] = None if self.manifest.semantic_models: legacy_time_spine_model = self.manifest.ref_lookup.find( LEGACY_TIME_SPINE_MODEL_NAME, None, None, self.manifest ) if legacy_time_spine_model: if ( not minimum_time_spine_granularity or LEGACY_TIME_SPINE_GRANULARITY.to_int() < minimum_time_spine_granularity.to_int() ): minimum_time_spine_granularity = LEGACY_TIME_SPINE_GRANULARITY # If no time spines have been configured at DAY or smaller AND legacy time spine model does not exist, error. if ( not minimum_time_spine_granularity or minimum_time_spine_granularity.to_int() > MINIMUM_REQUIRED_TIME_SPINE_GRANULARITY.to_int() ): raise ParsingError( "The semantic layer requires a time spine model with granularity DAY or smaller in the project, " "but none was found. Guidance on creating this model can be found on our docs site " "(https://docs.getdbt.com/docs/build/metricflow-time-spine)." ) # For backward compatibility: if legacy time spine exists without config, include it in the manifest. if legacy_time_spine_model and legacy_time_spine_model.time_spine is None: legacy_time_spine = LegacyTimeSpine( location=legacy_time_spine_model.relation_name, column_name="date_day", grain=LEGACY_TIME_SPINE_GRANULARITY, ) pydantic_semantic_manifest.project_configuration.time_spine_table_configurations = [ legacy_time_spine ] return pydantic_semantic_manifest ================================================ FILE: core/dbt/contracts/graph/unparsed.py ================================================ import datetime import re from dataclasses import dataclass, field from enum import Enum from pathlib import Path from typing import Any, Dict, List, Literal, Optional, Sequence, Union from typing_extensions import override # trigger the PathEncoder import dbt_common.helper_types # noqa:F401 from dbt import deprecations from dbt.artifacts.resources import ( ConstantPropertyInput, Defaults, DimensionValidityParams, Docs, ExposureType, ExternalTable, FreshnessThreshold, FunctionArgument, FunctionReturns, MacroArgument, MaturityType, MeasureAggregationParameters, NodeVersion, Owner, Quoting, TimeSpine, UnitTestInputFixture, UnitTestNodeVersions, UnitTestOutputFixture, UnitTestOverrides, list_str, metas, ) from dbt.exceptions import ParsingError from dbt.node_types import NodeType from dbt_common.contracts.config.base import CompareBehavior, MergeBehavior from dbt_common.contracts.config.metadata import ShowBehavior from dbt_common.contracts.config.properties import AdditionalPropertiesMixin from dbt_common.contracts.util import Mergeable from dbt_common.dataclass_schema import ( ExtensibleDbtClassMixin, StrEnum, ValidationError, dbtClassMixin, ) from dbt_common.exceptions import DbtInternalError from dbt_semantic_interfaces.type_enums import ( ConversionCalculationType, DimensionType, PeriodAggregation, ) @dataclass class UnparsedBaseNode(dbtClassMixin): package_name: str path: str original_file_path: str @property def file_id(self): return f"{self.package_name}://{self.original_file_path}" @dataclass class HasCode(dbtClassMixin): raw_code: str language: str @property def empty(self): return not self.raw_code.strip() @dataclass class UnparsedMacro(UnparsedBaseNode, HasCode): resource_type: Literal[NodeType.Macro] @dataclass class UnparsedGenericTest(UnparsedBaseNode, HasCode): resource_type: Literal[NodeType.Macro] @dataclass class UnparsedNode(UnparsedBaseNode, HasCode): name: str resource_type: NodeType @property def search_name(self): return self.name @dataclass class UnparsedRunHook(UnparsedNode): resource_type: Literal[NodeType.Operation] index: Optional[int] = None @dataclass class HasColumnProps(AdditionalPropertiesMixin, ExtensibleDbtClassMixin): name: str description: str = "" meta: Dict[str, Any] = field(default_factory=dict) data_type: Optional[str] = None constraints: List[Dict[str, Any]] = field(default_factory=list) docs: Docs = field(default_factory=Docs) config: Dict[str, Any] = field(default_factory=dict) _extra: Dict[str, Any] = field(default_factory=dict) TestDef = Union[Dict[str, Any], str] @dataclass class HasColumnAndTestProps(HasColumnProps): data_tests: List[TestDef] = field(default_factory=list) tests: List[TestDef] = field( default_factory=list ) # back compat for previous name of 'data_tests' @dataclass class HasColumnDocs(dbtClassMixin): columns: Sequence[HasColumnProps] = field(default_factory=list) @dataclass class HasYamlMetadata(dbtClassMixin): original_file_path: str yaml_key: str package_name: str @property def file_id(self): return f"{self.package_name}://{self.original_file_path}" @dataclass class HasConfig: config: Dict[str, Any] = field(default_factory=dict) @dataclass class UnparsedDimensionBase(dbtClassMixin): # Should we be limiting name length or otherwise validating this? type: str # actually a DimensionType enum value name: Optional[str] = None description: Optional[str] = None label: Optional[str] = None is_partition: bool = False config: Dict[str, Any] = field(default_factory=dict) @dataclass class UnparsedDimensionTypeParams(dbtClassMixin): """Used for dbt Semantic Layer dimensions (v1 YAML).""" time_granularity: str # TimeGranularity enum validity_params: Optional[DimensionValidityParams] = None @dataclass(kw_only=True) class UnparsedDimension(UnparsedDimensionBase): """Used for dbt Semantic Layer dimensions (v1 YAML).""" name: str type_params: Optional[UnparsedDimensionTypeParams] = None expr: Optional[str] = None @dataclass class UnparsedDimensionV2(UnparsedDimensionBase): """Used for dbt Semantic Layer dimensions (v2 YAML).""" validity_params: Optional[DimensionValidityParams] = None @dataclass(kw_only=True) class UnparsedDerivedDimensionV2(UnparsedDimensionV2): """Used for dbt Semantic Layer derived dimensions (v2 YAML).""" name: str expr: str granularity: Optional[str] = None # str is really a TimeGranularity Enum @classmethod @override def validate(cls, data): super().validate(data) # validity_params may only be set when the derived dimension has a granularity if data.get("validity_params") is not None and not data.get("granularity"): dim_name = data.get("name") raise ValidationError( f"Derived dimension {dim_name} has validity_params, " "so it must specify a granularity." ) @dataclass class UnparsedEntityBase(dbtClassMixin): name: str type: str # EntityType enum description: Optional[str] = None label: Optional[str] = None config: Dict[str, Any] = field(default_factory=dict) @dataclass class UnparsedEntity(UnparsedEntityBase): """Used for dbt Semantic Layer entities (v1 YAML only).""" role: Optional[str] = None expr: Optional[str] = None @dataclass class UnparsedColumnEntityV2(UnparsedEntityBase): """Used for dbt Semantic Layer column entities (v2 YAML).""" pass # kw_only allows this child to define required fields @dataclass(kw_only=True) class UnparsedDerivedEntityV2(UnparsedEntityBase): """Used for dbt Semantic Layer derived entities (v2 YAML).""" expr: str @dataclass class UnparsedColumn(HasConfig, HasColumnAndTestProps): quote: Optional[bool] = None tags: List[str] = field(default_factory=list) granularity: Optional[str] = None # str is really a TimeGranularity Enum # Note 1: Dimension str is a DimensionType enum value # Note 2: Don't ask me why, but str must come after UnparsedDimensionV2 here or else # this will be read as a dict object instead of a UnparsedDimensionV2 object # Only used in v2 semantic layer. dimension: Union[UnparsedDimensionV2, str, None] = None # UnparsedColumnEntityV2 must come before str to parse correctly. str is assumed to be EntityType enum value # Only used in v2 semantic layer. entity: Union[UnparsedColumnEntityV2, str, None] = None @classmethod @override def validate(cls, data): super().validate(data) if (dimension := data.get("dimension")) is not None: if isinstance(dimension, dict): dim_type_str = dimension.get("type") dim_name = dimension.get("name") else: dim_type_str = dimension dim_name = data.get("name") dim_type = DimensionType(dim_type_str) if dim_type_str is not None else None if dim_type is DimensionType.TIME and not data.get("granularity"): raise ValidationError( f"Dimension {dim_name} is a time dimension attached to " f"column {data.get('name')}, " "so that column must specify a granularity." ) # validity_params may only be set when the column has a granularity if ( isinstance(dimension, dict) and dimension.get("validity_params") is not None and not data.get("granularity") ): dim_name = dimension.get("name") or data.get("name") raise ValidationError( f"Dimension {dim_name} has validity_params attached to " f"column {data.get('name')}, " "so that column must specify a granularity." ) @dataclass class HasColumnTests(dbtClassMixin): columns: Sequence[UnparsedColumn] = field(default_factory=list) @dataclass class MetricFilter(dbtClassMixin): field: str operator: str # TODO : Can we make this Any? value: str class MetricTimePeriod(StrEnum): day = "day" week = "week" month = "month" year = "year" def plural(self) -> str: return str(self) + "s" @dataclass class MetricTime(dbtClassMixin, Mergeable): count: Optional[int] = None period: Optional[MetricTimePeriod] = None def __bool__(self): return self.count is not None and self.period is not None @dataclass class UnparsedMetricInputMeasure(dbtClassMixin): name: str # Note: `Union` must be the outermost part of the type annotation for serialization to work properly. filter: Union[str, List[str], None] = None alias: Optional[str] = None join_to_timespine: bool = False fill_nulls_with: Optional[int] = None @dataclass class UnparsedMetricInput(dbtClassMixin): name: str # Note: `Union` must be the outermost part of the type annotation for serialization to work properly. filter: Union[str, List[str], None] = None alias: Optional[str] = None offset_window: Optional[str] = None offset_to_grain: Optional[str] = None @dataclass class UnparsedConversionTypeParams(dbtClassMixin): """Only used in v1 Semantic YAML""" base_measure: Union[UnparsedMetricInputMeasure, str] conversion_measure: Union[UnparsedMetricInputMeasure, str] entity: str calculation: str = ( ConversionCalculationType.CONVERSION_RATE.value ) # ConversionCalculationType Enum window: Optional[str] = None constant_properties: Optional[List[ConstantPropertyInput]] = None @dataclass class UnparsedCumulativeTypeParams(dbtClassMixin): """Only used in v1 Semantic YAML""" window: Optional[str] = None grain_to_date: Optional[str] = None period_agg: str = PeriodAggregation.FIRST.value @dataclass class UnparsedMetricTypeParams(dbtClassMixin): """Used on v1 Semantic Metric YAML.""" measure: Optional[Union[UnparsedMetricInputMeasure, str]] = None numerator: Optional[Union[UnparsedMetricInput, str]] = None denominator: Optional[Union[UnparsedMetricInput, str]] = None expr: Optional[Union[str, bool]] = None window: Optional[str] = None grain_to_date: Optional[str] = None # str is really a TimeGranularity Enum metrics: Optional[List[Union[UnparsedMetricInput, str]]] = None conversion_type_params: Optional[UnparsedConversionTypeParams] = None cumulative_type_params: Optional[UnparsedCumulativeTypeParams] = None @dataclass(kw_only=True) class UnparsedMetricBase(dbtClassMixin): name: str type: str = "simple" label: Optional[str] = None # in v1 this is required, but in v2 it is optional description: str = "" # Note: `Union` must be the outermost part of the type annotation for serialization to work properly. filter: Union[str, List[str], None] = None time_granularity: Optional[str] = None config: Dict[str, Any] = field(default_factory=dict) @classmethod def validate(cls, data): super().validate(data) if "name" in data: errors = [] if " " in data["name"]: errors.append("cannot contain spaces") # This handles failing queries due to too long metric names. # It only occurs in BigQuery and Snowflake (Postgres/Redshift truncate) if len(data["name"]) > 250: errors.append("cannot contain more than 250 characters") if not (re.match(r"^[A-Za-z]", data["name"])): errors.append("must begin with a letter") if not (re.match(r"[\w]+$", data["name"])): errors.append("must contain only letters, numbers and underscores") if errors: raise ValidationError( f"The metric name '{data['name']}' is invalid. It {', '.join(e for e in errors)}" ) @dataclass(kw_only=True) class UnparsedMetric(UnparsedMetricBase): """Old-style YAML metric; prefer UnparsedMetricV2 instead as of late 2025.""" label: str type_params: UnparsedMetricTypeParams # old-style YAML # metadata: Optional[Unparsedetadata] = None # TODO meta: Dict[str, Any] = field(default_factory=dict) tags: List[str] = field(default_factory=list) @dataclass class UnparsedNonAdditiveDimensionV2(dbtClassMixin): name: str window_agg: str # AggregationType enum group_by: List[str] = field(default_factory=list) @dataclass class UnparsedMetricV2(UnparsedMetricBase): hidden: bool = False agg: Optional[str] = None percentile: Optional[float] = None percentile_type: Optional[str] = None join_to_timespine: Optional[bool] = None fill_nulls_with: Optional[int] = None expr: Optional[Union[str, int]] = None non_additive_dimension: Optional[UnparsedNonAdditiveDimensionV2] = None agg_time_dimension: Optional[str] = None # For cumulative metrics window: Optional[str] = None grain_to_date: Optional[str] = None period_agg: str = PeriodAggregation.FIRST.value input_metric: Optional[Union[UnparsedMetricInput, str]] = None # For ratio metrics numerator: Optional[Union[UnparsedMetricInput, str]] = None denominator: Optional[Union[UnparsedMetricInput, str]] = None # For derived metrics input_metrics: Optional[List[Union[UnparsedMetricInput, str]]] = None # For conversion metrics entity: Optional[str] = None calculation: Optional[str] = None base_metric: Optional[Union[UnparsedMetricInput, str]] = None conversion_metric: Optional[Union[UnparsedMetricInput, str]] = None constant_properties: Optional[List[ConstantPropertyInput]] = None @classmethod @override def validate(cls, data): super().validate(data) if data["type"] == "simple" and data.get("agg") is None: raise ValidationError("Simple metrics must have an agg param.") @dataclass class UnparsedVersion(dbtClassMixin): v: NodeVersion defined_in: Optional[str] = None description: str = "" access: Optional[str] = None config: Dict[str, Any] = field(default_factory=dict) constraints: List[Dict[str, Any]] = field(default_factory=list) docs: Docs = field(default_factory=Docs) data_tests: Optional[List[TestDef]] = None tests: Optional[List[TestDef]] = None # back compat for previous name of 'data_tests' columns: Sequence[Union[dbt_common.helper_types.IncludeExclude, UnparsedColumn]] = field( default_factory=list ) deprecation_date: Optional[datetime.datetime] = None def __lt__(self, other): try: return float(self.v) < float(other.v) except ValueError: return str(self.v) < str(other.v) @property def include_exclude(self) -> dbt_common.helper_types.IncludeExclude: return self._include_exclude @property def unparsed_columns(self) -> List: return self._unparsed_columns @property def formatted_v(self) -> str: return f"v{self.v}" def __post_init__(self): has_include_exclude = False self._include_exclude = dbt_common.helper_types.IncludeExclude(include="*") self._unparsed_columns = [] for column in self.columns: if isinstance(column, dbt_common.helper_types.IncludeExclude): if not has_include_exclude: self._include_exclude = column has_include_exclude = True else: raise ParsingError("version can have at most one include/exclude element") else: self._unparsed_columns.append(column) self.deprecation_date = normalize_date(self.deprecation_date) @dataclass class UnparsedAnalysisUpdate(HasConfig, HasColumnDocs, HasColumnProps, HasYamlMetadata): access: Optional[str] = None @dataclass class UnparsedSingularTestUpdate(HasConfig, HasColumnProps, HasYamlMetadata): pass @dataclass class UnparsedNodeUpdate(HasConfig, HasColumnTests, HasColumnAndTestProps, HasYamlMetadata): quote_columns: Optional[bool] = None access: Optional[str] = None @dataclass class UnparsedDerivedSemantics(dbtClassMixin): entities: List[UnparsedDerivedEntityV2] = field(default_factory=list) dimensions: List[UnparsedDerivedDimensionV2] = field(default_factory=list) @dataclass class UnparsedSemanticResourceConfig(dbtClassMixin): meta: Dict[str, Any] = field(default_factory=dict) @dataclass(kw_only=True) class UnparsedSemanticModelConfig(dbtClassMixin): name: Optional[str] = None enabled: bool = True group: Optional[str] = None config: Optional[UnparsedSemanticResourceConfig] = None @dataclass class UnparsedModelUpdate(UnparsedNodeUpdate): quote_columns: Optional[bool] = None access: Optional[str] = None latest_version: Optional[NodeVersion] = None versions: Sequence[UnparsedVersion] = field(default_factory=list) deprecation_date: Optional[datetime.datetime] = None time_spine: Optional[TimeSpine] = None # True indicates that the semantic model is enabeld and will have it's values populated # directly from the model. # Using an UnparsedSemanticModelConfig object allows user to override some of the # values instead. semantic_model: Union[UnparsedSemanticModelConfig, bool, None] = None primary_entity: Optional[str] = None agg_time_dimension: Optional[str] = None metrics: Optional[List[UnparsedMetricV2]] = None derived_semantics: Optional[UnparsedDerivedSemantics] = None def __post_init__(self) -> None: if self.latest_version: version_values = [version.v for version in self.versions] if self.latest_version not in version_values: raise ParsingError( f"latest_version: {self.latest_version} is not one of model '{self.name}' versions: {version_values} " ) seen_versions = set() for version in self.versions: if str(version.v) in seen_versions: raise ParsingError( f"Found duplicate version: '{version.v}' in versions list of model '{self.name}'" ) seen_versions.add(str(version.v)) self._version_map = {version.v: version for version in self.versions} self.deprecation_date = normalize_date(self.deprecation_date) if self.time_spine: columns = ( self.get_columns_for_version(self.latest_version) if self.latest_version else self.columns ) column_names_to_columns = {column.name: column for column in columns} if self.time_spine.standard_granularity_column not in column_names_to_columns: raise ParsingError( f"Time spine standard granularity column must be defined on the model. Got invalid " f"column name '{self.time_spine.standard_granularity_column}' for model '{self.name}'. Valid names" f"{' for latest version' if self.latest_version else ''}: {list(column_names_to_columns.keys())}." ) standard_column = column_names_to_columns[self.time_spine.standard_granularity_column] if not standard_column.granularity: raise ParsingError( f"Time spine standard granularity column must have a granularity defined. Please add one for " f"column '{self.time_spine.standard_granularity_column}' in model '{self.name}'." ) custom_granularity_columns_not_found = [] for custom_granularity in self.time_spine.custom_granularities: column_name = ( custom_granularity.column_name if custom_granularity.column_name else custom_granularity.name ) if column_name not in column_names_to_columns: custom_granularity_columns_not_found.append(column_name) if custom_granularity_columns_not_found: raise ParsingError( "Time spine custom granularity columns do not exist in the model. " f"Columns not found: {custom_granularity_columns_not_found}; " f"Available columns: {list(column_names_to_columns.keys())}" ) def get_columns_for_version(self, version: NodeVersion) -> List[UnparsedColumn]: if version not in self._version_map: raise DbtInternalError( f"get_columns_for_version called for version '{version}' not in version map" ) version_columns = [] unparsed_version = self._version_map[version] for base_column in self.columns: if unparsed_version.include_exclude.includes(base_column.name): version_columns.append(base_column) for column in unparsed_version.unparsed_columns: version_columns.append(column) return version_columns def get_tests_for_version(self, version: NodeVersion) -> List[TestDef]: if version not in self._version_map: raise DbtInternalError( f"get_tests_for_version called for version '{version}' not in version map" ) unparsed_version = self._version_map[version] return ( unparsed_version.data_tests if unparsed_version.data_tests is not None else self.data_tests ) @dataclass class UnparsedMacroUpdate(HasConfig, HasColumnProps, HasYamlMetadata): arguments: List[MacroArgument] = field(default_factory=list) @dataclass class UnparsedSourceTableDefinition(HasColumnTests, HasColumnAndTestProps): config: Dict[str, Any] = field(default_factory=dict) loaded_at_field: Optional[str] = None loaded_at_field_present: Optional[bool] = None loaded_at_query: Optional[str] = None identifier: Optional[str] = None quoting: Quoting = field(default_factory=Quoting) freshness: Optional[FreshnessThreshold] = field(default_factory=FreshnessThreshold) external: Optional[ExternalTable] = None tags: List[str] = field(default_factory=list) def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None): dct = super().__post_serialize__(dct, context) if "freshness" not in dct and self.freshness is None: dct["freshness"] = None return dct @dataclass class UnparsedSourceDefinition(dbtClassMixin): name: str description: str = "" meta: Dict[str, Any] = field(default_factory=dict) database: Optional[str] = None schema: Optional[str] = None loader: str = "" quoting: Quoting = field(default_factory=Quoting) freshness: Optional[FreshnessThreshold] = field(default_factory=FreshnessThreshold) loaded_at_field: Optional[str] = None loaded_at_field_present: Optional[bool] = None loaded_at_query: Optional[str] = None tables: List[UnparsedSourceTableDefinition] = field(default_factory=list) tags: List[str] = field(default_factory=list) config: Dict[str, Any] = field(default_factory=dict) unrendered_database: Optional[str] = None unrendered_schema: Optional[str] = None @classmethod def validate(cls, data): super(UnparsedSourceDefinition, cls).validate(data) if data.get("loaded_at_field", None) == "": raise ValidationError("loaded_at_field cannot be an empty string.") if "tables" in data: for table in data["tables"]: if table.get("loaded_at_field", None) == "": raise ValidationError("loaded_at_field cannot be an empty string.") @property def yaml_key(self) -> "str": return "sources" def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None): dct = super().__post_serialize__(dct, context) if "freshness" not in dct and self.freshness is None: dct["freshness"] = None return dct @dataclass class SourceTablePatch(dbtClassMixin): name: str description: Optional[str] = None meta: Optional[Dict[str, Any]] = None data_type: Optional[str] = None docs: Optional[Docs] = None loaded_at_field: Optional[str] = None loaded_at_field_present: Optional[bool] = None loaded_at_query: Optional[str] = None identifier: Optional[str] = None quoting: Quoting = field(default_factory=Quoting) freshness: Optional[FreshnessThreshold] = field(default_factory=FreshnessThreshold) external: Optional[ExternalTable] = None tags: Optional[List[str]] = None data_tests: Optional[List[TestDef]] = None tests: Optional[List[TestDef]] = None # back compat for previous name of 'data_tests' columns: Optional[Sequence[UnparsedColumn]] = None def to_patch_dict(self) -> Dict[str, Any]: dct = self.to_dict(omit_none=True) remove_keys = "name" for key in remove_keys: if key in dct: del dct[key] if self.freshness is None: dct["freshness"] = None return dct @dataclass class SourcePatch(dbtClassMixin): name: str = field( metadata=dict(description="The name of the source to override"), ) overrides: str = field( metadata=dict(description="The package of the source to override"), ) path: Path = field( metadata=dict(description="The path to the patch-defining yml file"), ) config: Dict[str, Any] = field(default_factory=dict) description: Optional[str] = None meta: Optional[Dict[str, Any]] = None database: Optional[str] = None schema: Optional[str] = None loader: Optional[str] = None quoting: Optional[Quoting] = None freshness: Optional[Optional[FreshnessThreshold]] = field(default_factory=FreshnessThreshold) loaded_at_field: Optional[str] = None loaded_at_field_present: Optional[bool] = None loaded_at_query: Optional[str] = None tables: Optional[List[SourceTablePatch]] = None tags: Optional[List[str]] = None def to_patch_dict(self) -> Dict[str, Any]: dct = self.to_dict(omit_none=True) remove_keys = ("name", "overrides", "tables", "path") for key in remove_keys: if key in dct: del dct[key] if self.freshness is None: dct["freshness"] = None return dct def get_table_named(self, name: str) -> Optional[SourceTablePatch]: if self.tables is not None: for table in self.tables: if table.name == name: return table return None @dataclass class UnparsedDocumentation(dbtClassMixin): package_name: str path: str original_file_path: str @property def file_id(self): return f"{self.package_name}://{self.original_file_path}" @property def resource_type(self): return NodeType.Documentation @dataclass class UnparsedDocumentationFile(UnparsedDocumentation): file_contents: str # can't use total_ordering decorator here, as str provides an ordering already # and it's not the one we want. class Maturity(StrEnum): low = "low" medium = "medium" high = "high" def __lt__(self, other): if not isinstance(other, Maturity): return NotImplemented order = (Maturity.low, Maturity.medium, Maturity.high) return order.index(self) < order.index(other) def __gt__(self, other): if not isinstance(other, Maturity): return NotImplemented return self != other and not (self < other) def __ge__(self, other): if not isinstance(other, Maturity): return NotImplemented return self == other or not (self < other) def __le__(self, other): if not isinstance(other, Maturity): return NotImplemented return self == other or self < other @dataclass class UnparsedExposure(dbtClassMixin): name: str type: ExposureType owner: Owner description: str = "" label: Optional[str] = None maturity: Optional[MaturityType] = None meta: Dict[str, Any] = field(default_factory=dict) tags: List[str] = field(default_factory=list) url: Optional[str] = None depends_on: List[str] = field(default_factory=list) config: Dict[str, Any] = field(default_factory=dict) @classmethod def validate(cls, data): super(UnparsedExposure, cls).validate(data) if "name" in data: # name can only contain alphanumeric chars and underscores if not (re.match(r"[\w-]+$", data["name"])): deprecations.warn("exposure-name", exposure=data["name"]) if data["owner"].get("name") is None and data["owner"].get("email") is None: raise ValidationError("Exposure owner must have at least one of 'name' or 'email'.") @dataclass class UnparsedGroup(dbtClassMixin): name: str owner: Owner description: Optional[str] = None config: Dict[str, Any] = field(default_factory=dict) @classmethod def validate(cls, data): super(UnparsedGroup, cls).validate(data) if data["owner"].get("name") is None and data["owner"].get("email") is None: raise ValidationError("Group owner must have at least one of 'name' or 'email'.") # TODO DI-4413: the following are not strictly necessary (they will be handled # in dsi validation), but they would be a better user experience # if we did it at parse time. # TODO: validate that conversion metrics have base_metric, conversion_metric, and entity # TODO: validate that cumulative metrics have all required inputs here # TODO: validate that derived metrics have all required inputs here # TODO: validate that ratio metrics have all required inputs here # TODO: validate that simple metrics have all required inputs here @dataclass class UnparsedFunctionReturns(dbtClassMixin): returns: FunctionReturns @dataclass class UnparsedFunctionUpdate(HasConfig, HasColumnProps, HasYamlMetadata, UnparsedFunctionReturns): access: Optional[str] = None arguments: List[FunctionArgument] = field(default_factory=list) # # semantic interfaces unparsed objects # @dataclass class UnparsedNonAdditiveDimension(dbtClassMixin): name: str window_choice: str # AggregationType enum window_groupings: List[str] = field(default_factory=list) class PercentileType(str, Enum): DISCRETE = "discrete" CONTINUOUS = "continuous" @dataclass class UnparsedMeasure(dbtClassMixin): name: str agg: str # actually an enum description: Optional[str] = None label: Optional[str] = None expr: Optional[Union[str, bool, int]] = None agg_params: Optional[MeasureAggregationParameters] = None non_additive_dimension: Optional[UnparsedNonAdditiveDimension] = None agg_time_dimension: Optional[str] = None create_metric: bool = False config: Dict[str, Any] = field(default_factory=dict) @dataclass class UnparsedSemanticModel(dbtClassMixin): name: str model: str # looks like "ref(...)" config: Dict[str, Any] = field(default_factory=dict) description: Optional[str] = None label: Optional[str] = None defaults: Optional[Defaults] = None entities: List[UnparsedEntity] = field(default_factory=list) measures: List[UnparsedMeasure] = field(default_factory=list) dimensions: List[UnparsedDimension] = field(default_factory=list) primary_entity: Optional[str] = None @dataclass class UnparsedQueryParams(dbtClassMixin): metrics: List[str] = field(default_factory=list) group_by: List[str] = field(default_factory=list) # Note: `Union` must be the outermost part of the type annotation for serialization to work properly. where: Union[str, List[str], None] = None order_by: List[str] = field(default_factory=list) limit: Optional[int] = None @dataclass class UnparsedExport(dbtClassMixin): """Configuration for writing query results to a table.""" name: str config: Dict[str, Any] = field(default_factory=dict) @dataclass class UnparsedSavedQuery(dbtClassMixin): name: str query_params: UnparsedQueryParams description: Optional[str] = None label: Optional[str] = None exports: List[UnparsedExport] = field(default_factory=list) config: Dict[str, Any] = field(default_factory=dict) # Note: the order of the types is critical; it's the order that they will be checked against inputs. # if reversed, a single-string tag like `tag: "good"` becomes ['g','o','o','d'] tags: Union[str, List[str]] = field( default_factory=list_str, metadata=metas(ShowBehavior.Hide, MergeBehavior.Append, CompareBehavior.Exclude), ) def normalize_date(d: Optional[datetime.date]) -> Optional[datetime.datetime]: """Convert date to datetime (at midnight), and add local time zone if naive""" if d is None: return None # convert date to datetime dt = d if type(d) == datetime.datetime else datetime.datetime(d.year, d.month, d.day) if not dt.tzinfo: # date is naive, re-interpret as system time zone dt = dt.astimezone() return dt @dataclass class UnparsedUnitTest(dbtClassMixin): name: str model: str # name of the model being unit tested given: Sequence[UnitTestInputFixture] expect: UnitTestOutputFixture description: str = "" overrides: Optional[UnitTestOverrides] = None config: Dict[str, Any] = field(default_factory=dict) versions: Optional[UnitTestNodeVersions] = None @classmethod def validate(cls, data): super(UnparsedUnitTest, cls).validate(data) if data.get("versions", None): if data["versions"].get("include") and data["versions"].get("exclude"): raise ValidationError("Unit tests can not both include and exclude versions.") ================================================ FILE: core/dbt/contracts/project.py ================================================ from dataclasses import dataclass, field from typing import Any, ClassVar, Dict, List, Optional, Union from mashumaro.jsonschema.annotations import Pattern from mashumaro.types import SerializableType from typing_extensions import Annotated from dbt.adapters.contracts.connection import QueryComment from dbt.contracts.util import Identifier, list_str from dbt_common.contracts.util import Mergeable from dbt_common.dataclass_schema import ( ExtensibleDbtClassMixin, ValidationError, dbtClassMixin, dbtMashConfig, ) from dbt_common.helper_types import NoValue DEFAULT_SEND_ANONYMOUS_USAGE_STATS = True class SemverString(str, SerializableType): def _serialize(self) -> str: return self @classmethod def _deserialize(cls, value: str) -> "SemverString": return SemverString(value) # This supports full semver, but also allows for 2 group version numbers, (allows '1.0'). sem_ver_pattern = r"^(0|[1-9]\d*)\.(0|[1-9]\d*)(\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)?$" @dataclass class Quoting(dbtClassMixin, Mergeable): schema: Optional[bool] = None database: Optional[bool] = None project: Optional[bool] = None identifier: Optional[bool] = None snowflake_ignore_case: Optional[bool] = None @dataclass class Package(dbtClassMixin): # Exclude {'name': None} from to_dict result to avoid changing sha1_hash result # when user has not changed their 'packages' configuration. def __post_serialize__(self, data, context: Optional[Dict]): if "name" in data.keys() and data["name"] is None: data.pop("name") return data return data @dataclass class LocalPackage(Package): local: str unrendered: Dict[str, Any] = field(default_factory=dict) name: Optional[str] = None # `float` also allows `int`, according to PEP484 (and jsonschema!) RawVersion = Union[str, float] @dataclass class TarballPackage(Package): tarball: str name: str unrendered: Dict[str, Any] = field(default_factory=dict) @dataclass class GitPackage(Package): git: str revision: Optional[RawVersion] = None warn_unpinned: Optional[bool] = field(default=None, metadata={"alias": "warn-unpinned"}) subdirectory: Optional[str] = None unrendered: Dict[str, Any] = field(default_factory=dict) name: Optional[str] = None def get_revisions(self) -> List[str]: if self.revision is None: return [] else: return [str(self.revision)] @dataclass class PrivatePackage(Package): private: str provider: Optional[str] = None revision: Optional[RawVersion] = None warn_unpinned: Optional[bool] = field(default=None, metadata={"alias": "warn-unpinned"}) subdirectory: Optional[str] = None unrendered: Dict[str, Any] = field(default_factory=dict) name: Optional[str] = None @dataclass class RegistryPackage(Package): package: str version: Union[RawVersion, List[RawVersion]] install_prerelease: Optional[bool] = False unrendered: Dict[str, Any] = field(default_factory=dict) name: Optional[str] = None def get_versions(self) -> List[str]: if isinstance(self.version, list): return [str(v) for v in self.version] else: return [str(self.version)] PackageSpec = Union[LocalPackage, TarballPackage, GitPackage, RegistryPackage, PrivatePackage] @dataclass class PackageConfig(dbtClassMixin): packages: List[PackageSpec] @classmethod def validate(cls, data): for package in data.get("packages", data): # This can happen when the target is a variable that is not filled and results in hangs if isinstance(package, dict): if package.get("package") == "": raise ValidationError( "A hub package is missing the value. It is a required property." ) if package.get("local") == "": raise ValidationError( "A local package is missing the value. It is a required property." ) if package.get("git") == "": raise ValidationError( "A git package is missing the value. It is a required property." ) if isinstance(package, dict) and package.get("package"): if not package["version"]: raise ValidationError( f"{package['package']} is missing the version. When installing from the Hub " "package index, version is a required property" ) if "/" not in package["package"]: raise ValidationError( f"{package['package']} was not found in the package index. Packages on the index " "require a namespace, e.g dbt-labs/dbt_utils" ) super().validate(data) @dataclass class ProjectPackageMetadata: name: str packages: List[PackageSpec] @classmethod def from_project(cls, project): return cls(name=project.project_name, packages=project.packages.packages) @dataclass class Downloads(ExtensibleDbtClassMixin): tarball: str @dataclass class RegistryPackageMetadata( ExtensibleDbtClassMixin, ProjectPackageMetadata, ): downloads: Downloads # A list of all the reserved words that packages may not have as names. BANNED_PROJECT_NAMES = { "_sql_results", "adapter", "api", "column", "config", "context", "database", "env", "env_var", "exceptions", "execute", "flags", "fromjson", "fromyaml", "graph", "invocation_id", "load_agate_table", "load_result", "log", "model", "modules", "post_hooks", "pre_hooks", "ref", "render", "return", "run_started_at", "schema", "source", "sql", "sql_now", "store_result", "store_raw_result", "target", "this", "tojson", "toyaml", "try_or_compiler_error", "var", "write", } @dataclass class Project(dbtClassMixin): _hyphenated: ClassVar[bool] = True # Annotated is used by mashumaro for jsonschema generation name: Annotated[Identifier, Pattern(r"^[^\d\W]\w*$")] config_version: Optional[int] = 2 # Annotated is used by mashumaro for jsonschema generation version: Optional[Union[Annotated[SemverString, Pattern(sem_ver_pattern)], float]] = None project_root: Optional[str] = None source_paths: Optional[List[str]] = None model_paths: Optional[List[str]] = None macro_paths: Optional[List[str]] = None data_paths: Optional[List[str]] = None # deprecated seed_paths: Optional[List[str]] = None test_paths: Optional[List[str]] = None analysis_paths: Optional[List[str]] = None docs_paths: Optional[List[str]] = None asset_paths: Optional[List[str]] = None function_paths: Optional[List[str]] = None target_path: Optional[str] = None snapshot_paths: Optional[List[str]] = None clean_targets: Optional[List[str]] = None profile: Optional[str] = None log_path: Optional[str] = None packages_install_path: Optional[str] = None quoting: Optional[Quoting] = None on_run_start: Optional[List[str]] = field(default_factory=list_str) on_run_end: Optional[List[str]] = field(default_factory=list_str) require_dbt_version: Optional[Union[List[str], str]] = None dispatch: List[Dict[str, Any]] = field(default_factory=list) models: Dict[str, Any] = field(default_factory=dict) seeds: Dict[str, Any] = field(default_factory=dict) snapshots: Dict[str, Any] = field(default_factory=dict) analyses: Dict[str, Any] = field(default_factory=dict) sources: Dict[str, Any] = field(default_factory=dict) tests: Dict[str, Any] = field(default_factory=dict) # deprecated data_tests: Dict[str, Any] = field(default_factory=dict) unit_tests: Dict[str, Any] = field(default_factory=dict) metrics: Dict[str, Any] = field(default_factory=dict) semantic_models: Dict[str, Any] = field(default_factory=dict) saved_queries: Dict[str, Any] = field(default_factory=dict) exposures: Dict[str, Any] = field(default_factory=dict) functions: Dict[str, Any] = field(default_factory=dict) vars: Optional[Dict[str, Any]] = field( default=None, metadata=dict( description="map project names to their vars override dicts", ), ) packages: List[PackageSpec] = field(default_factory=list) query_comment: Optional[Union[QueryComment, NoValue, str]] = field(default_factory=NoValue) restrict_access: bool = False dbt_cloud: Optional[Dict[str, Any]] = None flags: Dict[str, Any] = field(default_factory=dict) class Config(dbtMashConfig): # These tell mashumaro to use aliases for jsonschema and for "from_dict" aliases = { "config_version": "config-version", "project_root": "project-root", "source_paths": "source-paths", "model_paths": "model-paths", "macro_paths": "macro-paths", "data_paths": "data-paths", "seed_paths": "seed-paths", "test_paths": "test-paths", "analysis_paths": "analysis-paths", "docs_paths": "docs-paths", "asset_paths": "asset-paths", "function_paths": "function-paths", "target_path": "target-path", "snapshot_paths": "snapshot-paths", "clean_targets": "clean-targets", "log_path": "log-path", "packages_install_path": "packages-install-path", "on_run_start": "on-run-start", "on_run_end": "on-run-end", "require_dbt_version": "require-dbt-version", "query_comment": "query-comment", "restrict_access": "restrict-access", "semantic_models": "semantic-models", "saved_queries": "saved-queries", "dbt_cloud": "dbt-cloud", } @classmethod def validate(cls, data): super().validate(data) if data["name"] in BANNED_PROJECT_NAMES: raise ValidationError(f"Invalid project name: {data['name']} is a reserved word") # validate dispatch config if "dispatch" in data and data["dispatch"]: entries = data["dispatch"] for entry in entries: if ( "macro_namespace" not in entry or "search_order" not in entry or not isinstance(entry["search_order"], list) ): raise ValidationError(f"Invalid project dispatch config: {entry}") if "dbt_cloud" in data and not isinstance(data["dbt_cloud"], dict): raise ValidationError( f"Invalid dbt_cloud config. Expected a 'dict' but got '{type(data['dbt_cloud'])}'" ) if data.get("tests", None) and data.get("data_tests", None): raise ValidationError( "Invalid project config: cannot have both 'tests' and 'data_tests' defined" ) @dataclass class ProjectFlags(ExtensibleDbtClassMixin): cache_selected_only: Optional[bool] = None debug: Optional[bool] = None fail_fast: Optional[bool] = None indirect_selection: Optional[str] = None log_format: Optional[str] = None log_format_file: Optional[str] = None log_level: Optional[str] = None log_level_file: Optional[str] = None partial_parse: Optional[bool] = None populate_cache: Optional[bool] = None printer_width: Optional[int] = None send_anonymous_usage_stats: bool = DEFAULT_SEND_ANONYMOUS_USAGE_STATS static_parser: Optional[bool] = None use_colors: Optional[bool] = None use_colors_file: Optional[bool] = None use_experimental_parser: Optional[bool] = None version_check: Optional[bool] = None warn_error: Optional[bool] = None warn_error_options: Optional[Dict[str, Union[str, List[str]]]] = None write_json: Optional[bool] = None # legacy behaviors - https://github.com/dbt-labs/dbt-core/blob/main/docs/guides/behavior-change-flags.md require_batched_execution_for_custom_microbatch_strategy: bool = False require_event_names_in_deprecations: bool = False require_explicit_package_overrides_for_builtin_materializations: bool = True require_resource_names_without_spaces: bool = True source_freshness_run_project_hooks: bool = True skip_nodes_if_on_run_start_fails: bool = False state_modified_compare_more_unrendered_values: bool = False state_modified_compare_vars: bool = False require_yaml_configuration_for_mf_time_spines: bool = False require_nested_cumulative_type_params: bool = False validate_macro_args: bool = False require_all_warnings_handled_by_warn_error: bool = False require_generic_test_arguments_property: bool = True require_unique_project_resource_names: bool = False require_ref_searches_node_package_before_root: bool = False require_valid_schema_from_generate_schema_name: bool = False require_sql_header_in_test_configs: bool = False support_custom_ref_kwargs: bool = False @property def project_only_flags(self) -> Dict[str, Any]: return { "require_batched_execution_for_custom_microbatch_strategy": self.require_batched_execution_for_custom_microbatch_strategy, "require_explicit_package_overrides_for_builtin_materializations": self.require_explicit_package_overrides_for_builtin_materializations, "require_resource_names_without_spaces": self.require_resource_names_without_spaces, "source_freshness_run_project_hooks": self.source_freshness_run_project_hooks, "skip_nodes_if_on_run_start_fails": self.skip_nodes_if_on_run_start_fails, "state_modified_compare_more_unrendered_values": self.state_modified_compare_more_unrendered_values, "state_modified_compare_vars": self.state_modified_compare_vars, "require_yaml_configuration_for_mf_time_spines": self.require_yaml_configuration_for_mf_time_spines, "require_nested_cumulative_type_params": self.require_nested_cumulative_type_params, "validate_macro_args": self.validate_macro_args, "require_all_warnings_handled_by_warn_error": self.require_all_warnings_handled_by_warn_error, "require_generic_test_arguments_property": self.require_generic_test_arguments_property, "require_unique_project_resource_names": self.require_unique_project_resource_names, "require_ref_searches_node_package_before_root": self.require_ref_searches_node_package_before_root, "require_valid_schema_from_generate_schema_name": self.require_valid_schema_from_generate_schema_name, "require_sql_header_in_test_configs": self.require_sql_header_in_test_configs, "support_custom_ref_kwargs": self.support_custom_ref_kwargs, } @dataclass class ProfileConfig(dbtClassMixin): profile_name: str target_name: str threads: int # TODO: make this a dynamic union of some kind? credentials: Optional[Dict[str, Any]] @dataclass class ConfiguredQuoting(Quoting): identifier: bool = True schema: bool = True database: Optional[bool] = None project: Optional[bool] = None snowflake_ignore_case: Optional[bool] = None @dataclass class Configuration(Project, ProfileConfig): cli_vars: Dict[str, Any] = field( default_factory=dict, metadata={"preserve_underscore": True}, ) quoting: Optional[ConfiguredQuoting] = None @dataclass class ProjectList(dbtClassMixin): projects: Dict[str, Project] ================================================ FILE: core/dbt/contracts/results.py ================================================ # flake8: noqa # This file is temporary, in order to not break various adapter tests, etc, until # they are updated to use the new locations. from dbt.artifacts.schemas.base import ( ArtifactMixin, BaseArtifactMetadata, VersionedSchema, schema_version, ) from dbt.artifacts.schemas.catalog import ( CatalogArtifact, CatalogKey, CatalogMetadata, CatalogResults, CatalogTable, ColumnMetadata, StatsItem, TableMetadata, ) from dbt.artifacts.schemas.freshness import ( FreshnessErrorEnum, FreshnessExecutionResultArtifact, FreshnessMetadata, FreshnessNodeOutput, FreshnessNodeResult, FreshnessResult, PartialSourceFreshnessResult, SourceFreshnessOutput, SourceFreshnessResult, SourceFreshnessRuntimeError, process_freshness_result, ) from dbt.artifacts.schemas.results import ( BaseResult, ExecutionResult, FreshnessStatus, NodeResult, NodeStatus, RunningStatus, RunStatus, TestStatus, TimingInfo, collect_timing_info, ) from dbt.artifacts.schemas.run import ( RunExecutionResult, RunResult, RunResultsArtifact, RunResultsMetadata, process_run_result, ) ================================================ FILE: core/dbt/contracts/selection.py ================================================ from dataclasses import dataclass from typing import Any, Dict, List, Union from dbt_common.dataclass_schema import dbtClassMixin @dataclass class SelectorDefinition(dbtClassMixin): name: str definition: Union[str, Dict[str, Any]] description: str = "" default: bool = False @dataclass class SelectorFile(dbtClassMixin): selectors: List[SelectorDefinition] version: int = 2 # @dataclass # class SelectorCollection: # packages: Dict[str, List[SelectorFile]] = field(default_factory=dict) ================================================ FILE: core/dbt/contracts/sql.py ================================================ import uuid from dataclasses import dataclass, field from datetime import datetime, timezone from typing import Any, Dict, List, Optional, Sequence from dbt.artifacts.schemas.base import VersionedSchema, schema_version from dbt.artifacts.schemas.results import ExecutionResult, TimingInfo from dbt.artifacts.schemas.run import RunExecutionResult, RunResult, RunResultsArtifact from dbt.contracts.graph.nodes import ResultNode from dbt.events.types import ArtifactWritten from dbt_common.dataclass_schema import dbtClassMixin from dbt_common.events.functions import fire_event TaskTags = Optional[Dict[str, Any]] TaskID = uuid.UUID # Outputs @dataclass class RemoteCompileResultMixin(VersionedSchema): raw_code: str compiled_code: str node: ResultNode timing: List[TimingInfo] @dataclass @schema_version("remote-compile-result", 1) class RemoteCompileResult(RemoteCompileResultMixin): generated_at: datetime = field( default_factory=lambda: datetime.now(timezone.utc).replace(tzinfo=None) ) @property def error(self) -> None: # TODO: Can we delete this? It's never set anywhere else and never accessed return None @dataclass @schema_version("remote-execution-result", 1) class RemoteExecutionResult(ExecutionResult): results: Sequence[RunResult] args: Dict[str, Any] = field(default_factory=dict) generated_at: datetime = field( default_factory=lambda: datetime.now(timezone.utc).replace(tzinfo=None) ) def write(self, path: str) -> None: writable = RunResultsArtifact.from_execution_results( generated_at=self.generated_at, results=self.results, elapsed_time=self.elapsed_time, args=self.args, ) writable.write(path) fire_event(ArtifactWritten(artifact_type=writable.__class__.__name__, artifact_path=path)) @classmethod def from_local_result( cls, base: RunExecutionResult, ) -> "RemoteExecutionResult": return cls( generated_at=base.generated_at, results=base.results, elapsed_time=base.elapsed_time, args=base.args, ) @dataclass class ResultTable(dbtClassMixin): column_names: List[str] rows: List[Any] @dataclass @schema_version("remote-run-result", 1) class RemoteRunResult(RemoteCompileResultMixin): table: ResultTable generated_at: datetime = field( default_factory=lambda: datetime.now(timezone.utc).replace(tzinfo=None) ) ================================================ FILE: core/dbt/contracts/state.py ================================================ from pathlib import Path from typing import Optional from dbt.artifacts.exceptions import IncompatibleSchemaError from dbt.artifacts.schemas.freshness import FreshnessExecutionResultArtifact from dbt.artifacts.schemas.manifest import WritableManifest from dbt.artifacts.schemas.run import RunResultsArtifact from dbt.constants import RUN_RESULTS_FILE_NAME from dbt.contracts.graph.manifest import Manifest from dbt.events.types import WarnStateTargetEqual from dbt_common.events.functions import fire_event def load_result_state(results_path) -> Optional[RunResultsArtifact]: if results_path.exists() and results_path.is_file(): try: return RunResultsArtifact.read_and_check_versions(str(results_path)) except IncompatibleSchemaError as exc: exc.add_filename(str(results_path)) raise return None class PreviousState: def __init__(self, state_path: Path, target_path: Path, project_root: Path) -> None: self.state_path: Path = state_path self.target_path: Path = target_path self.project_root: Path = project_root self.manifest: Optional[Manifest] = None self.results: Optional[RunResultsArtifact] = None self.sources: Optional[FreshnessExecutionResultArtifact] = None self.sources_current: Optional[FreshnessExecutionResultArtifact] = None if self.state_path == self.target_path: fire_event(WarnStateTargetEqual(state_path=str(self.state_path))) # Note: if state_path is absolute, project_root will be ignored. manifest_path = self.project_root / self.state_path / "manifest.json" if manifest_path.exists() and manifest_path.is_file(): try: writable_manifest = WritableManifest.read_and_check_versions(str(manifest_path)) self.manifest = Manifest.from_writable_manifest(writable_manifest) except IncompatibleSchemaError as exc: exc.add_filename(str(manifest_path)) raise results_path = self.project_root / self.state_path / RUN_RESULTS_FILE_NAME self.results = load_result_state(results_path) sources_path = self.project_root / self.state_path / "sources.json" if sources_path.exists() and sources_path.is_file(): try: self.sources = FreshnessExecutionResultArtifact.read_and_check_versions( str(sources_path) ) except IncompatibleSchemaError as exc: exc.add_filename(str(sources_path)) raise sources_current_path = self.project_root / self.target_path / "sources.json" if sources_current_path.exists() and sources_current_path.is_file(): try: self.sources_current = FreshnessExecutionResultArtifact.read_and_check_versions( str(sources_current_path) ) except IncompatibleSchemaError as exc: exc.add_filename(str(sources_current_path)) raise ================================================ FILE: core/dbt/contracts/util.py ================================================ from typing import Any, List, Tuple # Leave imports of `Mergeable` to preserve import paths from dbt_common.contracts.util import Mergeable # noqa:F401 from dbt_common.dataclass_schema import ValidatedStringMixin, ValidationError SourceKey = Tuple[str, str] def list_str() -> List[str]: """Mypy gets upset about stuff like: from dataclasses import dataclass, field from typing import Optional, List @dataclass class Foo: x: Optional[List[str]] = field(default_factory=list) Because `list` could be any kind of list, I guess """ return [] class Identifier(ValidatedStringMixin): """Our definition of a valid Identifier is the same as what's valid for an unquoted database table name. That is: 1. It can contain a-z, A-Z, 0-9, and _ 1. It cannot start with a number """ ValidationRegex = r"^[^\d\W]\w*$" @classmethod def is_valid(cls, value: Any) -> bool: if not isinstance(value, str): return False try: cls.validate(value) except ValidationError: return False return True ================================================ FILE: core/dbt/deprecations.py ================================================ import abc from collections import defaultdict from dataclasses import dataclass from typing import Any, Callable, ClassVar, DefaultDict, Dict, List, Optional import dbt.tracking from dbt.events import types as core_types from dbt.flags import get_flags from dbt_common.dataclass_schema import dbtClassMixin from dbt_common.events.functions import fire_event, warn_or_error from dbt_common.events.types import Note class DBTDeprecation: _name: ClassVar[Optional[str]] = None _event: ClassVar[Optional[str]] = None _is_preview: ClassVar[bool] = False @property def name(self) -> str: if self._name is not None: return self._name raise NotImplementedError("name not implemented for {}".format(self)) def track_deprecation_warn(self) -> None: if dbt.tracking.active_user is not None: dbt.tracking.track_deprecation_warn({"deprecation_name": self.name}) @property def event(self) -> abc.ABCMeta: if self._event is not None: module_path = core_types class_name = self._event try: return getattr(module_path, class_name) except AttributeError: msg = f"Event Class `{class_name}` is not defined in `{module_path}`" raise NameError(msg) raise NotImplementedError("event not implemented for {}".format(self._event)) def preview(self, base_event: abc.ABCMeta) -> None: note_event = Note(msg=base_event.message()) # type: ignore fire_event(note_event) def show(self, *args, **kwargs) -> None: if self._is_preview: base_event = self.event(**kwargs) self.preview(base_event) else: flags = get_flags() if self.name not in active_deprecations or flags.show_all_deprecations: event = self.event(**kwargs) warn_or_error(event) self.track_deprecation_warn() active_deprecations[self.name] += 1 class PackageRedirectDeprecation(DBTDeprecation): _name = "package-redirect" _event = "PackageRedirectDeprecation" class PackageInstallPathDeprecation(DBTDeprecation): _name = "install-packages-path" _event = "PackageInstallPathDeprecation" # deprecations with a pattern of `project-config-*` for the name are not hardcoded # they are called programatically via the pattern below class ConfigSourcePathDeprecation(DBTDeprecation): _name = "project-config-source-paths" _event = "ConfigSourcePathDeprecation" class ConfigDataPathDeprecation(DBTDeprecation): _name = "project-config-data-paths" _event = "ConfigDataPathDeprecation" class ConfigLogPathDeprecation(DBTDeprecation): _name = "project-config-log-path" _event = "ConfigLogPathDeprecation" class ConfigTargetPathDeprecation(DBTDeprecation): _name = "project-config-target-path" _event = "ConfigTargetPathDeprecation" def renamed_method(old_name: str, new_name: str): class AdapterDeprecationWarning(DBTDeprecation): _name = "adapter:{}".format(old_name) _event = "AdapterDeprecationWarning" dep = AdapterDeprecationWarning() deprecations_list.append(dep) deprecations[dep.name] = dep class MetricAttributesRenamed(DBTDeprecation): _name = "metric-attr-renamed" _event = "MetricAttributesRenamed" class ExposureNameDeprecation(DBTDeprecation): _name = "exposure-name" _event = "ExposureNameDeprecation" class CollectFreshnessReturnSignature(DBTDeprecation): _name = "collect-freshness-return-signature" _event = "CollectFreshnessReturnSignature" class ProjectFlagsMovedDeprecation(DBTDeprecation): _name = "project-flags-moved" _event = "ProjectFlagsMovedDeprecation" class PackageMaterializationOverrideDeprecation(DBTDeprecation): _name = "package-materialization-override" _event = "PackageMaterializationOverrideDeprecation" class ResourceNamesWithSpacesDeprecation(DBTDeprecation): _name = "resource-names-with-spaces" _event = "ResourceNamesWithSpacesDeprecation" class SourceFreshnessProjectHooksNotRun(DBTDeprecation): _name = "source-freshness-project-hooks" _event = "SourceFreshnessProjectHooksNotRun" class MFTimespineWithoutYamlConfigurationDeprecation(DBTDeprecation): _name = "mf-timespine-without-yaml-configuration" _event = "MFTimespineWithoutYamlConfigurationDeprecation" class MFCumulativeTypeParamsDeprecation(DBTDeprecation): _name = "mf-cumulative-type-params-deprecation" _event = "MFCumulativeTypeParamsDeprecation" class MicrobatchMacroOutsideOfBatchesDeprecation(DBTDeprecation): _name = "microbatch-macro-outside-of-batches-deprecation" _event = "MicrobatchMacroOutsideOfBatchesDeprecation" class GenericJSONSchemaValidationDeprecation(DBTDeprecation): _name = "generic-json-schema-validation-deprecation" _event = "GenericJSONSchemaValidationDeprecation" _is_preview = True class UnexpectedJinjaBlockDeprecation(DBTDeprecation): _name = "unexpected-jinja-block-deprecation" _event = "UnexpectedJinjaBlockDeprecation" class DuplicateYAMLKeysDeprecation(DBTDeprecation): _name = "duplicate-yaml-keys-deprecation" _event = "DuplicateYAMLKeysDeprecation" class CustomTopLevelKeyDeprecation(DBTDeprecation): _name = "custom-top-level-key-deprecation" _event = "CustomTopLevelKeyDeprecation" class CustomKeyInConfigDeprecation(DBTDeprecation): _name = "custom-key-in-config-deprecation" _event = "CustomKeyInConfigDeprecation" class CustomKeyInObjectDeprecation(DBTDeprecation): _name = "custom-key-in-object-deprecation" _event = "CustomKeyInObjectDeprecation" class WEOInlcudeExcludeDeprecation(DBTDeprecation): _name = "weo-include-exclude-deprecation" _event = "WEOIncludeExcludeDeprecation" class CustomOutputPathInSourceFreshnessDeprecation(DBTDeprecation): _name = "custom-output-path-in-source-freshness-deprecation" _event = "CustomOutputPathInSourceFreshnessDeprecation" class SourceOverrideDeprecation(DBTDeprecation): _name = "source-override-deprecation" _event = "SourceOverrideDeprecation" class PropertyMovedToConfigDeprecation(DBTDeprecation): _name = "property-moved-to-config-deprecation" _event = "PropertyMovedToConfigDeprecation" class ModelParamUsageDeprecation(DBTDeprecation): _name = "model-param-usage-deprecation" _event = "ModelParamUsageDeprecation" class EnvironmentVariableNamespaceDeprecation(DBTDeprecation): _name = "environment-variable-namespace-deprecation" _event = "EnvironmentVariableNamespaceDeprecation" class MissingPlusPrefixDeprecation(DBTDeprecation): _name = "missing-plus-prefix-in-config-deprecation" _event = "MissingPlusPrefixDeprecation" class ArgumentsPropertyInGenericTestDeprecation(DBTDeprecation): _name = "arguments-property-in-generic-test-deprecation" _event = "ArgumentsPropertyInGenericTestDeprecation" class MissingArgumentsPropertyInGenericTestDeprecation(DBTDeprecation): _name = "missing-arguments-property-in-generic-test-deprecation" _event = "MissingArgumentsPropertyInGenericTestDeprecation" class ModulesItertoolsUsageDeprecation(DBTDeprecation): _name = "modules-itertools-usage-deprecation" _event = "ModulesItertoolsUsageDeprecation" class DuplicateNameDistinctNodeTypesDeprecation(DBTDeprecation): _name = "duplicate-name-distinct-node-types-deprecation" _event = "DuplicateNameDistinctNodeTypesDeprecation" class TimeDimensionsRequireGranularityDeprecation(DBTDeprecation): _name = "time-dimensions-require-granularity-deprecation" _event = "TimeDimensionsRequireGranularityDeprecation" class GenericSemanticLayerDeprecation(DBTDeprecation): _name = "generic-semantic-layer-deprecation" _event = "GenericSemanticLayerDeprecation" class GenerateSchemaNameNullValueDeprecation(DBTDeprecation): _name = "generate-schema-name-null-value-deprecation" _event = "GenerateSchemaNameNullValueDeprecation" def renamed_env_var(old_name: str, new_name: str): class EnvironmentVariableRenamed(DBTDeprecation): _name = f"environment-variable-renamed:{old_name}" _event = "EnvironmentVariableRenamed" dep = EnvironmentVariableRenamed() deprecations_list.append(dep) deprecations[dep.name] = dep def cb(): dep.show(old_name=old_name, new_name=new_name) return cb def warn(name: str, *args, **kwargs) -> None: if name not in deprecations: # this should (hopefully) never happen raise RuntimeError("Error showing deprecation warning: {}".format(name)) deprecations[name].show(*args, **kwargs) def buffer(name: str, *args, **kwargs): def show_callback(): deprecations[name].show(*args, **kwargs) buffered_deprecations.append(show_callback) def show_deprecations_summary() -> None: summaries: List[Dict[str, Any]] = [] for deprecation, occurrences in active_deprecations.items(): deprecation_event = deprecations[deprecation].event() summaries.append( DeprecationSummary( event_name=type(deprecation_event).__name__, event_code=deprecation_event.code(), occurrences=occurrences, ).to_msg_dict() ) if len(summaries) > 0: show_all_hint = not get_flags().show_all_deprecations warn_or_error( core_types.DeprecationsSummary(summaries=summaries, show_all_hint=show_all_hint) ) # these are globally available # since modules are only imported once, active_deprecations is a singleton active_deprecations: DefaultDict[str, int] = defaultdict(int) deprecations_list: List[DBTDeprecation] = [ PackageRedirectDeprecation(), PackageInstallPathDeprecation(), ConfigSourcePathDeprecation(), ConfigDataPathDeprecation(), ExposureNameDeprecation(), ConfigLogPathDeprecation(), ConfigTargetPathDeprecation(), CollectFreshnessReturnSignature(), ProjectFlagsMovedDeprecation(), PackageMaterializationOverrideDeprecation(), ResourceNamesWithSpacesDeprecation(), SourceFreshnessProjectHooksNotRun(), MFTimespineWithoutYamlConfigurationDeprecation(), MFCumulativeTypeParamsDeprecation(), MicrobatchMacroOutsideOfBatchesDeprecation(), GenericJSONSchemaValidationDeprecation(), UnexpectedJinjaBlockDeprecation(), DuplicateYAMLKeysDeprecation(), CustomTopLevelKeyDeprecation(), CustomKeyInConfigDeprecation(), CustomKeyInObjectDeprecation(), CustomOutputPathInSourceFreshnessDeprecation(), PropertyMovedToConfigDeprecation(), ModelParamUsageDeprecation(), WEOInlcudeExcludeDeprecation(), SourceOverrideDeprecation(), EnvironmentVariableNamespaceDeprecation(), MissingPlusPrefixDeprecation(), ArgumentsPropertyInGenericTestDeprecation(), MissingArgumentsPropertyInGenericTestDeprecation(), ModulesItertoolsUsageDeprecation(), DuplicateNameDistinctNodeTypesDeprecation(), TimeDimensionsRequireGranularityDeprecation(), GenericSemanticLayerDeprecation(), GenerateSchemaNameNullValueDeprecation(), ] deprecations: Dict[str, DBTDeprecation] = {d.name: d for d in deprecations_list} buffered_deprecations: List[Callable] = [] def reset_deprecations(): active_deprecations.clear() def fire_buffered_deprecations(): [dep_fn() for dep_fn in buffered_deprecations] buffered_deprecations.clear() @dataclass class DeprecationSummary(dbtClassMixin): event_name: str event_code: str occurrences: int def to_msg_dict(self) -> Dict[str, Any]: return { "event_name": self.event_name, "event_code": self.event_code, "occurrences": self.occurrences, } ================================================ FILE: core/dbt/deps/README.md ================================================ # Deps README The deps module is responsible for installing dbt packages into dbt projects. A dbt package is a standalone dbt project with models and macros that solve a specific problem area. More specific information on dbt packages is available on the [docs site](https://docs.getdbt.com/docs/building-a-dbt-project/package-management). # What's a package? See [How do I specify a package?](https://docs.getdbt.com/docs/building-a-dbt-project/package-management#how-do-i-specify-a-package) on the docs site for a detailed explination of the different types of packages supported and expected formats. # Files ## `base.py` Defines the base classes of `PinnedPackage` and `UnpinnedPackage`. `downloads_directory` sets the directory packages will be downloaded to. `_install` has retry logic if the download or untarring process hit exceptions (see `dbt_common.utils.connection_exception_retry`). ## `git.py` Extends `PinnedPackage` and `UnpinnedPackage` specific to dbt packages defined with git urls. ## `local.py` Extends `PinnedPackage` and `UnpinnedPackage` specific to dbt packages defined locally. ## `registry.py` Extends `PinnedPackage` and `UnpinnedPackage` specific to dbt packages defined on the dbt Hub registry. ## `resolver.py` Resolves the package definition into package objects to download. ## `tarball.py` Extends `PinnedPackage` and `UnpinnedPackage` specific to dbt packages defined by a URL to a tarball hosted on an HTTP server. ================================================ FILE: core/dbt/deps/__init__.py ================================================ ================================================ FILE: core/dbt/deps/base.py ================================================ import abc import functools import os import tempfile from contextlib import contextmanager from pathlib import Path from typing import Dict, Generic, List, Optional, TypeVar from dbt.contracts.project import ProjectPackageMetadata from dbt.events.types import DepsSetDownloadDirectory from dbt_common.clients import system from dbt_common.events.functions import fire_event from dbt_common.utils.connection import connection_exception_retry DOWNLOADS_PATH = None def get_downloads_path(): return DOWNLOADS_PATH @contextmanager def downloads_directory(): global DOWNLOADS_PATH remove_downloads = False # the user might have set an environment variable. Set it to that, and do # not remove it when finished. if DOWNLOADS_PATH is None: DOWNLOADS_PATH = os.getenv("DBT_DOWNLOADS_DIR") remove_downloads = False # if we are making a per-run temp directory, remove it at the end of # successful runs if DOWNLOADS_PATH is None: DOWNLOADS_PATH = tempfile.mkdtemp(prefix="dbt-downloads-") remove_downloads = True system.make_directory(DOWNLOADS_PATH) fire_event(DepsSetDownloadDirectory(path=DOWNLOADS_PATH)) yield DOWNLOADS_PATH if remove_downloads: system.rmtree(DOWNLOADS_PATH) DOWNLOADS_PATH = None class BasePackage(metaclass=abc.ABCMeta): @abc.abstractproperty def name(self) -> str: raise NotImplementedError def all_names(self) -> List[str]: return [self.name] @abc.abstractmethod def source_type(self) -> str: raise NotImplementedError class PinnedPackage(BasePackage): def __init__(self) -> None: self._cached_metadata: Optional[ProjectPackageMetadata] = None def __str__(self) -> str: version = self.get_version() if not version: return self.name return "{}@{}".format(self.name, version) @abc.abstractmethod def get_version(self) -> Optional[str]: raise NotImplementedError @abc.abstractmethod def _fetch_metadata(self, project, renderer): raise NotImplementedError @abc.abstractmethod def install(self, project, renderer): raise NotImplementedError @abc.abstractmethod def nice_version_name(self): raise NotImplementedError @abc.abstractmethod def to_dict(self) -> Dict[str, str]: raise NotImplementedError def fetch_metadata(self, project, renderer): if not self._cached_metadata: self._cached_metadata = self._fetch_metadata(project, renderer) return self._cached_metadata def get_project_name(self, project, renderer): metadata = self.fetch_metadata(project, renderer) return metadata.name def get_installation_path(self, project, renderer): dest_dirname = self.get_project_name(project, renderer) return os.path.join(project.packages_install_path, dest_dirname) def get_subdirectory(self): return None def _install(self, project, renderer): metadata = self.fetch_metadata(project, renderer) tar_name = f"{self.package}.{self.version}.tar.gz" tar_path = (Path(get_downloads_path()) / tar_name).resolve(strict=False) system.make_directory(str(tar_path.parent)) download_url = metadata.downloads.tarball deps_path = project.packages_install_path package_name = self.get_project_name(project, renderer) download_untar_fn = functools.partial( self.download_and_untar, download_url, str(tar_path), deps_path, package_name ) connection_exception_retry(download_untar_fn, 5) def download_and_untar(self, download_url, tar_path, deps_path, package_name): """ Sometimes the download of the files fails and we want to retry. Sometimes the download appears successful but the file did not make it through as expected (generally due to a github incident). Either way we want to retry downloading and untarring to see if we can get a success. Call this within `connection_exception_retry` """ system.download(download_url, tar_path) system.untar_package(tar_path, deps_path, package_name) SomePinned = TypeVar("SomePinned", bound=PinnedPackage) SomeUnpinned = TypeVar("SomeUnpinned", bound="UnpinnedPackage") class UnpinnedPackage(Generic[SomePinned], BasePackage): @classmethod @abc.abstractmethod def from_contract(cls, contract): raise NotImplementedError @abc.abstractmethod def incorporate(self: SomeUnpinned, other: SomeUnpinned) -> SomeUnpinned: raise NotImplementedError @abc.abstractmethod def resolved(self) -> SomePinned: raise NotImplementedError ================================================ FILE: core/dbt/deps/git.py ================================================ import os from typing import Dict, List, Optional from dbt.clients import git from dbt.config.project import PartialProject, Project from dbt.config.renderer import PackageRenderer from dbt.contracts.project import GitPackage, ProjectPackageMetadata from dbt.deps.base import PinnedPackage, UnpinnedPackage, get_downloads_path from dbt.events.types import DepsScrubbedPackageName, DepsUnpinned, EnsureGitInstalled from dbt.exceptions import MultipleVersionGitDepsError from dbt.utils import md5 from dbt_common.clients import system from dbt_common.events.functions import ( env_secrets, fire_event, scrub_secrets, warn_or_error, ) from dbt_common.exceptions import ExecutableError def md5sum(s: str): return md5(s, "latin-1") class GitPackageMixin: def __init__( self, git: str, git_unrendered: str, subdirectory: Optional[str] = None, ) -> None: super().__init__() self.git = git self.git_unrendered = git_unrendered self.subdirectory = subdirectory @property def name(self): return f"{self.git}/{self.subdirectory}" if self.subdirectory else self.git def source_type(self) -> str: return "git" class GitPinnedPackage(GitPackageMixin, PinnedPackage): def __init__( self, git: str, git_unrendered: str, revision: str, warn_unpinned: bool = True, subdirectory: Optional[str] = None, ) -> None: super().__init__(git, git_unrendered, subdirectory) self.revision = revision self.warn_unpinned = warn_unpinned self.subdirectory = subdirectory self._checkout_name = md5sum(self.name) def to_dict(self) -> Dict[str, str]: git_scrubbed = scrub_secrets(self.git_unrendered, env_secrets()) if self.git_unrendered != git_scrubbed: warn_or_error(DepsScrubbedPackageName(package_name=git_scrubbed)) ret = { "git": git_scrubbed, "revision": self.revision, } if self.subdirectory: ret["subdirectory"] = self.subdirectory return ret def get_version(self): return self.revision def get_subdirectory(self): return self.subdirectory def nice_version_name(self): if self.revision == "HEAD": return "HEAD (default revision)" else: return "revision {}".format(self.revision) def _checkout(self): """Performs a shallow clone of the repository into the downloads directory. This function can be called repeatedly. If the project has already been checked out at this version, it will be a no-op. Returns the path to the checked out directory.""" try: dir_ = git.clone_and_checkout( self.git, get_downloads_path(), revision=self.revision, dirname=self._checkout_name, subdirectory=self.subdirectory, ) except ExecutableError as exc: if exc.cmd and exc.cmd[0] == "git": fire_event(EnsureGitInstalled()) raise return os.path.join(get_downloads_path(), dir_) def _fetch_metadata( self, project: Project, renderer: PackageRenderer ) -> ProjectPackageMetadata: path = self._checkout() # raise warning (or error) if this package is not pinned if (self.revision == "HEAD" or self.revision in ("main", "master")) and self.warn_unpinned: warn_or_error(DepsUnpinned(revision=self.revision, git=self.git)) # now overwrite 'revision' with actual commit SHA self.revision = git.get_current_sha(path) partial = PartialProject.from_project_root(path) return partial.render_package_metadata(renderer) def install(self, project, renderer): dest_path = self.get_installation_path(project, renderer) if os.path.exists(dest_path): if system.path_is_symlink(dest_path): system.remove_file(dest_path) else: system.rmdir(dest_path) system.move(self._checkout(), dest_path) class GitUnpinnedPackage(GitPackageMixin, UnpinnedPackage[GitPinnedPackage]): def __init__( self, git: str, git_unrendered: str, revisions: List[str], warn_unpinned: bool = True, subdirectory: Optional[str] = None, ) -> None: super().__init__(git, git_unrendered, subdirectory) self.revisions = revisions self.warn_unpinned = warn_unpinned self.subdirectory = subdirectory @classmethod def from_contract(cls, contract: GitPackage) -> "GitUnpinnedPackage": revisions = contract.get_revisions() # we want to map None -> True warn_unpinned = contract.warn_unpinned is not False return cls( git=contract.git, git_unrendered=(contract.unrendered.get("git") or contract.git), revisions=revisions, warn_unpinned=warn_unpinned, subdirectory=contract.subdirectory, ) def all_names(self) -> List[str]: if self.git.endswith(".git"): other = self.git[:-4] else: other = self.git + ".git" if self.subdirectory: git_name = f"{self.git}/{self.subdirectory}" other = f"{other}/{self.subdirectory}" else: git_name = self.git return [git_name, other] def incorporate(self, other: "GitUnpinnedPackage") -> "GitUnpinnedPackage": warn_unpinned = self.warn_unpinned and other.warn_unpinned return GitUnpinnedPackage( git=self.git, git_unrendered=self.git_unrendered, revisions=self.revisions + other.revisions, warn_unpinned=warn_unpinned, subdirectory=self.subdirectory, ) def resolved(self) -> GitPinnedPackage: requested = set(self.revisions) if len(requested) == 0: requested = {"HEAD"} elif len(requested) > 1: raise MultipleVersionGitDepsError(self.name, requested) return GitPinnedPackage( git=self.git, git_unrendered=self.git_unrendered, revision=requested.pop(), warn_unpinned=self.warn_unpinned, subdirectory=self.subdirectory, ) ================================================ FILE: core/dbt/deps/local.py ================================================ import shutil from typing import Dict from dbt.config.project import PartialProject, Project from dbt.config.renderer import PackageRenderer from dbt.contracts.project import LocalPackage, ProjectPackageMetadata from dbt.deps.base import PinnedPackage, UnpinnedPackage from dbt.events.types import DepsCreatingLocalSymlink, DepsSymlinkNotAvailable from dbt_common.clients import system from dbt_common.events.functions import fire_event class LocalPackageMixin: def __init__(self, local: str) -> None: super().__init__() self.local = local @property def name(self): return self.local def source_type(self): return "local" class LocalPinnedPackage(LocalPackageMixin, PinnedPackage): def __init__(self, local: str) -> None: super().__init__(local) def to_dict(self) -> Dict[str, str]: return { "local": self.local, } def get_version(self): return None def nice_version_name(self): return "<local @ {}>".format(self.local) def resolve_path(self, project): return system.resolve_path_from_base( self.local, project.project_root, ) def _fetch_metadata( self, project: Project, renderer: PackageRenderer ) -> ProjectPackageMetadata: partial = PartialProject.from_project_root(self.resolve_path(project)) return partial.render_package_metadata(renderer) def install(self, project, renderer): src_path = self.resolve_path(project) dest_path = self.get_installation_path(project, renderer) if system.path_exists(dest_path): if not system.path_is_symlink(dest_path): system.rmdir(dest_path) else: system.remove_file(dest_path) try: fire_event(DepsCreatingLocalSymlink()) system.make_symlink(src_path, dest_path) except OSError: fire_event(DepsSymlinkNotAvailable()) shutil.copytree(src_path, dest_path) class LocalUnpinnedPackage(LocalPackageMixin, UnpinnedPackage[LocalPinnedPackage]): @classmethod def from_contract(cls, contract: LocalPackage) -> "LocalUnpinnedPackage": return cls(local=contract.local) def incorporate(self, other: "LocalUnpinnedPackage") -> "LocalUnpinnedPackage": return LocalUnpinnedPackage(local=self.local) def resolved(self) -> LocalPinnedPackage: return LocalPinnedPackage(local=self.local) ================================================ FILE: core/dbt/deps/registry.py ================================================ from typing import Dict, List from dbt.clients import registry from dbt.contracts.project import RegistryPackage, RegistryPackageMetadata from dbt.deps.base import PinnedPackage, UnpinnedPackage from dbt.exceptions import ( DependencyError, PackageNotFoundError, PackageVersionNotFoundError, ) from dbt.flags import get_flags from dbt.version import get_installed_version from dbt_common import semver from dbt_common.exceptions import VersionsNotCompatibleError class RegistryPackageMixin: def __init__(self, package: str) -> None: super().__init__() self.package = package @property def name(self): return self.package def source_type(self) -> str: return "hub" class RegistryPinnedPackage(RegistryPackageMixin, PinnedPackage): def __init__(self, package: str, version: str, version_latest: str) -> None: super().__init__(package) self.version = version self.version_latest = version_latest @property def name(self): return self.package def to_dict(self) -> Dict[str, str]: return { "package": self.package, "version": self.version, } def source_type(self): return "hub" def get_version(self): return self.version def get_version_latest(self): return self.version_latest def nice_version_name(self): return "version {}".format(self.version) def _fetch_metadata(self, project, renderer) -> RegistryPackageMetadata: dct = registry.package_version(self.package, self.version) return RegistryPackageMetadata.from_dict(dct) def install(self, project, renderer): self._install(project, renderer) class RegistryUnpinnedPackage(RegistryPackageMixin, UnpinnedPackage[RegistryPinnedPackage]): def __init__( self, package: str, versions: List[semver.VersionSpecifier], install_prerelease: bool ) -> None: super().__init__(package) self.versions = versions self.install_prerelease = install_prerelease def _check_in_index(self): index = registry.index_cached() if self.package not in index: raise PackageNotFoundError(self.package) @classmethod def from_contract(cls, contract: RegistryPackage) -> "RegistryUnpinnedPackage": raw_version = contract.get_versions() versions = [semver.VersionSpecifier.from_version_string(v) for v in raw_version] return cls( package=contract.package, versions=versions, install_prerelease=bool(contract.install_prerelease), ) def incorporate(self, other: "RegistryUnpinnedPackage") -> "RegistryUnpinnedPackage": return RegistryUnpinnedPackage( package=self.package, install_prerelease=self.install_prerelease, versions=self.versions + other.versions, ) def resolved(self) -> RegistryPinnedPackage: self._check_in_index() try: range_ = semver.reduce_versions(*self.versions) except VersionsNotCompatibleError as e: new_msg = "Version error for package {}: {}".format(self.name, e) raise DependencyError(new_msg) from e flags = get_flags() should_version_check = bool(flags.VERSION_CHECK) dbt_version = get_installed_version() compatible_versions = registry.get_compatible_versions( self.package, dbt_version, should_version_check ) prerelease_version_specified = any(bool(version.prerelease) for version in self.versions) installable = semver.filter_installable( compatible_versions, self.install_prerelease or prerelease_version_specified ) if installable: # for now, pick a version and then recurse. later on, # we'll probably want to traverse multiple options # so we can match packages. not going to make a difference # right now. target = semver.resolve_to_specific_version(range_, installable) else: target = None if not target: # raise an exception if no installable target version is found raise PackageVersionNotFoundError( self.package, range_, installable, should_version_check ) latest_compatible = installable[-1] return RegistryPinnedPackage( package=self.package, version=target, version_latest=latest_compatible ) ================================================ FILE: core/dbt/deps/resolver.py ================================================ from dataclasses import dataclass, field from typing import Any, Dict, Iterator, List, NoReturn, Set, Type from dbt.config import Project from dbt.config.renderer import PackageRenderer from dbt.contracts.project import ( GitPackage, LocalPackage, PackageSpec, PrivatePackage, RegistryPackage, TarballPackage, ) from dbt.deps.base import BasePackage, PinnedPackage, UnpinnedPackage from dbt.deps.git import GitUnpinnedPackage from dbt.deps.local import LocalUnpinnedPackage from dbt.deps.registry import RegistryUnpinnedPackage from dbt.deps.tarball import TarballUnpinnedPackage from dbt.exceptions import ( DependencyError, DuplicateDependencyToRootError, DuplicateProjectDependencyError, MismatchedDependencyTypeError, ) @dataclass class PackageListing: packages: Dict[str, UnpinnedPackage] = field(default_factory=dict) def __len__(self): return len(self.packages) def __bool__(self): return bool(self.packages) def _pick_key(self, key: BasePackage) -> str: for name in key.all_names(): if name in self.packages: return name return key.name def __contains__(self, key: BasePackage): for name in key.all_names(): if name in self.packages: return True def __getitem__(self, key: BasePackage): key_str: str = self._pick_key(key) return self.packages[key_str] def __setitem__(self, key: BasePackage, value): key_str: str = self._pick_key(key) self.packages[key_str] = value def _mismatched_types(self, old: UnpinnedPackage, new: UnpinnedPackage) -> NoReturn: raise MismatchedDependencyTypeError(new, old) def incorporate(self, package: UnpinnedPackage): key: str = self._pick_key(package) if key in self.packages: existing: UnpinnedPackage = self.packages[key] if not isinstance(existing, type(package)): self._mismatched_types(existing, package) self.packages[key] = existing.incorporate(package) else: self.packages[key] = package def update_from(self, src: List[PackageSpec]) -> None: pkg: UnpinnedPackage for contract in src: if isinstance(contract, LocalPackage): pkg = LocalUnpinnedPackage.from_contract(contract) elif isinstance(contract, TarballPackage): pkg = TarballUnpinnedPackage.from_contract(contract) elif isinstance(contract, GitPackage): pkg = GitUnpinnedPackage.from_contract(contract) elif isinstance(contract, PrivatePackage): raise DependencyError( f'Cannot resolve private package {contract.private} because git provider integration is missing. Please use a "git" package instead.' ) elif isinstance(contract, RegistryPackage): pkg = RegistryUnpinnedPackage.from_contract(contract) else: raise DependencyError("Invalid package type {}".format(type(contract))) self.incorporate(pkg) @classmethod def from_contracts(cls: Type["PackageListing"], src: List[PackageSpec]) -> "PackageListing": self = cls({}) self.update_from(src) return self def resolved(self) -> List[PinnedPackage]: return [p.resolved() for p in self.packages.values()] def __iter__(self) -> Iterator[UnpinnedPackage]: return iter(self.packages.values()) def _check_for_duplicate_project_names( final_deps: List[PinnedPackage], project: Project, renderer: PackageRenderer, ): seen: Set[str] = set() for package in final_deps: project_name = package.get_project_name(project, renderer) if project_name in seen: raise DuplicateProjectDependencyError(project_name) elif project_name == project.project_name: raise DuplicateDependencyToRootError(project_name) seen.add(project_name) def resolve_packages( packages: List[PackageSpec], project: Project, cli_vars: Dict[str, Any], ) -> List[PinnedPackage]: pending = PackageListing.from_contracts(packages) final = PackageListing() renderer = PackageRenderer(cli_vars) while pending: next_pending = PackageListing() # resolve the dependency in question for package in pending: final.incorporate(package) target = final[package].resolved().fetch_metadata(project, renderer) next_pending.update_from(target.packages) pending = next_pending resolved = final.resolved() _check_for_duplicate_project_names(resolved, project, renderer) return resolved def resolve_lock_packages(packages: List[PackageSpec]) -> List[PinnedPackage]: lock_packages = PackageListing.from_contracts(packages) final = PackageListing() for package in lock_packages: final.incorporate(package) resolved = final.resolved() return resolved ================================================ FILE: core/dbt/deps/tarball.py ================================================ import functools import os from pathlib import Path from typing import Dict from dbt.config.project import PartialProject from dbt.contracts.project import TarballPackage from dbt.deps.base import PinnedPackage, UnpinnedPackage, get_downloads_path from dbt.events.types import DepsScrubbedPackageName from dbt.exceptions import DependencyError, env_secrets, scrub_secrets from dbt_common.clients import system from dbt_common.events.functions import warn_or_error from dbt_common.utils.connection import connection_exception_retry class TarballPackageMixin: def __init__(self, tarball: str, tarball_unrendered: str) -> None: super().__init__() self.tarball = tarball self.tarball_unrendered = tarball_unrendered @property def name(self): return self.tarball def source_type(self) -> str: return "tarball" class TarballPinnedPackage(TarballPackageMixin, PinnedPackage): def __init__(self, tarball: str, tarball_unrendered: str, package: str) -> None: super().__init__(tarball, tarball_unrendered) self.package = package self.version = "tarball" self.tar_path = os.path.join(Path(get_downloads_path()), self.package) self.untarred_path = f"{self.tar_path}_untarred" @property def name(self): return self.package def to_dict(self) -> Dict[str, str]: tarball_scrubbed = scrub_secrets(self.tarball_unrendered, env_secrets()) if self.tarball_unrendered != tarball_scrubbed: warn_or_error(DepsScrubbedPackageName(package_name=tarball_scrubbed)) return { "tarball": tarball_scrubbed, "name": self.package, } def get_version(self): return self.version def nice_version_name(self): return f"tarball (url: {self.tarball})" def _fetch_metadata(self, project, renderer): """Download and untar the project and parse metadata from the project folder.""" download_untar_fn = functools.partial( self.download_and_untar, self.tarball, self.tar_path, self.untarred_path, self.name ) connection_exception_retry(download_untar_fn, 5) tar_contents = os.listdir(self.untarred_path) if len(tar_contents) != 1: raise DependencyError( f"Incorrect structure for package extracted from {self.tarball}." f"The extracted package needs to follow the structure {self.name}/<package_contents>." ) child_folder = os.listdir(self.untarred_path)[0] self.untarred_path = os.path.join(self.untarred_path, child_folder) partial = PartialProject.from_project_root(self.untarred_path) metadata = partial.render_package_metadata(renderer) metadata.name = self.package if self.package else metadata.name return metadata def install(self, project, renderer): download_untar_fn = functools.partial( self.download_and_untar, self.tarball, self.tar_path, self.untarred_path, self.name ) connection_exception_retry(download_untar_fn, 5) dest_path = self.get_installation_path(project, renderer) if os.path.exists(dest_path): if system.path_is_symlink(dest_path): system.remove_file(dest_path) else: system.rmdir(dest_path) system.move(self.untarred_path, dest_path) class TarballUnpinnedPackage(TarballPackageMixin, UnpinnedPackage[TarballPinnedPackage]): def __init__( self, tarball: str, tarball_unrendered: str, package: str, ) -> None: super().__init__(tarball, tarball_unrendered) # setup to recycle RegistryPinnedPackage fns self.package = package self.version = "tarball" @classmethod def from_contract(cls, contract: TarballPackage) -> "TarballUnpinnedPackage": return cls( tarball=contract.tarball, tarball_unrendered=(contract.unrendered.get("tarball") or contract.tarball), package=contract.name, ) def incorporate(self, other: "TarballUnpinnedPackage") -> "TarballUnpinnedPackage": return TarballUnpinnedPackage( tarball=self.tarball, tarball_unrendered=self.tarball_unrendered, package=self.package ) def resolved(self) -> TarballPinnedPackage: return TarballPinnedPackage( tarball=self.tarball, tarball_unrendered=self.tarball_unrendered, package=self.package ) ================================================ FILE: core/dbt/docs/Makefile ================================================ # Minimal makefile for Sphinx documentation # # You can set these variables from the command line, and also # from the environment for the first two. SPHINXOPTS ?= SPHINXBUILD ?= sphinx-build SOURCEDIR = source BUILDDIR = build # Put it first so that "make" without argument is like "make help". help: @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) .PHONY: help Makefile # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) ================================================ FILE: core/dbt/docs/build/html/.buildinfo ================================================ # Sphinx build info version 1 # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done. config: 0d25ef12a43286020bcd8b805064f01c tags: 645f666f9bcd5a90fca523b33c5a78b7 ================================================ FILE: core/dbt/docs/build/html/_sources/index.rst.txt ================================================ dbt-core's API documentation ============================ How to invoke dbt commands in python runtime -------------------------------------------- Right now the best way to invoke a command from python runtime is to use the `dbtRunner` we exposed .. code-block:: python from dbt.cli.main import dbtRunner cli_args = ['run', '--project-dir', 'jaffle_shop'] # initialize the dbt runner dbt = dbtRunner() # run the command res, success = dbt.invoke(args) You can also pass in pre constructed object into dbtRunner, and we will use those objects instead of loading up from the disk. .. code-block:: python # preload profile and project profile = load_profile(project_dir, {}, 'testing-postgres') project = load_project(project_dir, False, profile, {}) # initialize the runner with pre-loaded profile and project dbt = dbtRunner(profile=profile, project=project) # run the command, this will use the pre-loaded profile and project instead of loading res, success = dbt.invoke(cli_args) For the full example code, you can refer to `core/dbt/cli/example.py` API documentation ----------------- .. dbt_click:: dbt.cli.main:cli ================================================ FILE: core/dbt/docs/build/html/_static/_sphinx_javascript_frameworks_compat.js ================================================ /* * _sphinx_javascript_frameworks_compat.js * ~~~~~~~~~~ * * Compatability shim for jQuery and underscores.js. * * WILL BE REMOVED IN Sphinx 6.0 * xref RemovedInSphinx60Warning * */ /** * select a different prefix for underscore */ $u = _.noConflict(); /** * small helper function to urldecode strings * * See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/decodeURIComponent#Decoding_query_parameters_from_a_URL */ jQuery.urldecode = function(x) { if (!x) { return x } return decodeURIComponent(x.replace(/\+/g, ' ')); }; /** * small helper function to urlencode strings */ jQuery.urlencode = encodeURIComponent; /** * This function returns the parsed url parameters of the * current request. Multiple values per key are supported, * it will always return arrays of strings for the value parts. */ jQuery.getQueryParameters = function(s) { if (typeof s === 'undefined') s = document.location.search; var parts = s.substr(s.indexOf('?') + 1).split('&'); var result = {}; for (var i = 0; i < parts.length; i++) { var tmp = parts[i].split('=', 2); var key = jQuery.urldecode(tmp[0]); var value = jQuery.urldecode(tmp[1]); if (key in result) result[key].push(value); else result[key] = [value]; } return result; }; /** * highlight a given string on a jquery object by wrapping it in * span elements with the given class name. */ jQuery.fn.highlightText = function(text, className) { function highlight(node, addItems) { if (node.nodeType === 3) { var val = node.nodeValue; var pos = val.toLowerCase().indexOf(text); if (pos >= 0 && !jQuery(node.parentNode).hasClass(className) && !jQuery(node.parentNode).hasClass("nohighlight")) { var span; var isInSVG = jQuery(node).closest("body, svg, foreignObject").is("svg"); if (isInSVG) { span = document.createElementNS("http://www.w3.org/2000/svg", "tspan"); } else { span = document.createElement("span"); span.className = className; } span.appendChild(document.createTextNode(val.substr(pos, text.length))); node.parentNode.insertBefore(span, node.parentNode.insertBefore( document.createTextNode(val.substr(pos + text.length)), node.nextSibling)); node.nodeValue = val.substr(0, pos); if (isInSVG) { var rect = document.createElementNS("http://www.w3.org/2000/svg", "rect"); var bbox = node.parentElement.getBBox(); rect.x.baseVal.value = bbox.x; rect.y.baseVal.value = bbox.y; rect.width.baseVal.value = bbox.width; rect.height.baseVal.value = bbox.height; rect.setAttribute('class', className); addItems.push({ "parent": node.parentNode, "target": rect}); } } } else if (!jQuery(node).is("button, select, textarea")) { jQuery.each(node.childNodes, function() { highlight(this, addItems); }); } } var addItems = []; var result = this.each(function() { highlight(this, addItems); }); for (var i = 0; i < addItems.length; ++i) { jQuery(addItems[i].parent).before(addItems[i].target); } return result; }; /* * backward compatibility for jQuery.browser * This will be supported until firefox bug is fixed. */ if (!jQuery.browser) { jQuery.uaMatch = function(ua) { ua = ua.toLowerCase(); var match = /(chrome)[ \/]([\w.]+)/.exec(ua) || /(webkit)[ \/]([\w.]+)/.exec(ua) || /(opera)(?:.*version|)[ \/]([\w.]+)/.exec(ua) || /(msie) ([\w.]+)/.exec(ua) || ua.indexOf("compatible") < 0 && /(mozilla)(?:.*? rv:([\w.]+)|)/.exec(ua) || []; return { browser: match[ 1 ] || "", version: match[ 2 ] || "0" }; }; jQuery.browser = {}; jQuery.browser[jQuery.uaMatch(navigator.userAgent).browser] = true; } ================================================ FILE: core/dbt/docs/build/html/_static/alabaster.css ================================================ @import url("basic.css"); /* -- page layout ----------------------------------------------------------- */ body { font-family: Georgia, serif; font-size: 17px; background-color: #fff; color: #000; margin: 0; padding: 0; } div.document { width: 940px; margin: 30px auto 0 auto; } div.documentwrapper { float: left; width: 100%; } div.bodywrapper { margin: 0 0 0 220px; } div.sphinxsidebar { width: 220px; font-size: 14px; line-height: 1.5; } hr { border: 1px solid #B1B4B6; } div.body { background-color: #fff; color: #3E4349; padding: 0 30px 0 30px; } div.body > .section { text-align: left; } div.footer { width: 940px; margin: 20px auto 30px auto; font-size: 14px; color: #888; text-align: right; } div.footer a { color: #888; } p.caption { font-family: inherit; font-size: inherit; } div.relations { display: none; } div.sphinxsidebar a { color: #444; text-decoration: none; border-bottom: 1px dotted #999; } div.sphinxsidebar a:hover { border-bottom: 1px solid #999; } div.sphinxsidebarwrapper { padding: 18px 10px; } div.sphinxsidebarwrapper p.logo { padding: 0; margin: -10px 0 0 0px; text-align: center; } div.sphinxsidebarwrapper h1.logo { margin-top: -10px; text-align: center; margin-bottom: 5px; text-align: left; } div.sphinxsidebarwrapper h1.logo-name { margin-top: 0px; } div.sphinxsidebarwrapper p.blurb { margin-top: 0; font-style: normal; } div.sphinxsidebar h3, div.sphinxsidebar h4 { font-family: Georgia, serif; color: #444; font-size: 24px; font-weight: normal; margin: 0 0 5px 0; padding: 0; } div.sphinxsidebar h4 { font-size: 20px; } div.sphinxsidebar h3 a { color: #444; } div.sphinxsidebar p.logo a, div.sphinxsidebar h3 a, div.sphinxsidebar p.logo a:hover, div.sphinxsidebar h3 a:hover { border: none; } div.sphinxsidebar p { color: #555; margin: 10px 0; } div.sphinxsidebar ul { margin: 10px 0; padding: 0; color: #000; } div.sphinxsidebar ul li.toctree-l1 > a { font-size: 120%; } div.sphinxsidebar ul li.toctree-l2 > a { font-size: 110%; } div.sphinxsidebar input { border: 1px solid #CCC; font-family: Georgia, serif; font-size: 1em; } div.sphinxsidebar hr { border: none; height: 1px; color: #AAA; background: #AAA; text-align: left; margin-left: 0; width: 50%; } div.sphinxsidebar .badge { border-bottom: none; } div.sphinxsidebar .badge:hover { border-bottom: none; } /* To address an issue with donation coming after search */ div.sphinxsidebar h3.donation { margin-top: 10px; } /* -- body styles ----------------------------------------------------------- */ a { color: #004B6B; text-decoration: underline; } a:hover { color: #6D4100; text-decoration: underline; } div.body h1, div.body h2, div.body h3, div.body h4, div.body h5, div.body h6 { font-family: Georgia, serif; font-weight: normal; margin: 30px 0px 10px 0px; padding: 0; } div.body h1 { margin-top: 0; padding-top: 0; font-size: 240%; } div.body h2 { font-size: 180%; } div.body h3 { font-size: 150%; } div.body h4 { font-size: 130%; } div.body h5 { font-size: 100%; } div.body h6 { font-size: 100%; } a.headerlink { color: #DDD; padding: 0 4px; text-decoration: none; } a.headerlink:hover { color: #444; background: #EAEAEA; } div.body p, div.body dd, div.body li { line-height: 1.4em; } div.admonition { margin: 20px 0px; padding: 10px 30px; background-color: #EEE; border: 1px solid #CCC; } div.admonition tt.xref, div.admonition code.xref, div.admonition a tt { background-color: #FBFBFB; border-bottom: 1px solid #fafafa; } div.admonition p.admonition-title { font-family: Georgia, serif; font-weight: normal; font-size: 24px; margin: 0 0 10px 0; padding: 0; line-height: 1; } div.admonition p.last { margin-bottom: 0; } div.highlight { background-color: #fff; } dt:target, .highlight { background: #FAF3E8; } div.warning { background-color: #FCC; border: 1px solid #FAA; } div.danger { background-color: #FCC; border: 1px solid #FAA; -moz-box-shadow: 2px 2px 4px #D52C2C; -webkit-box-shadow: 2px 2px 4px #D52C2C; box-shadow: 2px 2px 4px #D52C2C; } div.error { background-color: #FCC; border: 1px solid #FAA; -moz-box-shadow: 2px 2px 4px #D52C2C; -webkit-box-shadow: 2px 2px 4px #D52C2C; box-shadow: 2px 2px 4px #D52C2C; } div.caution { background-color: #FCC; border: 1px solid #FAA; } div.attention { background-color: #FCC; border: 1px solid #FAA; } div.important { background-color: #EEE; border: 1px solid #CCC; } div.note { background-color: #EEE; border: 1px solid #CCC; } div.tip { background-color: #EEE; border: 1px solid #CCC; } div.hint { background-color: #EEE; border: 1px solid #CCC; } div.seealso { background-color: #EEE; border: 1px solid #CCC; } div.topic { background-color: #EEE; } p.admonition-title { display: inline; } p.admonition-title:after { content: ":"; } pre, tt, code { font-family: 'Consolas', 'Menlo', 'DejaVu Sans Mono', 'Bitstream Vera Sans Mono', monospace; font-size: 0.9em; } .hll { background-color: #FFC; margin: 0 -12px; padding: 0 12px; display: block; } img.screenshot { } tt.descname, tt.descclassname, code.descname, code.descclassname { font-size: 0.95em; } tt.descname, code.descname { padding-right: 0.08em; } img.screenshot { -moz-box-shadow: 2px 2px 4px #EEE; -webkit-box-shadow: 2px 2px 4px #EEE; box-shadow: 2px 2px 4px #EEE; } table.docutils { border: 1px solid #888; -moz-box-shadow: 2px 2px 4px #EEE; -webkit-box-shadow: 2px 2px 4px #EEE; box-shadow: 2px 2px 4px #EEE; } table.docutils td, table.docutils th { border: 1px solid #888; padding: 0.25em 0.7em; } table.field-list, table.footnote { border: none; -moz-box-shadow: none; -webkit-box-shadow: none; box-shadow: none; } table.footnote { margin: 15px 0; width: 100%; border: 1px solid #EEE; background: #FDFDFD; font-size: 0.9em; } table.footnote + table.footnote { margin-top: -15px; border-top: none; } table.field-list th { padding: 0 0.8em 0 0; } table.field-list td { padding: 0; } table.field-list p { margin-bottom: 0.8em; } /* Cloned from * https://github.com/sphinx-doc/sphinx/commit/ef60dbfce09286b20b7385333d63a60321784e68 */ .field-name { -moz-hyphens: manual; -ms-hyphens: manual; -webkit-hyphens: manual; hyphens: manual; } table.footnote td.label { width: .1px; padding: 0.3em 0 0.3em 0.5em; } table.footnote td { padding: 0.3em 0.5em; } dl { margin: 0; padding: 0; } dl dd { margin-left: 30px; } blockquote { margin: 0 0 0 30px; padding: 0; } ul, ol { /* Matches the 30px from the narrow-screen "li > ul" selector below */ margin: 10px 0 10px 30px; padding: 0; } pre { background: #EEE; padding: 7px 30px; margin: 15px 0px; line-height: 1.3em; } div.viewcode-block:target { background: #ffd; } dl pre, blockquote pre, li pre { margin-left: 0; padding-left: 30px; } tt, code { background-color: #ecf0f3; color: #222; /* padding: 1px 2px; */ } tt.xref, code.xref, a tt { background-color: #FBFBFB; border-bottom: 1px solid #fff; } a.reference { text-decoration: none; border-bottom: 1px dotted #004B6B; } /* Don't put an underline on images */ a.image-reference, a.image-reference:hover { border-bottom: none; } a.reference:hover { border-bottom: 1px solid #6D4100; } a.footnote-reference { text-decoration: none; font-size: 0.7em; vertical-align: top; border-bottom: 1px dotted #004B6B; } a.footnote-reference:hover { border-bottom: 1px solid #6D4100; } a:hover tt, a:hover code { background: #EEE; } @media screen and (max-width: 870px) { div.sphinxsidebar { display: none; } div.document { width: 100%; } div.documentwrapper { margin-left: 0; margin-top: 0; margin-right: 0; margin-bottom: 0; } div.bodywrapper { margin-top: 0; margin-right: 0; margin-bottom: 0; margin-left: 0; } ul { margin-left: 0; } li > ul { /* Matches the 30px from the "ul, ol" selector above */ margin-left: 30px; } .document { width: auto; } .footer { width: auto; } .bodywrapper { margin: 0; } .footer { width: auto; } .github { display: none; } } @media screen and (max-width: 875px) { body { margin: 0; padding: 20px 30px; } div.documentwrapper { float: none; background: #fff; } div.sphinxsidebar { display: block; float: none; width: 102.5%; margin: 50px -30px -20px -30px; padding: 10px 20px; background: #333; color: #FFF; } div.sphinxsidebar h3, div.sphinxsidebar h4, div.sphinxsidebar p, div.sphinxsidebar h3 a { color: #fff; } div.sphinxsidebar a { color: #AAA; } div.sphinxsidebar p.logo { display: none; } div.document { width: 100%; margin: 0; } div.footer { display: none; } div.bodywrapper { margin: 0; } div.body { min-height: 0; padding: 0; } .rtd_doc_footer { display: none; } .document { width: auto; } .footer { width: auto; } .footer { width: auto; } .github { display: none; } } /* misc. */ .revsys-inline { display: none!important; } /* Make nested-list/multi-paragraph items look better in Releases changelog * pages. Without this, docutils' magical list fuckery causes inconsistent * formatting between different release sub-lists. */ div#changelog > div.section > ul > li > p:only-child { margin-bottom: 0; } /* Hide fugly table cell borders in ..bibliography:: directive output */ table.docutils.citation, table.docutils.citation td, table.docutils.citation th { border: none; /* Below needed in some edge cases; if not applied, bottom shadows appear */ -moz-box-shadow: none; -webkit-box-shadow: none; box-shadow: none; } /* relbar */ .related { line-height: 30px; width: 100%; font-size: 0.9rem; } .related.top { border-bottom: 1px solid #EEE; margin-bottom: 20px; } .related.bottom { border-top: 1px solid #EEE; } .related ul { padding: 0; margin: 0; list-style: none; } .related li { display: inline; } nav#rellinks { float: right; } nav#rellinks li+li:before { content: "|"; } nav#breadcrumbs li+li:before { content: "\00BB"; } /* Hide certain items when printing */ @media print { div.related { display: none; } } ================================================ FILE: core/dbt/docs/build/html/_static/basic.css ================================================ /* * basic.css * ~~~~~~~~~ * * Sphinx stylesheet -- basic theme. * * :copyright: Copyright 2007-2022 by the Sphinx team, see AUTHORS. * :license: BSD, see LICENSE for details. * */ /* -- main layout ----------------------------------------------------------- */ div.clearer { clear: both; } div.section::after { display: block; content: ''; clear: left; } /* -- relbar ---------------------------------------------------------------- */ div.related { width: 100%; font-size: 90%; } div.related h3 { display: none; } div.related ul { margin: 0; padding: 0 0 0 10px; list-style: none; } div.related li { display: inline; } div.related li.right { float: right; margin-right: 5px; } /* -- sidebar --------------------------------------------------------------- */ div.sphinxsidebarwrapper { padding: 10px 5px 0 10px; } div.sphinxsidebar { float: left; width: 230px; margin-left: -100%; font-size: 90%; word-wrap: break-word; overflow-wrap : break-word; } div.sphinxsidebar ul { list-style: none; } div.sphinxsidebar ul ul, div.sphinxsidebar ul.want-points { margin-left: 20px; list-style: square; } div.sphinxsidebar ul ul { margin-top: 0; margin-bottom: 0; } div.sphinxsidebar form { margin-top: 10px; } div.sphinxsidebar input { border: 1px solid #98dbcc; font-family: sans-serif; font-size: 1em; } div.sphinxsidebar #searchbox form.search { overflow: hidden; } div.sphinxsidebar #searchbox input[type="text"] { float: left; width: 80%; padding: 0.25em; box-sizing: border-box; } div.sphinxsidebar #searchbox input[type="submit"] { float: left; width: 20%; border-left: none; padding: 0.25em; box-sizing: border-box; } img { border: 0; max-width: 100%; } /* -- search page ----------------------------------------------------------- */ ul.search { margin: 10px 0 0 20px; padding: 0; } ul.search li { padding: 5px 0 5px 20px; background-image: url(file.png); background-repeat: no-repeat; background-position: 0 7px; } ul.search li a { font-weight: bold; } ul.search li p.context { color: #888; margin: 2px 0 0 30px; text-align: left; } ul.keywordmatches li.goodmatch a { font-weight: bold; } /* -- index page ------------------------------------------------------------ */ table.contentstable { width: 90%; margin-left: auto; margin-right: auto; } table.contentstable p.biglink { line-height: 150%; } a.biglink { font-size: 1.3em; } span.linkdescr { font-style: italic; padding-top: 5px; font-size: 90%; } /* -- general index --------------------------------------------------------- */ table.indextable { width: 100%; } table.indextable td { text-align: left; vertical-align: top; } table.indextable ul { margin-top: 0; margin-bottom: 0; list-style-type: none; } table.indextable > tbody > tr > td > ul { padding-left: 0em; } table.indextable tr.pcap { height: 10px; } table.indextable tr.cap { margin-top: 10px; background-color: #f2f2f2; } img.toggler { margin-right: 3px; margin-top: 3px; cursor: pointer; } div.modindex-jumpbox { border-top: 1px solid #ddd; border-bottom: 1px solid #ddd; margin: 1em 0 1em 0; padding: 0.4em; } div.genindex-jumpbox { border-top: 1px solid #ddd; border-bottom: 1px solid #ddd; margin: 1em 0 1em 0; padding: 0.4em; } /* -- domain module index --------------------------------------------------- */ table.modindextable td { padding: 2px; border-collapse: collapse; } /* -- general body styles --------------------------------------------------- */ div.body { min-width: 360px; max-width: 800px; } div.body p, div.body dd, div.body li, div.body blockquote { -moz-hyphens: auto; -ms-hyphens: auto; -webkit-hyphens: auto; hyphens: auto; } a.headerlink { visibility: hidden; } h1:hover > a.headerlink, h2:hover > a.headerlink, h3:hover > a.headerlink, h4:hover > a.headerlink, h5:hover > a.headerlink, h6:hover > a.headerlink, dt:hover > a.headerlink, caption:hover > a.headerlink, p.caption:hover > a.headerlink, div.code-block-caption:hover > a.headerlink { visibility: visible; } div.body p.caption { text-align: inherit; } div.body td { text-align: left; } .first { margin-top: 0 !important; } p.rubric { margin-top: 30px; font-weight: bold; } img.align-left, figure.align-left, .figure.align-left, object.align-left { clear: left; float: left; margin-right: 1em; } img.align-right, figure.align-right, .figure.align-right, object.align-right { clear: right; float: right; margin-left: 1em; } img.align-center, figure.align-center, .figure.align-center, object.align-center { display: block; margin-left: auto; margin-right: auto; } img.align-default, figure.align-default, .figure.align-default { display: block; margin-left: auto; margin-right: auto; } .align-left { text-align: left; } .align-center { text-align: center; } .align-default { text-align: center; } .align-right { text-align: right; } /* -- sidebars -------------------------------------------------------------- */ div.sidebar, aside.sidebar { margin: 0 0 0.5em 1em; border: 1px solid #ddb; padding: 7px; background-color: #ffe; width: 40%; float: right; clear: right; overflow-x: auto; } p.sidebar-title { font-weight: bold; } nav.contents, aside.topic, div.admonition, div.topic, blockquote { clear: left; } /* -- topics ---------------------------------------------------------------- */ nav.contents, aside.topic, div.topic { border: 1px solid #ccc; padding: 7px; margin: 10px 0 10px 0; } p.topic-title { font-size: 1.1em; font-weight: bold; margin-top: 10px; } /* -- admonitions ----------------------------------------------------------- */ div.admonition { margin-top: 10px; margin-bottom: 10px; padding: 7px; } div.admonition dt { font-weight: bold; } p.admonition-title { margin: 0px 10px 5px 0px; font-weight: bold; } div.body p.centered { text-align: center; margin-top: 25px; } /* -- content of sidebars/topics/admonitions -------------------------------- */ div.sidebar > :last-child, aside.sidebar > :last-child, nav.contents > :last-child, aside.topic > :last-child, div.topic > :last-child, div.admonition > :last-child { margin-bottom: 0; } div.sidebar::after, aside.sidebar::after, nav.contents::after, aside.topic::after, div.topic::after, div.admonition::after, blockquote::after { display: block; content: ''; clear: both; } /* -- tables ---------------------------------------------------------------- */ table.docutils { margin-top: 10px; margin-bottom: 10px; border: 0; border-collapse: collapse; } table.align-center { margin-left: auto; margin-right: auto; } table.align-default { margin-left: auto; margin-right: auto; } table caption span.caption-number { font-style: italic; } table caption span.caption-text { } table.docutils td, table.docutils th { padding: 1px 8px 1px 5px; border-top: 0; border-left: 0; border-right: 0; border-bottom: 1px solid #aaa; } th { text-align: left; padding-right: 5px; } table.citation { border-left: solid 1px gray; margin-left: 1px; } table.citation td { border-bottom: none; } th > :first-child, td > :first-child { margin-top: 0px; } th > :last-child, td > :last-child { margin-bottom: 0px; } /* -- figures --------------------------------------------------------------- */ div.figure, figure { margin: 0.5em; padding: 0.5em; } div.figure p.caption, figcaption { padding: 0.3em; } div.figure p.caption span.caption-number, figcaption span.caption-number { font-style: italic; } div.figure p.caption span.caption-text, figcaption span.caption-text { } /* -- field list styles ----------------------------------------------------- */ table.field-list td, table.field-list th { border: 0 !important; } .field-list ul { margin: 0; padding-left: 1em; } .field-list p { margin: 0; } .field-name { -moz-hyphens: manual; -ms-hyphens: manual; -webkit-hyphens: manual; hyphens: manual; } /* -- hlist styles ---------------------------------------------------------- */ table.hlist { margin: 1em 0; } table.hlist td { vertical-align: top; } /* -- object description styles --------------------------------------------- */ .sig { font-family: 'Consolas', 'Menlo', 'DejaVu Sans Mono', 'Bitstream Vera Sans Mono', monospace; } .sig-name, code.descname { background-color: transparent; font-weight: bold; } .sig-name { font-size: 1.1em; } code.descname { font-size: 1.2em; } .sig-prename, code.descclassname { background-color: transparent; } .optional { font-size: 1.3em; } .sig-paren { font-size: larger; } .sig-param.n { font-style: italic; } /* C++ specific styling */ .sig-inline.c-texpr, .sig-inline.cpp-texpr { font-family: unset; } .sig.c .k, .sig.c .kt, .sig.cpp .k, .sig.cpp .kt { color: #0033B3; } .sig.c .m, .sig.cpp .m { color: #1750EB; } .sig.c .s, .sig.c .sc, .sig.cpp .s, .sig.cpp .sc { color: #067D17; } /* -- other body styles ----------------------------------------------------- */ ol.arabic { list-style: decimal; } ol.loweralpha { list-style: lower-alpha; } ol.upperalpha { list-style: upper-alpha; } ol.lowerroman { list-style: lower-roman; } ol.upperroman { list-style: upper-roman; } :not(li) > ol > li:first-child > :first-child, :not(li) > ul > li:first-child > :first-child { margin-top: 0px; } :not(li) > ol > li:last-child > :last-child, :not(li) > ul > li:last-child > :last-child { margin-bottom: 0px; } ol.simple ol p, ol.simple ul p, ul.simple ol p, ul.simple ul p { margin-top: 0; } ol.simple > li:not(:first-child) > p, ul.simple > li:not(:first-child) > p { margin-top: 0; } ol.simple p, ul.simple p { margin-bottom: 0; } aside.footnote > span, div.citation > span { float: left; } aside.footnote > span:last-of-type, div.citation > span:last-of-type { padding-right: 0.5em; } aside.footnote > p { margin-left: 2em; } div.citation > p { margin-left: 4em; } aside.footnote > p:last-of-type, div.citation > p:last-of-type { margin-bottom: 0em; } aside.footnote > p:last-of-type:after, div.citation > p:last-of-type:after { content: ""; clear: both; } dl.field-list { display: grid; grid-template-columns: fit-content(30%) auto; } dl.field-list > dt { font-weight: bold; word-break: break-word; padding-left: 0.5em; padding-right: 5px; } dl.field-list > dd { padding-left: 0.5em; margin-top: 0em; margin-left: 0em; margin-bottom: 0em; } dl { margin-bottom: 15px; } dd > :first-child { margin-top: 0px; } dd ul, dd table { margin-bottom: 10px; } dd { margin-top: 3px; margin-bottom: 10px; margin-left: 30px; } dl > dd:last-child, dl > dd:last-child > :last-child { margin-bottom: 0; } dt:target, span.highlighted { background-color: #fbe54e; } rect.highlighted { fill: #fbe54e; } dl.glossary dt { font-weight: bold; font-size: 1.1em; } .versionmodified { font-style: italic; } .system-message { background-color: #fda; padding: 5px; border: 3px solid red; } .footnote:target { background-color: #ffa; } .line-block { display: block; margin-top: 1em; margin-bottom: 1em; } .line-block .line-block { margin-top: 0; margin-bottom: 0; margin-left: 1.5em; } .guilabel, .menuselection { font-family: sans-serif; } .accelerator { text-decoration: underline; } .classifier { font-style: oblique; } .classifier:before { font-style: normal; margin: 0 0.5em; content: ":"; display: inline-block; } abbr, acronym { border-bottom: dotted 1px; cursor: help; } /* -- code displays --------------------------------------------------------- */ pre { overflow: auto; overflow-y: hidden; /* fixes display issues on Chrome browsers */ } pre, div[class*="highlight-"] { clear: both; } span.pre { -moz-hyphens: none; -ms-hyphens: none; -webkit-hyphens: none; hyphens: none; white-space: nowrap; } div[class*="highlight-"] { margin: 1em 0; } td.linenos pre { border: 0; background-color: transparent; color: #aaa; } table.highlighttable { display: block; } table.highlighttable tbody { display: block; } table.highlighttable tr { display: flex; } table.highlighttable td { margin: 0; padding: 0; } table.highlighttable td.linenos { padding-right: 0.5em; } table.highlighttable td.code { flex: 1; overflow: hidden; } .highlight .hll { display: block; } div.highlight pre, table.highlighttable pre { margin: 0; } div.code-block-caption + div { margin-top: 0; } div.code-block-caption { margin-top: 1em; padding: 2px 5px; font-size: small; } div.code-block-caption code { background-color: transparent; } table.highlighttable td.linenos, span.linenos, div.highlight span.gp { /* gp: Generic.Prompt */ user-select: none; -webkit-user-select: text; /* Safari fallback only */ -webkit-user-select: none; /* Chrome/Safari */ -moz-user-select: none; /* Firefox */ -ms-user-select: none; /* IE10+ */ } div.code-block-caption span.caption-number { padding: 0.1em 0.3em; font-style: italic; } div.code-block-caption span.caption-text { } div.literal-block-wrapper { margin: 1em 0; } code.xref, a code { background-color: transparent; font-weight: bold; } h1 code, h2 code, h3 code, h4 code, h5 code, h6 code { background-color: transparent; } .viewcode-link { float: right; } .viewcode-back { float: right; font-family: sans-serif; } div.viewcode-block:target { margin: -1px -10px; padding: 0 10px; } /* -- math display ---------------------------------------------------------- */ img.math { vertical-align: middle; } div.body div.math p { text-align: center; } span.eqno { float: right; } span.eqno a.headerlink { position: absolute; z-index: 1; } div.math:hover a.headerlink { visibility: visible; } /* -- printout stylesheet --------------------------------------------------- */ @media print { div.document, div.documentwrapper, div.bodywrapper { margin: 0 !important; width: 100%; } div.sphinxsidebar, div.related, div.footer, #top-link { display: none; } } ================================================ FILE: core/dbt/docs/build/html/_static/custom.css ================================================ /* This file intentionally left blank. */ ================================================ FILE: core/dbt/docs/build/html/_static/doctools.js ================================================ /* * doctools.js * ~~~~~~~~~~~ * * Base JavaScript utilities for all Sphinx HTML documentation. * * :copyright: Copyright 2007-2022 by the Sphinx team, see AUTHORS. * :license: BSD, see LICENSE for details. * */ "use strict"; const BLACKLISTED_KEY_CONTROL_ELEMENTS = new Set([ "TEXTAREA", "INPUT", "SELECT", "BUTTON", ]); const _ready = (callback) => { if (document.readyState !== "loading") { callback(); } else { document.addEventListener("DOMContentLoaded", callback); } }; /** * Small JavaScript module for the documentation. */ const Documentation = { init: () => { Documentation.initDomainIndexTable(); Documentation.initOnKeyListeners(); }, /** * i18n support */ TRANSLATIONS: {}, PLURAL_EXPR: (n) => (n === 1 ? 0 : 1), LOCALE: "unknown", // gettext and ngettext don't access this so that the functions // can safely bound to a different name (_ = Documentation.gettext) gettext: (string) => { const translated = Documentation.TRANSLATIONS[string]; switch (typeof translated) { case "undefined": return string; // no translation case "string": return translated; // translation exists default: return translated[0]; // (singular, plural) translation tuple exists } }, ngettext: (singular, plural, n) => { const translated = Documentation.TRANSLATIONS[singular]; if (typeof translated !== "undefined") return translated[Documentation.PLURAL_EXPR(n)]; return n === 1 ? singular : plural; }, addTranslations: (catalog) => { Object.assign(Documentation.TRANSLATIONS, catalog.messages); Documentation.PLURAL_EXPR = new Function( "n", `return (${catalog.plural_expr})` ); Documentation.LOCALE = catalog.locale; }, /** * helper function to focus on search bar */ focusSearchBar: () => { document.querySelectorAll("input[name=q]")[0]?.focus(); }, /** * Initialise the domain index toggle buttons */ initDomainIndexTable: () => { const toggler = (el) => { const idNumber = el.id.substr(7); const toggledRows = document.querySelectorAll(`tr.cg-${idNumber}`); if (el.src.substr(-9) === "minus.png") { el.src = `${el.src.substr(0, el.src.length - 9)}plus.png`; toggledRows.forEach((el) => (el.style.display = "none")); } else { el.src = `${el.src.substr(0, el.src.length - 8)}minus.png`; toggledRows.forEach((el) => (el.style.display = "")); } }; const togglerElements = document.querySelectorAll("img.toggler"); togglerElements.forEach((el) => el.addEventListener("click", (event) => toggler(event.currentTarget)) ); togglerElements.forEach((el) => (el.style.display = "")); if (DOCUMENTATION_OPTIONS.COLLAPSE_INDEX) togglerElements.forEach(toggler); }, initOnKeyListeners: () => { // only install a listener if it is really needed if ( !DOCUMENTATION_OPTIONS.NAVIGATION_WITH_KEYS && !DOCUMENTATION_OPTIONS.ENABLE_SEARCH_SHORTCUTS ) return; document.addEventListener("keydown", (event) => { // bail for input elements if (BLACKLISTED_KEY_CONTROL_ELEMENTS.has(document.activeElement.tagName)) return; // bail with special keys if (event.altKey || event.ctrlKey || event.metaKey) return; if (!event.shiftKey) { switch (event.key) { case "ArrowLeft": if (!DOCUMENTATION_OPTIONS.NAVIGATION_WITH_KEYS) break; const prevLink = document.querySelector('link[rel="prev"]'); if (prevLink && prevLink.href) { window.location.href = prevLink.href; event.preventDefault(); } break; case "ArrowRight": if (!DOCUMENTATION_OPTIONS.NAVIGATION_WITH_KEYS) break; const nextLink = document.querySelector('link[rel="next"]'); if (nextLink && nextLink.href) { window.location.href = nextLink.href; event.preventDefault(); } break; } } // some keyboard layouts may need Shift to get / switch (event.key) { case "/": if (!DOCUMENTATION_OPTIONS.ENABLE_SEARCH_SHORTCUTS) break; Documentation.focusSearchBar(); event.preventDefault(); } }); }, }; // quick alias for translations const _ = Documentation.gettext; _ready(Documentation.init); ================================================ FILE: core/dbt/docs/build/html/_static/documentation_options.js ================================================ var DOCUMENTATION_OPTIONS = { URL_ROOT: document.getElementById("documentation_options").getAttribute('data-url_root'), VERSION: '', LANGUAGE: 'en', COLLAPSE_INDEX: false, BUILDER: 'html', FILE_SUFFIX: '.html', LINK_SUFFIX: '.html', HAS_SOURCE: true, SOURCELINK_SUFFIX: '.txt', NAVIGATION_WITH_KEYS: false, SHOW_SEARCH_SUMMARY: true, ENABLE_SEARCH_SHORTCUTS: true, }; ================================================ FILE: core/dbt/docs/build/html/_static/jquery-3.6.0.js ================================================ /*! * jQuery JavaScript Library v3.6.0 * https://jquery.com/ * * Includes Sizzle.js * https://sizzlejs.com/ * * Copyright OpenJS Foundation and other contributors * Released under the MIT license * https://jquery.org/license * * Date: 2021-03-02T17:08Z */ ( function( global, factory ) { "use strict"; if ( typeof module === "object" && typeof module.exports === "object" ) { // For CommonJS and CommonJS-like environments where a proper `window` // is present, execute the factory and get jQuery. // For environments that do not have a `window` with a `document` // (such as Node.js), expose a factory as module.exports. // This accentuates the need for the creation of a real `window`. // e.g. var jQuery = require("jquery")(window); // See ticket #14549 for more info. module.exports = global.document ? factory( global, true ) : function( w ) { if ( !w.document ) { throw new Error( "jQuery requires a window with a document" ); } return factory( w ); }; } else { factory( global ); } // Pass this if window is not defined yet } )( typeof window !== "undefined" ? window : this, function( window, noGlobal ) { // Edge <= 12 - 13+, Firefox <=18 - 45+, IE 10 - 11, Safari 5.1 - 9+, iOS 6 - 9.1 // throw exceptions when non-strict code (e.g., ASP.NET 4.5) accesses strict mode // arguments.callee.caller (trac-13335). But as of jQuery 3.0 (2016), strict mode should be common // enough that all such attempts are guarded in a try block. "use strict"; var arr = []; var getProto = Object.getPrototypeOf; var slice = arr.slice; var flat = arr.flat ? function( array ) { return arr.flat.call( array ); } : function( array ) { return arr.concat.apply( [], array ); }; var push = arr.push; var indexOf = arr.indexOf; var class2type = {}; var toString = class2type.toString; var hasOwn = class2type.hasOwnProperty; var fnToString = hasOwn.toString; var ObjectFunctionString = fnToString.call( Object ); var support = {}; var isFunction = function isFunction( obj ) { // Support: Chrome <=57, Firefox <=52 // In some browsers, typeof returns "function" for HTML <object> elements // (i.e., `typeof document.createElement( "object" ) === "function"`). // We don't want to classify *any* DOM node as a function. // Support: QtWeb <=3.8.5, WebKit <=534.34, wkhtmltopdf tool <=0.12.5 // Plus for old WebKit, typeof returns "function" for HTML collections // (e.g., `typeof document.getElementsByTagName("div") === "function"`). (gh-4756) return typeof obj === "function" && typeof obj.nodeType !== "number" && typeof obj.item !== "function"; }; var isWindow = function isWindow( obj ) { return obj != null && obj === obj.window; }; var document = window.document; var preservedScriptAttributes = { type: true, src: true, nonce: true, noModule: true }; function DOMEval( code, node, doc ) { doc = doc || document; var i, val, script = doc.createElement( "script" ); script.text = code; if ( node ) { for ( i in preservedScriptAttributes ) { // Support: Firefox 64+, Edge 18+ // Some browsers don't support the "nonce" property on scripts. // On the other hand, just using `getAttribute` is not enough as // the `nonce` attribute is reset to an empty string whenever it // becomes browsing-context connected. // See https://github.com/whatwg/html/issues/2369 // See https://html.spec.whatwg.org/#nonce-attributes // The `node.getAttribute` check was added for the sake of // `jQuery.globalEval` so that it can fake a nonce-containing node // via an object. val = node[ i ] || node.getAttribute && node.getAttribute( i ); if ( val ) { script.setAttribute( i, val ); } } } doc.head.appendChild( script ).parentNode.removeChild( script ); } function toType( obj ) { if ( obj == null ) { return obj + ""; } // Support: Android <=2.3 only (functionish RegExp) return typeof obj === "object" || typeof obj === "function" ? class2type[ toString.call( obj ) ] || "object" : typeof obj; } /* global Symbol */ // Defining this global in .eslintrc.json would create a danger of using the global // unguarded in another place, it seems safer to define global only for this module var version = "3.6.0", // Define a local copy of jQuery jQuery = function( selector, context ) { // The jQuery object is actually just the init constructor 'enhanced' // Need init if jQuery is called (just allow error to be thrown if not included) return new jQuery.fn.init( selector, context ); }; jQuery.fn = jQuery.prototype = { // The current version of jQuery being used jquery: version, constructor: jQuery, // The default length of a jQuery object is 0 length: 0, toArray: function() { return slice.call( this ); }, // Get the Nth element in the matched element set OR // Get the whole matched element set as a clean array get: function( num ) { // Return all the elements in a clean array if ( num == null ) { return slice.call( this ); } // Return just the one element from the set return num < 0 ? this[ num + this.length ] : this[ num ]; }, // Take an array of elements and push it onto the stack // (returning the new matched element set) pushStack: function( elems ) { // Build a new jQuery matched element set var ret = jQuery.merge( this.constructor(), elems ); // Add the old object onto the stack (as a reference) ret.prevObject = this; // Return the newly-formed element set return ret; }, // Execute a callback for every element in the matched set. each: function( callback ) { return jQuery.each( this, callback ); }, map: function( callback ) { return this.pushStack( jQuery.map( this, function( elem, i ) { return callback.call( elem, i, elem ); } ) ); }, slice: function() { return this.pushStack( slice.apply( this, arguments ) ); }, first: function() { return this.eq( 0 ); }, last: function() { return this.eq( -1 ); }, even: function() { return this.pushStack( jQuery.grep( this, function( _elem, i ) { return ( i + 1 ) % 2; } ) ); }, odd: function() { return this.pushStack( jQuery.grep( this, function( _elem, i ) { return i % 2; } ) ); }, eq: function( i ) { var len = this.length, j = +i + ( i < 0 ? len : 0 ); return this.pushStack( j >= 0 && j < len ? [ this[ j ] ] : [] ); }, end: function() { return this.prevObject || this.constructor(); }, // For internal use only. // Behaves like an Array's method, not like a jQuery method. push: push, sort: arr.sort, splice: arr.splice }; jQuery.extend = jQuery.fn.extend = function() { var options, name, src, copy, copyIsArray, clone, target = arguments[ 0 ] || {}, i = 1, length = arguments.length, deep = false; // Handle a deep copy situation if ( typeof target === "boolean" ) { deep = target; // Skip the boolean and the target target = arguments[ i ] || {}; i++; } // Handle case when target is a string or something (possible in deep copy) if ( typeof target !== "object" && !isFunction( target ) ) { target = {}; } // Extend jQuery itself if only one argument is passed if ( i === length ) { target = this; i--; } for ( ; i < length; i++ ) { // Only deal with non-null/undefined values if ( ( options = arguments[ i ] ) != null ) { // Extend the base object for ( name in options ) { copy = options[ name ]; // Prevent Object.prototype pollution // Prevent never-ending loop if ( name === "__proto__" || target === copy ) { continue; } // Recurse if we're merging plain objects or arrays if ( deep && copy && ( jQuery.isPlainObject( copy ) || ( copyIsArray = Array.isArray( copy ) ) ) ) { src = target[ name ]; // Ensure proper type for the source value if ( copyIsArray && !Array.isArray( src ) ) { clone = []; } else if ( !copyIsArray && !jQuery.isPlainObject( src ) ) { clone = {}; } else { clone = src; } copyIsArray = false; // Never move original objects, clone them target[ name ] = jQuery.extend( deep, clone, copy ); // Don't bring in undefined values } else if ( copy !== undefined ) { target[ name ] = copy; } } } } // Return the modified object return target; }; jQuery.extend( { // Unique for each copy of jQuery on the page expando: "jQuery" + ( version + Math.random() ).replace( /\D/g, "" ), // Assume jQuery is ready without the ready module isReady: true, error: function( msg ) { throw new Error( msg ); }, noop: function() {}, isPlainObject: function( obj ) { var proto, Ctor; // Detect obvious negatives // Use toString instead of jQuery.type to catch host objects if ( !obj || toString.call( obj ) !== "[object Object]" ) { return false; } proto = getProto( obj ); // Objects with no prototype (e.g., `Object.create( null )`) are plain if ( !proto ) { return true; } // Objects with prototype are plain iff they were constructed by a global Object function Ctor = hasOwn.call( proto, "constructor" ) && proto.constructor; return typeof Ctor === "function" && fnToString.call( Ctor ) === ObjectFunctionString; }, isEmptyObject: function( obj ) { var name; for ( name in obj ) { return false; } return true; }, // Evaluates a script in a provided context; falls back to the global one // if not specified. globalEval: function( code, options, doc ) { DOMEval( code, { nonce: options && options.nonce }, doc ); }, each: function( obj, callback ) { var length, i = 0; if ( isArrayLike( obj ) ) { length = obj.length; for ( ; i < length; i++ ) { if ( callback.call( obj[ i ], i, obj[ i ] ) === false ) { break; } } } else { for ( i in obj ) { if ( callback.call( obj[ i ], i, obj[ i ] ) === false ) { break; } } } return obj; }, // results is for internal usage only makeArray: function( arr, results ) { var ret = results || []; if ( arr != null ) { if ( isArrayLike( Object( arr ) ) ) { jQuery.merge( ret, typeof arr === "string" ? [ arr ] : arr ); } else { push.call( ret, arr ); } } return ret; }, inArray: function( elem, arr, i ) { return arr == null ? -1 : indexOf.call( arr, elem, i ); }, // Support: Android <=4.0 only, PhantomJS 1 only // push.apply(_, arraylike) throws on ancient WebKit merge: function( first, second ) { var len = +second.length, j = 0, i = first.length; for ( ; j < len; j++ ) { first[ i++ ] = second[ j ]; } first.length = i; return first; }, grep: function( elems, callback, invert ) { var callbackInverse, matches = [], i = 0, length = elems.length, callbackExpect = !invert; // Go through the array, only saving the items // that pass the validator function for ( ; i < length; i++ ) { callbackInverse = !callback( elems[ i ], i ); if ( callbackInverse !== callbackExpect ) { matches.push( elems[ i ] ); } } return matches; }, // arg is for internal usage only map: function( elems, callback, arg ) { var length, value, i = 0, ret = []; // Go through the array, translating each of the items to their new values if ( isArrayLike( elems ) ) { length = elems.length; for ( ; i < length; i++ ) { value = callback( elems[ i ], i, arg ); if ( value != null ) { ret.push( value ); } } // Go through every key on the object, } else { for ( i in elems ) { value = callback( elems[ i ], i, arg ); if ( value != null ) { ret.push( value ); } } } // Flatten any nested arrays return flat( ret ); }, // A global GUID counter for objects guid: 1, // jQuery.support is not used in Core but other projects attach their // properties to it so it needs to exist. support: support } ); if ( typeof Symbol === "function" ) { jQuery.fn[ Symbol.iterator ] = arr[ Symbol.iterator ]; } // Populate the class2type map jQuery.each( "Boolean Number String Function Array Date RegExp Object Error Symbol".split( " " ), function( _i, name ) { class2type[ "[object " + name + "]" ] = name.toLowerCase(); } ); function isArrayLike( obj ) { // Support: real iOS 8.2 only (not reproducible in simulator) // `in` check used to prevent JIT error (gh-2145) // hasOwn isn't used here due to false negatives // regarding Nodelist length in IE var length = !!obj && "length" in obj && obj.length, type = toType( obj ); if ( isFunction( obj ) || isWindow( obj ) ) { return false; } return type === "array" || length === 0 || typeof length === "number" && length > 0 && ( length - 1 ) in obj; } var Sizzle = /*! * Sizzle CSS Selector Engine v2.3.6 * https://sizzlejs.com/ * * Copyright JS Foundation and other contributors * Released under the MIT license * https://js.foundation/ * * Date: 2021-02-16 */ ( function( window ) { var i, support, Expr, getText, isXML, tokenize, compile, select, outermostContext, sortInput, hasDuplicate, // Local document vars setDocument, document, docElem, documentIsHTML, rbuggyQSA, rbuggyMatches, matches, contains, // Instance-specific data expando = "sizzle" + 1 * new Date(), preferredDoc = window.document, dirruns = 0, done = 0, classCache = createCache(), tokenCache = createCache(), compilerCache = createCache(), nonnativeSelectorCache = createCache(), sortOrder = function( a, b ) { if ( a === b ) { hasDuplicate = true; } return 0; }, // Instance methods hasOwn = ( {} ).hasOwnProperty, arr = [], pop = arr.pop, pushNative = arr.push, push = arr.push, slice = arr.slice, // Use a stripped-down indexOf as it's faster than native // https://jsperf.com/thor-indexof-vs-for/5 indexOf = function( list, elem ) { var i = 0, len = list.length; for ( ; i < len; i++ ) { if ( list[ i ] === elem ) { return i; } } return -1; }, booleans = "checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|" + "ismap|loop|multiple|open|readonly|required|scoped", // Regular expressions // http://www.w3.org/TR/css3-selectors/#whitespace whitespace = "[\\x20\\t\\r\\n\\f]", // https://www.w3.org/TR/css-syntax-3/#ident-token-diagram identifier = "(?:\\\\[\\da-fA-F]{1,6}" + whitespace + "?|\\\\[^\\r\\n\\f]|[\\w-]|[^\0-\\x7f])+", // Attribute selectors: http://www.w3.org/TR/selectors/#attribute-selectors attributes = "\\[" + whitespace + "*(" + identifier + ")(?:" + whitespace + // Operator (capture 2) "*([*^$|!~]?=)" + whitespace + // "Attribute values must be CSS identifiers [capture 5] // or strings [capture 3 or capture 4]" "*(?:'((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\"|(" + identifier + "))|)" + whitespace + "*\\]", pseudos = ":(" + identifier + ")(?:\\((" + // To reduce the number of selectors needing tokenize in the preFilter, prefer arguments: // 1. quoted (capture 3; capture 4 or capture 5) "('((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\")|" + // 2. simple (capture 6) "((?:\\\\.|[^\\\\()[\\]]|" + attributes + ")*)|" + // 3. anything else (capture 2) ".*" + ")\\)|)", // Leading and non-escaped trailing whitespace, capturing some non-whitespace characters preceding the latter rwhitespace = new RegExp( whitespace + "+", "g" ), rtrim = new RegExp( "^" + whitespace + "+|((?:^|[^\\\\])(?:\\\\.)*)" + whitespace + "+$", "g" ), rcomma = new RegExp( "^" + whitespace + "*," + whitespace + "*" ), rcombinators = new RegExp( "^" + whitespace + "*([>+~]|" + whitespace + ")" + whitespace + "*" ), rdescend = new RegExp( whitespace + "|>" ), rpseudo = new RegExp( pseudos ), ridentifier = new RegExp( "^" + identifier + "$" ), matchExpr = { "ID": new RegExp( "^#(" + identifier + ")" ), "CLASS": new RegExp( "^\\.(" + identifier + ")" ), "TAG": new RegExp( "^(" + identifier + "|[*])" ), "ATTR": new RegExp( "^" + attributes ), "PSEUDO": new RegExp( "^" + pseudos ), "CHILD": new RegExp( "^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\(" + whitespace + "*(even|odd|(([+-]|)(\\d*)n|)" + whitespace + "*(?:([+-]|)" + whitespace + "*(\\d+)|))" + whitespace + "*\\)|)", "i" ), "bool": new RegExp( "^(?:" + booleans + ")$", "i" ), // For use in libraries implementing .is() // We use this for POS matching in `select` "needsContext": new RegExp( "^" + whitespace + "*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\(" + whitespace + "*((?:-\\d)?\\d*)" + whitespace + "*\\)|)(?=[^-]|$)", "i" ) }, rhtml = /HTML$/i, rinputs = /^(?:input|select|textarea|button)$/i, rheader = /^h\d$/i, rnative = /^[^{]+\{\s*\[native \w/, // Easily-parseable/retrievable ID or TAG or CLASS selectors rquickExpr = /^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/, rsibling = /[+~]/, // CSS escapes // http://www.w3.org/TR/CSS21/syndata.html#escaped-characters runescape = new RegExp( "\\\\[\\da-fA-F]{1,6}" + whitespace + "?|\\\\([^\\r\\n\\f])", "g" ), funescape = function( escape, nonHex ) { var high = "0x" + escape.slice( 1 ) - 0x10000; return nonHex ? // Strip the backslash prefix from a non-hex escape sequence nonHex : // Replace a hexadecimal escape sequence with the encoded Unicode code point // Support: IE <=11+ // For values outside the Basic Multilingual Plane (BMP), manually construct a // surrogate pair high < 0 ? String.fromCharCode( high + 0x10000 ) : String.fromCharCode( high >> 10 | 0xD800, high & 0x3FF | 0xDC00 ); }, // CSS string/identifier serialization // https://drafts.csswg.org/cssom/#common-serializing-idioms rcssescape = /([\0-\x1f\x7f]|^-?\d)|^-$|[^\0-\x1f\x7f-\uFFFF\w-]/g, fcssescape = function( ch, asCodePoint ) { if ( asCodePoint ) { // U+0000 NULL becomes U+FFFD REPLACEMENT CHARACTER if ( ch === "\0" ) { return "\uFFFD"; } // Control characters and (dependent upon position) numbers get escaped as code points return ch.slice( 0, -1 ) + "\\" + ch.charCodeAt( ch.length - 1 ).toString( 16 ) + " "; } // Other potentially-special ASCII characters get backslash-escaped return "\\" + ch; }, // Used for iframes // See setDocument() // Removing the function wrapper causes a "Permission Denied" // error in IE unloadHandler = function() { setDocument(); }, inDisabledFieldset = addCombinator( function( elem ) { return elem.disabled === true && elem.nodeName.toLowerCase() === "fieldset"; }, { dir: "parentNode", next: "legend" } ); // Optimize for push.apply( _, NodeList ) try { push.apply( ( arr = slice.call( preferredDoc.childNodes ) ), preferredDoc.childNodes ); // Support: Android<4.0 // Detect silently failing push.apply // eslint-disable-next-line no-unused-expressions arr[ preferredDoc.childNodes.length ].nodeType; } catch ( e ) { push = { apply: arr.length ? // Leverage slice if possible function( target, els ) { pushNative.apply( target, slice.call( els ) ); } : // Support: IE<9 // Otherwise append directly function( target, els ) { var j = target.length, i = 0; // Can't trust NodeList.length while ( ( target[ j++ ] = els[ i++ ] ) ) {} target.length = j - 1; } }; } function Sizzle( selector, context, results, seed ) { var m, i, elem, nid, match, groups, newSelector, newContext = context && context.ownerDocument, // nodeType defaults to 9, since context defaults to document nodeType = context ? context.nodeType : 9; results = results || []; // Return early from calls with invalid selector or context if ( typeof selector !== "string" || !selector || nodeType !== 1 && nodeType !== 9 && nodeType !== 11 ) { return results; } // Try to shortcut find operations (as opposed to filters) in HTML documents if ( !seed ) { setDocument( context ); context = context || document; if ( documentIsHTML ) { // If the selector is sufficiently simple, try using a "get*By*" DOM method // (excepting DocumentFragment context, where the methods don't exist) if ( nodeType !== 11 && ( match = rquickExpr.exec( selector ) ) ) { // ID selector if ( ( m = match[ 1 ] ) ) { // Document context if ( nodeType === 9 ) { if ( ( elem = context.getElementById( m ) ) ) { // Support: IE, Opera, Webkit // TODO: identify versions // getElementById can match elements by name instead of ID if ( elem.id === m ) { results.push( elem ); return results; } } else { return results; } // Element context } else { // Support: IE, Opera, Webkit // TODO: identify versions // getElementById can match elements by name instead of ID if ( newContext && ( elem = newContext.getElementById( m ) ) && contains( context, elem ) && elem.id === m ) { results.push( elem ); return results; } } // Type selector } else if ( match[ 2 ] ) { push.apply( results, context.getElementsByTagName( selector ) ); return results; // Class selector } else if ( ( m = match[ 3 ] ) && support.getElementsByClassName && context.getElementsByClassName ) { push.apply( results, context.getElementsByClassName( m ) ); return results; } } // Take advantage of querySelectorAll if ( support.qsa && !nonnativeSelectorCache[ selector + " " ] && ( !rbuggyQSA || !rbuggyQSA.test( selector ) ) && // Support: IE 8 only // Exclude object elements ( nodeType !== 1 || context.nodeName.toLowerCase() !== "object" ) ) { newSelector = selector; newContext = context; // qSA considers elements outside a scoping root when evaluating child or // descendant combinators, which is not what we want. // In such cases, we work around the behavior by prefixing every selector in the // list with an ID selector referencing the scope context. // The technique has to be used as well when a leading combinator is used // as such selectors are not recognized by querySelectorAll. // Thanks to Andrew Dupont for this technique. if ( nodeType === 1 && ( rdescend.test( selector ) || rcombinators.test( selector ) ) ) { // Expand context for sibling selectors newContext = rsibling.test( selector ) && testContext( context.parentNode ) || context; // We can use :scope instead of the ID hack if the browser // supports it & if we're not changing the context. if ( newContext !== context || !support.scope ) { // Capture the context ID, setting it first if necessary if ( ( nid = context.getAttribute( "id" ) ) ) { nid = nid.replace( rcssescape, fcssescape ); } else { context.setAttribute( "id", ( nid = expando ) ); } } // Prefix every selector in the list groups = tokenize( selector ); i = groups.length; while ( i-- ) { groups[ i ] = ( nid ? "#" + nid : ":scope" ) + " " + toSelector( groups[ i ] ); } newSelector = groups.join( "," ); } try { push.apply( results, newContext.querySelectorAll( newSelector ) ); return results; } catch ( qsaError ) { nonnativeSelectorCache( selector, true ); } finally { if ( nid === expando ) { context.removeAttribute( "id" ); } } } } } // All others return select( selector.replace( rtrim, "$1" ), context, results, seed ); } /** * Create key-value caches of limited size * @returns {function(string, object)} Returns the Object data after storing it on itself with * property name the (space-suffixed) string and (if the cache is larger than Expr.cacheLength) * deleting the oldest entry */ function createCache() { var keys = []; function cache( key, value ) { // Use (key + " ") to avoid collision with native prototype properties (see Issue #157) if ( keys.push( key + " " ) > Expr.cacheLength ) { // Only keep the most recent entries delete cache[ keys.shift() ]; } return ( cache[ key + " " ] = value ); } return cache; } /** * Mark a function for special use by Sizzle * @param {Function} fn The function to mark */ function markFunction( fn ) { fn[ expando ] = true; return fn; } /** * Support testing using an element * @param {Function} fn Passed the created element and returns a boolean result */ function assert( fn ) { var el = document.createElement( "fieldset" ); try { return !!fn( el ); } catch ( e ) { return false; } finally { // Remove from its parent by default if ( el.parentNode ) { el.parentNode.removeChild( el ); } // release memory in IE el = null; } } /** * Adds the same handler for all of the specified attrs * @param {String} attrs Pipe-separated list of attributes * @param {Function} handler The method that will be applied */ function addHandle( attrs, handler ) { var arr = attrs.split( "|" ), i = arr.length; while ( i-- ) { Expr.attrHandle[ arr[ i ] ] = handler; } } /** * Checks document order of two siblings * @param {Element} a * @param {Element} b * @returns {Number} Returns less than 0 if a precedes b, greater than 0 if a follows b */ function siblingCheck( a, b ) { var cur = b && a, diff = cur && a.nodeType === 1 && b.nodeType === 1 && a.sourceIndex - b.sourceIndex; // Use IE sourceIndex if available on both nodes if ( diff ) { return diff; } // Check if b follows a if ( cur ) { while ( ( cur = cur.nextSibling ) ) { if ( cur === b ) { return -1; } } } return a ? 1 : -1; } /** * Returns a function to use in pseudos for input types * @param {String} type */ function createInputPseudo( type ) { return function( elem ) { var name = elem.nodeName.toLowerCase(); return name === "input" && elem.type === type; }; } /** * Returns a function to use in pseudos for buttons * @param {String} type */ function createButtonPseudo( type ) { return function( elem ) { var name = elem.nodeName.toLowerCase(); return ( name === "input" || name === "button" ) && elem.type === type; }; } /** * Returns a function to use in pseudos for :enabled/:disabled * @param {Boolean} disabled true for :disabled; false for :enabled */ function createDisabledPseudo( disabled ) { // Known :disabled false positives: fieldset[disabled] > legend:nth-of-type(n+2) :can-disable return function( elem ) { // Only certain elements can match :enabled or :disabled // https://html.spec.whatwg.org/multipage/scripting.html#selector-enabled // https://html.spec.whatwg.org/multipage/scripting.html#selector-disabled if ( "form" in elem ) { // Check for inherited disabledness on relevant non-disabled elements: // * listed form-associated elements in a disabled fieldset // https://html.spec.whatwg.org/multipage/forms.html#category-listed // https://html.spec.whatwg.org/multipage/forms.html#concept-fe-disabled // * option elements in a disabled optgroup // https://html.spec.whatwg.org/multipage/forms.html#concept-option-disabled // All such elements have a "form" property. if ( elem.parentNode && elem.disabled === false ) { // Option elements defer to a parent optgroup if present if ( "label" in elem ) { if ( "label" in elem.parentNode ) { return elem.parentNode.disabled === disabled; } else { return elem.disabled === disabled; } } // Support: IE 6 - 11 // Use the isDisabled shortcut property to check for disabled fieldset ancestors return elem.isDisabled === disabled || // Where there is no isDisabled, check manually /* jshint -W018 */ elem.isDisabled !== !disabled && inDisabledFieldset( elem ) === disabled; } return elem.disabled === disabled; // Try to winnow out elements that can't be disabled before trusting the disabled property. // Some victims get caught in our net (label, legend, menu, track), but it shouldn't // even exist on them, let alone have a boolean value. } else if ( "label" in elem ) { return elem.disabled === disabled; } // Remaining elements are neither :enabled nor :disabled return false; }; } /** * Returns a function to use in pseudos for positionals * @param {Function} fn */ function createPositionalPseudo( fn ) { return markFunction( function( argument ) { argument = +argument; return markFunction( function( seed, matches ) { var j, matchIndexes = fn( [], seed.length, argument ), i = matchIndexes.length; // Match elements found at the specified indexes while ( i-- ) { if ( seed[ ( j = matchIndexes[ i ] ) ] ) { seed[ j ] = !( matches[ j ] = seed[ j ] ); } } } ); } ); } /** * Checks a node for validity as a Sizzle context * @param {Element|Object=} context * @returns {Element|Object|Boolean} The input node if acceptable, otherwise a falsy value */ function testContext( context ) { return context && typeof context.getElementsByTagName !== "undefined" && context; } // Expose support vars for convenience support = Sizzle.support = {}; /** * Detects XML nodes * @param {Element|Object} elem An element or a document * @returns {Boolean} True iff elem is a non-HTML XML node */ isXML = Sizzle.isXML = function( elem ) { var namespace = elem && elem.namespaceURI, docElem = elem && ( elem.ownerDocument || elem ).documentElement; // Support: IE <=8 // Assume HTML when documentElement doesn't yet exist, such as inside loading iframes // https://bugs.jquery.com/ticket/4833 return !rhtml.test( namespace || docElem && docElem.nodeName || "HTML" ); }; /** * Sets document-related variables once based on the current document * @param {Element|Object} [doc] An element or document object to use to set the document * @returns {Object} Returns the current document */ setDocument = Sizzle.setDocument = function( node ) { var hasCompare, subWindow, doc = node ? node.ownerDocument || node : preferredDoc; // Return early if doc is invalid or already selected // Support: IE 11+, Edge 17 - 18+ // IE/Edge sometimes throw a "Permission denied" error when strict-comparing // two documents; shallow comparisons work. // eslint-disable-next-line eqeqeq if ( doc == document || doc.nodeType !== 9 || !doc.documentElement ) { return document; } // Update global variables document = doc; docElem = document.documentElement; documentIsHTML = !isXML( document ); // Support: IE 9 - 11+, Edge 12 - 18+ // Accessing iframe documents after unload throws "permission denied" errors (jQuery #13936) // Support: IE 11+, Edge 17 - 18+ // IE/Edge sometimes throw a "Permission denied" error when strict-comparing // two documents; shallow comparisons work. // eslint-disable-next-line eqeqeq if ( preferredDoc != document && ( subWindow = document.defaultView ) && subWindow.top !== subWindow ) { // Support: IE 11, Edge if ( subWindow.addEventListener ) { subWindow.addEventListener( "unload", unloadHandler, false ); // Support: IE 9 - 10 only } else if ( subWindow.attachEvent ) { subWindow.attachEvent( "onunload", unloadHandler ); } } // Support: IE 8 - 11+, Edge 12 - 18+, Chrome <=16 - 25 only, Firefox <=3.6 - 31 only, // Safari 4 - 5 only, Opera <=11.6 - 12.x only // IE/Edge & older browsers don't support the :scope pseudo-class. // Support: Safari 6.0 only // Safari 6.0 supports :scope but it's an alias of :root there. support.scope = assert( function( el ) { docElem.appendChild( el ).appendChild( document.createElement( "div" ) ); return typeof el.querySelectorAll !== "undefined" && !el.querySelectorAll( ":scope fieldset div" ).length; } ); /* Attributes ---------------------------------------------------------------------- */ // Support: IE<8 // Verify that getAttribute really returns attributes and not properties // (excepting IE8 booleans) support.attributes = assert( function( el ) { el.className = "i"; return !el.getAttribute( "className" ); } ); /* getElement(s)By* ---------------------------------------------------------------------- */ // Check if getElementsByTagName("*") returns only elements support.getElementsByTagName = assert( function( el ) { el.appendChild( document.createComment( "" ) ); return !el.getElementsByTagName( "*" ).length; } ); // Support: IE<9 support.getElementsByClassName = rnative.test( document.getElementsByClassName ); // Support: IE<10 // Check if getElementById returns elements by name // The broken getElementById methods don't pick up programmatically-set names, // so use a roundabout getElementsByName test support.getById = assert( function( el ) { docElem.appendChild( el ).id = expando; return !document.getElementsByName || !document.getElementsByName( expando ).length; } ); // ID filter and find if ( support.getById ) { Expr.filter[ "ID" ] = function( id ) { var attrId = id.replace( runescape, funescape ); return function( elem ) { return elem.getAttribute( "id" ) === attrId; }; }; Expr.find[ "ID" ] = function( id, context ) { if ( typeof context.getElementById !== "undefined" && documentIsHTML ) { var elem = context.getElementById( id ); return elem ? [ elem ] : []; } }; } else { Expr.filter[ "ID" ] = function( id ) { var attrId = id.replace( runescape, funescape ); return function( elem ) { var node = typeof elem.getAttributeNode !== "undefined" && elem.getAttributeNode( "id" ); return node && node.value === attrId; }; }; // Support: IE 6 - 7 only // getElementById is not reliable as a find shortcut Expr.find[ "ID" ] = function( id, context ) { if ( typeof context.getElementById !== "undefined" && documentIsHTML ) { var node, i, elems, elem = context.getElementById( id ); if ( elem ) { // Verify the id attribute node = elem.getAttributeNode( "id" ); if ( node && node.value === id ) { return [ elem ]; } // Fall back on getElementsByName elems = context.getElementsByName( id ); i = 0; while ( ( elem = elems[ i++ ] ) ) { node = elem.getAttributeNode( "id" ); if ( node && node.value === id ) { return [ elem ]; } } } return []; } }; } // Tag Expr.find[ "TAG" ] = support.getElementsByTagName ? function( tag, context ) { if ( typeof context.getElementsByTagName !== "undefined" ) { return context.getElementsByTagName( tag ); // DocumentFragment nodes don't have gEBTN } else if ( support.qsa ) { return context.querySelectorAll( tag ); } } : function( tag, context ) { var elem, tmp = [], i = 0, // By happy coincidence, a (broken) gEBTN appears on DocumentFragment nodes too results = context.getElementsByTagName( tag ); // Filter out possible comments if ( tag === "*" ) { while ( ( elem = results[ i++ ] ) ) { if ( elem.nodeType === 1 ) { tmp.push( elem ); } } return tmp; } return results; }; // Class Expr.find[ "CLASS" ] = support.getElementsByClassName && function( className, context ) { if ( typeof context.getElementsByClassName !== "undefined" && documentIsHTML ) { return context.getElementsByClassName( className ); } }; /* QSA/matchesSelector ---------------------------------------------------------------------- */ // QSA and matchesSelector support // matchesSelector(:active) reports false when true (IE9/Opera 11.5) rbuggyMatches = []; // qSa(:focus) reports false when true (Chrome 21) // We allow this because of a bug in IE8/9 that throws an error // whenever `document.activeElement` is accessed on an iframe // So, we allow :focus to pass through QSA all the time to avoid the IE error // See https://bugs.jquery.com/ticket/13378 rbuggyQSA = []; if ( ( support.qsa = rnative.test( document.querySelectorAll ) ) ) { // Build QSA regex // Regex strategy adopted from Diego Perini assert( function( el ) { var input; // Select is set to empty string on purpose // This is to test IE's treatment of not explicitly // setting a boolean content attribute, // since its presence should be enough // https://bugs.jquery.com/ticket/12359 docElem.appendChild( el ).innerHTML = "<a id='" + expando + "'></a>" + "<select id='" + expando + "-\r\\' msallowcapture=''>" + "<option selected=''></option></select>"; // Support: IE8, Opera 11-12.16 // Nothing should be selected when empty strings follow ^= or $= or *= // The test attribute must be unknown in Opera but "safe" for WinRT // https://msdn.microsoft.com/en-us/library/ie/hh465388.aspx#attribute_section if ( el.querySelectorAll( "[msallowcapture^='']" ).length ) { rbuggyQSA.push( "[*^$]=" + whitespace + "*(?:''|\"\")" ); } // Support: IE8 // Boolean attributes and "value" are not treated correctly if ( !el.querySelectorAll( "[selected]" ).length ) { rbuggyQSA.push( "\\[" + whitespace + "*(?:value|" + booleans + ")" ); } // Support: Chrome<29, Android<4.4, Safari<7.0+, iOS<7.0+, PhantomJS<1.9.8+ if ( !el.querySelectorAll( "[id~=" + expando + "-]" ).length ) { rbuggyQSA.push( "~=" ); } // Support: IE 11+, Edge 15 - 18+ // IE 11/Edge don't find elements on a `[name='']` query in some cases. // Adding a temporary attribute to the document before the selection works // around the issue. // Interestingly, IE 10 & older don't seem to have the issue. input = document.createElement( "input" ); input.setAttribute( "name", "" ); el.appendChild( input ); if ( !el.querySelectorAll( "[name='']" ).length ) { rbuggyQSA.push( "\\[" + whitespace + "*name" + whitespace + "*=" + whitespace + "*(?:''|\"\")" ); } // Webkit/Opera - :checked should return selected option elements // http://www.w3.org/TR/2011/REC-css3-selectors-20110929/#checked // IE8 throws error here and will not see later tests if ( !el.querySelectorAll( ":checked" ).length ) { rbuggyQSA.push( ":checked" ); } // Support: Safari 8+, iOS 8+ // https://bugs.webkit.org/show_bug.cgi?id=136851 // In-page `selector#id sibling-combinator selector` fails if ( !el.querySelectorAll( "a#" + expando + "+*" ).length ) { rbuggyQSA.push( ".#.+[+~]" ); } // Support: Firefox <=3.6 - 5 only // Old Firefox doesn't throw on a badly-escaped identifier. el.querySelectorAll( "\\\f" ); rbuggyQSA.push( "[\\r\\n\\f]" ); } ); assert( function( el ) { el.innerHTML = "<a href='' disabled='disabled'></a>" + "<select disabled='disabled'><option/></select>"; // Support: Windows 8 Native Apps // The type and name attributes are restricted during .innerHTML assignment var input = document.createElement( "input" ); input.setAttribute( "type", "hidden" ); el.appendChild( input ).setAttribute( "name", "D" ); // Support: IE8 // Enforce case-sensitivity of name attribute if ( el.querySelectorAll( "[name=d]" ).length ) { rbuggyQSA.push( "name" + whitespace + "*[*^$|!~]?=" ); } // FF 3.5 - :enabled/:disabled and hidden elements (hidden elements are still enabled) // IE8 throws error here and will not see later tests if ( el.querySelectorAll( ":enabled" ).length !== 2 ) { rbuggyQSA.push( ":enabled", ":disabled" ); } // Support: IE9-11+ // IE's :disabled selector does not pick up the children of disabled fieldsets docElem.appendChild( el ).disabled = true; if ( el.querySelectorAll( ":disabled" ).length !== 2 ) { rbuggyQSA.push( ":enabled", ":disabled" ); } // Support: Opera 10 - 11 only // Opera 10-11 does not throw on post-comma invalid pseudos el.querySelectorAll( "*,:x" ); rbuggyQSA.push( ",.*:" ); } ); } if ( ( support.matchesSelector = rnative.test( ( matches = docElem.matches || docElem.webkitMatchesSelector || docElem.mozMatchesSelector || docElem.oMatchesSelector || docElem.msMatchesSelector ) ) ) ) { assert( function( el ) { // Check to see if it's possible to do matchesSelector // on a disconnected node (IE 9) support.disconnectedMatch = matches.call( el, "*" ); // This should fail with an exception // Gecko does not error, returns false instead matches.call( el, "[s!='']:x" ); rbuggyMatches.push( "!=", pseudos ); } ); } rbuggyQSA = rbuggyQSA.length && new RegExp( rbuggyQSA.join( "|" ) ); rbuggyMatches = rbuggyMatches.length && new RegExp( rbuggyMatches.join( "|" ) ); /* Contains ---------------------------------------------------------------------- */ hasCompare = rnative.test( docElem.compareDocumentPosition ); // Element contains another // Purposefully self-exclusive // As in, an element does not contain itself contains = hasCompare || rnative.test( docElem.contains ) ? function( a, b ) { var adown = a.nodeType === 9 ? a.documentElement : a, bup = b && b.parentNode; return a === bup || !!( bup && bup.nodeType === 1 && ( adown.contains ? adown.contains( bup ) : a.compareDocumentPosition && a.compareDocumentPosition( bup ) & 16 ) ); } : function( a, b ) { if ( b ) { while ( ( b = b.parentNode ) ) { if ( b === a ) { return true; } } } return false; }; /* Sorting ---------------------------------------------------------------------- */ // Document order sorting sortOrder = hasCompare ? function( a, b ) { // Flag for duplicate removal if ( a === b ) { hasDuplicate = true; return 0; } // Sort on method existence if only one input has compareDocumentPosition var compare = !a.compareDocumentPosition - !b.compareDocumentPosition; if ( compare ) { return compare; } // Calculate position if both inputs belong to the same document // Support: IE 11+, Edge 17 - 18+ // IE/Edge sometimes throw a "Permission denied" error when strict-comparing // two documents; shallow comparisons work. // eslint-disable-next-line eqeqeq compare = ( a.ownerDocument || a ) == ( b.ownerDocument || b ) ? a.compareDocumentPosition( b ) : // Otherwise we know they are disconnected 1; // Disconnected nodes if ( compare & 1 || ( !support.sortDetached && b.compareDocumentPosition( a ) === compare ) ) { // Choose the first element that is related to our preferred document // Support: IE 11+, Edge 17 - 18+ // IE/Edge sometimes throw a "Permission denied" error when strict-comparing // two documents; shallow comparisons work. // eslint-disable-next-line eqeqeq if ( a == document || a.ownerDocument == preferredDoc && contains( preferredDoc, a ) ) { return -1; } // Support: IE 11+, Edge 17 - 18+ // IE/Edge sometimes throw a "Permission denied" error when strict-comparing // two documents; shallow comparisons work. // eslint-disable-next-line eqeqeq if ( b == document || b.ownerDocument == preferredDoc && contains( preferredDoc, b ) ) { return 1; } // Maintain original order return sortInput ? ( indexOf( sortInput, a ) - indexOf( sortInput, b ) ) : 0; } return compare & 4 ? -1 : 1; } : function( a, b ) { // Exit early if the nodes are identical if ( a === b ) { hasDuplicate = true; return 0; } var cur, i = 0, aup = a.parentNode, bup = b.parentNode, ap = [ a ], bp = [ b ]; // Parentless nodes are either documents or disconnected if ( !aup || !bup ) { // Support: IE 11+, Edge 17 - 18+ // IE/Edge sometimes throw a "Permission denied" error when strict-comparing // two documents; shallow comparisons work. /* eslint-disable eqeqeq */ return a == document ? -1 : b == document ? 1 : /* eslint-enable eqeqeq */ aup ? -1 : bup ? 1 : sortInput ? ( indexOf( sortInput, a ) - indexOf( sortInput, b ) ) : 0; // If the nodes are siblings, we can do a quick check } else if ( aup === bup ) { return siblingCheck( a, b ); } // Otherwise we need full lists of their ancestors for comparison cur = a; while ( ( cur = cur.parentNode ) ) { ap.unshift( cur ); } cur = b; while ( ( cur = cur.parentNode ) ) { bp.unshift( cur ); } // Walk down the tree looking for a discrepancy while ( ap[ i ] === bp[ i ] ) { i++; } return i ? // Do a sibling check if the nodes have a common ancestor siblingCheck( ap[ i ], bp[ i ] ) : // Otherwise nodes in our document sort first // Support: IE 11+, Edge 17 - 18+ // IE/Edge sometimes throw a "Permission denied" error when strict-comparing // two documents; shallow comparisons work. /* eslint-disable eqeqeq */ ap[ i ] == preferredDoc ? -1 : bp[ i ] == preferredDoc ? 1 : /* eslint-enable eqeqeq */ 0; }; return document; }; Sizzle.matches = function( expr, elements ) { return Sizzle( expr, null, null, elements ); }; Sizzle.matchesSelector = function( elem, expr ) { setDocument( elem ); if ( support.matchesSelector && documentIsHTML && !nonnativeSelectorCache[ expr + " " ] && ( !rbuggyMatches || !rbuggyMatches.test( expr ) ) && ( !rbuggyQSA || !rbuggyQSA.test( expr ) ) ) { try { var ret = matches.call( elem, expr ); // IE 9's matchesSelector returns false on disconnected nodes if ( ret || support.disconnectedMatch || // As well, disconnected nodes are said to be in a document // fragment in IE 9 elem.document && elem.document.nodeType !== 11 ) { return ret; } } catch ( e ) { nonnativeSelectorCache( expr, true ); } } return Sizzle( expr, document, null, [ elem ] ).length > 0; }; Sizzle.contains = function( context, elem ) { // Set document vars if needed // Support: IE 11+, Edge 17 - 18+ // IE/Edge sometimes throw a "Permission denied" error when strict-comparing // two documents; shallow comparisons work. // eslint-disable-next-line eqeqeq if ( ( context.ownerDocument || context ) != document ) { setDocument( context ); } return contains( context, elem ); }; Sizzle.attr = function( elem, name ) { // Set document vars if needed // Support: IE 11+, Edge 17 - 18+ // IE/Edge sometimes throw a "Permission denied" error when strict-comparing // two documents; shallow comparisons work. // eslint-disable-next-line eqeqeq if ( ( elem.ownerDocument || elem ) != document ) { setDocument( elem ); } var fn = Expr.attrHandle[ name.toLowerCase() ], // Don't get fooled by Object.prototype properties (jQuery #13807) val = fn && hasOwn.call( Expr.attrHandle, name.toLowerCase() ) ? fn( elem, name, !documentIsHTML ) : undefined; return val !== undefined ? val : support.attributes || !documentIsHTML ? elem.getAttribute( name ) : ( val = elem.getAttributeNode( name ) ) && val.specified ? val.value : null; }; Sizzle.escape = function( sel ) { return ( sel + "" ).replace( rcssescape, fcssescape ); }; Sizzle.error = function( msg ) { throw new Error( "Syntax error, unrecognized expression: " + msg ); }; /** * Document sorting and removing duplicates * @param {ArrayLike} results */ Sizzle.uniqueSort = function( results ) { var elem, duplicates = [], j = 0, i = 0; // Unless we *know* we can detect duplicates, assume their presence hasDuplicate = !support.detectDuplicates; sortInput = !support.sortStable && results.slice( 0 ); results.sort( sortOrder ); if ( hasDuplicate ) { while ( ( elem = results[ i++ ] ) ) { if ( elem === results[ i ] ) { j = duplicates.push( i ); } } while ( j-- ) { results.splice( duplicates[ j ], 1 ); } } // Clear input after sorting to release objects // See https://github.com/jquery/sizzle/pull/225 sortInput = null; return results; }; /** * Utility function for retrieving the text value of an array of DOM nodes * @param {Array|Element} elem */ getText = Sizzle.getText = function( elem ) { var node, ret = "", i = 0, nodeType = elem.nodeType; if ( !nodeType ) { // If no nodeType, this is expected to be an array while ( ( node = elem[ i++ ] ) ) { // Do not traverse comment nodes ret += getText( node ); } } else if ( nodeType === 1 || nodeType === 9 || nodeType === 11 ) { // Use textContent for elements // innerText usage removed for consistency of new lines (jQuery #11153) if ( typeof elem.textContent === "string" ) { return elem.textContent; } else { // Traverse its children for ( elem = elem.firstChild; elem; elem = elem.nextSibling ) { ret += getText( elem ); } } } else if ( nodeType === 3 || nodeType === 4 ) { return elem.nodeValue; } // Do not include comment or processing instruction nodes return ret; }; Expr = Sizzle.selectors = { // Can be adjusted by the user cacheLength: 50, createPseudo: markFunction, match: matchExpr, attrHandle: {}, find: {}, relative: { ">": { dir: "parentNode", first: true }, " ": { dir: "parentNode" }, "+": { dir: "previousSibling", first: true }, "~": { dir: "previousSibling" } }, preFilter: { "ATTR": function( match ) { match[ 1 ] = match[ 1 ].replace( runescape, funescape ); // Move the given value to match[3] whether quoted or unquoted match[ 3 ] = ( match[ 3 ] || match[ 4 ] || match[ 5 ] || "" ).replace( runescape, funescape ); if ( match[ 2 ] === "~=" ) { match[ 3 ] = " " + match[ 3 ] + " "; } return match.slice( 0, 4 ); }, "CHILD": function( match ) { /* matches from matchExpr["CHILD"] 1 type (only|nth|...) 2 what (child|of-type) 3 argument (even|odd|\d*|\d*n([+-]\d+)?|...) 4 xn-component of xn+y argument ([+-]?\d*n|) 5 sign of xn-component 6 x of xn-component 7 sign of y-component 8 y of y-component */ match[ 1 ] = match[ 1 ].toLowerCase(); if ( match[ 1 ].slice( 0, 3 ) === "nth" ) { // nth-* requires argument if ( !match[ 3 ] ) { Sizzle.error( match[ 0 ] ); } // numeric x and y parameters for Expr.filter.CHILD // remember that false/true cast respectively to 0/1 match[ 4 ] = +( match[ 4 ] ? match[ 5 ] + ( match[ 6 ] || 1 ) : 2 * ( match[ 3 ] === "even" || match[ 3 ] === "odd" ) ); match[ 5 ] = +( ( match[ 7 ] + match[ 8 ] ) || match[ 3 ] === "odd" ); // other types prohibit arguments } else if ( match[ 3 ] ) { Sizzle.error( match[ 0 ] ); } return match; }, "PSEUDO": function( match ) { var excess, unquoted = !match[ 6 ] && match[ 2 ]; if ( matchExpr[ "CHILD" ].test( match[ 0 ] ) ) { return null; } // Accept quoted arguments as-is if ( match[ 3 ] ) { match[ 2 ] = match[ 4 ] || match[ 5 ] || ""; // Strip excess characters from unquoted arguments } else if ( unquoted && rpseudo.test( unquoted ) && // Get excess from tokenize (recursively) ( excess = tokenize( unquoted, true ) ) && // advance to the next closing parenthesis ( excess = unquoted.indexOf( ")", unquoted.length - excess ) - unquoted.length ) ) { // excess is a negative index match[ 0 ] = match[ 0 ].slice( 0, excess ); match[ 2 ] = unquoted.slice( 0, excess ); } // Return only captures needed by the pseudo filter method (type and argument) return match.slice( 0, 3 ); } }, filter: { "TAG": function( nodeNameSelector ) { var nodeName = nodeNameSelector.replace( runescape, funescape ).toLowerCase(); return nodeNameSelector === "*" ? function() { return true; } : function( elem ) { return elem.nodeName && elem.nodeName.toLowerCase() === nodeName; }; }, "CLASS": function( className ) { var pattern = classCache[ className + " " ]; return pattern || ( pattern = new RegExp( "(^|" + whitespace + ")" + className + "(" + whitespace + "|$)" ) ) && classCache( className, function( elem ) { return pattern.test( typeof elem.className === "string" && elem.className || typeof elem.getAttribute !== "undefined" && elem.getAttribute( "class" ) || "" ); } ); }, "ATTR": function( name, operator, check ) { return function( elem ) { var result = Sizzle.attr( elem, name ); if ( result == null ) { return operator === "!="; } if ( !operator ) { return true; } result += ""; /* eslint-disable max-len */ return operator === "=" ? result === check : operator === "!=" ? result !== check : operator === "^=" ? check && result.indexOf( check ) === 0 : operator === "*=" ? check && result.indexOf( check ) > -1 : operator === "$=" ? check && result.slice( -check.length ) === check : operator === "~=" ? ( " " + result.replace( rwhitespace, " " ) + " " ).indexOf( check ) > -1 : operator === "|=" ? result === check || result.slice( 0, check.length + 1 ) === check + "-" : false; /* eslint-enable max-len */ }; }, "CHILD": function( type, what, _argument, first, last ) { var simple = type.slice( 0, 3 ) !== "nth", forward = type.slice( -4 ) !== "last", ofType = what === "of-type"; return first === 1 && last === 0 ? // Shortcut for :nth-*(n) function( elem ) { return !!elem.parentNode; } : function( elem, _context, xml ) { var cache, uniqueCache, outerCache, node, nodeIndex, start, dir = simple !== forward ? "nextSibling" : "previousSibling", parent = elem.parentNode, name = ofType && elem.nodeName.toLowerCase(), useCache = !xml && !ofType, diff = false; if ( parent ) { // :(first|last|only)-(child|of-type) if ( simple ) { while ( dir ) { node = elem; while ( ( node = node[ dir ] ) ) { if ( ofType ? node.nodeName.toLowerCase() === name : node.nodeType === 1 ) { return false; } } // Reverse direction for :only-* (if we haven't yet done so) start = dir = type === "only" && !start && "nextSibling"; } return true; } start = [ forward ? parent.firstChild : parent.lastChild ]; // non-xml :nth-child(...) stores cache data on `parent` if ( forward && useCache ) { // Seek `elem` from a previously-cached index // ...in a gzip-friendly way node = parent; outerCache = node[ expando ] || ( node[ expando ] = {} ); // Support: IE <9 only // Defend against cloned attroperties (jQuery gh-1709) uniqueCache = outerCache[ node.uniqueID ] || ( outerCache[ node.uniqueID ] = {} ); cache = uniqueCache[ type ] || []; nodeIndex = cache[ 0 ] === dirruns && cache[ 1 ]; diff = nodeIndex && cache[ 2 ]; node = nodeIndex && parent.childNodes[ nodeIndex ]; while ( ( node = ++nodeIndex && node && node[ dir ] || // Fallback to seeking `elem` from the start ( diff = nodeIndex = 0 ) || start.pop() ) ) { // When found, cache indexes on `parent` and break if ( node.nodeType === 1 && ++diff && node === elem ) { uniqueCache[ type ] = [ dirruns, nodeIndex, diff ]; break; } } } else { // Use previously-cached element index if available if ( useCache ) { // ...in a gzip-friendly way node = elem; outerCache = node[ expando ] || ( node[ expando ] = {} ); // Support: IE <9 only // Defend against cloned attroperties (jQuery gh-1709) uniqueCache = outerCache[ node.uniqueID ] || ( outerCache[ node.uniqueID ] = {} ); cache = uniqueCache[ type ] || []; nodeIndex = cache[ 0 ] === dirruns && cache[ 1 ]; diff = nodeIndex; } // xml :nth-child(...) // or :nth-last-child(...) or :nth(-last)?-of-type(...) if ( diff === false ) { // Use the same loop as above to seek `elem` from the start while ( ( node = ++nodeIndex && node && node[ dir ] || ( diff = nodeIndex = 0 ) || start.pop() ) ) { if ( ( ofType ? node.nodeName.toLowerCase() === name : node.nodeType === 1 ) && ++diff ) { // Cache the index of each encountered element if ( useCache ) { outerCache = node[ expando ] || ( node[ expando ] = {} ); // Support: IE <9 only // Defend against cloned attroperties (jQuery gh-1709) uniqueCache = outerCache[ node.uniqueID ] || ( outerCache[ node.uniqueID ] = {} ); uniqueCache[ type ] = [ dirruns, diff ]; } if ( node === elem ) { break; } } } } } // Incorporate the offset, then check against cycle size diff -= last; return diff === first || ( diff % first === 0 && diff / first >= 0 ); } }; }, "PSEUDO": function( pseudo, argument ) { // pseudo-class names are case-insensitive // http://www.w3.org/TR/selectors/#pseudo-classes // Prioritize by case sensitivity in case custom pseudos are added with uppercase letters // Remember that setFilters inherits from pseudos var args, fn = Expr.pseudos[ pseudo ] || Expr.setFilters[ pseudo.toLowerCase() ] || Sizzle.error( "unsupported pseudo: " + pseudo ); // The user may use createPseudo to indicate that // arguments are needed to create the filter function // just as Sizzle does if ( fn[ expando ] ) { return fn( argument ); } // But maintain support for old signatures if ( fn.length > 1 ) { args = [ pseudo, pseudo, "", argument ]; return Expr.setFilters.hasOwnProperty( pseudo.toLowerCase() ) ? markFunction( function( seed, matches ) { var idx, matched = fn( seed, argument ), i = matched.length; while ( i-- ) { idx = indexOf( seed, matched[ i ] ); seed[ idx ] = !( matches[ idx ] = matched[ i ] ); } } ) : function( elem ) { return fn( elem, 0, args ); }; } return fn; } }, pseudos: { // Potentially complex pseudos "not": markFunction( function( selector ) { // Trim the selector passed to compile // to avoid treating leading and trailing // spaces as combinators var input = [], results = [], matcher = compile( selector.replace( rtrim, "$1" ) ); return matcher[ expando ] ? markFunction( function( seed, matches, _context, xml ) { var elem, unmatched = matcher( seed, null, xml, [] ), i = seed.length; // Match elements unmatched by `matcher` while ( i-- ) { if ( ( elem = unmatched[ i ] ) ) { seed[ i ] = !( matches[ i ] = elem ); } } } ) : function( elem, _context, xml ) { input[ 0 ] = elem; matcher( input, null, xml, results ); // Don't keep the element (issue #299) input[ 0 ] = null; return !results.pop(); }; } ), "has": markFunction( function( selector ) { return function( elem ) { return Sizzle( selector, elem ).length > 0; }; } ), "contains": markFunction( function( text ) { text = text.replace( runescape, funescape ); return function( elem ) { return ( elem.textContent || getText( elem ) ).indexOf( text ) > -1; }; } ), // "Whether an element is represented by a :lang() selector // is based solely on the element's language value // being equal to the identifier C, // or beginning with the identifier C immediately followed by "-". // The matching of C against the element's language value is performed case-insensitively. // The identifier C does not have to be a valid language name." // http://www.w3.org/TR/selectors/#lang-pseudo "lang": markFunction( function( lang ) { // lang value must be a valid identifier if ( !ridentifier.test( lang || "" ) ) { Sizzle.error( "unsupported lang: " + lang ); } lang = lang.replace( runescape, funescape ).toLowerCase(); return function( elem ) { var elemLang; do { if ( ( elemLang = documentIsHTML ? elem.lang : elem.getAttribute( "xml:lang" ) || elem.getAttribute( "lang" ) ) ) { elemLang = elemLang.toLowerCase(); return elemLang === lang || elemLang.indexOf( lang + "-" ) === 0; } } while ( ( elem = elem.parentNode ) && elem.nodeType === 1 ); return false; }; } ), // Miscellaneous "target": function( elem ) { var hash = window.location && window.location.hash; return hash && hash.slice( 1 ) === elem.id; }, "root": function( elem ) { return elem === docElem; }, "focus": function( elem ) { return elem === document.activeElement && ( !document.hasFocus || document.hasFocus() ) && !!( elem.type || elem.href || ~elem.tabIndex ); }, // Boolean properties "enabled": createDisabledPseudo( false ), "disabled": createDisabledPseudo( true ), "checked": function( elem ) { // In CSS3, :checked should return both checked and selected elements // http://www.w3.org/TR/2011/REC-css3-selectors-20110929/#checked var nodeName = elem.nodeName.toLowerCase(); return ( nodeName === "input" && !!elem.checked ) || ( nodeName === "option" && !!elem.selected ); }, "selected": function( elem ) { // Accessing this property makes selected-by-default // options in Safari work properly if ( elem.parentNode ) { // eslint-disable-next-line no-unused-expressions elem.parentNode.selectedIndex; } return elem.selected === true; }, // Contents "empty": function( elem ) { // http://www.w3.org/TR/selectors/#empty-pseudo // :empty is negated by element (1) or content nodes (text: 3; cdata: 4; entity ref: 5), // but not by others (comment: 8; processing instruction: 7; etc.) // nodeType < 6 works because attributes (2) do not appear as children for ( elem = elem.firstChild; elem; elem = elem.nextSibling ) { if ( elem.nodeType < 6 ) { return false; } } return true; }, "parent": function( elem ) { return !Expr.pseudos[ "empty" ]( elem ); }, // Element/input types "header": function( elem ) { return rheader.test( elem.nodeName ); }, "input": function( elem ) { return rinputs.test( elem.nodeName ); }, "button": function( elem ) { var name = elem.nodeName.toLowerCase(); return name === "input" && elem.type === "button" || name === "button"; }, "text": function( elem ) { var attr; return elem.nodeName.toLowerCase() === "input" && elem.type === "text" && // Support: IE<8 // New HTML5 attribute values (e.g., "search") appear with elem.type === "text" ( ( attr = elem.getAttribute( "type" ) ) == null || attr.toLowerCase() === "text" ); }, // Position-in-collection "first": createPositionalPseudo( function() { return [ 0 ]; } ), "last": createPositionalPseudo( function( _matchIndexes, length ) { return [ length - 1 ]; } ), "eq": createPositionalPseudo( function( _matchIndexes, length, argument ) { return [ argument < 0 ? argument + length : argument ]; } ), "even": createPositionalPseudo( function( matchIndexes, length ) { var i = 0; for ( ; i < length; i += 2 ) { matchIndexes.push( i ); } return matchIndexes; } ), "odd": createPositionalPseudo( function( matchIndexes, length ) { var i = 1; for ( ; i < length; i += 2 ) { matchIndexes.push( i ); } return matchIndexes; } ), "lt": createPositionalPseudo( function( matchIndexes, length, argument ) { var i = argument < 0 ? argument + length : argument > length ? length : argument; for ( ; --i >= 0; ) { matchIndexes.push( i ); } return matchIndexes; } ), "gt": createPositionalPseudo( function( matchIndexes, length, argument ) { var i = argument < 0 ? argument + length : argument; for ( ; ++i < length; ) { matchIndexes.push( i ); } return matchIndexes; } ) } }; Expr.pseudos[ "nth" ] = Expr.pseudos[ "eq" ]; // Add button/input type pseudos for ( i in { radio: true, checkbox: true, file: true, password: true, image: true } ) { Expr.pseudos[ i ] = createInputPseudo( i ); } for ( i in { submit: true, reset: true } ) { Expr.pseudos[ i ] = createButtonPseudo( i ); } // Easy API for creating new setFilters function setFilters() {} setFilters.prototype = Expr.filters = Expr.pseudos; Expr.setFilters = new setFilters(); tokenize = Sizzle.tokenize = function( selector, parseOnly ) { var matched, match, tokens, type, soFar, groups, preFilters, cached = tokenCache[ selector + " " ]; if ( cached ) { return parseOnly ? 0 : cached.slice( 0 ); } soFar = selector; groups = []; preFilters = Expr.preFilter; while ( soFar ) { // Comma and first run if ( !matched || ( match = rcomma.exec( soFar ) ) ) { if ( match ) { // Don't consume trailing commas as valid soFar = soFar.slice( match[ 0 ].length ) || soFar; } groups.push( ( tokens = [] ) ); } matched = false; // Combinators if ( ( match = rcombinators.exec( soFar ) ) ) { matched = match.shift(); tokens.push( { value: matched, // Cast descendant combinators to space type: match[ 0 ].replace( rtrim, " " ) } ); soFar = soFar.slice( matched.length ); } // Filters for ( type in Expr.filter ) { if ( ( match = matchExpr[ type ].exec( soFar ) ) && ( !preFilters[ type ] || ( match = preFilters[ type ]( match ) ) ) ) { matched = match.shift(); tokens.push( { value: matched, type: type, matches: match } ); soFar = soFar.slice( matched.length ); } } if ( !matched ) { break; } } // Return the length of the invalid excess // if we're just parsing // Otherwise, throw an error or return tokens return parseOnly ? soFar.length : soFar ? Sizzle.error( selector ) : // Cache the tokens tokenCache( selector, groups ).slice( 0 ); }; function toSelector( tokens ) { var i = 0, len = tokens.length, selector = ""; for ( ; i < len; i++ ) { selector += tokens[ i ].value; } return selector; } function addCombinator( matcher, combinator, base ) { var dir = combinator.dir, skip = combinator.next, key = skip || dir, checkNonElements = base && key === "parentNode", doneName = done++; return combinator.first ? // Check against closest ancestor/preceding element function( elem, context, xml ) { while ( ( elem = elem[ dir ] ) ) { if ( elem.nodeType === 1 || checkNonElements ) { return matcher( elem, context, xml ); } } return false; } : // Check against all ancestor/preceding elements function( elem, context, xml ) { var oldCache, uniqueCache, outerCache, newCache = [ dirruns, doneName ]; // We can't set arbitrary data on XML nodes, so they don't benefit from combinator caching if ( xml ) { while ( ( elem = elem[ dir ] ) ) { if ( elem.nodeType === 1 || checkNonElements ) { if ( matcher( elem, context, xml ) ) { return true; } } } } else { while ( ( elem = elem[ dir ] ) ) { if ( elem.nodeType === 1 || checkNonElements ) { outerCache = elem[ expando ] || ( elem[ expando ] = {} ); // Support: IE <9 only // Defend against cloned attroperties (jQuery gh-1709) uniqueCache = outerCache[ elem.uniqueID ] || ( outerCache[ elem.uniqueID ] = {} ); if ( skip && skip === elem.nodeName.toLowerCase() ) { elem = elem[ dir ] || elem; } else if ( ( oldCache = uniqueCache[ key ] ) && oldCache[ 0 ] === dirruns && oldCache[ 1 ] === doneName ) { // Assign to newCache so results back-propagate to previous elements return ( newCache[ 2 ] = oldCache[ 2 ] ); } else { // Reuse newcache so results back-propagate to previous elements uniqueCache[ key ] = newCache; // A match means we're done; a fail means we have to keep checking if ( ( newCache[ 2 ] = matcher( elem, context, xml ) ) ) { return true; } } } } } return false; }; } function elementMatcher( matchers ) { return matchers.length > 1 ? function( elem, context, xml ) { var i = matchers.length; while ( i-- ) { if ( !matchers[ i ]( elem, context, xml ) ) { return false; } } return true; } : matchers[ 0 ]; } function multipleContexts( selector, contexts, results ) { var i = 0, len = contexts.length; for ( ; i < len; i++ ) { Sizzle( selector, contexts[ i ], results ); } return results; } function condense( unmatched, map, filter, context, xml ) { var elem, newUnmatched = [], i = 0, len = unmatched.length, mapped = map != null; for ( ; i < len; i++ ) { if ( ( elem = unmatched[ i ] ) ) { if ( !filter || filter( elem, context, xml ) ) { newUnmatched.push( elem ); if ( mapped ) { map.push( i ); } } } } return newUnmatched; } function setMatcher( preFilter, selector, matcher, postFilter, postFinder, postSelector ) { if ( postFilter && !postFilter[ expando ] ) { postFilter = setMatcher( postFilter ); } if ( postFinder && !postFinder[ expando ] ) { postFinder = setMatcher( postFinder, postSelector ); } return markFunction( function( seed, results, context, xml ) { var temp, i, elem, preMap = [], postMap = [], preexisting = results.length, // Get initial elements from seed or context elems = seed || multipleContexts( selector || "*", context.nodeType ? [ context ] : context, [] ), // Prefilter to get matcher input, preserving a map for seed-results synchronization matcherIn = preFilter && ( seed || !selector ) ? condense( elems, preMap, preFilter, context, xml ) : elems, matcherOut = matcher ? // If we have a postFinder, or filtered seed, or non-seed postFilter or preexisting results, postFinder || ( seed ? preFilter : preexisting || postFilter ) ? // ...intermediate processing is necessary [] : // ...otherwise use results directly results : matcherIn; // Find primary matches if ( matcher ) { matcher( matcherIn, matcherOut, context, xml ); } // Apply postFilter if ( postFilter ) { temp = condense( matcherOut, postMap ); postFilter( temp, [], context, xml ); // Un-match failing elements by moving them back to matcherIn i = temp.length; while ( i-- ) { if ( ( elem = temp[ i ] ) ) { matcherOut[ postMap[ i ] ] = !( matcherIn[ postMap[ i ] ] = elem ); } } } if ( seed ) { if ( postFinder || preFilter ) { if ( postFinder ) { // Get the final matcherOut by condensing this intermediate into postFinder contexts temp = []; i = matcherOut.length; while ( i-- ) { if ( ( elem = matcherOut[ i ] ) ) { // Restore matcherIn since elem is not yet a final match temp.push( ( matcherIn[ i ] = elem ) ); } } postFinder( null, ( matcherOut = [] ), temp, xml ); } // Move matched elements from seed to results to keep them synchronized i = matcherOut.length; while ( i-- ) { if ( ( elem = matcherOut[ i ] ) && ( temp = postFinder ? indexOf( seed, elem ) : preMap[ i ] ) > -1 ) { seed[ temp ] = !( results[ temp ] = elem ); } } } // Add elements to results, through postFinder if defined } else { matcherOut = condense( matcherOut === results ? matcherOut.splice( preexisting, matcherOut.length ) : matcherOut ); if ( postFinder ) { postFinder( null, results, matcherOut, xml ); } else { push.apply( results, matcherOut ); } } } ); } function matcherFromTokens( tokens ) { var checkContext, matcher, j, len = tokens.length, leadingRelative = Expr.relative[ tokens[ 0 ].type ], implicitRelative = leadingRelative || Expr.relative[ " " ], i = leadingRelative ? 1 : 0, // The foundational matcher ensures that elements are reachable from top-level context(s) matchContext = addCombinator( function( elem ) { return elem === checkContext; }, implicitRelative, true ), matchAnyContext = addCombinator( function( elem ) { return indexOf( checkContext, elem ) > -1; }, implicitRelative, true ), matchers = [ function( elem, context, xml ) { var ret = ( !leadingRelative && ( xml || context !== outermostContext ) ) || ( ( checkContext = context ).nodeType ? matchContext( elem, context, xml ) : matchAnyContext( elem, context, xml ) ); // Avoid hanging onto element (issue #299) checkContext = null; return ret; } ]; for ( ; i < len; i++ ) { if ( ( matcher = Expr.relative[ tokens[ i ].type ] ) ) { matchers = [ addCombinator( elementMatcher( matchers ), matcher ) ]; } else { matcher = Expr.filter[ tokens[ i ].type ].apply( null, tokens[ i ].matches ); // Return special upon seeing a positional matcher if ( matcher[ expando ] ) { // Find the next relative operator (if any) for proper handling j = ++i; for ( ; j < len; j++ ) { if ( Expr.relative[ tokens[ j ].type ] ) { break; } } return setMatcher( i > 1 && elementMatcher( matchers ), i > 1 && toSelector( // If the preceding token was a descendant combinator, insert an implicit any-element `*` tokens .slice( 0, i - 1 ) .concat( { value: tokens[ i - 2 ].type === " " ? "*" : "" } ) ).replace( rtrim, "$1" ), matcher, i < j && matcherFromTokens( tokens.slice( i, j ) ), j < len && matcherFromTokens( ( tokens = tokens.slice( j ) ) ), j < len && toSelector( tokens ) ); } matchers.push( matcher ); } } return elementMatcher( matchers ); } function matcherFromGroupMatchers( elementMatchers, setMatchers ) { var bySet = setMatchers.length > 0, byElement = elementMatchers.length > 0, superMatcher = function( seed, context, xml, results, outermost ) { var elem, j, matcher, matchedCount = 0, i = "0", unmatched = seed && [], setMatched = [], contextBackup = outermostContext, // We must always have either seed elements or outermost context elems = seed || byElement && Expr.find[ "TAG" ]( "*", outermost ), // Use integer dirruns iff this is the outermost matcher dirrunsUnique = ( dirruns += contextBackup == null ? 1 : Math.random() || 0.1 ), len = elems.length; if ( outermost ) { // Support: IE 11+, Edge 17 - 18+ // IE/Edge sometimes throw a "Permission denied" error when strict-comparing // two documents; shallow comparisons work. // eslint-disable-next-line eqeqeq outermostContext = context == document || context || outermost; } // Add elements passing elementMatchers directly to results // Support: IE<9, Safari // Tolerate NodeList properties (IE: "length"; Safari: <number>) matching elements by id for ( ; i !== len && ( elem = elems[ i ] ) != null; i++ ) { if ( byElement && elem ) { j = 0; // Support: IE 11+, Edge 17 - 18+ // IE/Edge sometimes throw a "Permission denied" error when strict-comparing // two documents; shallow comparisons work. // eslint-disable-next-line eqeqeq if ( !context && elem.ownerDocument != document ) { setDocument( elem ); xml = !documentIsHTML; } while ( ( matcher = elementMatchers[ j++ ] ) ) { if ( matcher( elem, context || document, xml ) ) { results.push( elem ); break; } } if ( outermost ) { dirruns = dirrunsUnique; } } // Track unmatched elements for set filters if ( bySet ) { // They will have gone through all possible matchers if ( ( elem = !matcher && elem ) ) { matchedCount--; } // Lengthen the array for every element, matched or not if ( seed ) { unmatched.push( elem ); } } } // `i` is now the count of elements visited above, and adding it to `matchedCount` // makes the latter nonnegative. matchedCount += i; // Apply set filters to unmatched elements // NOTE: This can be skipped if there are no unmatched elements (i.e., `matchedCount` // equals `i`), unless we didn't visit _any_ elements in the above loop because we have // no element matchers and no seed. // Incrementing an initially-string "0" `i` allows `i` to remain a string only in that // case, which will result in a "00" `matchedCount` that differs from `i` but is also // numerically zero. if ( bySet && i !== matchedCount ) { j = 0; while ( ( matcher = setMatchers[ j++ ] ) ) { matcher( unmatched, setMatched, context, xml ); } if ( seed ) { // Reintegrate element matches to eliminate the need for sorting if ( matchedCount > 0 ) { while ( i-- ) { if ( !( unmatched[ i ] || setMatched[ i ] ) ) { setMatched[ i ] = pop.call( results ); } } } // Discard index placeholder values to get only actual matches setMatched = condense( setMatched ); } // Add matches to results push.apply( results, setMatched ); // Seedless set matches succeeding multiple successful matchers stipulate sorting if ( outermost && !seed && setMatched.length > 0 && ( matchedCount + setMatchers.length ) > 1 ) { Sizzle.uniqueSort( results ); } } // Override manipulation of globals by nested matchers if ( outermost ) { dirruns = dirrunsUnique; outermostContext = contextBackup; } return unmatched; }; return bySet ? markFunction( superMatcher ) : superMatcher; } compile = Sizzle.compile = function( selector, match /* Internal Use Only */ ) { var i, setMatchers = [], elementMatchers = [], cached = compilerCache[ selector + " " ]; if ( !cached ) { // Generate a function of recursive functions that can be used to check each element if ( !match ) { match = tokenize( selector ); } i = match.length; while ( i-- ) { cached = matcherFromTokens( match[ i ] ); if ( cached[ expando ] ) { setMatchers.push( cached ); } else { elementMatchers.push( cached ); } } // Cache the compiled function cached = compilerCache( selector, matcherFromGroupMatchers( elementMatchers, setMatchers ) ); // Save selector and tokenization cached.selector = selector; } return cached; }; /** * A low-level selection function that works with Sizzle's compiled * selector functions * @param {String|Function} selector A selector or a pre-compiled * selector function built with Sizzle.compile * @param {Element} context * @param {Array} [results] * @param {Array} [seed] A set of elements to match against */ select = Sizzle.select = function( selector, context, results, seed ) { var i, tokens, token, type, find, compiled = typeof selector === "function" && selector, match = !seed && tokenize( ( selector = compiled.selector || selector ) ); results = results || []; // Try to minimize operations if there is only one selector in the list and no seed // (the latter of which guarantees us context) if ( match.length === 1 ) { // Reduce context if the leading compound selector is an ID tokens = match[ 0 ] = match[ 0 ].slice( 0 ); if ( tokens.length > 2 && ( token = tokens[ 0 ] ).type === "ID" && context.nodeType === 9 && documentIsHTML && Expr.relative[ tokens[ 1 ].type ] ) { context = ( Expr.find[ "ID" ]( token.matches[ 0 ] .replace( runescape, funescape ), context ) || [] )[ 0 ]; if ( !context ) { return results; // Precompiled matchers will still verify ancestry, so step up a level } else if ( compiled ) { context = context.parentNode; } selector = selector.slice( tokens.shift().value.length ); } // Fetch a seed set for right-to-left matching i = matchExpr[ "needsContext" ].test( selector ) ? 0 : tokens.length; while ( i-- ) { token = tokens[ i ]; // Abort if we hit a combinator if ( Expr.relative[ ( type = token.type ) ] ) { break; } if ( ( find = Expr.find[ type ] ) ) { // Search, expanding context for leading sibling combinators if ( ( seed = find( token.matches[ 0 ].replace( runescape, funescape ), rsibling.test( tokens[ 0 ].type ) && testContext( context.parentNode ) || context ) ) ) { // If seed is empty or no tokens remain, we can return early tokens.splice( i, 1 ); selector = seed.length && toSelector( tokens ); if ( !selector ) { push.apply( results, seed ); return results; } break; } } } } // Compile and execute a filtering function if one is not provided // Provide `match` to avoid retokenization if we modified the selector above ( compiled || compile( selector, match ) )( seed, context, !documentIsHTML, results, !context || rsibling.test( selector ) && testContext( context.parentNode ) || context ); return results; }; // One-time assignments // Sort stability support.sortStable = expando.split( "" ).sort( sortOrder ).join( "" ) === expando; // Support: Chrome 14-35+ // Always assume duplicates if they aren't passed to the comparison function support.detectDuplicates = !!hasDuplicate; // Initialize against the default document setDocument(); // Support: Webkit<537.32 - Safari 6.0.3/Chrome 25 (fixed in Chrome 27) // Detached nodes confoundingly follow *each other* support.sortDetached = assert( function( el ) { // Should return 1, but returns 4 (following) return el.compareDocumentPosition( document.createElement( "fieldset" ) ) & 1; } ); // Support: IE<8 // Prevent attribute/property "interpolation" // https://msdn.microsoft.com/en-us/library/ms536429%28VS.85%29.aspx if ( !assert( function( el ) { el.innerHTML = "<a href='#'></a>"; return el.firstChild.getAttribute( "href" ) === "#"; } ) ) { addHandle( "type|href|height|width", function( elem, name, isXML ) { if ( !isXML ) { return elem.getAttribute( name, name.toLowerCase() === "type" ? 1 : 2 ); } } ); } // Support: IE<9 // Use defaultValue in place of getAttribute("value") if ( !support.attributes || !assert( function( el ) { el.innerHTML = "<input/>"; el.firstChild.setAttribute( "value", "" ); return el.firstChild.getAttribute( "value" ) === ""; } ) ) { addHandle( "value", function( elem, _name, isXML ) { if ( !isXML && elem.nodeName.toLowerCase() === "input" ) { return elem.defaultValue; } } ); } // Support: IE<9 // Use getAttributeNode to fetch booleans when getAttribute lies if ( !assert( function( el ) { return el.getAttribute( "disabled" ) == null; } ) ) { addHandle( booleans, function( elem, name, isXML ) { var val; if ( !isXML ) { return elem[ name ] === true ? name.toLowerCase() : ( val = elem.getAttributeNode( name ) ) && val.specified ? val.value : null; } } ); } return Sizzle; } )( window ); jQuery.find = Sizzle; jQuery.expr = Sizzle.selectors; // Deprecated jQuery.expr[ ":" ] = jQuery.expr.pseudos; jQuery.uniqueSort = jQuery.unique = Sizzle.uniqueSort; jQuery.text = Sizzle.getText; jQuery.isXMLDoc = Sizzle.isXML; jQuery.contains = Sizzle.contains; jQuery.escapeSelector = Sizzle.escape; var dir = function( elem, dir, until ) { var matched = [], truncate = until !== undefined; while ( ( elem = elem[ dir ] ) && elem.nodeType !== 9 ) { if ( elem.nodeType === 1 ) { if ( truncate && jQuery( elem ).is( until ) ) { break; } matched.push( elem ); } } return matched; }; var siblings = function( n, elem ) { var matched = []; for ( ; n; n = n.nextSibling ) { if ( n.nodeType === 1 && n !== elem ) { matched.push( n ); } } return matched; }; var rneedsContext = jQuery.expr.match.needsContext; function nodeName( elem, name ) { return elem.nodeName && elem.nodeName.toLowerCase() === name.toLowerCase(); } var rsingleTag = ( /^<([a-z][^\/\0>:\x20\t\r\n\f]*)[\x20\t\r\n\f]*\/?>(?:<\/\1>|)$/i ); // Implement the identical functionality for filter and not function winnow( elements, qualifier, not ) { if ( isFunction( qualifier ) ) { return jQuery.grep( elements, function( elem, i ) { return !!qualifier.call( elem, i, elem ) !== not; } ); } // Single element if ( qualifier.nodeType ) { return jQuery.grep( elements, function( elem ) { return ( elem === qualifier ) !== not; } ); } // Arraylike of elements (jQuery, arguments, Array) if ( typeof qualifier !== "string" ) { return jQuery.grep( elements, function( elem ) { return ( indexOf.call( qualifier, elem ) > -1 ) !== not; } ); } // Filtered directly for both simple and complex selectors return jQuery.filter( qualifier, elements, not ); } jQuery.filter = function( expr, elems, not ) { var elem = elems[ 0 ]; if ( not ) { expr = ":not(" + expr + ")"; } if ( elems.length === 1 && elem.nodeType === 1 ) { return jQuery.find.matchesSelector( elem, expr ) ? [ elem ] : []; } return jQuery.find.matches( expr, jQuery.grep( elems, function( elem ) { return elem.nodeType === 1; } ) ); }; jQuery.fn.extend( { find: function( selector ) { var i, ret, len = this.length, self = this; if ( typeof selector !== "string" ) { return this.pushStack( jQuery( selector ).filter( function() { for ( i = 0; i < len; i++ ) { if ( jQuery.contains( self[ i ], this ) ) { return true; } } } ) ); } ret = this.pushStack( [] ); for ( i = 0; i < len; i++ ) { jQuery.find( selector, self[ i ], ret ); } return len > 1 ? jQuery.uniqueSort( ret ) : ret; }, filter: function( selector ) { return this.pushStack( winnow( this, selector || [], false ) ); }, not: function( selector ) { return this.pushStack( winnow( this, selector || [], true ) ); }, is: function( selector ) { return !!winnow( this, // If this is a positional/relative selector, check membership in the returned set // so $("p:first").is("p:last") won't return true for a doc with two "p". typeof selector === "string" && rneedsContext.test( selector ) ? jQuery( selector ) : selector || [], false ).length; } } ); // Initialize a jQuery object // A central reference to the root jQuery(document) var rootjQuery, // A simple way to check for HTML strings // Prioritize #id over <tag> to avoid XSS via location.hash (#9521) // Strict HTML recognition (#11290: must start with <) // Shortcut simple #id case for speed rquickExpr = /^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]+))$/, init = jQuery.fn.init = function( selector, context, root ) { var match, elem; // HANDLE: $(""), $(null), $(undefined), $(false) if ( !selector ) { return this; } // Method init() accepts an alternate rootjQuery // so migrate can support jQuery.sub (gh-2101) root = root || rootjQuery; // Handle HTML strings if ( typeof selector === "string" ) { if ( selector[ 0 ] === "<" && selector[ selector.length - 1 ] === ">" && selector.length >= 3 ) { // Assume that strings that start and end with <> are HTML and skip the regex check match = [ null, selector, null ]; } else { match = rquickExpr.exec( selector ); } // Match html or make sure no context is specified for #id if ( match && ( match[ 1 ] || !context ) ) { // HANDLE: $(html) -> $(array) if ( match[ 1 ] ) { context = context instanceof jQuery ? context[ 0 ] : context; // Option to run scripts is true for back-compat // Intentionally let the error be thrown if parseHTML is not present jQuery.merge( this, jQuery.parseHTML( match[ 1 ], context && context.nodeType ? context.ownerDocument || context : document, true ) ); // HANDLE: $(html, props) if ( rsingleTag.test( match[ 1 ] ) && jQuery.isPlainObject( context ) ) { for ( match in context ) { // Properties of context are called as methods if possible if ( isFunction( this[ match ] ) ) { this[ match ]( context[ match ] ); // ...and otherwise set as attributes } else { this.attr( match, context[ match ] ); } } } return this; // HANDLE: $(#id) } else { elem = document.getElementById( match[ 2 ] ); if ( elem ) { // Inject the element directly into the jQuery object this[ 0 ] = elem; this.length = 1; } return this; } // HANDLE: $(expr, $(...)) } else if ( !context || context.jquery ) { return ( context || root ).find( selector ); // HANDLE: $(expr, context) // (which is just equivalent to: $(context).find(expr) } else { return this.constructor( context ).find( selector ); } // HANDLE: $(DOMElement) } else if ( selector.nodeType ) { this[ 0 ] = selector; this.length = 1; return this; // HANDLE: $(function) // Shortcut for document ready } else if ( isFunction( selector ) ) { return root.ready !== undefined ? root.ready( selector ) : // Execute immediately if ready is not present selector( jQuery ); } return jQuery.makeArray( selector, this ); }; // Give the init function the jQuery prototype for later instantiation init.prototype = jQuery.fn; // Initialize central reference rootjQuery = jQuery( document ); var rparentsprev = /^(?:parents|prev(?:Until|All))/, // Methods guaranteed to produce a unique set when starting from a unique set guaranteedUnique = { children: true, contents: true, next: true, prev: true }; jQuery.fn.extend( { has: function( target ) { var targets = jQuery( target, this ), l = targets.length; return this.filter( function() { var i = 0; for ( ; i < l; i++ ) { if ( jQuery.contains( this, targets[ i ] ) ) { return true; } } } ); }, closest: function( selectors, context ) { var cur, i = 0, l = this.length, matched = [], targets = typeof selectors !== "string" && jQuery( selectors ); // Positional selectors never match, since there's no _selection_ context if ( !rneedsContext.test( selectors ) ) { for ( ; i < l; i++ ) { for ( cur = this[ i ]; cur && cur !== context; cur = cur.parentNode ) { // Always skip document fragments if ( cur.nodeType < 11 && ( targets ? targets.index( cur ) > -1 : // Don't pass non-elements to Sizzle cur.nodeType === 1 && jQuery.find.matchesSelector( cur, selectors ) ) ) { matched.push( cur ); break; } } } } return this.pushStack( matched.length > 1 ? jQuery.uniqueSort( matched ) : matched ); }, // Determine the position of an element within the set index: function( elem ) { // No argument, return index in parent if ( !elem ) { return ( this[ 0 ] && this[ 0 ].parentNode ) ? this.first().prevAll().length : -1; } // Index in selector if ( typeof elem === "string" ) { return indexOf.call( jQuery( elem ), this[ 0 ] ); } // Locate the position of the desired element return indexOf.call( this, // If it receives a jQuery object, the first element is used elem.jquery ? elem[ 0 ] : elem ); }, add: function( selector, context ) { return this.pushStack( jQuery.uniqueSort( jQuery.merge( this.get(), jQuery( selector, context ) ) ) ); }, addBack: function( selector ) { return this.add( selector == null ? this.prevObject : this.prevObject.filter( selector ) ); } } ); function sibling( cur, dir ) { while ( ( cur = cur[ dir ] ) && cur.nodeType !== 1 ) {} return cur; } jQuery.each( { parent: function( elem ) { var parent = elem.parentNode; return parent && parent.nodeType !== 11 ? parent : null; }, parents: function( elem ) { return dir( elem, "parentNode" ); }, parentsUntil: function( elem, _i, until ) { return dir( elem, "parentNode", until ); }, next: function( elem ) { return sibling( elem, "nextSibling" ); }, prev: function( elem ) { return sibling( elem, "previousSibling" ); }, nextAll: function( elem ) { return dir( elem, "nextSibling" ); }, prevAll: function( elem ) { return dir( elem, "previousSibling" ); }, nextUntil: function( elem, _i, until ) { return dir( elem, "nextSibling", until ); }, prevUntil: function( elem, _i, until ) { return dir( elem, "previousSibling", until ); }, siblings: function( elem ) { return siblings( ( elem.parentNode || {} ).firstChild, elem ); }, children: function( elem ) { return siblings( elem.firstChild ); }, contents: function( elem ) { if ( elem.contentDocument != null && // Support: IE 11+ // <object> elements with no `data` attribute has an object // `contentDocument` with a `null` prototype. getProto( elem.contentDocument ) ) { return elem.contentDocument; } // Support: IE 9 - 11 only, iOS 7 only, Android Browser <=4.3 only // Treat the template element as a regular one in browsers that // don't support it. if ( nodeName( elem, "template" ) ) { elem = elem.content || elem; } return jQuery.merge( [], elem.childNodes ); } }, function( name, fn ) { jQuery.fn[ name ] = function( until, selector ) { var matched = jQuery.map( this, fn, until ); if ( name.slice( -5 ) !== "Until" ) { selector = until; } if ( selector && typeof selector === "string" ) { matched = jQuery.filter( selector, matched ); } if ( this.length > 1 ) { // Remove duplicates if ( !guaranteedUnique[ name ] ) { jQuery.uniqueSort( matched ); } // Reverse order for parents* and prev-derivatives if ( rparentsprev.test( name ) ) { matched.reverse(); } } return this.pushStack( matched ); }; } ); var rnothtmlwhite = ( /[^\x20\t\r\n\f]+/g ); // Convert String-formatted options into Object-formatted ones function createOptions( options ) { var object = {}; jQuery.each( options.match( rnothtmlwhite ) || [], function( _, flag ) { object[ flag ] = true; } ); return object; } /* * Create a callback list using the following parameters: * * options: an optional list of space-separated options that will change how * the callback list behaves or a more traditional option object * * By default a callback list will act like an event callback list and can be * "fired" multiple times. * * Possible options: * * once: will ensure the callback list can only be fired once (like a Deferred) * * memory: will keep track of previous values and will call any callback added * after the list has been fired right away with the latest "memorized" * values (like a Deferred) * * unique: will ensure a callback can only be added once (no duplicate in the list) * * stopOnFalse: interrupt callings when a callback returns false * */ jQuery.Callbacks = function( options ) { // Convert options from String-formatted to Object-formatted if needed // (we check in cache first) options = typeof options === "string" ? createOptions( options ) : jQuery.extend( {}, options ); var // Flag to know if list is currently firing firing, // Last fire value for non-forgettable lists memory, // Flag to know if list was already fired fired, // Flag to prevent firing locked, // Actual callback list list = [], // Queue of execution data for repeatable lists queue = [], // Index of currently firing callback (modified by add/remove as needed) firingIndex = -1, // Fire callbacks fire = function() { // Enforce single-firing locked = locked || options.once; // Execute callbacks for all pending executions, // respecting firingIndex overrides and runtime changes fired = firing = true; for ( ; queue.length; firingIndex = -1 ) { memory = queue.shift(); while ( ++firingIndex < list.length ) { // Run callback and check for early termination if ( list[ firingIndex ].apply( memory[ 0 ], memory[ 1 ] ) === false && options.stopOnFalse ) { // Jump to end and forget the data so .add doesn't re-fire firingIndex = list.length; memory = false; } } } // Forget the data if we're done with it if ( !options.memory ) { memory = false; } firing = false; // Clean up if we're done firing for good if ( locked ) { // Keep an empty list if we have data for future add calls if ( memory ) { list = []; // Otherwise, this object is spent } else { list = ""; } } }, // Actual Callbacks object self = { // Add a callback or a collection of callbacks to the list add: function() { if ( list ) { // If we have memory from a past run, we should fire after adding if ( memory && !firing ) { firingIndex = list.length - 1; queue.push( memory ); } ( function add( args ) { jQuery.each( args, function( _, arg ) { if ( isFunction( arg ) ) { if ( !options.unique || !self.has( arg ) ) { list.push( arg ); } } else if ( arg && arg.length && toType( arg ) !== "string" ) { // Inspect recursively add( arg ); } } ); } )( arguments ); if ( memory && !firing ) { fire(); } } return this; }, // Remove a callback from the list remove: function() { jQuery.each( arguments, function( _, arg ) { var index; while ( ( index = jQuery.inArray( arg, list, index ) ) > -1 ) { list.splice( index, 1 ); // Handle firing indexes if ( index <= firingIndex ) { firingIndex--; } } } ); return this; }, // Check if a given callback is in the list. // If no argument is given, return whether or not list has callbacks attached. has: function( fn ) { return fn ? jQuery.inArray( fn, list ) > -1 : list.length > 0; }, // Remove all callbacks from the list empty: function() { if ( list ) { list = []; } return this; }, // Disable .fire and .add // Abort any current/pending executions // Clear all callbacks and values disable: function() { locked = queue = []; list = memory = ""; return this; }, disabled: function() { return !list; }, // Disable .fire // Also disable .add unless we have memory (since it would have no effect) // Abort any pending executions lock: function() { locked = queue = []; if ( !memory && !firing ) { list = memory = ""; } return this; }, locked: function() { return !!locked; }, // Call all callbacks with the given context and arguments fireWith: function( context, args ) { if ( !locked ) { args = args || []; args = [ context, args.slice ? args.slice() : args ]; queue.push( args ); if ( !firing ) { fire(); } } return this; }, // Call all the callbacks with the given arguments fire: function() { self.fireWith( this, arguments ); return this; }, // To know if the callbacks have already been called at least once fired: function() { return !!fired; } }; return self; }; function Identity( v ) { return v; } function Thrower( ex ) { throw ex; } function adoptValue( value, resolve, reject, noValue ) { var method; try { // Check for promise aspect first to privilege synchronous behavior if ( value && isFunction( ( method = value.promise ) ) ) { method.call( value ).done( resolve ).fail( reject ); // Other thenables } else if ( value && isFunction( ( method = value.then ) ) ) { method.call( value, resolve, reject ); // Other non-thenables } else { // Control `resolve` arguments by letting Array#slice cast boolean `noValue` to integer: // * false: [ value ].slice( 0 ) => resolve( value ) // * true: [ value ].slice( 1 ) => resolve() resolve.apply( undefined, [ value ].slice( noValue ) ); } // For Promises/A+, convert exceptions into rejections // Since jQuery.when doesn't unwrap thenables, we can skip the extra checks appearing in // Deferred#then to conditionally suppress rejection. } catch ( value ) { // Support: Android 4.0 only // Strict mode functions invoked without .call/.apply get global-object context reject.apply( undefined, [ value ] ); } } jQuery.extend( { Deferred: function( func ) { var tuples = [ // action, add listener, callbacks, // ... .then handlers, argument index, [final state] [ "notify", "progress", jQuery.Callbacks( "memory" ), jQuery.Callbacks( "memory" ), 2 ], [ "resolve", "done", jQuery.Callbacks( "once memory" ), jQuery.Callbacks( "once memory" ), 0, "resolved" ], [ "reject", "fail", jQuery.Callbacks( "once memory" ), jQuery.Callbacks( "once memory" ), 1, "rejected" ] ], state = "pending", promise = { state: function() { return state; }, always: function() { deferred.done( arguments ).fail( arguments ); return this; }, "catch": function( fn ) { return promise.then( null, fn ); }, // Keep pipe for back-compat pipe: function( /* fnDone, fnFail, fnProgress */ ) { var fns = arguments; return jQuery.Deferred( function( newDefer ) { jQuery.each( tuples, function( _i, tuple ) { // Map tuples (progress, done, fail) to arguments (done, fail, progress) var fn = isFunction( fns[ tuple[ 4 ] ] ) && fns[ tuple[ 4 ] ]; // deferred.progress(function() { bind to newDefer or newDefer.notify }) // deferred.done(function() { bind to newDefer or newDefer.resolve }) // deferred.fail(function() { bind to newDefer or newDefer.reject }) deferred[ tuple[ 1 ] ]( function() { var returned = fn && fn.apply( this, arguments ); if ( returned && isFunction( returned.promise ) ) { returned.promise() .progress( newDefer.notify ) .done( newDefer.resolve ) .fail( newDefer.reject ); } else { newDefer[ tuple[ 0 ] + "With" ]( this, fn ? [ returned ] : arguments ); } } ); } ); fns = null; } ).promise(); }, then: function( onFulfilled, onRejected, onProgress ) { var maxDepth = 0; function resolve( depth, deferred, handler, special ) { return function() { var that = this, args = arguments, mightThrow = function() { var returned, then; // Support: Promises/A+ section 2.3.3.3.3 // https://promisesaplus.com/#point-59 // Ignore double-resolution attempts if ( depth < maxDepth ) { return; } returned = handler.apply( that, args ); // Support: Promises/A+ section 2.3.1 // https://promisesaplus.com/#point-48 if ( returned === deferred.promise() ) { throw new TypeError( "Thenable self-resolution" ); } // Support: Promises/A+ sections 2.3.3.1, 3.5 // https://promisesaplus.com/#point-54 // https://promisesaplus.com/#point-75 // Retrieve `then` only once then = returned && // Support: Promises/A+ section 2.3.4 // https://promisesaplus.com/#point-64 // Only check objects and functions for thenability ( typeof returned === "object" || typeof returned === "function" ) && returned.then; // Handle a returned thenable if ( isFunction( then ) ) { // Special processors (notify) just wait for resolution if ( special ) { then.call( returned, resolve( maxDepth, deferred, Identity, special ), resolve( maxDepth, deferred, Thrower, special ) ); // Normal processors (resolve) also hook into progress } else { // ...and disregard older resolution values maxDepth++; then.call( returned, resolve( maxDepth, deferred, Identity, special ), resolve( maxDepth, deferred, Thrower, special ), resolve( maxDepth, deferred, Identity, deferred.notifyWith ) ); } // Handle all other returned values } else { // Only substitute handlers pass on context // and multiple values (non-spec behavior) if ( handler !== Identity ) { that = undefined; args = [ returned ]; } // Process the value(s) // Default process is resolve ( special || deferred.resolveWith )( that, args ); } }, // Only normal processors (resolve) catch and reject exceptions process = special ? mightThrow : function() { try { mightThrow(); } catch ( e ) { if ( jQuery.Deferred.exceptionHook ) { jQuery.Deferred.exceptionHook( e, process.stackTrace ); } // Support: Promises/A+ section 2.3.3.3.4.1 // https://promisesaplus.com/#point-61 // Ignore post-resolution exceptions if ( depth + 1 >= maxDepth ) { // Only substitute handlers pass on context // and multiple values (non-spec behavior) if ( handler !== Thrower ) { that = undefined; args = [ e ]; } deferred.rejectWith( that, args ); } } }; // Support: Promises/A+ section 2.3.3.3.1 // https://promisesaplus.com/#point-57 // Re-resolve promises immediately to dodge false rejection from // subsequent errors if ( depth ) { process(); } else { // Call an optional hook to record the stack, in case of exception // since it's otherwise lost when execution goes async if ( jQuery.Deferred.getStackHook ) { process.stackTrace = jQuery.Deferred.getStackHook(); } window.setTimeout( process ); } }; } return jQuery.Deferred( function( newDefer ) { // progress_handlers.add( ... ) tuples[ 0 ][ 3 ].add( resolve( 0, newDefer, isFunction( onProgress ) ? onProgress : Identity, newDefer.notifyWith ) ); // fulfilled_handlers.add( ... ) tuples[ 1 ][ 3 ].add( resolve( 0, newDefer, isFunction( onFulfilled ) ? onFulfilled : Identity ) ); // rejected_handlers.add( ... ) tuples[ 2 ][ 3 ].add( resolve( 0, newDefer, isFunction( onRejected ) ? onRejected : Thrower ) ); } ).promise(); }, // Get a promise for this deferred // If obj is provided, the promise aspect is added to the object promise: function( obj ) { return obj != null ? jQuery.extend( obj, promise ) : promise; } }, deferred = {}; // Add list-specific methods jQuery.each( tuples, function( i, tuple ) { var list = tuple[ 2 ], stateString = tuple[ 5 ]; // promise.progress = list.add // promise.done = list.add // promise.fail = list.add promise[ tuple[ 1 ] ] = list.add; // Handle state if ( stateString ) { list.add( function() { // state = "resolved" (i.e., fulfilled) // state = "rejected" state = stateString; }, // rejected_callbacks.disable // fulfilled_callbacks.disable tuples[ 3 - i ][ 2 ].disable, // rejected_handlers.disable // fulfilled_handlers.disable tuples[ 3 - i ][ 3 ].disable, // progress_callbacks.lock tuples[ 0 ][ 2 ].lock, // progress_handlers.lock tuples[ 0 ][ 3 ].lock ); } // progress_handlers.fire // fulfilled_handlers.fire // rejected_handlers.fire list.add( tuple[ 3 ].fire ); // deferred.notify = function() { deferred.notifyWith(...) } // deferred.resolve = function() { deferred.resolveWith(...) } // deferred.reject = function() { deferred.rejectWith(...) } deferred[ tuple[ 0 ] ] = function() { deferred[ tuple[ 0 ] + "With" ]( this === deferred ? undefined : this, arguments ); return this; }; // deferred.notifyWith = list.fireWith // deferred.resolveWith = list.fireWith // deferred.rejectWith = list.fireWith deferred[ tuple[ 0 ] + "With" ] = list.fireWith; } ); // Make the deferred a promise promise.promise( deferred ); // Call given func if any if ( func ) { func.call( deferred, deferred ); } // All done! return deferred; }, // Deferred helper when: function( singleValue ) { var // count of uncompleted subordinates remaining = arguments.length, // count of unprocessed arguments i = remaining, // subordinate fulfillment data resolveContexts = Array( i ), resolveValues = slice.call( arguments ), // the primary Deferred primary = jQuery.Deferred(), // subordinate callback factory updateFunc = function( i ) { return function( value ) { resolveContexts[ i ] = this; resolveValues[ i ] = arguments.length > 1 ? slice.call( arguments ) : value; if ( !( --remaining ) ) { primary.resolveWith( resolveContexts, resolveValues ); } }; }; // Single- and empty arguments are adopted like Promise.resolve if ( remaining <= 1 ) { adoptValue( singleValue, primary.done( updateFunc( i ) ).resolve, primary.reject, !remaining ); // Use .then() to unwrap secondary thenables (cf. gh-3000) if ( primary.state() === "pending" || isFunction( resolveValues[ i ] && resolveValues[ i ].then ) ) { return primary.then(); } } // Multiple arguments are aggregated like Promise.all array elements while ( i-- ) { adoptValue( resolveValues[ i ], updateFunc( i ), primary.reject ); } return primary.promise(); } } ); // These usually indicate a programmer mistake during development, // warn about them ASAP rather than swallowing them by default. var rerrorNames = /^(Eval|Internal|Range|Reference|Syntax|Type|URI)Error$/; jQuery.Deferred.exceptionHook = function( error, stack ) { // Support: IE 8 - 9 only // Console exists when dev tools are open, which can happen at any time if ( window.console && window.console.warn && error && rerrorNames.test( error.name ) ) { window.console.warn( "jQuery.Deferred exception: " + error.message, error.stack, stack ); } }; jQuery.readyException = function( error ) { window.setTimeout( function() { throw error; } ); }; // The deferred used on DOM ready var readyList = jQuery.Deferred(); jQuery.fn.ready = function( fn ) { readyList .then( fn ) // Wrap jQuery.readyException in a function so that the lookup // happens at the time of error handling instead of callback // registration. .catch( function( error ) { jQuery.readyException( error ); } ); return this; }; jQuery.extend( { // Is the DOM ready to be used? Set to true once it occurs. isReady: false, // A counter to track how many items to wait for before // the ready event fires. See #6781 readyWait: 1, // Handle when the DOM is ready ready: function( wait ) { // Abort if there are pending holds or we're already ready if ( wait === true ? --jQuery.readyWait : jQuery.isReady ) { return; } // Remember that the DOM is ready jQuery.isReady = true; // If a normal DOM Ready event fired, decrement, and wait if need be if ( wait !== true && --jQuery.readyWait > 0 ) { return; } // If there are functions bound, to execute readyList.resolveWith( document, [ jQuery ] ); } } ); jQuery.ready.then = readyList.then; // The ready event handler and self cleanup method function completed() { document.removeEventListener( "DOMContentLoaded", completed ); window.removeEventListener( "load", completed ); jQuery.ready(); } // Catch cases where $(document).ready() is called // after the browser event has already occurred. // Support: IE <=9 - 10 only // Older IE sometimes signals "interactive" too soon if ( document.readyState === "complete" || ( document.readyState !== "loading" && !document.documentElement.doScroll ) ) { // Handle it asynchronously to allow scripts the opportunity to delay ready window.setTimeout( jQuery.ready ); } else { // Use the handy event callback document.addEventListener( "DOMContentLoaded", completed ); // A fallback to window.onload, that will always work window.addEventListener( "load", completed ); } // Multifunctional method to get and set values of a collection // The value/s can optionally be executed if it's a function var access = function( elems, fn, key, value, chainable, emptyGet, raw ) { var i = 0, len = elems.length, bulk = key == null; // Sets many values if ( toType( key ) === "object" ) { chainable = true; for ( i in key ) { access( elems, fn, i, key[ i ], true, emptyGet, raw ); } // Sets one value } else if ( value !== undefined ) { chainable = true; if ( !isFunction( value ) ) { raw = true; } if ( bulk ) { // Bulk operations run against the entire set if ( raw ) { fn.call( elems, value ); fn = null; // ...except when executing function values } else { bulk = fn; fn = function( elem, _key, value ) { return bulk.call( jQuery( elem ), value ); }; } } if ( fn ) { for ( ; i < len; i++ ) { fn( elems[ i ], key, raw ? value : value.call( elems[ i ], i, fn( elems[ i ], key ) ) ); } } } if ( chainable ) { return elems; } // Gets if ( bulk ) { return fn.call( elems ); } return len ? fn( elems[ 0 ], key ) : emptyGet; }; // Matches dashed string for camelizing var rmsPrefix = /^-ms-/, rdashAlpha = /-([a-z])/g; // Used by camelCase as callback to replace() function fcamelCase( _all, letter ) { return letter.toUpperCase(); } // Convert dashed to camelCase; used by the css and data modules // Support: IE <=9 - 11, Edge 12 - 15 // Microsoft forgot to hump their vendor prefix (#9572) function camelCase( string ) { return string.replace( rmsPrefix, "ms-" ).replace( rdashAlpha, fcamelCase ); } var acceptData = function( owner ) { // Accepts only: // - Node // - Node.ELEMENT_NODE // - Node.DOCUMENT_NODE // - Object // - Any return owner.nodeType === 1 || owner.nodeType === 9 || !( +owner.nodeType ); }; function Data() { this.expando = jQuery.expando + Data.uid++; } Data.uid = 1; Data.prototype = { cache: function( owner ) { // Check if the owner object already has a cache var value = owner[ this.expando ]; // If not, create one if ( !value ) { value = {}; // We can accept data for non-element nodes in modern browsers, // but we should not, see #8335. // Always return an empty object. if ( acceptData( owner ) ) { // If it is a node unlikely to be stringify-ed or looped over // use plain assignment if ( owner.nodeType ) { owner[ this.expando ] = value; // Otherwise secure it in a non-enumerable property // configurable must be true to allow the property to be // deleted when data is removed } else { Object.defineProperty( owner, this.expando, { value: value, configurable: true } ); } } } return value; }, set: function( owner, data, value ) { var prop, cache = this.cache( owner ); // Handle: [ owner, key, value ] args // Always use camelCase key (gh-2257) if ( typeof data === "string" ) { cache[ camelCase( data ) ] = value; // Handle: [ owner, { properties } ] args } else { // Copy the properties one-by-one to the cache object for ( prop in data ) { cache[ camelCase( prop ) ] = data[ prop ]; } } return cache; }, get: function( owner, key ) { return key === undefined ? this.cache( owner ) : // Always use camelCase key (gh-2257) owner[ this.expando ] && owner[ this.expando ][ camelCase( key ) ]; }, access: function( owner, key, value ) { // In cases where either: // // 1. No key was specified // 2. A string key was specified, but no value provided // // Take the "read" path and allow the get method to determine // which value to return, respectively either: // // 1. The entire cache object // 2. The data stored at the key // if ( key === undefined || ( ( key && typeof key === "string" ) && value === undefined ) ) { return this.get( owner, key ); } // When the key is not a string, or both a key and value // are specified, set or extend (existing objects) with either: // // 1. An object of properties // 2. A key and value // this.set( owner, key, value ); // Since the "set" path can have two possible entry points // return the expected data based on which path was taken[*] return value !== undefined ? value : key; }, remove: function( owner, key ) { var i, cache = owner[ this.expando ]; if ( cache === undefined ) { return; } if ( key !== undefined ) { // Support array or space separated string of keys if ( Array.isArray( key ) ) { // If key is an array of keys... // We always set camelCase keys, so remove that. key = key.map( camelCase ); } else { key = camelCase( key ); // If a key with the spaces exists, use it. // Otherwise, create an array by matching non-whitespace key = key in cache ? [ key ] : ( key.match( rnothtmlwhite ) || [] ); } i = key.length; while ( i-- ) { delete cache[ key[ i ] ]; } } // Remove the expando if there's no more data if ( key === undefined || jQuery.isEmptyObject( cache ) ) { // Support: Chrome <=35 - 45 // Webkit & Blink performance suffers when deleting properties // from DOM nodes, so set to undefined instead // https://bugs.chromium.org/p/chromium/issues/detail?id=378607 (bug restricted) if ( owner.nodeType ) { owner[ this.expando ] = undefined; } else { delete owner[ this.expando ]; } } }, hasData: function( owner ) { var cache = owner[ this.expando ]; return cache !== undefined && !jQuery.isEmptyObject( cache ); } }; var dataPriv = new Data(); var dataUser = new Data(); // Implementation Summary // // 1. Enforce API surface and semantic compatibility with 1.9.x branch // 2. Improve the module's maintainability by reducing the storage // paths to a single mechanism. // 3. Use the same single mechanism to support "private" and "user" data. // 4. _Never_ expose "private" data to user code (TODO: Drop _data, _removeData) // 5. Avoid exposing implementation details on user objects (eg. expando properties) // 6. Provide a clear path for implementation upgrade to WeakMap in 2014 var rbrace = /^(?:\{[\w\W]*\}|\[[\w\W]*\])$/, rmultiDash = /[A-Z]/g; function getData( data ) { if ( data === "true" ) { return true; } if ( data === "false" ) { return false; } if ( data === "null" ) { return null; } // Only convert to a number if it doesn't change the string if ( data === +data + "" ) { return +data; } if ( rbrace.test( data ) ) { return JSON.parse( data ); } return data; } function dataAttr( elem, key, data ) { var name; // If nothing was found internally, try to fetch any // data from the HTML5 data-* attribute if ( data === undefined && elem.nodeType === 1 ) { name = "data-" + key.replace( rmultiDash, "-$&" ).toLowerCase(); data = elem.getAttribute( name ); if ( typeof data === "string" ) { try { data = getData( data ); } catch ( e ) {} // Make sure we set the data so it isn't changed later dataUser.set( elem, key, data ); } else { data = undefined; } } return data; } jQuery.extend( { hasData: function( elem ) { return dataUser.hasData( elem ) || dataPriv.hasData( elem ); }, data: function( elem, name, data ) { return dataUser.access( elem, name, data ); }, removeData: function( elem, name ) { dataUser.remove( elem, name ); }, // TODO: Now that all calls to _data and _removeData have been replaced // with direct calls to dataPriv methods, these can be deprecated. _data: function( elem, name, data ) { return dataPriv.access( elem, name, data ); }, _removeData: function( elem, name ) { dataPriv.remove( elem, name ); } } ); jQuery.fn.extend( { data: function( key, value ) { var i, name, data, elem = this[ 0 ], attrs = elem && elem.attributes; // Gets all values if ( key === undefined ) { if ( this.length ) { data = dataUser.get( elem ); if ( elem.nodeType === 1 && !dataPriv.get( elem, "hasDataAttrs" ) ) { i = attrs.length; while ( i-- ) { // Support: IE 11 only // The attrs elements can be null (#14894) if ( attrs[ i ] ) { name = attrs[ i ].name; if ( name.indexOf( "data-" ) === 0 ) { name = camelCase( name.slice( 5 ) ); dataAttr( elem, name, data[ name ] ); } } } dataPriv.set( elem, "hasDataAttrs", true ); } } return data; } // Sets multiple values if ( typeof key === "object" ) { return this.each( function() { dataUser.set( this, key ); } ); } return access( this, function( value ) { var data; // The calling jQuery object (element matches) is not empty // (and therefore has an element appears at this[ 0 ]) and the // `value` parameter was not undefined. An empty jQuery object // will result in `undefined` for elem = this[ 0 ] which will // throw an exception if an attempt to read a data cache is made. if ( elem && value === undefined ) { // Attempt to get data from the cache // The key will always be camelCased in Data data = dataUser.get( elem, key ); if ( data !== undefined ) { return data; } // Attempt to "discover" the data in // HTML5 custom data-* attrs data = dataAttr( elem, key ); if ( data !== undefined ) { return data; } // We tried really hard, but the data doesn't exist. return; } // Set the data... this.each( function() { // We always store the camelCased key dataUser.set( this, key, value ); } ); }, null, value, arguments.length > 1, null, true ); }, removeData: function( key ) { return this.each( function() { dataUser.remove( this, key ); } ); } } ); jQuery.extend( { queue: function( elem, type, data ) { var queue; if ( elem ) { type = ( type || "fx" ) + "queue"; queue = dataPriv.get( elem, type ); // Speed up dequeue by getting out quickly if this is just a lookup if ( data ) { if ( !queue || Array.isArray( data ) ) { queue = dataPriv.access( elem, type, jQuery.makeArray( data ) ); } else { queue.push( data ); } } return queue || []; } }, dequeue: function( elem, type ) { type = type || "fx"; var queue = jQuery.queue( elem, type ), startLength = queue.length, fn = queue.shift(), hooks = jQuery._queueHooks( elem, type ), next = function() { jQuery.dequeue( elem, type ); }; // If the fx queue is dequeued, always remove the progress sentinel if ( fn === "inprogress" ) { fn = queue.shift(); startLength--; } if ( fn ) { // Add a progress sentinel to prevent the fx queue from being // automatically dequeued if ( type === "fx" ) { queue.unshift( "inprogress" ); } // Clear up the last queue stop function delete hooks.stop; fn.call( elem, next, hooks ); } if ( !startLength && hooks ) { hooks.empty.fire(); } }, // Not public - generate a queueHooks object, or return the current one _queueHooks: function( elem, type ) { var key = type + "queueHooks"; return dataPriv.get( elem, key ) || dataPriv.access( elem, key, { empty: jQuery.Callbacks( "once memory" ).add( function() { dataPriv.remove( elem, [ type + "queue", key ] ); } ) } ); } } ); jQuery.fn.extend( { queue: function( type, data ) { var setter = 2; if ( typeof type !== "string" ) { data = type; type = "fx"; setter--; } if ( arguments.length < setter ) { return jQuery.queue( this[ 0 ], type ); } return data === undefined ? this : this.each( function() { var queue = jQuery.queue( this, type, data ); // Ensure a hooks for this queue jQuery._queueHooks( this, type ); if ( type === "fx" && queue[ 0 ] !== "inprogress" ) { jQuery.dequeue( this, type ); } } ); }, dequeue: function( type ) { return this.each( function() { jQuery.dequeue( this, type ); } ); }, clearQueue: function( type ) { return this.queue( type || "fx", [] ); }, // Get a promise resolved when queues of a certain type // are emptied (fx is the type by default) promise: function( type, obj ) { var tmp, count = 1, defer = jQuery.Deferred(), elements = this, i = this.length, resolve = function() { if ( !( --count ) ) { defer.resolveWith( elements, [ elements ] ); } }; if ( typeof type !== "string" ) { obj = type; type = undefined; } type = type || "fx"; while ( i-- ) { tmp = dataPriv.get( elements[ i ], type + "queueHooks" ); if ( tmp && tmp.empty ) { count++; tmp.empty.add( resolve ); } } resolve(); return defer.promise( obj ); } } ); var pnum = ( /[+-]?(?:\d*\.|)\d+(?:[eE][+-]?\d+|)/ ).source; var rcssNum = new RegExp( "^(?:([+-])=|)(" + pnum + ")([a-z%]*)$", "i" ); var cssExpand = [ "Top", "Right", "Bottom", "Left" ]; var documentElement = document.documentElement; var isAttached = function( elem ) { return jQuery.contains( elem.ownerDocument, elem ); }, composed = { composed: true }; // Support: IE 9 - 11+, Edge 12 - 18+, iOS 10.0 - 10.2 only // Check attachment across shadow DOM boundaries when possible (gh-3504) // Support: iOS 10.0-10.2 only // Early iOS 10 versions support `attachShadow` but not `getRootNode`, // leading to errors. We need to check for `getRootNode`. if ( documentElement.getRootNode ) { isAttached = function( elem ) { return jQuery.contains( elem.ownerDocument, elem ) || elem.getRootNode( composed ) === elem.ownerDocument; }; } var isHiddenWithinTree = function( elem, el ) { // isHiddenWithinTree might be called from jQuery#filter function; // in that case, element will be second argument elem = el || elem; // Inline style trumps all return elem.style.display === "none" || elem.style.display === "" && // Otherwise, check computed style // Support: Firefox <=43 - 45 // Disconnected elements can have computed display: none, so first confirm that elem is // in the document. isAttached( elem ) && jQuery.css( elem, "display" ) === "none"; }; function adjustCSS( elem, prop, valueParts, tween ) { var adjusted, scale, maxIterations = 20, currentValue = tween ? function() { return tween.cur(); } : function() { return jQuery.css( elem, prop, "" ); }, initial = currentValue(), unit = valueParts && valueParts[ 3 ] || ( jQuery.cssNumber[ prop ] ? "" : "px" ), // Starting value computation is required for potential unit mismatches initialInUnit = elem.nodeType && ( jQuery.cssNumber[ prop ] || unit !== "px" && +initial ) && rcssNum.exec( jQuery.css( elem, prop ) ); if ( initialInUnit && initialInUnit[ 3 ] !== unit ) { // Support: Firefox <=54 // Halve the iteration target value to prevent interference from CSS upper bounds (gh-2144) initial = initial / 2; // Trust units reported by jQuery.css unit = unit || initialInUnit[ 3 ]; // Iteratively approximate from a nonzero starting point initialInUnit = +initial || 1; while ( maxIterations-- ) { // Evaluate and update our best guess (doubling guesses that zero out). // Finish if the scale equals or crosses 1 (making the old*new product non-positive). jQuery.style( elem, prop, initialInUnit + unit ); if ( ( 1 - scale ) * ( 1 - ( scale = currentValue() / initial || 0.5 ) ) <= 0 ) { maxIterations = 0; } initialInUnit = initialInUnit / scale; } initialInUnit = initialInUnit * 2; jQuery.style( elem, prop, initialInUnit + unit ); // Make sure we update the tween properties later on valueParts = valueParts || []; } if ( valueParts ) { initialInUnit = +initialInUnit || +initial || 0; // Apply relative offset (+=/-=) if specified adjusted = valueParts[ 1 ] ? initialInUnit + ( valueParts[ 1 ] + 1 ) * valueParts[ 2 ] : +valueParts[ 2 ]; if ( tween ) { tween.unit = unit; tween.start = initialInUnit; tween.end = adjusted; } } return adjusted; } var defaultDisplayMap = {}; function getDefaultDisplay( elem ) { var temp, doc = elem.ownerDocument, nodeName = elem.nodeName, display = defaultDisplayMap[ nodeName ]; if ( display ) { return display; } temp = doc.body.appendChild( doc.createElement( nodeName ) ); display = jQuery.css( temp, "display" ); temp.parentNode.removeChild( temp ); if ( display === "none" ) { display = "block"; } defaultDisplayMap[ nodeName ] = display; return display; } function showHide( elements, show ) { var display, elem, values = [], index = 0, length = elements.length; // Determine new display value for elements that need to change for ( ; index < length; index++ ) { elem = elements[ index ]; if ( !elem.style ) { continue; } display = elem.style.display; if ( show ) { // Since we force visibility upon cascade-hidden elements, an immediate (and slow) // check is required in this first loop unless we have a nonempty display value (either // inline or about-to-be-restored) if ( display === "none" ) { values[ index ] = dataPriv.get( elem, "display" ) || null; if ( !values[ index ] ) { elem.style.display = ""; } } if ( elem.style.display === "" && isHiddenWithinTree( elem ) ) { values[ index ] = getDefaultDisplay( elem ); } } else { if ( display !== "none" ) { values[ index ] = "none"; // Remember what we're overwriting dataPriv.set( elem, "display", display ); } } } // Set the display of the elements in a second loop to avoid constant reflow for ( index = 0; index < length; index++ ) { if ( values[ index ] != null ) { elements[ index ].style.display = values[ index ]; } } return elements; } jQuery.fn.extend( { show: function() { return showHide( this, true ); }, hide: function() { return showHide( this ); }, toggle: function( state ) { if ( typeof state === "boolean" ) { return state ? this.show() : this.hide(); } return this.each( function() { if ( isHiddenWithinTree( this ) ) { jQuery( this ).show(); } else { jQuery( this ).hide(); } } ); } } ); var rcheckableType = ( /^(?:checkbox|radio)$/i ); var rtagName = ( /<([a-z][^\/\0>\x20\t\r\n\f]*)/i ); var rscriptType = ( /^$|^module$|\/(?:java|ecma)script/i ); ( function() { var fragment = document.createDocumentFragment(), div = fragment.appendChild( document.createElement( "div" ) ), input = document.createElement( "input" ); // Support: Android 4.0 - 4.3 only // Check state lost if the name is set (#11217) // Support: Windows Web Apps (WWA) // `name` and `type` must use .setAttribute for WWA (#14901) input.setAttribute( "type", "radio" ); input.setAttribute( "checked", "checked" ); input.setAttribute( "name", "t" ); div.appendChild( input ); // Support: Android <=4.1 only // Older WebKit doesn't clone checked state correctly in fragments support.checkClone = div.cloneNode( true ).cloneNode( true ).lastChild.checked; // Support: IE <=11 only // Make sure textarea (and checkbox) defaultValue is properly cloned div.innerHTML = "<textarea>x</textarea>"; support.noCloneChecked = !!div.cloneNode( true ).lastChild.defaultValue; // Support: IE <=9 only // IE <=9 replaces <option> tags with their contents when inserted outside of // the select element. div.innerHTML = "<option></option>"; support.option = !!div.lastChild; } )(); // We have to close these tags to support XHTML (#13200) var wrapMap = { // XHTML parsers do not magically insert elements in the // same way that tag soup parsers do. So we cannot shorten // this by omitting <tbody> or other required elements. thead: [ 1, "<table>", "</table>" ], col: [ 2, "<table><colgroup>", "</colgroup></table>" ], tr: [ 2, "<table><tbody>", "</tbody></table>" ], td: [ 3, "<table><tbody><tr>", "</tr></tbody></table>" ], _default: [ 0, "", "" ] }; wrapMap.tbody = wrapMap.tfoot = wrapMap.colgroup = wrapMap.caption = wrapMap.thead; wrapMap.th = wrapMap.td; // Support: IE <=9 only if ( !support.option ) { wrapMap.optgroup = wrapMap.option = [ 1, "<select multiple='multiple'>", "</select>" ]; } function getAll( context, tag ) { // Support: IE <=9 - 11 only // Use typeof to avoid zero-argument method invocation on host objects (#15151) var ret; if ( typeof context.getElementsByTagName !== "undefined" ) { ret = context.getElementsByTagName( tag || "*" ); } else if ( typeof context.querySelectorAll !== "undefined" ) { ret = context.querySelectorAll( tag || "*" ); } else { ret = []; } if ( tag === undefined || tag && nodeName( context, tag ) ) { return jQuery.merge( [ context ], ret ); } return ret; } // Mark scripts as having already been evaluated function setGlobalEval( elems, refElements ) { var i = 0, l = elems.length; for ( ; i < l; i++ ) { dataPriv.set( elems[ i ], "globalEval", !refElements || dataPriv.get( refElements[ i ], "globalEval" ) ); } } var rhtml = /<|&#?\w+;/; function buildFragment( elems, context, scripts, selection, ignored ) { var elem, tmp, tag, wrap, attached, j, fragment = context.createDocumentFragment(), nodes = [], i = 0, l = elems.length; for ( ; i < l; i++ ) { elem = elems[ i ]; if ( elem || elem === 0 ) { // Add nodes directly if ( toType( elem ) === "object" ) { // Support: Android <=4.0 only, PhantomJS 1 only // push.apply(_, arraylike) throws on ancient WebKit jQuery.merge( nodes, elem.nodeType ? [ elem ] : elem ); // Convert non-html into a text node } else if ( !rhtml.test( elem ) ) { nodes.push( context.createTextNode( elem ) ); // Convert html into DOM nodes } else { tmp = tmp || fragment.appendChild( context.createElement( "div" ) ); // Deserialize a standard representation tag = ( rtagName.exec( elem ) || [ "", "" ] )[ 1 ].toLowerCase(); wrap = wrapMap[ tag ] || wrapMap._default; tmp.innerHTML = wrap[ 1 ] + jQuery.htmlPrefilter( elem ) + wrap[ 2 ]; // Descend through wrappers to the right content j = wrap[ 0 ]; while ( j-- ) { tmp = tmp.lastChild; } // Support: Android <=4.0 only, PhantomJS 1 only // push.apply(_, arraylike) throws on ancient WebKit jQuery.merge( nodes, tmp.childNodes ); // Remember the top-level container tmp = fragment.firstChild; // Ensure the created nodes are orphaned (#12392) tmp.textContent = ""; } } } // Remove wrapper from fragment fragment.textContent = ""; i = 0; while ( ( elem = nodes[ i++ ] ) ) { // Skip elements already in the context collection (trac-4087) if ( selection && jQuery.inArray( elem, selection ) > -1 ) { if ( ignored ) { ignored.push( elem ); } continue; } attached = isAttached( elem ); // Append to fragment tmp = getAll( fragment.appendChild( elem ), "script" ); // Preserve script evaluation history if ( attached ) { setGlobalEval( tmp ); } // Capture executables if ( scripts ) { j = 0; while ( ( elem = tmp[ j++ ] ) ) { if ( rscriptType.test( elem.type || "" ) ) { scripts.push( elem ); } } } } return fragment; } var rtypenamespace = /^([^.]*)(?:\.(.+)|)/; function returnTrue() { return true; } function returnFalse() { return false; } // Support: IE <=9 - 11+ // focus() and blur() are asynchronous, except when they are no-op. // So expect focus to be synchronous when the element is already active, // and blur to be synchronous when the element is not already active. // (focus and blur are always synchronous in other supported browsers, // this just defines when we can count on it). function expectSync( elem, type ) { return ( elem === safeActiveElement() ) === ( type === "focus" ); } // Support: IE <=9 only // Accessing document.activeElement can throw unexpectedly // https://bugs.jquery.com/ticket/13393 function safeActiveElement() { try { return document.activeElement; } catch ( err ) { } } function on( elem, types, selector, data, fn, one ) { var origFn, type; // Types can be a map of types/handlers if ( typeof types === "object" ) { // ( types-Object, selector, data ) if ( typeof selector !== "string" ) { // ( types-Object, data ) data = data || selector; selector = undefined; } for ( type in types ) { on( elem, type, selector, data, types[ type ], one ); } return elem; } if ( data == null && fn == null ) { // ( types, fn ) fn = selector; data = selector = undefined; } else if ( fn == null ) { if ( typeof selector === "string" ) { // ( types, selector, fn ) fn = data; data = undefined; } else { // ( types, data, fn ) fn = data; data = selector; selector = undefined; } } if ( fn === false ) { fn = returnFalse; } else if ( !fn ) { return elem; } if ( one === 1 ) { origFn = fn; fn = function( event ) { // Can use an empty set, since event contains the info jQuery().off( event ); return origFn.apply( this, arguments ); }; // Use same guid so caller can remove using origFn fn.guid = origFn.guid || ( origFn.guid = jQuery.guid++ ); } return elem.each( function() { jQuery.event.add( this, types, fn, data, selector ); } ); } /* * Helper functions for managing events -- not part of the public interface. * Props to Dean Edwards' addEvent library for many of the ideas. */ jQuery.event = { global: {}, add: function( elem, types, handler, data, selector ) { var handleObjIn, eventHandle, tmp, events, t, handleObj, special, handlers, type, namespaces, origType, elemData = dataPriv.get( elem ); // Only attach events to objects that accept data if ( !acceptData( elem ) ) { return; } // Caller can pass in an object of custom data in lieu of the handler if ( handler.handler ) { handleObjIn = handler; handler = handleObjIn.handler; selector = handleObjIn.selector; } // Ensure that invalid selectors throw exceptions at attach time // Evaluate against documentElement in case elem is a non-element node (e.g., document) if ( selector ) { jQuery.find.matchesSelector( documentElement, selector ); } // Make sure that the handler has a unique ID, used to find/remove it later if ( !handler.guid ) { handler.guid = jQuery.guid++; } // Init the element's event structure and main handler, if this is the first if ( !( events = elemData.events ) ) { events = elemData.events = Object.create( null ); } if ( !( eventHandle = elemData.handle ) ) { eventHandle = elemData.handle = function( e ) { // Discard the second event of a jQuery.event.trigger() and // when an event is called after a page has unloaded return typeof jQuery !== "undefined" && jQuery.event.triggered !== e.type ? jQuery.event.dispatch.apply( elem, arguments ) : undefined; }; } // Handle multiple events separated by a space types = ( types || "" ).match( rnothtmlwhite ) || [ "" ]; t = types.length; while ( t-- ) { tmp = rtypenamespace.exec( types[ t ] ) || []; type = origType = tmp[ 1 ]; namespaces = ( tmp[ 2 ] || "" ).split( "." ).sort(); // There *must* be a type, no attaching namespace-only handlers if ( !type ) { continue; } // If event changes its type, use the special event handlers for the changed type special = jQuery.event.special[ type ] || {}; // If selector defined, determine special event api type, otherwise given type type = ( selector ? special.delegateType : special.bindType ) || type; // Update special based on newly reset type special = jQuery.event.special[ type ] || {}; // handleObj is passed to all event handlers handleObj = jQuery.extend( { type: type, origType: origType, data: data, handler: handler, guid: handler.guid, selector: selector, needsContext: selector && jQuery.expr.match.needsContext.test( selector ), namespace: namespaces.join( "." ) }, handleObjIn ); // Init the event handler queue if we're the first if ( !( handlers = events[ type ] ) ) { handlers = events[ type ] = []; handlers.delegateCount = 0; // Only use addEventListener if the special events handler returns false if ( !special.setup || special.setup.call( elem, data, namespaces, eventHandle ) === false ) { if ( elem.addEventListener ) { elem.addEventListener( type, eventHandle ); } } } if ( special.add ) { special.add.call( elem, handleObj ); if ( !handleObj.handler.guid ) { handleObj.handler.guid = handler.guid; } } // Add to the element's handler list, delegates in front if ( selector ) { handlers.splice( handlers.delegateCount++, 0, handleObj ); } else { handlers.push( handleObj ); } // Keep track of which events have ever been used, for event optimization jQuery.event.global[ type ] = true; } }, // Detach an event or set of events from an element remove: function( elem, types, handler, selector, mappedTypes ) { var j, origCount, tmp, events, t, handleObj, special, handlers, type, namespaces, origType, elemData = dataPriv.hasData( elem ) && dataPriv.get( elem ); if ( !elemData || !( events = elemData.events ) ) { return; } // Once for each type.namespace in types; type may be omitted types = ( types || "" ).match( rnothtmlwhite ) || [ "" ]; t = types.length; while ( t-- ) { tmp = rtypenamespace.exec( types[ t ] ) || []; type = origType = tmp[ 1 ]; namespaces = ( tmp[ 2 ] || "" ).split( "." ).sort(); // Unbind all events (on this namespace, if provided) for the element if ( !type ) { for ( type in events ) { jQuery.event.remove( elem, type + types[ t ], handler, selector, true ); } continue; } special = jQuery.event.special[ type ] || {}; type = ( selector ? special.delegateType : special.bindType ) || type; handlers = events[ type ] || []; tmp = tmp[ 2 ] && new RegExp( "(^|\\.)" + namespaces.join( "\\.(?:.*\\.|)" ) + "(\\.|$)" ); // Remove matching events origCount = j = handlers.length; while ( j-- ) { handleObj = handlers[ j ]; if ( ( mappedTypes || origType === handleObj.origType ) && ( !handler || handler.guid === handleObj.guid ) && ( !tmp || tmp.test( handleObj.namespace ) ) && ( !selector || selector === handleObj.selector || selector === "**" && handleObj.selector ) ) { handlers.splice( j, 1 ); if ( handleObj.selector ) { handlers.delegateCount--; } if ( special.remove ) { special.remove.call( elem, handleObj ); } } } // Remove generic event handler if we removed something and no more handlers exist // (avoids potential for endless recursion during removal of special event handlers) if ( origCount && !handlers.length ) { if ( !special.teardown || special.teardown.call( elem, namespaces, elemData.handle ) === false ) { jQuery.removeEvent( elem, type, elemData.handle ); } delete events[ type ]; } } // Remove data and the expando if it's no longer used if ( jQuery.isEmptyObject( events ) ) { dataPriv.remove( elem, "handle events" ); } }, dispatch: function( nativeEvent ) { var i, j, ret, matched, handleObj, handlerQueue, args = new Array( arguments.length ), // Make a writable jQuery.Event from the native event object event = jQuery.event.fix( nativeEvent ), handlers = ( dataPriv.get( this, "events" ) || Object.create( null ) )[ event.type ] || [], special = jQuery.event.special[ event.type ] || {}; // Use the fix-ed jQuery.Event rather than the (read-only) native event args[ 0 ] = event; for ( i = 1; i < arguments.length; i++ ) { args[ i ] = arguments[ i ]; } event.delegateTarget = this; // Call the preDispatch hook for the mapped type, and let it bail if desired if ( special.preDispatch && special.preDispatch.call( this, event ) === false ) { return; } // Determine handlers handlerQueue = jQuery.event.handlers.call( this, event, handlers ); // Run delegates first; they may want to stop propagation beneath us i = 0; while ( ( matched = handlerQueue[ i++ ] ) && !event.isPropagationStopped() ) { event.currentTarget = matched.elem; j = 0; while ( ( handleObj = matched.handlers[ j++ ] ) && !event.isImmediatePropagationStopped() ) { // If the event is namespaced, then each handler is only invoked if it is // specially universal or its namespaces are a superset of the event's. if ( !event.rnamespace || handleObj.namespace === false || event.rnamespace.test( handleObj.namespace ) ) { event.handleObj = handleObj; event.data = handleObj.data; ret = ( ( jQuery.event.special[ handleObj.origType ] || {} ).handle || handleObj.handler ).apply( matched.elem, args ); if ( ret !== undefined ) { if ( ( event.result = ret ) === false ) { event.preventDefault(); event.stopPropagation(); } } } } } // Call the postDispatch hook for the mapped type if ( special.postDispatch ) { special.postDispatch.call( this, event ); } return event.result; }, handlers: function( event, handlers ) { var i, handleObj, sel, matchedHandlers, matchedSelectors, handlerQueue = [], delegateCount = handlers.delegateCount, cur = event.target; // Find delegate handlers if ( delegateCount && // Support: IE <=9 // Black-hole SVG <use> instance trees (trac-13180) cur.nodeType && // Support: Firefox <=42 // Suppress spec-violating clicks indicating a non-primary pointer button (trac-3861) // https://www.w3.org/TR/DOM-Level-3-Events/#event-type-click // Support: IE 11 only // ...but not arrow key "clicks" of radio inputs, which can have `button` -1 (gh-2343) !( event.type === "click" && event.button >= 1 ) ) { for ( ; cur !== this; cur = cur.parentNode || this ) { // Don't check non-elements (#13208) // Don't process clicks on disabled elements (#6911, #8165, #11382, #11764) if ( cur.nodeType === 1 && !( event.type === "click" && cur.disabled === true ) ) { matchedHandlers = []; matchedSelectors = {}; for ( i = 0; i < delegateCount; i++ ) { handleObj = handlers[ i ]; // Don't conflict with Object.prototype properties (#13203) sel = handleObj.selector + " "; if ( matchedSelectors[ sel ] === undefined ) { matchedSelectors[ sel ] = handleObj.needsContext ? jQuery( sel, this ).index( cur ) > -1 : jQuery.find( sel, this, null, [ cur ] ).length; } if ( matchedSelectors[ sel ] ) { matchedHandlers.push( handleObj ); } } if ( matchedHandlers.length ) { handlerQueue.push( { elem: cur, handlers: matchedHandlers } ); } } } } // Add the remaining (directly-bound) handlers cur = this; if ( delegateCount < handlers.length ) { handlerQueue.push( { elem: cur, handlers: handlers.slice( delegateCount ) } ); } return handlerQueue; }, addProp: function( name, hook ) { Object.defineProperty( jQuery.Event.prototype, name, { enumerable: true, configurable: true, get: isFunction( hook ) ? function() { if ( this.originalEvent ) { return hook( this.originalEvent ); } } : function() { if ( this.originalEvent ) { return this.originalEvent[ name ]; } }, set: function( value ) { Object.defineProperty( this, name, { enumerable: true, configurable: true, writable: true, value: value } ); } } ); }, fix: function( originalEvent ) { return originalEvent[ jQuery.expando ] ? originalEvent : new jQuery.Event( originalEvent ); }, special: { load: { // Prevent triggered image.load events from bubbling to window.load noBubble: true }, click: { // Utilize native event to ensure correct state for checkable inputs setup: function( data ) { // For mutual compressibility with _default, replace `this` access with a local var. // `|| data` is dead code meant only to preserve the variable through minification. var el = this || data; // Claim the first handler if ( rcheckableType.test( el.type ) && el.click && nodeName( el, "input" ) ) { // dataPriv.set( el, "click", ... ) leverageNative( el, "click", returnTrue ); } // Return false to allow normal processing in the caller return false; }, trigger: function( data ) { // For mutual compressibility with _default, replace `this` access with a local var. // `|| data` is dead code meant only to preserve the variable through minification. var el = this || data; // Force setup before triggering a click if ( rcheckableType.test( el.type ) && el.click && nodeName( el, "input" ) ) { leverageNative( el, "click" ); } // Return non-false to allow normal event-path propagation return true; }, // For cross-browser consistency, suppress native .click() on links // Also prevent it if we're currently inside a leveraged native-event stack _default: function( event ) { var target = event.target; return rcheckableType.test( target.type ) && target.click && nodeName( target, "input" ) && dataPriv.get( target, "click" ) || nodeName( target, "a" ); } }, beforeunload: { postDispatch: function( event ) { // Support: Firefox 20+ // Firefox doesn't alert if the returnValue field is not set. if ( event.result !== undefined && event.originalEvent ) { event.originalEvent.returnValue = event.result; } } } } }; // Ensure the presence of an event listener that handles manually-triggered // synthetic events by interrupting progress until reinvoked in response to // *native* events that it fires directly, ensuring that state changes have // already occurred before other listeners are invoked. function leverageNative( el, type, expectSync ) { // Missing expectSync indicates a trigger call, which must force setup through jQuery.event.add if ( !expectSync ) { if ( dataPriv.get( el, type ) === undefined ) { jQuery.event.add( el, type, returnTrue ); } return; } // Register the controller as a special universal handler for all event namespaces dataPriv.set( el, type, false ); jQuery.event.add( el, type, { namespace: false, handler: function( event ) { var notAsync, result, saved = dataPriv.get( this, type ); if ( ( event.isTrigger & 1 ) && this[ type ] ) { // Interrupt processing of the outer synthetic .trigger()ed event // Saved data should be false in such cases, but might be a leftover capture object // from an async native handler (gh-4350) if ( !saved.length ) { // Store arguments for use when handling the inner native event // There will always be at least one argument (an event object), so this array // will not be confused with a leftover capture object. saved = slice.call( arguments ); dataPriv.set( this, type, saved ); // Trigger the native event and capture its result // Support: IE <=9 - 11+ // focus() and blur() are asynchronous notAsync = expectSync( this, type ); this[ type ](); result = dataPriv.get( this, type ); if ( saved !== result || notAsync ) { dataPriv.set( this, type, false ); } else { result = {}; } if ( saved !== result ) { // Cancel the outer synthetic event event.stopImmediatePropagation(); event.preventDefault(); // Support: Chrome 86+ // In Chrome, if an element having a focusout handler is blurred by // clicking outside of it, it invokes the handler synchronously. If // that handler calls `.remove()` on the element, the data is cleared, // leaving `result` undefined. We need to guard against this. return result && result.value; } // If this is an inner synthetic event for an event with a bubbling surrogate // (focus or blur), assume that the surrogate already propagated from triggering the // native event and prevent that from happening again here. // This technically gets the ordering wrong w.r.t. to `.trigger()` (in which the // bubbling surrogate propagates *after* the non-bubbling base), but that seems // less bad than duplication. } else if ( ( jQuery.event.special[ type ] || {} ).delegateType ) { event.stopPropagation(); } // If this is a native event triggered above, everything is now in order // Fire an inner synthetic event with the original arguments } else if ( saved.length ) { // ...and capture the result dataPriv.set( this, type, { value: jQuery.event.trigger( // Support: IE <=9 - 11+ // Extend with the prototype to reset the above stopImmediatePropagation() jQuery.extend( saved[ 0 ], jQuery.Event.prototype ), saved.slice( 1 ), this ) } ); // Abort handling of the native event event.stopImmediatePropagation(); } } } ); } jQuery.removeEvent = function( elem, type, handle ) { // This "if" is needed for plain objects if ( elem.removeEventListener ) { elem.removeEventListener( type, handle ); } }; jQuery.Event = function( src, props ) { // Allow instantiation without the 'new' keyword if ( !( this instanceof jQuery.Event ) ) { return new jQuery.Event( src, props ); } // Event object if ( src && src.type ) { this.originalEvent = src; this.type = src.type; // Events bubbling up the document may have been marked as prevented // by a handler lower down the tree; reflect the correct value. this.isDefaultPrevented = src.defaultPrevented || src.defaultPrevented === undefined && // Support: Android <=2.3 only src.returnValue === false ? returnTrue : returnFalse; // Create target properties // Support: Safari <=6 - 7 only // Target should not be a text node (#504, #13143) this.target = ( src.target && src.target.nodeType === 3 ) ? src.target.parentNode : src.target; this.currentTarget = src.currentTarget; this.relatedTarget = src.relatedTarget; // Event type } else { this.type = src; } // Put explicitly provided properties onto the event object if ( props ) { jQuery.extend( this, props ); } // Create a timestamp if incoming event doesn't have one this.timeStamp = src && src.timeStamp || Date.now(); // Mark it as fixed this[ jQuery.expando ] = true; }; // jQuery.Event is based on DOM3 Events as specified by the ECMAScript Language Binding // https://www.w3.org/TR/2003/WD-DOM-Level-3-Events-20030331/ecma-script-binding.html jQuery.Event.prototype = { constructor: jQuery.Event, isDefaultPrevented: returnFalse, isPropagationStopped: returnFalse, isImmediatePropagationStopped: returnFalse, isSimulated: false, preventDefault: function() { var e = this.originalEvent; this.isDefaultPrevented = returnTrue; if ( e && !this.isSimulated ) { e.preventDefault(); } }, stopPropagation: function() { var e = this.originalEvent; this.isPropagationStopped = returnTrue; if ( e && !this.isSimulated ) { e.stopPropagation(); } }, stopImmediatePropagation: function() { var e = this.originalEvent; this.isImmediatePropagationStopped = returnTrue; if ( e && !this.isSimulated ) { e.stopImmediatePropagation(); } this.stopPropagation(); } }; // Includes all common event props including KeyEvent and MouseEvent specific props jQuery.each( { altKey: true, bubbles: true, cancelable: true, changedTouches: true, ctrlKey: true, detail: true, eventPhase: true, metaKey: true, pageX: true, pageY: true, shiftKey: true, view: true, "char": true, code: true, charCode: true, key: true, keyCode: true, button: true, buttons: true, clientX: true, clientY: true, offsetX: true, offsetY: true, pointerId: true, pointerType: true, screenX: true, screenY: true, targetTouches: true, toElement: true, touches: true, which: true }, jQuery.event.addProp ); jQuery.each( { focus: "focusin", blur: "focusout" }, function( type, delegateType ) { jQuery.event.special[ type ] = { // Utilize native event if possible so blur/focus sequence is correct setup: function() { // Claim the first handler // dataPriv.set( this, "focus", ... ) // dataPriv.set( this, "blur", ... ) leverageNative( this, type, expectSync ); // Return false to allow normal processing in the caller return false; }, trigger: function() { // Force setup before trigger leverageNative( this, type ); // Return non-false to allow normal event-path propagation return true; }, // Suppress native focus or blur as it's already being fired // in leverageNative. _default: function() { return true; }, delegateType: delegateType }; } ); // Create mouseenter/leave events using mouseover/out and event-time checks // so that event delegation works in jQuery. // Do the same for pointerenter/pointerleave and pointerover/pointerout // // Support: Safari 7 only // Safari sends mouseenter too often; see: // https://bugs.chromium.org/p/chromium/issues/detail?id=470258 // for the description of the bug (it existed in older Chrome versions as well). jQuery.each( { mouseenter: "mouseover", mouseleave: "mouseout", pointerenter: "pointerover", pointerleave: "pointerout" }, function( orig, fix ) { jQuery.event.special[ orig ] = { delegateType: fix, bindType: fix, handle: function( event ) { var ret, target = this, related = event.relatedTarget, handleObj = event.handleObj; // For mouseenter/leave call the handler if related is outside the target. // NB: No relatedTarget if the mouse left/entered the browser window if ( !related || ( related !== target && !jQuery.contains( target, related ) ) ) { event.type = handleObj.origType; ret = handleObj.handler.apply( this, arguments ); event.type = fix; } return ret; } }; } ); jQuery.fn.extend( { on: function( types, selector, data, fn ) { return on( this, types, selector, data, fn ); }, one: function( types, selector, data, fn ) { return on( this, types, selector, data, fn, 1 ); }, off: function( types, selector, fn ) { var handleObj, type; if ( types && types.preventDefault && types.handleObj ) { // ( event ) dispatched jQuery.Event handleObj = types.handleObj; jQuery( types.delegateTarget ).off( handleObj.namespace ? handleObj.origType + "." + handleObj.namespace : handleObj.origType, handleObj.selector, handleObj.handler ); return this; } if ( typeof types === "object" ) { // ( types-object [, selector] ) for ( type in types ) { this.off( type, selector, types[ type ] ); } return this; } if ( selector === false || typeof selector === "function" ) { // ( types [, fn] ) fn = selector; selector = undefined; } if ( fn === false ) { fn = returnFalse; } return this.each( function() { jQuery.event.remove( this, types, fn, selector ); } ); } } ); var // Support: IE <=10 - 11, Edge 12 - 13 only // In IE/Edge using regex groups here causes severe slowdowns. // See https://connect.microsoft.com/IE/feedback/details/1736512/ rnoInnerhtml = /<script|<style|<link/i, // checked="checked" or checked rchecked = /checked\s*(?:[^=]|=\s*.checked.)/i, rcleanScript = /^\s*<!(?:\[CDATA\[|--)|(?:\]\]|--)>\s*$/g; // Prefer a tbody over its parent table for containing new rows function manipulationTarget( elem, content ) { if ( nodeName( elem, "table" ) && nodeName( content.nodeType !== 11 ? content : content.firstChild, "tr" ) ) { return jQuery( elem ).children( "tbody" )[ 0 ] || elem; } return elem; } // Replace/restore the type attribute of script elements for safe DOM manipulation function disableScript( elem ) { elem.type = ( elem.getAttribute( "type" ) !== null ) + "/" + elem.type; return elem; } function restoreScript( elem ) { if ( ( elem.type || "" ).slice( 0, 5 ) === "true/" ) { elem.type = elem.type.slice( 5 ); } else { elem.removeAttribute( "type" ); } return elem; } function cloneCopyEvent( src, dest ) { var i, l, type, pdataOld, udataOld, udataCur, events; if ( dest.nodeType !== 1 ) { return; } // 1. Copy private data: events, handlers, etc. if ( dataPriv.hasData( src ) ) { pdataOld = dataPriv.get( src ); events = pdataOld.events; if ( events ) { dataPriv.remove( dest, "handle events" ); for ( type in events ) { for ( i = 0, l = events[ type ].length; i < l; i++ ) { jQuery.event.add( dest, type, events[ type ][ i ] ); } } } } // 2. Copy user data if ( dataUser.hasData( src ) ) { udataOld = dataUser.access( src ); udataCur = jQuery.extend( {}, udataOld ); dataUser.set( dest, udataCur ); } } // Fix IE bugs, see support tests function fixInput( src, dest ) { var nodeName = dest.nodeName.toLowerCase(); // Fails to persist the checked state of a cloned checkbox or radio button. if ( nodeName === "input" && rcheckableType.test( src.type ) ) { dest.checked = src.checked; // Fails to return the selected option to the default selected state when cloning options } else if ( nodeName === "input" || nodeName === "textarea" ) { dest.defaultValue = src.defaultValue; } } function domManip( collection, args, callback, ignored ) { // Flatten any nested arrays args = flat( args ); var fragment, first, scripts, hasScripts, node, doc, i = 0, l = collection.length, iNoClone = l - 1, value = args[ 0 ], valueIsFunction = isFunction( value ); // We can't cloneNode fragments that contain checked, in WebKit if ( valueIsFunction || ( l > 1 && typeof value === "string" && !support.checkClone && rchecked.test( value ) ) ) { return collection.each( function( index ) { var self = collection.eq( index ); if ( valueIsFunction ) { args[ 0 ] = value.call( this, index, self.html() ); } domManip( self, args, callback, ignored ); } ); } if ( l ) { fragment = buildFragment( args, collection[ 0 ].ownerDocument, false, collection, ignored ); first = fragment.firstChild; if ( fragment.childNodes.length === 1 ) { fragment = first; } // Require either new content or an interest in ignored elements to invoke the callback if ( first || ignored ) { scripts = jQuery.map( getAll( fragment, "script" ), disableScript ); hasScripts = scripts.length; // Use the original fragment for the last item // instead of the first because it can end up // being emptied incorrectly in certain situations (#8070). for ( ; i < l; i++ ) { node = fragment; if ( i !== iNoClone ) { node = jQuery.clone( node, true, true ); // Keep references to cloned scripts for later restoration if ( hasScripts ) { // Support: Android <=4.0 only, PhantomJS 1 only // push.apply(_, arraylike) throws on ancient WebKit jQuery.merge( scripts, getAll( node, "script" ) ); } } callback.call( collection[ i ], node, i ); } if ( hasScripts ) { doc = scripts[ scripts.length - 1 ].ownerDocument; // Reenable scripts jQuery.map( scripts, restoreScript ); // Evaluate executable scripts on first document insertion for ( i = 0; i < hasScripts; i++ ) { node = scripts[ i ]; if ( rscriptType.test( node.type || "" ) && !dataPriv.access( node, "globalEval" ) && jQuery.contains( doc, node ) ) { if ( node.src && ( node.type || "" ).toLowerCase() !== "module" ) { // Optional AJAX dependency, but won't run scripts if not present if ( jQuery._evalUrl && !node.noModule ) { jQuery._evalUrl( node.src, { nonce: node.nonce || node.getAttribute( "nonce" ) }, doc ); } } else { DOMEval( node.textContent.replace( rcleanScript, "" ), node, doc ); } } } } } } return collection; } function remove( elem, selector, keepData ) { var node, nodes = selector ? jQuery.filter( selector, elem ) : elem, i = 0; for ( ; ( node = nodes[ i ] ) != null; i++ ) { if ( !keepData && node.nodeType === 1 ) { jQuery.cleanData( getAll( node ) ); } if ( node.parentNode ) { if ( keepData && isAttached( node ) ) { setGlobalEval( getAll( node, "script" ) ); } node.parentNode.removeChild( node ); } } return elem; } jQuery.extend( { htmlPrefilter: function( html ) { return html; }, clone: function( elem, dataAndEvents, deepDataAndEvents ) { var i, l, srcElements, destElements, clone = elem.cloneNode( true ), inPage = isAttached( elem ); // Fix IE cloning issues if ( !support.noCloneChecked && ( elem.nodeType === 1 || elem.nodeType === 11 ) && !jQuery.isXMLDoc( elem ) ) { // We eschew Sizzle here for performance reasons: https://jsperf.com/getall-vs-sizzle/2 destElements = getAll( clone ); srcElements = getAll( elem ); for ( i = 0, l = srcElements.length; i < l; i++ ) { fixInput( srcElements[ i ], destElements[ i ] ); } } // Copy the events from the original to the clone if ( dataAndEvents ) { if ( deepDataAndEvents ) { srcElements = srcElements || getAll( elem ); destElements = destElements || getAll( clone ); for ( i = 0, l = srcElements.length; i < l; i++ ) { cloneCopyEvent( srcElements[ i ], destElements[ i ] ); } } else { cloneCopyEvent( elem, clone ); } } // Preserve script evaluation history destElements = getAll( clone, "script" ); if ( destElements.length > 0 ) { setGlobalEval( destElements, !inPage && getAll( elem, "script" ) ); } // Return the cloned set return clone; }, cleanData: function( elems ) { var data, elem, type, special = jQuery.event.special, i = 0; for ( ; ( elem = elems[ i ] ) !== undefined; i++ ) { if ( acceptData( elem ) ) { if ( ( data = elem[ dataPriv.expando ] ) ) { if ( data.events ) { for ( type in data.events ) { if ( special[ type ] ) { jQuery.event.remove( elem, type ); // This is a shortcut to avoid jQuery.event.remove's overhead } else { jQuery.removeEvent( elem, type, data.handle ); } } } // Support: Chrome <=35 - 45+ // Assign undefined instead of using delete, see Data#remove elem[ dataPriv.expando ] = undefined; } if ( elem[ dataUser.expando ] ) { // Support: Chrome <=35 - 45+ // Assign undefined instead of using delete, see Data#remove elem[ dataUser.expando ] = undefined; } } } } } ); jQuery.fn.extend( { detach: function( selector ) { return remove( this, selector, true ); }, remove: function( selector ) { return remove( this, selector ); }, text: function( value ) { return access( this, function( value ) { return value === undefined ? jQuery.text( this ) : this.empty().each( function() { if ( this.nodeType === 1 || this.nodeType === 11 || this.nodeType === 9 ) { this.textContent = value; } } ); }, null, value, arguments.length ); }, append: function() { return domManip( this, arguments, function( elem ) { if ( this.nodeType === 1 || this.nodeType === 11 || this.nodeType === 9 ) { var target = manipulationTarget( this, elem ); target.appendChild( elem ); } } ); }, prepend: function() { return domManip( this, arguments, function( elem ) { if ( this.nodeType === 1 || this.nodeType === 11 || this.nodeType === 9 ) { var target = manipulationTarget( this, elem ); target.insertBefore( elem, target.firstChild ); } } ); }, before: function() { return domManip( this, arguments, function( elem ) { if ( this.parentNode ) { this.parentNode.insertBefore( elem, this ); } } ); }, after: function() { return domManip( this, arguments, function( elem ) { if ( this.parentNode ) { this.parentNode.insertBefore( elem, this.nextSibling ); } } ); }, empty: function() { var elem, i = 0; for ( ; ( elem = this[ i ] ) != null; i++ ) { if ( elem.nodeType === 1 ) { // Prevent memory leaks jQuery.cleanData( getAll( elem, false ) ); // Remove any remaining nodes elem.textContent = ""; } } return this; }, clone: function( dataAndEvents, deepDataAndEvents ) { dataAndEvents = dataAndEvents == null ? false : dataAndEvents; deepDataAndEvents = deepDataAndEvents == null ? dataAndEvents : deepDataAndEvents; return this.map( function() { return jQuery.clone( this, dataAndEvents, deepDataAndEvents ); } ); }, html: function( value ) { return access( this, function( value ) { var elem = this[ 0 ] || {}, i = 0, l = this.length; if ( value === undefined && elem.nodeType === 1 ) { return elem.innerHTML; } // See if we can take a shortcut and just use innerHTML if ( typeof value === "string" && !rnoInnerhtml.test( value ) && !wrapMap[ ( rtagName.exec( value ) || [ "", "" ] )[ 1 ].toLowerCase() ] ) { value = jQuery.htmlPrefilter( value ); try { for ( ; i < l; i++ ) { elem = this[ i ] || {}; // Remove element nodes and prevent memory leaks if ( elem.nodeType === 1 ) { jQuery.cleanData( getAll( elem, false ) ); elem.innerHTML = value; } } elem = 0; // If using innerHTML throws an exception, use the fallback method } catch ( e ) {} } if ( elem ) { this.empty().append( value ); } }, null, value, arguments.length ); }, replaceWith: function() { var ignored = []; // Make the changes, replacing each non-ignored context element with the new content return domManip( this, arguments, function( elem ) { var parent = this.parentNode; if ( jQuery.inArray( this, ignored ) < 0 ) { jQuery.cleanData( getAll( this ) ); if ( parent ) { parent.replaceChild( elem, this ); } } // Force callback invocation }, ignored ); } } ); jQuery.each( { appendTo: "append", prependTo: "prepend", insertBefore: "before", insertAfter: "after", replaceAll: "replaceWith" }, function( name, original ) { jQuery.fn[ name ] = function( selector ) { var elems, ret = [], insert = jQuery( selector ), last = insert.length - 1, i = 0; for ( ; i <= last; i++ ) { elems = i === last ? this : this.clone( true ); jQuery( insert[ i ] )[ original ]( elems ); // Support: Android <=4.0 only, PhantomJS 1 only // .get() because push.apply(_, arraylike) throws on ancient WebKit push.apply( ret, elems.get() ); } return this.pushStack( ret ); }; } ); var rnumnonpx = new RegExp( "^(" + pnum + ")(?!px)[a-z%]+$", "i" ); var getStyles = function( elem ) { // Support: IE <=11 only, Firefox <=30 (#15098, #14150) // IE throws on elements created in popups // FF meanwhile throws on frame elements through "defaultView.getComputedStyle" var view = elem.ownerDocument.defaultView; if ( !view || !view.opener ) { view = window; } return view.getComputedStyle( elem ); }; var swap = function( elem, options, callback ) { var ret, name, old = {}; // Remember the old values, and insert the new ones for ( name in options ) { old[ name ] = elem.style[ name ]; elem.style[ name ] = options[ name ]; } ret = callback.call( elem ); // Revert the old values for ( name in options ) { elem.style[ name ] = old[ name ]; } return ret; }; var rboxStyle = new RegExp( cssExpand.join( "|" ), "i" ); ( function() { // Executing both pixelPosition & boxSizingReliable tests require only one layout // so they're executed at the same time to save the second computation. function computeStyleTests() { // This is a singleton, we need to execute it only once if ( !div ) { return; } container.style.cssText = "position:absolute;left:-11111px;width:60px;" + "margin-top:1px;padding:0;border:0"; div.style.cssText = "position:relative;display:block;box-sizing:border-box;overflow:scroll;" + "margin:auto;border:1px;padding:1px;" + "width:60%;top:1%"; documentElement.appendChild( container ).appendChild( div ); var divStyle = window.getComputedStyle( div ); pixelPositionVal = divStyle.top !== "1%"; // Support: Android 4.0 - 4.3 only, Firefox <=3 - 44 reliableMarginLeftVal = roundPixelMeasures( divStyle.marginLeft ) === 12; // Support: Android 4.0 - 4.3 only, Safari <=9.1 - 10.1, iOS <=7.0 - 9.3 // Some styles come back with percentage values, even though they shouldn't div.style.right = "60%"; pixelBoxStylesVal = roundPixelMeasures( divStyle.right ) === 36; // Support: IE 9 - 11 only // Detect misreporting of content dimensions for box-sizing:border-box elements boxSizingReliableVal = roundPixelMeasures( divStyle.width ) === 36; // Support: IE 9 only // Detect overflow:scroll screwiness (gh-3699) // Support: Chrome <=64 // Don't get tricked when zoom affects offsetWidth (gh-4029) div.style.position = "absolute"; scrollboxSizeVal = roundPixelMeasures( div.offsetWidth / 3 ) === 12; documentElement.removeChild( container ); // Nullify the div so it wouldn't be stored in the memory and // it will also be a sign that checks already performed div = null; } function roundPixelMeasures( measure ) { return Math.round( parseFloat( measure ) ); } var pixelPositionVal, boxSizingReliableVal, scrollboxSizeVal, pixelBoxStylesVal, reliableTrDimensionsVal, reliableMarginLeftVal, container = document.createElement( "div" ), div = document.createElement( "div" ); // Finish early in limited (non-browser) environments if ( !div.style ) { return; } // Support: IE <=9 - 11 only // Style of cloned element affects source element cloned (#8908) div.style.backgroundClip = "content-box"; div.cloneNode( true ).style.backgroundClip = ""; support.clearCloneStyle = div.style.backgroundClip === "content-box"; jQuery.extend( support, { boxSizingReliable: function() { computeStyleTests(); return boxSizingReliableVal; }, pixelBoxStyles: function() { computeStyleTests(); return pixelBoxStylesVal; }, pixelPosition: function() { computeStyleTests(); return pixelPositionVal; }, reliableMarginLeft: function() { computeStyleTests(); return reliableMarginLeftVal; }, scrollboxSize: function() { computeStyleTests(); return scrollboxSizeVal; }, // Support: IE 9 - 11+, Edge 15 - 18+ // IE/Edge misreport `getComputedStyle` of table rows with width/height // set in CSS while `offset*` properties report correct values. // Behavior in IE 9 is more subtle than in newer versions & it passes // some versions of this test; make sure not to make it pass there! // // Support: Firefox 70+ // Only Firefox includes border widths // in computed dimensions. (gh-4529) reliableTrDimensions: function() { var table, tr, trChild, trStyle; if ( reliableTrDimensionsVal == null ) { table = document.createElement( "table" ); tr = document.createElement( "tr" ); trChild = document.createElement( "div" ); table.style.cssText = "position:absolute;left:-11111px;border-collapse:separate"; tr.style.cssText = "border:1px solid"; // Support: Chrome 86+ // Height set through cssText does not get applied. // Computed height then comes back as 0. tr.style.height = "1px"; trChild.style.height = "9px"; // Support: Android 8 Chrome 86+ // In our bodyBackground.html iframe, // display for all div elements is set to "inline", // which causes a problem only in Android 8 Chrome 86. // Ensuring the div is display: block // gets around this issue. trChild.style.display = "block"; documentElement .appendChild( table ) .appendChild( tr ) .appendChild( trChild ); trStyle = window.getComputedStyle( tr ); reliableTrDimensionsVal = ( parseInt( trStyle.height, 10 ) + parseInt( trStyle.borderTopWidth, 10 ) + parseInt( trStyle.borderBottomWidth, 10 ) ) === tr.offsetHeight; documentElement.removeChild( table ); } return reliableTrDimensionsVal; } } ); } )(); function curCSS( elem, name, computed ) { var width, minWidth, maxWidth, ret, // Support: Firefox 51+ // Retrieving style before computed somehow // fixes an issue with getting wrong values // on detached elements style = elem.style; computed = computed || getStyles( elem ); // getPropertyValue is needed for: // .css('filter') (IE 9 only, #12537) // .css('--customProperty) (#3144) if ( computed ) { ret = computed.getPropertyValue( name ) || computed[ name ]; if ( ret === "" && !isAttached( elem ) ) { ret = jQuery.style( elem, name ); } // A tribute to the "awesome hack by Dean Edwards" // Android Browser returns percentage for some values, // but width seems to be reliably pixels. // This is against the CSSOM draft spec: // https://drafts.csswg.org/cssom/#resolved-values if ( !support.pixelBoxStyles() && rnumnonpx.test( ret ) && rboxStyle.test( name ) ) { // Remember the original values width = style.width; minWidth = style.minWidth; maxWidth = style.maxWidth; // Put in the new values to get a computed value out style.minWidth = style.maxWidth = style.width = ret; ret = computed.width; // Revert the changed values style.width = width; style.minWidth = minWidth; style.maxWidth = maxWidth; } } return ret !== undefined ? // Support: IE <=9 - 11 only // IE returns zIndex value as an integer. ret + "" : ret; } function addGetHookIf( conditionFn, hookFn ) { // Define the hook, we'll check on the first run if it's really needed. return { get: function() { if ( conditionFn() ) { // Hook not needed (or it's not possible to use it due // to missing dependency), remove it. delete this.get; return; } // Hook needed; redefine it so that the support test is not executed again. return ( this.get = hookFn ).apply( this, arguments ); } }; } var cssPrefixes = [ "Webkit", "Moz", "ms" ], emptyStyle = document.createElement( "div" ).style, vendorProps = {}; // Return a vendor-prefixed property or undefined function vendorPropName( name ) { // Check for vendor prefixed names var capName = name[ 0 ].toUpperCase() + name.slice( 1 ), i = cssPrefixes.length; while ( i-- ) { name = cssPrefixes[ i ] + capName; if ( name in emptyStyle ) { return name; } } } // Return a potentially-mapped jQuery.cssProps or vendor prefixed property function finalPropName( name ) { var final = jQuery.cssProps[ name ] || vendorProps[ name ]; if ( final ) { return final; } if ( name in emptyStyle ) { return name; } return vendorProps[ name ] = vendorPropName( name ) || name; } var // Swappable if display is none or starts with table // except "table", "table-cell", or "table-caption" // See here for display values: https://developer.mozilla.org/en-US/docs/CSS/display rdisplayswap = /^(none|table(?!-c[ea]).+)/, rcustomProp = /^--/, cssShow = { position: "absolute", visibility: "hidden", display: "block" }, cssNormalTransform = { letterSpacing: "0", fontWeight: "400" }; function setPositiveNumber( _elem, value, subtract ) { // Any relative (+/-) values have already been // normalized at this point var matches = rcssNum.exec( value ); return matches ? // Guard against undefined "subtract", e.g., when used as in cssHooks Math.max( 0, matches[ 2 ] - ( subtract || 0 ) ) + ( matches[ 3 ] || "px" ) : value; } function boxModelAdjustment( elem, dimension, box, isBorderBox, styles, computedVal ) { var i = dimension === "width" ? 1 : 0, extra = 0, delta = 0; // Adjustment may not be necessary if ( box === ( isBorderBox ? "border" : "content" ) ) { return 0; } for ( ; i < 4; i += 2 ) { // Both box models exclude margin if ( box === "margin" ) { delta += jQuery.css( elem, box + cssExpand[ i ], true, styles ); } // If we get here with a content-box, we're seeking "padding" or "border" or "margin" if ( !isBorderBox ) { // Add padding delta += jQuery.css( elem, "padding" + cssExpand[ i ], true, styles ); // For "border" or "margin", add border if ( box !== "padding" ) { delta += jQuery.css( elem, "border" + cssExpand[ i ] + "Width", true, styles ); // But still keep track of it otherwise } else { extra += jQuery.css( elem, "border" + cssExpand[ i ] + "Width", true, styles ); } // If we get here with a border-box (content + padding + border), we're seeking "content" or // "padding" or "margin" } else { // For "content", subtract padding if ( box === "content" ) { delta -= jQuery.css( elem, "padding" + cssExpand[ i ], true, styles ); } // For "content" or "padding", subtract border if ( box !== "margin" ) { delta -= jQuery.css( elem, "border" + cssExpand[ i ] + "Width", true, styles ); } } } // Account for positive content-box scroll gutter when requested by providing computedVal if ( !isBorderBox && computedVal >= 0 ) { // offsetWidth/offsetHeight is a rounded sum of content, padding, scroll gutter, and border // Assuming integer scroll gutter, subtract the rest and round down delta += Math.max( 0, Math.ceil( elem[ "offset" + dimension[ 0 ].toUpperCase() + dimension.slice( 1 ) ] - computedVal - delta - extra - 0.5 // If offsetWidth/offsetHeight is unknown, then we can't determine content-box scroll gutter // Use an explicit zero to avoid NaN (gh-3964) ) ) || 0; } return delta; } function getWidthOrHeight( elem, dimension, extra ) { // Start with computed style var styles = getStyles( elem ), // To avoid forcing a reflow, only fetch boxSizing if we need it (gh-4322). // Fake content-box until we know it's needed to know the true value. boxSizingNeeded = !support.boxSizingReliable() || extra, isBorderBox = boxSizingNeeded && jQuery.css( elem, "boxSizing", false, styles ) === "border-box", valueIsBorderBox = isBorderBox, val = curCSS( elem, dimension, styles ), offsetProp = "offset" + dimension[ 0 ].toUpperCase() + dimension.slice( 1 ); // Support: Firefox <=54 // Return a confounding non-pixel value or feign ignorance, as appropriate. if ( rnumnonpx.test( val ) ) { if ( !extra ) { return val; } val = "auto"; } // Support: IE 9 - 11 only // Use offsetWidth/offsetHeight for when box sizing is unreliable. // In those cases, the computed value can be trusted to be border-box. if ( ( !support.boxSizingReliable() && isBorderBox || // Support: IE 10 - 11+, Edge 15 - 18+ // IE/Edge misreport `getComputedStyle` of table rows with width/height // set in CSS while `offset*` properties report correct values. // Interestingly, in some cases IE 9 doesn't suffer from this issue. !support.reliableTrDimensions() && nodeName( elem, "tr" ) || // Fall back to offsetWidth/offsetHeight when value is "auto" // This happens for inline elements with no explicit setting (gh-3571) val === "auto" || // Support: Android <=4.1 - 4.3 only // Also use offsetWidth/offsetHeight for misreported inline dimensions (gh-3602) !parseFloat( val ) && jQuery.css( elem, "display", false, styles ) === "inline" ) && // Make sure the element is visible & connected elem.getClientRects().length ) { isBorderBox = jQuery.css( elem, "boxSizing", false, styles ) === "border-box"; // Where available, offsetWidth/offsetHeight approximate border box dimensions. // Where not available (e.g., SVG), assume unreliable box-sizing and interpret the // retrieved value as a content box dimension. valueIsBorderBox = offsetProp in elem; if ( valueIsBorderBox ) { val = elem[ offsetProp ]; } } // Normalize "" and auto val = parseFloat( val ) || 0; // Adjust for the element's box model return ( val + boxModelAdjustment( elem, dimension, extra || ( isBorderBox ? "border" : "content" ), valueIsBorderBox, styles, // Provide the current computed size to request scroll gutter calculation (gh-3589) val ) ) + "px"; } jQuery.extend( { // Add in style property hooks for overriding the default // behavior of getting and setting a style property cssHooks: { opacity: { get: function( elem, computed ) { if ( computed ) { // We should always get a number back from opacity var ret = curCSS( elem, "opacity" ); return ret === "" ? "1" : ret; } } } }, // Don't automatically add "px" to these possibly-unitless properties cssNumber: { "animationIterationCount": true, "columnCount": true, "fillOpacity": true, "flexGrow": true, "flexShrink": true, "fontWeight": true, "gridArea": true, "gridColumn": true, "gridColumnEnd": true, "gridColumnStart": true, "gridRow": true, "gridRowEnd": true, "gridRowStart": true, "lineHeight": true, "opacity": true, "order": true, "orphans": true, "widows": true, "zIndex": true, "zoom": true }, // Add in properties whose names you wish to fix before // setting or getting the value cssProps: {}, // Get and set the style property on a DOM Node style: function( elem, name, value, extra ) { // Don't set styles on text and comment nodes if ( !elem || elem.nodeType === 3 || elem.nodeType === 8 || !elem.style ) { return; } // Make sure that we're working with the right name var ret, type, hooks, origName = camelCase( name ), isCustomProp = rcustomProp.test( name ), style = elem.style; // Make sure that we're working with the right name. We don't // want to query the value if it is a CSS custom property // since they are user-defined. if ( !isCustomProp ) { name = finalPropName( origName ); } // Gets hook for the prefixed version, then unprefixed version hooks = jQuery.cssHooks[ name ] || jQuery.cssHooks[ origName ]; // Check if we're setting a value if ( value !== undefined ) { type = typeof value; // Convert "+=" or "-=" to relative numbers (#7345) if ( type === "string" && ( ret = rcssNum.exec( value ) ) && ret[ 1 ] ) { value = adjustCSS( elem, name, ret ); // Fixes bug #9237 type = "number"; } // Make sure that null and NaN values aren't set (#7116) if ( value == null || value !== value ) { return; } // If a number was passed in, add the unit (except for certain CSS properties) // The isCustomProp check can be removed in jQuery 4.0 when we only auto-append // "px" to a few hardcoded values. if ( type === "number" && !isCustomProp ) { value += ret && ret[ 3 ] || ( jQuery.cssNumber[ origName ] ? "" : "px" ); } // background-* props affect original clone's values if ( !support.clearCloneStyle && value === "" && name.indexOf( "background" ) === 0 ) { style[ name ] = "inherit"; } // If a hook was provided, use that value, otherwise just set the specified value if ( !hooks || !( "set" in hooks ) || ( value = hooks.set( elem, value, extra ) ) !== undefined ) { if ( isCustomProp ) { style.setProperty( name, value ); } else { style[ name ] = value; } } } else { // If a hook was provided get the non-computed value from there if ( hooks && "get" in hooks && ( ret = hooks.get( elem, false, extra ) ) !== undefined ) { return ret; } // Otherwise just get the value from the style object return style[ name ]; } }, css: function( elem, name, extra, styles ) { var val, num, hooks, origName = camelCase( name ), isCustomProp = rcustomProp.test( name ); // Make sure that we're working with the right name. We don't // want to modify the value if it is a CSS custom property // since they are user-defined. if ( !isCustomProp ) { name = finalPropName( origName ); } // Try prefixed name followed by the unprefixed name hooks = jQuery.cssHooks[ name ] || jQuery.cssHooks[ origName ]; // If a hook was provided get the computed value from there if ( hooks && "get" in hooks ) { val = hooks.get( elem, true, extra ); } // Otherwise, if a way to get the computed value exists, use that if ( val === undefined ) { val = curCSS( elem, name, styles ); } // Convert "normal" to computed value if ( val === "normal" && name in cssNormalTransform ) { val = cssNormalTransform[ name ]; } // Make numeric if forced or a qualifier was provided and val looks numeric if ( extra === "" || extra ) { num = parseFloat( val ); return extra === true || isFinite( num ) ? num || 0 : val; } return val; } } ); jQuery.each( [ "height", "width" ], function( _i, dimension ) { jQuery.cssHooks[ dimension ] = { get: function( elem, computed, extra ) { if ( computed ) { // Certain elements can have dimension info if we invisibly show them // but it must have a current display style that would benefit return rdisplayswap.test( jQuery.css( elem, "display" ) ) && // Support: Safari 8+ // Table columns in Safari have non-zero offsetWidth & zero // getBoundingClientRect().width unless display is changed. // Support: IE <=11 only // Running getBoundingClientRect on a disconnected node // in IE throws an error. ( !elem.getClientRects().length || !elem.getBoundingClientRect().width ) ? swap( elem, cssShow, function() { return getWidthOrHeight( elem, dimension, extra ); } ) : getWidthOrHeight( elem, dimension, extra ); } }, set: function( elem, value, extra ) { var matches, styles = getStyles( elem ), // Only read styles.position if the test has a chance to fail // to avoid forcing a reflow. scrollboxSizeBuggy = !support.scrollboxSize() && styles.position === "absolute", // To avoid forcing a reflow, only fetch boxSizing if we need it (gh-3991) boxSizingNeeded = scrollboxSizeBuggy || extra, isBorderBox = boxSizingNeeded && jQuery.css( elem, "boxSizing", false, styles ) === "border-box", subtract = extra ? boxModelAdjustment( elem, dimension, extra, isBorderBox, styles ) : 0; // Account for unreliable border-box dimensions by comparing offset* to computed and // faking a content-box to get border and padding (gh-3699) if ( isBorderBox && scrollboxSizeBuggy ) { subtract -= Math.ceil( elem[ "offset" + dimension[ 0 ].toUpperCase() + dimension.slice( 1 ) ] - parseFloat( styles[ dimension ] ) - boxModelAdjustment( elem, dimension, "border", false, styles ) - 0.5 ); } // Convert to pixels if value adjustment is needed if ( subtract && ( matches = rcssNum.exec( value ) ) && ( matches[ 3 ] || "px" ) !== "px" ) { elem.style[ dimension ] = value; value = jQuery.css( elem, dimension ); } return setPositiveNumber( elem, value, subtract ); } }; } ); jQuery.cssHooks.marginLeft = addGetHookIf( support.reliableMarginLeft, function( elem, computed ) { if ( computed ) { return ( parseFloat( curCSS( elem, "marginLeft" ) ) || elem.getBoundingClientRect().left - swap( elem, { marginLeft: 0 }, function() { return elem.getBoundingClientRect().left; } ) ) + "px"; } } ); // These hooks are used by animate to expand properties jQuery.each( { margin: "", padding: "", border: "Width" }, function( prefix, suffix ) { jQuery.cssHooks[ prefix + suffix ] = { expand: function( value ) { var i = 0, expanded = {}, // Assumes a single number if not a string parts = typeof value === "string" ? value.split( " " ) : [ value ]; for ( ; i < 4; i++ ) { expanded[ prefix + cssExpand[ i ] + suffix ] = parts[ i ] || parts[ i - 2 ] || parts[ 0 ]; } return expanded; } }; if ( prefix !== "margin" ) { jQuery.cssHooks[ prefix + suffix ].set = setPositiveNumber; } } ); jQuery.fn.extend( { css: function( name, value ) { return access( this, function( elem, name, value ) { var styles, len, map = {}, i = 0; if ( Array.isArray( name ) ) { styles = getStyles( elem ); len = name.length; for ( ; i < len; i++ ) { map[ name[ i ] ] = jQuery.css( elem, name[ i ], false, styles ); } return map; } return value !== undefined ? jQuery.style( elem, name, value ) : jQuery.css( elem, name ); }, name, value, arguments.length > 1 ); } } ); function Tween( elem, options, prop, end, easing ) { return new Tween.prototype.init( elem, options, prop, end, easing ); } jQuery.Tween = Tween; Tween.prototype = { constructor: Tween, init: function( elem, options, prop, end, easing, unit ) { this.elem = elem; this.prop = prop; this.easing = easing || jQuery.easing._default; this.options = options; this.start = this.now = this.cur(); this.end = end; this.unit = unit || ( jQuery.cssNumber[ prop ] ? "" : "px" ); }, cur: function() { var hooks = Tween.propHooks[ this.prop ]; return hooks && hooks.get ? hooks.get( this ) : Tween.propHooks._default.get( this ); }, run: function( percent ) { var eased, hooks = Tween.propHooks[ this.prop ]; if ( this.options.duration ) { this.pos = eased = jQuery.easing[ this.easing ]( percent, this.options.duration * percent, 0, 1, this.options.duration ); } else { this.pos = eased = percent; } this.now = ( this.end - this.start ) * eased + this.start; if ( this.options.step ) { this.options.step.call( this.elem, this.now, this ); } if ( hooks && hooks.set ) { hooks.set( this ); } else { Tween.propHooks._default.set( this ); } return this; } }; Tween.prototype.init.prototype = Tween.prototype; Tween.propHooks = { _default: { get: function( tween ) { var result; // Use a property on the element directly when it is not a DOM element, // or when there is no matching style property that exists. if ( tween.elem.nodeType !== 1 || tween.elem[ tween.prop ] != null && tween.elem.style[ tween.prop ] == null ) { return tween.elem[ tween.prop ]; } // Passing an empty string as a 3rd parameter to .css will automatically // attempt a parseFloat and fallback to a string if the parse fails. // Simple values such as "10px" are parsed to Float; // complex values such as "rotate(1rad)" are returned as-is. result = jQuery.css( tween.elem, tween.prop, "" ); // Empty strings, null, undefined and "auto" are converted to 0. return !result || result === "auto" ? 0 : result; }, set: function( tween ) { // Use step hook for back compat. // Use cssHook if its there. // Use .style if available and use plain properties where available. if ( jQuery.fx.step[ tween.prop ] ) { jQuery.fx.step[ tween.prop ]( tween ); } else if ( tween.elem.nodeType === 1 && ( jQuery.cssHooks[ tween.prop ] || tween.elem.style[ finalPropName( tween.prop ) ] != null ) ) { jQuery.style( tween.elem, tween.prop, tween.now + tween.unit ); } else { tween.elem[ tween.prop ] = tween.now; } } } }; // Support: IE <=9 only // Panic based approach to setting things on disconnected nodes Tween.propHooks.scrollTop = Tween.propHooks.scrollLeft = { set: function( tween ) { if ( tween.elem.nodeType && tween.elem.parentNode ) { tween.elem[ tween.prop ] = tween.now; } } }; jQuery.easing = { linear: function( p ) { return p; }, swing: function( p ) { return 0.5 - Math.cos( p * Math.PI ) / 2; }, _default: "swing" }; jQuery.fx = Tween.prototype.init; // Back compat <1.8 extension point jQuery.fx.step = {}; var fxNow, inProgress, rfxtypes = /^(?:toggle|show|hide)$/, rrun = /queueHooks$/; function schedule() { if ( inProgress ) { if ( document.hidden === false && window.requestAnimationFrame ) { window.requestAnimationFrame( schedule ); } else { window.setTimeout( schedule, jQuery.fx.interval ); } jQuery.fx.tick(); } } // Animations created synchronously will run synchronously function createFxNow() { window.setTimeout( function() { fxNow = undefined; } ); return ( fxNow = Date.now() ); } // Generate parameters to create a standard animation function genFx( type, includeWidth ) { var which, i = 0, attrs = { height: type }; // If we include width, step value is 1 to do all cssExpand values, // otherwise step value is 2 to skip over Left and Right includeWidth = includeWidth ? 1 : 0; for ( ; i < 4; i += 2 - includeWidth ) { which = cssExpand[ i ]; attrs[ "margin" + which ] = attrs[ "padding" + which ] = type; } if ( includeWidth ) { attrs.opacity = attrs.width = type; } return attrs; } function createTween( value, prop, animation ) { var tween, collection = ( Animation.tweeners[ prop ] || [] ).concat( Animation.tweeners[ "*" ] ), index = 0, length = collection.length; for ( ; index < length; index++ ) { if ( ( tween = collection[ index ].call( animation, prop, value ) ) ) { // We're done with this property return tween; } } } function defaultPrefilter( elem, props, opts ) { var prop, value, toggle, hooks, oldfire, propTween, restoreDisplay, display, isBox = "width" in props || "height" in props, anim = this, orig = {}, style = elem.style, hidden = elem.nodeType && isHiddenWithinTree( elem ), dataShow = dataPriv.get( elem, "fxshow" ); // Queue-skipping animations hijack the fx hooks if ( !opts.queue ) { hooks = jQuery._queueHooks( elem, "fx" ); if ( hooks.unqueued == null ) { hooks.unqueued = 0; oldfire = hooks.empty.fire; hooks.empty.fire = function() { if ( !hooks.unqueued ) { oldfire(); } }; } hooks.unqueued++; anim.always( function() { // Ensure the complete handler is called before this completes anim.always( function() { hooks.unqueued--; if ( !jQuery.queue( elem, "fx" ).length ) { hooks.empty.fire(); } } ); } ); } // Detect show/hide animations for ( prop in props ) { value = props[ prop ]; if ( rfxtypes.test( value ) ) { delete props[ prop ]; toggle = toggle || value === "toggle"; if ( value === ( hidden ? "hide" : "show" ) ) { // Pretend to be hidden if this is a "show" and // there is still data from a stopped show/hide if ( value === "show" && dataShow && dataShow[ prop ] !== undefined ) { hidden = true; // Ignore all other no-op show/hide data } else { continue; } } orig[ prop ] = dataShow && dataShow[ prop ] || jQuery.style( elem, prop ); } } // Bail out if this is a no-op like .hide().hide() propTween = !jQuery.isEmptyObject( props ); if ( !propTween && jQuery.isEmptyObject( orig ) ) { return; } // Restrict "overflow" and "display" styles during box animations if ( isBox && elem.nodeType === 1 ) { // Support: IE <=9 - 11, Edge 12 - 15 // Record all 3 overflow attributes because IE does not infer the shorthand // from identically-valued overflowX and overflowY and Edge just mirrors // the overflowX value there. opts.overflow = [ style.overflow, style.overflowX, style.overflowY ]; // Identify a display type, preferring old show/hide data over the CSS cascade restoreDisplay = dataShow && dataShow.display; if ( restoreDisplay == null ) { restoreDisplay = dataPriv.get( elem, "display" ); } display = jQuery.css( elem, "display" ); if ( display === "none" ) { if ( restoreDisplay ) { display = restoreDisplay; } else { // Get nonempty value(s) by temporarily forcing visibility showHide( [ elem ], true ); restoreDisplay = elem.style.display || restoreDisplay; display = jQuery.css( elem, "display" ); showHide( [ elem ] ); } } // Animate inline elements as inline-block if ( display === "inline" || display === "inline-block" && restoreDisplay != null ) { if ( jQuery.css( elem, "float" ) === "none" ) { // Restore the original display value at the end of pure show/hide animations if ( !propTween ) { anim.done( function() { style.display = restoreDisplay; } ); if ( restoreDisplay == null ) { display = style.display; restoreDisplay = display === "none" ? "" : display; } } style.display = "inline-block"; } } } if ( opts.overflow ) { style.overflow = "hidden"; anim.always( function() { style.overflow = opts.overflow[ 0 ]; style.overflowX = opts.overflow[ 1 ]; style.overflowY = opts.overflow[ 2 ]; } ); } // Implement show/hide animations propTween = false; for ( prop in orig ) { // General show/hide setup for this element animation if ( !propTween ) { if ( dataShow ) { if ( "hidden" in dataShow ) { hidden = dataShow.hidden; } } else { dataShow = dataPriv.access( elem, "fxshow", { display: restoreDisplay } ); } // Store hidden/visible for toggle so `.stop().toggle()` "reverses" if ( toggle ) { dataShow.hidden = !hidden; } // Show elements before animating them if ( hidden ) { showHide( [ elem ], true ); } /* eslint-disable no-loop-func */ anim.done( function() { /* eslint-enable no-loop-func */ // The final step of a "hide" animation is actually hiding the element if ( !hidden ) { showHide( [ elem ] ); } dataPriv.remove( elem, "fxshow" ); for ( prop in orig ) { jQuery.style( elem, prop, orig[ prop ] ); } } ); } // Per-property setup propTween = createTween( hidden ? dataShow[ prop ] : 0, prop, anim ); if ( !( prop in dataShow ) ) { dataShow[ prop ] = propTween.start; if ( hidden ) { propTween.end = propTween.start; propTween.start = 0; } } } } function propFilter( props, specialEasing ) { var index, name, easing, value, hooks; // camelCase, specialEasing and expand cssHook pass for ( index in props ) { name = camelCase( index ); easing = specialEasing[ name ]; value = props[ index ]; if ( Array.isArray( value ) ) { easing = value[ 1 ]; value = props[ index ] = value[ 0 ]; } if ( index !== name ) { props[ name ] = value; delete props[ index ]; } hooks = jQuery.cssHooks[ name ]; if ( hooks && "expand" in hooks ) { value = hooks.expand( value ); delete props[ name ]; // Not quite $.extend, this won't overwrite existing keys. // Reusing 'index' because we have the correct "name" for ( index in value ) { if ( !( index in props ) ) { props[ index ] = value[ index ]; specialEasing[ index ] = easing; } } } else { specialEasing[ name ] = easing; } } } function Animation( elem, properties, options ) { var result, stopped, index = 0, length = Animation.prefilters.length, deferred = jQuery.Deferred().always( function() { // Don't match elem in the :animated selector delete tick.elem; } ), tick = function() { if ( stopped ) { return false; } var currentTime = fxNow || createFxNow(), remaining = Math.max( 0, animation.startTime + animation.duration - currentTime ), // Support: Android 2.3 only // Archaic crash bug won't allow us to use `1 - ( 0.5 || 0 )` (#12497) temp = remaining / animation.duration || 0, percent = 1 - temp, index = 0, length = animation.tweens.length; for ( ; index < length; index++ ) { animation.tweens[ index ].run( percent ); } deferred.notifyWith( elem, [ animation, percent, remaining ] ); // If there's more to do, yield if ( percent < 1 && length ) { return remaining; } // If this was an empty animation, synthesize a final progress notification if ( !length ) { deferred.notifyWith( elem, [ animation, 1, 0 ] ); } // Resolve the animation and report its conclusion deferred.resolveWith( elem, [ animation ] ); return false; }, animation = deferred.promise( { elem: elem, props: jQuery.extend( {}, properties ), opts: jQuery.extend( true, { specialEasing: {}, easing: jQuery.easing._default }, options ), originalProperties: properties, originalOptions: options, startTime: fxNow || createFxNow(), duration: options.duration, tweens: [], createTween: function( prop, end ) { var tween = jQuery.Tween( elem, animation.opts, prop, end, animation.opts.specialEasing[ prop ] || animation.opts.easing ); animation.tweens.push( tween ); return tween; }, stop: function( gotoEnd ) { var index = 0, // If we are going to the end, we want to run all the tweens // otherwise we skip this part length = gotoEnd ? animation.tweens.length : 0; if ( stopped ) { return this; } stopped = true; for ( ; index < length; index++ ) { animation.tweens[ index ].run( 1 ); } // Resolve when we played the last frame; otherwise, reject if ( gotoEnd ) { deferred.notifyWith( elem, [ animation, 1, 0 ] ); deferred.resolveWith( elem, [ animation, gotoEnd ] ); } else { deferred.rejectWith( elem, [ animation, gotoEnd ] ); } return this; } } ), props = animation.props; propFilter( props, animation.opts.specialEasing ); for ( ; index < length; index++ ) { result = Animation.prefilters[ index ].call( animation, elem, props, animation.opts ); if ( result ) { if ( isFunction( result.stop ) ) { jQuery._queueHooks( animation.elem, animation.opts.queue ).stop = result.stop.bind( result ); } return result; } } jQuery.map( props, createTween, animation ); if ( isFunction( animation.opts.start ) ) { animation.opts.start.call( elem, animation ); } // Attach callbacks from options animation .progress( animation.opts.progress ) .done( animation.opts.done, animation.opts.complete ) .fail( animation.opts.fail ) .always( animation.opts.always ); jQuery.fx.timer( jQuery.extend( tick, { elem: elem, anim: animation, queue: animation.opts.queue } ) ); return animation; } jQuery.Animation = jQuery.extend( Animation, { tweeners: { "*": [ function( prop, value ) { var tween = this.createTween( prop, value ); adjustCSS( tween.elem, prop, rcssNum.exec( value ), tween ); return tween; } ] }, tweener: function( props, callback ) { if ( isFunction( props ) ) { callback = props; props = [ "*" ]; } else { props = props.match( rnothtmlwhite ); } var prop, index = 0, length = props.length; for ( ; index < length; index++ ) { prop = props[ index ]; Animation.tweeners[ prop ] = Animation.tweeners[ prop ] || []; Animation.tweeners[ prop ].unshift( callback ); } }, prefilters: [ defaultPrefilter ], prefilter: function( callback, prepend ) { if ( prepend ) { Animation.prefilters.unshift( callback ); } else { Animation.prefilters.push( callback ); } } } ); jQuery.speed = function( speed, easing, fn ) { var opt = speed && typeof speed === "object" ? jQuery.extend( {}, speed ) : { complete: fn || !fn && easing || isFunction( speed ) && speed, duration: speed, easing: fn && easing || easing && !isFunction( easing ) && easing }; // Go to the end state if fx are off if ( jQuery.fx.off ) { opt.duration = 0; } else { if ( typeof opt.duration !== "number" ) { if ( opt.duration in jQuery.fx.speeds ) { opt.duration = jQuery.fx.speeds[ opt.duration ]; } else { opt.duration = jQuery.fx.speeds._default; } } } // Normalize opt.queue - true/undefined/null -> "fx" if ( opt.queue == null || opt.queue === true ) { opt.queue = "fx"; } // Queueing opt.old = opt.complete; opt.complete = function() { if ( isFunction( opt.old ) ) { opt.old.call( this ); } if ( opt.queue ) { jQuery.dequeue( this, opt.queue ); } }; return opt; }; jQuery.fn.extend( { fadeTo: function( speed, to, easing, callback ) { // Show any hidden elements after setting opacity to 0 return this.filter( isHiddenWithinTree ).css( "opacity", 0 ).show() // Animate to the value specified .end().animate( { opacity: to }, speed, easing, callback ); }, animate: function( prop, speed, easing, callback ) { var empty = jQuery.isEmptyObject( prop ), optall = jQuery.speed( speed, easing, callback ), doAnimation = function() { // Operate on a copy of prop so per-property easing won't be lost var anim = Animation( this, jQuery.extend( {}, prop ), optall ); // Empty animations, or finishing resolves immediately if ( empty || dataPriv.get( this, "finish" ) ) { anim.stop( true ); } }; doAnimation.finish = doAnimation; return empty || optall.queue === false ? this.each( doAnimation ) : this.queue( optall.queue, doAnimation ); }, stop: function( type, clearQueue, gotoEnd ) { var stopQueue = function( hooks ) { var stop = hooks.stop; delete hooks.stop; stop( gotoEnd ); }; if ( typeof type !== "string" ) { gotoEnd = clearQueue; clearQueue = type; type = undefined; } if ( clearQueue ) { this.queue( type || "fx", [] ); } return this.each( function() { var dequeue = true, index = type != null && type + "queueHooks", timers = jQuery.timers, data = dataPriv.get( this ); if ( index ) { if ( data[ index ] && data[ index ].stop ) { stopQueue( data[ index ] ); } } else { for ( index in data ) { if ( data[ index ] && data[ index ].stop && rrun.test( index ) ) { stopQueue( data[ index ] ); } } } for ( index = timers.length; index--; ) { if ( timers[ index ].elem === this && ( type == null || timers[ index ].queue === type ) ) { timers[ index ].anim.stop( gotoEnd ); dequeue = false; timers.splice( index, 1 ); } } // Start the next in the queue if the last step wasn't forced. // Timers currently will call their complete callbacks, which // will dequeue but only if they were gotoEnd. if ( dequeue || !gotoEnd ) { jQuery.dequeue( this, type ); } } ); }, finish: function( type ) { if ( type !== false ) { type = type || "fx"; } return this.each( function() { var index, data = dataPriv.get( this ), queue = data[ type + "queue" ], hooks = data[ type + "queueHooks" ], timers = jQuery.timers, length = queue ? queue.length : 0; // Enable finishing flag on private data data.finish = true; // Empty the queue first jQuery.queue( this, type, [] ); if ( hooks && hooks.stop ) { hooks.stop.call( this, true ); } // Look for any active animations, and finish them for ( index = timers.length; index--; ) { if ( timers[ index ].elem === this && timers[ index ].queue === type ) { timers[ index ].anim.stop( true ); timers.splice( index, 1 ); } } // Look for any animations in the old queue and finish them for ( index = 0; index < length; index++ ) { if ( queue[ index ] && queue[ index ].finish ) { queue[ index ].finish.call( this ); } } // Turn off finishing flag delete data.finish; } ); } } ); jQuery.each( [ "toggle", "show", "hide" ], function( _i, name ) { var cssFn = jQuery.fn[ name ]; jQuery.fn[ name ] = function( speed, easing, callback ) { return speed == null || typeof speed === "boolean" ? cssFn.apply( this, arguments ) : this.animate( genFx( name, true ), speed, easing, callback ); }; } ); // Generate shortcuts for custom animations jQuery.each( { slideDown: genFx( "show" ), slideUp: genFx( "hide" ), slideToggle: genFx( "toggle" ), fadeIn: { opacity: "show" }, fadeOut: { opacity: "hide" }, fadeToggle: { opacity: "toggle" } }, function( name, props ) { jQuery.fn[ name ] = function( speed, easing, callback ) { return this.animate( props, speed, easing, callback ); }; } ); jQuery.timers = []; jQuery.fx.tick = function() { var timer, i = 0, timers = jQuery.timers; fxNow = Date.now(); for ( ; i < timers.length; i++ ) { timer = timers[ i ]; // Run the timer and safely remove it when done (allowing for external removal) if ( !timer() && timers[ i ] === timer ) { timers.splice( i--, 1 ); } } if ( !timers.length ) { jQuery.fx.stop(); } fxNow = undefined; }; jQuery.fx.timer = function( timer ) { jQuery.timers.push( timer ); jQuery.fx.start(); }; jQuery.fx.interval = 13; jQuery.fx.start = function() { if ( inProgress ) { return; } inProgress = true; schedule(); }; jQuery.fx.stop = function() { inProgress = null; }; jQuery.fx.speeds = { slow: 600, fast: 200, // Default speed _default: 400 }; // Based off of the plugin by Clint Helfers, with permission. // https://web.archive.org/web/20100324014747/http://blindsignals.com/index.php/2009/07/jquery-delay/ jQuery.fn.delay = function( time, type ) { time = jQuery.fx ? jQuery.fx.speeds[ time ] || time : time; type = type || "fx"; return this.queue( type, function( next, hooks ) { var timeout = window.setTimeout( next, time ); hooks.stop = function() { window.clearTimeout( timeout ); }; } ); }; ( function() { var input = document.createElement( "input" ), select = document.createElement( "select" ), opt = select.appendChild( document.createElement( "option" ) ); input.type = "checkbox"; // Support: Android <=4.3 only // Default value for a checkbox should be "on" support.checkOn = input.value !== ""; // Support: IE <=11 only // Must access selectedIndex to make default options select support.optSelected = opt.selected; // Support: IE <=11 only // An input loses its value after becoming a radio input = document.createElement( "input" ); input.value = "t"; input.type = "radio"; support.radioValue = input.value === "t"; } )(); var boolHook, attrHandle = jQuery.expr.attrHandle; jQuery.fn.extend( { attr: function( name, value ) { return access( this, jQuery.attr, name, value, arguments.length > 1 ); }, removeAttr: function( name ) { return this.each( function() { jQuery.removeAttr( this, name ); } ); } } ); jQuery.extend( { attr: function( elem, name, value ) { var ret, hooks, nType = elem.nodeType; // Don't get/set attributes on text, comment and attribute nodes if ( nType === 3 || nType === 8 || nType === 2 ) { return; } // Fallback to prop when attributes are not supported if ( typeof elem.getAttribute === "undefined" ) { return jQuery.prop( elem, name, value ); } // Attribute hooks are determined by the lowercase version // Grab necessary hook if one is defined if ( nType !== 1 || !jQuery.isXMLDoc( elem ) ) { hooks = jQuery.attrHooks[ name.toLowerCase() ] || ( jQuery.expr.match.bool.test( name ) ? boolHook : undefined ); } if ( value !== undefined ) { if ( value === null ) { jQuery.removeAttr( elem, name ); return; } if ( hooks && "set" in hooks && ( ret = hooks.set( elem, value, name ) ) !== undefined ) { return ret; } elem.setAttribute( name, value + "" ); return value; } if ( hooks && "get" in hooks && ( ret = hooks.get( elem, name ) ) !== null ) { return ret; } ret = jQuery.find.attr( elem, name ); // Non-existent attributes return null, we normalize to undefined return ret == null ? undefined : ret; }, attrHooks: { type: { set: function( elem, value ) { if ( !support.radioValue && value === "radio" && nodeName( elem, "input" ) ) { var val = elem.value; elem.setAttribute( "type", value ); if ( val ) { elem.value = val; } return value; } } } }, removeAttr: function( elem, value ) { var name, i = 0, // Attribute names can contain non-HTML whitespace characters // https://html.spec.whatwg.org/multipage/syntax.html#attributes-2 attrNames = value && value.match( rnothtmlwhite ); if ( attrNames && elem.nodeType === 1 ) { while ( ( name = attrNames[ i++ ] ) ) { elem.removeAttribute( name ); } } } } ); // Hooks for boolean attributes boolHook = { set: function( elem, value, name ) { if ( value === false ) { // Remove boolean attributes when set to false jQuery.removeAttr( elem, name ); } else { elem.setAttribute( name, name ); } return name; } }; jQuery.each( jQuery.expr.match.bool.source.match( /\w+/g ), function( _i, name ) { var getter = attrHandle[ name ] || jQuery.find.attr; attrHandle[ name ] = function( elem, name, isXML ) { var ret, handle, lowercaseName = name.toLowerCase(); if ( !isXML ) { // Avoid an infinite loop by temporarily removing this function from the getter handle = attrHandle[ lowercaseName ]; attrHandle[ lowercaseName ] = ret; ret = getter( elem, name, isXML ) != null ? lowercaseName : null; attrHandle[ lowercaseName ] = handle; } return ret; }; } ); var rfocusable = /^(?:input|select|textarea|button)$/i, rclickable = /^(?:a|area)$/i; jQuery.fn.extend( { prop: function( name, value ) { return access( this, jQuery.prop, name, value, arguments.length > 1 ); }, removeProp: function( name ) { return this.each( function() { delete this[ jQuery.propFix[ name ] || name ]; } ); } } ); jQuery.extend( { prop: function( elem, name, value ) { var ret, hooks, nType = elem.nodeType; // Don't get/set properties on text, comment and attribute nodes if ( nType === 3 || nType === 8 || nType === 2 ) { return; } if ( nType !== 1 || !jQuery.isXMLDoc( elem ) ) { // Fix name and attach hooks name = jQuery.propFix[ name ] || name; hooks = jQuery.propHooks[ name ]; } if ( value !== undefined ) { if ( hooks && "set" in hooks && ( ret = hooks.set( elem, value, name ) ) !== undefined ) { return ret; } return ( elem[ name ] = value ); } if ( hooks && "get" in hooks && ( ret = hooks.get( elem, name ) ) !== null ) { return ret; } return elem[ name ]; }, propHooks: { tabIndex: { get: function( elem ) { // Support: IE <=9 - 11 only // elem.tabIndex doesn't always return the // correct value when it hasn't been explicitly set // https://web.archive.org/web/20141116233347/http://fluidproject.org/blog/2008/01/09/getting-setting-and-removing-tabindex-values-with-javascript/ // Use proper attribute retrieval(#12072) var tabindex = jQuery.find.attr( elem, "tabindex" ); if ( tabindex ) { return parseInt( tabindex, 10 ); } if ( rfocusable.test( elem.nodeName ) || rclickable.test( elem.nodeName ) && elem.href ) { return 0; } return -1; } } }, propFix: { "for": "htmlFor", "class": "className" } } ); // Support: IE <=11 only // Accessing the selectedIndex property // forces the browser to respect setting selected // on the option // The getter ensures a default option is selected // when in an optgroup // eslint rule "no-unused-expressions" is disabled for this code // since it considers such accessions noop if ( !support.optSelected ) { jQuery.propHooks.selected = { get: function( elem ) { /* eslint no-unused-expressions: "off" */ var parent = elem.parentNode; if ( parent && parent.parentNode ) { parent.parentNode.selectedIndex; } return null; }, set: function( elem ) { /* eslint no-unused-expressions: "off" */ var parent = elem.parentNode; if ( parent ) { parent.selectedIndex; if ( parent.parentNode ) { parent.parentNode.selectedIndex; } } } }; } jQuery.each( [ "tabIndex", "readOnly", "maxLength", "cellSpacing", "cellPadding", "rowSpan", "colSpan", "useMap", "frameBorder", "contentEditable" ], function() { jQuery.propFix[ this.toLowerCase() ] = this; } ); // Strip and collapse whitespace according to HTML spec // https://infra.spec.whatwg.org/#strip-and-collapse-ascii-whitespace function stripAndCollapse( value ) { var tokens = value.match( rnothtmlwhite ) || []; return tokens.join( " " ); } function getClass( elem ) { return elem.getAttribute && elem.getAttribute( "class" ) || ""; } function classesToArray( value ) { if ( Array.isArray( value ) ) { return value; } if ( typeof value === "string" ) { return value.match( rnothtmlwhite ) || []; } return []; } jQuery.fn.extend( { addClass: function( value ) { var classes, elem, cur, curValue, clazz, j, finalValue, i = 0; if ( isFunction( value ) ) { return this.each( function( j ) { jQuery( this ).addClass( value.call( this, j, getClass( this ) ) ); } ); } classes = classesToArray( value ); if ( classes.length ) { while ( ( elem = this[ i++ ] ) ) { curValue = getClass( elem ); cur = elem.nodeType === 1 && ( " " + stripAndCollapse( curValue ) + " " ); if ( cur ) { j = 0; while ( ( clazz = classes[ j++ ] ) ) { if ( cur.indexOf( " " + clazz + " " ) < 0 ) { cur += clazz + " "; } } // Only assign if different to avoid unneeded rendering. finalValue = stripAndCollapse( cur ); if ( curValue !== finalValue ) { elem.setAttribute( "class", finalValue ); } } } } return this; }, removeClass: function( value ) { var classes, elem, cur, curValue, clazz, j, finalValue, i = 0; if ( isFunction( value ) ) { return this.each( function( j ) { jQuery( this ).removeClass( value.call( this, j, getClass( this ) ) ); } ); } if ( !arguments.length ) { return this.attr( "class", "" ); } classes = classesToArray( value ); if ( classes.length ) { while ( ( elem = this[ i++ ] ) ) { curValue = getClass( elem ); // This expression is here for better compressibility (see addClass) cur = elem.nodeType === 1 && ( " " + stripAndCollapse( curValue ) + " " ); if ( cur ) { j = 0; while ( ( clazz = classes[ j++ ] ) ) { // Remove *all* instances while ( cur.indexOf( " " + clazz + " " ) > -1 ) { cur = cur.replace( " " + clazz + " ", " " ); } } // Only assign if different to avoid unneeded rendering. finalValue = stripAndCollapse( cur ); if ( curValue !== finalValue ) { elem.setAttribute( "class", finalValue ); } } } } return this; }, toggleClass: function( value, stateVal ) { var type = typeof value, isValidValue = type === "string" || Array.isArray( value ); if ( typeof stateVal === "boolean" && isValidValue ) { return stateVal ? this.addClass( value ) : this.removeClass( value ); } if ( isFunction( value ) ) { return this.each( function( i ) { jQuery( this ).toggleClass( value.call( this, i, getClass( this ), stateVal ), stateVal ); } ); } return this.each( function() { var className, i, self, classNames; if ( isValidValue ) { // Toggle individual class names i = 0; self = jQuery( this ); classNames = classesToArray( value ); while ( ( className = classNames[ i++ ] ) ) { // Check each className given, space separated list if ( self.hasClass( className ) ) { self.removeClass( className ); } else { self.addClass( className ); } } // Toggle whole class name } else if ( value === undefined || type === "boolean" ) { className = getClass( this ); if ( className ) { // Store className if set dataPriv.set( this, "__className__", className ); } // If the element has a class name or if we're passed `false`, // then remove the whole classname (if there was one, the above saved it). // Otherwise bring back whatever was previously saved (if anything), // falling back to the empty string if nothing was stored. if ( this.setAttribute ) { this.setAttribute( "class", className || value === false ? "" : dataPriv.get( this, "__className__" ) || "" ); } } } ); }, hasClass: function( selector ) { var className, elem, i = 0; className = " " + selector + " "; while ( ( elem = this[ i++ ] ) ) { if ( elem.nodeType === 1 && ( " " + stripAndCollapse( getClass( elem ) ) + " " ).indexOf( className ) > -1 ) { return true; } } return false; } } ); var rreturn = /\r/g; jQuery.fn.extend( { val: function( value ) { var hooks, ret, valueIsFunction, elem = this[ 0 ]; if ( !arguments.length ) { if ( elem ) { hooks = jQuery.valHooks[ elem.type ] || jQuery.valHooks[ elem.nodeName.toLowerCase() ]; if ( hooks && "get" in hooks && ( ret = hooks.get( elem, "value" ) ) !== undefined ) { return ret; } ret = elem.value; // Handle most common string cases if ( typeof ret === "string" ) { return ret.replace( rreturn, "" ); } // Handle cases where value is null/undef or number return ret == null ? "" : ret; } return; } valueIsFunction = isFunction( value ); return this.each( function( i ) { var val; if ( this.nodeType !== 1 ) { return; } if ( valueIsFunction ) { val = value.call( this, i, jQuery( this ).val() ); } else { val = value; } // Treat null/undefined as ""; convert numbers to string if ( val == null ) { val = ""; } else if ( typeof val === "number" ) { val += ""; } else if ( Array.isArray( val ) ) { val = jQuery.map( val, function( value ) { return value == null ? "" : value + ""; } ); } hooks = jQuery.valHooks[ this.type ] || jQuery.valHooks[ this.nodeName.toLowerCase() ]; // If set returns undefined, fall back to normal setting if ( !hooks || !( "set" in hooks ) || hooks.set( this, val, "value" ) === undefined ) { this.value = val; } } ); } } ); jQuery.extend( { valHooks: { option: { get: function( elem ) { var val = jQuery.find.attr( elem, "value" ); return val != null ? val : // Support: IE <=10 - 11 only // option.text throws exceptions (#14686, #14858) // Strip and collapse whitespace // https://html.spec.whatwg.org/#strip-and-collapse-whitespace stripAndCollapse( jQuery.text( elem ) ); } }, select: { get: function( elem ) { var value, option, i, options = elem.options, index = elem.selectedIndex, one = elem.type === "select-one", values = one ? null : [], max = one ? index + 1 : options.length; if ( index < 0 ) { i = max; } else { i = one ? index : 0; } // Loop through all the selected options for ( ; i < max; i++ ) { option = options[ i ]; // Support: IE <=9 only // IE8-9 doesn't update selected after form reset (#2551) if ( ( option.selected || i === index ) && // Don't return options that are disabled or in a disabled optgroup !option.disabled && ( !option.parentNode.disabled || !nodeName( option.parentNode, "optgroup" ) ) ) { // Get the specific value for the option value = jQuery( option ).val(); // We don't need an array for one selects if ( one ) { return value; } // Multi-Selects return an array values.push( value ); } } return values; }, set: function( elem, value ) { var optionSet, option, options = elem.options, values = jQuery.makeArray( value ), i = options.length; while ( i-- ) { option = options[ i ]; /* eslint-disable no-cond-assign */ if ( option.selected = jQuery.inArray( jQuery.valHooks.option.get( option ), values ) > -1 ) { optionSet = true; } /* eslint-enable no-cond-assign */ } // Force browsers to behave consistently when non-matching value is set if ( !optionSet ) { elem.selectedIndex = -1; } return values; } } } } ); // Radios and checkboxes getter/setter jQuery.each( [ "radio", "checkbox" ], function() { jQuery.valHooks[ this ] = { set: function( elem, value ) { if ( Array.isArray( value ) ) { return ( elem.checked = jQuery.inArray( jQuery( elem ).val(), value ) > -1 ); } } }; if ( !support.checkOn ) { jQuery.valHooks[ this ].get = function( elem ) { return elem.getAttribute( "value" ) === null ? "on" : elem.value; }; } } ); // Return jQuery for attributes-only inclusion support.focusin = "onfocusin" in window; var rfocusMorph = /^(?:focusinfocus|focusoutblur)$/, stopPropagationCallback = function( e ) { e.stopPropagation(); }; jQuery.extend( jQuery.event, { trigger: function( event, data, elem, onlyHandlers ) { var i, cur, tmp, bubbleType, ontype, handle, special, lastElement, eventPath = [ elem || document ], type = hasOwn.call( event, "type" ) ? event.type : event, namespaces = hasOwn.call( event, "namespace" ) ? event.namespace.split( "." ) : []; cur = lastElement = tmp = elem = elem || document; // Don't do events on text and comment nodes if ( elem.nodeType === 3 || elem.nodeType === 8 ) { return; } // focus/blur morphs to focusin/out; ensure we're not firing them right now if ( rfocusMorph.test( type + jQuery.event.triggered ) ) { return; } if ( type.indexOf( "." ) > -1 ) { // Namespaced trigger; create a regexp to match event type in handle() namespaces = type.split( "." ); type = namespaces.shift(); namespaces.sort(); } ontype = type.indexOf( ":" ) < 0 && "on" + type; // Caller can pass in a jQuery.Event object, Object, or just an event type string event = event[ jQuery.expando ] ? event : new jQuery.Event( type, typeof event === "object" && event ); // Trigger bitmask: & 1 for native handlers; & 2 for jQuery (always true) event.isTrigger = onlyHandlers ? 2 : 3; event.namespace = namespaces.join( "." ); event.rnamespace = event.namespace ? new RegExp( "(^|\\.)" + namespaces.join( "\\.(?:.*\\.|)" ) + "(\\.|$)" ) : null; // Clean up the event in case it is being reused event.result = undefined; if ( !event.target ) { event.target = elem; } // Clone any incoming data and prepend the event, creating the handler arg list data = data == null ? [ event ] : jQuery.makeArray( data, [ event ] ); // Allow special events to draw outside the lines special = jQuery.event.special[ type ] || {}; if ( !onlyHandlers && special.trigger && special.trigger.apply( elem, data ) === false ) { return; } // Determine event propagation path in advance, per W3C events spec (#9951) // Bubble up to document, then to window; watch for a global ownerDocument var (#9724) if ( !onlyHandlers && !special.noBubble && !isWindow( elem ) ) { bubbleType = special.delegateType || type; if ( !rfocusMorph.test( bubbleType + type ) ) { cur = cur.parentNode; } for ( ; cur; cur = cur.parentNode ) { eventPath.push( cur ); tmp = cur; } // Only add window if we got to document (e.g., not plain obj or detached DOM) if ( tmp === ( elem.ownerDocument || document ) ) { eventPath.push( tmp.defaultView || tmp.parentWindow || window ); } } // Fire handlers on the event path i = 0; while ( ( cur = eventPath[ i++ ] ) && !event.isPropagationStopped() ) { lastElement = cur; event.type = i > 1 ? bubbleType : special.bindType || type; // jQuery handler handle = ( dataPriv.get( cur, "events" ) || Object.create( null ) )[ event.type ] && dataPriv.get( cur, "handle" ); if ( handle ) { handle.apply( cur, data ); } // Native handler handle = ontype && cur[ ontype ]; if ( handle && handle.apply && acceptData( cur ) ) { event.result = handle.apply( cur, data ); if ( event.result === false ) { event.preventDefault(); } } } event.type = type; // If nobody prevented the default action, do it now if ( !onlyHandlers && !event.isDefaultPrevented() ) { if ( ( !special._default || special._default.apply( eventPath.pop(), data ) === false ) && acceptData( elem ) ) { // Call a native DOM method on the target with the same name as the event. // Don't do default actions on window, that's where global variables be (#6170) if ( ontype && isFunction( elem[ type ] ) && !isWindow( elem ) ) { // Don't re-trigger an onFOO event when we call its FOO() method tmp = elem[ ontype ]; if ( tmp ) { elem[ ontype ] = null; } // Prevent re-triggering of the same event, since we already bubbled it above jQuery.event.triggered = type; if ( event.isPropagationStopped() ) { lastElement.addEventListener( type, stopPropagationCallback ); } elem[ type ](); if ( event.isPropagationStopped() ) { lastElement.removeEventListener( type, stopPropagationCallback ); } jQuery.event.triggered = undefined; if ( tmp ) { elem[ ontype ] = tmp; } } } } return event.result; }, // Piggyback on a donor event to simulate a different one // Used only for `focus(in | out)` events simulate: function( type, elem, event ) { var e = jQuery.extend( new jQuery.Event(), event, { type: type, isSimulated: true } ); jQuery.event.trigger( e, null, elem ); } } ); jQuery.fn.extend( { trigger: function( type, data ) { return this.each( function() { jQuery.event.trigger( type, data, this ); } ); }, triggerHandler: function( type, data ) { var elem = this[ 0 ]; if ( elem ) { return jQuery.event.trigger( type, data, elem, true ); } } } ); // Support: Firefox <=44 // Firefox doesn't have focus(in | out) events // Related ticket - https://bugzilla.mozilla.org/show_bug.cgi?id=687787 // // Support: Chrome <=48 - 49, Safari <=9.0 - 9.1 // focus(in | out) events fire after focus & blur events, // which is spec violation - http://www.w3.org/TR/DOM-Level-3-Events/#events-focusevent-event-order // Related ticket - https://bugs.chromium.org/p/chromium/issues/detail?id=449857 if ( !support.focusin ) { jQuery.each( { focus: "focusin", blur: "focusout" }, function( orig, fix ) { // Attach a single capturing handler on the document while someone wants focusin/focusout var handler = function( event ) { jQuery.event.simulate( fix, event.target, jQuery.event.fix( event ) ); }; jQuery.event.special[ fix ] = { setup: function() { // Handle: regular nodes (via `this.ownerDocument`), window // (via `this.document`) & document (via `this`). var doc = this.ownerDocument || this.document || this, attaches = dataPriv.access( doc, fix ); if ( !attaches ) { doc.addEventListener( orig, handler, true ); } dataPriv.access( doc, fix, ( attaches || 0 ) + 1 ); }, teardown: function() { var doc = this.ownerDocument || this.document || this, attaches = dataPriv.access( doc, fix ) - 1; if ( !attaches ) { doc.removeEventListener( orig, handler, true ); dataPriv.remove( doc, fix ); } else { dataPriv.access( doc, fix, attaches ); } } }; } ); } var location = window.location; var nonce = { guid: Date.now() }; var rquery = ( /\?/ ); // Cross-browser xml parsing jQuery.parseXML = function( data ) { var xml, parserErrorElem; if ( !data || typeof data !== "string" ) { return null; } // Support: IE 9 - 11 only // IE throws on parseFromString with invalid input. try { xml = ( new window.DOMParser() ).parseFromString( data, "text/xml" ); } catch ( e ) {} parserErrorElem = xml && xml.getElementsByTagName( "parsererror" )[ 0 ]; if ( !xml || parserErrorElem ) { jQuery.error( "Invalid XML: " + ( parserErrorElem ? jQuery.map( parserErrorElem.childNodes, function( el ) { return el.textContent; } ).join( "\n" ) : data ) ); } return xml; }; var rbracket = /\[\]$/, rCRLF = /\r?\n/g, rsubmitterTypes = /^(?:submit|button|image|reset|file)$/i, rsubmittable = /^(?:input|select|textarea|keygen)/i; function buildParams( prefix, obj, traditional, add ) { var name; if ( Array.isArray( obj ) ) { // Serialize array item. jQuery.each( obj, function( i, v ) { if ( traditional || rbracket.test( prefix ) ) { // Treat each array item as a scalar. add( prefix, v ); } else { // Item is non-scalar (array or object), encode its numeric index. buildParams( prefix + "[" + ( typeof v === "object" && v != null ? i : "" ) + "]", v, traditional, add ); } } ); } else if ( !traditional && toType( obj ) === "object" ) { // Serialize object item. for ( name in obj ) { buildParams( prefix + "[" + name + "]", obj[ name ], traditional, add ); } } else { // Serialize scalar item. add( prefix, obj ); } } // Serialize an array of form elements or a set of // key/values into a query string jQuery.param = function( a, traditional ) { var prefix, s = [], add = function( key, valueOrFunction ) { // If value is a function, invoke it and use its return value var value = isFunction( valueOrFunction ) ? valueOrFunction() : valueOrFunction; s[ s.length ] = encodeURIComponent( key ) + "=" + encodeURIComponent( value == null ? "" : value ); }; if ( a == null ) { return ""; } // If an array was passed in, assume that it is an array of form elements. if ( Array.isArray( a ) || ( a.jquery && !jQuery.isPlainObject( a ) ) ) { // Serialize the form elements jQuery.each( a, function() { add( this.name, this.value ); } ); } else { // If traditional, encode the "old" way (the way 1.3.2 or older // did it), otherwise encode params recursively. for ( prefix in a ) { buildParams( prefix, a[ prefix ], traditional, add ); } } // Return the resulting serialization return s.join( "&" ); }; jQuery.fn.extend( { serialize: function() { return jQuery.param( this.serializeArray() ); }, serializeArray: function() { return this.map( function() { // Can add propHook for "elements" to filter or add form elements var elements = jQuery.prop( this, "elements" ); return elements ? jQuery.makeArray( elements ) : this; } ).filter( function() { var type = this.type; // Use .is( ":disabled" ) so that fieldset[disabled] works return this.name && !jQuery( this ).is( ":disabled" ) && rsubmittable.test( this.nodeName ) && !rsubmitterTypes.test( type ) && ( this.checked || !rcheckableType.test( type ) ); } ).map( function( _i, elem ) { var val = jQuery( this ).val(); if ( val == null ) { return null; } if ( Array.isArray( val ) ) { return jQuery.map( val, function( val ) { return { name: elem.name, value: val.replace( rCRLF, "\r\n" ) }; } ); } return { name: elem.name, value: val.replace( rCRLF, "\r\n" ) }; } ).get(); } } ); var r20 = /%20/g, rhash = /#.*$/, rantiCache = /([?&])_=[^&]*/, rheaders = /^(.*?):[ \t]*([^\r\n]*)$/mg, // #7653, #8125, #8152: local protocol detection rlocalProtocol = /^(?:about|app|app-storage|.+-extension|file|res|widget):$/, rnoContent = /^(?:GET|HEAD)$/, rprotocol = /^\/\//, /* Prefilters * 1) They are useful to introduce custom dataTypes (see ajax/jsonp.js for an example) * 2) These are called: * - BEFORE asking for a transport * - AFTER param serialization (s.data is a string if s.processData is true) * 3) key is the dataType * 4) the catchall symbol "*" can be used * 5) execution will start with transport dataType and THEN continue down to "*" if needed */ prefilters = {}, /* Transports bindings * 1) key is the dataType * 2) the catchall symbol "*" can be used * 3) selection will start with transport dataType and THEN go to "*" if needed */ transports = {}, // Avoid comment-prolog char sequence (#10098); must appease lint and evade compression allTypes = "*/".concat( "*" ), // Anchor tag for parsing the document origin originAnchor = document.createElement( "a" ); originAnchor.href = location.href; // Base "constructor" for jQuery.ajaxPrefilter and jQuery.ajaxTransport function addToPrefiltersOrTransports( structure ) { // dataTypeExpression is optional and defaults to "*" return function( dataTypeExpression, func ) { if ( typeof dataTypeExpression !== "string" ) { func = dataTypeExpression; dataTypeExpression = "*"; } var dataType, i = 0, dataTypes = dataTypeExpression.toLowerCase().match( rnothtmlwhite ) || []; if ( isFunction( func ) ) { // For each dataType in the dataTypeExpression while ( ( dataType = dataTypes[ i++ ] ) ) { // Prepend if requested if ( dataType[ 0 ] === "+" ) { dataType = dataType.slice( 1 ) || "*"; ( structure[ dataType ] = structure[ dataType ] || [] ).unshift( func ); // Otherwise append } else { ( structure[ dataType ] = structure[ dataType ] || [] ).push( func ); } } } }; } // Base inspection function for prefilters and transports function inspectPrefiltersOrTransports( structure, options, originalOptions, jqXHR ) { var inspected = {}, seekingTransport = ( structure === transports ); function inspect( dataType ) { var selected; inspected[ dataType ] = true; jQuery.each( structure[ dataType ] || [], function( _, prefilterOrFactory ) { var dataTypeOrTransport = prefilterOrFactory( options, originalOptions, jqXHR ); if ( typeof dataTypeOrTransport === "string" && !seekingTransport && !inspected[ dataTypeOrTransport ] ) { options.dataTypes.unshift( dataTypeOrTransport ); inspect( dataTypeOrTransport ); return false; } else if ( seekingTransport ) { return !( selected = dataTypeOrTransport ); } } ); return selected; } return inspect( options.dataTypes[ 0 ] ) || !inspected[ "*" ] && inspect( "*" ); } // A special extend for ajax options // that takes "flat" options (not to be deep extended) // Fixes #9887 function ajaxExtend( target, src ) { var key, deep, flatOptions = jQuery.ajaxSettings.flatOptions || {}; for ( key in src ) { if ( src[ key ] !== undefined ) { ( flatOptions[ key ] ? target : ( deep || ( deep = {} ) ) )[ key ] = src[ key ]; } } if ( deep ) { jQuery.extend( true, target, deep ); } return target; } /* Handles responses to an ajax request: * - finds the right dataType (mediates between content-type and expected dataType) * - returns the corresponding response */ function ajaxHandleResponses( s, jqXHR, responses ) { var ct, type, finalDataType, firstDataType, contents = s.contents, dataTypes = s.dataTypes; // Remove auto dataType and get content-type in the process while ( dataTypes[ 0 ] === "*" ) { dataTypes.shift(); if ( ct === undefined ) { ct = s.mimeType || jqXHR.getResponseHeader( "Content-Type" ); } } // Check if we're dealing with a known content-type if ( ct ) { for ( type in contents ) { if ( contents[ type ] && contents[ type ].test( ct ) ) { dataTypes.unshift( type ); break; } } } // Check to see if we have a response for the expected dataType if ( dataTypes[ 0 ] in responses ) { finalDataType = dataTypes[ 0 ]; } else { // Try convertible dataTypes for ( type in responses ) { if ( !dataTypes[ 0 ] || s.converters[ type + " " + dataTypes[ 0 ] ] ) { finalDataType = type; break; } if ( !firstDataType ) { firstDataType = type; } } // Or just use first one finalDataType = finalDataType || firstDataType; } // If we found a dataType // We add the dataType to the list if needed // and return the corresponding response if ( finalDataType ) { if ( finalDataType !== dataTypes[ 0 ] ) { dataTypes.unshift( finalDataType ); } return responses[ finalDataType ]; } } /* Chain conversions given the request and the original response * Also sets the responseXXX fields on the jqXHR instance */ function ajaxConvert( s, response, jqXHR, isSuccess ) { var conv2, current, conv, tmp, prev, converters = {}, // Work with a copy of dataTypes in case we need to modify it for conversion dataTypes = s.dataTypes.slice(); // Create converters map with lowercased keys if ( dataTypes[ 1 ] ) { for ( conv in s.converters ) { converters[ conv.toLowerCase() ] = s.converters[ conv ]; } } current = dataTypes.shift(); // Convert to each sequential dataType while ( current ) { if ( s.responseFields[ current ] ) { jqXHR[ s.responseFields[ current ] ] = response; } // Apply the dataFilter if provided if ( !prev && isSuccess && s.dataFilter ) { response = s.dataFilter( response, s.dataType ); } prev = current; current = dataTypes.shift(); if ( current ) { // There's only work to do if current dataType is non-auto if ( current === "*" ) { current = prev; // Convert response if prev dataType is non-auto and differs from current } else if ( prev !== "*" && prev !== current ) { // Seek a direct converter conv = converters[ prev + " " + current ] || converters[ "* " + current ]; // If none found, seek a pair if ( !conv ) { for ( conv2 in converters ) { // If conv2 outputs current tmp = conv2.split( " " ); if ( tmp[ 1 ] === current ) { // If prev can be converted to accepted input conv = converters[ prev + " " + tmp[ 0 ] ] || converters[ "* " + tmp[ 0 ] ]; if ( conv ) { // Condense equivalence converters if ( conv === true ) { conv = converters[ conv2 ]; // Otherwise, insert the intermediate dataType } else if ( converters[ conv2 ] !== true ) { current = tmp[ 0 ]; dataTypes.unshift( tmp[ 1 ] ); } break; } } } } // Apply converter (if not an equivalence) if ( conv !== true ) { // Unless errors are allowed to bubble, catch and return them if ( conv && s.throws ) { response = conv( response ); } else { try { response = conv( response ); } catch ( e ) { return { state: "parsererror", error: conv ? e : "No conversion from " + prev + " to " + current }; } } } } } } return { state: "success", data: response }; } jQuery.extend( { // Counter for holding the number of active queries active: 0, // Last-Modified header cache for next request lastModified: {}, etag: {}, ajaxSettings: { url: location.href, type: "GET", isLocal: rlocalProtocol.test( location.protocol ), global: true, processData: true, async: true, contentType: "application/x-www-form-urlencoded; charset=UTF-8", /* timeout: 0, data: null, dataType: null, username: null, password: null, cache: null, throws: false, traditional: false, headers: {}, */ accepts: { "*": allTypes, text: "text/plain", html: "text/html", xml: "application/xml, text/xml", json: "application/json, text/javascript" }, contents: { xml: /\bxml\b/, html: /\bhtml/, json: /\bjson\b/ }, responseFields: { xml: "responseXML", text: "responseText", json: "responseJSON" }, // Data converters // Keys separate source (or catchall "*") and destination types with a single space converters: { // Convert anything to text "* text": String, // Text to html (true = no transformation) "text html": true, // Evaluate text as a json expression "text json": JSON.parse, // Parse text as xml "text xml": jQuery.parseXML }, // For options that shouldn't be deep extended: // you can add your own custom options here if // and when you create one that shouldn't be // deep extended (see ajaxExtend) flatOptions: { url: true, context: true } }, // Creates a full fledged settings object into target // with both ajaxSettings and settings fields. // If target is omitted, writes into ajaxSettings. ajaxSetup: function( target, settings ) { return settings ? // Building a settings object ajaxExtend( ajaxExtend( target, jQuery.ajaxSettings ), settings ) : // Extending ajaxSettings ajaxExtend( jQuery.ajaxSettings, target ); }, ajaxPrefilter: addToPrefiltersOrTransports( prefilters ), ajaxTransport: addToPrefiltersOrTransports( transports ), // Main method ajax: function( url, options ) { // If url is an object, simulate pre-1.5 signature if ( typeof url === "object" ) { options = url; url = undefined; } // Force options to be an object options = options || {}; var transport, // URL without anti-cache param cacheURL, // Response headers responseHeadersString, responseHeaders, // timeout handle timeoutTimer, // Url cleanup var urlAnchor, // Request state (becomes false upon send and true upon completion) completed, // To know if global events are to be dispatched fireGlobals, // Loop variable i, // uncached part of the url uncached, // Create the final options object s = jQuery.ajaxSetup( {}, options ), // Callbacks context callbackContext = s.context || s, // Context for global events is callbackContext if it is a DOM node or jQuery collection globalEventContext = s.context && ( callbackContext.nodeType || callbackContext.jquery ) ? jQuery( callbackContext ) : jQuery.event, // Deferreds deferred = jQuery.Deferred(), completeDeferred = jQuery.Callbacks( "once memory" ), // Status-dependent callbacks statusCode = s.statusCode || {}, // Headers (they are sent all at once) requestHeaders = {}, requestHeadersNames = {}, // Default abort message strAbort = "canceled", // Fake xhr jqXHR = { readyState: 0, // Builds headers hashtable if needed getResponseHeader: function( key ) { var match; if ( completed ) { if ( !responseHeaders ) { responseHeaders = {}; while ( ( match = rheaders.exec( responseHeadersString ) ) ) { responseHeaders[ match[ 1 ].toLowerCase() + " " ] = ( responseHeaders[ match[ 1 ].toLowerCase() + " " ] || [] ) .concat( match[ 2 ] ); } } match = responseHeaders[ key.toLowerCase() + " " ]; } return match == null ? null : match.join( ", " ); }, // Raw string getAllResponseHeaders: function() { return completed ? responseHeadersString : null; }, // Caches the header setRequestHeader: function( name, value ) { if ( completed == null ) { name = requestHeadersNames[ name.toLowerCase() ] = requestHeadersNames[ name.toLowerCase() ] || name; requestHeaders[ name ] = value; } return this; }, // Overrides response content-type header overrideMimeType: function( type ) { if ( completed == null ) { s.mimeType = type; } return this; }, // Status-dependent callbacks statusCode: function( map ) { var code; if ( map ) { if ( completed ) { // Execute the appropriate callbacks jqXHR.always( map[ jqXHR.status ] ); } else { // Lazy-add the new callbacks in a way that preserves old ones for ( code in map ) { statusCode[ code ] = [ statusCode[ code ], map[ code ] ]; } } } return this; }, // Cancel the request abort: function( statusText ) { var finalText = statusText || strAbort; if ( transport ) { transport.abort( finalText ); } done( 0, finalText ); return this; } }; // Attach deferreds deferred.promise( jqXHR ); // Add protocol if not provided (prefilters might expect it) // Handle falsy url in the settings object (#10093: consistency with old signature) // We also use the url parameter if available s.url = ( ( url || s.url || location.href ) + "" ) .replace( rprotocol, location.protocol + "//" ); // Alias method option to type as per ticket #12004 s.type = options.method || options.type || s.method || s.type; // Extract dataTypes list s.dataTypes = ( s.dataType || "*" ).toLowerCase().match( rnothtmlwhite ) || [ "" ]; // A cross-domain request is in order when the origin doesn't match the current origin. if ( s.crossDomain == null ) { urlAnchor = document.createElement( "a" ); // Support: IE <=8 - 11, Edge 12 - 15 // IE throws exception on accessing the href property if url is malformed, // e.g. http://example.com:80x/ try { urlAnchor.href = s.url; // Support: IE <=8 - 11 only // Anchor's host property isn't correctly set when s.url is relative urlAnchor.href = urlAnchor.href; s.crossDomain = originAnchor.protocol + "//" + originAnchor.host !== urlAnchor.protocol + "//" + urlAnchor.host; } catch ( e ) { // If there is an error parsing the URL, assume it is crossDomain, // it can be rejected by the transport if it is invalid s.crossDomain = true; } } // Convert data if not already a string if ( s.data && s.processData && typeof s.data !== "string" ) { s.data = jQuery.param( s.data, s.traditional ); } // Apply prefilters inspectPrefiltersOrTransports( prefilters, s, options, jqXHR ); // If request was aborted inside a prefilter, stop there if ( completed ) { return jqXHR; } // We can fire global events as of now if asked to // Don't fire events if jQuery.event is undefined in an AMD-usage scenario (#15118) fireGlobals = jQuery.event && s.global; // Watch for a new set of requests if ( fireGlobals && jQuery.active++ === 0 ) { jQuery.event.trigger( "ajaxStart" ); } // Uppercase the type s.type = s.type.toUpperCase(); // Determine if request has content s.hasContent = !rnoContent.test( s.type ); // Save the URL in case we're toying with the If-Modified-Since // and/or If-None-Match header later on // Remove hash to simplify url manipulation cacheURL = s.url.replace( rhash, "" ); // More options handling for requests with no content if ( !s.hasContent ) { // Remember the hash so we can put it back uncached = s.url.slice( cacheURL.length ); // If data is available and should be processed, append data to url if ( s.data && ( s.processData || typeof s.data === "string" ) ) { cacheURL += ( rquery.test( cacheURL ) ? "&" : "?" ) + s.data; // #9682: remove data so that it's not used in an eventual retry delete s.data; } // Add or update anti-cache param if needed if ( s.cache === false ) { cacheURL = cacheURL.replace( rantiCache, "$1" ); uncached = ( rquery.test( cacheURL ) ? "&" : "?" ) + "_=" + ( nonce.guid++ ) + uncached; } // Put hash and anti-cache on the URL that will be requested (gh-1732) s.url = cacheURL + uncached; // Change '%20' to '+' if this is encoded form body content (gh-2658) } else if ( s.data && s.processData && ( s.contentType || "" ).indexOf( "application/x-www-form-urlencoded" ) === 0 ) { s.data = s.data.replace( r20, "+" ); } // Set the If-Modified-Since and/or If-None-Match header, if in ifModified mode. if ( s.ifModified ) { if ( jQuery.lastModified[ cacheURL ] ) { jqXHR.setRequestHeader( "If-Modified-Since", jQuery.lastModified[ cacheURL ] ); } if ( jQuery.etag[ cacheURL ] ) { jqXHR.setRequestHeader( "If-None-Match", jQuery.etag[ cacheURL ] ); } } // Set the correct header, if data is being sent if ( s.data && s.hasContent && s.contentType !== false || options.contentType ) { jqXHR.setRequestHeader( "Content-Type", s.contentType ); } // Set the Accepts header for the server, depending on the dataType jqXHR.setRequestHeader( "Accept", s.dataTypes[ 0 ] && s.accepts[ s.dataTypes[ 0 ] ] ? s.accepts[ s.dataTypes[ 0 ] ] + ( s.dataTypes[ 0 ] !== "*" ? ", " + allTypes + "; q=0.01" : "" ) : s.accepts[ "*" ] ); // Check for headers option for ( i in s.headers ) { jqXHR.setRequestHeader( i, s.headers[ i ] ); } // Allow custom headers/mimetypes and early abort if ( s.beforeSend && ( s.beforeSend.call( callbackContext, jqXHR, s ) === false || completed ) ) { // Abort if not done already and return return jqXHR.abort(); } // Aborting is no longer a cancellation strAbort = "abort"; // Install callbacks on deferreds completeDeferred.add( s.complete ); jqXHR.done( s.success ); jqXHR.fail( s.error ); // Get transport transport = inspectPrefiltersOrTransports( transports, s, options, jqXHR ); // If no transport, we auto-abort if ( !transport ) { done( -1, "No Transport" ); } else { jqXHR.readyState = 1; // Send global event if ( fireGlobals ) { globalEventContext.trigger( "ajaxSend", [ jqXHR, s ] ); } // If request was aborted inside ajaxSend, stop there if ( completed ) { return jqXHR; } // Timeout if ( s.async && s.timeout > 0 ) { timeoutTimer = window.setTimeout( function() { jqXHR.abort( "timeout" ); }, s.timeout ); } try { completed = false; transport.send( requestHeaders, done ); } catch ( e ) { // Rethrow post-completion exceptions if ( completed ) { throw e; } // Propagate others as results done( -1, e ); } } // Callback for when everything is done function done( status, nativeStatusText, responses, headers ) { var isSuccess, success, error, response, modified, statusText = nativeStatusText; // Ignore repeat invocations if ( completed ) { return; } completed = true; // Clear timeout if it exists if ( timeoutTimer ) { window.clearTimeout( timeoutTimer ); } // Dereference transport for early garbage collection // (no matter how long the jqXHR object will be used) transport = undefined; // Cache response headers responseHeadersString = headers || ""; // Set readyState jqXHR.readyState = status > 0 ? 4 : 0; // Determine if successful isSuccess = status >= 200 && status < 300 || status === 304; // Get response data if ( responses ) { response = ajaxHandleResponses( s, jqXHR, responses ); } // Use a noop converter for missing script but not if jsonp if ( !isSuccess && jQuery.inArray( "script", s.dataTypes ) > -1 && jQuery.inArray( "json", s.dataTypes ) < 0 ) { s.converters[ "text script" ] = function() {}; } // Convert no matter what (that way responseXXX fields are always set) response = ajaxConvert( s, response, jqXHR, isSuccess ); // If successful, handle type chaining if ( isSuccess ) { // Set the If-Modified-Since and/or If-None-Match header, if in ifModified mode. if ( s.ifModified ) { modified = jqXHR.getResponseHeader( "Last-Modified" ); if ( modified ) { jQuery.lastModified[ cacheURL ] = modified; } modified = jqXHR.getResponseHeader( "etag" ); if ( modified ) { jQuery.etag[ cacheURL ] = modified; } } // if no content if ( status === 204 || s.type === "HEAD" ) { statusText = "nocontent"; // if not modified } else if ( status === 304 ) { statusText = "notmodified"; // If we have data, let's convert it } else { statusText = response.state; success = response.data; error = response.error; isSuccess = !error; } } else { // Extract error from statusText and normalize for non-aborts error = statusText; if ( status || !statusText ) { statusText = "error"; if ( status < 0 ) { status = 0; } } } // Set data for the fake xhr object jqXHR.status = status; jqXHR.statusText = ( nativeStatusText || statusText ) + ""; // Success/Error if ( isSuccess ) { deferred.resolveWith( callbackContext, [ success, statusText, jqXHR ] ); } else { deferred.rejectWith( callbackContext, [ jqXHR, statusText, error ] ); } // Status-dependent callbacks jqXHR.statusCode( statusCode ); statusCode = undefined; if ( fireGlobals ) { globalEventContext.trigger( isSuccess ? "ajaxSuccess" : "ajaxError", [ jqXHR, s, isSuccess ? success : error ] ); } // Complete completeDeferred.fireWith( callbackContext, [ jqXHR, statusText ] ); if ( fireGlobals ) { globalEventContext.trigger( "ajaxComplete", [ jqXHR, s ] ); // Handle the global AJAX counter if ( !( --jQuery.active ) ) { jQuery.event.trigger( "ajaxStop" ); } } } return jqXHR; }, getJSON: function( url, data, callback ) { return jQuery.get( url, data, callback, "json" ); }, getScript: function( url, callback ) { return jQuery.get( url, undefined, callback, "script" ); } } ); jQuery.each( [ "get", "post" ], function( _i, method ) { jQuery[ method ] = function( url, data, callback, type ) { // Shift arguments if data argument was omitted if ( isFunction( data ) ) { type = type || callback; callback = data; data = undefined; } // The url can be an options object (which then must have .url) return jQuery.ajax( jQuery.extend( { url: url, type: method, dataType: type, data: data, success: callback }, jQuery.isPlainObject( url ) && url ) ); }; } ); jQuery.ajaxPrefilter( function( s ) { var i; for ( i in s.headers ) { if ( i.toLowerCase() === "content-type" ) { s.contentType = s.headers[ i ] || ""; } } } ); jQuery._evalUrl = function( url, options, doc ) { return jQuery.ajax( { url: url, // Make this explicit, since user can override this through ajaxSetup (#11264) type: "GET", dataType: "script", cache: true, async: false, global: false, // Only evaluate the response if it is successful (gh-4126) // dataFilter is not invoked for failure responses, so using it instead // of the default converter is kludgy but it works. converters: { "text script": function() {} }, dataFilter: function( response ) { jQuery.globalEval( response, options, doc ); } } ); }; jQuery.fn.extend( { wrapAll: function( html ) { var wrap; if ( this[ 0 ] ) { if ( isFunction( html ) ) { html = html.call( this[ 0 ] ); } // The elements to wrap the target around wrap = jQuery( html, this[ 0 ].ownerDocument ).eq( 0 ).clone( true ); if ( this[ 0 ].parentNode ) { wrap.insertBefore( this[ 0 ] ); } wrap.map( function() { var elem = this; while ( elem.firstElementChild ) { elem = elem.firstElementChild; } return elem; } ).append( this ); } return this; }, wrapInner: function( html ) { if ( isFunction( html ) ) { return this.each( function( i ) { jQuery( this ).wrapInner( html.call( this, i ) ); } ); } return this.each( function() { var self = jQuery( this ), contents = self.contents(); if ( contents.length ) { contents.wrapAll( html ); } else { self.append( html ); } } ); }, wrap: function( html ) { var htmlIsFunction = isFunction( html ); return this.each( function( i ) { jQuery( this ).wrapAll( htmlIsFunction ? html.call( this, i ) : html ); } ); }, unwrap: function( selector ) { this.parent( selector ).not( "body" ).each( function() { jQuery( this ).replaceWith( this.childNodes ); } ); return this; } } ); jQuery.expr.pseudos.hidden = function( elem ) { return !jQuery.expr.pseudos.visible( elem ); }; jQuery.expr.pseudos.visible = function( elem ) { return !!( elem.offsetWidth || elem.offsetHeight || elem.getClientRects().length ); }; jQuery.ajaxSettings.xhr = function() { try { return new window.XMLHttpRequest(); } catch ( e ) {} }; var xhrSuccessStatus = { // File protocol always yields status code 0, assume 200 0: 200, // Support: IE <=9 only // #1450: sometimes IE returns 1223 when it should be 204 1223: 204 }, xhrSupported = jQuery.ajaxSettings.xhr(); support.cors = !!xhrSupported && ( "withCredentials" in xhrSupported ); support.ajax = xhrSupported = !!xhrSupported; jQuery.ajaxTransport( function( options ) { var callback, errorCallback; // Cross domain only allowed if supported through XMLHttpRequest if ( support.cors || xhrSupported && !options.crossDomain ) { return { send: function( headers, complete ) { var i, xhr = options.xhr(); xhr.open( options.type, options.url, options.async, options.username, options.password ); // Apply custom fields if provided if ( options.xhrFields ) { for ( i in options.xhrFields ) { xhr[ i ] = options.xhrFields[ i ]; } } // Override mime type if needed if ( options.mimeType && xhr.overrideMimeType ) { xhr.overrideMimeType( options.mimeType ); } // X-Requested-With header // For cross-domain requests, seeing as conditions for a preflight are // akin to a jigsaw puzzle, we simply never set it to be sure. // (it can always be set on a per-request basis or even using ajaxSetup) // For same-domain requests, won't change header if already provided. if ( !options.crossDomain && !headers[ "X-Requested-With" ] ) { headers[ "X-Requested-With" ] = "XMLHttpRequest"; } // Set headers for ( i in headers ) { xhr.setRequestHeader( i, headers[ i ] ); } // Callback callback = function( type ) { return function() { if ( callback ) { callback = errorCallback = xhr.onload = xhr.onerror = xhr.onabort = xhr.ontimeout = xhr.onreadystatechange = null; if ( type === "abort" ) { xhr.abort(); } else if ( type === "error" ) { // Support: IE <=9 only // On a manual native abort, IE9 throws // errors on any property access that is not readyState if ( typeof xhr.status !== "number" ) { complete( 0, "error" ); } else { complete( // File: protocol always yields status 0; see #8605, #14207 xhr.status, xhr.statusText ); } } else { complete( xhrSuccessStatus[ xhr.status ] || xhr.status, xhr.statusText, // Support: IE <=9 only // IE9 has no XHR2 but throws on binary (trac-11426) // For XHR2 non-text, let the caller handle it (gh-2498) ( xhr.responseType || "text" ) !== "text" || typeof xhr.responseText !== "string" ? { binary: xhr.response } : { text: xhr.responseText }, xhr.getAllResponseHeaders() ); } } }; }; // Listen to events xhr.onload = callback(); errorCallback = xhr.onerror = xhr.ontimeout = callback( "error" ); // Support: IE 9 only // Use onreadystatechange to replace onabort // to handle uncaught aborts if ( xhr.onabort !== undefined ) { xhr.onabort = errorCallback; } else { xhr.onreadystatechange = function() { // Check readyState before timeout as it changes if ( xhr.readyState === 4 ) { // Allow onerror to be called first, // but that will not handle a native abort // Also, save errorCallback to a variable // as xhr.onerror cannot be accessed window.setTimeout( function() { if ( callback ) { errorCallback(); } } ); } }; } // Create the abort callback callback = callback( "abort" ); try { // Do send the request (this may raise an exception) xhr.send( options.hasContent && options.data || null ); } catch ( e ) { // #14683: Only rethrow if this hasn't been notified as an error yet if ( callback ) { throw e; } } }, abort: function() { if ( callback ) { callback(); } } }; } } ); // Prevent auto-execution of scripts when no explicit dataType was provided (See gh-2432) jQuery.ajaxPrefilter( function( s ) { if ( s.crossDomain ) { s.contents.script = false; } } ); // Install script dataType jQuery.ajaxSetup( { accepts: { script: "text/javascript, application/javascript, " + "application/ecmascript, application/x-ecmascript" }, contents: { script: /\b(?:java|ecma)script\b/ }, converters: { "text script": function( text ) { jQuery.globalEval( text ); return text; } } } ); // Handle cache's special case and crossDomain jQuery.ajaxPrefilter( "script", function( s ) { if ( s.cache === undefined ) { s.cache = false; } if ( s.crossDomain ) { s.type = "GET"; } } ); // Bind script tag hack transport jQuery.ajaxTransport( "script", function( s ) { // This transport only deals with cross domain or forced-by-attrs requests if ( s.crossDomain || s.scriptAttrs ) { var script, callback; return { send: function( _, complete ) { script = jQuery( "<script>" ) .attr( s.scriptAttrs || {} ) .prop( { charset: s.scriptCharset, src: s.url } ) .on( "load error", callback = function( evt ) { script.remove(); callback = null; if ( evt ) { complete( evt.type === "error" ? 404 : 200, evt.type ); } } ); // Use native DOM manipulation to avoid our domManip AJAX trickery document.head.appendChild( script[ 0 ] ); }, abort: function() { if ( callback ) { callback(); } } }; } } ); var oldCallbacks = [], rjsonp = /(=)\?(?=&|$)|\?\?/; // Default jsonp settings jQuery.ajaxSetup( { jsonp: "callback", jsonpCallback: function() { var callback = oldCallbacks.pop() || ( jQuery.expando + "_" + ( nonce.guid++ ) ); this[ callback ] = true; return callback; } } ); // Detect, normalize options and install callbacks for jsonp requests jQuery.ajaxPrefilter( "json jsonp", function( s, originalSettings, jqXHR ) { var callbackName, overwritten, responseContainer, jsonProp = s.jsonp !== false && ( rjsonp.test( s.url ) ? "url" : typeof s.data === "string" && ( s.contentType || "" ) .indexOf( "application/x-www-form-urlencoded" ) === 0 && rjsonp.test( s.data ) && "data" ); // Handle iff the expected data type is "jsonp" or we have a parameter to set if ( jsonProp || s.dataTypes[ 0 ] === "jsonp" ) { // Get callback name, remembering preexisting value associated with it callbackName = s.jsonpCallback = isFunction( s.jsonpCallback ) ? s.jsonpCallback() : s.jsonpCallback; // Insert callback into url or form data if ( jsonProp ) { s[ jsonProp ] = s[ jsonProp ].replace( rjsonp, "$1" + callbackName ); } else if ( s.jsonp !== false ) { s.url += ( rquery.test( s.url ) ? "&" : "?" ) + s.jsonp + "=" + callbackName; } // Use data converter to retrieve json after script execution s.converters[ "script json" ] = function() { if ( !responseContainer ) { jQuery.error( callbackName + " was not called" ); } return responseContainer[ 0 ]; }; // Force json dataType s.dataTypes[ 0 ] = "json"; // Install callback overwritten = window[ callbackName ]; window[ callbackName ] = function() { responseContainer = arguments; }; // Clean-up function (fires after converters) jqXHR.always( function() { // If previous value didn't exist - remove it if ( overwritten === undefined ) { jQuery( window ).removeProp( callbackName ); // Otherwise restore preexisting value } else { window[ callbackName ] = overwritten; } // Save back as free if ( s[ callbackName ] ) { // Make sure that re-using the options doesn't screw things around s.jsonpCallback = originalSettings.jsonpCallback; // Save the callback name for future use oldCallbacks.push( callbackName ); } // Call if it was a function and we have a response if ( responseContainer && isFunction( overwritten ) ) { overwritten( responseContainer[ 0 ] ); } responseContainer = overwritten = undefined; } ); // Delegate to script return "script"; } } ); // Support: Safari 8 only // In Safari 8 documents created via document.implementation.createHTMLDocument // collapse sibling forms: the second one becomes a child of the first one. // Because of that, this security measure has to be disabled in Safari 8. // https://bugs.webkit.org/show_bug.cgi?id=137337 support.createHTMLDocument = ( function() { var body = document.implementation.createHTMLDocument( "" ).body; body.innerHTML = "<form></form><form></form>"; return body.childNodes.length === 2; } )(); // Argument "data" should be string of html // context (optional): If specified, the fragment will be created in this context, // defaults to document // keepScripts (optional): If true, will include scripts passed in the html string jQuery.parseHTML = function( data, context, keepScripts ) { if ( typeof data !== "string" ) { return []; } if ( typeof context === "boolean" ) { keepScripts = context; context = false; } var base, parsed, scripts; if ( !context ) { // Stop scripts or inline event handlers from being executed immediately // by using document.implementation if ( support.createHTMLDocument ) { context = document.implementation.createHTMLDocument( "" ); // Set the base href for the created document // so any parsed elements with URLs // are based on the document's URL (gh-2965) base = context.createElement( "base" ); base.href = document.location.href; context.head.appendChild( base ); } else { context = document; } } parsed = rsingleTag.exec( data ); scripts = !keepScripts && []; // Single tag if ( parsed ) { return [ context.createElement( parsed[ 1 ] ) ]; } parsed = buildFragment( [ data ], context, scripts ); if ( scripts && scripts.length ) { jQuery( scripts ).remove(); } return jQuery.merge( [], parsed.childNodes ); }; /** * Load a url into a page */ jQuery.fn.load = function( url, params, callback ) { var selector, type, response, self = this, off = url.indexOf( " " ); if ( off > -1 ) { selector = stripAndCollapse( url.slice( off ) ); url = url.slice( 0, off ); } // If it's a function if ( isFunction( params ) ) { // We assume that it's the callback callback = params; params = undefined; // Otherwise, build a param string } else if ( params && typeof params === "object" ) { type = "POST"; } // If we have elements to modify, make the request if ( self.length > 0 ) { jQuery.ajax( { url: url, // If "type" variable is undefined, then "GET" method will be used. // Make value of this field explicit since // user can override it through ajaxSetup method type: type || "GET", dataType: "html", data: params } ).done( function( responseText ) { // Save response for use in complete callback response = arguments; self.html( selector ? // If a selector was specified, locate the right elements in a dummy div // Exclude scripts to avoid IE 'Permission Denied' errors jQuery( "<div>" ).append( jQuery.parseHTML( responseText ) ).find( selector ) : // Otherwise use the full result responseText ); // If the request succeeds, this function gets "data", "status", "jqXHR" // but they are ignored because response was set above. // If it fails, this function gets "jqXHR", "status", "error" } ).always( callback && function( jqXHR, status ) { self.each( function() { callback.apply( this, response || [ jqXHR.responseText, status, jqXHR ] ); } ); } ); } return this; }; jQuery.expr.pseudos.animated = function( elem ) { return jQuery.grep( jQuery.timers, function( fn ) { return elem === fn.elem; } ).length; }; jQuery.offset = { setOffset: function( elem, options, i ) { var curPosition, curLeft, curCSSTop, curTop, curOffset, curCSSLeft, calculatePosition, position = jQuery.css( elem, "position" ), curElem = jQuery( elem ), props = {}; // Set position first, in-case top/left are set even on static elem if ( position === "static" ) { elem.style.position = "relative"; } curOffset = curElem.offset(); curCSSTop = jQuery.css( elem, "top" ); curCSSLeft = jQuery.css( elem, "left" ); calculatePosition = ( position === "absolute" || position === "fixed" ) && ( curCSSTop + curCSSLeft ).indexOf( "auto" ) > -1; // Need to be able to calculate position if either // top or left is auto and position is either absolute or fixed if ( calculatePosition ) { curPosition = curElem.position(); curTop = curPosition.top; curLeft = curPosition.left; } else { curTop = parseFloat( curCSSTop ) || 0; curLeft = parseFloat( curCSSLeft ) || 0; } if ( isFunction( options ) ) { // Use jQuery.extend here to allow modification of coordinates argument (gh-1848) options = options.call( elem, i, jQuery.extend( {}, curOffset ) ); } if ( options.top != null ) { props.top = ( options.top - curOffset.top ) + curTop; } if ( options.left != null ) { props.left = ( options.left - curOffset.left ) + curLeft; } if ( "using" in options ) { options.using.call( elem, props ); } else { curElem.css( props ); } } }; jQuery.fn.extend( { // offset() relates an element's border box to the document origin offset: function( options ) { // Preserve chaining for setter if ( arguments.length ) { return options === undefined ? this : this.each( function( i ) { jQuery.offset.setOffset( this, options, i ); } ); } var rect, win, elem = this[ 0 ]; if ( !elem ) { return; } // Return zeros for disconnected and hidden (display: none) elements (gh-2310) // Support: IE <=11 only // Running getBoundingClientRect on a // disconnected node in IE throws an error if ( !elem.getClientRects().length ) { return { top: 0, left: 0 }; } // Get document-relative position by adding viewport scroll to viewport-relative gBCR rect = elem.getBoundingClientRect(); win = elem.ownerDocument.defaultView; return { top: rect.top + win.pageYOffset, left: rect.left + win.pageXOffset }; }, // position() relates an element's margin box to its offset parent's padding box // This corresponds to the behavior of CSS absolute positioning position: function() { if ( !this[ 0 ] ) { return; } var offsetParent, offset, doc, elem = this[ 0 ], parentOffset = { top: 0, left: 0 }; // position:fixed elements are offset from the viewport, which itself always has zero offset if ( jQuery.css( elem, "position" ) === "fixed" ) { // Assume position:fixed implies availability of getBoundingClientRect offset = elem.getBoundingClientRect(); } else { offset = this.offset(); // Account for the *real* offset parent, which can be the document or its root element // when a statically positioned element is identified doc = elem.ownerDocument; offsetParent = elem.offsetParent || doc.documentElement; while ( offsetParent && ( offsetParent === doc.body || offsetParent === doc.documentElement ) && jQuery.css( offsetParent, "position" ) === "static" ) { offsetParent = offsetParent.parentNode; } if ( offsetParent && offsetParent !== elem && offsetParent.nodeType === 1 ) { // Incorporate borders into its offset, since they are outside its content origin parentOffset = jQuery( offsetParent ).offset(); parentOffset.top += jQuery.css( offsetParent, "borderTopWidth", true ); parentOffset.left += jQuery.css( offsetParent, "borderLeftWidth", true ); } } // Subtract parent offsets and element margins return { top: offset.top - parentOffset.top - jQuery.css( elem, "marginTop", true ), left: offset.left - parentOffset.left - jQuery.css( elem, "marginLeft", true ) }; }, // This method will return documentElement in the following cases: // 1) For the element inside the iframe without offsetParent, this method will return // documentElement of the parent window // 2) For the hidden or detached element // 3) For body or html element, i.e. in case of the html node - it will return itself // // but those exceptions were never presented as a real life use-cases // and might be considered as more preferable results. // // This logic, however, is not guaranteed and can change at any point in the future offsetParent: function() { return this.map( function() { var offsetParent = this.offsetParent; while ( offsetParent && jQuery.css( offsetParent, "position" ) === "static" ) { offsetParent = offsetParent.offsetParent; } return offsetParent || documentElement; } ); } } ); // Create scrollLeft and scrollTop methods jQuery.each( { scrollLeft: "pageXOffset", scrollTop: "pageYOffset" }, function( method, prop ) { var top = "pageYOffset" === prop; jQuery.fn[ method ] = function( val ) { return access( this, function( elem, method, val ) { // Coalesce documents and windows var win; if ( isWindow( elem ) ) { win = elem; } else if ( elem.nodeType === 9 ) { win = elem.defaultView; } if ( val === undefined ) { return win ? win[ prop ] : elem[ method ]; } if ( win ) { win.scrollTo( !top ? val : win.pageXOffset, top ? val : win.pageYOffset ); } else { elem[ method ] = val; } }, method, val, arguments.length ); }; } ); // Support: Safari <=7 - 9.1, Chrome <=37 - 49 // Add the top/left cssHooks using jQuery.fn.position // Webkit bug: https://bugs.webkit.org/show_bug.cgi?id=29084 // Blink bug: https://bugs.chromium.org/p/chromium/issues/detail?id=589347 // getComputedStyle returns percent when specified for top/left/bottom/right; // rather than make the css module depend on the offset module, just check for it here jQuery.each( [ "top", "left" ], function( _i, prop ) { jQuery.cssHooks[ prop ] = addGetHookIf( support.pixelPosition, function( elem, computed ) { if ( computed ) { computed = curCSS( elem, prop ); // If curCSS returns percentage, fallback to offset return rnumnonpx.test( computed ) ? jQuery( elem ).position()[ prop ] + "px" : computed; } } ); } ); // Create innerHeight, innerWidth, height, width, outerHeight and outerWidth methods jQuery.each( { Height: "height", Width: "width" }, function( name, type ) { jQuery.each( { padding: "inner" + name, content: type, "": "outer" + name }, function( defaultExtra, funcName ) { // Margin is only for outerHeight, outerWidth jQuery.fn[ funcName ] = function( margin, value ) { var chainable = arguments.length && ( defaultExtra || typeof margin !== "boolean" ), extra = defaultExtra || ( margin === true || value === true ? "margin" : "border" ); return access( this, function( elem, type, value ) { var doc; if ( isWindow( elem ) ) { // $( window ).outerWidth/Height return w/h including scrollbars (gh-1729) return funcName.indexOf( "outer" ) === 0 ? elem[ "inner" + name ] : elem.document.documentElement[ "client" + name ]; } // Get document width or height if ( elem.nodeType === 9 ) { doc = elem.documentElement; // Either scroll[Width/Height] or offset[Width/Height] or client[Width/Height], // whichever is greatest return Math.max( elem.body[ "scroll" + name ], doc[ "scroll" + name ], elem.body[ "offset" + name ], doc[ "offset" + name ], doc[ "client" + name ] ); } return value === undefined ? // Get width or height on the element, requesting but not forcing parseFloat jQuery.css( elem, type, extra ) : // Set width or height on the element jQuery.style( elem, type, value, extra ); }, type, chainable ? margin : undefined, chainable ); }; } ); } ); jQuery.each( [ "ajaxStart", "ajaxStop", "ajaxComplete", "ajaxError", "ajaxSuccess", "ajaxSend" ], function( _i, type ) { jQuery.fn[ type ] = function( fn ) { return this.on( type, fn ); }; } ); jQuery.fn.extend( { bind: function( types, data, fn ) { return this.on( types, null, data, fn ); }, unbind: function( types, fn ) { return this.off( types, null, fn ); }, delegate: function( selector, types, data, fn ) { return this.on( types, selector, data, fn ); }, undelegate: function( selector, types, fn ) { // ( namespace ) or ( selector, types [, fn] ) return arguments.length === 1 ? this.off( selector, "**" ) : this.off( types, selector || "**", fn ); }, hover: function( fnOver, fnOut ) { return this.mouseenter( fnOver ).mouseleave( fnOut || fnOver ); } } ); jQuery.each( ( "blur focus focusin focusout resize scroll click dblclick " + "mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave " + "change select submit keydown keypress keyup contextmenu" ).split( " " ), function( _i, name ) { // Handle event binding jQuery.fn[ name ] = function( data, fn ) { return arguments.length > 0 ? this.on( name, null, data, fn ) : this.trigger( name ); }; } ); // Support: Android <=4.0 only // Make sure we trim BOM and NBSP var rtrim = /^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g; // Bind a function to a context, optionally partially applying any // arguments. // jQuery.proxy is deprecated to promote standards (specifically Function#bind) // However, it is not slated for removal any time soon jQuery.proxy = function( fn, context ) { var tmp, args, proxy; if ( typeof context === "string" ) { tmp = fn[ context ]; context = fn; fn = tmp; } // Quick check to determine if target is callable, in the spec // this throws a TypeError, but we will just return undefined. if ( !isFunction( fn ) ) { return undefined; } // Simulated bind args = slice.call( arguments, 2 ); proxy = function() { return fn.apply( context || this, args.concat( slice.call( arguments ) ) ); }; // Set the guid of unique handler to the same of original handler, so it can be removed proxy.guid = fn.guid = fn.guid || jQuery.guid++; return proxy; }; jQuery.holdReady = function( hold ) { if ( hold ) { jQuery.readyWait++; } else { jQuery.ready( true ); } }; jQuery.isArray = Array.isArray; jQuery.parseJSON = JSON.parse; jQuery.nodeName = nodeName; jQuery.isFunction = isFunction; jQuery.isWindow = isWindow; jQuery.camelCase = camelCase; jQuery.type = toType; jQuery.now = Date.now; jQuery.isNumeric = function( obj ) { // As of jQuery 3.0, isNumeric is limited to // strings and numbers (primitives or objects) // that can be coerced to finite numbers (gh-2662) var type = jQuery.type( obj ); return ( type === "number" || type === "string" ) && // parseFloat NaNs numeric-cast false positives ("") // ...but misinterprets leading-number strings, particularly hex literals ("0x...") // subtraction forces infinities to NaN !isNaN( obj - parseFloat( obj ) ); }; jQuery.trim = function( text ) { return text == null ? "" : ( text + "" ).replace( rtrim, "" ); }; // Register as a named AMD module, since jQuery can be concatenated with other // files that may use define, but not via a proper concatenation script that // understands anonymous AMD modules. A named AMD is safest and most robust // way to register. Lowercase jquery is used because AMD module names are // derived from file names, and jQuery is normally delivered in a lowercase // file name. Do this after creating the global so that if an AMD module wants // to call noConflict to hide this version of jQuery, it will work. // Note that for maximum portability, libraries that are not jQuery should // declare themselves as anonymous modules, and avoid setting a global if an // AMD loader is present. jQuery is a special case. For more information, see // https://github.com/jrburke/requirejs/wiki/Updating-existing-libraries#wiki-anon if ( typeof define === "function" && define.amd ) { define( "jquery", [], function() { return jQuery; } ); } var // Map over jQuery in case of overwrite _jQuery = window.jQuery, // Map over the $ in case of overwrite _$ = window.$; jQuery.noConflict = function( deep ) { if ( window.$ === jQuery ) { window.$ = _$; } if ( deep && window.jQuery === jQuery ) { window.jQuery = _jQuery; } return jQuery; }; // Expose jQuery and $ identifiers, even in AMD // (#7102#comment:10, https://github.com/jquery/jquery/pull/557) // and CommonJS for browser emulators (#13566) if ( typeof noGlobal === "undefined" ) { window.jQuery = window.$ = jQuery; } return jQuery; } ); ================================================ FILE: core/dbt/docs/build/html/_static/jquery.js ================================================ /*! jQuery v3.6.0 | (c) OpenJS Foundation and other contributors | jquery.org/license */ !function(e,t){"use strict";"object"==typeof module&&"object"==typeof module.exports?module.exports=e.document?t(e,!0):function(e){if(!e.document)throw new Error("jQuery requires a window with a document");return t(e)}:t(e)}("undefined"!=typeof window?window:this,function(C,e){"use strict";var t=[],r=Object.getPrototypeOf,s=t.slice,g=t.flat?function(e){return t.flat.call(e)}:function(e){return t.concat.apply([],e)},u=t.push,i=t.indexOf,n={},o=n.toString,v=n.hasOwnProperty,a=v.toString,l=a.call(Object),y={},m=function(e){return"function"==typeof e&&"number"!=typeof e.nodeType&&"function"!=typeof e.item},x=function(e){return null!=e&&e===e.window},E=C.document,c={type:!0,src:!0,nonce:!0,noModule:!0};function b(e,t,n){var r,i,o=(n=n||E).createElement("script");if(o.text=e,t)for(r in c)(i=t[r]||t.getAttribute&&t.getAttribute(r))&&o.setAttribute(r,i);n.head.appendChild(o).parentNode.removeChild(o)}function w(e){return null==e?e+"":"object"==typeof e||"function"==typeof e?n[o.call(e)]||"object":typeof e}var f="3.6.0",S=function(e,t){return new S.fn.init(e,t)};function p(e){var t=!!e&&"length"in e&&e.length,n=w(e);return!m(e)&&!x(e)&&("array"===n||0===t||"number"==typeof t&&0<t&&t-1 in e)}S.fn=S.prototype={jquery:f,constructor:S,length:0,toArray:function(){return s.call(this)},get:function(e){return null==e?s.call(this):e<0?this[e+this.length]:this[e]},pushStack:function(e){var t=S.merge(this.constructor(),e);return t.prevObject=this,t},each:function(e){return S.each(this,e)},map:function(n){return this.pushStack(S.map(this,function(e,t){return n.call(e,t,e)}))},slice:function(){return this.pushStack(s.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},even:function(){return this.pushStack(S.grep(this,function(e,t){return(t+1)%2}))},odd:function(){return this.pushStack(S.grep(this,function(e,t){return t%2}))},eq:function(e){var t=this.length,n=+e+(e<0?t:0);return this.pushStack(0<=n&&n<t?[this[n]]:[])},end:function(){return this.prevObject||this.constructor()},push:u,sort:t.sort,splice:t.splice},S.extend=S.fn.extend=function(){var e,t,n,r,i,o,a=arguments[0]||{},s=1,u=arguments.length,l=!1;for("boolean"==typeof a&&(l=a,a=arguments[s]||{},s++),"object"==typeof a||m(a)||(a={}),s===u&&(a=this,s--);s<u;s++)if(null!=(e=arguments[s]))for(t in e)r=e[t],"__proto__"!==t&&a!==r&&(l&&r&&(S.isPlainObject(r)||(i=Array.isArray(r)))?(n=a[t],o=i&&!Array.isArray(n)?[]:i||S.isPlainObject(n)?n:{},i=!1,a[t]=S.extend(l,o,r)):void 0!==r&&(a[t]=r));return a},S.extend({expando:"jQuery"+(f+Math.random()).replace(/\D/g,""),isReady:!0,error:function(e){throw new Error(e)},noop:function(){},isPlainObject:function(e){var t,n;return!(!e||"[object Object]"!==o.call(e))&&(!(t=r(e))||"function"==typeof(n=v.call(t,"constructor")&&t.constructor)&&a.call(n)===l)},isEmptyObject:function(e){var t;for(t in e)return!1;return!0},globalEval:function(e,t,n){b(e,{nonce:t&&t.nonce},n)},each:function(e,t){var n,r=0;if(p(e)){for(n=e.length;r<n;r++)if(!1===t.call(e[r],r,e[r]))break}else for(r in e)if(!1===t.call(e[r],r,e[r]))break;return e},makeArray:function(e,t){var n=t||[];return null!=e&&(p(Object(e))?S.merge(n,"string"==typeof e?[e]:e):u.call(n,e)),n},inArray:function(e,t,n){return null==t?-1:i.call(t,e,n)},merge:function(e,t){for(var n=+t.length,r=0,i=e.length;r<n;r++)e[i++]=t[r];return e.length=i,e},grep:function(e,t,n){for(var r=[],i=0,o=e.length,a=!n;i<o;i++)!t(e[i],i)!==a&&r.push(e[i]);return r},map:function(e,t,n){var r,i,o=0,a=[];if(p(e))for(r=e.length;o<r;o++)null!=(i=t(e[o],o,n))&&a.push(i);else for(o in e)null!=(i=t(e[o],o,n))&&a.push(i);return g(a)},guid:1,support:y}),"function"==typeof Symbol&&(S.fn[Symbol.iterator]=t[Symbol.iterator]),S.each("Boolean Number String Function Array Date RegExp Object Error Symbol".split(" "),function(e,t){n["[object "+t+"]"]=t.toLowerCase()});var d=function(n){var e,d,b,o,i,h,f,g,w,u,l,T,C,a,E,v,s,c,y,S="sizzle"+1*new Date,p=n.document,k=0,r=0,m=ue(),x=ue(),A=ue(),N=ue(),j=function(e,t){return e===t&&(l=!0),0},D={}.hasOwnProperty,t=[],q=t.pop,L=t.push,H=t.push,O=t.slice,P=function(e,t){for(var n=0,r=e.length;n<r;n++)if(e[n]===t)return n;return-1},R="checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped",M="[\\x20\\t\\r\\n\\f]",I="(?:\\\\[\\da-fA-F]{1,6}"+M+"?|\\\\[^\\r\\n\\f]|[\\w-]|[^\0-\\x7f])+",W="\\["+M+"*("+I+")(?:"+M+"*([*^$|!~]?=)"+M+"*(?:'((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\"|("+I+"))|)"+M+"*\\]",F=":("+I+")(?:\\((('((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\")|((?:\\\\.|[^\\\\()[\\]]|"+W+")*)|.*)\\)|)",B=new RegExp(M+"+","g"),$=new RegExp("^"+M+"+|((?:^|[^\\\\])(?:\\\\.)*)"+M+"+$","g"),_=new RegExp("^"+M+"*,"+M+"*"),z=new RegExp("^"+M+"*([>+~]|"+M+")"+M+"*"),U=new RegExp(M+"|>"),X=new RegExp(F),V=new RegExp("^"+I+"$"),G={ID:new RegExp("^#("+I+")"),CLASS:new RegExp("^\\.("+I+")"),TAG:new RegExp("^("+I+"|[*])"),ATTR:new RegExp("^"+W),PSEUDO:new RegExp("^"+F),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+M+"*(even|odd|(([+-]|)(\\d*)n|)"+M+"*(?:([+-]|)"+M+"*(\\d+)|))"+M+"*\\)|)","i"),bool:new RegExp("^(?:"+R+")$","i"),needsContext:new RegExp("^"+M+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+M+"*((?:-\\d)?\\d*)"+M+"*\\)|)(?=[^-]|$)","i")},Y=/HTML$/i,Q=/^(?:input|select|textarea|button)$/i,J=/^h\d$/i,K=/^[^{]+\{\s*\[native \w/,Z=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,ee=/[+~]/,te=new RegExp("\\\\[\\da-fA-F]{1,6}"+M+"?|\\\\([^\\r\\n\\f])","g"),ne=function(e,t){var n="0x"+e.slice(1)-65536;return t||(n<0?String.fromCharCode(n+65536):String.fromCharCode(n>>10|55296,1023&n|56320))},re=/([\0-\x1f\x7f]|^-?\d)|^-$|[^\0-\x1f\x7f-\uFFFF\w-]/g,ie=function(e,t){return t?"\0"===e?"\ufffd":e.slice(0,-1)+"\\"+e.charCodeAt(e.length-1).toString(16)+" ":"\\"+e},oe=function(){T()},ae=be(function(e){return!0===e.disabled&&"fieldset"===e.nodeName.toLowerCase()},{dir:"parentNode",next:"legend"});try{H.apply(t=O.call(p.childNodes),p.childNodes),t[p.childNodes.length].nodeType}catch(e){H={apply:t.length?function(e,t){L.apply(e,O.call(t))}:function(e,t){var n=e.length,r=0;while(e[n++]=t[r++]);e.length=n-1}}}function se(t,e,n,r){var i,o,a,s,u,l,c,f=e&&e.ownerDocument,p=e?e.nodeType:9;if(n=n||[],"string"!=typeof t||!t||1!==p&&9!==p&&11!==p)return n;if(!r&&(T(e),e=e||C,E)){if(11!==p&&(u=Z.exec(t)))if(i=u[1]){if(9===p){if(!(a=e.getElementById(i)))return n;if(a.id===i)return n.push(a),n}else if(f&&(a=f.getElementById(i))&&y(e,a)&&a.id===i)return n.push(a),n}else{if(u[2])return H.apply(n,e.getElementsByTagName(t)),n;if((i=u[3])&&d.getElementsByClassName&&e.getElementsByClassName)return H.apply(n,e.getElementsByClassName(i)),n}if(d.qsa&&!N[t+" "]&&(!v||!v.test(t))&&(1!==p||"object"!==e.nodeName.toLowerCase())){if(c=t,f=e,1===p&&(U.test(t)||z.test(t))){(f=ee.test(t)&&ye(e.parentNode)||e)===e&&d.scope||((s=e.getAttribute("id"))?s=s.replace(re,ie):e.setAttribute("id",s=S)),o=(l=h(t)).length;while(o--)l[o]=(s?"#"+s:":scope")+" "+xe(l[o]);c=l.join(",")}try{return H.apply(n,f.querySelectorAll(c)),n}catch(e){N(t,!0)}finally{s===S&&e.removeAttribute("id")}}}return g(t.replace($,"$1"),e,n,r)}function ue(){var r=[];return function e(t,n){return r.push(t+" ")>b.cacheLength&&delete e[r.shift()],e[t+" "]=n}}function le(e){return e[S]=!0,e}function ce(e){var t=C.createElement("fieldset");try{return!!e(t)}catch(e){return!1}finally{t.parentNode&&t.parentNode.removeChild(t),t=null}}function fe(e,t){var n=e.split("|"),r=n.length;while(r--)b.attrHandle[n[r]]=t}function pe(e,t){var n=t&&e,r=n&&1===e.nodeType&&1===t.nodeType&&e.sourceIndex-t.sourceIndex;if(r)return r;if(n)while(n=n.nextSibling)if(n===t)return-1;return e?1:-1}function de(t){return function(e){return"input"===e.nodeName.toLowerCase()&&e.type===t}}function he(n){return function(e){var t=e.nodeName.toLowerCase();return("input"===t||"button"===t)&&e.type===n}}function ge(t){return function(e){return"form"in e?e.parentNode&&!1===e.disabled?"label"in e?"label"in e.parentNode?e.parentNode.disabled===t:e.disabled===t:e.isDisabled===t||e.isDisabled!==!t&&ae(e)===t:e.disabled===t:"label"in e&&e.disabled===t}}function ve(a){return le(function(o){return o=+o,le(function(e,t){var n,r=a([],e.length,o),i=r.length;while(i--)e[n=r[i]]&&(e[n]=!(t[n]=e[n]))})})}function ye(e){return e&&"undefined"!=typeof e.getElementsByTagName&&e}for(e in d=se.support={},i=se.isXML=function(e){var t=e&&e.namespaceURI,n=e&&(e.ownerDocument||e).documentElement;return!Y.test(t||n&&n.nodeName||"HTML")},T=se.setDocument=function(e){var t,n,r=e?e.ownerDocument||e:p;return r!=C&&9===r.nodeType&&r.documentElement&&(a=(C=r).documentElement,E=!i(C),p!=C&&(n=C.defaultView)&&n.top!==n&&(n.addEventListener?n.addEventListener("unload",oe,!1):n.attachEvent&&n.attachEvent("onunload",oe)),d.scope=ce(function(e){return a.appendChild(e).appendChild(C.createElement("div")),"undefined"!=typeof e.querySelectorAll&&!e.querySelectorAll(":scope fieldset div").length}),d.attributes=ce(function(e){return e.className="i",!e.getAttribute("className")}),d.getElementsByTagName=ce(function(e){return e.appendChild(C.createComment("")),!e.getElementsByTagName("*").length}),d.getElementsByClassName=K.test(C.getElementsByClassName),d.getById=ce(function(e){return a.appendChild(e).id=S,!C.getElementsByName||!C.getElementsByName(S).length}),d.getById?(b.filter.ID=function(e){var t=e.replace(te,ne);return function(e){return e.getAttribute("id")===t}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n=t.getElementById(e);return n?[n]:[]}}):(b.filter.ID=function(e){var n=e.replace(te,ne);return function(e){var t="undefined"!=typeof e.getAttributeNode&&e.getAttributeNode("id");return t&&t.value===n}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n,r,i,o=t.getElementById(e);if(o){if((n=o.getAttributeNode("id"))&&n.value===e)return[o];i=t.getElementsByName(e),r=0;while(o=i[r++])if((n=o.getAttributeNode("id"))&&n.value===e)return[o]}return[]}}),b.find.TAG=d.getElementsByTagName?function(e,t){return"undefined"!=typeof t.getElementsByTagName?t.getElementsByTagName(e):d.qsa?t.querySelectorAll(e):void 0}:function(e,t){var n,r=[],i=0,o=t.getElementsByTagName(e);if("*"===e){while(n=o[i++])1===n.nodeType&&r.push(n);return r}return o},b.find.CLASS=d.getElementsByClassName&&function(e,t){if("undefined"!=typeof t.getElementsByClassName&&E)return t.getElementsByClassName(e)},s=[],v=[],(d.qsa=K.test(C.querySelectorAll))&&(ce(function(e){var t;a.appendChild(e).innerHTML="<a id='"+S+"'></a><select id='"+S+"-\r\\' msallowcapture=''><option selected=''></option></select>",e.querySelectorAll("[msallowcapture^='']").length&&v.push("[*^$]="+M+"*(?:''|\"\")"),e.querySelectorAll("[selected]").length||v.push("\\["+M+"*(?:value|"+R+")"),e.querySelectorAll("[id~="+S+"-]").length||v.push("~="),(t=C.createElement("input")).setAttribute("name",""),e.appendChild(t),e.querySelectorAll("[name='']").length||v.push("\\["+M+"*name"+M+"*="+M+"*(?:''|\"\")"),e.querySelectorAll(":checked").length||v.push(":checked"),e.querySelectorAll("a#"+S+"+*").length||v.push(".#.+[+~]"),e.querySelectorAll("\\\f"),v.push("[\\r\\n\\f]")}),ce(function(e){e.innerHTML="<a href='' disabled='disabled'></a><select disabled='disabled'><option/></select>";var t=C.createElement("input");t.setAttribute("type","hidden"),e.appendChild(t).setAttribute("name","D"),e.querySelectorAll("[name=d]").length&&v.push("name"+M+"*[*^$|!~]?="),2!==e.querySelectorAll(":enabled").length&&v.push(":enabled",":disabled"),a.appendChild(e).disabled=!0,2!==e.querySelectorAll(":disabled").length&&v.push(":enabled",":disabled"),e.querySelectorAll("*,:x"),v.push(",.*:")})),(d.matchesSelector=K.test(c=a.matches||a.webkitMatchesSelector||a.mozMatchesSelector||a.oMatchesSelector||a.msMatchesSelector))&&ce(function(e){d.disconnectedMatch=c.call(e,"*"),c.call(e,"[s!='']:x"),s.push("!=",F)}),v=v.length&&new RegExp(v.join("|")),s=s.length&&new RegExp(s.join("|")),t=K.test(a.compareDocumentPosition),y=t||K.test(a.contains)?function(e,t){var n=9===e.nodeType?e.documentElement:e,r=t&&t.parentNode;return e===r||!(!r||1!==r.nodeType||!(n.contains?n.contains(r):e.compareDocumentPosition&&16&e.compareDocumentPosition(r)))}:function(e,t){if(t)while(t=t.parentNode)if(t===e)return!0;return!1},j=t?function(e,t){if(e===t)return l=!0,0;var n=!e.compareDocumentPosition-!t.compareDocumentPosition;return n||(1&(n=(e.ownerDocument||e)==(t.ownerDocument||t)?e.compareDocumentPosition(t):1)||!d.sortDetached&&t.compareDocumentPosition(e)===n?e==C||e.ownerDocument==p&&y(p,e)?-1:t==C||t.ownerDocument==p&&y(p,t)?1:u?P(u,e)-P(u,t):0:4&n?-1:1)}:function(e,t){if(e===t)return l=!0,0;var n,r=0,i=e.parentNode,o=t.parentNode,a=[e],s=[t];if(!i||!o)return e==C?-1:t==C?1:i?-1:o?1:u?P(u,e)-P(u,t):0;if(i===o)return pe(e,t);n=e;while(n=n.parentNode)a.unshift(n);n=t;while(n=n.parentNode)s.unshift(n);while(a[r]===s[r])r++;return r?pe(a[r],s[r]):a[r]==p?-1:s[r]==p?1:0}),C},se.matches=function(e,t){return se(e,null,null,t)},se.matchesSelector=function(e,t){if(T(e),d.matchesSelector&&E&&!N[t+" "]&&(!s||!s.test(t))&&(!v||!v.test(t)))try{var n=c.call(e,t);if(n||d.disconnectedMatch||e.document&&11!==e.document.nodeType)return n}catch(e){N(t,!0)}return 0<se(t,C,null,[e]).length},se.contains=function(e,t){return(e.ownerDocument||e)!=C&&T(e),y(e,t)},se.attr=function(e,t){(e.ownerDocument||e)!=C&&T(e);var n=b.attrHandle[t.toLowerCase()],r=n&&D.call(b.attrHandle,t.toLowerCase())?n(e,t,!E):void 0;return void 0!==r?r:d.attributes||!E?e.getAttribute(t):(r=e.getAttributeNode(t))&&r.specified?r.value:null},se.escape=function(e){return(e+"").replace(re,ie)},se.error=function(e){throw new Error("Syntax error, unrecognized expression: "+e)},se.uniqueSort=function(e){var t,n=[],r=0,i=0;if(l=!d.detectDuplicates,u=!d.sortStable&&e.slice(0),e.sort(j),l){while(t=e[i++])t===e[i]&&(r=n.push(i));while(r--)e.splice(n[r],1)}return u=null,e},o=se.getText=function(e){var t,n="",r=0,i=e.nodeType;if(i){if(1===i||9===i||11===i){if("string"==typeof e.textContent)return e.textContent;for(e=e.firstChild;e;e=e.nextSibling)n+=o(e)}else if(3===i||4===i)return e.nodeValue}else while(t=e[r++])n+=o(t);return n},(b=se.selectors={cacheLength:50,createPseudo:le,match:G,attrHandle:{},find:{},relative:{">":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(e){return e[1]=e[1].replace(te,ne),e[3]=(e[3]||e[4]||e[5]||"").replace(te,ne),"~="===e[2]&&(e[3]=" "+e[3]+" "),e.slice(0,4)},CHILD:function(e){return e[1]=e[1].toLowerCase(),"nth"===e[1].slice(0,3)?(e[3]||se.error(e[0]),e[4]=+(e[4]?e[5]+(e[6]||1):2*("even"===e[3]||"odd"===e[3])),e[5]=+(e[7]+e[8]||"odd"===e[3])):e[3]&&se.error(e[0]),e},PSEUDO:function(e){var t,n=!e[6]&&e[2];return G.CHILD.test(e[0])?null:(e[3]?e[2]=e[4]||e[5]||"":n&&X.test(n)&&(t=h(n,!0))&&(t=n.indexOf(")",n.length-t)-n.length)&&(e[0]=e[0].slice(0,t),e[2]=n.slice(0,t)),e.slice(0,3))}},filter:{TAG:function(e){var t=e.replace(te,ne).toLowerCase();return"*"===e?function(){return!0}:function(e){return e.nodeName&&e.nodeName.toLowerCase()===t}},CLASS:function(e){var t=m[e+" "];return t||(t=new RegExp("(^|"+M+")"+e+"("+M+"|$)"))&&m(e,function(e){return t.test("string"==typeof e.className&&e.className||"undefined"!=typeof e.getAttribute&&e.getAttribute("class")||"")})},ATTR:function(n,r,i){return function(e){var t=se.attr(e,n);return null==t?"!="===r:!r||(t+="","="===r?t===i:"!="===r?t!==i:"^="===r?i&&0===t.indexOf(i):"*="===r?i&&-1<t.indexOf(i):"$="===r?i&&t.slice(-i.length)===i:"~="===r?-1<(" "+t.replace(B," ")+" ").indexOf(i):"|="===r&&(t===i||t.slice(0,i.length+1)===i+"-"))}},CHILD:function(h,e,t,g,v){var y="nth"!==h.slice(0,3),m="last"!==h.slice(-4),x="of-type"===e;return 1===g&&0===v?function(e){return!!e.parentNode}:function(e,t,n){var r,i,o,a,s,u,l=y!==m?"nextSibling":"previousSibling",c=e.parentNode,f=x&&e.nodeName.toLowerCase(),p=!n&&!x,d=!1;if(c){if(y){while(l){a=e;while(a=a[l])if(x?a.nodeName.toLowerCase()===f:1===a.nodeType)return!1;u=l="only"===h&&!u&&"nextSibling"}return!0}if(u=[m?c.firstChild:c.lastChild],m&&p){d=(s=(r=(i=(o=(a=c)[S]||(a[S]={}))[a.uniqueID]||(o[a.uniqueID]={}))[h]||[])[0]===k&&r[1])&&r[2],a=s&&c.childNodes[s];while(a=++s&&a&&a[l]||(d=s=0)||u.pop())if(1===a.nodeType&&++d&&a===e){i[h]=[k,s,d];break}}else if(p&&(d=s=(r=(i=(o=(a=e)[S]||(a[S]={}))[a.uniqueID]||(o[a.uniqueID]={}))[h]||[])[0]===k&&r[1]),!1===d)while(a=++s&&a&&a[l]||(d=s=0)||u.pop())if((x?a.nodeName.toLowerCase()===f:1===a.nodeType)&&++d&&(p&&((i=(o=a[S]||(a[S]={}))[a.uniqueID]||(o[a.uniqueID]={}))[h]=[k,d]),a===e))break;return(d-=v)===g||d%g==0&&0<=d/g}}},PSEUDO:function(e,o){var t,a=b.pseudos[e]||b.setFilters[e.toLowerCase()]||se.error("unsupported pseudo: "+e);return a[S]?a(o):1<a.length?(t=[e,e,"",o],b.setFilters.hasOwnProperty(e.toLowerCase())?le(function(e,t){var n,r=a(e,o),i=r.length;while(i--)e[n=P(e,r[i])]=!(t[n]=r[i])}):function(e){return a(e,0,t)}):a}},pseudos:{not:le(function(e){var r=[],i=[],s=f(e.replace($,"$1"));return s[S]?le(function(e,t,n,r){var i,o=s(e,null,r,[]),a=e.length;while(a--)(i=o[a])&&(e[a]=!(t[a]=i))}):function(e,t,n){return r[0]=e,s(r,null,n,i),r[0]=null,!i.pop()}}),has:le(function(t){return function(e){return 0<se(t,e).length}}),contains:le(function(t){return t=t.replace(te,ne),function(e){return-1<(e.textContent||o(e)).indexOf(t)}}),lang:le(function(n){return V.test(n||"")||se.error("unsupported lang: "+n),n=n.replace(te,ne).toLowerCase(),function(e){var t;do{if(t=E?e.lang:e.getAttribute("xml:lang")||e.getAttribute("lang"))return(t=t.toLowerCase())===n||0===t.indexOf(n+"-")}while((e=e.parentNode)&&1===e.nodeType);return!1}}),target:function(e){var t=n.location&&n.location.hash;return t&&t.slice(1)===e.id},root:function(e){return e===a},focus:function(e){return e===C.activeElement&&(!C.hasFocus||C.hasFocus())&&!!(e.type||e.href||~e.tabIndex)},enabled:ge(!1),disabled:ge(!0),checked:function(e){var t=e.nodeName.toLowerCase();return"input"===t&&!!e.checked||"option"===t&&!!e.selected},selected:function(e){return e.parentNode&&e.parentNode.selectedIndex,!0===e.selected},empty:function(e){for(e=e.firstChild;e;e=e.nextSibling)if(e.nodeType<6)return!1;return!0},parent:function(e){return!b.pseudos.empty(e)},header:function(e){return J.test(e.nodeName)},input:function(e){return Q.test(e.nodeName)},button:function(e){var t=e.nodeName.toLowerCase();return"input"===t&&"button"===e.type||"button"===t},text:function(e){var t;return"input"===e.nodeName.toLowerCase()&&"text"===e.type&&(null==(t=e.getAttribute("type"))||"text"===t.toLowerCase())},first:ve(function(){return[0]}),last:ve(function(e,t){return[t-1]}),eq:ve(function(e,t,n){return[n<0?n+t:n]}),even:ve(function(e,t){for(var n=0;n<t;n+=2)e.push(n);return e}),odd:ve(function(e,t){for(var n=1;n<t;n+=2)e.push(n);return e}),lt:ve(function(e,t,n){for(var r=n<0?n+t:t<n?t:n;0<=--r;)e.push(r);return e}),gt:ve(function(e,t,n){for(var r=n<0?n+t:n;++r<t;)e.push(r);return e})}}).pseudos.nth=b.pseudos.eq,{radio:!0,checkbox:!0,file:!0,password:!0,image:!0})b.pseudos[e]=de(e);for(e in{submit:!0,reset:!0})b.pseudos[e]=he(e);function me(){}function xe(e){for(var t=0,n=e.length,r="";t<n;t++)r+=e[t].value;return r}function be(s,e,t){var u=e.dir,l=e.next,c=l||u,f=t&&"parentNode"===c,p=r++;return e.first?function(e,t,n){while(e=e[u])if(1===e.nodeType||f)return s(e,t,n);return!1}:function(e,t,n){var r,i,o,a=[k,p];if(n){while(e=e[u])if((1===e.nodeType||f)&&s(e,t,n))return!0}else while(e=e[u])if(1===e.nodeType||f)if(i=(o=e[S]||(e[S]={}))[e.uniqueID]||(o[e.uniqueID]={}),l&&l===e.nodeName.toLowerCase())e=e[u]||e;else{if((r=i[c])&&r[0]===k&&r[1]===p)return a[2]=r[2];if((i[c]=a)[2]=s(e,t,n))return!0}return!1}}function we(i){return 1<i.length?function(e,t,n){var r=i.length;while(r--)if(!i[r](e,t,n))return!1;return!0}:i[0]}function Te(e,t,n,r,i){for(var o,a=[],s=0,u=e.length,l=null!=t;s<u;s++)(o=e[s])&&(n&&!n(o,r,i)||(a.push(o),l&&t.push(s)));return a}function Ce(d,h,g,v,y,e){return v&&!v[S]&&(v=Ce(v)),y&&!y[S]&&(y=Ce(y,e)),le(function(e,t,n,r){var i,o,a,s=[],u=[],l=t.length,c=e||function(e,t,n){for(var r=0,i=t.length;r<i;r++)se(e,t[r],n);return n}(h||"*",n.nodeType?[n]:n,[]),f=!d||!e&&h?c:Te(c,s,d,n,r),p=g?y||(e?d:l||v)?[]:t:f;if(g&&g(f,p,n,r),v){i=Te(p,u),v(i,[],n,r),o=i.length;while(o--)(a=i[o])&&(p[u[o]]=!(f[u[o]]=a))}if(e){if(y||d){if(y){i=[],o=p.length;while(o--)(a=p[o])&&i.push(f[o]=a);y(null,p=[],i,r)}o=p.length;while(o--)(a=p[o])&&-1<(i=y?P(e,a):s[o])&&(e[i]=!(t[i]=a))}}else p=Te(p===t?p.splice(l,p.length):p),y?y(null,t,p,r):H.apply(t,p)})}function Ee(e){for(var i,t,n,r=e.length,o=b.relative[e[0].type],a=o||b.relative[" "],s=o?1:0,u=be(function(e){return e===i},a,!0),l=be(function(e){return-1<P(i,e)},a,!0),c=[function(e,t,n){var r=!o&&(n||t!==w)||((i=t).nodeType?u(e,t,n):l(e,t,n));return i=null,r}];s<r;s++)if(t=b.relative[e[s].type])c=[be(we(c),t)];else{if((t=b.filter[e[s].type].apply(null,e[s].matches))[S]){for(n=++s;n<r;n++)if(b.relative[e[n].type])break;return Ce(1<s&&we(c),1<s&&xe(e.slice(0,s-1).concat({value:" "===e[s-2].type?"*":""})).replace($,"$1"),t,s<n&&Ee(e.slice(s,n)),n<r&&Ee(e=e.slice(n)),n<r&&xe(e))}c.push(t)}return we(c)}return me.prototype=b.filters=b.pseudos,b.setFilters=new me,h=se.tokenize=function(e,t){var n,r,i,o,a,s,u,l=x[e+" "];if(l)return t?0:l.slice(0);a=e,s=[],u=b.preFilter;while(a){for(o in n&&!(r=_.exec(a))||(r&&(a=a.slice(r[0].length)||a),s.push(i=[])),n=!1,(r=z.exec(a))&&(n=r.shift(),i.push({value:n,type:r[0].replace($," ")}),a=a.slice(n.length)),b.filter)!(r=G[o].exec(a))||u[o]&&!(r=u[o](r))||(n=r.shift(),i.push({value:n,type:o,matches:r}),a=a.slice(n.length));if(!n)break}return t?a.length:a?se.error(e):x(e,s).slice(0)},f=se.compile=function(e,t){var n,v,y,m,x,r,i=[],o=[],a=A[e+" "];if(!a){t||(t=h(e)),n=t.length;while(n--)(a=Ee(t[n]))[S]?i.push(a):o.push(a);(a=A(e,(v=o,m=0<(y=i).length,x=0<v.length,r=function(e,t,n,r,i){var o,a,s,u=0,l="0",c=e&&[],f=[],p=w,d=e||x&&b.find.TAG("*",i),h=k+=null==p?1:Math.random()||.1,g=d.length;for(i&&(w=t==C||t||i);l!==g&&null!=(o=d[l]);l++){if(x&&o){a=0,t||o.ownerDocument==C||(T(o),n=!E);while(s=v[a++])if(s(o,t||C,n)){r.push(o);break}i&&(k=h)}m&&((o=!s&&o)&&u--,e&&c.push(o))}if(u+=l,m&&l!==u){a=0;while(s=y[a++])s(c,f,t,n);if(e){if(0<u)while(l--)c[l]||f[l]||(f[l]=q.call(r));f=Te(f)}H.apply(r,f),i&&!e&&0<f.length&&1<u+y.length&&se.uniqueSort(r)}return i&&(k=h,w=p),c},m?le(r):r))).selector=e}return a},g=se.select=function(e,t,n,r){var i,o,a,s,u,l="function"==typeof e&&e,c=!r&&h(e=l.selector||e);if(n=n||[],1===c.length){if(2<(o=c[0]=c[0].slice(0)).length&&"ID"===(a=o[0]).type&&9===t.nodeType&&E&&b.relative[o[1].type]){if(!(t=(b.find.ID(a.matches[0].replace(te,ne),t)||[])[0]))return n;l&&(t=t.parentNode),e=e.slice(o.shift().value.length)}i=G.needsContext.test(e)?0:o.length;while(i--){if(a=o[i],b.relative[s=a.type])break;if((u=b.find[s])&&(r=u(a.matches[0].replace(te,ne),ee.test(o[0].type)&&ye(t.parentNode)||t))){if(o.splice(i,1),!(e=r.length&&xe(o)))return H.apply(n,r),n;break}}}return(l||f(e,c))(r,t,!E,n,!t||ee.test(e)&&ye(t.parentNode)||t),n},d.sortStable=S.split("").sort(j).join("")===S,d.detectDuplicates=!!l,T(),d.sortDetached=ce(function(e){return 1&e.compareDocumentPosition(C.createElement("fieldset"))}),ce(function(e){return e.innerHTML="<a href='#'></a>","#"===e.firstChild.getAttribute("href")})||fe("type|href|height|width",function(e,t,n){if(!n)return e.getAttribute(t,"type"===t.toLowerCase()?1:2)}),d.attributes&&ce(function(e){return e.innerHTML="<input/>",e.firstChild.setAttribute("value",""),""===e.firstChild.getAttribute("value")})||fe("value",function(e,t,n){if(!n&&"input"===e.nodeName.toLowerCase())return e.defaultValue}),ce(function(e){return null==e.getAttribute("disabled")})||fe(R,function(e,t,n){var r;if(!n)return!0===e[t]?t.toLowerCase():(r=e.getAttributeNode(t))&&r.specified?r.value:null}),se}(C);S.find=d,S.expr=d.selectors,S.expr[":"]=S.expr.pseudos,S.uniqueSort=S.unique=d.uniqueSort,S.text=d.getText,S.isXMLDoc=d.isXML,S.contains=d.contains,S.escapeSelector=d.escape;var h=function(e,t,n){var r=[],i=void 0!==n;while((e=e[t])&&9!==e.nodeType)if(1===e.nodeType){if(i&&S(e).is(n))break;r.push(e)}return r},T=function(e,t){for(var n=[];e;e=e.nextSibling)1===e.nodeType&&e!==t&&n.push(e);return n},k=S.expr.match.needsContext;function A(e,t){return e.nodeName&&e.nodeName.toLowerCase()===t.toLowerCase()}var N=/^<([a-z][^\/\0>:\x20\t\r\n\f]*)[\x20\t\r\n\f]*\/?>(?:<\/\1>|)$/i;function j(e,n,r){return m(n)?S.grep(e,function(e,t){return!!n.call(e,t,e)!==r}):n.nodeType?S.grep(e,function(e){return e===n!==r}):"string"!=typeof n?S.grep(e,function(e){return-1<i.call(n,e)!==r}):S.filter(n,e,r)}S.filter=function(e,t,n){var r=t[0];return n&&(e=":not("+e+")"),1===t.length&&1===r.nodeType?S.find.matchesSelector(r,e)?[r]:[]:S.find.matches(e,S.grep(t,function(e){return 1===e.nodeType}))},S.fn.extend({find:function(e){var t,n,r=this.length,i=this;if("string"!=typeof e)return this.pushStack(S(e).filter(function(){for(t=0;t<r;t++)if(S.contains(i[t],this))return!0}));for(n=this.pushStack([]),t=0;t<r;t++)S.find(e,i[t],n);return 1<r?S.uniqueSort(n):n},filter:function(e){return this.pushStack(j(this,e||[],!1))},not:function(e){return this.pushStack(j(this,e||[],!0))},is:function(e){return!!j(this,"string"==typeof e&&k.test(e)?S(e):e||[],!1).length}});var D,q=/^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]+))$/;(S.fn.init=function(e,t,n){var r,i;if(!e)return this;if(n=n||D,"string"==typeof e){if(!(r="<"===e[0]&&">"===e[e.length-1]&&3<=e.length?[null,e,null]:q.exec(e))||!r[1]&&t)return!t||t.jquery?(t||n).find(e):this.constructor(t).find(e);if(r[1]){if(t=t instanceof S?t[0]:t,S.merge(this,S.parseHTML(r[1],t&&t.nodeType?t.ownerDocument||t:E,!0)),N.test(r[1])&&S.isPlainObject(t))for(r in t)m(this[r])?this[r](t[r]):this.attr(r,t[r]);return this}return(i=E.getElementById(r[2]))&&(this[0]=i,this.length=1),this}return e.nodeType?(this[0]=e,this.length=1,this):m(e)?void 0!==n.ready?n.ready(e):e(S):S.makeArray(e,this)}).prototype=S.fn,D=S(E);var L=/^(?:parents|prev(?:Until|All))/,H={children:!0,contents:!0,next:!0,prev:!0};function O(e,t){while((e=e[t])&&1!==e.nodeType);return e}S.fn.extend({has:function(e){var t=S(e,this),n=t.length;return this.filter(function(){for(var e=0;e<n;e++)if(S.contains(this,t[e]))return!0})},closest:function(e,t){var n,r=0,i=this.length,o=[],a="string"!=typeof e&&S(e);if(!k.test(e))for(;r<i;r++)for(n=this[r];n&&n!==t;n=n.parentNode)if(n.nodeType<11&&(a?-1<a.index(n):1===n.nodeType&&S.find.matchesSelector(n,e))){o.push(n);break}return this.pushStack(1<o.length?S.uniqueSort(o):o)},index:function(e){return e?"string"==typeof e?i.call(S(e),this[0]):i.call(this,e.jquery?e[0]:e):this[0]&&this[0].parentNode?this.first().prevAll().length:-1},add:function(e,t){return this.pushStack(S.uniqueSort(S.merge(this.get(),S(e,t))))},addBack:function(e){return this.add(null==e?this.prevObject:this.prevObject.filter(e))}}),S.each({parent:function(e){var t=e.parentNode;return t&&11!==t.nodeType?t:null},parents:function(e){return h(e,"parentNode")},parentsUntil:function(e,t,n){return h(e,"parentNode",n)},next:function(e){return O(e,"nextSibling")},prev:function(e){return O(e,"previousSibling")},nextAll:function(e){return h(e,"nextSibling")},prevAll:function(e){return h(e,"previousSibling")},nextUntil:function(e,t,n){return h(e,"nextSibling",n)},prevUntil:function(e,t,n){return h(e,"previousSibling",n)},siblings:function(e){return T((e.parentNode||{}).firstChild,e)},children:function(e){return T(e.firstChild)},contents:function(e){return null!=e.contentDocument&&r(e.contentDocument)?e.contentDocument:(A(e,"template")&&(e=e.content||e),S.merge([],e.childNodes))}},function(r,i){S.fn[r]=function(e,t){var n=S.map(this,i,e);return"Until"!==r.slice(-5)&&(t=e),t&&"string"==typeof t&&(n=S.filter(t,n)),1<this.length&&(H[r]||S.uniqueSort(n),L.test(r)&&n.reverse()),this.pushStack(n)}});var P=/[^\x20\t\r\n\f]+/g;function R(e){return e}function M(e){throw e}function I(e,t,n,r){var i;try{e&&m(i=e.promise)?i.call(e).done(t).fail(n):e&&m(i=e.then)?i.call(e,t,n):t.apply(void 0,[e].slice(r))}catch(e){n.apply(void 0,[e])}}S.Callbacks=function(r){var e,n;r="string"==typeof r?(e=r,n={},S.each(e.match(P)||[],function(e,t){n[t]=!0}),n):S.extend({},r);var i,t,o,a,s=[],u=[],l=-1,c=function(){for(a=a||r.once,o=i=!0;u.length;l=-1){t=u.shift();while(++l<s.length)!1===s[l].apply(t[0],t[1])&&r.stopOnFalse&&(l=s.length,t=!1)}r.memory||(t=!1),i=!1,a&&(s=t?[]:"")},f={add:function(){return s&&(t&&!i&&(l=s.length-1,u.push(t)),function n(e){S.each(e,function(e,t){m(t)?r.unique&&f.has(t)||s.push(t):t&&t.length&&"string"!==w(t)&&n(t)})}(arguments),t&&!i&&c()),this},remove:function(){return S.each(arguments,function(e,t){var n;while(-1<(n=S.inArray(t,s,n)))s.splice(n,1),n<=l&&l--}),this},has:function(e){return e?-1<S.inArray(e,s):0<s.length},empty:function(){return s&&(s=[]),this},disable:function(){return a=u=[],s=t="",this},disabled:function(){return!s},lock:function(){return a=u=[],t||i||(s=t=""),this},locked:function(){return!!a},fireWith:function(e,t){return a||(t=[e,(t=t||[]).slice?t.slice():t],u.push(t),i||c()),this},fire:function(){return f.fireWith(this,arguments),this},fired:function(){return!!o}};return f},S.extend({Deferred:function(e){var o=[["notify","progress",S.Callbacks("memory"),S.Callbacks("memory"),2],["resolve","done",S.Callbacks("once memory"),S.Callbacks("once memory"),0,"resolved"],["reject","fail",S.Callbacks("once memory"),S.Callbacks("once memory"),1,"rejected"]],i="pending",a={state:function(){return i},always:function(){return s.done(arguments).fail(arguments),this},"catch":function(e){return a.then(null,e)},pipe:function(){var i=arguments;return S.Deferred(function(r){S.each(o,function(e,t){var n=m(i[t[4]])&&i[t[4]];s[t[1]](function(){var e=n&&n.apply(this,arguments);e&&m(e.promise)?e.promise().progress(r.notify).done(r.resolve).fail(r.reject):r[t[0]+"With"](this,n?[e]:arguments)})}),i=null}).promise()},then:function(t,n,r){var u=0;function l(i,o,a,s){return function(){var n=this,r=arguments,e=function(){var e,t;if(!(i<u)){if((e=a.apply(n,r))===o.promise())throw new TypeError("Thenable self-resolution");t=e&&("object"==typeof e||"function"==typeof e)&&e.then,m(t)?s?t.call(e,l(u,o,R,s),l(u,o,M,s)):(u++,t.call(e,l(u,o,R,s),l(u,o,M,s),l(u,o,R,o.notifyWith))):(a!==R&&(n=void 0,r=[e]),(s||o.resolveWith)(n,r))}},t=s?e:function(){try{e()}catch(e){S.Deferred.exceptionHook&&S.Deferred.exceptionHook(e,t.stackTrace),u<=i+1&&(a!==M&&(n=void 0,r=[e]),o.rejectWith(n,r))}};i?t():(S.Deferred.getStackHook&&(t.stackTrace=S.Deferred.getStackHook()),C.setTimeout(t))}}return S.Deferred(function(e){o[0][3].add(l(0,e,m(r)?r:R,e.notifyWith)),o[1][3].add(l(0,e,m(t)?t:R)),o[2][3].add(l(0,e,m(n)?n:M))}).promise()},promise:function(e){return null!=e?S.extend(e,a):a}},s={};return S.each(o,function(e,t){var n=t[2],r=t[5];a[t[1]]=n.add,r&&n.add(function(){i=r},o[3-e][2].disable,o[3-e][3].disable,o[0][2].lock,o[0][3].lock),n.add(t[3].fire),s[t[0]]=function(){return s[t[0]+"With"](this===s?void 0:this,arguments),this},s[t[0]+"With"]=n.fireWith}),a.promise(s),e&&e.call(s,s),s},when:function(e){var n=arguments.length,t=n,r=Array(t),i=s.call(arguments),o=S.Deferred(),a=function(t){return function(e){r[t]=this,i[t]=1<arguments.length?s.call(arguments):e,--n||o.resolveWith(r,i)}};if(n<=1&&(I(e,o.done(a(t)).resolve,o.reject,!n),"pending"===o.state()||m(i[t]&&i[t].then)))return o.then();while(t--)I(i[t],a(t),o.reject);return o.promise()}});var W=/^(Eval|Internal|Range|Reference|Syntax|Type|URI)Error$/;S.Deferred.exceptionHook=function(e,t){C.console&&C.console.warn&&e&&W.test(e.name)&&C.console.warn("jQuery.Deferred exception: "+e.message,e.stack,t)},S.readyException=function(e){C.setTimeout(function(){throw e})};var F=S.Deferred();function B(){E.removeEventListener("DOMContentLoaded",B),C.removeEventListener("load",B),S.ready()}S.fn.ready=function(e){return F.then(e)["catch"](function(e){S.readyException(e)}),this},S.extend({isReady:!1,readyWait:1,ready:function(e){(!0===e?--S.readyWait:S.isReady)||(S.isReady=!0)!==e&&0<--S.readyWait||F.resolveWith(E,[S])}}),S.ready.then=F.then,"complete"===E.readyState||"loading"!==E.readyState&&!E.documentElement.doScroll?C.setTimeout(S.ready):(E.addEventListener("DOMContentLoaded",B),C.addEventListener("load",B));var $=function(e,t,n,r,i,o,a){var s=0,u=e.length,l=null==n;if("object"===w(n))for(s in i=!0,n)$(e,t,s,n[s],!0,o,a);else if(void 0!==r&&(i=!0,m(r)||(a=!0),l&&(a?(t.call(e,r),t=null):(l=t,t=function(e,t,n){return l.call(S(e),n)})),t))for(;s<u;s++)t(e[s],n,a?r:r.call(e[s],s,t(e[s],n)));return i?e:l?t.call(e):u?t(e[0],n):o},_=/^-ms-/,z=/-([a-z])/g;function U(e,t){return t.toUpperCase()}function X(e){return e.replace(_,"ms-").replace(z,U)}var V=function(e){return 1===e.nodeType||9===e.nodeType||!+e.nodeType};function G(){this.expando=S.expando+G.uid++}G.uid=1,G.prototype={cache:function(e){var t=e[this.expando];return t||(t={},V(e)&&(e.nodeType?e[this.expando]=t:Object.defineProperty(e,this.expando,{value:t,configurable:!0}))),t},set:function(e,t,n){var r,i=this.cache(e);if("string"==typeof t)i[X(t)]=n;else for(r in t)i[X(r)]=t[r];return i},get:function(e,t){return void 0===t?this.cache(e):e[this.expando]&&e[this.expando][X(t)]},access:function(e,t,n){return void 0===t||t&&"string"==typeof t&&void 0===n?this.get(e,t):(this.set(e,t,n),void 0!==n?n:t)},remove:function(e,t){var n,r=e[this.expando];if(void 0!==r){if(void 0!==t){n=(t=Array.isArray(t)?t.map(X):(t=X(t))in r?[t]:t.match(P)||[]).length;while(n--)delete r[t[n]]}(void 0===t||S.isEmptyObject(r))&&(e.nodeType?e[this.expando]=void 0:delete e[this.expando])}},hasData:function(e){var t=e[this.expando];return void 0!==t&&!S.isEmptyObject(t)}};var Y=new G,Q=new G,J=/^(?:\{[\w\W]*\}|\[[\w\W]*\])$/,K=/[A-Z]/g;function Z(e,t,n){var r,i;if(void 0===n&&1===e.nodeType)if(r="data-"+t.replace(K,"-$&").toLowerCase(),"string"==typeof(n=e.getAttribute(r))){try{n="true"===(i=n)||"false"!==i&&("null"===i?null:i===+i+""?+i:J.test(i)?JSON.parse(i):i)}catch(e){}Q.set(e,t,n)}else n=void 0;return n}S.extend({hasData:function(e){return Q.hasData(e)||Y.hasData(e)},data:function(e,t,n){return Q.access(e,t,n)},removeData:function(e,t){Q.remove(e,t)},_data:function(e,t,n){return Y.access(e,t,n)},_removeData:function(e,t){Y.remove(e,t)}}),S.fn.extend({data:function(n,e){var t,r,i,o=this[0],a=o&&o.attributes;if(void 0===n){if(this.length&&(i=Q.get(o),1===o.nodeType&&!Y.get(o,"hasDataAttrs"))){t=a.length;while(t--)a[t]&&0===(r=a[t].name).indexOf("data-")&&(r=X(r.slice(5)),Z(o,r,i[r]));Y.set(o,"hasDataAttrs",!0)}return i}return"object"==typeof n?this.each(function(){Q.set(this,n)}):$(this,function(e){var t;if(o&&void 0===e)return void 0!==(t=Q.get(o,n))?t:void 0!==(t=Z(o,n))?t:void 0;this.each(function(){Q.set(this,n,e)})},null,e,1<arguments.length,null,!0)},removeData:function(e){return this.each(function(){Q.remove(this,e)})}}),S.extend({queue:function(e,t,n){var r;if(e)return t=(t||"fx")+"queue",r=Y.get(e,t),n&&(!r||Array.isArray(n)?r=Y.access(e,t,S.makeArray(n)):r.push(n)),r||[]},dequeue:function(e,t){t=t||"fx";var n=S.queue(e,t),r=n.length,i=n.shift(),o=S._queueHooks(e,t);"inprogress"===i&&(i=n.shift(),r--),i&&("fx"===t&&n.unshift("inprogress"),delete o.stop,i.call(e,function(){S.dequeue(e,t)},o)),!r&&o&&o.empty.fire()},_queueHooks:function(e,t){var n=t+"queueHooks";return Y.get(e,n)||Y.access(e,n,{empty:S.Callbacks("once memory").add(function(){Y.remove(e,[t+"queue",n])})})}}),S.fn.extend({queue:function(t,n){var e=2;return"string"!=typeof t&&(n=t,t="fx",e--),arguments.length<e?S.queue(this[0],t):void 0===n?this:this.each(function(){var e=S.queue(this,t,n);S._queueHooks(this,t),"fx"===t&&"inprogress"!==e[0]&&S.dequeue(this,t)})},dequeue:function(e){return this.each(function(){S.dequeue(this,e)})},clearQueue:function(e){return this.queue(e||"fx",[])},promise:function(e,t){var n,r=1,i=S.Deferred(),o=this,a=this.length,s=function(){--r||i.resolveWith(o,[o])};"string"!=typeof e&&(t=e,e=void 0),e=e||"fx";while(a--)(n=Y.get(o[a],e+"queueHooks"))&&n.empty&&(r++,n.empty.add(s));return s(),i.promise(t)}});var ee=/[+-]?(?:\d*\.|)\d+(?:[eE][+-]?\d+|)/.source,te=new RegExp("^(?:([+-])=|)("+ee+")([a-z%]*)$","i"),ne=["Top","Right","Bottom","Left"],re=E.documentElement,ie=function(e){return S.contains(e.ownerDocument,e)},oe={composed:!0};re.getRootNode&&(ie=function(e){return S.contains(e.ownerDocument,e)||e.getRootNode(oe)===e.ownerDocument});var ae=function(e,t){return"none"===(e=t||e).style.display||""===e.style.display&&ie(e)&&"none"===S.css(e,"display")};function se(e,t,n,r){var i,o,a=20,s=r?function(){return r.cur()}:function(){return S.css(e,t,"")},u=s(),l=n&&n[3]||(S.cssNumber[t]?"":"px"),c=e.nodeType&&(S.cssNumber[t]||"px"!==l&&+u)&&te.exec(S.css(e,t));if(c&&c[3]!==l){u/=2,l=l||c[3],c=+u||1;while(a--)S.style(e,t,c+l),(1-o)*(1-(o=s()/u||.5))<=0&&(a=0),c/=o;c*=2,S.style(e,t,c+l),n=n||[]}return n&&(c=+c||+u||0,i=n[1]?c+(n[1]+1)*n[2]:+n[2],r&&(r.unit=l,r.start=c,r.end=i)),i}var ue={};function le(e,t){for(var n,r,i,o,a,s,u,l=[],c=0,f=e.length;c<f;c++)(r=e[c]).style&&(n=r.style.display,t?("none"===n&&(l[c]=Y.get(r,"display")||null,l[c]||(r.style.display="")),""===r.style.display&&ae(r)&&(l[c]=(u=a=o=void 0,a=(i=r).ownerDocument,s=i.nodeName,(u=ue[s])||(o=a.body.appendChild(a.createElement(s)),u=S.css(o,"display"),o.parentNode.removeChild(o),"none"===u&&(u="block"),ue[s]=u)))):"none"!==n&&(l[c]="none",Y.set(r,"display",n)));for(c=0;c<f;c++)null!=l[c]&&(e[c].style.display=l[c]);return e}S.fn.extend({show:function(){return le(this,!0)},hide:function(){return le(this)},toggle:function(e){return"boolean"==typeof e?e?this.show():this.hide():this.each(function(){ae(this)?S(this).show():S(this).hide()})}});var ce,fe,pe=/^(?:checkbox|radio)$/i,de=/<([a-z][^\/\0>\x20\t\r\n\f]*)/i,he=/^$|^module$|\/(?:java|ecma)script/i;ce=E.createDocumentFragment().appendChild(E.createElement("div")),(fe=E.createElement("input")).setAttribute("type","radio"),fe.setAttribute("checked","checked"),fe.setAttribute("name","t"),ce.appendChild(fe),y.checkClone=ce.cloneNode(!0).cloneNode(!0).lastChild.checked,ce.innerHTML="<textarea>x</textarea>",y.noCloneChecked=!!ce.cloneNode(!0).lastChild.defaultValue,ce.innerHTML="<option></option>",y.option=!!ce.lastChild;var ge={thead:[1,"<table>","</table>"],col:[2,"<table><colgroup>","</colgroup></table>"],tr:[2,"<table><tbody>","</tbody></table>"],td:[3,"<table><tbody><tr>","</tr></tbody></table>"],_default:[0,"",""]};function ve(e,t){var n;return n="undefined"!=typeof e.getElementsByTagName?e.getElementsByTagName(t||"*"):"undefined"!=typeof e.querySelectorAll?e.querySelectorAll(t||"*"):[],void 0===t||t&&A(e,t)?S.merge([e],n):n}function ye(e,t){for(var n=0,r=e.length;n<r;n++)Y.set(e[n],"globalEval",!t||Y.get(t[n],"globalEval"))}ge.tbody=ge.tfoot=ge.colgroup=ge.caption=ge.thead,ge.th=ge.td,y.option||(ge.optgroup=ge.option=[1,"<select multiple='multiple'>","</select>"]);var me=/<|&#?\w+;/;function xe(e,t,n,r,i){for(var o,a,s,u,l,c,f=t.createDocumentFragment(),p=[],d=0,h=e.length;d<h;d++)if((o=e[d])||0===o)if("object"===w(o))S.merge(p,o.nodeType?[o]:o);else if(me.test(o)){a=a||f.appendChild(t.createElement("div")),s=(de.exec(o)||["",""])[1].toLowerCase(),u=ge[s]||ge._default,a.innerHTML=u[1]+S.htmlPrefilter(o)+u[2],c=u[0];while(c--)a=a.lastChild;S.merge(p,a.childNodes),(a=f.firstChild).textContent=""}else p.push(t.createTextNode(o));f.textContent="",d=0;while(o=p[d++])if(r&&-1<S.inArray(o,r))i&&i.push(o);else if(l=ie(o),a=ve(f.appendChild(o),"script"),l&&ye(a),n){c=0;while(o=a[c++])he.test(o.type||"")&&n.push(o)}return f}var be=/^([^.]*)(?:\.(.+)|)/;function we(){return!0}function Te(){return!1}function Ce(e,t){return e===function(){try{return E.activeElement}catch(e){}}()==("focus"===t)}function Ee(e,t,n,r,i,o){var a,s;if("object"==typeof t){for(s in"string"!=typeof n&&(r=r||n,n=void 0),t)Ee(e,s,n,r,t[s],o);return e}if(null==r&&null==i?(i=n,r=n=void 0):null==i&&("string"==typeof n?(i=r,r=void 0):(i=r,r=n,n=void 0)),!1===i)i=Te;else if(!i)return e;return 1===o&&(a=i,(i=function(e){return S().off(e),a.apply(this,arguments)}).guid=a.guid||(a.guid=S.guid++)),e.each(function(){S.event.add(this,t,i,r,n)})}function Se(e,i,o){o?(Y.set(e,i,!1),S.event.add(e,i,{namespace:!1,handler:function(e){var t,n,r=Y.get(this,i);if(1&e.isTrigger&&this[i]){if(r.length)(S.event.special[i]||{}).delegateType&&e.stopPropagation();else if(r=s.call(arguments),Y.set(this,i,r),t=o(this,i),this[i](),r!==(n=Y.get(this,i))||t?Y.set(this,i,!1):n={},r!==n)return e.stopImmediatePropagation(),e.preventDefault(),n&&n.value}else r.length&&(Y.set(this,i,{value:S.event.trigger(S.extend(r[0],S.Event.prototype),r.slice(1),this)}),e.stopImmediatePropagation())}})):void 0===Y.get(e,i)&&S.event.add(e,i,we)}S.event={global:{},add:function(t,e,n,r,i){var o,a,s,u,l,c,f,p,d,h,g,v=Y.get(t);if(V(t)){n.handler&&(n=(o=n).handler,i=o.selector),i&&S.find.matchesSelector(re,i),n.guid||(n.guid=S.guid++),(u=v.events)||(u=v.events=Object.create(null)),(a=v.handle)||(a=v.handle=function(e){return"undefined"!=typeof S&&S.event.triggered!==e.type?S.event.dispatch.apply(t,arguments):void 0}),l=(e=(e||"").match(P)||[""]).length;while(l--)d=g=(s=be.exec(e[l])||[])[1],h=(s[2]||"").split(".").sort(),d&&(f=S.event.special[d]||{},d=(i?f.delegateType:f.bindType)||d,f=S.event.special[d]||{},c=S.extend({type:d,origType:g,data:r,handler:n,guid:n.guid,selector:i,needsContext:i&&S.expr.match.needsContext.test(i),namespace:h.join(".")},o),(p=u[d])||((p=u[d]=[]).delegateCount=0,f.setup&&!1!==f.setup.call(t,r,h,a)||t.addEventListener&&t.addEventListener(d,a)),f.add&&(f.add.call(t,c),c.handler.guid||(c.handler.guid=n.guid)),i?p.splice(p.delegateCount++,0,c):p.push(c),S.event.global[d]=!0)}},remove:function(e,t,n,r,i){var o,a,s,u,l,c,f,p,d,h,g,v=Y.hasData(e)&&Y.get(e);if(v&&(u=v.events)){l=(t=(t||"").match(P)||[""]).length;while(l--)if(d=g=(s=be.exec(t[l])||[])[1],h=(s[2]||"").split(".").sort(),d){f=S.event.special[d]||{},p=u[d=(r?f.delegateType:f.bindType)||d]||[],s=s[2]&&new RegExp("(^|\\.)"+h.join("\\.(?:.*\\.|)")+"(\\.|$)"),a=o=p.length;while(o--)c=p[o],!i&&g!==c.origType||n&&n.guid!==c.guid||s&&!s.test(c.namespace)||r&&r!==c.selector&&("**"!==r||!c.selector)||(p.splice(o,1),c.selector&&p.delegateCount--,f.remove&&f.remove.call(e,c));a&&!p.length&&(f.teardown&&!1!==f.teardown.call(e,h,v.handle)||S.removeEvent(e,d,v.handle),delete u[d])}else for(d in u)S.event.remove(e,d+t[l],n,r,!0);S.isEmptyObject(u)&&Y.remove(e,"handle events")}},dispatch:function(e){var t,n,r,i,o,a,s=new Array(arguments.length),u=S.event.fix(e),l=(Y.get(this,"events")||Object.create(null))[u.type]||[],c=S.event.special[u.type]||{};for(s[0]=u,t=1;t<arguments.length;t++)s[t]=arguments[t];if(u.delegateTarget=this,!c.preDispatch||!1!==c.preDispatch.call(this,u)){a=S.event.handlers.call(this,u,l),t=0;while((i=a[t++])&&!u.isPropagationStopped()){u.currentTarget=i.elem,n=0;while((o=i.handlers[n++])&&!u.isImmediatePropagationStopped())u.rnamespace&&!1!==o.namespace&&!u.rnamespace.test(o.namespace)||(u.handleObj=o,u.data=o.data,void 0!==(r=((S.event.special[o.origType]||{}).handle||o.handler).apply(i.elem,s))&&!1===(u.result=r)&&(u.preventDefault(),u.stopPropagation()))}return c.postDispatch&&c.postDispatch.call(this,u),u.result}},handlers:function(e,t){var n,r,i,o,a,s=[],u=t.delegateCount,l=e.target;if(u&&l.nodeType&&!("click"===e.type&&1<=e.button))for(;l!==this;l=l.parentNode||this)if(1===l.nodeType&&("click"!==e.type||!0!==l.disabled)){for(o=[],a={},n=0;n<u;n++)void 0===a[i=(r=t[n]).selector+" "]&&(a[i]=r.needsContext?-1<S(i,this).index(l):S.find(i,this,null,[l]).length),a[i]&&o.push(r);o.length&&s.push({elem:l,handlers:o})}return l=this,u<t.length&&s.push({elem:l,handlers:t.slice(u)}),s},addProp:function(t,e){Object.defineProperty(S.Event.prototype,t,{enumerable:!0,configurable:!0,get:m(e)?function(){if(this.originalEvent)return e(this.originalEvent)}:function(){if(this.originalEvent)return this.originalEvent[t]},set:function(e){Object.defineProperty(this,t,{enumerable:!0,configurable:!0,writable:!0,value:e})}})},fix:function(e){return e[S.expando]?e:new S.Event(e)},special:{load:{noBubble:!0},click:{setup:function(e){var t=this||e;return pe.test(t.type)&&t.click&&A(t,"input")&&Se(t,"click",we),!1},trigger:function(e){var t=this||e;return pe.test(t.type)&&t.click&&A(t,"input")&&Se(t,"click"),!0},_default:function(e){var t=e.target;return pe.test(t.type)&&t.click&&A(t,"input")&&Y.get(t,"click")||A(t,"a")}},beforeunload:{postDispatch:function(e){void 0!==e.result&&e.originalEvent&&(e.originalEvent.returnValue=e.result)}}}},S.removeEvent=function(e,t,n){e.removeEventListener&&e.removeEventListener(t,n)},S.Event=function(e,t){if(!(this instanceof S.Event))return new S.Event(e,t);e&&e.type?(this.originalEvent=e,this.type=e.type,this.isDefaultPrevented=e.defaultPrevented||void 0===e.defaultPrevented&&!1===e.returnValue?we:Te,this.target=e.target&&3===e.target.nodeType?e.target.parentNode:e.target,this.currentTarget=e.currentTarget,this.relatedTarget=e.relatedTarget):this.type=e,t&&S.extend(this,t),this.timeStamp=e&&e.timeStamp||Date.now(),this[S.expando]=!0},S.Event.prototype={constructor:S.Event,isDefaultPrevented:Te,isPropagationStopped:Te,isImmediatePropagationStopped:Te,isSimulated:!1,preventDefault:function(){var e=this.originalEvent;this.isDefaultPrevented=we,e&&!this.isSimulated&&e.preventDefault()},stopPropagation:function(){var e=this.originalEvent;this.isPropagationStopped=we,e&&!this.isSimulated&&e.stopPropagation()},stopImmediatePropagation:function(){var e=this.originalEvent;this.isImmediatePropagationStopped=we,e&&!this.isSimulated&&e.stopImmediatePropagation(),this.stopPropagation()}},S.each({altKey:!0,bubbles:!0,cancelable:!0,changedTouches:!0,ctrlKey:!0,detail:!0,eventPhase:!0,metaKey:!0,pageX:!0,pageY:!0,shiftKey:!0,view:!0,"char":!0,code:!0,charCode:!0,key:!0,keyCode:!0,button:!0,buttons:!0,clientX:!0,clientY:!0,offsetX:!0,offsetY:!0,pointerId:!0,pointerType:!0,screenX:!0,screenY:!0,targetTouches:!0,toElement:!0,touches:!0,which:!0},S.event.addProp),S.each({focus:"focusin",blur:"focusout"},function(e,t){S.event.special[e]={setup:function(){return Se(this,e,Ce),!1},trigger:function(){return Se(this,e),!0},_default:function(){return!0},delegateType:t}}),S.each({mouseenter:"mouseover",mouseleave:"mouseout",pointerenter:"pointerover",pointerleave:"pointerout"},function(e,i){S.event.special[e]={delegateType:i,bindType:i,handle:function(e){var t,n=e.relatedTarget,r=e.handleObj;return n&&(n===this||S.contains(this,n))||(e.type=r.origType,t=r.handler.apply(this,arguments),e.type=i),t}}}),S.fn.extend({on:function(e,t,n,r){return Ee(this,e,t,n,r)},one:function(e,t,n,r){return Ee(this,e,t,n,r,1)},off:function(e,t,n){var r,i;if(e&&e.preventDefault&&e.handleObj)return r=e.handleObj,S(e.delegateTarget).off(r.namespace?r.origType+"."+r.namespace:r.origType,r.selector,r.handler),this;if("object"==typeof e){for(i in e)this.off(i,t,e[i]);return this}return!1!==t&&"function"!=typeof t||(n=t,t=void 0),!1===n&&(n=Te),this.each(function(){S.event.remove(this,e,n,t)})}});var ke=/<script|<style|<link/i,Ae=/checked\s*(?:[^=]|=\s*.checked.)/i,Ne=/^\s*<!(?:\[CDATA\[|--)|(?:\]\]|--)>\s*$/g;function je(e,t){return A(e,"table")&&A(11!==t.nodeType?t:t.firstChild,"tr")&&S(e).children("tbody")[0]||e}function De(e){return e.type=(null!==e.getAttribute("type"))+"/"+e.type,e}function qe(e){return"true/"===(e.type||"").slice(0,5)?e.type=e.type.slice(5):e.removeAttribute("type"),e}function Le(e,t){var n,r,i,o,a,s;if(1===t.nodeType){if(Y.hasData(e)&&(s=Y.get(e).events))for(i in Y.remove(t,"handle events"),s)for(n=0,r=s[i].length;n<r;n++)S.event.add(t,i,s[i][n]);Q.hasData(e)&&(o=Q.access(e),a=S.extend({},o),Q.set(t,a))}}function He(n,r,i,o){r=g(r);var e,t,a,s,u,l,c=0,f=n.length,p=f-1,d=r[0],h=m(d);if(h||1<f&&"string"==typeof d&&!y.checkClone&&Ae.test(d))return n.each(function(e){var t=n.eq(e);h&&(r[0]=d.call(this,e,t.html())),He(t,r,i,o)});if(f&&(t=(e=xe(r,n[0].ownerDocument,!1,n,o)).firstChild,1===e.childNodes.length&&(e=t),t||o)){for(s=(a=S.map(ve(e,"script"),De)).length;c<f;c++)u=e,c!==p&&(u=S.clone(u,!0,!0),s&&S.merge(a,ve(u,"script"))),i.call(n[c],u,c);if(s)for(l=a[a.length-1].ownerDocument,S.map(a,qe),c=0;c<s;c++)u=a[c],he.test(u.type||"")&&!Y.access(u,"globalEval")&&S.contains(l,u)&&(u.src&&"module"!==(u.type||"").toLowerCase()?S._evalUrl&&!u.noModule&&S._evalUrl(u.src,{nonce:u.nonce||u.getAttribute("nonce")},l):b(u.textContent.replace(Ne,""),u,l))}return n}function Oe(e,t,n){for(var r,i=t?S.filter(t,e):e,o=0;null!=(r=i[o]);o++)n||1!==r.nodeType||S.cleanData(ve(r)),r.parentNode&&(n&&ie(r)&&ye(ve(r,"script")),r.parentNode.removeChild(r));return e}S.extend({htmlPrefilter:function(e){return e},clone:function(e,t,n){var r,i,o,a,s,u,l,c=e.cloneNode(!0),f=ie(e);if(!(y.noCloneChecked||1!==e.nodeType&&11!==e.nodeType||S.isXMLDoc(e)))for(a=ve(c),r=0,i=(o=ve(e)).length;r<i;r++)s=o[r],u=a[r],void 0,"input"===(l=u.nodeName.toLowerCase())&&pe.test(s.type)?u.checked=s.checked:"input"!==l&&"textarea"!==l||(u.defaultValue=s.defaultValue);if(t)if(n)for(o=o||ve(e),a=a||ve(c),r=0,i=o.length;r<i;r++)Le(o[r],a[r]);else Le(e,c);return 0<(a=ve(c,"script")).length&&ye(a,!f&&ve(e,"script")),c},cleanData:function(e){for(var t,n,r,i=S.event.special,o=0;void 0!==(n=e[o]);o++)if(V(n)){if(t=n[Y.expando]){if(t.events)for(r in t.events)i[r]?S.event.remove(n,r):S.removeEvent(n,r,t.handle);n[Y.expando]=void 0}n[Q.expando]&&(n[Q.expando]=void 0)}}}),S.fn.extend({detach:function(e){return Oe(this,e,!0)},remove:function(e){return Oe(this,e)},text:function(e){return $(this,function(e){return void 0===e?S.text(this):this.empty().each(function(){1!==this.nodeType&&11!==this.nodeType&&9!==this.nodeType||(this.textContent=e)})},null,e,arguments.length)},append:function(){return He(this,arguments,function(e){1!==this.nodeType&&11!==this.nodeType&&9!==this.nodeType||je(this,e).appendChild(e)})},prepend:function(){return He(this,arguments,function(e){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var t=je(this,e);t.insertBefore(e,t.firstChild)}})},before:function(){return He(this,arguments,function(e){this.parentNode&&this.parentNode.insertBefore(e,this)})},after:function(){return He(this,arguments,function(e){this.parentNode&&this.parentNode.insertBefore(e,this.nextSibling)})},empty:function(){for(var e,t=0;null!=(e=this[t]);t++)1===e.nodeType&&(S.cleanData(ve(e,!1)),e.textContent="");return this},clone:function(e,t){return e=null!=e&&e,t=null==t?e:t,this.map(function(){return S.clone(this,e,t)})},html:function(e){return $(this,function(e){var t=this[0]||{},n=0,r=this.length;if(void 0===e&&1===t.nodeType)return t.innerHTML;if("string"==typeof e&&!ke.test(e)&&!ge[(de.exec(e)||["",""])[1].toLowerCase()]){e=S.htmlPrefilter(e);try{for(;n<r;n++)1===(t=this[n]||{}).nodeType&&(S.cleanData(ve(t,!1)),t.innerHTML=e);t=0}catch(e){}}t&&this.empty().append(e)},null,e,arguments.length)},replaceWith:function(){var n=[];return He(this,arguments,function(e){var t=this.parentNode;S.inArray(this,n)<0&&(S.cleanData(ve(this)),t&&t.replaceChild(e,this))},n)}}),S.each({appendTo:"append",prependTo:"prepend",insertBefore:"before",insertAfter:"after",replaceAll:"replaceWith"},function(e,a){S.fn[e]=function(e){for(var t,n=[],r=S(e),i=r.length-1,o=0;o<=i;o++)t=o===i?this:this.clone(!0),S(r[o])[a](t),u.apply(n,t.get());return this.pushStack(n)}});var Pe=new RegExp("^("+ee+")(?!px)[a-z%]+$","i"),Re=function(e){var t=e.ownerDocument.defaultView;return t&&t.opener||(t=C),t.getComputedStyle(e)},Me=function(e,t,n){var r,i,o={};for(i in t)o[i]=e.style[i],e.style[i]=t[i];for(i in r=n.call(e),t)e.style[i]=o[i];return r},Ie=new RegExp(ne.join("|"),"i");function We(e,t,n){var r,i,o,a,s=e.style;return(n=n||Re(e))&&(""!==(a=n.getPropertyValue(t)||n[t])||ie(e)||(a=S.style(e,t)),!y.pixelBoxStyles()&&Pe.test(a)&&Ie.test(t)&&(r=s.width,i=s.minWidth,o=s.maxWidth,s.minWidth=s.maxWidth=s.width=a,a=n.width,s.width=r,s.minWidth=i,s.maxWidth=o)),void 0!==a?a+"":a}function Fe(e,t){return{get:function(){if(!e())return(this.get=t).apply(this,arguments);delete this.get}}}!function(){function e(){if(l){u.style.cssText="position:absolute;left:-11111px;width:60px;margin-top:1px;padding:0;border:0",l.style.cssText="position:relative;display:block;box-sizing:border-box;overflow:scroll;margin:auto;border:1px;padding:1px;width:60%;top:1%",re.appendChild(u).appendChild(l);var e=C.getComputedStyle(l);n="1%"!==e.top,s=12===t(e.marginLeft),l.style.right="60%",o=36===t(e.right),r=36===t(e.width),l.style.position="absolute",i=12===t(l.offsetWidth/3),re.removeChild(u),l=null}}function t(e){return Math.round(parseFloat(e))}var n,r,i,o,a,s,u=E.createElement("div"),l=E.createElement("div");l.style&&(l.style.backgroundClip="content-box",l.cloneNode(!0).style.backgroundClip="",y.clearCloneStyle="content-box"===l.style.backgroundClip,S.extend(y,{boxSizingReliable:function(){return e(),r},pixelBoxStyles:function(){return e(),o},pixelPosition:function(){return e(),n},reliableMarginLeft:function(){return e(),s},scrollboxSize:function(){return e(),i},reliableTrDimensions:function(){var e,t,n,r;return null==a&&(e=E.createElement("table"),t=E.createElement("tr"),n=E.createElement("div"),e.style.cssText="position:absolute;left:-11111px;border-collapse:separate",t.style.cssText="border:1px solid",t.style.height="1px",n.style.height="9px",n.style.display="block",re.appendChild(e).appendChild(t).appendChild(n),r=C.getComputedStyle(t),a=parseInt(r.height,10)+parseInt(r.borderTopWidth,10)+parseInt(r.borderBottomWidth,10)===t.offsetHeight,re.removeChild(e)),a}}))}();var Be=["Webkit","Moz","ms"],$e=E.createElement("div").style,_e={};function ze(e){var t=S.cssProps[e]||_e[e];return t||(e in $e?e:_e[e]=function(e){var t=e[0].toUpperCase()+e.slice(1),n=Be.length;while(n--)if((e=Be[n]+t)in $e)return e}(e)||e)}var Ue=/^(none|table(?!-c[ea]).+)/,Xe=/^--/,Ve={position:"absolute",visibility:"hidden",display:"block"},Ge={letterSpacing:"0",fontWeight:"400"};function Ye(e,t,n){var r=te.exec(t);return r?Math.max(0,r[2]-(n||0))+(r[3]||"px"):t}function Qe(e,t,n,r,i,o){var a="width"===t?1:0,s=0,u=0;if(n===(r?"border":"content"))return 0;for(;a<4;a+=2)"margin"===n&&(u+=S.css(e,n+ne[a],!0,i)),r?("content"===n&&(u-=S.css(e,"padding"+ne[a],!0,i)),"margin"!==n&&(u-=S.css(e,"border"+ne[a]+"Width",!0,i))):(u+=S.css(e,"padding"+ne[a],!0,i),"padding"!==n?u+=S.css(e,"border"+ne[a]+"Width",!0,i):s+=S.css(e,"border"+ne[a]+"Width",!0,i));return!r&&0<=o&&(u+=Math.max(0,Math.ceil(e["offset"+t[0].toUpperCase()+t.slice(1)]-o-u-s-.5))||0),u}function Je(e,t,n){var r=Re(e),i=(!y.boxSizingReliable()||n)&&"border-box"===S.css(e,"boxSizing",!1,r),o=i,a=We(e,t,r),s="offset"+t[0].toUpperCase()+t.slice(1);if(Pe.test(a)){if(!n)return a;a="auto"}return(!y.boxSizingReliable()&&i||!y.reliableTrDimensions()&&A(e,"tr")||"auto"===a||!parseFloat(a)&&"inline"===S.css(e,"display",!1,r))&&e.getClientRects().length&&(i="border-box"===S.css(e,"boxSizing",!1,r),(o=s in e)&&(a=e[s])),(a=parseFloat(a)||0)+Qe(e,t,n||(i?"border":"content"),o,r,a)+"px"}function Ke(e,t,n,r,i){return new Ke.prototype.init(e,t,n,r,i)}S.extend({cssHooks:{opacity:{get:function(e,t){if(t){var n=We(e,"opacity");return""===n?"1":n}}}},cssNumber:{animationIterationCount:!0,columnCount:!0,fillOpacity:!0,flexGrow:!0,flexShrink:!0,fontWeight:!0,gridArea:!0,gridColumn:!0,gridColumnEnd:!0,gridColumnStart:!0,gridRow:!0,gridRowEnd:!0,gridRowStart:!0,lineHeight:!0,opacity:!0,order:!0,orphans:!0,widows:!0,zIndex:!0,zoom:!0},cssProps:{},style:function(e,t,n,r){if(e&&3!==e.nodeType&&8!==e.nodeType&&e.style){var i,o,a,s=X(t),u=Xe.test(t),l=e.style;if(u||(t=ze(s)),a=S.cssHooks[t]||S.cssHooks[s],void 0===n)return a&&"get"in a&&void 0!==(i=a.get(e,!1,r))?i:l[t];"string"===(o=typeof n)&&(i=te.exec(n))&&i[1]&&(n=se(e,t,i),o="number"),null!=n&&n==n&&("number"!==o||u||(n+=i&&i[3]||(S.cssNumber[s]?"":"px")),y.clearCloneStyle||""!==n||0!==t.indexOf("background")||(l[t]="inherit"),a&&"set"in a&&void 0===(n=a.set(e,n,r))||(u?l.setProperty(t,n):l[t]=n))}},css:function(e,t,n,r){var i,o,a,s=X(t);return Xe.test(t)||(t=ze(s)),(a=S.cssHooks[t]||S.cssHooks[s])&&"get"in a&&(i=a.get(e,!0,n)),void 0===i&&(i=We(e,t,r)),"normal"===i&&t in Ge&&(i=Ge[t]),""===n||n?(o=parseFloat(i),!0===n||isFinite(o)?o||0:i):i}}),S.each(["height","width"],function(e,u){S.cssHooks[u]={get:function(e,t,n){if(t)return!Ue.test(S.css(e,"display"))||e.getClientRects().length&&e.getBoundingClientRect().width?Je(e,u,n):Me(e,Ve,function(){return Je(e,u,n)})},set:function(e,t,n){var r,i=Re(e),o=!y.scrollboxSize()&&"absolute"===i.position,a=(o||n)&&"border-box"===S.css(e,"boxSizing",!1,i),s=n?Qe(e,u,n,a,i):0;return a&&o&&(s-=Math.ceil(e["offset"+u[0].toUpperCase()+u.slice(1)]-parseFloat(i[u])-Qe(e,u,"border",!1,i)-.5)),s&&(r=te.exec(t))&&"px"!==(r[3]||"px")&&(e.style[u]=t,t=S.css(e,u)),Ye(0,t,s)}}}),S.cssHooks.marginLeft=Fe(y.reliableMarginLeft,function(e,t){if(t)return(parseFloat(We(e,"marginLeft"))||e.getBoundingClientRect().left-Me(e,{marginLeft:0},function(){return e.getBoundingClientRect().left}))+"px"}),S.each({margin:"",padding:"",border:"Width"},function(i,o){S.cssHooks[i+o]={expand:function(e){for(var t=0,n={},r="string"==typeof e?e.split(" "):[e];t<4;t++)n[i+ne[t]+o]=r[t]||r[t-2]||r[0];return n}},"margin"!==i&&(S.cssHooks[i+o].set=Ye)}),S.fn.extend({css:function(e,t){return $(this,function(e,t,n){var r,i,o={},a=0;if(Array.isArray(t)){for(r=Re(e),i=t.length;a<i;a++)o[t[a]]=S.css(e,t[a],!1,r);return o}return void 0!==n?S.style(e,t,n):S.css(e,t)},e,t,1<arguments.length)}}),((S.Tween=Ke).prototype={constructor:Ke,init:function(e,t,n,r,i,o){this.elem=e,this.prop=n,this.easing=i||S.easing._default,this.options=t,this.start=this.now=this.cur(),this.end=r,this.unit=o||(S.cssNumber[n]?"":"px")},cur:function(){var e=Ke.propHooks[this.prop];return e&&e.get?e.get(this):Ke.propHooks._default.get(this)},run:function(e){var t,n=Ke.propHooks[this.prop];return this.options.duration?this.pos=t=S.easing[this.easing](e,this.options.duration*e,0,1,this.options.duration):this.pos=t=e,this.now=(this.end-this.start)*t+this.start,this.options.step&&this.options.step.call(this.elem,this.now,this),n&&n.set?n.set(this):Ke.propHooks._default.set(this),this}}).init.prototype=Ke.prototype,(Ke.propHooks={_default:{get:function(e){var t;return 1!==e.elem.nodeType||null!=e.elem[e.prop]&&null==e.elem.style[e.prop]?e.elem[e.prop]:(t=S.css(e.elem,e.prop,""))&&"auto"!==t?t:0},set:function(e){S.fx.step[e.prop]?S.fx.step[e.prop](e):1!==e.elem.nodeType||!S.cssHooks[e.prop]&&null==e.elem.style[ze(e.prop)]?e.elem[e.prop]=e.now:S.style(e.elem,e.prop,e.now+e.unit)}}}).scrollTop=Ke.propHooks.scrollLeft={set:function(e){e.elem.nodeType&&e.elem.parentNode&&(e.elem[e.prop]=e.now)}},S.easing={linear:function(e){return e},swing:function(e){return.5-Math.cos(e*Math.PI)/2},_default:"swing"},S.fx=Ke.prototype.init,S.fx.step={};var Ze,et,tt,nt,rt=/^(?:toggle|show|hide)$/,it=/queueHooks$/;function ot(){et&&(!1===E.hidden&&C.requestAnimationFrame?C.requestAnimationFrame(ot):C.setTimeout(ot,S.fx.interval),S.fx.tick())}function at(){return C.setTimeout(function(){Ze=void 0}),Ze=Date.now()}function st(e,t){var n,r=0,i={height:e};for(t=t?1:0;r<4;r+=2-t)i["margin"+(n=ne[r])]=i["padding"+n]=e;return t&&(i.opacity=i.width=e),i}function ut(e,t,n){for(var r,i=(lt.tweeners[t]||[]).concat(lt.tweeners["*"]),o=0,a=i.length;o<a;o++)if(r=i[o].call(n,t,e))return r}function lt(o,e,t){var n,a,r=0,i=lt.prefilters.length,s=S.Deferred().always(function(){delete u.elem}),u=function(){if(a)return!1;for(var e=Ze||at(),t=Math.max(0,l.startTime+l.duration-e),n=1-(t/l.duration||0),r=0,i=l.tweens.length;r<i;r++)l.tweens[r].run(n);return s.notifyWith(o,[l,n,t]),n<1&&i?t:(i||s.notifyWith(o,[l,1,0]),s.resolveWith(o,[l]),!1)},l=s.promise({elem:o,props:S.extend({},e),opts:S.extend(!0,{specialEasing:{},easing:S.easing._default},t),originalProperties:e,originalOptions:t,startTime:Ze||at(),duration:t.duration,tweens:[],createTween:function(e,t){var n=S.Tween(o,l.opts,e,t,l.opts.specialEasing[e]||l.opts.easing);return l.tweens.push(n),n},stop:function(e){var t=0,n=e?l.tweens.length:0;if(a)return this;for(a=!0;t<n;t++)l.tweens[t].run(1);return e?(s.notifyWith(o,[l,1,0]),s.resolveWith(o,[l,e])):s.rejectWith(o,[l,e]),this}}),c=l.props;for(!function(e,t){var n,r,i,o,a;for(n in e)if(i=t[r=X(n)],o=e[n],Array.isArray(o)&&(i=o[1],o=e[n]=o[0]),n!==r&&(e[r]=o,delete e[n]),(a=S.cssHooks[r])&&"expand"in a)for(n in o=a.expand(o),delete e[r],o)n in e||(e[n]=o[n],t[n]=i);else t[r]=i}(c,l.opts.specialEasing);r<i;r++)if(n=lt.prefilters[r].call(l,o,c,l.opts))return m(n.stop)&&(S._queueHooks(l.elem,l.opts.queue).stop=n.stop.bind(n)),n;return S.map(c,ut,l),m(l.opts.start)&&l.opts.start.call(o,l),l.progress(l.opts.progress).done(l.opts.done,l.opts.complete).fail(l.opts.fail).always(l.opts.always),S.fx.timer(S.extend(u,{elem:o,anim:l,queue:l.opts.queue})),l}S.Animation=S.extend(lt,{tweeners:{"*":[function(e,t){var n=this.createTween(e,t);return se(n.elem,e,te.exec(t),n),n}]},tweener:function(e,t){m(e)?(t=e,e=["*"]):e=e.match(P);for(var n,r=0,i=e.length;r<i;r++)n=e[r],lt.tweeners[n]=lt.tweeners[n]||[],lt.tweeners[n].unshift(t)},prefilters:[function(e,t,n){var r,i,o,a,s,u,l,c,f="width"in t||"height"in t,p=this,d={},h=e.style,g=e.nodeType&&ae(e),v=Y.get(e,"fxshow");for(r in n.queue||(null==(a=S._queueHooks(e,"fx")).unqueued&&(a.unqueued=0,s=a.empty.fire,a.empty.fire=function(){a.unqueued||s()}),a.unqueued++,p.always(function(){p.always(function(){a.unqueued--,S.queue(e,"fx").length||a.empty.fire()})})),t)if(i=t[r],rt.test(i)){if(delete t[r],o=o||"toggle"===i,i===(g?"hide":"show")){if("show"!==i||!v||void 0===v[r])continue;g=!0}d[r]=v&&v[r]||S.style(e,r)}if((u=!S.isEmptyObject(t))||!S.isEmptyObject(d))for(r in f&&1===e.nodeType&&(n.overflow=[h.overflow,h.overflowX,h.overflowY],null==(l=v&&v.display)&&(l=Y.get(e,"display")),"none"===(c=S.css(e,"display"))&&(l?c=l:(le([e],!0),l=e.style.display||l,c=S.css(e,"display"),le([e]))),("inline"===c||"inline-block"===c&&null!=l)&&"none"===S.css(e,"float")&&(u||(p.done(function(){h.display=l}),null==l&&(c=h.display,l="none"===c?"":c)),h.display="inline-block")),n.overflow&&(h.overflow="hidden",p.always(function(){h.overflow=n.overflow[0],h.overflowX=n.overflow[1],h.overflowY=n.overflow[2]})),u=!1,d)u||(v?"hidden"in v&&(g=v.hidden):v=Y.access(e,"fxshow",{display:l}),o&&(v.hidden=!g),g&&le([e],!0),p.done(function(){for(r in g||le([e]),Y.remove(e,"fxshow"),d)S.style(e,r,d[r])})),u=ut(g?v[r]:0,r,p),r in v||(v[r]=u.start,g&&(u.end=u.start,u.start=0))}],prefilter:function(e,t){t?lt.prefilters.unshift(e):lt.prefilters.push(e)}}),S.speed=function(e,t,n){var r=e&&"object"==typeof e?S.extend({},e):{complete:n||!n&&t||m(e)&&e,duration:e,easing:n&&t||t&&!m(t)&&t};return S.fx.off?r.duration=0:"number"!=typeof r.duration&&(r.duration in S.fx.speeds?r.duration=S.fx.speeds[r.duration]:r.duration=S.fx.speeds._default),null!=r.queue&&!0!==r.queue||(r.queue="fx"),r.old=r.complete,r.complete=function(){m(r.old)&&r.old.call(this),r.queue&&S.dequeue(this,r.queue)},r},S.fn.extend({fadeTo:function(e,t,n,r){return this.filter(ae).css("opacity",0).show().end().animate({opacity:t},e,n,r)},animate:function(t,e,n,r){var i=S.isEmptyObject(t),o=S.speed(e,n,r),a=function(){var e=lt(this,S.extend({},t),o);(i||Y.get(this,"finish"))&&e.stop(!0)};return a.finish=a,i||!1===o.queue?this.each(a):this.queue(o.queue,a)},stop:function(i,e,o){var a=function(e){var t=e.stop;delete e.stop,t(o)};return"string"!=typeof i&&(o=e,e=i,i=void 0),e&&this.queue(i||"fx",[]),this.each(function(){var e=!0,t=null!=i&&i+"queueHooks",n=S.timers,r=Y.get(this);if(t)r[t]&&r[t].stop&&a(r[t]);else for(t in r)r[t]&&r[t].stop&&it.test(t)&&a(r[t]);for(t=n.length;t--;)n[t].elem!==this||null!=i&&n[t].queue!==i||(n[t].anim.stop(o),e=!1,n.splice(t,1));!e&&o||S.dequeue(this,i)})},finish:function(a){return!1!==a&&(a=a||"fx"),this.each(function(){var e,t=Y.get(this),n=t[a+"queue"],r=t[a+"queueHooks"],i=S.timers,o=n?n.length:0;for(t.finish=!0,S.queue(this,a,[]),r&&r.stop&&r.stop.call(this,!0),e=i.length;e--;)i[e].elem===this&&i[e].queue===a&&(i[e].anim.stop(!0),i.splice(e,1));for(e=0;e<o;e++)n[e]&&n[e].finish&&n[e].finish.call(this);delete t.finish})}}),S.each(["toggle","show","hide"],function(e,r){var i=S.fn[r];S.fn[r]=function(e,t,n){return null==e||"boolean"==typeof e?i.apply(this,arguments):this.animate(st(r,!0),e,t,n)}}),S.each({slideDown:st("show"),slideUp:st("hide"),slideToggle:st("toggle"),fadeIn:{opacity:"show"},fadeOut:{opacity:"hide"},fadeToggle:{opacity:"toggle"}},function(e,r){S.fn[e]=function(e,t,n){return this.animate(r,e,t,n)}}),S.timers=[],S.fx.tick=function(){var e,t=0,n=S.timers;for(Ze=Date.now();t<n.length;t++)(e=n[t])()||n[t]!==e||n.splice(t--,1);n.length||S.fx.stop(),Ze=void 0},S.fx.timer=function(e){S.timers.push(e),S.fx.start()},S.fx.interval=13,S.fx.start=function(){et||(et=!0,ot())},S.fx.stop=function(){et=null},S.fx.speeds={slow:600,fast:200,_default:400},S.fn.delay=function(r,e){return r=S.fx&&S.fx.speeds[r]||r,e=e||"fx",this.queue(e,function(e,t){var n=C.setTimeout(e,r);t.stop=function(){C.clearTimeout(n)}})},tt=E.createElement("input"),nt=E.createElement("select").appendChild(E.createElement("option")),tt.type="checkbox",y.checkOn=""!==tt.value,y.optSelected=nt.selected,(tt=E.createElement("input")).value="t",tt.type="radio",y.radioValue="t"===tt.value;var ct,ft=S.expr.attrHandle;S.fn.extend({attr:function(e,t){return $(this,S.attr,e,t,1<arguments.length)},removeAttr:function(e){return this.each(function(){S.removeAttr(this,e)})}}),S.extend({attr:function(e,t,n){var r,i,o=e.nodeType;if(3!==o&&8!==o&&2!==o)return"undefined"==typeof e.getAttribute?S.prop(e,t,n):(1===o&&S.isXMLDoc(e)||(i=S.attrHooks[t.toLowerCase()]||(S.expr.match.bool.test(t)?ct:void 0)),void 0!==n?null===n?void S.removeAttr(e,t):i&&"set"in i&&void 0!==(r=i.set(e,n,t))?r:(e.setAttribute(t,n+""),n):i&&"get"in i&&null!==(r=i.get(e,t))?r:null==(r=S.find.attr(e,t))?void 0:r)},attrHooks:{type:{set:function(e,t){if(!y.radioValue&&"radio"===t&&A(e,"input")){var n=e.value;return e.setAttribute("type",t),n&&(e.value=n),t}}}},removeAttr:function(e,t){var n,r=0,i=t&&t.match(P);if(i&&1===e.nodeType)while(n=i[r++])e.removeAttribute(n)}}),ct={set:function(e,t,n){return!1===t?S.removeAttr(e,n):e.setAttribute(n,n),n}},S.each(S.expr.match.bool.source.match(/\w+/g),function(e,t){var a=ft[t]||S.find.attr;ft[t]=function(e,t,n){var r,i,o=t.toLowerCase();return n||(i=ft[o],ft[o]=r,r=null!=a(e,t,n)?o:null,ft[o]=i),r}});var pt=/^(?:input|select|textarea|button)$/i,dt=/^(?:a|area)$/i;function ht(e){return(e.match(P)||[]).join(" ")}function gt(e){return e.getAttribute&&e.getAttribute("class")||""}function vt(e){return Array.isArray(e)?e:"string"==typeof e&&e.match(P)||[]}S.fn.extend({prop:function(e,t){return $(this,S.prop,e,t,1<arguments.length)},removeProp:function(e){return this.each(function(){delete this[S.propFix[e]||e]})}}),S.extend({prop:function(e,t,n){var r,i,o=e.nodeType;if(3!==o&&8!==o&&2!==o)return 1===o&&S.isXMLDoc(e)||(t=S.propFix[t]||t,i=S.propHooks[t]),void 0!==n?i&&"set"in i&&void 0!==(r=i.set(e,n,t))?r:e[t]=n:i&&"get"in i&&null!==(r=i.get(e,t))?r:e[t]},propHooks:{tabIndex:{get:function(e){var t=S.find.attr(e,"tabindex");return t?parseInt(t,10):pt.test(e.nodeName)||dt.test(e.nodeName)&&e.href?0:-1}}},propFix:{"for":"htmlFor","class":"className"}}),y.optSelected||(S.propHooks.selected={get:function(e){var t=e.parentNode;return t&&t.parentNode&&t.parentNode.selectedIndex,null},set:function(e){var t=e.parentNode;t&&(t.selectedIndex,t.parentNode&&t.parentNode.selectedIndex)}}),S.each(["tabIndex","readOnly","maxLength","cellSpacing","cellPadding","rowSpan","colSpan","useMap","frameBorder","contentEditable"],function(){S.propFix[this.toLowerCase()]=this}),S.fn.extend({addClass:function(t){var e,n,r,i,o,a,s,u=0;if(m(t))return this.each(function(e){S(this).addClass(t.call(this,e,gt(this)))});if((e=vt(t)).length)while(n=this[u++])if(i=gt(n),r=1===n.nodeType&&" "+ht(i)+" "){a=0;while(o=e[a++])r.indexOf(" "+o+" ")<0&&(r+=o+" ");i!==(s=ht(r))&&n.setAttribute("class",s)}return this},removeClass:function(t){var e,n,r,i,o,a,s,u=0;if(m(t))return this.each(function(e){S(this).removeClass(t.call(this,e,gt(this)))});if(!arguments.length)return this.attr("class","");if((e=vt(t)).length)while(n=this[u++])if(i=gt(n),r=1===n.nodeType&&" "+ht(i)+" "){a=0;while(o=e[a++])while(-1<r.indexOf(" "+o+" "))r=r.replace(" "+o+" "," ");i!==(s=ht(r))&&n.setAttribute("class",s)}return this},toggleClass:function(i,t){var o=typeof i,a="string"===o||Array.isArray(i);return"boolean"==typeof t&&a?t?this.addClass(i):this.removeClass(i):m(i)?this.each(function(e){S(this).toggleClass(i.call(this,e,gt(this),t),t)}):this.each(function(){var e,t,n,r;if(a){t=0,n=S(this),r=vt(i);while(e=r[t++])n.hasClass(e)?n.removeClass(e):n.addClass(e)}else void 0!==i&&"boolean"!==o||((e=gt(this))&&Y.set(this,"__className__",e),this.setAttribute&&this.setAttribute("class",e||!1===i?"":Y.get(this,"__className__")||""))})},hasClass:function(e){var t,n,r=0;t=" "+e+" ";while(n=this[r++])if(1===n.nodeType&&-1<(" "+ht(gt(n))+" ").indexOf(t))return!0;return!1}});var yt=/\r/g;S.fn.extend({val:function(n){var r,e,i,t=this[0];return arguments.length?(i=m(n),this.each(function(e){var t;1===this.nodeType&&(null==(t=i?n.call(this,e,S(this).val()):n)?t="":"number"==typeof t?t+="":Array.isArray(t)&&(t=S.map(t,function(e){return null==e?"":e+""})),(r=S.valHooks[this.type]||S.valHooks[this.nodeName.toLowerCase()])&&"set"in r&&void 0!==r.set(this,t,"value")||(this.value=t))})):t?(r=S.valHooks[t.type]||S.valHooks[t.nodeName.toLowerCase()])&&"get"in r&&void 0!==(e=r.get(t,"value"))?e:"string"==typeof(e=t.value)?e.replace(yt,""):null==e?"":e:void 0}}),S.extend({valHooks:{option:{get:function(e){var t=S.find.attr(e,"value");return null!=t?t:ht(S.text(e))}},select:{get:function(e){var t,n,r,i=e.options,o=e.selectedIndex,a="select-one"===e.type,s=a?null:[],u=a?o+1:i.length;for(r=o<0?u:a?o:0;r<u;r++)if(((n=i[r]).selected||r===o)&&!n.disabled&&(!n.parentNode.disabled||!A(n.parentNode,"optgroup"))){if(t=S(n).val(),a)return t;s.push(t)}return s},set:function(e,t){var n,r,i=e.options,o=S.makeArray(t),a=i.length;while(a--)((r=i[a]).selected=-1<S.inArray(S.valHooks.option.get(r),o))&&(n=!0);return n||(e.selectedIndex=-1),o}}}}),S.each(["radio","checkbox"],function(){S.valHooks[this]={set:function(e,t){if(Array.isArray(t))return e.checked=-1<S.inArray(S(e).val(),t)}},y.checkOn||(S.valHooks[this].get=function(e){return null===e.getAttribute("value")?"on":e.value})}),y.focusin="onfocusin"in C;var mt=/^(?:focusinfocus|focusoutblur)$/,xt=function(e){e.stopPropagation()};S.extend(S.event,{trigger:function(e,t,n,r){var i,o,a,s,u,l,c,f,p=[n||E],d=v.call(e,"type")?e.type:e,h=v.call(e,"namespace")?e.namespace.split("."):[];if(o=f=a=n=n||E,3!==n.nodeType&&8!==n.nodeType&&!mt.test(d+S.event.triggered)&&(-1<d.indexOf(".")&&(d=(h=d.split(".")).shift(),h.sort()),u=d.indexOf(":")<0&&"on"+d,(e=e[S.expando]?e:new S.Event(d,"object"==typeof e&&e)).isTrigger=r?2:3,e.namespace=h.join("."),e.rnamespace=e.namespace?new RegExp("(^|\\.)"+h.join("\\.(?:.*\\.|)")+"(\\.|$)"):null,e.result=void 0,e.target||(e.target=n),t=null==t?[e]:S.makeArray(t,[e]),c=S.event.special[d]||{},r||!c.trigger||!1!==c.trigger.apply(n,t))){if(!r&&!c.noBubble&&!x(n)){for(s=c.delegateType||d,mt.test(s+d)||(o=o.parentNode);o;o=o.parentNode)p.push(o),a=o;a===(n.ownerDocument||E)&&p.push(a.defaultView||a.parentWindow||C)}i=0;while((o=p[i++])&&!e.isPropagationStopped())f=o,e.type=1<i?s:c.bindType||d,(l=(Y.get(o,"events")||Object.create(null))[e.type]&&Y.get(o,"handle"))&&l.apply(o,t),(l=u&&o[u])&&l.apply&&V(o)&&(e.result=l.apply(o,t),!1===e.result&&e.preventDefault());return e.type=d,r||e.isDefaultPrevented()||c._default&&!1!==c._default.apply(p.pop(),t)||!V(n)||u&&m(n[d])&&!x(n)&&((a=n[u])&&(n[u]=null),S.event.triggered=d,e.isPropagationStopped()&&f.addEventListener(d,xt),n[d](),e.isPropagationStopped()&&f.removeEventListener(d,xt),S.event.triggered=void 0,a&&(n[u]=a)),e.result}},simulate:function(e,t,n){var r=S.extend(new S.Event,n,{type:e,isSimulated:!0});S.event.trigger(r,null,t)}}),S.fn.extend({trigger:function(e,t){return this.each(function(){S.event.trigger(e,t,this)})},triggerHandler:function(e,t){var n=this[0];if(n)return S.event.trigger(e,t,n,!0)}}),y.focusin||S.each({focus:"focusin",blur:"focusout"},function(n,r){var i=function(e){S.event.simulate(r,e.target,S.event.fix(e))};S.event.special[r]={setup:function(){var e=this.ownerDocument||this.document||this,t=Y.access(e,r);t||e.addEventListener(n,i,!0),Y.access(e,r,(t||0)+1)},teardown:function(){var e=this.ownerDocument||this.document||this,t=Y.access(e,r)-1;t?Y.access(e,r,t):(e.removeEventListener(n,i,!0),Y.remove(e,r))}}});var bt=C.location,wt={guid:Date.now()},Tt=/\?/;S.parseXML=function(e){var t,n;if(!e||"string"!=typeof e)return null;try{t=(new C.DOMParser).parseFromString(e,"text/xml")}catch(e){}return n=t&&t.getElementsByTagName("parsererror")[0],t&&!n||S.error("Invalid XML: "+(n?S.map(n.childNodes,function(e){return e.textContent}).join("\n"):e)),t};var Ct=/\[\]$/,Et=/\r?\n/g,St=/^(?:submit|button|image|reset|file)$/i,kt=/^(?:input|select|textarea|keygen)/i;function At(n,e,r,i){var t;if(Array.isArray(e))S.each(e,function(e,t){r||Ct.test(n)?i(n,t):At(n+"["+("object"==typeof t&&null!=t?e:"")+"]",t,r,i)});else if(r||"object"!==w(e))i(n,e);else for(t in e)At(n+"["+t+"]",e[t],r,i)}S.param=function(e,t){var n,r=[],i=function(e,t){var n=m(t)?t():t;r[r.length]=encodeURIComponent(e)+"="+encodeURIComponent(null==n?"":n)};if(null==e)return"";if(Array.isArray(e)||e.jquery&&!S.isPlainObject(e))S.each(e,function(){i(this.name,this.value)});else for(n in e)At(n,e[n],t,i);return r.join("&")},S.fn.extend({serialize:function(){return S.param(this.serializeArray())},serializeArray:function(){return this.map(function(){var e=S.prop(this,"elements");return e?S.makeArray(e):this}).filter(function(){var e=this.type;return this.name&&!S(this).is(":disabled")&&kt.test(this.nodeName)&&!St.test(e)&&(this.checked||!pe.test(e))}).map(function(e,t){var n=S(this).val();return null==n?null:Array.isArray(n)?S.map(n,function(e){return{name:t.name,value:e.replace(Et,"\r\n")}}):{name:t.name,value:n.replace(Et,"\r\n")}}).get()}});var Nt=/%20/g,jt=/#.*$/,Dt=/([?&])_=[^&]*/,qt=/^(.*?):[ \t]*([^\r\n]*)$/gm,Lt=/^(?:GET|HEAD)$/,Ht=/^\/\//,Ot={},Pt={},Rt="*/".concat("*"),Mt=E.createElement("a");function It(o){return function(e,t){"string"!=typeof e&&(t=e,e="*");var n,r=0,i=e.toLowerCase().match(P)||[];if(m(t))while(n=i[r++])"+"===n[0]?(n=n.slice(1)||"*",(o[n]=o[n]||[]).unshift(t)):(o[n]=o[n]||[]).push(t)}}function Wt(t,i,o,a){var s={},u=t===Pt;function l(e){var r;return s[e]=!0,S.each(t[e]||[],function(e,t){var n=t(i,o,a);return"string"!=typeof n||u||s[n]?u?!(r=n):void 0:(i.dataTypes.unshift(n),l(n),!1)}),r}return l(i.dataTypes[0])||!s["*"]&&l("*")}function Ft(e,t){var n,r,i=S.ajaxSettings.flatOptions||{};for(n in t)void 0!==t[n]&&((i[n]?e:r||(r={}))[n]=t[n]);return r&&S.extend(!0,e,r),e}Mt.href=bt.href,S.extend({active:0,lastModified:{},etag:{},ajaxSettings:{url:bt.href,type:"GET",isLocal:/^(?:about|app|app-storage|.+-extension|file|res|widget):$/.test(bt.protocol),global:!0,processData:!0,async:!0,contentType:"application/x-www-form-urlencoded; charset=UTF-8",accepts:{"*":Rt,text:"text/plain",html:"text/html",xml:"application/xml, text/xml",json:"application/json, text/javascript"},contents:{xml:/\bxml\b/,html:/\bhtml/,json:/\bjson\b/},responseFields:{xml:"responseXML",text:"responseText",json:"responseJSON"},converters:{"* text":String,"text html":!0,"text json":JSON.parse,"text xml":S.parseXML},flatOptions:{url:!0,context:!0}},ajaxSetup:function(e,t){return t?Ft(Ft(e,S.ajaxSettings),t):Ft(S.ajaxSettings,e)},ajaxPrefilter:It(Ot),ajaxTransport:It(Pt),ajax:function(e,t){"object"==typeof e&&(t=e,e=void 0),t=t||{};var c,f,p,n,d,r,h,g,i,o,v=S.ajaxSetup({},t),y=v.context||v,m=v.context&&(y.nodeType||y.jquery)?S(y):S.event,x=S.Deferred(),b=S.Callbacks("once memory"),w=v.statusCode||{},a={},s={},u="canceled",T={readyState:0,getResponseHeader:function(e){var t;if(h){if(!n){n={};while(t=qt.exec(p))n[t[1].toLowerCase()+" "]=(n[t[1].toLowerCase()+" "]||[]).concat(t[2])}t=n[e.toLowerCase()+" "]}return null==t?null:t.join(", ")},getAllResponseHeaders:function(){return h?p:null},setRequestHeader:function(e,t){return null==h&&(e=s[e.toLowerCase()]=s[e.toLowerCase()]||e,a[e]=t),this},overrideMimeType:function(e){return null==h&&(v.mimeType=e),this},statusCode:function(e){var t;if(e)if(h)T.always(e[T.status]);else for(t in e)w[t]=[w[t],e[t]];return this},abort:function(e){var t=e||u;return c&&c.abort(t),l(0,t),this}};if(x.promise(T),v.url=((e||v.url||bt.href)+"").replace(Ht,bt.protocol+"//"),v.type=t.method||t.type||v.method||v.type,v.dataTypes=(v.dataType||"*").toLowerCase().match(P)||[""],null==v.crossDomain){r=E.createElement("a");try{r.href=v.url,r.href=r.href,v.crossDomain=Mt.protocol+"//"+Mt.host!=r.protocol+"//"+r.host}catch(e){v.crossDomain=!0}}if(v.data&&v.processData&&"string"!=typeof v.data&&(v.data=S.param(v.data,v.traditional)),Wt(Ot,v,t,T),h)return T;for(i in(g=S.event&&v.global)&&0==S.active++&&S.event.trigger("ajaxStart"),v.type=v.type.toUpperCase(),v.hasContent=!Lt.test(v.type),f=v.url.replace(jt,""),v.hasContent?v.data&&v.processData&&0===(v.contentType||"").indexOf("application/x-www-form-urlencoded")&&(v.data=v.data.replace(Nt,"+")):(o=v.url.slice(f.length),v.data&&(v.processData||"string"==typeof v.data)&&(f+=(Tt.test(f)?"&":"?")+v.data,delete v.data),!1===v.cache&&(f=f.replace(Dt,"$1"),o=(Tt.test(f)?"&":"?")+"_="+wt.guid+++o),v.url=f+o),v.ifModified&&(S.lastModified[f]&&T.setRequestHeader("If-Modified-Since",S.lastModified[f]),S.etag[f]&&T.setRequestHeader("If-None-Match",S.etag[f])),(v.data&&v.hasContent&&!1!==v.contentType||t.contentType)&&T.setRequestHeader("Content-Type",v.contentType),T.setRequestHeader("Accept",v.dataTypes[0]&&v.accepts[v.dataTypes[0]]?v.accepts[v.dataTypes[0]]+("*"!==v.dataTypes[0]?", "+Rt+"; q=0.01":""):v.accepts["*"]),v.headers)T.setRequestHeader(i,v.headers[i]);if(v.beforeSend&&(!1===v.beforeSend.call(y,T,v)||h))return T.abort();if(u="abort",b.add(v.complete),T.done(v.success),T.fail(v.error),c=Wt(Pt,v,t,T)){if(T.readyState=1,g&&m.trigger("ajaxSend",[T,v]),h)return T;v.async&&0<v.timeout&&(d=C.setTimeout(function(){T.abort("timeout")},v.timeout));try{h=!1,c.send(a,l)}catch(e){if(h)throw e;l(-1,e)}}else l(-1,"No Transport");function l(e,t,n,r){var i,o,a,s,u,l=t;h||(h=!0,d&&C.clearTimeout(d),c=void 0,p=r||"",T.readyState=0<e?4:0,i=200<=e&&e<300||304===e,n&&(s=function(e,t,n){var r,i,o,a,s=e.contents,u=e.dataTypes;while("*"===u[0])u.shift(),void 0===r&&(r=e.mimeType||t.getResponseHeader("Content-Type"));if(r)for(i in s)if(s[i]&&s[i].test(r)){u.unshift(i);break}if(u[0]in n)o=u[0];else{for(i in n){if(!u[0]||e.converters[i+" "+u[0]]){o=i;break}a||(a=i)}o=o||a}if(o)return o!==u[0]&&u.unshift(o),n[o]}(v,T,n)),!i&&-1<S.inArray("script",v.dataTypes)&&S.inArray("json",v.dataTypes)<0&&(v.converters["text script"]=function(){}),s=function(e,t,n,r){var i,o,a,s,u,l={},c=e.dataTypes.slice();if(c[1])for(a in e.converters)l[a.toLowerCase()]=e.converters[a];o=c.shift();while(o)if(e.responseFields[o]&&(n[e.responseFields[o]]=t),!u&&r&&e.dataFilter&&(t=e.dataFilter(t,e.dataType)),u=o,o=c.shift())if("*"===o)o=u;else if("*"!==u&&u!==o){if(!(a=l[u+" "+o]||l["* "+o]))for(i in l)if((s=i.split(" "))[1]===o&&(a=l[u+" "+s[0]]||l["* "+s[0]])){!0===a?a=l[i]:!0!==l[i]&&(o=s[0],c.unshift(s[1]));break}if(!0!==a)if(a&&e["throws"])t=a(t);else try{t=a(t)}catch(e){return{state:"parsererror",error:a?e:"No conversion from "+u+" to "+o}}}return{state:"success",data:t}}(v,s,T,i),i?(v.ifModified&&((u=T.getResponseHeader("Last-Modified"))&&(S.lastModified[f]=u),(u=T.getResponseHeader("etag"))&&(S.etag[f]=u)),204===e||"HEAD"===v.type?l="nocontent":304===e?l="notmodified":(l=s.state,o=s.data,i=!(a=s.error))):(a=l,!e&&l||(l="error",e<0&&(e=0))),T.status=e,T.statusText=(t||l)+"",i?x.resolveWith(y,[o,l,T]):x.rejectWith(y,[T,l,a]),T.statusCode(w),w=void 0,g&&m.trigger(i?"ajaxSuccess":"ajaxError",[T,v,i?o:a]),b.fireWith(y,[T,l]),g&&(m.trigger("ajaxComplete",[T,v]),--S.active||S.event.trigger("ajaxStop")))}return T},getJSON:function(e,t,n){return S.get(e,t,n,"json")},getScript:function(e,t){return S.get(e,void 0,t,"script")}}),S.each(["get","post"],function(e,i){S[i]=function(e,t,n,r){return m(t)&&(r=r||n,n=t,t=void 0),S.ajax(S.extend({url:e,type:i,dataType:r,data:t,success:n},S.isPlainObject(e)&&e))}}),S.ajaxPrefilter(function(e){var t;for(t in e.headers)"content-type"===t.toLowerCase()&&(e.contentType=e.headers[t]||"")}),S._evalUrl=function(e,t,n){return S.ajax({url:e,type:"GET",dataType:"script",cache:!0,async:!1,global:!1,converters:{"text script":function(){}},dataFilter:function(e){S.globalEval(e,t,n)}})},S.fn.extend({wrapAll:function(e){var t;return this[0]&&(m(e)&&(e=e.call(this[0])),t=S(e,this[0].ownerDocument).eq(0).clone(!0),this[0].parentNode&&t.insertBefore(this[0]),t.map(function(){var e=this;while(e.firstElementChild)e=e.firstElementChild;return e}).append(this)),this},wrapInner:function(n){return m(n)?this.each(function(e){S(this).wrapInner(n.call(this,e))}):this.each(function(){var e=S(this),t=e.contents();t.length?t.wrapAll(n):e.append(n)})},wrap:function(t){var n=m(t);return this.each(function(e){S(this).wrapAll(n?t.call(this,e):t)})},unwrap:function(e){return this.parent(e).not("body").each(function(){S(this).replaceWith(this.childNodes)}),this}}),S.expr.pseudos.hidden=function(e){return!S.expr.pseudos.visible(e)},S.expr.pseudos.visible=function(e){return!!(e.offsetWidth||e.offsetHeight||e.getClientRects().length)},S.ajaxSettings.xhr=function(){try{return new C.XMLHttpRequest}catch(e){}};var Bt={0:200,1223:204},$t=S.ajaxSettings.xhr();y.cors=!!$t&&"withCredentials"in $t,y.ajax=$t=!!$t,S.ajaxTransport(function(i){var o,a;if(y.cors||$t&&!i.crossDomain)return{send:function(e,t){var n,r=i.xhr();if(r.open(i.type,i.url,i.async,i.username,i.password),i.xhrFields)for(n in i.xhrFields)r[n]=i.xhrFields[n];for(n in i.mimeType&&r.overrideMimeType&&r.overrideMimeType(i.mimeType),i.crossDomain||e["X-Requested-With"]||(e["X-Requested-With"]="XMLHttpRequest"),e)r.setRequestHeader(n,e[n]);o=function(e){return function(){o&&(o=a=r.onload=r.onerror=r.onabort=r.ontimeout=r.onreadystatechange=null,"abort"===e?r.abort():"error"===e?"number"!=typeof r.status?t(0,"error"):t(r.status,r.statusText):t(Bt[r.status]||r.status,r.statusText,"text"!==(r.responseType||"text")||"string"!=typeof r.responseText?{binary:r.response}:{text:r.responseText},r.getAllResponseHeaders()))}},r.onload=o(),a=r.onerror=r.ontimeout=o("error"),void 0!==r.onabort?r.onabort=a:r.onreadystatechange=function(){4===r.readyState&&C.setTimeout(function(){o&&a()})},o=o("abort");try{r.send(i.hasContent&&i.data||null)}catch(e){if(o)throw e}},abort:function(){o&&o()}}}),S.ajaxPrefilter(function(e){e.crossDomain&&(e.contents.script=!1)}),S.ajaxSetup({accepts:{script:"text/javascript, application/javascript, application/ecmascript, application/x-ecmascript"},contents:{script:/\b(?:java|ecma)script\b/},converters:{"text script":function(e){return S.globalEval(e),e}}}),S.ajaxPrefilter("script",function(e){void 0===e.cache&&(e.cache=!1),e.crossDomain&&(e.type="GET")}),S.ajaxTransport("script",function(n){var r,i;if(n.crossDomain||n.scriptAttrs)return{send:function(e,t){r=S("<script>").attr(n.scriptAttrs||{}).prop({charset:n.scriptCharset,src:n.url}).on("load error",i=function(e){r.remove(),i=null,e&&t("error"===e.type?404:200,e.type)}),E.head.appendChild(r[0])},abort:function(){i&&i()}}});var _t,zt=[],Ut=/(=)\?(?=&|$)|\?\?/;S.ajaxSetup({jsonp:"callback",jsonpCallback:function(){var e=zt.pop()||S.expando+"_"+wt.guid++;return this[e]=!0,e}}),S.ajaxPrefilter("json jsonp",function(e,t,n){var r,i,o,a=!1!==e.jsonp&&(Ut.test(e.url)?"url":"string"==typeof e.data&&0===(e.contentType||"").indexOf("application/x-www-form-urlencoded")&&Ut.test(e.data)&&"data");if(a||"jsonp"===e.dataTypes[0])return r=e.jsonpCallback=m(e.jsonpCallback)?e.jsonpCallback():e.jsonpCallback,a?e[a]=e[a].replace(Ut,"$1"+r):!1!==e.jsonp&&(e.url+=(Tt.test(e.url)?"&":"?")+e.jsonp+"="+r),e.converters["script json"]=function(){return o||S.error(r+" was not called"),o[0]},e.dataTypes[0]="json",i=C[r],C[r]=function(){o=arguments},n.always(function(){void 0===i?S(C).removeProp(r):C[r]=i,e[r]&&(e.jsonpCallback=t.jsonpCallback,zt.push(r)),o&&m(i)&&i(o[0]),o=i=void 0}),"script"}),y.createHTMLDocument=((_t=E.implementation.createHTMLDocument("").body).innerHTML="<form></form><form></form>",2===_t.childNodes.length),S.parseHTML=function(e,t,n){return"string"!=typeof e?[]:("boolean"==typeof t&&(n=t,t=!1),t||(y.createHTMLDocument?((r=(t=E.implementation.createHTMLDocument("")).createElement("base")).href=E.location.href,t.head.appendChild(r)):t=E),o=!n&&[],(i=N.exec(e))?[t.createElement(i[1])]:(i=xe([e],t,o),o&&o.length&&S(o).remove(),S.merge([],i.childNodes)));var r,i,o},S.fn.load=function(e,t,n){var r,i,o,a=this,s=e.indexOf(" ");return-1<s&&(r=ht(e.slice(s)),e=e.slice(0,s)),m(t)?(n=t,t=void 0):t&&"object"==typeof t&&(i="POST"),0<a.length&&S.ajax({url:e,type:i||"GET",dataType:"html",data:t}).done(function(e){o=arguments,a.html(r?S("<div>").append(S.parseHTML(e)).find(r):e)}).always(n&&function(e,t){a.each(function(){n.apply(this,o||[e.responseText,t,e])})}),this},S.expr.pseudos.animated=function(t){return S.grep(S.timers,function(e){return t===e.elem}).length},S.offset={setOffset:function(e,t,n){var r,i,o,a,s,u,l=S.css(e,"position"),c=S(e),f={};"static"===l&&(e.style.position="relative"),s=c.offset(),o=S.css(e,"top"),u=S.css(e,"left"),("absolute"===l||"fixed"===l)&&-1<(o+u).indexOf("auto")?(a=(r=c.position()).top,i=r.left):(a=parseFloat(o)||0,i=parseFloat(u)||0),m(t)&&(t=t.call(e,n,S.extend({},s))),null!=t.top&&(f.top=t.top-s.top+a),null!=t.left&&(f.left=t.left-s.left+i),"using"in t?t.using.call(e,f):c.css(f)}},S.fn.extend({offset:function(t){if(arguments.length)return void 0===t?this:this.each(function(e){S.offset.setOffset(this,t,e)});var e,n,r=this[0];return r?r.getClientRects().length?(e=r.getBoundingClientRect(),n=r.ownerDocument.defaultView,{top:e.top+n.pageYOffset,left:e.left+n.pageXOffset}):{top:0,left:0}:void 0},position:function(){if(this[0]){var e,t,n,r=this[0],i={top:0,left:0};if("fixed"===S.css(r,"position"))t=r.getBoundingClientRect();else{t=this.offset(),n=r.ownerDocument,e=r.offsetParent||n.documentElement;while(e&&(e===n.body||e===n.documentElement)&&"static"===S.css(e,"position"))e=e.parentNode;e&&e!==r&&1===e.nodeType&&((i=S(e).offset()).top+=S.css(e,"borderTopWidth",!0),i.left+=S.css(e,"borderLeftWidth",!0))}return{top:t.top-i.top-S.css(r,"marginTop",!0),left:t.left-i.left-S.css(r,"marginLeft",!0)}}},offsetParent:function(){return this.map(function(){var e=this.offsetParent;while(e&&"static"===S.css(e,"position"))e=e.offsetParent;return e||re})}}),S.each({scrollLeft:"pageXOffset",scrollTop:"pageYOffset"},function(t,i){var o="pageYOffset"===i;S.fn[t]=function(e){return $(this,function(e,t,n){var r;if(x(e)?r=e:9===e.nodeType&&(r=e.defaultView),void 0===n)return r?r[i]:e[t];r?r.scrollTo(o?r.pageXOffset:n,o?n:r.pageYOffset):e[t]=n},t,e,arguments.length)}}),S.each(["top","left"],function(e,n){S.cssHooks[n]=Fe(y.pixelPosition,function(e,t){if(t)return t=We(e,n),Pe.test(t)?S(e).position()[n]+"px":t})}),S.each({Height:"height",Width:"width"},function(a,s){S.each({padding:"inner"+a,content:s,"":"outer"+a},function(r,o){S.fn[o]=function(e,t){var n=arguments.length&&(r||"boolean"!=typeof e),i=r||(!0===e||!0===t?"margin":"border");return $(this,function(e,t,n){var r;return x(e)?0===o.indexOf("outer")?e["inner"+a]:e.document.documentElement["client"+a]:9===e.nodeType?(r=e.documentElement,Math.max(e.body["scroll"+a],r["scroll"+a],e.body["offset"+a],r["offset"+a],r["client"+a])):void 0===n?S.css(e,t,i):S.style(e,t,n,i)},s,n?e:void 0,n)}})}),S.each(["ajaxStart","ajaxStop","ajaxComplete","ajaxError","ajaxSuccess","ajaxSend"],function(e,t){S.fn[t]=function(e){return this.on(t,e)}}),S.fn.extend({bind:function(e,t,n){return this.on(e,null,t,n)},unbind:function(e,t){return this.off(e,null,t)},delegate:function(e,t,n,r){return this.on(t,e,n,r)},undelegate:function(e,t,n){return 1===arguments.length?this.off(e,"**"):this.off(t,e||"**",n)},hover:function(e,t){return this.mouseenter(e).mouseleave(t||e)}}),S.each("blur focus focusin focusout resize scroll click dblclick mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave change select submit keydown keypress keyup contextmenu".split(" "),function(e,n){S.fn[n]=function(e,t){return 0<arguments.length?this.on(n,null,e,t):this.trigger(n)}});var Xt=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g;S.proxy=function(e,t){var n,r,i;if("string"==typeof t&&(n=e[t],t=e,e=n),m(e))return r=s.call(arguments,2),(i=function(){return e.apply(t||this,r.concat(s.call(arguments)))}).guid=e.guid=e.guid||S.guid++,i},S.holdReady=function(e){e?S.readyWait++:S.ready(!0)},S.isArray=Array.isArray,S.parseJSON=JSON.parse,S.nodeName=A,S.isFunction=m,S.isWindow=x,S.camelCase=X,S.type=w,S.now=Date.now,S.isNumeric=function(e){var t=S.type(e);return("number"===t||"string"===t)&&!isNaN(e-parseFloat(e))},S.trim=function(e){return null==e?"":(e+"").replace(Xt,"")},"function"==typeof define&&define.amd&&define("jquery",[],function(){return S});var Vt=C.jQuery,Gt=C.$;return S.noConflict=function(e){return C.$===S&&(C.$=Gt),e&&C.jQuery===S&&(C.jQuery=Vt),S},"undefined"==typeof e&&(C.jQuery=C.$=S),S}); ================================================ FILE: core/dbt/docs/build/html/_static/language_data.js ================================================ /* * language_data.js * ~~~~~~~~~~~~~~~~ * * This script contains the language-specific data used by searchtools.js, * namely the list of stopwords, stemmer, scorer and splitter. * * :copyright: Copyright 2007-2022 by the Sphinx team, see AUTHORS. * :license: BSD, see LICENSE for details. * */ var stopwords = ["a", "and", "are", "as", "at", "be", "but", "by", "for", "if", "in", "into", "is", "it", "near", "no", "not", "of", "on", "or", "such", "that", "the", "their", "then", "there", "these", "they", "this", "to", "was", "will", "with"]; /* Non-minified version is copied as a separate JS file, is available */ /** * Porter Stemmer */ var Stemmer = function() { var step2list = { ational: 'ate', tional: 'tion', enci: 'ence', anci: 'ance', izer: 'ize', bli: 'ble', alli: 'al', entli: 'ent', eli: 'e', ousli: 'ous', ization: 'ize', ation: 'ate', ator: 'ate', alism: 'al', iveness: 'ive', fulness: 'ful', ousness: 'ous', aliti: 'al', iviti: 'ive', biliti: 'ble', logi: 'log' }; var step3list = { icate: 'ic', ative: '', alize: 'al', iciti: 'ic', ical: 'ic', ful: '', ness: '' }; var c = "[^aeiou]"; // consonant var v = "[aeiouy]"; // vowel var C = c + "[^aeiouy]*"; // consonant sequence var V = v + "[aeiou]*"; // vowel sequence var mgr0 = "^(" + C + ")?" + V + C; // [C]VC... is m>0 var meq1 = "^(" + C + ")?" + V + C + "(" + V + ")?$"; // [C]VC[V] is m=1 var mgr1 = "^(" + C + ")?" + V + C + V + C; // [C]VCVC... is m>1 var s_v = "^(" + C + ")?" + v; // vowel in stem this.stemWord = function (w) { var stem; var suffix; var firstch; var origword = w; if (w.length < 3) return w; var re; var re2; var re3; var re4; firstch = w.substr(0,1); if (firstch == "y") w = firstch.toUpperCase() + w.substr(1); // Step 1a re = /^(.+?)(ss|i)es$/; re2 = /^(.+?)([^s])s$/; if (re.test(w)) w = w.replace(re,"$1$2"); else if (re2.test(w)) w = w.replace(re2,"$1$2"); // Step 1b re = /^(.+?)eed$/; re2 = /^(.+?)(ed|ing)$/; if (re.test(w)) { var fp = re.exec(w); re = new RegExp(mgr0); if (re.test(fp[1])) { re = /.$/; w = w.replace(re,""); } } else if (re2.test(w)) { var fp = re2.exec(w); stem = fp[1]; re2 = new RegExp(s_v); if (re2.test(stem)) { w = stem; re2 = /(at|bl|iz)$/; re3 = new RegExp("([^aeiouylsz])\\1$"); re4 = new RegExp("^" + C + v + "[^aeiouwxy]$"); if (re2.test(w)) w = w + "e"; else if (re3.test(w)) { re = /.$/; w = w.replace(re,""); } else if (re4.test(w)) w = w + "e"; } } // Step 1c re = /^(.+?)y$/; if (re.test(w)) { var fp = re.exec(w); stem = fp[1]; re = new RegExp(s_v); if (re.test(stem)) w = stem + "i"; } // Step 2 re = /^(.+?)(ational|tional|enci|anci|izer|bli|alli|entli|eli|ousli|ization|ation|ator|alism|iveness|fulness|ousness|aliti|iviti|biliti|logi)$/; if (re.test(w)) { var fp = re.exec(w); stem = fp[1]; suffix = fp[2]; re = new RegExp(mgr0); if (re.test(stem)) w = stem + step2list[suffix]; } // Step 3 re = /^(.+?)(icate|ative|alize|iciti|ical|ful|ness)$/; if (re.test(w)) { var fp = re.exec(w); stem = fp[1]; suffix = fp[2]; re = new RegExp(mgr0); if (re.test(stem)) w = stem + step3list[suffix]; } // Step 4 re = /^(.+?)(al|ance|ence|er|ic|able|ible|ant|ement|ment|ent|ou|ism|ate|iti|ous|ive|ize)$/; re2 = /^(.+?)(s|t)(ion)$/; if (re.test(w)) { var fp = re.exec(w); stem = fp[1]; re = new RegExp(mgr1); if (re.test(stem)) w = stem; } else if (re2.test(w)) { var fp = re2.exec(w); stem = fp[1] + fp[2]; re2 = new RegExp(mgr1); if (re2.test(stem)) w = stem; } // Step 5 re = /^(.+?)e$/; if (re.test(w)) { var fp = re.exec(w); stem = fp[1]; re = new RegExp(mgr1); re2 = new RegExp(meq1); re3 = new RegExp("^" + C + v + "[^aeiouwxy]$"); if (re.test(stem) || (re2.test(stem) && !(re3.test(stem)))) w = stem; } re = /ll$/; re2 = new RegExp(mgr1); if (re.test(w) && re2.test(w)) { re = /.$/; w = w.replace(re,""); } // and turn initial Y back to y if (firstch == "y") w = firstch.toLowerCase() + w.substr(1); return w; } } ================================================ FILE: core/dbt/docs/build/html/_static/pygments.css ================================================ pre { line-height: 125%; } td.linenos .normal { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; } span.linenos { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; } td.linenos .special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; } span.linenos.special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; } .highlight .hll { background-color: #ffffcc } .highlight { background: #f8f8f8; } .highlight .c { color: #8f5902; font-style: italic } /* Comment */ .highlight .err { color: #a40000; border: 1px solid #ef2929 } /* Error */ .highlight .g { color: #000000 } /* Generic */ .highlight .k { color: #004461; font-weight: bold } /* Keyword */ .highlight .l { color: #000000 } /* Literal */ .highlight .n { color: #000000 } /* Name */ .highlight .o { color: #582800 } /* Operator */ .highlight .x { color: #000000 } /* Other */ .highlight .p { color: #000000; font-weight: bold } /* Punctuation */ .highlight .ch { color: #8f5902; font-style: italic } /* Comment.Hashbang */ .highlight .cm { color: #8f5902; font-style: italic } /* Comment.Multiline */ .highlight .cp { color: #8f5902 } /* Comment.Preproc */ .highlight .cpf { color: #8f5902; font-style: italic } /* Comment.PreprocFile */ .highlight .c1 { color: #8f5902; font-style: italic } /* Comment.Single */ .highlight .cs { color: #8f5902; font-style: italic } /* Comment.Special */ .highlight .gd { color: #a40000 } /* Generic.Deleted */ .highlight .ge { color: #000000; font-style: italic } /* Generic.Emph */ .highlight .gr { color: #ef2929 } /* Generic.Error */ .highlight .gh { color: #000080; font-weight: bold } /* Generic.Heading */ .highlight .gi { color: #00A000 } /* Generic.Inserted */ .highlight .go { color: #888888 } /* Generic.Output */ .highlight .gp { color: #745334 } /* Generic.Prompt */ .highlight .gs { color: #000000; font-weight: bold } /* Generic.Strong */ .highlight .gu { color: #800080; font-weight: bold } /* Generic.Subheading */ .highlight .gt { color: #a40000; font-weight: bold } /* Generic.Traceback */ .highlight .kc { color: #004461; font-weight: bold } /* Keyword.Constant */ .highlight .kd { color: #004461; font-weight: bold } /* Keyword.Declaration */ .highlight .kn { color: #004461; font-weight: bold } /* Keyword.Namespace */ .highlight .kp { color: #004461; font-weight: bold } /* Keyword.Pseudo */ .highlight .kr { color: #004461; font-weight: bold } /* Keyword.Reserved */ .highlight .kt { color: #004461; font-weight: bold } /* Keyword.Type */ .highlight .ld { color: #000000 } /* Literal.Date */ .highlight .m { color: #990000 } /* Literal.Number */ .highlight .s { color: #4e9a06 } /* Literal.String */ .highlight .na { color: #c4a000 } /* Name.Attribute */ .highlight .nb { color: #004461 } /* Name.Builtin */ .highlight .nc { color: #000000 } /* Name.Class */ .highlight .no { color: #000000 } /* Name.Constant */ .highlight .nd { color: #888888 } /* Name.Decorator */ .highlight .ni { color: #ce5c00 } /* Name.Entity */ .highlight .ne { color: #cc0000; font-weight: bold } /* Name.Exception */ .highlight .nf { color: #000000 } /* Name.Function */ .highlight .nl { color: #f57900 } /* Name.Label */ .highlight .nn { color: #000000 } /* Name.Namespace */ .highlight .nx { color: #000000 } /* Name.Other */ .highlight .py { color: #000000 } /* Name.Property */ .highlight .nt { color: #004461; font-weight: bold } /* Name.Tag */ .highlight .nv { color: #000000 } /* Name.Variable */ .highlight .ow { color: #004461; font-weight: bold } /* Operator.Word */ .highlight .w { color: #f8f8f8; text-decoration: underline } /* Text.Whitespace */ .highlight .mb { color: #990000 } /* Literal.Number.Bin */ .highlight .mf { color: #990000 } /* Literal.Number.Float */ .highlight .mh { color: #990000 } /* Literal.Number.Hex */ .highlight .mi { color: #990000 } /* Literal.Number.Integer */ .highlight .mo { color: #990000 } /* Literal.Number.Oct */ .highlight .sa { color: #4e9a06 } /* Literal.String.Affix */ .highlight .sb { color: #4e9a06 } /* Literal.String.Backtick */ .highlight .sc { color: #4e9a06 } /* Literal.String.Char */ .highlight .dl { color: #4e9a06 } /* Literal.String.Delimiter */ .highlight .sd { color: #8f5902; font-style: italic } /* Literal.String.Doc */ .highlight .s2 { color: #4e9a06 } /* Literal.String.Double */ .highlight .se { color: #4e9a06 } /* Literal.String.Escape */ .highlight .sh { color: #4e9a06 } /* Literal.String.Heredoc */ .highlight .si { color: #4e9a06 } /* Literal.String.Interpol */ .highlight .sx { color: #4e9a06 } /* Literal.String.Other */ .highlight .sr { color: #4e9a06 } /* Literal.String.Regex */ .highlight .s1 { color: #4e9a06 } /* Literal.String.Single */ .highlight .ss { color: #4e9a06 } /* Literal.String.Symbol */ .highlight .bp { color: #3465a4 } /* Name.Builtin.Pseudo */ .highlight .fm { color: #000000 } /* Name.Function.Magic */ .highlight .vc { color: #000000 } /* Name.Variable.Class */ .highlight .vg { color: #000000 } /* Name.Variable.Global */ .highlight .vi { color: #000000 } /* Name.Variable.Instance */ .highlight .vm { color: #000000 } /* Name.Variable.Magic */ .highlight .il { color: #990000 } /* Literal.Number.Integer.Long */ ================================================ FILE: core/dbt/docs/build/html/_static/searchtools.js ================================================ /* * searchtools.js * ~~~~~~~~~~~~~~~~ * * Sphinx JavaScript utilities for the full-text search. * * :copyright: Copyright 2007-2022 by the Sphinx team, see AUTHORS. * :license: BSD, see LICENSE for details. * */ "use strict"; /** * Simple result scoring code. */ if (typeof Scorer === "undefined") { var Scorer = { // Implement the following function to further tweak the score for each result // The function takes a result array [docname, title, anchor, descr, score, filename] // and returns the new score. /* score: result => { const [docname, title, anchor, descr, score, filename] = result return score }, */ // query matches the full name of an object objNameMatch: 11, // or matches in the last dotted part of the object name objPartialMatch: 6, // Additive scores depending on the priority of the object objPrio: { 0: 15, // used to be importantResults 1: 5, // used to be objectResults 2: -5, // used to be unimportantResults }, // Used when the priority is not in the mapping. objPrioDefault: 0, // query found in title title: 15, partialTitle: 7, // query found in terms term: 5, partialTerm: 2, }; } const _removeChildren = (element) => { while (element && element.lastChild) element.removeChild(element.lastChild); }; /** * See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions#escaping */ const _escapeRegExp = (string) => string.replace(/[.*+\-?^${}()|[\]\\]/g, "\\$&"); // $& means the whole matched string const _displayItem = (item, searchTerms) => { const docBuilder = DOCUMENTATION_OPTIONS.BUILDER; const docUrlRoot = DOCUMENTATION_OPTIONS.URL_ROOT; const docFileSuffix = DOCUMENTATION_OPTIONS.FILE_SUFFIX; const docLinkSuffix = DOCUMENTATION_OPTIONS.LINK_SUFFIX; const showSearchSummary = DOCUMENTATION_OPTIONS.SHOW_SEARCH_SUMMARY; const [docName, title, anchor, descr, score, _filename] = item; let listItem = document.createElement("li"); let requestUrl; let linkUrl; if (docBuilder === "dirhtml") { // dirhtml builder let dirname = docName + "/"; if (dirname.match(/\/index\/$/)) dirname = dirname.substring(0, dirname.length - 6); else if (dirname === "index/") dirname = ""; requestUrl = docUrlRoot + dirname; linkUrl = requestUrl; } else { // normal html builders requestUrl = docUrlRoot + docName + docFileSuffix; linkUrl = docName + docLinkSuffix; } let linkEl = listItem.appendChild(document.createElement("a")); linkEl.href = linkUrl + anchor; linkEl.dataset.score = score; linkEl.innerHTML = title; if (descr) listItem.appendChild(document.createElement("span")).innerHTML = " (" + descr + ")"; else if (showSearchSummary) fetch(requestUrl) .then((responseData) => responseData.text()) .then((data) => { if (data) listItem.appendChild( Search.makeSearchSummary(data, searchTerms) ); }); Search.output.appendChild(listItem); }; const _finishSearch = (resultCount) => { Search.stopPulse(); Search.title.innerText = _("Search Results"); if (!resultCount) Search.status.innerText = Documentation.gettext( "Your search did not match any documents. Please make sure that all words are spelled correctly and that you've selected enough categories." ); else Search.status.innerText = _( `Search finished, found ${resultCount} page(s) matching the search query.` ); }; const _displayNextItem = ( results, resultCount, searchTerms ) => { // results left, load the summary and display it // this is intended to be dynamic (don't sub resultsCount) if (results.length) { _displayItem(results.pop(), searchTerms); setTimeout( () => _displayNextItem(results, resultCount, searchTerms), 5 ); } // search finished, update title and status message else _finishSearch(resultCount); }; /** * Default splitQuery function. Can be overridden in ``sphinx.search`` with a * custom function per language. * * The regular expression works by splitting the string on consecutive characters * that are not Unicode letters, numbers, underscores, or emoji characters. * This is the same as ``\W+`` in Python, preserving the surrogate pair area. */ if (typeof splitQuery === "undefined") { var splitQuery = (query) => query .split(/[^\p{Letter}\p{Number}_\p{Emoji_Presentation}]+/gu) .filter(term => term) // remove remaining empty strings } /** * Search Module */ const Search = { _index: null, _queued_query: null, _pulse_status: -1, htmlToText: (htmlString) => { const htmlElement = new DOMParser().parseFromString(htmlString, 'text/html'); htmlElement.querySelectorAll(".headerlink").forEach((el) => { el.remove() }); const docContent = htmlElement.querySelector('[role="main"]'); if (docContent !== undefined) return docContent.textContent; console.warn( "Content block not found. Sphinx search tries to obtain it via '[role=main]'. Could you check your theme or template." ); return ""; }, init: () => { const query = new URLSearchParams(window.location.search).get("q"); document .querySelectorAll('input[name="q"]') .forEach((el) => (el.value = query)); if (query) Search.performSearch(query); }, loadIndex: (url) => (document.body.appendChild(document.createElement("script")).src = url), setIndex: (index) => { Search._index = index; if (Search._queued_query !== null) { const query = Search._queued_query; Search._queued_query = null; Search.query(query); } }, hasIndex: () => Search._index !== null, deferQuery: (query) => (Search._queued_query = query), stopPulse: () => (Search._pulse_status = -1), startPulse: () => { if (Search._pulse_status >= 0) return; const pulse = () => { Search._pulse_status = (Search._pulse_status + 1) % 4; Search.dots.innerText = ".".repeat(Search._pulse_status); if (Search._pulse_status >= 0) window.setTimeout(pulse, 500); }; pulse(); }, /** * perform a search for something (or wait until index is loaded) */ performSearch: (query) => { // create the required interface elements const searchText = document.createElement("h2"); searchText.textContent = _("Searching"); const searchSummary = document.createElement("p"); searchSummary.classList.add("search-summary"); searchSummary.innerText = ""; const searchList = document.createElement("ul"); searchList.classList.add("search"); const out = document.getElementById("search-results"); Search.title = out.appendChild(searchText); Search.dots = Search.title.appendChild(document.createElement("span")); Search.status = out.appendChild(searchSummary); Search.output = out.appendChild(searchList); const searchProgress = document.getElementById("search-progress"); // Some themes don't use the search progress node if (searchProgress) { searchProgress.innerText = _("Preparing search..."); } Search.startPulse(); // index already loaded, the browser was quick! if (Search.hasIndex()) Search.query(query); else Search.deferQuery(query); }, /** * execute search (requires search index to be loaded) */ query: (query) => { const filenames = Search._index.filenames; const docNames = Search._index.docnames; const titles = Search._index.titles; const allTitles = Search._index.alltitles; const indexEntries = Search._index.indexentries; // stem the search terms and add them to the correct list const stemmer = new Stemmer(); const searchTerms = new Set(); const excludedTerms = new Set(); const highlightTerms = new Set(); const objectTerms = new Set(splitQuery(query.toLowerCase().trim())); splitQuery(query.trim()).forEach((queryTerm) => { const queryTermLower = queryTerm.toLowerCase(); // maybe skip this "word" // stopwords array is from language_data.js if ( stopwords.indexOf(queryTermLower) !== -1 || queryTerm.match(/^\d+$/) ) return; // stem the word let word = stemmer.stemWord(queryTermLower); // select the correct list if (word[0] === "-") excludedTerms.add(word.substr(1)); else { searchTerms.add(word); highlightTerms.add(queryTermLower); } }); if (SPHINX_HIGHLIGHT_ENABLED) { // set in sphinx_highlight.js localStorage.setItem("sphinx_highlight_terms", [...highlightTerms].join(" ")) } // console.debug("SEARCH: searching for:"); // console.info("required: ", [...searchTerms]); // console.info("excluded: ", [...excludedTerms]); // array of [docname, title, anchor, descr, score, filename] let results = []; _removeChildren(document.getElementById("search-progress")); const queryLower = query.toLowerCase(); for (const [title, foundTitles] of Object.entries(allTitles)) { if (title.toLowerCase().includes(queryLower) && (queryLower.length >= title.length/2)) { for (const [file, id] of foundTitles) { let score = Math.round(100 * queryLower.length / title.length) results.push([ docNames[file], titles[file] !== title ? `${titles[file]} > ${title}` : title, id !== null ? "#" + id : "", null, score, filenames[file], ]); } } } // search for explicit entries in index directives for (const [entry, foundEntries] of Object.entries(indexEntries)) { if (entry.includes(queryLower) && (queryLower.length >= entry.length/2)) { for (const [file, id] of foundEntries) { let score = Math.round(100 * queryLower.length / entry.length) results.push([ docNames[file], titles[file], id ? "#" + id : "", null, score, filenames[file], ]); } } } // lookup as object objectTerms.forEach((term) => results.push(...Search.performObjectSearch(term, objectTerms)) ); // lookup as search terms in fulltext results.push(...Search.performTermsSearch(searchTerms, excludedTerms)); // let the scorer override scores with a custom scoring function if (Scorer.score) results.forEach((item) => (item[4] = Scorer.score(item))); // now sort the results by score (in opposite order of appearance, since the // display function below uses pop() to retrieve items) and then // alphabetically results.sort((a, b) => { const leftScore = a[4]; const rightScore = b[4]; if (leftScore === rightScore) { // same score: sort alphabetically const leftTitle = a[1].toLowerCase(); const rightTitle = b[1].toLowerCase(); if (leftTitle === rightTitle) return 0; return leftTitle > rightTitle ? -1 : 1; // inverted is intentional } return leftScore > rightScore ? 1 : -1; }); // remove duplicate search results // note the reversing of results, so that in the case of duplicates, the highest-scoring entry is kept let seen = new Set(); results = results.reverse().reduce((acc, result) => { let resultStr = result.slice(0, 4).concat([result[5]]).map(v => String(v)).join(','); if (!seen.has(resultStr)) { acc.push(result); seen.add(resultStr); } return acc; }, []); results = results.reverse(); // for debugging //Search.lastresults = results.slice(); // a copy // console.info("search results:", Search.lastresults); // print the results _displayNextItem(results, results.length, searchTerms); }, /** * search for object names */ performObjectSearch: (object, objectTerms) => { const filenames = Search._index.filenames; const docNames = Search._index.docnames; const objects = Search._index.objects; const objNames = Search._index.objnames; const titles = Search._index.titles; const results = []; const objectSearchCallback = (prefix, match) => { const name = match[4] const fullname = (prefix ? prefix + "." : "") + name; const fullnameLower = fullname.toLowerCase(); if (fullnameLower.indexOf(object) < 0) return; let score = 0; const parts = fullnameLower.split("."); // check for different match types: exact matches of full name or // "last name" (i.e. last dotted part) if (fullnameLower === object || parts.slice(-1)[0] === object) score += Scorer.objNameMatch; else if (parts.slice(-1)[0].indexOf(object) > -1) score += Scorer.objPartialMatch; // matches in last name const objName = objNames[match[1]][2]; const title = titles[match[0]]; // If more than one term searched for, we require other words to be // found in the name/title/description const otherTerms = new Set(objectTerms); otherTerms.delete(object); if (otherTerms.size > 0) { const haystack = `${prefix} ${name} ${objName} ${title}`.toLowerCase(); if ( [...otherTerms].some((otherTerm) => haystack.indexOf(otherTerm) < 0) ) return; } let anchor = match[3]; if (anchor === "") anchor = fullname; else if (anchor === "-") anchor = objNames[match[1]][1] + "-" + fullname; const descr = objName + _(", in ") + title; // add custom score for some objects according to scorer if (Scorer.objPrio.hasOwnProperty(match[2])) score += Scorer.objPrio[match[2]]; else score += Scorer.objPrioDefault; results.push([ docNames[match[0]], fullname, "#" + anchor, descr, score, filenames[match[0]], ]); }; Object.keys(objects).forEach((prefix) => objects[prefix].forEach((array) => objectSearchCallback(prefix, array) ) ); return results; }, /** * search for full-text terms in the index */ performTermsSearch: (searchTerms, excludedTerms) => { // prepare search const terms = Search._index.terms; const titleTerms = Search._index.titleterms; const filenames = Search._index.filenames; const docNames = Search._index.docnames; const titles = Search._index.titles; const scoreMap = new Map(); const fileMap = new Map(); // perform the search on the required terms searchTerms.forEach((word) => { const files = []; const arr = [ { files: terms[word], score: Scorer.term }, { files: titleTerms[word], score: Scorer.title }, ]; // add support for partial matches if (word.length > 2) { const escapedWord = _escapeRegExp(word); Object.keys(terms).forEach((term) => { if (term.match(escapedWord) && !terms[word]) arr.push({ files: terms[term], score: Scorer.partialTerm }); }); Object.keys(titleTerms).forEach((term) => { if (term.match(escapedWord) && !titleTerms[word]) arr.push({ files: titleTerms[word], score: Scorer.partialTitle }); }); } // no match but word was a required one if (arr.every((record) => record.files === undefined)) return; // found search word in contents arr.forEach((record) => { if (record.files === undefined) return; let recordFiles = record.files; if (recordFiles.length === undefined) recordFiles = [recordFiles]; files.push(...recordFiles); // set score for the word in each file recordFiles.forEach((file) => { if (!scoreMap.has(file)) scoreMap.set(file, {}); scoreMap.get(file)[word] = record.score; }); }); // create the mapping files.forEach((file) => { if (fileMap.has(file) && fileMap.get(file).indexOf(word) === -1) fileMap.get(file).push(word); else fileMap.set(file, [word]); }); }); // now check if the files don't contain excluded terms const results = []; for (const [file, wordList] of fileMap) { // check if all requirements are matched // as search terms with length < 3 are discarded const filteredTermCount = [...searchTerms].filter( (term) => term.length > 2 ).length; if ( wordList.length !== searchTerms.size && wordList.length !== filteredTermCount ) continue; // ensure that none of the excluded terms is in the search result if ( [...excludedTerms].some( (term) => terms[term] === file || titleTerms[term] === file || (terms[term] || []).includes(file) || (titleTerms[term] || []).includes(file) ) ) break; // select one (max) score for the file. const score = Math.max(...wordList.map((w) => scoreMap.get(file)[w])); // add result to the result list results.push([ docNames[file], titles[file], "", null, score, filenames[file], ]); } return results; }, /** * helper function to return a node containing the * search summary for a given text. keywords is a list * of stemmed words. */ makeSearchSummary: (htmlText, keywords) => { const text = Search.htmlToText(htmlText); if (text === "") return null; const textLower = text.toLowerCase(); const actualStartPosition = [...keywords] .map((k) => textLower.indexOf(k.toLowerCase())) .filter((i) => i > -1) .slice(-1)[0]; const startWithContext = Math.max(actualStartPosition - 120, 0); const top = startWithContext === 0 ? "" : "..."; const tail = startWithContext + 240 < text.length ? "..." : ""; let summary = document.createElement("p"); summary.classList.add("context"); summary.textContent = top + text.substr(startWithContext, 240).trim() + tail; return summary; }, }; _ready(Search.init); ================================================ FILE: core/dbt/docs/build/html/_static/sphinx_highlight.js ================================================ /* Highlighting utilities for Sphinx HTML documentation. */ "use strict"; const SPHINX_HIGHLIGHT_ENABLED = true /** * highlight a given string on a node by wrapping it in * span elements with the given class name. */ const _highlight = (node, addItems, text, className) => { if (node.nodeType === Node.TEXT_NODE) { const val = node.nodeValue; const parent = node.parentNode; const pos = val.toLowerCase().indexOf(text); if ( pos >= 0 && !parent.classList.contains(className) && !parent.classList.contains("nohighlight") ) { let span; const closestNode = parent.closest("body, svg, foreignObject"); const isInSVG = closestNode && closestNode.matches("svg"); if (isInSVG) { span = document.createElementNS("http://www.w3.org/2000/svg", "tspan"); } else { span = document.createElement("span"); span.classList.add(className); } span.appendChild(document.createTextNode(val.substr(pos, text.length))); parent.insertBefore( span, parent.insertBefore( document.createTextNode(val.substr(pos + text.length)), node.nextSibling ) ); node.nodeValue = val.substr(0, pos); if (isInSVG) { const rect = document.createElementNS( "http://www.w3.org/2000/svg", "rect" ); const bbox = parent.getBBox(); rect.x.baseVal.value = bbox.x; rect.y.baseVal.value = bbox.y; rect.width.baseVal.value = bbox.width; rect.height.baseVal.value = bbox.height; rect.setAttribute("class", className); addItems.push({ parent: parent, target: rect }); } } } else if (node.matches && !node.matches("button, select, textarea")) { node.childNodes.forEach((el) => _highlight(el, addItems, text, className)); } }; const _highlightText = (thisNode, text, className) => { let addItems = []; _highlight(thisNode, addItems, text, className); addItems.forEach((obj) => obj.parent.insertAdjacentElement("beforebegin", obj.target) ); }; /** * Small JavaScript module for the documentation. */ const SphinxHighlight = { /** * highlight the search words provided in localstorage in the text */ highlightSearchWords: () => { if (!SPHINX_HIGHLIGHT_ENABLED) return; // bail if no highlight // get and clear terms from localstorage const url = new URL(window.location); const highlight = localStorage.getItem("sphinx_highlight_terms") || url.searchParams.get("highlight") || ""; localStorage.removeItem("sphinx_highlight_terms") url.searchParams.delete("highlight"); window.history.replaceState({}, "", url); // get individual terms from highlight string const terms = highlight.toLowerCase().split(/\s+/).filter(x => x); if (terms.length === 0) return; // nothing to do // There should never be more than one element matching "div.body" const divBody = document.querySelectorAll("div.body"); const body = divBody.length ? divBody[0] : document.querySelector("body"); window.setTimeout(() => { terms.forEach((term) => _highlightText(body, term, "highlighted")); }, 10); const searchBox = document.getElementById("searchbox"); if (searchBox === null) return; searchBox.appendChild( document .createRange() .createContextualFragment( '<p class="highlight-link">' + '<a href="javascript:SphinxHighlight.hideSearchWords()">' + _("Hide Search Matches") + "</a></p>" ) ); }, /** * helper function to hide the search marks again */ hideSearchWords: () => { document .querySelectorAll("#searchbox .highlight-link") .forEach((el) => el.remove()); document .querySelectorAll("span.highlighted") .forEach((el) => el.classList.remove("highlighted")); localStorage.removeItem("sphinx_highlight_terms") }, initEscapeListener: () => { // only install a listener if it is really needed if (!DOCUMENTATION_OPTIONS.ENABLE_SEARCH_SHORTCUTS) return; document.addEventListener("keydown", (event) => { // bail for input elements if (BLACKLISTED_KEY_CONTROL_ELEMENTS.has(document.activeElement.tagName)) return; // bail with special keys if (event.shiftKey || event.altKey || event.ctrlKey || event.metaKey) return; if (DOCUMENTATION_OPTIONS.ENABLE_SEARCH_SHORTCUTS && (event.key === "Escape")) { SphinxHighlight.hideSearchWords(); event.preventDefault(); } }); }, }; _ready(SphinxHighlight.highlightSearchWords); _ready(SphinxHighlight.initEscapeListener); ================================================ FILE: core/dbt/docs/build/html/_static/underscore-1.13.1.js ================================================ (function (global, factory) { typeof exports === 'object' && typeof module !== 'undefined' ? module.exports = factory() : typeof define === 'function' && define.amd ? define('underscore', factory) : (global = typeof globalThis !== 'undefined' ? globalThis : global || self, (function () { var current = global._; var exports = global._ = factory(); exports.noConflict = function () { global._ = current; return exports; }; }())); }(this, (function () { // Underscore.js 1.13.1 // https://underscorejs.org // (c) 2009-2021 Jeremy Ashkenas, Julian Gonggrijp, and DocumentCloud and Investigative Reporters & Editors // Underscore may be freely distributed under the MIT license. // Current version. var VERSION = '1.13.1'; // Establish the root object, `window` (`self`) in the browser, `global` // on the server, or `this` in some virtual machines. We use `self` // instead of `window` for `WebWorker` support. var root = typeof self == 'object' && self.self === self && self || typeof global == 'object' && global.global === global && global || Function('return this')() || {}; // Save bytes in the minified (but not gzipped) version: var ArrayProto = Array.prototype, ObjProto = Object.prototype; var SymbolProto = typeof Symbol !== 'undefined' ? Symbol.prototype : null; // Create quick reference variables for speed access to core prototypes. var push = ArrayProto.push, slice = ArrayProto.slice, toString = ObjProto.toString, hasOwnProperty = ObjProto.hasOwnProperty; // Modern feature detection. var supportsArrayBuffer = typeof ArrayBuffer !== 'undefined', supportsDataView = typeof DataView !== 'undefined'; // All **ECMAScript 5+** native function implementations that we hope to use // are declared here. var nativeIsArray = Array.isArray, nativeKeys = Object.keys, nativeCreate = Object.create, nativeIsView = supportsArrayBuffer && ArrayBuffer.isView; // Create references to these builtin functions because we override them. var _isNaN = isNaN, _isFinite = isFinite; // Keys in IE < 9 that won't be iterated by `for key in ...` and thus missed. var hasEnumBug = !{toString: null}.propertyIsEnumerable('toString'); var nonEnumerableProps = ['valueOf', 'isPrototypeOf', 'toString', 'propertyIsEnumerable', 'hasOwnProperty', 'toLocaleString']; // The largest integer that can be represented exactly. var MAX_ARRAY_INDEX = Math.pow(2, 53) - 1; // Some functions take a variable number of arguments, or a few expected // arguments at the beginning and then a variable number of values to operate // on. This helper accumulates all remaining arguments past the function’s // argument length (or an explicit `startIndex`), into an array that becomes // the last argument. Similar to ES6’s "rest parameter". function restArguments(func, startIndex) { startIndex = startIndex == null ? func.length - 1 : +startIndex; return function() { var length = Math.max(arguments.length - startIndex, 0), rest = Array(length), index = 0; for (; index < length; index++) { rest[index] = arguments[index + startIndex]; } switch (startIndex) { case 0: return func.call(this, rest); case 1: return func.call(this, arguments[0], rest); case 2: return func.call(this, arguments[0], arguments[1], rest); } var args = Array(startIndex + 1); for (index = 0; index < startIndex; index++) { args[index] = arguments[index]; } args[startIndex] = rest; return func.apply(this, args); }; } // Is a given variable an object? function isObject(obj) { var type = typeof obj; return type === 'function' || type === 'object' && !!obj; } // Is a given value equal to null? function isNull(obj) { return obj === null; } // Is a given variable undefined? function isUndefined(obj) { return obj === void 0; } // Is a given value a boolean? function isBoolean(obj) { return obj === true || obj === false || toString.call(obj) === '[object Boolean]'; } // Is a given value a DOM element? function isElement(obj) { return !!(obj && obj.nodeType === 1); } // Internal function for creating a `toString`-based type tester. function tagTester(name) { var tag = '[object ' + name + ']'; return function(obj) { return toString.call(obj) === tag; }; } var isString = tagTester('String'); var isNumber = tagTester('Number'); var isDate = tagTester('Date'); var isRegExp = tagTester('RegExp'); var isError = tagTester('Error'); var isSymbol = tagTester('Symbol'); var isArrayBuffer = tagTester('ArrayBuffer'); var isFunction = tagTester('Function'); // Optimize `isFunction` if appropriate. Work around some `typeof` bugs in old // v8, IE 11 (#1621), Safari 8 (#1929), and PhantomJS (#2236). var nodelist = root.document && root.document.childNodes; if (typeof /./ != 'function' && typeof Int8Array != 'object' && typeof nodelist != 'function') { isFunction = function(obj) { return typeof obj == 'function' || false; }; } var isFunction$1 = isFunction; var hasObjectTag = tagTester('Object'); // In IE 10 - Edge 13, `DataView` has string tag `'[object Object]'`. // In IE 11, the most common among them, this problem also applies to // `Map`, `WeakMap` and `Set`. var hasStringTagBug = ( supportsDataView && hasObjectTag(new DataView(new ArrayBuffer(8))) ), isIE11 = (typeof Map !== 'undefined' && hasObjectTag(new Map)); var isDataView = tagTester('DataView'); // In IE 10 - Edge 13, we need a different heuristic // to determine whether an object is a `DataView`. function ie10IsDataView(obj) { return obj != null && isFunction$1(obj.getInt8) && isArrayBuffer(obj.buffer); } var isDataView$1 = (hasStringTagBug ? ie10IsDataView : isDataView); // Is a given value an array? // Delegates to ECMA5's native `Array.isArray`. var isArray = nativeIsArray || tagTester('Array'); // Internal function to check whether `key` is an own property name of `obj`. function has$1(obj, key) { return obj != null && hasOwnProperty.call(obj, key); } var isArguments = tagTester('Arguments'); // Define a fallback version of the method in browsers (ahem, IE < 9), where // there isn't any inspectable "Arguments" type. (function() { if (!isArguments(arguments)) { isArguments = function(obj) { return has$1(obj, 'callee'); }; } }()); var isArguments$1 = isArguments; // Is a given object a finite number? function isFinite$1(obj) { return !isSymbol(obj) && _isFinite(obj) && !isNaN(parseFloat(obj)); } // Is the given value `NaN`? function isNaN$1(obj) { return isNumber(obj) && _isNaN(obj); } // Predicate-generating function. Often useful outside of Underscore. function constant(value) { return function() { return value; }; } // Common internal logic for `isArrayLike` and `isBufferLike`. function createSizePropertyCheck(getSizeProperty) { return function(collection) { var sizeProperty = getSizeProperty(collection); return typeof sizeProperty == 'number' && sizeProperty >= 0 && sizeProperty <= MAX_ARRAY_INDEX; } } // Internal helper to generate a function to obtain property `key` from `obj`. function shallowProperty(key) { return function(obj) { return obj == null ? void 0 : obj[key]; }; } // Internal helper to obtain the `byteLength` property of an object. var getByteLength = shallowProperty('byteLength'); // Internal helper to determine whether we should spend extensive checks against // `ArrayBuffer` et al. var isBufferLike = createSizePropertyCheck(getByteLength); // Is a given value a typed array? var typedArrayPattern = /\[object ((I|Ui)nt(8|16|32)|Float(32|64)|Uint8Clamped|Big(I|Ui)nt64)Array\]/; function isTypedArray(obj) { // `ArrayBuffer.isView` is the most future-proof, so use it when available. // Otherwise, fall back on the above regular expression. return nativeIsView ? (nativeIsView(obj) && !isDataView$1(obj)) : isBufferLike(obj) && typedArrayPattern.test(toString.call(obj)); } var isTypedArray$1 = supportsArrayBuffer ? isTypedArray : constant(false); // Internal helper to obtain the `length` property of an object. var getLength = shallowProperty('length'); // Internal helper to create a simple lookup structure. // `collectNonEnumProps` used to depend on `_.contains`, but this led to // circular imports. `emulatedSet` is a one-off solution that only works for // arrays of strings. function emulatedSet(keys) { var hash = {}; for (var l = keys.length, i = 0; i < l; ++i) hash[keys[i]] = true; return { contains: function(key) { return hash[key]; }, push: function(key) { hash[key] = true; return keys.push(key); } }; } // Internal helper. Checks `keys` for the presence of keys in IE < 9 that won't // be iterated by `for key in ...` and thus missed. Extends `keys` in place if // needed. function collectNonEnumProps(obj, keys) { keys = emulatedSet(keys); var nonEnumIdx = nonEnumerableProps.length; var constructor = obj.constructor; var proto = isFunction$1(constructor) && constructor.prototype || ObjProto; // Constructor is a special case. var prop = 'constructor'; if (has$1(obj, prop) && !keys.contains(prop)) keys.push(prop); while (nonEnumIdx--) { prop = nonEnumerableProps[nonEnumIdx]; if (prop in obj && obj[prop] !== proto[prop] && !keys.contains(prop)) { keys.push(prop); } } } // Retrieve the names of an object's own properties. // Delegates to **ECMAScript 5**'s native `Object.keys`. function keys(obj) { if (!isObject(obj)) return []; if (nativeKeys) return nativeKeys(obj); var keys = []; for (var key in obj) if (has$1(obj, key)) keys.push(key); // Ahem, IE < 9. if (hasEnumBug) collectNonEnumProps(obj, keys); return keys; } // Is a given array, string, or object empty? // An "empty" object has no enumerable own-properties. function isEmpty(obj) { if (obj == null) return true; // Skip the more expensive `toString`-based type checks if `obj` has no // `.length`. var length = getLength(obj); if (typeof length == 'number' && ( isArray(obj) || isString(obj) || isArguments$1(obj) )) return length === 0; return getLength(keys(obj)) === 0; } // Returns whether an object has a given set of `key:value` pairs. function isMatch(object, attrs) { var _keys = keys(attrs), length = _keys.length; if (object == null) return !length; var obj = Object(object); for (var i = 0; i < length; i++) { var key = _keys[i]; if (attrs[key] !== obj[key] || !(key in obj)) return false; } return true; } // If Underscore is called as a function, it returns a wrapped object that can // be used OO-style. This wrapper holds altered versions of all functions added // through `_.mixin`. Wrapped objects may be chained. function _$1(obj) { if (obj instanceof _$1) return obj; if (!(this instanceof _$1)) return new _$1(obj); this._wrapped = obj; } _$1.VERSION = VERSION; // Extracts the result from a wrapped and chained object. _$1.prototype.value = function() { return this._wrapped; }; // Provide unwrapping proxies for some methods used in engine operations // such as arithmetic and JSON stringification. _$1.prototype.valueOf = _$1.prototype.toJSON = _$1.prototype.value; _$1.prototype.toString = function() { return String(this._wrapped); }; // Internal function to wrap or shallow-copy an ArrayBuffer, // typed array or DataView to a new view, reusing the buffer. function toBufferView(bufferSource) { return new Uint8Array( bufferSource.buffer || bufferSource, bufferSource.byteOffset || 0, getByteLength(bufferSource) ); } // We use this string twice, so give it a name for minification. var tagDataView = '[object DataView]'; // Internal recursive comparison function for `_.isEqual`. function eq(a, b, aStack, bStack) { // Identical objects are equal. `0 === -0`, but they aren't identical. // See the [Harmony `egal` proposal](https://wiki.ecmascript.org/doku.php?id=harmony:egal). if (a === b) return a !== 0 || 1 / a === 1 / b; // `null` or `undefined` only equal to itself (strict comparison). if (a == null || b == null) return false; // `NaN`s are equivalent, but non-reflexive. if (a !== a) return b !== b; // Exhaust primitive checks var type = typeof a; if (type !== 'function' && type !== 'object' && typeof b != 'object') return false; return deepEq(a, b, aStack, bStack); } // Internal recursive comparison function for `_.isEqual`. function deepEq(a, b, aStack, bStack) { // Unwrap any wrapped objects. if (a instanceof _$1) a = a._wrapped; if (b instanceof _$1) b = b._wrapped; // Compare `[[Class]]` names. var className = toString.call(a); if (className !== toString.call(b)) return false; // Work around a bug in IE 10 - Edge 13. if (hasStringTagBug && className == '[object Object]' && isDataView$1(a)) { if (!isDataView$1(b)) return false; className = tagDataView; } switch (className) { // These types are compared by value. case '[object RegExp]': // RegExps are coerced to strings for comparison (Note: '' + /a/i === '/a/i') case '[object String]': // Primitives and their corresponding object wrappers are equivalent; thus, `"5"` is // equivalent to `new String("5")`. return '' + a === '' + b; case '[object Number]': // `NaN`s are equivalent, but non-reflexive. // Object(NaN) is equivalent to NaN. if (+a !== +a) return +b !== +b; // An `egal` comparison is performed for other numeric values. return +a === 0 ? 1 / +a === 1 / b : +a === +b; case '[object Date]': case '[object Boolean]': // Coerce dates and booleans to numeric primitive values. Dates are compared by their // millisecond representations. Note that invalid dates with millisecond representations // of `NaN` are not equivalent. return +a === +b; case '[object Symbol]': return SymbolProto.valueOf.call(a) === SymbolProto.valueOf.call(b); case '[object ArrayBuffer]': case tagDataView: // Coerce to typed array so we can fall through. return deepEq(toBufferView(a), toBufferView(b), aStack, bStack); } var areArrays = className === '[object Array]'; if (!areArrays && isTypedArray$1(a)) { var byteLength = getByteLength(a); if (byteLength !== getByteLength(b)) return false; if (a.buffer === b.buffer && a.byteOffset === b.byteOffset) return true; areArrays = true; } if (!areArrays) { if (typeof a != 'object' || typeof b != 'object') return false; // Objects with different constructors are not equivalent, but `Object`s or `Array`s // from different frames are. var aCtor = a.constructor, bCtor = b.constructor; if (aCtor !== bCtor && !(isFunction$1(aCtor) && aCtor instanceof aCtor && isFunction$1(bCtor) && bCtor instanceof bCtor) && ('constructor' in a && 'constructor' in b)) { return false; } } // Assume equality for cyclic structures. The algorithm for detecting cyclic // structures is adapted from ES 5.1 section 15.12.3, abstract operation `JO`. // Initializing stack of traversed objects. // It's done here since we only need them for objects and arrays comparison. aStack = aStack || []; bStack = bStack || []; var length = aStack.length; while (length--) { // Linear search. Performance is inversely proportional to the number of // unique nested structures. if (aStack[length] === a) return bStack[length] === b; } // Add the first object to the stack of traversed objects. aStack.push(a); bStack.push(b); // Recursively compare objects and arrays. if (areArrays) { // Compare array lengths to determine if a deep comparison is necessary. length = a.length; if (length !== b.length) return false; // Deep compare the contents, ignoring non-numeric properties. while (length--) { if (!eq(a[length], b[length], aStack, bStack)) return false; } } else { // Deep compare objects. var _keys = keys(a), key; length = _keys.length; // Ensure that both objects contain the same number of properties before comparing deep equality. if (keys(b).length !== length) return false; while (length--) { // Deep compare each member key = _keys[length]; if (!(has$1(b, key) && eq(a[key], b[key], aStack, bStack))) return false; } } // Remove the first object from the stack of traversed objects. aStack.pop(); bStack.pop(); return true; } // Perform a deep comparison to check if two objects are equal. function isEqual(a, b) { return eq(a, b); } // Retrieve all the enumerable property names of an object. function allKeys(obj) { if (!isObject(obj)) return []; var keys = []; for (var key in obj) keys.push(key); // Ahem, IE < 9. if (hasEnumBug) collectNonEnumProps(obj, keys); return keys; } // Since the regular `Object.prototype.toString` type tests don't work for // some types in IE 11, we use a fingerprinting heuristic instead, based // on the methods. It's not great, but it's the best we got. // The fingerprint method lists are defined below. function ie11fingerprint(methods) { var length = getLength(methods); return function(obj) { if (obj == null) return false; // `Map`, `WeakMap` and `Set` have no enumerable keys. var keys = allKeys(obj); if (getLength(keys)) return false; for (var i = 0; i < length; i++) { if (!isFunction$1(obj[methods[i]])) return false; } // If we are testing against `WeakMap`, we need to ensure that // `obj` doesn't have a `forEach` method in order to distinguish // it from a regular `Map`. return methods !== weakMapMethods || !isFunction$1(obj[forEachName]); }; } // In the interest of compact minification, we write // each string in the fingerprints only once. var forEachName = 'forEach', hasName = 'has', commonInit = ['clear', 'delete'], mapTail = ['get', hasName, 'set']; // `Map`, `WeakMap` and `Set` each have slightly different // combinations of the above sublists. var mapMethods = commonInit.concat(forEachName, mapTail), weakMapMethods = commonInit.concat(mapTail), setMethods = ['add'].concat(commonInit, forEachName, hasName); var isMap = isIE11 ? ie11fingerprint(mapMethods) : tagTester('Map'); var isWeakMap = isIE11 ? ie11fingerprint(weakMapMethods) : tagTester('WeakMap'); var isSet = isIE11 ? ie11fingerprint(setMethods) : tagTester('Set'); var isWeakSet = tagTester('WeakSet'); // Retrieve the values of an object's properties. function values(obj) { var _keys = keys(obj); var length = _keys.length; var values = Array(length); for (var i = 0; i < length; i++) { values[i] = obj[_keys[i]]; } return values; } // Convert an object into a list of `[key, value]` pairs. // The opposite of `_.object` with one argument. function pairs(obj) { var _keys = keys(obj); var length = _keys.length; var pairs = Array(length); for (var i = 0; i < length; i++) { pairs[i] = [_keys[i], obj[_keys[i]]]; } return pairs; } // Invert the keys and values of an object. The values must be serializable. function invert(obj) { var result = {}; var _keys = keys(obj); for (var i = 0, length = _keys.length; i < length; i++) { result[obj[_keys[i]]] = _keys[i]; } return result; } // Return a sorted list of the function names available on the object. function functions(obj) { var names = []; for (var key in obj) { if (isFunction$1(obj[key])) names.push(key); } return names.sort(); } // An internal function for creating assigner functions. function createAssigner(keysFunc, defaults) { return function(obj) { var length = arguments.length; if (defaults) obj = Object(obj); if (length < 2 || obj == null) return obj; for (var index = 1; index < length; index++) { var source = arguments[index], keys = keysFunc(source), l = keys.length; for (var i = 0; i < l; i++) { var key = keys[i]; if (!defaults || obj[key] === void 0) obj[key] = source[key]; } } return obj; }; } // Extend a given object with all the properties in passed-in object(s). var extend = createAssigner(allKeys); // Assigns a given object with all the own properties in the passed-in // object(s). // (https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object/assign) var extendOwn = createAssigner(keys); // Fill in a given object with default properties. var defaults = createAssigner(allKeys, true); // Create a naked function reference for surrogate-prototype-swapping. function ctor() { return function(){}; } // An internal function for creating a new object that inherits from another. function baseCreate(prototype) { if (!isObject(prototype)) return {}; if (nativeCreate) return nativeCreate(prototype); var Ctor = ctor(); Ctor.prototype = prototype; var result = new Ctor; Ctor.prototype = null; return result; } // Creates an object that inherits from the given prototype object. // If additional properties are provided then they will be added to the // created object. function create(prototype, props) { var result = baseCreate(prototype); if (props) extendOwn(result, props); return result; } // Create a (shallow-cloned) duplicate of an object. function clone(obj) { if (!isObject(obj)) return obj; return isArray(obj) ? obj.slice() : extend({}, obj); } // Invokes `interceptor` with the `obj` and then returns `obj`. // The primary purpose of this method is to "tap into" a method chain, in // order to perform operations on intermediate results within the chain. function tap(obj, interceptor) { interceptor(obj); return obj; } // Normalize a (deep) property `path` to array. // Like `_.iteratee`, this function can be customized. function toPath$1(path) { return isArray(path) ? path : [path]; } _$1.toPath = toPath$1; // Internal wrapper for `_.toPath` to enable minification. // Similar to `cb` for `_.iteratee`. function toPath(path) { return _$1.toPath(path); } // Internal function to obtain a nested property in `obj` along `path`. function deepGet(obj, path) { var length = path.length; for (var i = 0; i < length; i++) { if (obj == null) return void 0; obj = obj[path[i]]; } return length ? obj : void 0; } // Get the value of the (deep) property on `path` from `object`. // If any property in `path` does not exist or if the value is // `undefined`, return `defaultValue` instead. // The `path` is normalized through `_.toPath`. function get(object, path, defaultValue) { var value = deepGet(object, toPath(path)); return isUndefined(value) ? defaultValue : value; } // Shortcut function for checking if an object has a given property directly on // itself (in other words, not on a prototype). Unlike the internal `has` // function, this public version can also traverse nested properties. function has(obj, path) { path = toPath(path); var length = path.length; for (var i = 0; i < length; i++) { var key = path[i]; if (!has$1(obj, key)) return false; obj = obj[key]; } return !!length; } // Keep the identity function around for default iteratees. function identity(value) { return value; } // Returns a predicate for checking whether an object has a given set of // `key:value` pairs. function matcher(attrs) { attrs = extendOwn({}, attrs); return function(obj) { return isMatch(obj, attrs); }; } // Creates a function that, when passed an object, will traverse that object’s // properties down the given `path`, specified as an array of keys or indices. function property(path) { path = toPath(path); return function(obj) { return deepGet(obj, path); }; } // Internal function that returns an efficient (for current engines) version // of the passed-in callback, to be repeatedly applied in other Underscore // functions. function optimizeCb(func, context, argCount) { if (context === void 0) return func; switch (argCount == null ? 3 : argCount) { case 1: return function(value) { return func.call(context, value); }; // The 2-argument case is omitted because we’re not using it. case 3: return function(value, index, collection) { return func.call(context, value, index, collection); }; case 4: return function(accumulator, value, index, collection) { return func.call(context, accumulator, value, index, collection); }; } return function() { return func.apply(context, arguments); }; } // An internal function to generate callbacks that can be applied to each // element in a collection, returning the desired result — either `_.identity`, // an arbitrary callback, a property matcher, or a property accessor. function baseIteratee(value, context, argCount) { if (value == null) return identity; if (isFunction$1(value)) return optimizeCb(value, context, argCount); if (isObject(value) && !isArray(value)) return matcher(value); return property(value); } // External wrapper for our callback generator. Users may customize // `_.iteratee` if they want additional predicate/iteratee shorthand styles. // This abstraction hides the internal-only `argCount` argument. function iteratee(value, context) { return baseIteratee(value, context, Infinity); } _$1.iteratee = iteratee; // The function we call internally to generate a callback. It invokes // `_.iteratee` if overridden, otherwise `baseIteratee`. function cb(value, context, argCount) { if (_$1.iteratee !== iteratee) return _$1.iteratee(value, context); return baseIteratee(value, context, argCount); } // Returns the results of applying the `iteratee` to each element of `obj`. // In contrast to `_.map` it returns an object. function mapObject(obj, iteratee, context) { iteratee = cb(iteratee, context); var _keys = keys(obj), length = _keys.length, results = {}; for (var index = 0; index < length; index++) { var currentKey = _keys[index]; results[currentKey] = iteratee(obj[currentKey], currentKey, obj); } return results; } // Predicate-generating function. Often useful outside of Underscore. function noop(){} // Generates a function for a given object that returns a given property. function propertyOf(obj) { if (obj == null) return noop; return function(path) { return get(obj, path); }; } // Run a function **n** times. function times(n, iteratee, context) { var accum = Array(Math.max(0, n)); iteratee = optimizeCb(iteratee, context, 1); for (var i = 0; i < n; i++) accum[i] = iteratee(i); return accum; } // Return a random integer between `min` and `max` (inclusive). function random(min, max) { if (max == null) { max = min; min = 0; } return min + Math.floor(Math.random() * (max - min + 1)); } // A (possibly faster) way to get the current timestamp as an integer. var now = Date.now || function() { return new Date().getTime(); }; // Internal helper to generate functions for escaping and unescaping strings // to/from HTML interpolation. function createEscaper(map) { var escaper = function(match) { return map[match]; }; // Regexes for identifying a key that needs to be escaped. var source = '(?:' + keys(map).join('|') + ')'; var testRegexp = RegExp(source); var replaceRegexp = RegExp(source, 'g'); return function(string) { string = string == null ? '' : '' + string; return testRegexp.test(string) ? string.replace(replaceRegexp, escaper) : string; }; } // Internal list of HTML entities for escaping. var escapeMap = { '&': '&', '<': '<', '>': '>', '"': '"', "'": ''', '`': '`' }; // Function for escaping strings to HTML interpolation. var _escape = createEscaper(escapeMap); // Internal list of HTML entities for unescaping. var unescapeMap = invert(escapeMap); // Function for unescaping strings from HTML interpolation. var _unescape = createEscaper(unescapeMap); // By default, Underscore uses ERB-style template delimiters. Change the // following template settings to use alternative delimiters. var templateSettings = _$1.templateSettings = { evaluate: /<%([\s\S]+?)%>/g, interpolate: /<%=([\s\S]+?)%>/g, escape: /<%-([\s\S]+?)%>/g }; // When customizing `_.templateSettings`, if you don't want to define an // interpolation, evaluation or escaping regex, we need one that is // guaranteed not to match. var noMatch = /(.)^/; // Certain characters need to be escaped so that they can be put into a // string literal. var escapes = { "'": "'", '\\': '\\', '\r': 'r', '\n': 'n', '\u2028': 'u2028', '\u2029': 'u2029' }; var escapeRegExp = /\\|'|\r|\n|\u2028|\u2029/g; function escapeChar(match) { return '\\' + escapes[match]; } // In order to prevent third-party code injection through // `_.templateSettings.variable`, we test it against the following regular // expression. It is intentionally a bit more liberal than just matching valid // identifiers, but still prevents possible loopholes through defaults or // destructuring assignment. var bareIdentifier = /^\s*(\w|\$)+\s*$/; // JavaScript micro-templating, similar to John Resig's implementation. // Underscore templating handles arbitrary delimiters, preserves whitespace, // and correctly escapes quotes within interpolated code. // NB: `oldSettings` only exists for backwards compatibility. function template(text, settings, oldSettings) { if (!settings && oldSettings) settings = oldSettings; settings = defaults({}, settings, _$1.templateSettings); // Combine delimiters into one regular expression via alternation. var matcher = RegExp([ (settings.escape || noMatch).source, (settings.interpolate || noMatch).source, (settings.evaluate || noMatch).source ].join('|') + '|$', 'g'); // Compile the template source, escaping string literals appropriately. var index = 0; var source = "__p+='"; text.replace(matcher, function(match, escape, interpolate, evaluate, offset) { source += text.slice(index, offset).replace(escapeRegExp, escapeChar); index = offset + match.length; if (escape) { source += "'+\n((__t=(" + escape + "))==null?'':_.escape(__t))+\n'"; } else if (interpolate) { source += "'+\n((__t=(" + interpolate + "))==null?'':__t)+\n'"; } else if (evaluate) { source += "';\n" + evaluate + "\n__p+='"; } // Adobe VMs need the match returned to produce the correct offset. return match; }); source += "';\n"; var argument = settings.variable; if (argument) { // Insure against third-party code injection. (CVE-2021-23358) if (!bareIdentifier.test(argument)) throw new Error( 'variable is not a bare identifier: ' + argument ); } else { // If a variable is not specified, place data values in local scope. source = 'with(obj||{}){\n' + source + '}\n'; argument = 'obj'; } source = "var __t,__p='',__j=Array.prototype.join," + "print=function(){__p+=__j.call(arguments,'');};\n" + source + 'return __p;\n'; var render; try { render = new Function(argument, '_', source); } catch (e) { e.source = source; throw e; } var template = function(data) { return render.call(this, data, _$1); }; // Provide the compiled source as a convenience for precompilation. template.source = 'function(' + argument + '){\n' + source + '}'; return template; } // Traverses the children of `obj` along `path`. If a child is a function, it // is invoked with its parent as context. Returns the value of the final // child, or `fallback` if any child is undefined. function result(obj, path, fallback) { path = toPath(path); var length = path.length; if (!length) { return isFunction$1(fallback) ? fallback.call(obj) : fallback; } for (var i = 0; i < length; i++) { var prop = obj == null ? void 0 : obj[path[i]]; if (prop === void 0) { prop = fallback; i = length; // Ensure we don't continue iterating. } obj = isFunction$1(prop) ? prop.call(obj) : prop; } return obj; } // Generate a unique integer id (unique within the entire client session). // Useful for temporary DOM ids. var idCounter = 0; function uniqueId(prefix) { var id = ++idCounter + ''; return prefix ? prefix + id : id; } // Start chaining a wrapped Underscore object. function chain(obj) { var instance = _$1(obj); instance._chain = true; return instance; } // Internal function to execute `sourceFunc` bound to `context` with optional // `args`. Determines whether to execute a function as a constructor or as a // normal function. function executeBound(sourceFunc, boundFunc, context, callingContext, args) { if (!(callingContext instanceof boundFunc)) return sourceFunc.apply(context, args); var self = baseCreate(sourceFunc.prototype); var result = sourceFunc.apply(self, args); if (isObject(result)) return result; return self; } // Partially apply a function by creating a version that has had some of its // arguments pre-filled, without changing its dynamic `this` context. `_` acts // as a placeholder by default, allowing any combination of arguments to be // pre-filled. Set `_.partial.placeholder` for a custom placeholder argument. var partial = restArguments(function(func, boundArgs) { var placeholder = partial.placeholder; var bound = function() { var position = 0, length = boundArgs.length; var args = Array(length); for (var i = 0; i < length; i++) { args[i] = boundArgs[i] === placeholder ? arguments[position++] : boundArgs[i]; } while (position < arguments.length) args.push(arguments[position++]); return executeBound(func, bound, this, this, args); }; return bound; }); partial.placeholder = _$1; // Create a function bound to a given object (assigning `this`, and arguments, // optionally). var bind = restArguments(function(func, context, args) { if (!isFunction$1(func)) throw new TypeError('Bind must be called on a function'); var bound = restArguments(function(callArgs) { return executeBound(func, bound, context, this, args.concat(callArgs)); }); return bound; }); // Internal helper for collection methods to determine whether a collection // should be iterated as an array or as an object. // Related: https://people.mozilla.org/~jorendorff/es6-draft.html#sec-tolength // Avoids a very nasty iOS 8 JIT bug on ARM-64. #2094 var isArrayLike = createSizePropertyCheck(getLength); // Internal implementation of a recursive `flatten` function. function flatten$1(input, depth, strict, output) { output = output || []; if (!depth && depth !== 0) { depth = Infinity; } else if (depth <= 0) { return output.concat(input); } var idx = output.length; for (var i = 0, length = getLength(input); i < length; i++) { var value = input[i]; if (isArrayLike(value) && (isArray(value) || isArguments$1(value))) { // Flatten current level of array or arguments object. if (depth > 1) { flatten$1(value, depth - 1, strict, output); idx = output.length; } else { var j = 0, len = value.length; while (j < len) output[idx++] = value[j++]; } } else if (!strict) { output[idx++] = value; } } return output; } // Bind a number of an object's methods to that object. Remaining arguments // are the method names to be bound. Useful for ensuring that all callbacks // defined on an object belong to it. var bindAll = restArguments(function(obj, keys) { keys = flatten$1(keys, false, false); var index = keys.length; if (index < 1) throw new Error('bindAll must be passed function names'); while (index--) { var key = keys[index]; obj[key] = bind(obj[key], obj); } return obj; }); // Memoize an expensive function by storing its results. function memoize(func, hasher) { var memoize = function(key) { var cache = memoize.cache; var address = '' + (hasher ? hasher.apply(this, arguments) : key); if (!has$1(cache, address)) cache[address] = func.apply(this, arguments); return cache[address]; }; memoize.cache = {}; return memoize; } // Delays a function for the given number of milliseconds, and then calls // it with the arguments supplied. var delay = restArguments(function(func, wait, args) { return setTimeout(function() { return func.apply(null, args); }, wait); }); // Defers a function, scheduling it to run after the current call stack has // cleared. var defer = partial(delay, _$1, 1); // Returns a function, that, when invoked, will only be triggered at most once // during a given window of time. Normally, the throttled function will run // as much as it can, without ever going more than once per `wait` duration; // but if you'd like to disable the execution on the leading edge, pass // `{leading: false}`. To disable execution on the trailing edge, ditto. function throttle(func, wait, options) { var timeout, context, args, result; var previous = 0; if (!options) options = {}; var later = function() { previous = options.leading === false ? 0 : now(); timeout = null; result = func.apply(context, args); if (!timeout) context = args = null; }; var throttled = function() { var _now = now(); if (!previous && options.leading === false) previous = _now; var remaining = wait - (_now - previous); context = this; args = arguments; if (remaining <= 0 || remaining > wait) { if (timeout) { clearTimeout(timeout); timeout = null; } previous = _now; result = func.apply(context, args); if (!timeout) context = args = null; } else if (!timeout && options.trailing !== false) { timeout = setTimeout(later, remaining); } return result; }; throttled.cancel = function() { clearTimeout(timeout); previous = 0; timeout = context = args = null; }; return throttled; } // When a sequence of calls of the returned function ends, the argument // function is triggered. The end of a sequence is defined by the `wait` // parameter. If `immediate` is passed, the argument function will be // triggered at the beginning of the sequence instead of at the end. function debounce(func, wait, immediate) { var timeout, previous, args, result, context; var later = function() { var passed = now() - previous; if (wait > passed) { timeout = setTimeout(later, wait - passed); } else { timeout = null; if (!immediate) result = func.apply(context, args); // This check is needed because `func` can recursively invoke `debounced`. if (!timeout) args = context = null; } }; var debounced = restArguments(function(_args) { context = this; args = _args; previous = now(); if (!timeout) { timeout = setTimeout(later, wait); if (immediate) result = func.apply(context, args); } return result; }); debounced.cancel = function() { clearTimeout(timeout); timeout = args = context = null; }; return debounced; } // Returns the first function passed as an argument to the second, // allowing you to adjust arguments, run code before and after, and // conditionally execute the original function. function wrap(func, wrapper) { return partial(wrapper, func); } // Returns a negated version of the passed-in predicate. function negate(predicate) { return function() { return !predicate.apply(this, arguments); }; } // Returns a function that is the composition of a list of functions, each // consuming the return value of the function that follows. function compose() { var args = arguments; var start = args.length - 1; return function() { var i = start; var result = args[start].apply(this, arguments); while (i--) result = args[i].call(this, result); return result; }; } // Returns a function that will only be executed on and after the Nth call. function after(times, func) { return function() { if (--times < 1) { return func.apply(this, arguments); } }; } // Returns a function that will only be executed up to (but not including) the // Nth call. function before(times, func) { var memo; return function() { if (--times > 0) { memo = func.apply(this, arguments); } if (times <= 1) func = null; return memo; }; } // Returns a function that will be executed at most one time, no matter how // often you call it. Useful for lazy initialization. var once = partial(before, 2); // Returns the first key on an object that passes a truth test. function findKey(obj, predicate, context) { predicate = cb(predicate, context); var _keys = keys(obj), key; for (var i = 0, length = _keys.length; i < length; i++) { key = _keys[i]; if (predicate(obj[key], key, obj)) return key; } } // Internal function to generate `_.findIndex` and `_.findLastIndex`. function createPredicateIndexFinder(dir) { return function(array, predicate, context) { predicate = cb(predicate, context); var length = getLength(array); var index = dir > 0 ? 0 : length - 1; for (; index >= 0 && index < length; index += dir) { if (predicate(array[index], index, array)) return index; } return -1; }; } // Returns the first index on an array-like that passes a truth test. var findIndex = createPredicateIndexFinder(1); // Returns the last index on an array-like that passes a truth test. var findLastIndex = createPredicateIndexFinder(-1); // Use a comparator function to figure out the smallest index at which // an object should be inserted so as to maintain order. Uses binary search. function sortedIndex(array, obj, iteratee, context) { iteratee = cb(iteratee, context, 1); var value = iteratee(obj); var low = 0, high = getLength(array); while (low < high) { var mid = Math.floor((low + high) / 2); if (iteratee(array[mid]) < value) low = mid + 1; else high = mid; } return low; } // Internal function to generate the `_.indexOf` and `_.lastIndexOf` functions. function createIndexFinder(dir, predicateFind, sortedIndex) { return function(array, item, idx) { var i = 0, length = getLength(array); if (typeof idx == 'number') { if (dir > 0) { i = idx >= 0 ? idx : Math.max(idx + length, i); } else { length = idx >= 0 ? Math.min(idx + 1, length) : idx + length + 1; } } else if (sortedIndex && idx && length) { idx = sortedIndex(array, item); return array[idx] === item ? idx : -1; } if (item !== item) { idx = predicateFind(slice.call(array, i, length), isNaN$1); return idx >= 0 ? idx + i : -1; } for (idx = dir > 0 ? i : length - 1; idx >= 0 && idx < length; idx += dir) { if (array[idx] === item) return idx; } return -1; }; } // Return the position of the first occurrence of an item in an array, // or -1 if the item is not included in the array. // If the array is large and already in sort order, pass `true` // for **isSorted** to use binary search. var indexOf = createIndexFinder(1, findIndex, sortedIndex); // Return the position of the last occurrence of an item in an array, // or -1 if the item is not included in the array. var lastIndexOf = createIndexFinder(-1, findLastIndex); // Return the first value which passes a truth test. function find(obj, predicate, context) { var keyFinder = isArrayLike(obj) ? findIndex : findKey; var key = keyFinder(obj, predicate, context); if (key !== void 0 && key !== -1) return obj[key]; } // Convenience version of a common use case of `_.find`: getting the first // object containing specific `key:value` pairs. function findWhere(obj, attrs) { return find(obj, matcher(attrs)); } // The cornerstone for collection functions, an `each` // implementation, aka `forEach`. // Handles raw objects in addition to array-likes. Treats all // sparse array-likes as if they were dense. function each(obj, iteratee, context) { iteratee = optimizeCb(iteratee, context); var i, length; if (isArrayLike(obj)) { for (i = 0, length = obj.length; i < length; i++) { iteratee(obj[i], i, obj); } } else { var _keys = keys(obj); for (i = 0, length = _keys.length; i < length; i++) { iteratee(obj[_keys[i]], _keys[i], obj); } } return obj; } // Return the results of applying the iteratee to each element. function map(obj, iteratee, context) { iteratee = cb(iteratee, context); var _keys = !isArrayLike(obj) && keys(obj), length = (_keys || obj).length, results = Array(length); for (var index = 0; index < length; index++) { var currentKey = _keys ? _keys[index] : index; results[index] = iteratee(obj[currentKey], currentKey, obj); } return results; } // Internal helper to create a reducing function, iterating left or right. function createReduce(dir) { // Wrap code that reassigns argument variables in a separate function than // the one that accesses `arguments.length` to avoid a perf hit. (#1991) var reducer = function(obj, iteratee, memo, initial) { var _keys = !isArrayLike(obj) && keys(obj), length = (_keys || obj).length, index = dir > 0 ? 0 : length - 1; if (!initial) { memo = obj[_keys ? _keys[index] : index]; index += dir; } for (; index >= 0 && index < length; index += dir) { var currentKey = _keys ? _keys[index] : index; memo = iteratee(memo, obj[currentKey], currentKey, obj); } return memo; }; return function(obj, iteratee, memo, context) { var initial = arguments.length >= 3; return reducer(obj, optimizeCb(iteratee, context, 4), memo, initial); }; } // **Reduce** builds up a single result from a list of values, aka `inject`, // or `foldl`. var reduce = createReduce(1); // The right-associative version of reduce, also known as `foldr`. var reduceRight = createReduce(-1); // Return all the elements that pass a truth test. function filter(obj, predicate, context) { var results = []; predicate = cb(predicate, context); each(obj, function(value, index, list) { if (predicate(value, index, list)) results.push(value); }); return results; } // Return all the elements for which a truth test fails. function reject(obj, predicate, context) { return filter(obj, negate(cb(predicate)), context); } // Determine whether all of the elements pass a truth test. function every(obj, predicate, context) { predicate = cb(predicate, context); var _keys = !isArrayLike(obj) && keys(obj), length = (_keys || obj).length; for (var index = 0; index < length; index++) { var currentKey = _keys ? _keys[index] : index; if (!predicate(obj[currentKey], currentKey, obj)) return false; } return true; } // Determine if at least one element in the object passes a truth test. function some(obj, predicate, context) { predicate = cb(predicate, context); var _keys = !isArrayLike(obj) && keys(obj), length = (_keys || obj).length; for (var index = 0; index < length; index++) { var currentKey = _keys ? _keys[index] : index; if (predicate(obj[currentKey], currentKey, obj)) return true; } return false; } // Determine if the array or object contains a given item (using `===`). function contains(obj, item, fromIndex, guard) { if (!isArrayLike(obj)) obj = values(obj); if (typeof fromIndex != 'number' || guard) fromIndex = 0; return indexOf(obj, item, fromIndex) >= 0; } // Invoke a method (with arguments) on every item in a collection. var invoke = restArguments(function(obj, path, args) { var contextPath, func; if (isFunction$1(path)) { func = path; } else { path = toPath(path); contextPath = path.slice(0, -1); path = path[path.length - 1]; } return map(obj, function(context) { var method = func; if (!method) { if (contextPath && contextPath.length) { context = deepGet(context, contextPath); } if (context == null) return void 0; method = context[path]; } return method == null ? method : method.apply(context, args); }); }); // Convenience version of a common use case of `_.map`: fetching a property. function pluck(obj, key) { return map(obj, property(key)); } // Convenience version of a common use case of `_.filter`: selecting only // objects containing specific `key:value` pairs. function where(obj, attrs) { return filter(obj, matcher(attrs)); } // Return the maximum element (or element-based computation). function max(obj, iteratee, context) { var result = -Infinity, lastComputed = -Infinity, value, computed; if (iteratee == null || typeof iteratee == 'number' && typeof obj[0] != 'object' && obj != null) { obj = isArrayLike(obj) ? obj : values(obj); for (var i = 0, length = obj.length; i < length; i++) { value = obj[i]; if (value != null && value > result) { result = value; } } } else { iteratee = cb(iteratee, context); each(obj, function(v, index, list) { computed = iteratee(v, index, list); if (computed > lastComputed || computed === -Infinity && result === -Infinity) { result = v; lastComputed = computed; } }); } return result; } // Return the minimum element (or element-based computation). function min(obj, iteratee, context) { var result = Infinity, lastComputed = Infinity, value, computed; if (iteratee == null || typeof iteratee == 'number' && typeof obj[0] != 'object' && obj != null) { obj = isArrayLike(obj) ? obj : values(obj); for (var i = 0, length = obj.length; i < length; i++) { value = obj[i]; if (value != null && value < result) { result = value; } } } else { iteratee = cb(iteratee, context); each(obj, function(v, index, list) { computed = iteratee(v, index, list); if (computed < lastComputed || computed === Infinity && result === Infinity) { result = v; lastComputed = computed; } }); } return result; } // Sample **n** random values from a collection using the modern version of the // [Fisher-Yates shuffle](https://en.wikipedia.org/wiki/Fisher–Yates_shuffle). // If **n** is not specified, returns a single random element. // The internal `guard` argument allows it to work with `_.map`. function sample(obj, n, guard) { if (n == null || guard) { if (!isArrayLike(obj)) obj = values(obj); return obj[random(obj.length - 1)]; } var sample = isArrayLike(obj) ? clone(obj) : values(obj); var length = getLength(sample); n = Math.max(Math.min(n, length), 0); var last = length - 1; for (var index = 0; index < n; index++) { var rand = random(index, last); var temp = sample[index]; sample[index] = sample[rand]; sample[rand] = temp; } return sample.slice(0, n); } // Shuffle a collection. function shuffle(obj) { return sample(obj, Infinity); } // Sort the object's values by a criterion produced by an iteratee. function sortBy(obj, iteratee, context) { var index = 0; iteratee = cb(iteratee, context); return pluck(map(obj, function(value, key, list) { return { value: value, index: index++, criteria: iteratee(value, key, list) }; }).sort(function(left, right) { var a = left.criteria; var b = right.criteria; if (a !== b) { if (a > b || a === void 0) return 1; if (a < b || b === void 0) return -1; } return left.index - right.index; }), 'value'); } // An internal function used for aggregate "group by" operations. function group(behavior, partition) { return function(obj, iteratee, context) { var result = partition ? [[], []] : {}; iteratee = cb(iteratee, context); each(obj, function(value, index) { var key = iteratee(value, index, obj); behavior(result, value, key); }); return result; }; } // Groups the object's values by a criterion. Pass either a string attribute // to group by, or a function that returns the criterion. var groupBy = group(function(result, value, key) { if (has$1(result, key)) result[key].push(value); else result[key] = [value]; }); // Indexes the object's values by a criterion, similar to `_.groupBy`, but for // when you know that your index values will be unique. var indexBy = group(function(result, value, key) { result[key] = value; }); // Counts instances of an object that group by a certain criterion. Pass // either a string attribute to count by, or a function that returns the // criterion. var countBy = group(function(result, value, key) { if (has$1(result, key)) result[key]++; else result[key] = 1; }); // Split a collection into two arrays: one whose elements all pass the given // truth test, and one whose elements all do not pass the truth test. var partition = group(function(result, value, pass) { result[pass ? 0 : 1].push(value); }, true); // Safely create a real, live array from anything iterable. var reStrSymbol = /[^\ud800-\udfff]|[\ud800-\udbff][\udc00-\udfff]|[\ud800-\udfff]/g; function toArray(obj) { if (!obj) return []; if (isArray(obj)) return slice.call(obj); if (isString(obj)) { // Keep surrogate pair characters together. return obj.match(reStrSymbol); } if (isArrayLike(obj)) return map(obj, identity); return values(obj); } // Return the number of elements in a collection. function size(obj) { if (obj == null) return 0; return isArrayLike(obj) ? obj.length : keys(obj).length; } // Internal `_.pick` helper function to determine whether `key` is an enumerable // property name of `obj`. function keyInObj(value, key, obj) { return key in obj; } // Return a copy of the object only containing the allowed properties. var pick = restArguments(function(obj, keys) { var result = {}, iteratee = keys[0]; if (obj == null) return result; if (isFunction$1(iteratee)) { if (keys.length > 1) iteratee = optimizeCb(iteratee, keys[1]); keys = allKeys(obj); } else { iteratee = keyInObj; keys = flatten$1(keys, false, false); obj = Object(obj); } for (var i = 0, length = keys.length; i < length; i++) { var key = keys[i]; var value = obj[key]; if (iteratee(value, key, obj)) result[key] = value; } return result; }); // Return a copy of the object without the disallowed properties. var omit = restArguments(function(obj, keys) { var iteratee = keys[0], context; if (isFunction$1(iteratee)) { iteratee = negate(iteratee); if (keys.length > 1) context = keys[1]; } else { keys = map(flatten$1(keys, false, false), String); iteratee = function(value, key) { return !contains(keys, key); }; } return pick(obj, iteratee, context); }); // Returns everything but the last entry of the array. Especially useful on // the arguments object. Passing **n** will return all the values in // the array, excluding the last N. function initial(array, n, guard) { return slice.call(array, 0, Math.max(0, array.length - (n == null || guard ? 1 : n))); } // Get the first element of an array. Passing **n** will return the first N // values in the array. The **guard** check allows it to work with `_.map`. function first(array, n, guard) { if (array == null || array.length < 1) return n == null || guard ? void 0 : []; if (n == null || guard) return array[0]; return initial(array, array.length - n); } // Returns everything but the first entry of the `array`. Especially useful on // the `arguments` object. Passing an **n** will return the rest N values in the // `array`. function rest(array, n, guard) { return slice.call(array, n == null || guard ? 1 : n); } // Get the last element of an array. Passing **n** will return the last N // values in the array. function last(array, n, guard) { if (array == null || array.length < 1) return n == null || guard ? void 0 : []; if (n == null || guard) return array[array.length - 1]; return rest(array, Math.max(0, array.length - n)); } // Trim out all falsy values from an array. function compact(array) { return filter(array, Boolean); } // Flatten out an array, either recursively (by default), or up to `depth`. // Passing `true` or `false` as `depth` means `1` or `Infinity`, respectively. function flatten(array, depth) { return flatten$1(array, depth, false); } // Take the difference between one array and a number of other arrays. // Only the elements present in just the first array will remain. var difference = restArguments(function(array, rest) { rest = flatten$1(rest, true, true); return filter(array, function(value){ return !contains(rest, value); }); }); // Return a version of the array that does not contain the specified value(s). var without = restArguments(function(array, otherArrays) { return difference(array, otherArrays); }); // Produce a duplicate-free version of the array. If the array has already // been sorted, you have the option of using a faster algorithm. // The faster algorithm will not work with an iteratee if the iteratee // is not a one-to-one function, so providing an iteratee will disable // the faster algorithm. function uniq(array, isSorted, iteratee, context) { if (!isBoolean(isSorted)) { context = iteratee; iteratee = isSorted; isSorted = false; } if (iteratee != null) iteratee = cb(iteratee, context); var result = []; var seen = []; for (var i = 0, length = getLength(array); i < length; i++) { var value = array[i], computed = iteratee ? iteratee(value, i, array) : value; if (isSorted && !iteratee) { if (!i || seen !== computed) result.push(value); seen = computed; } else if (iteratee) { if (!contains(seen, computed)) { seen.push(computed); result.push(value); } } else if (!contains(result, value)) { result.push(value); } } return result; } // Produce an array that contains the union: each distinct element from all of // the passed-in arrays. var union = restArguments(function(arrays) { return uniq(flatten$1(arrays, true, true)); }); // Produce an array that contains every item shared between all the // passed-in arrays. function intersection(array) { var result = []; var argsLength = arguments.length; for (var i = 0, length = getLength(array); i < length; i++) { var item = array[i]; if (contains(result, item)) continue; var j; for (j = 1; j < argsLength; j++) { if (!contains(arguments[j], item)) break; } if (j === argsLength) result.push(item); } return result; } // Complement of zip. Unzip accepts an array of arrays and groups // each array's elements on shared indices. function unzip(array) { var length = array && max(array, getLength).length || 0; var result = Array(length); for (var index = 0; index < length; index++) { result[index] = pluck(array, index); } return result; } // Zip together multiple lists into a single array -- elements that share // an index go together. var zip = restArguments(unzip); // Converts lists into objects. Pass either a single array of `[key, value]` // pairs, or two parallel arrays of the same length -- one of keys, and one of // the corresponding values. Passing by pairs is the reverse of `_.pairs`. function object(list, values) { var result = {}; for (var i = 0, length = getLength(list); i < length; i++) { if (values) { result[list[i]] = values[i]; } else { result[list[i][0]] = list[i][1]; } } return result; } // Generate an integer Array containing an arithmetic progression. A port of // the native Python `range()` function. See // [the Python documentation](https://docs.python.org/library/functions.html#range). function range(start, stop, step) { if (stop == null) { stop = start || 0; start = 0; } if (!step) { step = stop < start ? -1 : 1; } var length = Math.max(Math.ceil((stop - start) / step), 0); var range = Array(length); for (var idx = 0; idx < length; idx++, start += step) { range[idx] = start; } return range; } // Chunk a single array into multiple arrays, each containing `count` or fewer // items. function chunk(array, count) { if (count == null || count < 1) return []; var result = []; var i = 0, length = array.length; while (i < length) { result.push(slice.call(array, i, i += count)); } return result; } // Helper function to continue chaining intermediate results. function chainResult(instance, obj) { return instance._chain ? _$1(obj).chain() : obj; } // Add your own custom functions to the Underscore object. function mixin(obj) { each(functions(obj), function(name) { var func = _$1[name] = obj[name]; _$1.prototype[name] = function() { var args = [this._wrapped]; push.apply(args, arguments); return chainResult(this, func.apply(_$1, args)); }; }); return _$1; } // Add all mutator `Array` functions to the wrapper. each(['pop', 'push', 'reverse', 'shift', 'sort', 'splice', 'unshift'], function(name) { var method = ArrayProto[name]; _$1.prototype[name] = function() { var obj = this._wrapped; if (obj != null) { method.apply(obj, arguments); if ((name === 'shift' || name === 'splice') && obj.length === 0) { delete obj[0]; } } return chainResult(this, obj); }; }); // Add all accessor `Array` functions to the wrapper. each(['concat', 'join', 'slice'], function(name) { var method = ArrayProto[name]; _$1.prototype[name] = function() { var obj = this._wrapped; if (obj != null) obj = method.apply(obj, arguments); return chainResult(this, obj); }; }); // Named Exports var allExports = { __proto__: null, VERSION: VERSION, restArguments: restArguments, isObject: isObject, isNull: isNull, isUndefined: isUndefined, isBoolean: isBoolean, isElement: isElement, isString: isString, isNumber: isNumber, isDate: isDate, isRegExp: isRegExp, isError: isError, isSymbol: isSymbol, isArrayBuffer: isArrayBuffer, isDataView: isDataView$1, isArray: isArray, isFunction: isFunction$1, isArguments: isArguments$1, isFinite: isFinite$1, isNaN: isNaN$1, isTypedArray: isTypedArray$1, isEmpty: isEmpty, isMatch: isMatch, isEqual: isEqual, isMap: isMap, isWeakMap: isWeakMap, isSet: isSet, isWeakSet: isWeakSet, keys: keys, allKeys: allKeys, values: values, pairs: pairs, invert: invert, functions: functions, methods: functions, extend: extend, extendOwn: extendOwn, assign: extendOwn, defaults: defaults, create: create, clone: clone, tap: tap, get: get, has: has, mapObject: mapObject, identity: identity, constant: constant, noop: noop, toPath: toPath$1, property: property, propertyOf: propertyOf, matcher: matcher, matches: matcher, times: times, random: random, now: now, escape: _escape, unescape: _unescape, templateSettings: templateSettings, template: template, result: result, uniqueId: uniqueId, chain: chain, iteratee: iteratee, partial: partial, bind: bind, bindAll: bindAll, memoize: memoize, delay: delay, defer: defer, throttle: throttle, debounce: debounce, wrap: wrap, negate: negate, compose: compose, after: after, before: before, once: once, findKey: findKey, findIndex: findIndex, findLastIndex: findLastIndex, sortedIndex: sortedIndex, indexOf: indexOf, lastIndexOf: lastIndexOf, find: find, detect: find, findWhere: findWhere, each: each, forEach: each, map: map, collect: map, reduce: reduce, foldl: reduce, inject: reduce, reduceRight: reduceRight, foldr: reduceRight, filter: filter, select: filter, reject: reject, every: every, all: every, some: some, any: some, contains: contains, includes: contains, include: contains, invoke: invoke, pluck: pluck, where: where, max: max, min: min, shuffle: shuffle, sample: sample, sortBy: sortBy, groupBy: groupBy, indexBy: indexBy, countBy: countBy, partition: partition, toArray: toArray, size: size, pick: pick, omit: omit, first: first, head: first, take: first, initial: initial, last: last, rest: rest, tail: rest, drop: rest, compact: compact, flatten: flatten, without: without, uniq: uniq, unique: uniq, union: union, intersection: intersection, difference: difference, unzip: unzip, transpose: unzip, zip: zip, object: object, range: range, chunk: chunk, mixin: mixin, 'default': _$1 }; // Default Export // Add all of the Underscore functions to the wrapper object. var _ = mixin(allExports); // Legacy Node.js API. _._ = _; return _; }))); //# sourceMappingURL=underscore-umd.js.map ================================================ FILE: core/dbt/docs/build/html/_static/underscore.js ================================================ !function(n,r){"object"==typeof exports&&"undefined"!=typeof module?module.exports=r():"function"==typeof define&&define.amd?define("underscore",r):(n="undefined"!=typeof globalThis?globalThis:n||self,function(){var t=n._,e=n._=r();e.noConflict=function(){return n._=t,e}}())}(this,(function(){ // Underscore.js 1.13.1 // https://underscorejs.org // (c) 2009-2021 Jeremy Ashkenas, Julian Gonggrijp, and DocumentCloud and Investigative Reporters & Editors // Underscore may be freely distributed under the MIT license. var n="1.13.1",r="object"==typeof self&&self.self===self&&self||"object"==typeof global&&global.global===global&&global||Function("return this")()||{},t=Array.prototype,e=Object.prototype,u="undefined"!=typeof Symbol?Symbol.prototype:null,o=t.push,i=t.slice,a=e.toString,f=e.hasOwnProperty,c="undefined"!=typeof ArrayBuffer,l="undefined"!=typeof DataView,s=Array.isArray,p=Object.keys,v=Object.create,h=c&&ArrayBuffer.isView,y=isNaN,d=isFinite,g=!{toString:null}.propertyIsEnumerable("toString"),b=["valueOf","isPrototypeOf","toString","propertyIsEnumerable","hasOwnProperty","toLocaleString"],m=Math.pow(2,53)-1;function j(n,r){return r=null==r?n.length-1:+r,function(){for(var t=Math.max(arguments.length-r,0),e=Array(t),u=0;u<t;u++)e[u]=arguments[u+r];switch(r){case 0:return n.call(this,e);case 1:return n.call(this,arguments[0],e);case 2:return n.call(this,arguments[0],arguments[1],e)}var o=Array(r+1);for(u=0;u<r;u++)o[u]=arguments[u];return o[r]=e,n.apply(this,o)}}function _(n){var r=typeof n;return"function"===r||"object"===r&&!!n}function w(n){return void 0===n}function A(n){return!0===n||!1===n||"[object Boolean]"===a.call(n)}function x(n){var r="[object "+n+"]";return function(n){return a.call(n)===r}}var S=x("String"),O=x("Number"),M=x("Date"),E=x("RegExp"),B=x("Error"),N=x("Symbol"),I=x("ArrayBuffer"),T=x("Function"),k=r.document&&r.document.childNodes;"function"!=typeof/./&&"object"!=typeof Int8Array&&"function"!=typeof k&&(T=function(n){return"function"==typeof n||!1});var D=T,R=x("Object"),F=l&&R(new DataView(new ArrayBuffer(8))),V="undefined"!=typeof Map&&R(new Map),P=x("DataView");var q=F?function(n){return null!=n&&D(n.getInt8)&&I(n.buffer)}:P,U=s||x("Array");function W(n,r){return null!=n&&f.call(n,r)}var z=x("Arguments");!function(){z(arguments)||(z=function(n){return W(n,"callee")})}();var L=z;function $(n){return O(n)&&y(n)}function C(n){return function(){return n}}function K(n){return function(r){var t=n(r);return"number"==typeof t&&t>=0&&t<=m}}function J(n){return function(r){return null==r?void 0:r[n]}}var G=J("byteLength"),H=K(G),Q=/\[object ((I|Ui)nt(8|16|32)|Float(32|64)|Uint8Clamped|Big(I|Ui)nt64)Array\]/;var X=c?function(n){return h?h(n)&&!q(n):H(n)&&Q.test(a.call(n))}:C(!1),Y=J("length");function Z(n,r){r=function(n){for(var r={},t=n.length,e=0;e<t;++e)r[n[e]]=!0;return{contains:function(n){return r[n]},push:function(t){return r[t]=!0,n.push(t)}}}(r);var t=b.length,u=n.constructor,o=D(u)&&u.prototype||e,i="constructor";for(W(n,i)&&!r.contains(i)&&r.push(i);t--;)(i=b[t])in n&&n[i]!==o[i]&&!r.contains(i)&&r.push(i)}function nn(n){if(!_(n))return[];if(p)return p(n);var r=[];for(var t in n)W(n,t)&&r.push(t);return g&&Z(n,r),r}function rn(n,r){var t=nn(r),e=t.length;if(null==n)return!e;for(var u=Object(n),o=0;o<e;o++){var i=t[o];if(r[i]!==u[i]||!(i in u))return!1}return!0}function tn(n){return n instanceof tn?n:this instanceof tn?void(this._wrapped=n):new tn(n)}function en(n){return new Uint8Array(n.buffer||n,n.byteOffset||0,G(n))}tn.VERSION=n,tn.prototype.value=function(){return this._wrapped},tn.prototype.valueOf=tn.prototype.toJSON=tn.prototype.value,tn.prototype.toString=function(){return String(this._wrapped)};var un="[object DataView]";function on(n,r,t,e){if(n===r)return 0!==n||1/n==1/r;if(null==n||null==r)return!1;if(n!=n)return r!=r;var o=typeof n;return("function"===o||"object"===o||"object"==typeof r)&&function n(r,t,e,o){r instanceof tn&&(r=r._wrapped);t instanceof tn&&(t=t._wrapped);var i=a.call(r);if(i!==a.call(t))return!1;if(F&&"[object Object]"==i&&q(r)){if(!q(t))return!1;i=un}switch(i){case"[object RegExp]":case"[object String]":return""+r==""+t;case"[object Number]":return+r!=+r?+t!=+t:0==+r?1/+r==1/t:+r==+t;case"[object Date]":case"[object Boolean]":return+r==+t;case"[object Symbol]":return u.valueOf.call(r)===u.valueOf.call(t);case"[object ArrayBuffer]":case un:return n(en(r),en(t),e,o)}var f="[object Array]"===i;if(!f&&X(r)){if(G(r)!==G(t))return!1;if(r.buffer===t.buffer&&r.byteOffset===t.byteOffset)return!0;f=!0}if(!f){if("object"!=typeof r||"object"!=typeof t)return!1;var c=r.constructor,l=t.constructor;if(c!==l&&!(D(c)&&c instanceof c&&D(l)&&l instanceof l)&&"constructor"in r&&"constructor"in t)return!1}o=o||[];var s=(e=e||[]).length;for(;s--;)if(e[s]===r)return o[s]===t;if(e.push(r),o.push(t),f){if((s=r.length)!==t.length)return!1;for(;s--;)if(!on(r[s],t[s],e,o))return!1}else{var p,v=nn(r);if(s=v.length,nn(t).length!==s)return!1;for(;s--;)if(p=v[s],!W(t,p)||!on(r[p],t[p],e,o))return!1}return e.pop(),o.pop(),!0}(n,r,t,e)}function an(n){if(!_(n))return[];var r=[];for(var t in n)r.push(t);return g&&Z(n,r),r}function fn(n){var r=Y(n);return function(t){if(null==t)return!1;var e=an(t);if(Y(e))return!1;for(var u=0;u<r;u++)if(!D(t[n[u]]))return!1;return n!==hn||!D(t[cn])}}var cn="forEach",ln="has",sn=["clear","delete"],pn=["get",ln,"set"],vn=sn.concat(cn,pn),hn=sn.concat(pn),yn=["add"].concat(sn,cn,ln),dn=V?fn(vn):x("Map"),gn=V?fn(hn):x("WeakMap"),bn=V?fn(yn):x("Set"),mn=x("WeakSet");function jn(n){for(var r=nn(n),t=r.length,e=Array(t),u=0;u<t;u++)e[u]=n[r[u]];return e}function _n(n){for(var r={},t=nn(n),e=0,u=t.length;e<u;e++)r[n[t[e]]]=t[e];return r}function wn(n){var r=[];for(var t in n)D(n[t])&&r.push(t);return r.sort()}function An(n,r){return function(t){var e=arguments.length;if(r&&(t=Object(t)),e<2||null==t)return t;for(var u=1;u<e;u++)for(var o=arguments[u],i=n(o),a=i.length,f=0;f<a;f++){var c=i[f];r&&void 0!==t[c]||(t[c]=o[c])}return t}}var xn=An(an),Sn=An(nn),On=An(an,!0);function Mn(n){if(!_(n))return{};if(v)return v(n);var r=function(){};r.prototype=n;var t=new r;return r.prototype=null,t}function En(n){return _(n)?U(n)?n.slice():xn({},n):n}function Bn(n){return U(n)?n:[n]}function Nn(n){return tn.toPath(n)}function In(n,r){for(var t=r.length,e=0;e<t;e++){if(null==n)return;n=n[r[e]]}return t?n:void 0}function Tn(n,r,t){var e=In(n,Nn(r));return w(e)?t:e}function kn(n){return n}function Dn(n){return n=Sn({},n),function(r){return rn(r,n)}}function Rn(n){return n=Nn(n),function(r){return In(r,n)}}function Fn(n,r,t){if(void 0===r)return n;switch(null==t?3:t){case 1:return function(t){return n.call(r,t)};case 3:return function(t,e,u){return n.call(r,t,e,u)};case 4:return function(t,e,u,o){return n.call(r,t,e,u,o)}}return function(){return n.apply(r,arguments)}}function Vn(n,r,t){return null==n?kn:D(n)?Fn(n,r,t):_(n)&&!U(n)?Dn(n):Rn(n)}function Pn(n,r){return Vn(n,r,1/0)}function qn(n,r,t){return tn.iteratee!==Pn?tn.iteratee(n,r):Vn(n,r,t)}function Un(){}function Wn(n,r){return null==r&&(r=n,n=0),n+Math.floor(Math.random()*(r-n+1))}tn.toPath=Bn,tn.iteratee=Pn;var zn=Date.now||function(){return(new Date).getTime()};function Ln(n){var r=function(r){return n[r]},t="(?:"+nn(n).join("|")+")",e=RegExp(t),u=RegExp(t,"g");return function(n){return n=null==n?"":""+n,e.test(n)?n.replace(u,r):n}}var $n={"&":"&","<":"<",">":">",'"':""","'":"'","`":"`"},Cn=Ln($n),Kn=Ln(_n($n)),Jn=tn.templateSettings={evaluate:/<%([\s\S]+?)%>/g,interpolate:/<%=([\s\S]+?)%>/g,escape:/<%-([\s\S]+?)%>/g},Gn=/(.)^/,Hn={"'":"'","\\":"\\","\r":"r","\n":"n","\u2028":"u2028","\u2029":"u2029"},Qn=/\\|'|\r|\n|\u2028|\u2029/g;function Xn(n){return"\\"+Hn[n]}var Yn=/^\s*(\w|\$)+\s*$/;var Zn=0;function nr(n,r,t,e,u){if(!(e instanceof r))return n.apply(t,u);var o=Mn(n.prototype),i=n.apply(o,u);return _(i)?i:o}var rr=j((function(n,r){var t=rr.placeholder,e=function(){for(var u=0,o=r.length,i=Array(o),a=0;a<o;a++)i[a]=r[a]===t?arguments[u++]:r[a];for(;u<arguments.length;)i.push(arguments[u++]);return nr(n,e,this,this,i)};return e}));rr.placeholder=tn;var tr=j((function(n,r,t){if(!D(n))throw new TypeError("Bind must be called on a function");var e=j((function(u){return nr(n,e,r,this,t.concat(u))}));return e})),er=K(Y);function ur(n,r,t,e){if(e=e||[],r||0===r){if(r<=0)return e.concat(n)}else r=1/0;for(var u=e.length,o=0,i=Y(n);o<i;o++){var a=n[o];if(er(a)&&(U(a)||L(a)))if(r>1)ur(a,r-1,t,e),u=e.length;else for(var f=0,c=a.length;f<c;)e[u++]=a[f++];else t||(e[u++]=a)}return e}var or=j((function(n,r){var t=(r=ur(r,!1,!1)).length;if(t<1)throw new Error("bindAll must be passed function names");for(;t--;){var e=r[t];n[e]=tr(n[e],n)}return n}));var ir=j((function(n,r,t){return setTimeout((function(){return n.apply(null,t)}),r)})),ar=rr(ir,tn,1);function fr(n){return function(){return!n.apply(this,arguments)}}function cr(n,r){var t;return function(){return--n>0&&(t=r.apply(this,arguments)),n<=1&&(r=null),t}}var lr=rr(cr,2);function sr(n,r,t){r=qn(r,t);for(var e,u=nn(n),o=0,i=u.length;o<i;o++)if(r(n[e=u[o]],e,n))return e}function pr(n){return function(r,t,e){t=qn(t,e);for(var u=Y(r),o=n>0?0:u-1;o>=0&&o<u;o+=n)if(t(r[o],o,r))return o;return-1}}var vr=pr(1),hr=pr(-1);function yr(n,r,t,e){for(var u=(t=qn(t,e,1))(r),o=0,i=Y(n);o<i;){var a=Math.floor((o+i)/2);t(n[a])<u?o=a+1:i=a}return o}function dr(n,r,t){return function(e,u,o){var a=0,f=Y(e);if("number"==typeof o)n>0?a=o>=0?o:Math.max(o+f,a):f=o>=0?Math.min(o+1,f):o+f+1;else if(t&&o&&f)return e[o=t(e,u)]===u?o:-1;if(u!=u)return(o=r(i.call(e,a,f),$))>=0?o+a:-1;for(o=n>0?a:f-1;o>=0&&o<f;o+=n)if(e[o]===u)return o;return-1}}var gr=dr(1,vr,yr),br=dr(-1,hr);function mr(n,r,t){var e=(er(n)?vr:sr)(n,r,t);if(void 0!==e&&-1!==e)return n[e]}function jr(n,r,t){var e,u;if(r=Fn(r,t),er(n))for(e=0,u=n.length;e<u;e++)r(n[e],e,n);else{var o=nn(n);for(e=0,u=o.length;e<u;e++)r(n[o[e]],o[e],n)}return n}function _r(n,r,t){r=qn(r,t);for(var e=!er(n)&&nn(n),u=(e||n).length,o=Array(u),i=0;i<u;i++){var a=e?e[i]:i;o[i]=r(n[a],a,n)}return o}function wr(n){var r=function(r,t,e,u){var o=!er(r)&&nn(r),i=(o||r).length,a=n>0?0:i-1;for(u||(e=r[o?o[a]:a],a+=n);a>=0&&a<i;a+=n){var f=o?o[a]:a;e=t(e,r[f],f,r)}return e};return function(n,t,e,u){var o=arguments.length>=3;return r(n,Fn(t,u,4),e,o)}}var Ar=wr(1),xr=wr(-1);function Sr(n,r,t){var e=[];return r=qn(r,t),jr(n,(function(n,t,u){r(n,t,u)&&e.push(n)})),e}function Or(n,r,t){r=qn(r,t);for(var e=!er(n)&&nn(n),u=(e||n).length,o=0;o<u;o++){var i=e?e[o]:o;if(!r(n[i],i,n))return!1}return!0}function Mr(n,r,t){r=qn(r,t);for(var e=!er(n)&&nn(n),u=(e||n).length,o=0;o<u;o++){var i=e?e[o]:o;if(r(n[i],i,n))return!0}return!1}function Er(n,r,t,e){return er(n)||(n=jn(n)),("number"!=typeof t||e)&&(t=0),gr(n,r,t)>=0}var Br=j((function(n,r,t){var e,u;return D(r)?u=r:(r=Nn(r),e=r.slice(0,-1),r=r[r.length-1]),_r(n,(function(n){var o=u;if(!o){if(e&&e.length&&(n=In(n,e)),null==n)return;o=n[r]}return null==o?o:o.apply(n,t)}))}));function Nr(n,r){return _r(n,Rn(r))}function Ir(n,r,t){var e,u,o=-1/0,i=-1/0;if(null==r||"number"==typeof r&&"object"!=typeof n[0]&&null!=n)for(var a=0,f=(n=er(n)?n:jn(n)).length;a<f;a++)null!=(e=n[a])&&e>o&&(o=e);else r=qn(r,t),jr(n,(function(n,t,e){((u=r(n,t,e))>i||u===-1/0&&o===-1/0)&&(o=n,i=u)}));return o}function Tr(n,r,t){if(null==r||t)return er(n)||(n=jn(n)),n[Wn(n.length-1)];var e=er(n)?En(n):jn(n),u=Y(e);r=Math.max(Math.min(r,u),0);for(var o=u-1,i=0;i<r;i++){var a=Wn(i,o),f=e[i];e[i]=e[a],e[a]=f}return e.slice(0,r)}function kr(n,r){return function(t,e,u){var o=r?[[],[]]:{};return e=qn(e,u),jr(t,(function(r,u){var i=e(r,u,t);n(o,r,i)})),o}}var Dr=kr((function(n,r,t){W(n,t)?n[t].push(r):n[t]=[r]})),Rr=kr((function(n,r,t){n[t]=r})),Fr=kr((function(n,r,t){W(n,t)?n[t]++:n[t]=1})),Vr=kr((function(n,r,t){n[t?0:1].push(r)}),!0),Pr=/[^\ud800-\udfff]|[\ud800-\udbff][\udc00-\udfff]|[\ud800-\udfff]/g;function qr(n,r,t){return r in t}var Ur=j((function(n,r){var t={},e=r[0];if(null==n)return t;D(e)?(r.length>1&&(e=Fn(e,r[1])),r=an(n)):(e=qr,r=ur(r,!1,!1),n=Object(n));for(var u=0,o=r.length;u<o;u++){var i=r[u],a=n[i];e(a,i,n)&&(t[i]=a)}return t})),Wr=j((function(n,r){var t,e=r[0];return D(e)?(e=fr(e),r.length>1&&(t=r[1])):(r=_r(ur(r,!1,!1),String),e=function(n,t){return!Er(r,t)}),Ur(n,e,t)}));function zr(n,r,t){return i.call(n,0,Math.max(0,n.length-(null==r||t?1:r)))}function Lr(n,r,t){return null==n||n.length<1?null==r||t?void 0:[]:null==r||t?n[0]:zr(n,n.length-r)}function $r(n,r,t){return i.call(n,null==r||t?1:r)}var Cr=j((function(n,r){return r=ur(r,!0,!0),Sr(n,(function(n){return!Er(r,n)}))})),Kr=j((function(n,r){return Cr(n,r)}));function Jr(n,r,t,e){A(r)||(e=t,t=r,r=!1),null!=t&&(t=qn(t,e));for(var u=[],o=[],i=0,a=Y(n);i<a;i++){var f=n[i],c=t?t(f,i,n):f;r&&!t?(i&&o===c||u.push(f),o=c):t?Er(o,c)||(o.push(c),u.push(f)):Er(u,f)||u.push(f)}return u}var Gr=j((function(n){return Jr(ur(n,!0,!0))}));function Hr(n){for(var r=n&&Ir(n,Y).length||0,t=Array(r),e=0;e<r;e++)t[e]=Nr(n,e);return t}var Qr=j(Hr);function Xr(n,r){return n._chain?tn(r).chain():r}function Yr(n){return jr(wn(n),(function(r){var t=tn[r]=n[r];tn.prototype[r]=function(){var n=[this._wrapped];return o.apply(n,arguments),Xr(this,t.apply(tn,n))}})),tn}jr(["pop","push","reverse","shift","sort","splice","unshift"],(function(n){var r=t[n];tn.prototype[n]=function(){var t=this._wrapped;return null!=t&&(r.apply(t,arguments),"shift"!==n&&"splice"!==n||0!==t.length||delete t[0]),Xr(this,t)}})),jr(["concat","join","slice"],(function(n){var r=t[n];tn.prototype[n]=function(){var n=this._wrapped;return null!=n&&(n=r.apply(n,arguments)),Xr(this,n)}}));var Zr=Yr({__proto__:null,VERSION:n,restArguments:j,isObject:_,isNull:function(n){return null===n},isUndefined:w,isBoolean:A,isElement:function(n){return!(!n||1!==n.nodeType)},isString:S,isNumber:O,isDate:M,isRegExp:E,isError:B,isSymbol:N,isArrayBuffer:I,isDataView:q,isArray:U,isFunction:D,isArguments:L,isFinite:function(n){return!N(n)&&d(n)&&!isNaN(parseFloat(n))},isNaN:$,isTypedArray:X,isEmpty:function(n){if(null==n)return!0;var r=Y(n);return"number"==typeof r&&(U(n)||S(n)||L(n))?0===r:0===Y(nn(n))},isMatch:rn,isEqual:function(n,r){return on(n,r)},isMap:dn,isWeakMap:gn,isSet:bn,isWeakSet:mn,keys:nn,allKeys:an,values:jn,pairs:function(n){for(var r=nn(n),t=r.length,e=Array(t),u=0;u<t;u++)e[u]=[r[u],n[r[u]]];return e},invert:_n,functions:wn,methods:wn,extend:xn,extendOwn:Sn,assign:Sn,defaults:On,create:function(n,r){var t=Mn(n);return r&&Sn(t,r),t},clone:En,tap:function(n,r){return r(n),n},get:Tn,has:function(n,r){for(var t=(r=Nn(r)).length,e=0;e<t;e++){var u=r[e];if(!W(n,u))return!1;n=n[u]}return!!t},mapObject:function(n,r,t){r=qn(r,t);for(var e=nn(n),u=e.length,o={},i=0;i<u;i++){var a=e[i];o[a]=r(n[a],a,n)}return o},identity:kn,constant:C,noop:Un,toPath:Bn,property:Rn,propertyOf:function(n){return null==n?Un:function(r){return Tn(n,r)}},matcher:Dn,matches:Dn,times:function(n,r,t){var e=Array(Math.max(0,n));r=Fn(r,t,1);for(var u=0;u<n;u++)e[u]=r(u);return e},random:Wn,now:zn,escape:Cn,unescape:Kn,templateSettings:Jn,template:function(n,r,t){!r&&t&&(r=t),r=On({},r,tn.templateSettings);var e=RegExp([(r.escape||Gn).source,(r.interpolate||Gn).source,(r.evaluate||Gn).source].join("|")+"|$","g"),u=0,o="__p+='";n.replace(e,(function(r,t,e,i,a){return o+=n.slice(u,a).replace(Qn,Xn),u=a+r.length,t?o+="'+\n((__t=("+t+"))==null?'':_.escape(__t))+\n'":e?o+="'+\n((__t=("+e+"))==null?'':__t)+\n'":i&&(o+="';\n"+i+"\n__p+='"),r})),o+="';\n";var i,a=r.variable;if(a){if(!Yn.test(a))throw new Error("variable is not a bare identifier: "+a)}else o="with(obj||{}){\n"+o+"}\n",a="obj";o="var __t,__p='',__j=Array.prototype.join,"+"print=function(){__p+=__j.call(arguments,'');};\n"+o+"return __p;\n";try{i=new Function(a,"_",o)}catch(n){throw n.source=o,n}var f=function(n){return i.call(this,n,tn)};return f.source="function("+a+"){\n"+o+"}",f},result:function(n,r,t){var e=(r=Nn(r)).length;if(!e)return D(t)?t.call(n):t;for(var u=0;u<e;u++){var o=null==n?void 0:n[r[u]];void 0===o&&(o=t,u=e),n=D(o)?o.call(n):o}return n},uniqueId:function(n){var r=++Zn+"";return n?n+r:r},chain:function(n){var r=tn(n);return r._chain=!0,r},iteratee:Pn,partial:rr,bind:tr,bindAll:or,memoize:function(n,r){var t=function(e){var u=t.cache,o=""+(r?r.apply(this,arguments):e);return W(u,o)||(u[o]=n.apply(this,arguments)),u[o]};return t.cache={},t},delay:ir,defer:ar,throttle:function(n,r,t){var e,u,o,i,a=0;t||(t={});var f=function(){a=!1===t.leading?0:zn(),e=null,i=n.apply(u,o),e||(u=o=null)},c=function(){var c=zn();a||!1!==t.leading||(a=c);var l=r-(c-a);return u=this,o=arguments,l<=0||l>r?(e&&(clearTimeout(e),e=null),a=c,i=n.apply(u,o),e||(u=o=null)):e||!1===t.trailing||(e=setTimeout(f,l)),i};return c.cancel=function(){clearTimeout(e),a=0,e=u=o=null},c},debounce:function(n,r,t){var e,u,o,i,a,f=function(){var c=zn()-u;r>c?e=setTimeout(f,r-c):(e=null,t||(i=n.apply(a,o)),e||(o=a=null))},c=j((function(c){return a=this,o=c,u=zn(),e||(e=setTimeout(f,r),t&&(i=n.apply(a,o))),i}));return c.cancel=function(){clearTimeout(e),e=o=a=null},c},wrap:function(n,r){return rr(r,n)},negate:fr,compose:function(){var n=arguments,r=n.length-1;return function(){for(var t=r,e=n[r].apply(this,arguments);t--;)e=n[t].call(this,e);return e}},after:function(n,r){return function(){if(--n<1)return r.apply(this,arguments)}},before:cr,once:lr,findKey:sr,findIndex:vr,findLastIndex:hr,sortedIndex:yr,indexOf:gr,lastIndexOf:br,find:mr,detect:mr,findWhere:function(n,r){return mr(n,Dn(r))},each:jr,forEach:jr,map:_r,collect:_r,reduce:Ar,foldl:Ar,inject:Ar,reduceRight:xr,foldr:xr,filter:Sr,select:Sr,reject:function(n,r,t){return Sr(n,fr(qn(r)),t)},every:Or,all:Or,some:Mr,any:Mr,contains:Er,includes:Er,include:Er,invoke:Br,pluck:Nr,where:function(n,r){return Sr(n,Dn(r))},max:Ir,min:function(n,r,t){var e,u,o=1/0,i=1/0;if(null==r||"number"==typeof r&&"object"!=typeof n[0]&&null!=n)for(var a=0,f=(n=er(n)?n:jn(n)).length;a<f;a++)null!=(e=n[a])&&e<o&&(o=e);else r=qn(r,t),jr(n,(function(n,t,e){((u=r(n,t,e))<i||u===1/0&&o===1/0)&&(o=n,i=u)}));return o},shuffle:function(n){return Tr(n,1/0)},sample:Tr,sortBy:function(n,r,t){var e=0;return r=qn(r,t),Nr(_r(n,(function(n,t,u){return{value:n,index:e++,criteria:r(n,t,u)}})).sort((function(n,r){var t=n.criteria,e=r.criteria;if(t!==e){if(t>e||void 0===t)return 1;if(t<e||void 0===e)return-1}return n.index-r.index})),"value")},groupBy:Dr,indexBy:Rr,countBy:Fr,partition:Vr,toArray:function(n){return n?U(n)?i.call(n):S(n)?n.match(Pr):er(n)?_r(n,kn):jn(n):[]},size:function(n){return null==n?0:er(n)?n.length:nn(n).length},pick:Ur,omit:Wr,first:Lr,head:Lr,take:Lr,initial:zr,last:function(n,r,t){return null==n||n.length<1?null==r||t?void 0:[]:null==r||t?n[n.length-1]:$r(n,Math.max(0,n.length-r))},rest:$r,tail:$r,drop:$r,compact:function(n){return Sr(n,Boolean)},flatten:function(n,r){return ur(n,r,!1)},without:Kr,uniq:Jr,unique:Jr,union:Gr,intersection:function(n){for(var r=[],t=arguments.length,e=0,u=Y(n);e<u;e++){var o=n[e];if(!Er(r,o)){var i;for(i=1;i<t&&Er(arguments[i],o);i++);i===t&&r.push(o)}}return r},difference:Cr,unzip:Hr,transpose:Hr,zip:Qr,object:function(n,r){for(var t={},e=0,u=Y(n);e<u;e++)r?t[n[e]]=r[e]:t[n[e][0]]=n[e][1];return t},range:function(n,r,t){null==r&&(r=n||0,n=0),t||(t=r<n?-1:1);for(var e=Math.max(Math.ceil((r-n)/t),0),u=Array(e),o=0;o<e;o++,n+=t)u[o]=n;return u},chunk:function(n,r){if(null==r||r<1)return[];for(var t=[],e=0,u=n.length;e<u;)t.push(i.call(n,e,e+=r));return t},mixin:Yr,default:tn});return Zr._=Zr,Zr})); ================================================ FILE: core/dbt/docs/build/html/genindex.html ================================================ <!DOCTYPE html> <html lang="en"> <head> <meta charset="utf-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> <title>Index — dbt-core documentation

Index

©2022, dbt Labs. | Powered by Sphinx 5.2.3 & Alabaster 0.7.12
================================================ FILE: core/dbt/docs/build/html/search.html ================================================ Search — dbt-core documentation

Search

Searching for multiple words only shows matches that contain all words.

©2022, dbt Labs. | Powered by Sphinx 5.2.3 & Alabaster 0.7.12
================================================ FILE: core/dbt/docs/build/html/searchindex.js ================================================ Search.setIndex({"docnames": ["index"], "filenames": ["index.rst"], "titles": ["dbt-core\u2019s API documentation"], "terms": {"right": 0, "now": 0, "best": 0, "wai": 0, "from": 0, "i": 0, "us": 0, "dbtrunner": 0, "we": 0, "expos": 0, "cli": 0, "main": 0, "import": 0, "cli_arg": 0, "project": 0, "dir": 0, "jaffle_shop": 0, "initi": 0, "runner": 0, "re": 0, "success": 0, "you": 0, "can": 0, "also": 0, "pass": 0, "pre": 0, "construct": 0, "object": 0, "those": 0, "instead": 0, "load": 0, "up": 0, "disk": 0, "preload": 0, "load_profil": 0, "postgr": 0, "load_project": 0, "fals": 0, "thi": 0, "For": 0, "full": 0, "exampl": 0, "code": 0, "refer": 0, "py": 0, "type": 0, "boolean": 0, "If": 0, "set": 0, "variabl": 0, "resolv": 0, "unselect": 0, "node": 0, "unknown": 0, "specifi": 0, "stop": 0, "execut": 0, "first": 0, "failur": 0, "argument": 0, "provid": 0, "flag": 0, "even": 0, "exist": 0, "databas": 0, "current": 0, "environ": 0, "drop": 0, "increment": 0, "fulli": 0, "recalcul": 0, "tabl": 0, "definit": 0, "choic": 0, "eager": 0, "cautiou": 0, "buildabl": 0, "all": 0, "ar": 0, "adjac": 0, "resourc": 0, "thei": 0, "have": 0, "been": 0, "explicitli": 0, "string": 0, "which": 0, "overrid": 0, "dbt_project": 0, "yml": 0, "path": 0, "directori": 0, "look": 0, "file": 0, "work": 0, "home": 0, "default": 0, "its": 0, "parent": 0, "todo": 0, "No": 0, "help": 0, "text": 0, "includ": 0, "The": 0, "name": 0, "defin": 0, "sampl": 0, "data": 0, "termin": 0, "given": 0, "json": 0, "compar": 0, "store": 0, "result": 0, "fail": 0, "row": 0, "configur": 0, "onli": 0, "appli": 0, "dbt_target_path": 0, "int": 0, "number": 0, "while": 0, "yaml": 0, "suppli": 0, "your": 0, "should": 0, "eg": 0, "my_vari": 0, "my_valu": 0, "ensur": 0, "version": 0, "match": 0, "one": 0, "requir": 0, "whether": 0, "scaffold": 0, "queri": 0, "part": 0, "avail": 0, "inform": 0, "skip": 0, "interact": 0, "setup": 0, "dictionari": 0, "map": 0, "keyword": 0}, "objects": {}, "objtypes": {}, "objnames": {}, "titleterms": {"dbt": 0, "core": 0, "": 0, "api": 0, "document": 0, "how": 0, "invok": 0, "command": 0, "python": 0, "runtim": 0, "build": 0, "defer": 0, "exclud": 0, "fail_fast": 0, "favor_st": 0, "full_refresh": 0, "indirect_select": 0, "profil": 0, "profiles_dir": 0, "project_dir": 0, "resource_typ": 0, "select": 0, "selector": 0, "show": 0, "state": 0, "store_failur": 0, "target": 0, "target_path": 0, "thread": 0, "var": 0, "version_check": 0, "clean": 0, "compil": 0, "introspect": 0, "parse_onli": 0, "debug": 0, "config_dir": 0, "dep": 0, "doc": 0, "init": 0, "project_nam": 0, "skip_profile_setup": 0, "list": 0, "model": 0, "output": 0, "output_kei": 0, "pars": 0, "write_manifest": 0, "run": 0, "run_oper": 0, "macro": 0, "arg": 0, "seed": 0, "snapshot": 0, "sourc": 0, "test": 0}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 8, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx": 57}, "alltitles": {"dbt-core\u2019s API documentation": [[0, "dbt-core-s-api-documentation"]], "How to invoke dbt commands in python runtime": [[0, "how-to-invoke-dbt-commands-in-python-runtime"]], "API documentation": [[0, "api-documentation"]], "Command: build": [[0, "dbt-section"]], "defer": [[0, "build|defer"], [0, "compile|defer"], [0, "run|defer"], [0, "snapshot|defer"], [0, "test|defer"]], "exclude": [[0, "build|exclude"], [0, "compile|exclude"], [0, "list|exclude"], [0, "list|exclude"], [0, "run|exclude"], [0, "seed|exclude"], [0, "snapshot|exclude"], [0, "test|exclude"]], "fail_fast": [[0, "build|fail_fast"], [0, "run|fail_fast"], [0, "test|fail_fast"]], "favor_state": [[0, "build|favor_state"], [0, "compile|favor_state"], [0, "run|favor_state"], [0, "snapshot|favor_state"], [0, "test|favor_state"]], "full_refresh": [[0, "build|full_refresh"], [0, "compile|full_refresh"], [0, "run|full_refresh"], [0, "seed|full_refresh"]], "indirect_selection": [[0, "build|indirect_selection"], [0, "list|indirect_selection"], [0, "list|indirect_selection"], [0, "test|indirect_selection"]], "profile": [[0, "build|profile"], [0, "clean|profile"], [0, "compile|profile"], [0, "debug|profile"], [0, "deps|profile"], [0, "init|profile"], [0, "list|profile"], [0, "list|profile"], [0, "parse|profile"], [0, "run|profile"], [0, "run-operation|profile"], [0, "seed|profile"], [0, "snapshot|profile"], [0, "test|profile"]], "profiles_dir": [[0, "build|profiles_dir"], [0, "clean|profiles_dir"], [0, "compile|profiles_dir"], [0, "debug|profiles_dir"], [0, "deps|profiles_dir"], [0, "init|profiles_dir"], [0, "list|profiles_dir"], [0, "list|profiles_dir"], [0, "parse|profiles_dir"], [0, "run|profiles_dir"], [0, "run-operation|profiles_dir"], [0, "seed|profiles_dir"], [0, "snapshot|profiles_dir"], [0, "test|profiles_dir"]], "project_dir": [[0, "build|project_dir"], [0, "clean|project_dir"], [0, "compile|project_dir"], [0, "debug|project_dir"], [0, "deps|project_dir"], [0, "init|project_dir"], [0, "list|project_dir"], [0, "list|project_dir"], [0, "parse|project_dir"], [0, "run|project_dir"], [0, "run-operation|project_dir"], [0, "seed|project_dir"], [0, "snapshot|project_dir"], [0, "test|project_dir"]], "resource_types": [[0, "build|resource_types"], [0, "list|resource_types"], [0, "list|resource_types"]], "select": [[0, "build|select"], [0, "compile|select"], [0, "list|select"], [0, "list|select"], [0, "run|select"], [0, "seed|select"], [0, "snapshot|select"], [0, "test|select"]], "selector": [[0, "build|selector"], [0, "compile|selector"], [0, "list|selector"], [0, "list|selector"], [0, "run|selector"], [0, "seed|selector"], [0, "snapshot|selector"], [0, "test|selector"]], "show": [[0, "build|show"], [0, "seed|show"]], "state": [[0, "build|state"], [0, "compile|state"], [0, "list|state"], [0, "list|state"], [0, "run|state"], [0, "seed|state"], [0, "snapshot|state"], [0, "test|state"]], "store_failures": [[0, "build|store_failures"], [0, "test|store_failures"]], "target": [[0, "build|target"], [0, "clean|target"], [0, "compile|target"], [0, "debug|target"], [0, "deps|target"], [0, "init|target"], [0, "list|target"], [0, "list|target"], [0, "parse|target"], [0, "run|target"], [0, "run-operation|target"], [0, "seed|target"], [0, "snapshot|target"], [0, "test|target"]], "target_path": [[0, "build|target_path"], [0, "compile|target_path"], [0, "parse|target_path"], [0, "run|target_path"], [0, "seed|target_path"], [0, "test|target_path"]], "threads": [[0, "build|threads"], [0, "compile|threads"], [0, "parse|threads"], [0, "run|threads"], [0, "seed|threads"], [0, "snapshot|threads"], [0, "test|threads"]], "vars": [[0, "build|vars"], [0, "clean|vars"], [0, "compile|vars"], [0, "debug|vars"], [0, "deps|vars"], [0, "init|vars"], [0, "list|vars"], [0, "list|vars"], [0, "parse|vars"], [0, "run|vars"], [0, "run-operation|vars"], [0, "seed|vars"], [0, "snapshot|vars"], [0, "test|vars"]], "version_check": [[0, "build|version_check"], [0, "compile|version_check"], [0, "debug|version_check"], [0, "parse|version_check"], [0, "run|version_check"], [0, "seed|version_check"], [0, "test|version_check"]], "Command: clean": [[0, "dbt-section"]], "Command: compile": [[0, "dbt-section"]], "introspect": [[0, "compile|introspect"]], "parse_only": [[0, "compile|parse_only"]], "Command: debug": [[0, "dbt-section"]], "config_dir": [[0, "debug|config_dir"]], "Command: deps": [[0, "dbt-section"]], "Command: docs": [[0, "dbt-section"]], "Command: init": [[0, "dbt-section"]], "project_name": [[0, "init|project_name"]], "skip_profile_setup": [[0, "init|skip_profile_setup"]], "Command: list": [[0, "dbt-section"], [0, "dbt-section"]], "models": [[0, "list|models"], [0, "list|models"]], "output": [[0, "list|output"], [0, "list|output"]], "output_keys": [[0, "list|output_keys"], [0, "list|output_keys"]], "Command: parse": [[0, "dbt-section"]], "compile": [[0, "parse|compile"]], "write_manifest": [[0, "parse|write_manifest"]], "Command: run": [[0, "dbt-section"]], "Command: run_operation": [[0, "dbt-section"]], "macro": [[0, "run-operation|macro"]], "args": [[0, "run-operation|args"]], "Command: seed": [[0, "dbt-section"]], "Command: snapshot": [[0, "dbt-section"]], "Command: source": [[0, "dbt-section"]], "Command: test": [[0, "dbt-section"]]}, "indexentries": {}}) ================================================ FILE: core/dbt/docs/make.bat ================================================ @ECHO OFF pushd %~dp0 REM Command file for Sphinx documentation if "%SPHINXBUILD%" == "" ( set SPHINXBUILD=sphinx-build ) set SOURCEDIR=source set BUILDDIR=build %SPHINXBUILD% >NUL 2>NUL if errorlevel 9009 ( echo. echo.The 'sphinx-build' command was not found. Make sure you have Sphinx echo.installed, then set the SPHINXBUILD environment variable to point echo.to the full path of the 'sphinx-build' executable. Alternatively you echo.may add the Sphinx directory to PATH. echo. echo.If you don't have Sphinx installed, grab it from echo.https://www.sphinx-doc.org/ exit /b 1 ) if "%1" == "" goto help %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% goto end :help %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% :end popd ================================================ FILE: core/dbt/docs/source/_ext/dbt_click.py ================================================ import traceback import typing as t import click import click.types as click_t from docutils import nodes from docutils.parsers.rst import Directive import dbt.cli.option_types as dbt_t PARAM_TYPE_MAP = { click_t.BoolParamType: lambda _: "boolean", click_t.Choice: lambda c: f"choice: {c.choices}", click_t.IntParamType: lambda _: "int", click_t.Path: lambda _: "path", click_t.StringParamType: lambda _: "string", dbt_t.YAML: lambda _: "YAML", } def format_command(cmd) -> nodes.section: cmd_name = cmd.name.replace("-", "_") section = nodes.section( "", nodes.title(text=f"Command: {cmd_name}"), ids=[cmd_name], names=[cmd_name], ) section.extend(format_params(cmd)) return section def format_params(cmd) -> t.List[nodes.section]: lines = [] for param in cmd.params: uid = f"{cmd.name}|{param.name}" param_section = nodes.section( "", nodes.title(text=param.name), ids=[uid], names=[uid], ) get_type_str = PARAM_TYPE_MAP.get(type(param.type), lambda _: "unknown") type_str = get_type_str(param.type) param_section.append(nodes.paragraph(text=f"Type: {type_str}")) help_txt = getattr(param, "help", None) if help_txt is not None: param_section.append(nodes.paragraph(text=help_txt)) lines.append(param_section) return lines def load_module(module_path: str, error) -> t.Union[click.Command, click.Group]: try: module_name, attr_name = module_path.split(":", 1) except ValueError: # noqa raise error(f'"{module_path}" is not of format "module:parser"') try: mod = __import__(module_name, globals(), locals(), [attr_name]) except Exception: # noqa raise error( f'Failed to import "{attr_name}" from "{module_name}". ' f"The following exception was raised:\n{traceback.format_exc()}" ) if not hasattr(mod, attr_name): raise error(f'Module "{module_name}" has no attribute "{attr_name}"') parser = getattr(mod, attr_name) if not isinstance(parser, (click.Command, click.Group)): raise error( f'"{type(parser)}" of type "{module_path}" is not click.Command' ' or click.Group."click.BaseCommand"' ) return parser class DBTClick(Directive): has_content = False required_arguments = 1 def run(self): section = nodes.section( "", ids=["dbt-section"], names=["dbt-section"], ) cmds = self._get_commands(self.arguments[0]) for cmd in cmds: command_section = format_command(cmd) section.extend(command_section) return [section] def _get_commands(self, module: str) -> t.List[click.Command]: click_group = load_module(module, self.error) if type(click_group) is not click.Group: raise self.error('Type "click.Group" not supported in dbt_click extension') cmd_strs = [cmd for cmd in click_group.commands] cmd_strs.sort() cmds = [] for cmd_str in cmd_strs: cmd = click_group.commands.get(cmd_str) if cmd is not None: cmds.append(cmd) return cmds def setup(app) -> t.Dict[str, t.Any]: app.add_directive("dbt_click", DBTClick) return { "version": "0.1", "parallel_read_safe": True, "parallel_write_safe": True, } ================================================ FILE: core/dbt/docs/source/conf.py ================================================ import os import sys import typing as t # Configuration file for the Sphinx documentation builder. # # For the full list of built-in configuration values, see the documentation: # https://www.sphinx-doc.org/en/master/usage/configuration.html sys.path.insert(0, os.path.abspath("../../..")) sys.path.insert(0, os.path.abspath("./_ext")) # -- Project information ----------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information project = "dbt-core" copyright = "2022, dbt Labs" author = "dbt Labs" # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration extensions = ["dbt_click"] templates_path = ["_templates"] exclude_patterns: t.List[str] = [] # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output html_theme = "alabaster" html_static_path = ["_static"] ================================================ FILE: core/dbt/docs/source/index.rst ================================================ dbt-core's API documentation ============================ Programmatic invocations -------------------------------------------- In v1.5, dbt-core added support for programmatic invocations. The intent of this entry point is provide **exact parity** with CLI functionality, callable from within a Python script or application. The main entry point is a ``dbtRunner`` class that wraps around ``dbt-core``'s CLI, and allows you to "invoke" CLI commands as Python methods. Each command returns a `dbtRunnerResult` object, which has three attributes: * ``success`` (bool): Whether the command succeeded. * ``result``: If the command completed (successfully or with handled errors), its result(s). Return type varies by command. * ``exception``: If the dbt invocation encountered an unhandled error and did not complete, the exception it encountered. .. code-block:: python from dbt.cli.main import dbtRunner, dbtRunnerResult # initialize dbt = dbtRunner() # create CLI args as a list of strings cli_args = ["run", "--select", "tag:my_tag"] # run the command res: dbtRunnerResult = dbt.invoke(cli_args) # inspect the results for r in res.result: print(f"{r.node.name}: {r.status}") For more information and examples, consult the documentation: https://docs.getdbt.com/reference/programmatic-invocations API documentation ----------------- .. dbt_click:: dbt.cli.main:cli ================================================ FILE: core/dbt/env_vars.py ================================================ from typing import List from dbt.cli import params from dbt.deprecations import warn from dbt_common.constants import ENGINE_ENV_PREFIX from dbt_common.context import get_invocation_context # These are env vars that are not in the params module, but are still allowed to be set. # New additions to this list should use the new naming scheme, unless they are being added because # they already existed, but we didn't know about them previously. # TODO: Should at least some of these become (undocumented) cli param options? _ADDITIONAL_ENGINE_ENV_VARS: List[str] = [ "DBT_INVOCATION_ENV", "DBT_RECORDED_FILE_PATH", "DBT_TEST_STATE_MODIFIED", # TODO: This is testing related, should we do this differently? "DBT_PACKAGE_HUB_URL", "DBT_DOWNLOAD_DIR", "DBT_PP_FILE_DIFF_TEST", # TODO: This is testing related, should we do this differently? "DBT_PP_TEST", # TODO: This is testing related, should we do this differently? ] # Here we are creating a set of all known engine env vars. This is used in this moduleto create an allow list of dbt # engine env vars. We also use it in the cli flags module to cross propagate engine env vars with their old non-engine prefixed names. KNOWN_ENGINE_ENV_VARS: set[params.EngineEnvVar] = set(params.KNOWN_ENV_VARS).union( {params.EngineEnvVar(envvar=envvar) for envvar in _ADDITIONAL_ENGINE_ENV_VARS} ) _ALLOWED_ENV_VARS: set[str] = {engine_env_var.name for engine_env_var in KNOWN_ENGINE_ENV_VARS} def validate_engine_env_vars() -> None: """ Validate that any set environment variables that begin with the engine prefix are allowed. """ env_vars = get_invocation_context()._env for env_var in env_vars.keys(): if env_var.startswith(ENGINE_ENV_PREFIX) and env_var not in _ALLOWED_ENV_VARS: warn( "environment-variable-namespace-deprecation", env_var=env_var, reserved_prefix=ENGINE_ENV_PREFIX, ) ================================================ FILE: core/dbt/event_time/event_time.py ================================================ from datetime import datetime from dateutil.relativedelta import relativedelta from dbt.artifacts.resources.types import BatchSize from dbt_common.exceptions import DbtRuntimeError def offset_timestamp(timestamp=datetime, batch_size=BatchSize, offset=int) -> datetime: """Offsets the passed in timestamp based on the batch_size and offset. Note: THIS IS DIFFERENT FROM MicrobatchBuilder.offset_timestamp. That function first `truncates` the timestamp, and then does delta addition subtraction from there. This function _doesn't_ truncate the timestamp and uses `relativedelta` for specific edge case handling (months, years), which may produce different results than the delta math done in `MicrobatchBuilder.offset_timestamp` Examples 2024-09-17 16:06:00 + Batchsize.hour -1 -> 2024-09-17 15:06:00 2024-09-17 16:06:00 + Batchsize.hour +1 -> 2024-09-17 17:06:00 2024-09-17 16:06:00 + Batchsize.day -1 -> 2024-09-16 16:06:00 2024-09-17 16:06:00 + Batchsize.day +1 -> 2024-09-18 16:06:00 2024-09-17 16:06:00 + Batchsize.month -1 -> 2024-08-17 16:06:00 2024-09-17 16:06:00 + Batchsize.month +1 -> 2024-10-17 16:06:00 2024-09-17 16:06:00 + Batchsize.year -1 -> 2023-09-17 16:06:00 2024-09-17 16:06:00 + Batchsize.year +1 -> 2025-09-17 16:06:00 2024-01-31 16:06:00 + Batchsize.month +1 -> 2024-02-29 16:06:00 2024-02-29 16:06:00 + Batchsize.year +1 -> 2025-02-28 16:06:00 """ if batch_size == BatchSize.hour: return timestamp + relativedelta(hours=offset) elif batch_size == BatchSize.day: return timestamp + relativedelta(days=offset) elif batch_size == BatchSize.month: return timestamp + relativedelta(months=offset) elif batch_size == BatchSize.year: return timestamp + relativedelta(years=offset) else: raise DbtRuntimeError(f"Unhandled batch_size '{batch_size}'") ================================================ FILE: core/dbt/event_time/sample_window.py ================================================ from __future__ import annotations from datetime import datetime import pytz from attr import dataclass from dbt.artifacts.resources.types import BatchSize from dbt.event_time.event_time import offset_timestamp from dbt_common.dataclass_schema import dbtClassMixin from dbt_common.exceptions import DbtRuntimeError @dataclass class SampleWindow(dbtClassMixin): start: datetime end: datetime def __post_serialize__(self, data, context): # This is insane, but necessary, I apologize. Mashumaro handles the # dictification of this class via a compile time generated `to_dict` # method based off of the _typing_ of th class. By default `datetime` # types are converted to strings. We don't want that, we want them to # stay datetimes. # Note: This is safe because the `SampleWindow` isn't part of the artifact # and thus doesn't get written out. new_data = super().__post_serialize__(data, context) new_data["start"] = self.start new_data["end"] = self.end return new_data @classmethod def from_relative_string(cls, relative_string: str) -> SampleWindow: end = datetime.now(tz=pytz.UTC) relative_window = relative_string.split(" ") if len(relative_window) != 2: raise DbtRuntimeError( f"Cannot load SAMPLE_WINDOW from '{relative_string}'. Must be of form 'DAYS_INT GRAIN_SIZE'." ) try: lookback = int(relative_window[0]) except Exception: raise DbtRuntimeError(f"Unable to convert '{relative_window[0]}' to an integer.") try: batch_size_string = relative_window[1].lower().rstrip("s") batch_size = BatchSize[batch_size_string] except Exception: grains = [size.value for size in BatchSize] grain_plurals = [BatchSize.plural(size) for size in BatchSize] valid_grains = grains + grain_plurals raise DbtRuntimeError( f"Invalid grain size '{relative_window[1]}'. Must be one of {valid_grains}." ) start = offset_timestamp(timestamp=end, batch_size=batch_size, offset=-1 * lookback) return cls(start=start, end=end) ================================================ FILE: core/dbt/events/README.md ================================================ # Events Module The Events module is responsible for communicating internal dbt structures into a consumable interface. Because the "event" classes are based entirely on protobuf definitions, the interface is really clearly defined, whether or not protobufs are used to consume it. We use protoc for compiling the protobuf message definitions into Python classes. # Using the Events Module The event module provides types that represent what is happening in dbt in `events.types`. These types are intended to represent an exhaustive list of all things happening within dbt that will need to be logged, streamed, or printed. To fire an event, `common.events.functions::fire_event` is the entry point to the module from everywhere in dbt. # Logging When events are processed via `fire_event`, nearly everything is logged. Whether or not the user has enabled the debug flag, all debug messages are still logged to the file. However, some events are particularly time consuming to construct because they return a huge amount of data. Today, the only messages in this category are cache events and are only logged if the `--log-cache-events` flag is on. This is important because these messages should not be created unless they are going to be logged, because they cause a noticable performance degredation. These events use a "fire_event_if" functions. # Adding a New Event All protos have been moved into the central protos repository. To edit an event proto, edit https://github.com/dbt-labs/proto-python-public or open an issue on that repository. ## Required for Every Event - a method `code`, that's unique across events - assign a log level by using the Level mixin: `DebugLevel`, `InfoLevel`, `WarnLevel`, or `ErrorLevel` - a message() Example ``` class PartialParsingDeletedExposure(DebugLevel): def code(self): return "I049" def message(self) -> str: return f"Partial parsing: deleted exposure {self.unique_id}" ``` ## Compiling core_types.proto After adding a new message in `core_types.proto`, either: - In the repository root directory: `make core_proto_types` - In the `core/dbt/events` directory: `protoc -I=. --python_out=. types.proto` ================================================ FILE: core/dbt/events/__init__.py ================================================ from typing import Any, Dict, Set import dbt.adapters.events.types as adapter_dbt_event_types import dbt.events.types as core_dbt_event_types import dbt_common.events.types as dbt_event_types ALL_EVENT_TYPES: Dict[str, Any] = { **dbt_event_types.__dict__, **core_dbt_event_types.__dict__, **adapter_dbt_event_types.__dict__, } ALL_EVENT_NAMES: Set[str] = set( [name for name, cls in ALL_EVENT_TYPES.items() if isinstance(cls, type)] ) ================================================ FILE: core/dbt/events/base_types.py ================================================ from dbt.events import core_types_pb2 from dbt_common.events.base_types import BaseEvent from dbt_common.events.base_types import DebugLevel as CommonDebugLevel from dbt_common.events.base_types import DynamicLevel as CommonDyanicLevel from dbt_common.events.base_types import ErrorLevel as CommonErrorLevel from dbt_common.events.base_types import InfoLevel as CommonInfoLevel from dbt_common.events.base_types import TestLevel as CommonTestLevel from dbt_common.events.base_types import WarnLevel as CommonWarnLevel class CoreBaseEvent(BaseEvent): PROTO_TYPES_MODULE = core_types_pb2 class DynamicLevel(CommonDyanicLevel, CoreBaseEvent): pass class TestLevel(CommonTestLevel, CoreBaseEvent): pass class DebugLevel(CommonDebugLevel, CoreBaseEvent): pass class InfoLevel(CommonInfoLevel, CoreBaseEvent): pass class WarnLevel(CommonWarnLevel, CoreBaseEvent): pass class ErrorLevel(CommonErrorLevel, CoreBaseEvent): pass ================================================ FILE: core/dbt/events/core_types_pb2.py ================================================ # preserving import path during dbtlabs.proto refactor from dbtlabs.proto.public.v1.fields.core_types_pb2 import * # noqa ================================================ FILE: core/dbt/events/logging.py ================================================ import os from functools import partial from typing import Callable, List from dbt.tracking import track_behavior_change_warn from dbt_common.events.base_types import EventLevel, EventMsg from dbt_common.events.event_manager_client import ( add_callback_to_manager, add_logger_to_manager, cleanup_event_logger, get_event_manager, ) from dbt_common.events.functions import ( env_scrubber, get_capture_stream, get_stdout_config, make_log_dir_if_missing, ) from dbt_common.events.logger import LineFormat, LoggerConfig from dbt_common.invocation import get_invocation_id # These are the logging events issued by the "clean" command, # where we can't count on having a log directory. We've removed # the "class" flags on the events in types.py. If necessary we # could still use class or method flags, but we'd have to get # the type class from the msg and then get the information from the class. _NOFILE_CODES = ["Z012", "Z013", "Z014", "Z015"] def _line_format_from_str(format_str: str, default: LineFormat) -> LineFormat: if format_str == "text": return LineFormat.PlainText elif format_str == "debug": return LineFormat.DebugText elif format_str == "json": return LineFormat.Json return default def _get_logfile_config( log_path: str, use_colors: bool, line_format: LineFormat, level: EventLevel, log_file_max_bytes: int, log_cache_events: bool = False, ) -> LoggerConfig: return LoggerConfig( name="file_log", line_format=line_format, use_colors=use_colors, level=level, # File log is *always* debug level scrubber=env_scrubber, filter=partial(_logfile_filter, log_cache_events, line_format), invocation_id=get_invocation_id(), output_file_name=log_path, output_file_max_bytes=log_file_max_bytes, ) def _logfile_filter(log_cache_events: bool, line_format: LineFormat, msg: EventMsg) -> bool: return msg.info.code not in _NOFILE_CODES and not ( msg.info.name in ["CacheAction", "CacheDumpGraph"] and not log_cache_events ) def setup_event_logger(flags, callbacks: List[Callable[[EventMsg], None]] = []) -> None: cleanup_event_logger() make_log_dir_if_missing(flags.LOG_PATH) event_manager = get_event_manager() event_manager.callbacks = callbacks.copy() add_callback_to_manager(track_behavior_change_warn) if flags.LOG_LEVEL != "none": line_format = _line_format_from_str(flags.LOG_FORMAT, LineFormat.PlainText) log_level = ( EventLevel.ERROR if flags.QUIET else EventLevel.DEBUG if flags.DEBUG else EventLevel(flags.LOG_LEVEL) ) console_config = get_stdout_config( line_format, flags.USE_COLORS, log_level, flags.LOG_CACHE_EVENTS, ) if get_capture_stream(): # Create second stdout logger to support test which want to know what's # being sent to stdout. console_config.output_stream = get_capture_stream() add_logger_to_manager(console_config) if flags.LOG_LEVEL_FILE != "none": # create and add the file logger to the event manager log_file = os.path.join(flags.LOG_PATH, "dbt.log") log_file_format = _line_format_from_str(flags.LOG_FORMAT_FILE, LineFormat.DebugText) log_level_file = EventLevel.DEBUG if flags.DEBUG else EventLevel(flags.LOG_LEVEL_FILE) add_logger_to_manager( _get_logfile_config( log_file, flags.USE_COLORS_FILE, log_file_format, log_level_file, flags.LOG_FILE_MAX_BYTES, ) ) ================================================ FILE: core/dbt/events/types.py ================================================ import json import os from typing import List from dbt.constants import MAXIMUM_SEED_SIZE_NAME, PIN_PACKAGE_URL from dbt.events.base_types import ( DebugLevel, DynamicLevel, ErrorLevel, InfoLevel, WarnLevel, ) from dbt.flags import get_flags from dbt_common.events.base_types import EventLevel from dbt_common.events.format import ( format_fancy_output_line, pluralize, timestamp_to_datetime_string, ) from dbt_common.ui import deprecation_tag as deprecation_tag_less_strict from dbt_common.ui import error_tag, green, line_wrap_message, red, warning_tag, yellow # This makes it so that mypy will complain if a deprecation tag is used without an event name def _deprecation_tag(description: str, event_name: str) -> str: return deprecation_tag_less_strict(description, event_name) # Event codes have prefixes which follow this table # # | Code | Description | # |:----:|:-------------------:| # | A | Pre-project loading | # | D | Deprecations | # | E | DB adapter | # | I | Project parsing | # | M | Deps generation | # | P | Artifacts | # | Q | Node execution | # | W | Node testing | # | Z | Misc | # | T | Test only | # # The basic idea is that event codes roughly translate to the natural order of running a dbt task # ======================================================= # A - Pre-project loading # ======================================================= class MainReportVersion(InfoLevel): def code(self) -> str: return "A001" def message(self) -> str: return f"Running with dbt{self.version}" class MainReportArgs(DebugLevel): def code(self) -> str: return "A002" def message(self) -> str: return f"running dbt with arguments {str(self.args)}" class MainTrackingUserState(DebugLevel): def code(self) -> str: return "A003" def message(self) -> str: return f"Tracking: {self.user_state}" # Removed A004: MergedFromState class MissingProfileTarget(InfoLevel): def code(self) -> str: return "A005" def message(self) -> str: return f"target not specified in profile '{self.profile_name}', using '{self.target_name}'" # Skipped A006, A007 class InvalidOptionYAML(ErrorLevel): def code(self) -> str: return "A008" def message(self) -> str: return error_tag(f"The YAML provided in the --{self.option_name} argument is not valid.") class LogDbtProjectError(ErrorLevel): def code(self) -> str: return "A009" def message(self) -> str: msg = "Encountered an error while reading the project:" if self.exc: msg += f"\n {str(self.exc)}" return error_tag(msg) # Skipped A010 class LogDbtProfileError(ErrorLevel): def code(self) -> str: return "A011" def message(self) -> str: msg = "Encountered an error while reading profiles:\n" f" {str(self.exc)}" if self.profiles: msg += "Defined profiles:\n" for profile in self.profiles: msg += f" - {profile}" else: msg += "There are no profiles defined in your profiles.yml file" msg += """ For more information on configuring profiles, please consult the dbt docs: https://docs.getdbt.com/docs/configure-your-profile """ return error_tag(msg) class StarterProjectPath(DebugLevel): def code(self) -> str: return "A017" def message(self) -> str: return f"Starter project path: {self.dir}" class ConfigFolderDirectory(InfoLevel): def code(self) -> str: return "A018" def message(self) -> str: return f"Creating dbt configuration folder at {self.dir}" class NoSampleProfileFound(InfoLevel): def code(self) -> str: return "A019" def message(self) -> str: return f"No sample profile found for {self.adapter}." class ProfileWrittenWithSample(InfoLevel): def code(self) -> str: return "A020" def message(self) -> str: return ( f"Profile {self.name} written to {self.path} " "using target's sample configuration. Once updated, you'll be able to " "start developing with dbt." ) class ProfileWrittenWithTargetTemplateYAML(InfoLevel): def code(self) -> str: return "A021" def message(self) -> str: return ( f"Profile {self.name} written to {self.path} using target's " "profile_template.yml and your supplied values." ) class ProfileWrittenWithProjectTemplateYAML(InfoLevel): def code(self) -> str: return "A022" def message(self) -> str: return ( f"Profile {self.name} written to {self.path} using project's " "profile_template.yml and your supplied values." ) class SettingUpProfile(InfoLevel): def code(self) -> str: return "A023" def message(self) -> str: return "Setting up your profile." class InvalidProfileTemplateYAML(InfoLevel): def code(self) -> str: return "A024" def message(self) -> str: return "Invalid profile_template.yml in project." class ProjectNameAlreadyExists(InfoLevel): def code(self) -> str: return "A025" def message(self) -> str: return f"A project called {self.name} already exists here." class ProjectCreated(InfoLevel): def code(self) -> str: return "A026" def message(self) -> str: return f"""Your new dbt project "{self.project_name}" was created! Initialized new project in {os.path.abspath(self.project_name)} For more information on how to configure the profiles.yml file, please consult the dbt documentation here: {self.docs_url} One more thing: Need help? Don't hesitate to reach out to us via GitHub issues or on Slack: {self.slack_url} Happy modeling! """ # ======================================================= # D - Deprecations # ======================================================= def require_event_names_in_deprecations(): # The require_event_names_in_deprecations flag isn't guaranteed to be set by the # time some deprecations are fired. We could have done the following ever single deprecation # that needs it, but this makes it simpler to flip the flag later. return getattr(get_flags(), "require_event_names_in_deprecations", False) class DeprecatedModel(WarnLevel): def code(self) -> str: return "I065" def message(self) -> str: version = ".v" + self.model_version if self.model_version else "" msg = ( f"Model {self.model_name}{version} has passed its deprecation date of {self.deprecation_date}. " "This model should be disabled or removed." ) if require_event_names_in_deprecations(): return line_wrap_message(_deprecation_tag(msg, self.__class__.__name__)) else: return warning_tag(msg) class PackageRedirectDeprecation(WarnLevel): def code(self) -> str: return "D001" def message(self) -> str: description = ( f"The `{self.old_name}` package is deprecated in favor of `{self.new_name}`. Please " f"update your `packages.yml` configuration to use `{self.new_name}` instead." ) if require_event_names_in_deprecations(): return line_wrap_message(_deprecation_tag(description, self.__class__.__name__)) else: return line_wrap_message(deprecation_tag_less_strict(description)) class PackageInstallPathDeprecation(WarnLevel): def code(self) -> str: return "D002" def message(self) -> str: description = """\ The default package install path has changed from `dbt_modules` to `dbt_packages`. Please update `clean-targets` in `dbt_project.yml` and check `.gitignore` as well. Or, set `packages-install-path: dbt_modules` if you'd like to keep the current value. """ if require_event_names_in_deprecations(): return line_wrap_message(_deprecation_tag(description, self.__class__.__name__)) else: return line_wrap_message(deprecation_tag_less_strict(description)) class ConfigSourcePathDeprecation(WarnLevel): def code(self) -> str: return "D003" def message(self) -> str: description = ( f"The `{self.deprecated_path}` config has been renamed to `{self.exp_path}`. " "Please update your `dbt_project.yml` configuration to reflect this change." ) if require_event_names_in_deprecations(): return line_wrap_message(_deprecation_tag(description, self.__class__.__name__)) else: return line_wrap_message(deprecation_tag_less_strict(description)) class ConfigDataPathDeprecation(WarnLevel): def code(self) -> str: return "D004" def message(self) -> str: description = ( f"The `{self.deprecated_path}` config has been renamed to `{self.exp_path}`. " "Please update your `dbt_project.yml` configuration to reflect this change." ) if require_event_names_in_deprecations(): return line_wrap_message(_deprecation_tag(description, self.__class__.__name__)) else: return line_wrap_message(deprecation_tag_less_strict(description)) class MetricAttributesRenamed(WarnLevel): def code(self) -> str: return "D006" def message(self) -> str: description = ( "dbt-core v1.3 renamed attributes for metrics:" "\n 'sql' -> 'expression'" "\n 'type' -> 'calculation_method'" "\n 'type: expression' -> 'calculation_method: derived'" f"\nPlease remove them from the metric definition of metric '{self.metric_name}'" "\nRelevant issue here: https://github.com/dbt-labs/dbt-core/issues/5849" ) if require_event_names_in_deprecations(): return line_wrap_message(_deprecation_tag(description, self.__class__.__name__)) else: return deprecation_tag_less_strict(description) class ExposureNameDeprecation(WarnLevel): def code(self) -> str: return "D007" def message(self) -> str: description = ( "Starting in v1.3, the 'name' of an exposure should contain only letters, " "numbers, and underscores. Exposures support a new property, 'label', which may " f"contain spaces, capital letters, and special characters. {self.exposure} does not " "follow this pattern. Please update the 'name', and use the 'label' property for a " "human-friendly title. This will raise an error in a future version of dbt-core." ) if require_event_names_in_deprecations(): return line_wrap_message(_deprecation_tag(description, self.__class__.__name__)) else: return line_wrap_message(deprecation_tag_less_strict(description)) class InternalDeprecation(WarnLevel): def code(self) -> str: return "D008" def message(self) -> str: extra_reason = "" if self.reason: extra_reason = f"\n{self.reason}" msg = ( f"`{self.name}` is deprecated and will be removed in dbt-core version {self.version}\n\n" f"Adapter maintainers can resolve this deprecation by {self.suggested_action}. {extra_reason}" ) if require_event_names_in_deprecations(): return _deprecation_tag(msg, self.__class__.__name__) else: return warning_tag(msg) class EnvironmentVariableRenamed(WarnLevel): def code(self) -> str: return "D009" def message(self) -> str: description = ( f"The environment variable `{self.old_name}` has been renamed as `{self.new_name}`.\n" f"If `{self.old_name}` is currently set, its value will be used instead of `{self.new_name}`.\n" f"Set `{self.new_name}` and unset `{self.old_name}` to avoid this deprecation warning and " "ensure it works properly in a future release." ) if require_event_names_in_deprecations(): return line_wrap_message(_deprecation_tag(description, self.__class__.__name__)) else: return line_wrap_message(deprecation_tag_less_strict(description)) class ConfigLogPathDeprecation(WarnLevel): def code(self) -> str: return "D010" def message(self) -> str: output = "logs" cli_flag = "--log-path" env_var = "DBT_LOG_PATH" description = ( f"The `{self.deprecated_path}` config in `dbt_project.yml` has been deprecated, " f"and will no longer be supported in a future version of dbt-core. " f"If you wish to write dbt {output} to a custom directory, please use " f"the {cli_flag} CLI flag or {env_var} env var instead." ) if require_event_names_in_deprecations(): return line_wrap_message(_deprecation_tag(description, self.__class__.__name__)) else: return line_wrap_message(deprecation_tag_less_strict(description)) class ConfigTargetPathDeprecation(WarnLevel): def code(self) -> str: return "D011" def message(self) -> str: output = "artifacts" cli_flag = "--target-path" env_var = "DBT_TARGET_PATH" description = ( f"The `{self.deprecated_path}` config in `dbt_project.yml` has been deprecated, " f"and will no longer be supported in a future version of dbt-core. " f"If you wish to write dbt {output} to a custom directory, please use " f"the {cli_flag} CLI flag or {env_var} env var instead." ) if require_event_names_in_deprecations(): return line_wrap_message(_deprecation_tag(description, self.__class__.__name__)) else: return line_wrap_message(deprecation_tag_less_strict(description)) # Note: this deprecation has been removed, but we are leaving # the event class here, because users may have specified it in # warn_error_options. class TestsConfigDeprecation(WarnLevel): def code(self) -> str: return "D012" def message(self) -> str: description = ( f"The `{self.deprecated_path}` config has been renamed to `{self.exp_path}`. " "Please see https://docs.getdbt.com/docs/build/data-tests#new-data_tests-syntax for more information." ) if require_event_names_in_deprecations(): return line_wrap_message(_deprecation_tag(description, self.__class__.__name__)) else: return line_wrap_message(deprecation_tag_less_strict(description)) class ProjectFlagsMovedDeprecation(WarnLevel): def code(self) -> str: return "D013" def message(self) -> str: description = ( "User config should be moved from the 'config' key in profiles.yml to the 'flags' " "key in dbt_project.yml." ) # Can't use line_wrap_message here because flags.printer_width isn't available yet if require_event_names_in_deprecations(): return _deprecation_tag(description, self.__class__.__name__) else: return deprecation_tag_less_strict(description) class SpacesInResourceNameDeprecation(DynamicLevel): def code(self) -> str: return "D014" def message(self) -> str: description = f"Found spaces in the name of `{self.unique_id}`" if self.level == EventLevel.ERROR.value: description = error_tag(description) elif self.level == EventLevel.WARN.value: description = warning_tag(description) if require_event_names_in_deprecations(): return line_wrap_message(_deprecation_tag(description, self.__class__.__name__)) else: return line_wrap_message(description) class ResourceNamesWithSpacesDeprecation(WarnLevel): def code(self) -> str: return "D015" def message(self) -> str: description = f"Spaces found in {self.count_invalid_names} resource name(s). This is deprecated, and may lead to errors when using dbt." if self.show_debug_hint: description += " Run again with `--debug` to see them all." description += " For more information: https://docs.getdbt.com/reference/global-configs/legacy-behaviors" if require_event_names_in_deprecations(): return line_wrap_message(_deprecation_tag(description, self.__class__.__name__)) else: return line_wrap_message(warning_tag(description)) class PackageMaterializationOverrideDeprecation(WarnLevel): def code(self) -> str: return "D016" def message(self) -> str: description = f"Installed package '{self.package_name}' is overriding the built-in materialization '{self.materialization_name}'. Overrides of built-in materializations from installed packages will be deprecated in future versions of dbt. For more information: https://docs.getdbt.com/reference/global-configs/legacy-behaviors" if require_event_names_in_deprecations(): return line_wrap_message(_deprecation_tag(description, self.__class__.__name__)) else: return line_wrap_message(warning_tag(description)) class SourceFreshnessProjectHooksNotRun(WarnLevel): def code(self) -> str: return "D017" def message(self) -> str: description = "In a future version of dbt, the `source freshness` command will start running `on-run-start` and `on-run-end` hooks by default. For more information: https://docs.getdbt.com/reference/global-configs/legacy-behaviors" if require_event_names_in_deprecations(): return line_wrap_message(_deprecation_tag(description, self.__class__.__name__)) else: return line_wrap_message(warning_tag(description)) class MFTimespineWithoutYamlConfigurationDeprecation(WarnLevel): def code(self) -> str: return "D018" def message(self) -> str: description = "Time spines without YAML configuration are in the process of deprecation. Please add YAML configuration for your 'metricflow_time_spine' model. See documentation on MetricFlow time spines: https://docs.getdbt.com/docs/build/metricflow-time-spine and behavior change documentation: https://docs.getdbt.com/reference/global-configs/behavior-changes." if require_event_names_in_deprecations(): return line_wrap_message(_deprecation_tag(description, self.__class__.__name__)) else: return line_wrap_message(warning_tag(description)) class MFCumulativeTypeParamsDeprecation(WarnLevel): def code(self) -> str: return "D019" def message(self) -> str: description = "Cumulative fields `type_params.window` and `type_params.grain_to_date` have been moved and will soon be deprecated. Please nest those values under `type_params.cumulative_type_params.window` and `type_params.cumulative_type_params.grain_to_date`. See documentation on behavior changes: https://docs.getdbt.com/reference/global-configs/behavior-changes." if require_event_names_in_deprecations(): return line_wrap_message(_deprecation_tag(description, self.__class__.__name__)) else: return line_wrap_message(warning_tag(description)) class MicrobatchMacroOutsideOfBatchesDeprecation(WarnLevel): def code(self) -> str: return "D020" def message(self) -> str: description = "The use of a custom microbatch macro outside of batched execution is deprecated. To use it with batched execution, set `flags.require_batched_execution_for_custom_microbatch_strategy` to `True` in `dbt_project.yml`. In the future this will be the default behavior." if require_event_names_in_deprecations(): return line_wrap_message(_deprecation_tag(description, self.__class__.__name__)) else: return line_wrap_message(warning_tag(description)) # Skipping D021. It belonged to the now deleted PackageRedirectDeprecationSummary event. class GenericJSONSchemaValidationDeprecation(WarnLevel): def code(self) -> str: return "D022" def message(self) -> str: possible_causes = "This generally means that either we failed to catch this as a more specific deprecation type OR our JSONSchema had a regression (and this deprecation was erroneous)." if self.key_path == "": description = f"{self.violation} at top level in file `{self.file}` is possibly a deprecation. {possible_causes}" else: description = f"{self.violation} in file `{self.file}` at path `{self.key_path}` is possibly a deprecation. {possible_causes}" return line_wrap_message(_deprecation_tag(description, self.__class__.__name__)) class UnexpectedJinjaBlockDeprecation(WarnLevel): def code(self) -> str: return "D023" def message(self) -> str: description = f"{self.msg} in file `{self.file}`" return line_wrap_message(_deprecation_tag(description, self.__class__.__name__)) class DuplicateYAMLKeysDeprecation(WarnLevel): def code(self) -> str: return "D024" def message(self) -> str: description = f"{self.duplicate_description} in file `{self.file}`" return line_wrap_message(_deprecation_tag(description, self.__class__.__name__)) class CustomTopLevelKeyDeprecation(WarnLevel): def code(self) -> str: return "D025" def message(self) -> str: description = f"{self.msg} in file `{self.file}`" return line_wrap_message(_deprecation_tag(description, self.__class__.__name__)) class CustomKeyInConfigDeprecation(WarnLevel): def code(self) -> str: return "D026" def message(self) -> str: path_specification = "" if self.key_path != "": path_specification = f" at path `{self.key_path}`" description = f"Custom key `{self.key}` found in `config`{path_specification} in file `{self.file}`. Custom config keys should move into the `config.meta`." return line_wrap_message(_deprecation_tag(description, self.__class__.__name__)) class CustomKeyInObjectDeprecation(WarnLevel): def code(self) -> str: return "D027" def message(self) -> str: description = f"Custom key `{self.key}` found at `{self.key_path}` in file `{self.file}`. This may mean the key is a typo, or is simply not a key supported by the object." return line_wrap_message(_deprecation_tag(description, self.__class__.__name__)) class DeprecationsSummary(WarnLevel): def code(self) -> str: return "D028" def message(self) -> str: description = "Summary of encountered deprecations:" for summary in self.summaries: description += ( f"\n\n- {summary.event_name}: {pluralize(summary.occurrences, 'occurrence')}" ) if self.show_all_hint: description += "\n\nTo see all deprecation instances instead of just the first occurrence of each, run command again with the `--show-all-deprecations` flag. You may also need to run with `--no-partial-parse` as some deprecations are only encountered during parsing." return line_wrap_message(_deprecation_tag(description, self.__class__.__name__)) class CustomOutputPathInSourceFreshnessDeprecation(WarnLevel): def code(self) -> str: return "D029" def message(self) -> str: description = f"Custom output path usage `--output {self.path}` usage detected in `dbt source freshness` command." return line_wrap_message(_deprecation_tag(description, self.__class__.__name__)) class PropertyMovedToConfigDeprecation(WarnLevel): def code(self) -> str: return "D030" def message(self) -> str: description = f"Found `{self.key}` as a top-level property of `{self.key_path}` in file `{self.file}`. The `{self.key}` top-level property should be moved into the `config` of `{self.key_path}`." return line_wrap_message(_deprecation_tag(description, self.__class__.__name__)) class WEOIncludeExcludeDeprecation(WarnLevel): def code(self) -> str: return "D031" def message(self) -> str: found_keys: List[str] = [] if self.found_include: found_keys.append("`include`") if self.found_exclude: found_keys.append("`exclude`") description = f"Found {' and '.join(found_keys)} in `warn_error_options` specification." if self.found_include: description += " Please use `error` instead of `include`." if self.found_exclude: description += " Please use `warn` instead of `exclude`." return line_wrap_message(_deprecation_tag(description, self.__class__.__name__)) class ModelParamUsageDeprecation(WarnLevel): def code(self) -> str: return "D032" def message(self) -> str: description = "Usage of `--models`, `--model`, and `-m` is deprecated in favor of `--select` or `-s`." return line_wrap_message(_deprecation_tag(description, self.__class__.__name__)) class ModulesItertoolsUsageDeprecation(WarnLevel): def code(self) -> str: return "D034" def message(self) -> str: description = ( "Usage of itertools modules is deprecated. Please use the built-in functions instead." ) return line_wrap_message(_deprecation_tag(description, self.__class__.__name__)) class SourceOverrideDeprecation(WarnLevel): def code(self) -> str: return "D035" def message(self) -> str: description = f"The source property `overrides` is deprecated but was found on source `{self.source_name}` in file `{self.file}`. Instead, `enabled` should be used to disable the unwanted source." return line_wrap_message(_deprecation_tag(description, self.__class__.__name__)) class EnvironmentVariableNamespaceDeprecation(WarnLevel): def code(self) -> str: return "D036" def message(self) -> str: description = f"Found custom environment variable `{self.env_var}` in the environment. The prefix `{self.reserved_prefix}` is reserved for dbt engine environment variables. Custom environment variables with the prefix `{self.reserved_prefix}` may cause collisions and runtime errors." return line_wrap_message(_deprecation_tag(description, self.__class__.__name__)) class MissingPlusPrefixDeprecation(WarnLevel): def code(self) -> str: return "D037" def message(self) -> str: description = f"Missing '+' prefix on `{self.key}` found at `{self.key_path}` in file `{self.file}`. Hierarchical config values without a '+' prefix are deprecated in dbt_project.yml." return line_wrap_message(_deprecation_tag(description, self.__class__.__name__)) class ArgumentsPropertyInGenericTestDeprecation(WarnLevel): def code(self) -> str: return "D038" def message(self) -> str: description = f"Found `arguments` property in test definition of {self.test_name} without usage of `require_generic_test_arguments_property` behavior change flag. The `arguments` property is deprecated for custom usage and will be used to nest keyword arguments in future versions of dbt." return line_wrap_message(_deprecation_tag(description, self.__class__.__name__)) class MissingArgumentsPropertyInGenericTestDeprecation(WarnLevel): def code(self) -> str: return "D039" def message(self) -> str: description = f"Found top-level arguments to test {self.test_name}. Arguments to generic tests should be nested under the `arguments` property." return line_wrap_message(_deprecation_tag(description, self.__class__.__name__)) class DuplicateNameDistinctNodeTypesDeprecation(WarnLevel): def code(self) -> str: return "D040" def message(self) -> str: description = f"Found resources with the same name '{self.resource_name}' in package '{self.package_name}': '{self.unique_id1}' and '{self.unique_id2}'. Please update one of the resources to have a unique name." return line_wrap_message(_deprecation_tag(description, self.__class__.__name__)) class TimeDimensionsRequireGranularityDeprecation(WarnLevel): def code(self) -> str: return "D041" def message(self) -> str: return line_wrap_message(_deprecation_tag(self.msg, self.__class__.__name__)) class GenericSemanticLayerDeprecation(WarnLevel): def code(self) -> str: return "D042" def message(self) -> str: return line_wrap_message(_deprecation_tag(self.msg, self.__class__.__name__)) class GenerateSchemaNameNullValueDeprecation(WarnLevel): def code(self) -> str: return "D044" def message(self) -> str: description = f"Node '{self.resource_unique_id}' has a schema set to None as a result of a generate_schema_name call. Please set a valid schema name." return line_wrap_message(_deprecation_tag(description, self.__class__.__name__)) # ======================================================= # I - Project parsing # ======================================================= class InputFileDiffError(DebugLevel): def code(self) -> str: return "I001" def message(self) -> str: return f"Error processing file diff: {self.category}, {self.file_id}" # Skipping I003, I004, I005, I006, I007 class InvalidValueForField(WarnLevel): def code(self) -> str: return "I008" def message(self) -> str: return warning_tag(f"Invalid value ({self.field_value}) for field {self.field_name}") class ValidationWarning(WarnLevel): def code(self) -> str: return "I009" def message(self) -> str: return warning_tag( f"Field {self.field_name} is not valid for {self.resource_type} ({self.node_name})" ) class ParsePerfInfoPath(InfoLevel): def code(self) -> str: return "I010" def message(self) -> str: return f"Performance info: {self.path}" # Removed I011: GenericTestFileParse # Removed I012: MacroFileParse # Skipping I013 class PartialParsingErrorProcessingFile(DebugLevel): def code(self) -> str: return "I014" def message(self) -> str: return f"Partial parsing exception processing file {self.file}" # Skipped I015 class PartialParsingError(DebugLevel): def code(self) -> str: return "I016" def message(self) -> str: return f"PP exception info: {self.exc_info}" class PartialParsingSkipParsing(DebugLevel): def code(self) -> str: return "I017" def message(self) -> str: return "Partial parsing enabled, no changes found, skipping parsing" # Skipped I018, I019, I020, I021, I022, I023 class UnableToPartialParse(InfoLevel): def code(self) -> str: return "I024" def message(self) -> str: return f"Unable to do partial parsing because {self.reason}" class StateCheckVarsHash(DebugLevel): def code(self) -> str: return "I025" def message(self) -> str: return f"checksum: {self.checksum}, vars: {self.vars}, profile: {self.profile}, target: {self.target}, version: {self.version}" # Skipped I025, I026, I026, I027 class PartialParsingNotEnabled(DebugLevel): def code(self) -> str: return "I028" def message(self) -> str: return "Partial parsing not enabled" class ParsedFileLoadFailed(DebugLevel): def code(self) -> str: return "I029" def message(self) -> str: return f"Failed to load parsed file from disk at {self.path}: {self.exc}" # Skipped I030-I039 class PartialParsingEnabled(DebugLevel): def code(self) -> str: return "I040" def message(self) -> str: return ( f"Partial parsing enabled: " f"{self.deleted} files deleted, " f"{self.added} files added, " f"{self.changed} files changed." ) class PartialParsingFile(DebugLevel): def code(self) -> str: return "I041" def message(self) -> str: return f"Partial parsing: {self.operation} file: {self.file_id}" # Skipped I042, I043, I044, I045, I046, I047, I048, I049 class InvalidDisabledTargetInTestNode(DebugLevel): def code(self) -> str: return "I050" def message(self) -> str: target_package_string = "" if self.target_package != target_package_string: target_package_string = f"in package '{self.target_package}' " msg = ( f"{self.resource_type_title} '{self.unique_id}' " f"({self.original_file_path}) depends on a {self.target_kind} " f"named '{self.target_name}' {target_package_string}which is disabled" ) return warning_tag(msg) class UnusedResourceConfigPath(WarnLevel): def code(self) -> str: return "I051" def message(self) -> str: path_list = "\n".join(f"- {u}" for u in self.unused_config_paths) msg = ( "Configuration paths exist in your dbt_project.yml file which do not " "apply to any resources.\n" f"There are {len(self.unused_config_paths)} unused configuration paths:\n{path_list}" ) return warning_tag(msg) class SeedIncreased(WarnLevel): def code(self) -> str: return "I052" def message(self) -> str: msg = ( f"Found a seed ({self.package_name}.{self.name}) " f">{MAXIMUM_SEED_SIZE_NAME} in size. The previous file was " f"<={MAXIMUM_SEED_SIZE_NAME}, so it has changed" ) return warning_tag(msg) class SeedExceedsLimitSamePath(WarnLevel): def code(self) -> str: return "I053" def message(self) -> str: msg = ( f"Found a seed ({self.package_name}.{self.name}) " f">{MAXIMUM_SEED_SIZE_NAME} in size at the same path, dbt " f"cannot tell if it has changed: assuming they are the same" ) return warning_tag(msg) class SeedExceedsLimitAndPathChanged(WarnLevel): def code(self) -> str: return "I054" def message(self) -> str: msg = ( f"Found a seed ({self.package_name}.{self.name}) " f">{MAXIMUM_SEED_SIZE_NAME} in size. The previous file was in " f"a different location, assuming it has changed" ) return warning_tag(msg) class SeedExceedsLimitChecksumChanged(WarnLevel): def code(self) -> str: return "I055" def message(self) -> str: msg = ( f"Found a seed ({self.package_name}.{self.name}) " f">{MAXIMUM_SEED_SIZE_NAME} in size. The previous file had a " f"checksum type of {self.checksum_name}, so it has changed" ) return warning_tag(msg) class UnusedTables(WarnLevel): def code(self) -> str: return "I056" def message(self) -> str: msg = [ "During parsing, dbt encountered source overrides that had no target:", ] msg += self.unused_tables msg.append("") return warning_tag("\n".join(msg)) class WrongResourceSchemaFile(WarnLevel): def code(self) -> str: return "I057" def message(self) -> str: msg = line_wrap_message( f"""\ '{self.patch_name}' is a {self.resource_type} node, but it is specified in the {self.yaml_key} section of {self.file_path}. To fix this error, place the `{self.patch_name}` specification under the {self.plural_resource_type} key instead. """ ) return warning_tag(msg) class NoNodeForYamlKey(WarnLevel): def code(self) -> str: return "I058" def message(self) -> str: msg = ( f"Did not find matching node for patch with name '{self.patch_name}' " f"in the '{self.yaml_key}' section of " f"file '{self.file_path}'" ) return warning_tag(msg) class MacroNotFoundForPatch(WarnLevel): def code(self) -> str: return "I059" def message(self) -> str: msg = f'Found patch for macro "{self.patch_name}" which was not found' return warning_tag(msg) class NodeNotFoundOrDisabled(WarnLevel): def code(self) -> str: return "I060" def message(self) -> str: # this is duplicated logic from exceptions.get_not_found_or_disabled_msg # when we convert exceptions to be structured maybe it can be combined? # converting the bool to a string since None is also valid if self.disabled == "None": reason = "was not found or is disabled" elif self.disabled == "True": reason = "is disabled" else: reason = "was not found" target_package_string = "" if self.target_package is not None: target_package_string = f"in package '{self.target_package}' " msg = ( f"{self.resource_type_title} '{self.unique_id}' " f"({self.original_file_path}) depends on a {self.target_kind} " f"named '{self.target_name}' {target_package_string}which {reason}" ) return warning_tag(msg) class JinjaLogWarning(WarnLevel): def code(self) -> str: return "I061" def message(self) -> str: return warning_tag(self.msg) class JinjaLogInfo(InfoLevel): def code(self) -> str: return "I062" def message(self) -> str: # This is for the log method used in macros so msg cannot be built here return self.msg class JinjaLogDebug(DebugLevel): def code(self) -> str: return "I063" def message(self) -> str: # This is for the log method used in macros so msg cannot be built here return self.msg class UnpinnedRefNewVersionAvailable(InfoLevel): def code(self) -> str: return "I064" def message(self) -> str: msg = ( f"While compiling '{self.node_info.node_name}':\n" f"Found an unpinned reference to versioned model '{self.ref_node_name}' in project '{self.ref_node_package}'.\n" f"Resolving to latest version: {self.ref_node_name}.v{self.ref_node_version}\n" f"A prerelease version {self.ref_max_version} is available. It has not yet been marked 'latest' by its maintainer.\n" f"When that happens, this reference will resolve to {self.ref_node_name}.v{self.ref_max_version} instead.\n\n" f" Try out v{self.ref_max_version}: {{{{ ref('{self.ref_node_package}', '{self.ref_node_name}', v='{self.ref_max_version}') }}}}\n" f" Pin to v{self.ref_node_version}: {{{{ ref('{self.ref_node_package}', '{self.ref_node_name}', v='{self.ref_node_version}') }}}}\n" ) return msg class UpcomingReferenceDeprecation(WarnLevel): def code(self) -> str: return "I066" def message(self) -> str: ref_model_version = ".v" + self.ref_model_version if self.ref_model_version else "" msg = ( f"While compiling '{self.model_name}': Found a reference to {self.ref_model_name}{ref_model_version}, " f"which is slated for deprecation on '{self.ref_model_deprecation_date}'. " ) if self.ref_model_version and self.ref_model_version != self.ref_model_latest_version: coda = ( f"A new version of '{self.ref_model_name}' is available. Try it out: " f"{{{{ ref('{self.ref_model_package}', '{self.ref_model_name}', " f"v='{self.ref_model_latest_version}') }}}}." ) msg = msg + coda return warning_tag(msg) class DeprecatedReference(WarnLevel): def code(self) -> str: return "I067" def message(self) -> str: ref_model_version = ".v" + self.ref_model_version if self.ref_model_version else "" msg = ( f"While compiling '{self.model_name}': Found a reference to {self.ref_model_name}{ref_model_version}, " f"which was deprecated on '{self.ref_model_deprecation_date}'. " ) if self.ref_model_version and self.ref_model_version != self.ref_model_latest_version: coda = ( f"A new version of '{self.ref_model_name}' is available. Migrate now: " f"{{{{ ref('{self.ref_model_package}', '{self.ref_model_name}', " f"v='{self.ref_model_latest_version}') }}}}." ) msg = msg + coda return warning_tag(msg) class UnsupportedConstraintMaterialization(WarnLevel): def code(self) -> str: return "I068" def message(self) -> str: msg = ( f"Constraint types are not supported for {self.materialized} materializations and will " "be ignored. Set 'warn_unsupported: false' on this constraint to ignore this warning." ) return line_wrap_message(warning_tag(msg)) class ParseInlineNodeError(ErrorLevel): def code(self) -> str: return "I069" def message(self) -> str: return error_tag("Error while parsing node: " + self.node_info.node_name + "\n" + self.exc) class SemanticValidationFailure(WarnLevel): def code(self) -> str: return "I070" def message(self) -> str: return warning_tag(self.msg) class UnversionedBreakingChange(WarnLevel): def code(self) -> str: return "I071" def message(self) -> str: reasons = "\n - ".join(self.breaking_changes) msg = ( f"Breaking change to contracted, unversioned model {self.model_name} ({self.model_file_path})" "\nWhile comparing to previous project state, dbt detected a breaking change to an unversioned model." f"\n - {reasons}\n" ) return warning_tag(msg) class WarnStateTargetEqual(WarnLevel): def code(self) -> str: return "I072" def message(self) -> str: return warning_tag( f"The state and target directories are the same: '{self.state_path}'. " f"This could lead to missing changes due to overwritten state including non-idempotent retries." ) class FreshnessConfigProblem(WarnLevel): def code(self) -> str: return "I073" def message(self) -> str: return warning_tag(self.msg) class MicrobatchModelNoEventTimeInputs(WarnLevel): def code(self) -> str: return "I074" def message(self) -> str: msg = ( f"The microbatch model '{self.model_name}' has no 'ref' or 'source' input with an 'event_time' configuration. " "\nThis means no filtering can be applied and can result in unexpected duplicate records in the resulting microbatch model." ) return warning_tag(msg) class InvalidConcurrentBatchesConfig(WarnLevel): def code(self) -> str: return "I075" def message(self) -> str: maybe_plural_count_of_models = pluralize(self.num_models, "microbatch model") description = f"Found {maybe_plural_count_of_models} with the `concurrent_batches` config set to true, but the {self.adapter_type} adapter does not support running batches concurrently. Batches will be run sequentially." return line_wrap_message(warning_tag(description)) class InvalidMacroAnnotation(WarnLevel): def code(self) -> str: return "I076" def message(self) -> str: return warning_tag(self.msg) class PackageNodeDependsOnRootProjectNode(WarnLevel): def code(self) -> str: return "I077" def message(self) -> str: msg = ( f"The node '{self.node_name}'in package '{self.package_name}' depends on the root project node '{self.root_project_unique_id}'." "This may lead to unexpected cycles downstream. Please set the 'require_ref_searches_node_package_before_root' behavior change flag to True to avoid this issue." "For more information, see the documentation at https://docs.getdbt.com/reference/global-configs/behavior-changes#package-ref-search-order" ) return warning_tag(msg) # ======================================================= # M - Deps generation # ======================================================= class GitSparseCheckoutSubdirectory(DebugLevel): def code(self) -> str: return "M001" def message(self) -> str: return f"Subdirectory specified: {self.subdir}, using sparse checkout." class GitProgressCheckoutRevision(DebugLevel): def code(self) -> str: return "M002" def message(self) -> str: return f"Checking out revision {self.revision}." class GitProgressUpdatingExistingDependency(DebugLevel): def code(self) -> str: return "M003" def message(self) -> str: return f"Updating existing dependency {self.dir}." class GitProgressPullingNewDependency(DebugLevel): def code(self) -> str: return "M004" def message(self) -> str: return f"Pulling new dependency {self.dir}." class GitNothingToDo(DebugLevel): def code(self) -> str: return "M005" def message(self) -> str: return f"Already at {self.sha}, nothing to do." class GitProgressUpdatedCheckoutRange(DebugLevel): def code(self) -> str: return "M006" def message(self) -> str: return f"Updated checkout from {self.start_sha} to {self.end_sha}." class GitProgressCheckedOutAt(DebugLevel): def code(self) -> str: return "M007" def message(self) -> str: return f"Checked out at {self.end_sha}." class RegistryProgressGETRequest(DebugLevel): def code(self) -> str: return "M008" def message(self) -> str: return f"Making package registry request: GET {self.url}" class RegistryProgressGETResponse(DebugLevel): def code(self) -> str: return "M009" def message(self) -> str: return f"Response from registry: GET {self.url} {self.resp_code}" class SelectorReportInvalidSelector(InfoLevel): def code(self) -> str: return "M010" def message(self) -> str: return ( f"The '{self.spec_method}' selector specified in {self.raw_spec} is " f"invalid. Must be one of [{self.valid_selectors}]" ) class DepsNoPackagesFound(InfoLevel): def code(self) -> str: return "M013" def message(self) -> str: return "Warning: No packages were found in packages.yml" class DepsStartPackageInstall(InfoLevel): def code(self) -> str: return "M014" def message(self) -> str: return f"Installing {self.package_name}" class DepsInstallInfo(InfoLevel): def code(self) -> str: return "M015" def message(self) -> str: return f"Installed from {self.version_name}" class DepsUpdateAvailable(InfoLevel): def code(self) -> str: return "M016" def message(self) -> str: return f"Updated version available: {self.version_latest}" class DepsUpToDate(InfoLevel): def code(self) -> str: return "M017" def message(self) -> str: return "Up to date!" class DepsListSubdirectory(InfoLevel): def code(self) -> str: return "M018" def message(self) -> str: return f"and subdirectory {self.subdirectory}" class DepsNotifyUpdatesAvailable(InfoLevel): def code(self) -> str: return "M019" def message(self) -> str: return f"Updates available for packages: {self.packages} \ \nUpdate your versions in packages.yml, then run dbt deps" class RegistryIndexProgressGETRequest(DebugLevel): def code(self) -> str: return "M022" def message(self) -> str: return f"Making package index registry request: GET {self.url}" class RegistryIndexProgressGETResponse(DebugLevel): def code(self) -> str: return "M023" def message(self) -> str: return f"Response from registry index: GET {self.url} {self.resp_code}" class RegistryResponseUnexpectedType(DebugLevel): def code(self) -> str: return "M024" def message(self) -> str: return f"Response was None: {self.response}" class RegistryResponseMissingTopKeys(DebugLevel): def code(self) -> str: return "M025" def message(self) -> str: # expected/actual keys logged in exception return f"Response missing top level keys: {self.response}" class RegistryResponseMissingNestedKeys(DebugLevel): def code(self) -> str: return "M026" def message(self) -> str: # expected/actual keys logged in exception return f"Response missing nested keys: {self.response}" class RegistryResponseExtraNestedKeys(DebugLevel): def code(self) -> str: return "M027" def message(self) -> str: # expected/actual keys logged in exception return f"Response contained inconsistent keys: {self.response}" class DepsSetDownloadDirectory(DebugLevel): def code(self) -> str: return "M028" def message(self) -> str: return f"Set downloads directory='{self.path}'" class DepsUnpinned(WarnLevel): def code(self) -> str: return "M029" def message(self) -> str: if self.revision == "HEAD": unpinned_msg = "not pinned, using HEAD (default branch)" elif self.revision in ("main", "master"): unpinned_msg = f'pinned to the "{self.revision}" branch' else: unpinned_msg = None msg = ( f'The git package "{self.git}" \n\tis {unpinned_msg}.\n\tThis can introduce ' f"breaking changes into your project without warning!\n\nSee {PIN_PACKAGE_URL}" ) return warning_tag(msg) class NoNodesForSelectionCriteria(WarnLevel): def code(self) -> str: return "M030" def message(self) -> str: return warning_tag( f"The selection criterion '{self.spec_raw}' does not match any enabled nodes" ) class DepsLockUpdating(InfoLevel): def code(self): return "M031" def message(self) -> str: return f"Updating lock file in file path: {self.lock_filepath}" class DepsAddPackage(InfoLevel): def code(self): return "M032" def message(self) -> str: return f"Added new package {self.package_name}@{self.version} to {self.packages_filepath}" class DepsFoundDuplicatePackage(InfoLevel): def code(self): return "M033" def message(self) -> str: return f"Found duplicate package in packages.yml, removing: {self.removed_package}" class DepsScrubbedPackageName(WarnLevel): def code(self): return "M035" def message(self) -> str: return warning_tag( f"Detected secret env var in {self.package_name}. dbt will write a scrubbed representation to the lock file. This will cause issues with subsequent 'dbt deps' using the lock file, requiring 'dbt deps --upgrade'" ) # ======================================================= # P - Artifacts # ======================================================= class ArtifactWritten(DebugLevel): def code(self): return "P001" def message(self) -> str: return f"Wrote artifact {self.artifact_type} to {self.artifact_path}" # ======================================================= # Q - Node execution # ======================================================= class RunningOperationCaughtError(ErrorLevel): def code(self) -> str: return "Q001" def message(self) -> str: return error_tag(f"Encountered an error while running operation: {self.exc}") class CompileComplete(InfoLevel): def code(self) -> str: return "Q002" def message(self) -> str: return "Done." class FreshnessCheckComplete(InfoLevel): def code(self) -> str: return "Q003" def message(self) -> str: return "Done." class SeedHeader(InfoLevel): def code(self) -> str: return "Q004" def message(self) -> str: return self.header class SQLRunnerException(DebugLevel): def code(self) -> str: return "Q006" def message(self) -> str: return f"Got an exception: {self.exc}" class LogTestResult(DynamicLevel): def code(self) -> str: return "Q007" def message(self) -> str: if self.status == "error": info = "ERROR" status = red( info, ) elif self.status == "pass": info = "PASS" status = green(info) elif self.status == "warn": info = f"WARN {self.num_failures}" status = yellow(info) else: # self.status == "fail": info = f"FAIL {self.num_failures}" status = red(info) msg = f"{info} {self.name}" return format_fancy_output_line( msg=msg, status=status, index=self.index, total=self.num_models, execution_time=self.execution_time, ) @classmethod def status_to_level(cls, status): # The statuses come from TestStatus level_lookup = { "fail": EventLevel.ERROR, "pass": EventLevel.INFO, "warn": EventLevel.WARN, "error": EventLevel.ERROR, } if status in level_lookup: return level_lookup[status] else: return EventLevel.INFO class LogNodeResult(DynamicLevel): def code(self) -> str: return "Q008" def message(self) -> str: return self.msg # Skipped Q009, Q010 class LogStartLine(InfoLevel): def code(self) -> str: return "Q011" def message(self) -> str: msg = f"START {self.description}" return format_fancy_output_line(msg=msg, status="RUN", index=self.index, total=self.total) class LogModelResult(DynamicLevel): def code(self) -> str: return "Q012" def message(self) -> str: if self.status == "error": info = "ERROR creating" status = red(self.status.upper()) elif "PARTIAL SUCCESS" in self.status: info = "PARTIALLY created" status = yellow(self.status.upper()) else: info = "OK created" status = green(self.status) msg = f"{info} {self.description}" return format_fancy_output_line( msg=msg, status=status, index=self.index, total=self.total, execution_time=self.execution_time, ) # Skipped Q013, Q014 class LogSnapshotResult(DynamicLevel): def code(self) -> str: return "Q015" def message(self) -> str: if self.status == "error": info = "ERROR snapshotting" status = red(self.status.upper()) else: info = "OK snapshotted" status = green(self.result_message) msg = "{info} {description}".format(info=info, description=self.description, **self.cfg) return format_fancy_output_line( msg=msg, status=status, index=self.index, total=self.total, execution_time=self.execution_time, ) class LogSeedResult(DynamicLevel): def code(self) -> str: return "Q016" def message(self) -> str: if self.status == "error": info = "ERROR loading" status = red(self.status.upper()) else: info = "OK loaded" status = green(self.result_message) msg = f"{info} seed file {self.schema}.{self.relation}" return format_fancy_output_line( msg=msg, status=status, index=self.index, total=self.total, execution_time=self.execution_time, ) # Skipped Q017 class LogFreshnessResult(DynamicLevel): def code(self) -> str: return "Q018" def message(self) -> str: if self.status == "runtime error": info = "ERROR" status = red(info) elif self.status == "error": info = "ERROR STALE" status = red(info) elif self.status == "warn": info = "WARN" status = yellow(info) else: info = "PASS" status = green(info) msg = f"{info} freshness of {self.source_name}.{self.table_name}" return format_fancy_output_line( msg=msg, status=status, index=self.index, total=self.total, execution_time=self.execution_time, ) @classmethod def status_to_level(cls, status): # The statuses come from FreshnessStatus # TODO should this return EventLevel enum instead? level_lookup = { "runtime error": EventLevel.ERROR, "pass": EventLevel.INFO, "warn": EventLevel.WARN, "error": EventLevel.ERROR, } if status in level_lookup: return level_lookup[status] else: return EventLevel.INFO class LogNodeNoOpResult(InfoLevel): def code(self) -> str: return "Q019" def message(self) -> str: msg = f"NO-OP {self.description}" return format_fancy_output_line( msg=msg, status=yellow("NO-OP"), index=self.index, total=self.total, execution_time=self.execution_time, ) # Skipped Q020, Q021 class LogCancelLine(ErrorLevel): def code(self) -> str: return "Q022" def message(self) -> str: msg = f"CANCEL query {self.conn_name}" return format_fancy_output_line(msg=msg, status=red("CANCEL"), index=None, total=None) class DefaultSelector(InfoLevel): def code(self) -> str: return "Q023" def message(self) -> str: return f"Using default selector {self.name}" class NodeStart(DebugLevel): def code(self) -> str: return "Q024" def message(self) -> str: return f"Began running node {self.node_info.unique_id}" class NodeFinished(DebugLevel): def code(self) -> str: return "Q025" def message(self) -> str: return f"Finished running node {self.node_info.unique_id}" class QueryCancelationUnsupported(InfoLevel): def code(self) -> str: return "Q026" def message(self) -> str: msg = ( f"The {self.type} adapter does not support query " "cancellation. Some queries may still be " "running!" ) return yellow(msg) class ConcurrencyLine(InfoLevel): def code(self) -> str: return "Q027" def message(self) -> str: return f"Concurrency: {self.num_threads} threads (target='{self.target_name}')" class WritingInjectedSQLForNode(DebugLevel): def code(self) -> str: return "Q029" def message(self) -> str: return f'Writing injected SQL for node "{self.node_info.unique_id}"' class NodeCompiling(DebugLevel): def code(self) -> str: return "Q030" def message(self) -> str: return f"Began compiling node {self.node_info.unique_id}" class NodeExecuting(DebugLevel): def code(self) -> str: return "Q031" def message(self) -> str: return f"Began executing node {self.node_info.unique_id}" class LogHookStartLine(InfoLevel): def code(self) -> str: return "Q032" def message(self) -> str: msg = f"START hook: {self.statement}" return format_fancy_output_line( msg=msg, status="RUN", index=self.index, total=self.total, truncate=True ) class LogHookEndLine(InfoLevel): def code(self) -> str: return "Q033" def message(self) -> str: if self.status == "success": info = "OK" status = green(info) elif self.status == "skipped": info = "SKIP" status = yellow(info) else: info = "ERROR" status = red(info) msg = f"{info} hook: {self.statement}" return format_fancy_output_line( msg=msg, status=status, index=self.index, total=self.total, execution_time=self.execution_time, truncate=True, ) class SkippingDetails(InfoLevel): def code(self) -> str: return "Q034" def message(self) -> str: # ToDo: move to core or figure out NodeType if self.resource_type in ["model", "seed", "snapshot"]: msg = f"SKIP relation {self.schema}.{self.node_name}" else: msg = f"SKIP {self.resource_type} {self.node_name}" return format_fancy_output_line( msg=msg, status=yellow("SKIP"), index=self.index, total=self.total ) class NothingToDo(WarnLevel): def code(self) -> str: return "Q035" def message(self) -> str: return warning_tag( "Nothing to do. Try checking your model configs and model specification args" ) class RunningOperationUncaughtError(ErrorLevel): def code(self) -> str: return "Q036" def message(self) -> str: return error_tag(f"Encountered an error while running operation: {self.exc}") class EndRunResult(DebugLevel): def code(self) -> str: return "Q037" def message(self) -> str: return "Command end result" class NoNodesSelected(WarnLevel): def code(self) -> str: return "Q038" def message(self) -> str: return warning_tag("No nodes selected!") class CommandCompleted(DebugLevel): def code(self) -> str: return "Q039" def message(self) -> str: status = "succeeded" if self.success else "failed" completed_at = timestamp_to_datetime_string(self.completed_at) return f"Command `{self.command}` {status} at {completed_at} after {self.elapsed:0.2f} seconds" class ShowNode(InfoLevel): def code(self) -> str: return "Q041" def message(self) -> str: if self.output_format == "json": if self.is_inline: return json.dumps({"show": json.loads(self.preview)}, indent=2) else: return json.dumps( {"node": self.node_name, "show": json.loads(self.preview)}, indent=2 ) else: if self.quiet: return self.preview elif self.is_inline: return f"Previewing inline node:\n{self.preview}" else: return f"Previewing node '{self.node_name}':\n{self.preview}" class CompiledNode(InfoLevel): def code(self) -> str: return "Q042" def message(self) -> str: if self.output_format == "json": if self.is_inline: return json.dumps({"compiled": self.compiled}, indent=2) else: return json.dumps({"node": self.node_name, "compiled": self.compiled}, indent=2) else: if self.quiet: return self.compiled elif self.is_inline: return f"Compiled inline node is:\n{self.compiled}" else: return f"Compiled node '{self.node_name}' is:\n{self.compiled}" class SnapshotTimestampWarning(WarnLevel): def code(self) -> str: return "Q043" def message(self) -> str: msg = ( f"Data type of snapshot table timestamp columns ({self.snapshot_time_data_type}) " f"doesn't match derived column 'updated_at' ({self.updated_at_data_type}). " "Please update snapshot config 'updated_at'." ) return warning_tag(msg) class MicrobatchExecutionDebug(DebugLevel): def code(self) -> str: return "Q044" def message(self) -> str: return self.msg class LogStartBatch(InfoLevel): def code(self) -> str: return "Q045" def message(self) -> str: msg = f"START {self.description}" # TODO update common so that we can append "batch" in `format_fancy_output_line` formatted = format_fancy_output_line( msg=msg, status="RUN", index=self.batch_index, total=self.total_batches, ) return f"Batch {formatted}" class LogBatchResult(DynamicLevel): def code(self) -> str: return "Q046" def message(self) -> str: if self.status == "error": info = "ERROR creating" status = red(self.status.upper()) elif self.status == "skipped": info = "SKIP" status = yellow(self.status.upper()) else: info = "OK created" status = green(self.status) msg = f"{info} {self.description}" # TODO update common so that we can append "batch" in `format_fancy_output_line` formatted = format_fancy_output_line( msg=msg, status=status, index=self.batch_index, total=self.total_batches, execution_time=self.execution_time, ) return f"Batch {formatted}" class LogFunctionResult(DynamicLevel): def code(self) -> str: return "Q047" def message(self) -> str: if self.status == "error": info = "ERROR creating" status = red(self.status.upper()) elif self.status == "skipped": info = "SKIP" status = yellow(self.status.upper()) else: info = "OK created" status = green(self.status.upper()) msg = f"{info} {self.description}" return format_fancy_output_line( msg=msg, status=status, index=self.index, total=self.total, execution_time=self.execution_time, ) # ======================================================= # W - Node testing # ======================================================= # Skipped W001 class CatchableExceptionOnRun(DebugLevel): def code(self) -> str: return "W002" def message(self) -> str: return str(self.exc) class InternalErrorOnRun(DebugLevel): def code(self) -> str: return "W003" def message(self) -> str: prefix = f"Internal error executing {self.build_path}" internal_error_string = """This is an error in dbt. Please try again. If \ the error persists, open an issue at https://github.com/dbt-labs/dbt-core """.strip() return f"{red(prefix)}\n" f"{str(self.exc).strip()}\n\n" f"{internal_error_string}" class GenericExceptionOnRun(ErrorLevel): def code(self) -> str: return "W004" def message(self) -> str: node_description = self.build_path if node_description is None: node_description = self.unique_id msg = f"Unhandled error while executing {node_description}\n{str(self.exc).strip()}" return error_tag(msg) class NodeConnectionReleaseError(DebugLevel): def code(self) -> str: return "W005" def message(self) -> str: return f"Error releasing connection for node {self.node_name}: {str(self.exc)}" class FoundStats(InfoLevel): def code(self) -> str: return "W006" def message(self) -> str: return f"Found {self.stat_line}" # ======================================================= # Z - Misc # ======================================================= class MainKeyboardInterrupt(InfoLevel): def code(self) -> str: return "Z001" def message(self) -> str: return "ctrl-c" class MainEncounteredError(ErrorLevel): def code(self) -> str: return "Z002" def message(self) -> str: return error_tag(f"Encountered an error:\n{self.exc}") class MainStackTrace(ErrorLevel): def code(self) -> str: return "Z003" def message(self) -> str: return self.stack_trace # Skipped Z004 class TimingInfoCollected(DebugLevel): def code(self) -> str: return "Z010" def message(self) -> str: started_at = timestamp_to_datetime_string(self.timing_info.started_at) completed_at = timestamp_to_datetime_string(self.timing_info.completed_at) return f"Timing info for {self.node_info.unique_id} ({self.timing_info.name}): {started_at} => {completed_at}" # This prints the stack trace at the debug level while allowing just the nice exception message # at the error level - or whatever other level chosen. Used in multiple places. class LogDebugStackTrace(DebugLevel): def code(self) -> str: return "Z011" def message(self) -> str: return f"{self.exc_info}" # We don't write "clean" events to the log, because the clean command # may have removed the log directory. class CheckCleanPath(InfoLevel): def code(self) -> str: return "Z012" def message(self) -> str: return f"Checking {self.path}/*" class ConfirmCleanPath(InfoLevel): def code(self) -> str: return "Z013" def message(self) -> str: return f"Cleaned {self.path}/*" class ProtectedCleanPath(InfoLevel): def code(self) -> str: return "Z014" def message(self) -> str: return f"ERROR: not cleaning {self.path}/* because it is protected" class FinishedCleanPaths(InfoLevel): def code(self) -> str: return "Z015" def message(self) -> str: return "Finished cleaning all paths." class OpenCommand(InfoLevel): def code(self) -> str: return "Z016" def message(self) -> str: msg = f"""To view your profiles.yml file, run: {self.open_cmd} {self.profiles_dir}""" return msg class RunResultWarning(WarnLevel): def code(self) -> str: return "Z021" def message(self) -> str: return warning_tag(f"in {self.resource_type} {self.node_name} ({self.path})") class RunResultFailure(ErrorLevel): def code(self) -> str: return "Z022" def message(self) -> str: return error_tag(f"in {self.resource_type} {self.node_name} ({self.path})") class StatsLine(InfoLevel): def code(self) -> str: return "Z023" def message(self) -> str: stats_line = ( "Done. PASS={pass} WARN={warn} ERROR={error} SKIP={skip} NO-OP={noop} TOTAL={total}" ) return stats_line.format(**self.stats) class RunResultError(ErrorLevel): def code(self) -> str: return "Z024" def message(self) -> str: # This is the message on the result object, cannot be built here return f" {self.msg}" class RunResultErrorNoMessage(ErrorLevel): def code(self) -> str: return "Z025" def message(self) -> str: return f" Status: {self.status}" class SQLCompiledPath(InfoLevel): def code(self) -> str: return "Z026" def message(self) -> str: return f" compiled code at {self.path}" class CheckNodeTestFailure(InfoLevel): def code(self) -> str: return "Z027" def message(self) -> str: msg = f"select * from {self.relation_name}" border = "-" * len(msg) return f" See test failures:\n {border}\n {msg}\n {border}" # Skipped Z028, Z029 class EndOfRunSummary(InfoLevel): def code(self) -> str: return "Z030" def message(self) -> str: error_plural = pluralize(self.num_errors, "error") warn_plural = pluralize(self.num_warnings, "warning") partial_success_plural = f"""{self.num_partial_success} partial {"success" if self.num_partial_success == 1 else "successes"}""" if self.keyboard_interrupt: message = yellow("Exited because of keyboard interrupt") elif self.num_errors > 0: message = red( f"Completed with {error_plural}, {partial_success_plural}, and {warn_plural}:" ) elif self.num_partial_success > 0: message = yellow(f"Completed with {partial_success_plural} and {warn_plural}") elif self.num_warnings > 0: message = yellow(f"Completed with {warn_plural}:") else: message = green("Completed successfully") return message # Skipped Z031, Z032 class MarkSkippedChildren(DebugLevel): def code(self) -> str: return "Z033" def message(self) -> str: msg = ( f"Marking all children of '{self.unique_id}' to be skipped " f"because of status '{self.status}'. " ) if self.run_result.message: msg = msg + f" Reason: {self.run_result.message}." return msg class LogSkipBecauseError(ErrorLevel): def code(self) -> str: return "Z034" def message(self) -> str: msg = f"SKIP relation {self.schema}.{self.relation} due to ephemeral model status '{self.status}'" return format_fancy_output_line( msg=msg, status=red("ERROR SKIP"), index=self.index, total=self.total ) # Skipped Z035 class EnsureGitInstalled(ErrorLevel): def code(self) -> str: return "Z036" def message(self) -> str: return error_tag( "Make sure git is installed on your machine. More " "information: " "https://docs.getdbt.com/docs/package-management" ) class DepsCreatingLocalSymlink(DebugLevel): def code(self) -> str: return "Z037" def message(self) -> str: return "Creating symlink to local dependency." class DepsSymlinkNotAvailable(DebugLevel): def code(self) -> str: return "Z038" def message(self) -> str: return "Symlinks are not available on this OS, copying dependency." class DisableTracking(DebugLevel): def code(self) -> str: return "Z039" def message(self) -> str: return ( "Error sending anonymous usage statistics. Disabling tracking for this execution. " "If you wish to permanently disable tracking, see: " "https://docs.getdbt.com/reference/global-configs#send-anonymous-usage-stats." ) class SendingEvent(DebugLevel): def code(self) -> str: return "Z040" def message(self) -> str: return f"Sending event: {self.kwargs}" class SendEventFailure(DebugLevel): def code(self) -> str: return "Z041" def message(self) -> str: return "An error was encountered while trying to send an event" class FlushEvents(DebugLevel): def code(self) -> str: return "Z042" def message(self) -> str: return "Flushing usage events" class FlushEventsFailure(DebugLevel): def code(self) -> str: return "Z043" def message(self) -> str: return "An error was encountered while trying to flush usage events" class TrackingInitializeFailure(DebugLevel): def code(self) -> str: return "Z044" def message(self) -> str: return "Got an exception trying to initialize tracking" # this is the message from the result object class RunResultWarningMessage(WarnLevel): def code(self) -> str: return "Z046" def message(self) -> str: # This is the message on the result object, cannot be formatted in event return warning_tag(self.msg) class DebugCmdOut(InfoLevel): def code(self) -> str: return "Z047" def message(self) -> str: return self.msg class DebugCmdResult(InfoLevel): def code(self) -> str: return "Z048" def message(self) -> str: return self.msg class ListCmdOut(InfoLevel): # No longer in use, switching to Z051 PrintEvent in dbt-common def code(self) -> str: return "Z049" def message(self) -> str: return self.msg class ResourceReport(DebugLevel): def code(self) -> str: return "Z051" def message(self) -> str: return f"Resource report: {self.to_json()}" # Artifact Upload Events # class ArtifactUploadError(ErrorLevel): def code(self) -> str: return "Z061" def message(self) -> str: return error_tag(f"Error uploading artifacts to artifact ingestion API: {self.msg}") class ArtifactUploadSuccess(InfoLevel): def code(self) -> str: return "Z062" def message(self) -> str: return f"Artifacts uploaded successfully to artifact ingestion API: {self.msg}" class ArtifactUploadSkipped(DebugLevel): def code(self) -> str: return "Z063" def message(self) -> str: return f"Artifacts skipped for command : {self.msg}" class SelectExcludeIgnoredWithSelectorWarning(WarnLevel): def code(self) -> str: return "Z064" def message(self) -> str: return "The --select and --exclude arguments are being ignored for node selection because --selector is provided" ================================================ FILE: core/dbt/exceptions.py ================================================ import io import json import re from typing import TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Tuple, Union from dbt.node_types import REFABLE_NODE_TYPES, AccessType, NodeType from dbt_common.constants import SECRET_ENV_PREFIX from dbt_common.dataclass_schema import ValidationError from dbt_common.exceptions import ( CommandResultError, CompilationError, DbtConfigError, DbtInternalError, DbtRuntimeError, DbtValidationError, env_secrets, scrub_secrets, ) if TYPE_CHECKING: import agate class ContractBreakingChangeError(DbtRuntimeError): CODE = 10016 MESSAGE = "Breaking Change to Contract" def __init__( self, breaking_changes: List[str], node=None, ) -> None: self.breaking_changes = breaking_changes super().__init__(self.message(), node) @property def type(self): return "Breaking change to contract" def message(self): reasons = "\n - ".join(self.breaking_changes) return ( "While comparing to previous project state, dbt detected a breaking change to an enforced contract." f"\n - {reasons}\n" "Consider making an additive (non-breaking) change instead, if possible.\n" "Otherwise, create a new model version: https://docs.getdbt.com/docs/collaborate/govern/model-versions" ) class ParsingError(DbtRuntimeError): CODE = 10015 MESSAGE = "Parsing Error" @property def type(self): return "Parsing" class dbtPluginError(DbtRuntimeError): CODE = 10020 MESSAGE = "Plugin Error" # TODO: this isn't raised in the core codebase. Is it raised elsewhere? class JSONValidationError(DbtValidationError): def __init__(self, typename, errors) -> None: self.typename = typename self.errors = errors self.errors_message = ", ".join(errors) msg = f'Invalid arguments passed to "{self.typename}" instance: {self.errors_message}' super().__init__(msg) def __reduce__(self): # see https://stackoverflow.com/a/36342588 for why this is necessary return (JSONValidationError, (self.typename, self.errors)) class AliasError(DbtValidationError): pass class DependencyError(DbtRuntimeError): CODE = 10006 MESSAGE = "Dependency Error" class FailFastError(DbtRuntimeError): CODE = 10013 MESSAGE = "FailFast Error" def __init__(self, msg: str, result=None, node=None) -> None: super().__init__(msg=msg, node=node) self.result = result @property def type(self): return "FailFast" class DbtProjectError(DbtConfigError): pass class DbtSelectorsError(DbtConfigError): pass class DbtProfileError(DbtConfigError): pass class DbtExclusivePropertyUseError(DbtConfigError): pass class InvalidSelectorError(DbtRuntimeError): def __init__(self, name: str) -> None: self.name = name super().__init__(name) class DuplicateYamlKeyError(CompilationError): pass # compilation level exceptions class GraphDependencyNotFoundError(CompilationError): def __init__(self, node, dependency: str) -> None: self.node = node self.dependency = dependency super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = f"'{self.node.unique_id}' depends on '{self.dependency}' which is not in the graph!" return msg class ForeignKeyConstraintToSyntaxError(CompilationError): def __init__(self, node, expression: str) -> None: self.expression = expression self.node = node super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = f"'{self.node.unique_id}' defines a foreign key constraint 'to' expression which is not valid 'ref' or 'source' syntax: {self.expression}." return msg # client level exceptions class NoSupportedLanguagesFoundError(CompilationError): def __init__(self, node) -> None: self.node = node self.msg = f"No supported_languages found in materialization macro {self.node.name}" super().__init__(msg=self.msg) class MaterializtionMacroNotUsedError(CompilationError): def __init__(self, node) -> None: self.node = node self.msg = "Only materialization macros can be used with this function" super().__init__(msg=self.msg) class MacroNamespaceNotStringError(CompilationError): def __init__(self, kwarg_type: Any) -> None: self.kwarg_type = kwarg_type super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = ( "The macro_namespace parameter to adapter.dispatch " f"is a {self.kwarg_type}, not a string" ) return msg class UnknownGitCloningProblemError(DbtRuntimeError): def __init__(self, repo: str) -> None: self.repo = scrub_secrets(repo, env_secrets()) super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = f"""\ Something went wrong while cloning {self.repo} Check the debug logs for more information """ return msg class NoAdaptersAvailableError(DbtRuntimeError): def __init__(self) -> None: super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = "No adapters available. Learn how to install an adapter by going to https://docs.getdbt.com/docs/connect-adapters#install-using-the-cli" return msg class BadSpecError(DbtInternalError): def __init__(self, repo, revision, error) -> None: self.repo = repo self.revision = revision self.stderr = scrub_secrets(error.stderr.strip(), env_secrets()) super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = f"Error checking out spec='{self.revision}' for repo {self.repo}\n{self.stderr}" return msg class GitCloningError(DbtInternalError): def __init__(self, repo: str, revision: str, error: CommandResultError) -> None: self.repo = repo self.revision = revision self.error = error super().__init__(msg=self.get_message()) def get_message(self) -> str: stderr = self.error.stderr.strip() if "usage: git" in stderr: stderr = stderr.split("\nusage: git")[0] if re.match("fatal: destination path '(.+)' already exists", stderr): self.error.cmd = list(scrub_secrets(str(self.error.cmd), env_secrets())) raise self.error msg = f"Error checking out spec='{self.revision}' for repo {self.repo}\n{stderr}" return scrub_secrets(msg, env_secrets()) class GitCheckoutError(BadSpecError): pass class OperationError(CompilationError): def __init__(self, operation_name) -> None: self.operation_name = operation_name super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = ( f"dbt encountered an error when attempting to create a {self.operation_name}. " "If this error persists, please create an issue at: \n\n" "https://github.com/dbt-labs/dbt-core" ) return msg # context level exceptions class ZipStrictWrongTypeError(CompilationError): def __init__(self, exc) -> None: self.exc = exc msg = str(self.exc) super().__init__(msg=msg) class SetStrictWrongTypeError(CompilationError): def __init__(self, exc) -> None: self.exc = exc msg = str(self.exc) super().__init__(msg=msg) class LoadAgateTableValueError(CompilationError): def __init__(self, exc: ValueError, node) -> None: self.exc = exc self.node = node msg = str(self.exc) super().__init__(msg=msg) class LoadAgateTableNotSeedError(CompilationError): def __init__(self, resource_type, node) -> None: self.resource_type = resource_type self.node = node msg = f"can only load_agate_table for seeds (got a {self.resource_type})" super().__init__(msg=msg) class PackageNotInDepsError(CompilationError): def __init__(self, package_name: str, node) -> None: self.package_name = package_name self.node = node msg = f"Node package named {self.package_name} not found!" super().__init__(msg=msg) class OperationsCannotRefEphemeralNodesError(CompilationError): def __init__(self, target_name: str, node) -> None: self.target_name = target_name self.node = node msg = f"Operations can not ref() ephemeral nodes, but {target_name} is ephemeral" super().__init__(msg=msg) class PersistDocsValueTypeError(CompilationError): def __init__(self, persist_docs: Any) -> None: self.persist_docs = persist_docs msg = ( "Invalid value provided for 'persist_docs'. Expected dict " f"but received {type(self.persist_docs)}" ) super().__init__(msg=msg) class InlineModelConfigError(CompilationError): def __init__(self, node) -> None: self.node = node msg = "Invalid inline model config" super().__init__(msg=msg) class ConflictingConfigKeysError(CompilationError): def __init__(self, oldkey: str, newkey: str, node) -> None: self.oldkey = oldkey self.newkey = newkey self.node = node msg = f'Invalid config, has conflicting keys "{self.oldkey}" and "{self.newkey}"' super().__init__(msg=msg) class NumberSourceArgsError(CompilationError): def __init__(self, args, node) -> None: self.args = args self.node = node msg = f"source() takes exactly two arguments ({len(self.args)} given)" super().__init__(msg=msg) class RequiredVarNotFoundError(CompilationError): def __init__(self, var_name: str, merged: Dict, node) -> None: self.var_name = var_name self.merged = merged self.node = node super().__init__(msg=self.get_message()) def get_message(self) -> str: if self.node is not None: node_name = self.node.name else: node_name = "" dct = {k: self.merged[k] for k in self.merged} pretty_vars = json.dumps(dct, sort_keys=True, indent=4) msg = f"Required var '{self.var_name}' not found in config:\nVars supplied to {node_name} = {pretty_vars}" return scrub_secrets(msg, self.var_secrets()) def var_secrets(self) -> List[str]: return [v for k, v in self.merged.items() if k.startswith(SECRET_ENV_PREFIX) and v.strip()] class PackageNotFoundForMacroError(CompilationError): def __init__(self, package_name: str) -> None: self.package_name = package_name msg = f"Could not find package '{self.package_name}'" super().__init__(msg=msg) class SecretEnvVarLocationError(ParsingError): def __init__(self, env_var_name: str) -> None: self.env_var_name = env_var_name super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = ( "Secret env vars are allowed only in profiles.yml or packages.yml. " f"Found '{self.env_var_name}' referenced elsewhere." ) return msg class BooleanError(CompilationError): def __init__(self, return_value: Any, macro_name: str) -> None: self.return_value = return_value self.macro_name = macro_name super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = ( f"Macro '{self.macro_name}' returns '{self.return_value}'. It is not type 'bool' " "and cannot not be converted reliably to a bool." ) return msg class RefArgsError(CompilationError): def __init__(self, node, args) -> None: self.node = node self.args = args super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = f"ref() takes at most two arguments ({len(self.args)} given)" return msg class MetricArgsError(CompilationError): def __init__(self, node, args) -> None: self.node = node self.args = args super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = f"metric() takes at most two arguments ({len(self.args)} given)" return msg class RefBadContextError(CompilationError): def __init__(self, node, args) -> None: self.node = node self.args = args.positional_args # type: ignore self.kwargs = args.keyword_args # type: ignore super().__init__(msg=self.get_message()) def get_message(self) -> str: # This explicitly references model['name'], instead of model['alias'], for # better error messages. Ex. If models foo_users and bar_users are aliased # to 'users', in their respective schemas, then you would want to see # 'bar_users' in your error messge instead of just 'users'. if isinstance(self.node, dict): model_name = self.node["name"] else: model_name = self.node.name ref_args = ", ".join("'{}'".format(a) for a in self.args) keyword_args = "" if self.kwargs: keyword_args = ", ".join( "{}='{}'".format(k, v) for k, v in self.kwargs.items() # type: ignore ) keyword_args = "," + keyword_args ref_string = f"{{{{ ref({ref_args}{keyword_args}) }}}}" msg = f"""dbt was unable to infer all dependencies for the model "{model_name}". This typically happens when ref() is placed within a conditional block. To fix this, add the following hint to the top of the model "{model_name}": -- depends_on: {ref_string}""" return msg class DocArgsError(CompilationError): def __init__(self, node, args) -> None: self.node = node self.args = args super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = f"doc() takes at most two arguments ({len(self.args)} given)" return msg class DocTargetNotFoundError(CompilationError): def __init__( self, node, target_doc_name: str, target_doc_package: Optional[str] = None ) -> None: self.node = node self.target_doc_name = target_doc_name self.target_doc_package = target_doc_package super().__init__(msg=self.get_message()) def get_message(self) -> str: target_package_string = "" if self.target_doc_package is not None: target_package_string = f"in package '{self.target_doc_package}' " msg = f"Documentation for '{self.node.unique_id}' depends on doc '{self.target_doc_name}' {target_package_string} which was not found" return msg class MacroDispatchArgError(CompilationError): def __init__(self, macro_name: str) -> None: self.macro_name = macro_name super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = f"""\ The "packages" argument of adapter.dispatch() has been deprecated. Use the "macro_namespace" argument instead. Raised during dispatch for: {self.macro_name} For more information, see: https://docs.getdbt.com/reference/dbt-jinja-functions/dispatch """ return msg class DuplicateMacroNameError(CompilationError): def __init__(self, node_1, node_2, namespace: str) -> None: self.node_1 = node_1 self.node_2 = node_2 self.namespace = namespace super().__init__(msg=self.get_message()) def get_message(self) -> str: duped_name = self.node_1.name if self.node_1.package_name != self.node_2.package_name: extra = f' ("{self.node_1.package_name}" and "{self.node_2.package_name}" are both in the "{self.namespace}" namespace)' else: extra = "" msg = ( f'dbt found two macros with the name "{duped_name}" in the namespace "{self.namespace}"{extra}. ' "Since these macros have the same name and exist in the same " "namespace, dbt will be unable to decide which to call. To fix this, " f"change the name of one of these macros:\n- {self.node_1.unique_id} " f"({self.node_1.original_file_path})\n- {self.node_2.unique_id} ({self.node_2.original_file_path})" ) return msg class MacroResultAlreadyLoadedError(CompilationError): def __init__(self, result_name) -> None: self.result_name = result_name super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = f"The 'statement' result named '{self.result_name}' has already been loaded into a variable" return msg # parser level exceptions class DictParseError(ParsingError): def __init__(self, exc: ValidationError, node) -> None: self.exc = exc self.node = node msg = self.validator_error_message(exc) super().__init__(msg=msg) class ConfigUpdateError(ParsingError): def __init__(self, exc: ValidationError, node) -> None: self.exc = exc self.node = node msg = self.validator_error_message(exc) super().__init__(msg=msg) class PythonParsingError(ParsingError): def __init__(self, exc: SyntaxError, node) -> None: self.exc = exc self.node = node super().__init__(msg=self.get_message()) def get_message(self) -> str: validated_exc = self.validator_error_message(self.exc) msg = f"{validated_exc}\n{self.exc.text}" return msg class PythonLiteralEvalError(ParsingError): def __init__(self, exc: Exception, node) -> None: self.exc = exc self.node = node super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = ( f"Error when trying to literal_eval an arg to dbt.ref(), dbt.source(), dbt.config() or dbt.config.get() \n{self.exc}\n" "https://docs.python.org/3/library/ast.html#ast.literal_eval\n" "In dbt python model, `dbt.ref`, `dbt.source`, `dbt.config`, `dbt.config.get` function args only support Python literal structures" ) return msg class ModelConfigError(ParsingError): def __init__(self, exc: ValidationError, node) -> None: self.msg = self.validator_error_message(exc) self.node = node super().__init__(msg=self.msg) class YamlParseListError(ParsingError): def __init__( self, path: str, key: str, yaml_data: List, cause, ) -> None: self.path = path self.key = key self.yaml_data = yaml_data self.cause = cause super().__init__(msg=self.get_message()) def get_message(self) -> str: if isinstance(self.cause, str): reason = self.cause elif isinstance(self.cause, ValidationError): reason = self.validator_error_message(self.cause) else: reason = self.cause.msg msg = f"Invalid {self.key} config given in {self.path} @ {self.key}: {self.yaml_data} - {reason}" return msg class YamlParseDictError(ParsingError): def __init__( self, path: str, key: str, yaml_data: Dict[str, Any], cause, ) -> None: self.path = path self.key = key self.yaml_data = yaml_data self.cause = cause super().__init__(msg=self.get_message()) def get_message(self) -> str: if isinstance(self.cause, str): reason = self.cause elif isinstance(self.cause, ValidationError): reason = self.validator_error_message(self.cause) else: reason = self.cause.msg msg = f"Invalid {self.key} config given in {self.path} @ {self.key}: {self.yaml_data} - {reason}" return msg class YamlLoadError(ParsingError): def __init__( self, path: str, exc: DbtValidationError, project_name: Optional[str] = None, ) -> None: self.project_name = project_name self.path = path self.exc = exc super().__init__(msg=self.get_message()) def get_message(self) -> str: reason = self.validator_error_message(self.exc) msg = f"Error reading {self.project_name}: {self.path} - {reason}" return msg class TestConfigError(ParsingError): def __init__(self, exc: ValidationError, node) -> None: self.msg = self.validator_error_message(exc) self.node = node super().__init__(msg=self.msg) class SchemaConfigError(ParsingError): def __init__(self, exc: ValidationError, node) -> None: self.msg = self.validator_error_message(exc) self.node = node super().__init__(msg=self.msg) class SnapshopConfigError(ParsingError): def __init__(self, exc: ValidationError, node) -> None: self.msg = self.validator_error_message(exc) self.node = node super().__init__(msg=self.msg) class DbtReferenceError(ParsingError): def __init__(self, unique_id: str, ref_unique_id: str, access: AccessType, scope: str) -> None: self.unique_id = unique_id self.ref_unique_id = ref_unique_id self.access = access self.scope = scope self.scope_type = "group" if self.access == AccessType.Private else "package" super().__init__(msg=self.get_message()) def get_message(self) -> str: return ( f"Node {self.unique_id} attempted to reference node {self.ref_unique_id}, " f"which is not allowed because the referenced node is {self.access} to the '{self.scope}' {self.scope_type}." ) class InvalidAccessTypeError(ParsingError): def __init__( self, unique_id: str, field_value: str, materialization: Optional[str] = None ) -> None: self.unique_id = unique_id self.field_value = field_value self.materialization = materialization with_materialization = ( f"with '{self.materialization}' materialization " if self.materialization else "" ) msg = f"Node {self.unique_id} {with_materialization}has an invalid value ({self.field_value}) for the access field" super().__init__(msg=msg) class InvalidUnitTestGivenInput(ParsingError): def __init__(self, input: str) -> None: msg = f"Unit test given inputs must be either a 'ref', 'source' or 'this' call. Got: '{input}'." super().__init__(msg=msg) class SameKeyNestedError(CompilationError): def __init__(self) -> None: msg = "Test cannot have the same key at the top-level and in config" super().__init__(msg=msg) class TestArgIncludesModelError(CompilationError): def __init__(self) -> None: msg = 'Test arguments include "model", which is a reserved argument' super().__init__(msg=msg) class UnexpectedTestNamePatternError(CompilationError): def __init__(self, test_name: str) -> None: self.test_name = test_name msg = f"Test name string did not match expected pattern: {self.test_name}" super().__init__(msg=msg) class CustomMacroPopulatingConfigValueError(CompilationError): def __init__( self, target_name: str, name: str, key: str, err_msg: str, column_name: Optional[str] = None, ) -> None: self.target_name = target_name self.column_name = column_name self.name = name self.key = key self.err_msg = err_msg super().__init__(msg=self.get_message()) def get_message(self) -> str: # Generic tests do not include custom macros in the Jinja # rendering context, so this will almost always fail. As it # currently stands, the error message is inscrutable, which # has caused issues for some projects migrating from # pre-0.20.0 to post-0.20.0. # See https://github.com/dbt-labs/dbt-core/issues/4103 # and https://github.com/dbt-labs/dbt-core/issues/5294 msg = ( f"The {self.target_name}.{self.column_name} column's " f'"{self.name}" test references an undefined ' f"macro in its {self.key} configuration argument. " f"The macro {self.err_msg}.\n" "Please note that the generic test configuration parser " "currently does not support using custom macros to " "populate configuration values" ) return msg class TagsNotListOfStringsError(CompilationError): def __init__(self, tags: Any) -> None: self.tags = tags msg = f"got {self.tags} ({type(self.tags)}) for tags, expected a list of strings" super().__init__(msg=msg) class TagNotStringError(CompilationError): def __init__(self, tag: Any) -> None: self.tag = tag msg = f"got {self.tag} ({type(self.tag)}) for tag, expected a str" super().__init__(msg=msg) class TestNameNotStringError(ParsingError): def __init__(self, test_name: Any) -> None: self.test_name = test_name super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = f"test name must be a str, got {type(self.test_name)} (value {self.test_name})" return msg class TestArgsNotDictError(ParsingError): def __init__(self, test_args: Any) -> None: self.test_args = test_args super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = f"test arguments must be a dict, got {type(self.test_args)} (value {self.test_args})" return msg class TestConfigNotDictError(ParsingError): def __init__(self, config: Any) -> None: self.config = config super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = f"'config' must be a dict, got {type(self.config)} (value {self.config})" return msg class TestDefinitionDictLengthError(ParsingError): def __init__(self, test): self.test = test super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = ( "test definition dictionary must have exactly one key, got" f" {self.test} instead ({len(self.test)} keys)" ) return msg class TestTypeError(ParsingError): def __init__(self, test: Any): self.test = test super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = f"test must be dict or str, got {type(self.test)} (value {self.test})" return msg # This is triggered across multiple files class EnvVarMissingError(ParsingError): def __init__(self, var: str): self.var = var super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = f"Env var required but not provided: '{self.var}'" return msg class TargetNotFoundError(CompilationError): def __init__( self, node, target_name: str, target_kind: str, target_package: Optional[str] = None, target_version: Optional[Union[str, float]] = None, disabled: Optional[bool] = None, ): self.node = node self.target_name = target_name self.target_kind = target_kind self.target_package = target_package self.target_version = target_version self.disabled = disabled super().__init__(msg=self.get_message()) def get_message(self) -> str: original_file_path = self.node.original_file_path unique_id = self.node.unique_id resource_type_title = self.node.resource_type.title() if self.disabled is None: reason = "was not found or is disabled" elif self.disabled is True: reason = "is disabled" else: reason = "was not found" target_version_string = "" if self.target_version is not None: target_version_string = f"with version '{self.target_version}' " target_package_string = "" if self.target_package is not None: target_package_string = f"in package or project '{self.target_package}' " msg = ( f"{resource_type_title} '{unique_id}' ({original_file_path}) depends on a " f"{self.target_kind} named '{self.target_name}' {target_version_string}{target_package_string}which {reason}" ) return msg class DuplicateSourcePatchNameError(CompilationError): def __init__(self, patch_1, patch_2): self.patch_1 = patch_1 self.patch_2 = patch_2 super().__init__(msg=self.get_message()) def get_message(self) -> str: name = f"{self.patch_1.overrides}.{self.patch_1.name}" fix = self._fix_dupe_msg( self.patch_1.path, self.patch_2.path, name, "sources", ) msg = ( f"dbt found two schema.yml entries for the same source named " f"{self.patch_1.name} in package {self.patch_1.overrides}. Sources may only be " f"overridden a single time. To fix this, {fix}" ) return msg class DuplicateMacroPatchNameError(CompilationError): def __init__(self, patch_1, existing_patch_path): self.patch_1 = patch_1 self.existing_patch_path = existing_patch_path super().__init__(msg=self.get_message()) def get_message(self) -> str: package_name = self.patch_1.package_name name = self.patch_1.name fix = self._fix_dupe_msg( self.patch_1.original_file_path, self.existing_patch_path, name, "macros" ) msg = ( f"dbt found two schema.yml entries for the same macro in package " f"{package_name} named {name}. Macros may only be described a single " f"time. To fix this, {fix}" ) return msg # core level exceptions class DuplicateAliasError(AliasError): def __init__(self, kwargs: Mapping[str, Any], aliases: Mapping[str, str], canonical_key: str): self.kwargs = kwargs self.aliases = aliases self.canonical_key = canonical_key super().__init__(msg=self.get_message()) def get_message(self) -> str: # dupe found: go through the dict so we can have a nice-ish error key_names = ", ".join( "{}".format(k) for k in self.kwargs if self.aliases.get(k) == self.canonical_key ) msg = f'Got duplicate keys: ({key_names}) all map to "{self.canonical_key}"' return msg # deps exceptions class MultipleVersionGitDepsError(DependencyError): def __init__(self, git: str, requested): self.git = git self.requested = requested msg = ( "git dependencies should contain exactly one version. " f"{self.git} contains: {self.requested}" ) super().__init__(msg) class DuplicateProjectDependencyError(DependencyError): def __init__(self, project_name: str): self.project_name = project_name msg = ( f'Found duplicate project "{self.project_name}". This occurs when ' "a dependency has the same project name as some other dependency." ) super().__init__(msg) class DuplicateDependencyToRootError(DependencyError): def __init__(self, project_name: str): self.project_name = project_name msg = ( "Found a dependency with the same name as the root project " f'"{self.project_name}". Package names must be unique in a project.' " Please rename one of these packages." ) super().__init__(msg) class MismatchedDependencyTypeError(DependencyError): def __init__(self, new, old): self.new = new self.old = old msg = ( f"Cannot incorporate {self.new} ({self.new.__class__.__name__}) in {self.old} " f"({self.old.__class__.__name__}): mismatched types" ) super().__init__(msg) class PackageVersionNotFoundError(DependencyError): def __init__( self, package_name: str, version_range, available_versions: List[str], should_version_check: bool, ): self.package_name = package_name self.version_range = version_range self.available_versions = available_versions self.should_version_check = should_version_check super().__init__(self.get_message()) def get_message(self) -> str: base_msg = ( "Could not find a matching compatible version for package {}\n" " Requested range: {}\n" " Compatible versions: {}\n" ) addendum = ( ( "\n" " Not shown: package versions incompatible with installed version of dbt-core\n" " To include them, run 'dbt --no-version-check deps'" ) if self.should_version_check else "" ) msg = ( base_msg.format(self.package_name, self.version_range, self.available_versions) + addendum ) return msg class PackageNotFoundError(DependencyError): def __init__(self, package_name: str): self.package_name = package_name msg = f"Package {self.package_name} was not found in the package index" super().__init__(msg) # config level exceptions class ProfileConfigError(DbtProfileError): def __init__(self, exc: ValidationError): self.exc = exc msg = self.validator_error_message(self.exc) super().__init__(msg=msg) class ProjectContractError(DbtProjectError): def __init__(self, exc: ValidationError): self.exc = exc msg = self.validator_error_message(self.exc) super().__init__(msg=msg) class ProjectContractBrokenError(DbtProjectError): def __init__(self, exc: ValidationError): self.exc = exc msg = self.validator_error_message(self.exc) super().__init__(msg=msg) class ConfigContractBrokenError(DbtProjectError): def __init__(self, exc: ValidationError): self.exc = exc msg = self.validator_error_message(self.exc) super().__init__(msg=msg) class NonUniquePackageNameError(CompilationError): def __init__(self, project_name: str): self.project_name = project_name super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = ( "dbt found more than one package with the name " f'"{self.project_name}" included in this project. Package ' "names must be unique in a project. Please rename " "one of these packages." ) return msg class UninstalledPackagesFoundError(CompilationError): def __init__( self, count_packages_installed: int, count_packages_specified: int, packages_specified_path: str, packages_install_path: str, uninstalled_packages: Tuple[str, ...] = tuple(), ): self.count_packages_specified = count_packages_specified self.count_packages_installed = count_packages_installed self.uninstalled_packages = uninstalled_packages self.packages_specified_path = packages_specified_path self.packages_install_path = packages_install_path super().__init__(msg=self.get_message()) def get_message(self) -> str: uninstalled_packages_str = ", ".join(self.uninstalled_packages) msg = ( f"dbt expects {self.count_packages_specified} package(s) " f"based on packages specified in {self.packages_specified_path}, but " f"found only {self.count_packages_installed} package(s) installed " f"in {self.packages_install_path}. " ) if self.uninstalled_packages: msg += f"Following packages were not found: {uninstalled_packages_str}. " msg += 'Run "dbt deps" to install package dependencies.' return msg class OptionNotYamlDictError(CompilationError): def __init__(self, var_type, option_name): self.var_type = var_type self.option_name = option_name super().__init__(msg=self.get_message()) def get_message(self) -> str: type_name = self.var_type.__name__ msg = f"The --{self.option_name} argument must be a YAML dictionary, but was of type '{type_name}'" return msg # contracts level class UnrecognizedCredentialTypeError(CompilationError): def __init__(self, typename: str, supported_types: List): self.typename = typename self.supported_types = supported_types super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = 'Unrecognized credentials type "{}" - supported types are ({})'.format( self.typename, ", ".join('"{}"'.format(t) for t in self.supported_types) ) return msg # jinja exceptions class PatchTargetNotFoundError(CompilationError): def __init__(self, patches: Dict): self.patches = patches super().__init__(msg=self.get_message()) def get_message(self) -> str: patch_list = "\n\t".join( f"model {p.name} (referenced in path {p.original_file_path})" for p in self.patches.values() ) msg = f"dbt could not find models for the following patches:\n\t{patch_list}" return msg class MissingRelationError(CompilationError): def __init__(self, relation, model=None): self.relation = relation self.model = model msg = f"Relation {self.relation} not found!" super().__init__(msg=msg) class AmbiguousAliasError(CompilationError): def __init__(self, node_1, node_2, duped_name=None): self.node_1 = node_1 self.node_2 = node_2 if duped_name is None: self.duped_name = f"{self.node_1.database}.{self.node_1.schema}.{self.node_1.alias}" else: self.duped_name = duped_name super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = ( f'dbt found two resources with the database representation "{self.duped_name}".\ndbt ' "cannot create two resources with identical database representations. " "To fix this,\nchange the configuration of one of these resources:" f"\n- {self.node_1.unique_id} ({self.node_1.original_file_path})\n- {self.node_2.unique_id} ({self.node_2.original_file_path})" ) return msg class AmbiguousResourceNameRefError(CompilationError): def __init__(self, duped_name, unique_ids, node=None): self.duped_name = duped_name self.unique_ids = unique_ids self.packages = [unique_id.split(".")[1] for unique_id in unique_ids] super().__init__(msg=self.get_message(), node=node) def get_message(self) -> str: formatted_unique_ids = "'{0}'".format("', '".join(self.unique_ids)) formatted_packages = "'{0}'".format("' or '".join(self.packages)) msg = ( f"When referencing '{self.duped_name}', dbt found nodes in multiple packages: {formatted_unique_ids}" f"\nTo fix this, use two-argument 'ref', with the package name first: {formatted_packages}" ) return msg class AmbiguousCatalogMatchError(CompilationError): def __init__(self, unique_id: str, match_1, match_2): self.unique_id = unique_id self.match_1 = match_1 self.match_2 = match_2 super().__init__(msg=self.get_message()) def get_match_string(self, match): match_schema = match.get("metadata", {}).get("schema") match_name = match.get("metadata", {}).get("name") return f"{match_schema}.{match_name}" def get_message(self) -> str: msg = ( "dbt found two relations in your warehouse with similar database identifiers. " "dbt\nis unable to determine which of these relations was created by the model " f'"{self.unique_id}".\nIn order for dbt to correctly generate the catalog, one ' "of the following relations must be deleted or renamed:\n\n - " f"{self.get_match_string(self.match_1)}\n - {self.get_match_string(self.match_2)}" ) return msg class DependencyNotFoundError(CompilationError): def __init__(self, node, node_description, required_pkg): self.node = node self.node_description = node_description self.required_pkg = required_pkg super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = ( f"Error while parsing {self.node_description}.\nThe required package " f'"{self.required_pkg}" was not found. Is the package installed?\n' "Hint: You may need to run `dbt deps`." ) return msg class DuplicatePatchPathError(CompilationError): def __init__(self, patch_1, existing_patch_path): self.patch_1 = patch_1 self.existing_patch_path = existing_patch_path super().__init__(msg=self.get_message()) def get_message(self) -> str: name = self.patch_1.name fix = self._fix_dupe_msg( self.patch_1.original_file_path, self.existing_patch_path, name, "resource", ) msg = ( f"dbt found two schema.yml entries for the same resource named " f"{name}. Resources and their associated columns may only be " f"described a single time. To fix this, {fix}" ) return msg # should this inherit ParsingError instead? class DuplicateResourceNameError(CompilationError): def __init__(self, node_1, node_2): self.node_1 = node_1 self.node_2 = node_2 super().__init__(msg=self.get_message()) def get_message(self) -> str: duped_name = self.node_1.name node_type = NodeType(self.node_1.resource_type) pluralized = ( node_type.pluralize() if self.node_1.resource_type == self.node_2.resource_type else "resources" # still raise if ref() collision, e.g. model + seed ) action = "looking for" # duplicate 'ref' targets if node_type in REFABLE_NODE_TYPES: formatted_name = f'ref("{duped_name}")' # duplicate sources elif node_type == NodeType.Source: duped_name = self.node_1.get_full_source_name() formatted_name = self.node_1.get_source_representation() # duplicate docs blocks elif node_type == NodeType.Documentation: formatted_name = f'doc("{duped_name}")' # duplicate generic tests elif node_type == NodeType.Test and hasattr(self.node_1, "test_metadata"): column_name = ( f'column "{self.node_1.column_name}" in ' if self.node_1.column_name else "" ) model_name = self.node_1.file_key_name duped_name = f'{self.node_1.name}" defined on {column_name}"{model_name}' action = "running" formatted_name = "tests" # all other resource types else: formatted_name = duped_name msg = f""" dbt found two {pluralized} with the name "{duped_name}". Since these resources have the same name, dbt will be unable to find the correct resource when {action} {formatted_name}. To fix this, change the name of one of these resources: - {self.node_1.unique_id} ({self.node_1.original_file_path}) - {self.node_2.unique_id} ({self.node_2.original_file_path}) """.strip() return msg class DuplicateVersionedUnversionedError(ParsingError): def __init__(self, versioned_node, unversioned_node): self.versioned_node = versioned_node self.unversioned_node = unversioned_node super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = f""" dbt found versioned and unversioned models with the name "{self.versioned_node.name}". Since these resources have the same name, dbt will be unable to find the correct resource when looking for ref('{self.versioned_node.name}'). To fix this, change the name of the unversioned resource {self.unversioned_node.unique_id} ({self.unversioned_node.original_file_path}) or add the unversioned model to the versions in {self.versioned_node.patch_path} """.strip() return msg class PropertyYMLError(CompilationError): def __init__(self, path: str, issue: str): self.path = path self.issue = issue super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = ( f"The yml property file at {self.path} is invalid because {self.issue}. " "Please consult the documentation for more information on yml property file " "syntax:\n\nhttps://docs.getdbt.com/reference/configs-and-properties" ) return msg class ContractError(CompilationError): def __init__(self, yaml_columns, sql_columns): self.yaml_columns = yaml_columns self.sql_columns = sql_columns super().__init__(msg=self.get_message()) def get_mismatches(self) -> "agate.Table": # avoid a circular import from dbt_common.clients.agate_helper import table_from_data_flat column_names = ["column_name", "definition_type", "contract_type", "mismatch_reason"] # list of mismatches mismatches: List[Dict[str, str]] = [] # track sql cols so we don't need another for loop later sql_col_set = set() # for each sql col list for sql_col in self.sql_columns: # add sql col to set sql_col_set.add(sql_col["name"]) # for each yaml col list for i, yaml_col in enumerate(self.yaml_columns): # if name matches if sql_col["name"] == yaml_col["name"]: # if type matches if sql_col["data_type"] == yaml_col["data_type"]: # its a perfect match! don't include in mismatch table break else: # same name, diff type row = [ sql_col["name"], sql_col["data_type"], yaml_col["data_type"], "data type mismatch", ] mismatches += [dict(zip(column_names, row))] break # if last loop, then no name match if i == len(self.yaml_columns) - 1: row = [sql_col["name"], sql_col["data_type"], "", "missing in contract"] mismatches += [dict(zip(column_names, row))] # now add all yaml cols without a match for yaml_col in self.yaml_columns: if yaml_col["name"] not in sql_col_set: row = [yaml_col["name"], "", yaml_col["data_type"], "missing in definition"] mismatches += [dict(zip(column_names, row))] mismatches_sorted = sorted(mismatches, key=lambda d: d["column_name"]) return table_from_data_flat(mismatches_sorted, column_names) def get_message(self) -> str: if not self.yaml_columns: return ( "This model has an enforced contract, and its 'columns' specification is missing" ) table: "agate.Table" = self.get_mismatches() # Hack to get Agate table output as string output = io.StringIO() table.print_table(output=output, max_rows=None, max_column_width=50) # type: ignore mismatches = output.getvalue() msg = ( "This model has an enforced contract that failed.\n" "Please ensure the name, data_type, and number of columns in your contract " "match the columns in your model's definition.\n\n" f"{mismatches}" ) return msg # not modifying these since rpc should be deprecated soon class UnknownAsyncIDException(Exception): CODE = 10012 MESSAGE = "RPC server got an unknown async ID" def __init__(self, task_id): self.task_id = task_id def __str__(self): return f"{self.MESSAGE}: {self.task_id}" class RPCFailureResult(DbtRuntimeError): CODE = 10002 MESSAGE = "RPC execution error" class RPCTimeoutException(DbtRuntimeError): CODE = 10008 MESSAGE = "RPC timeout error" def __init__(self, timeout: Optional[float] = None): super().__init__(self.MESSAGE) self.timeout = timeout def data(self): result = super().data() result.update( { "timeout": self.timeout, "message": f"RPC timed out after {self.timeout}s", } ) return result class RPCKilledException(DbtRuntimeError): CODE = 10009 MESSAGE = "RPC process killed" def __init__(self, signum: int): self.signum = signum self.msg = f"RPC process killed by signal {self.signum}" super().__init__(self.msg) def data(self): return { "signum": self.signum, "message": self.msg, } class RPCCompiling(DbtRuntimeError): CODE = 10010 MESSAGE = 'RPC server is compiling the project, call the "status" method for' " compile status" def __init__(self, msg: Optional[str] = None, node=None): if msg is None: msg = "compile in progress" super().__init__(msg, node) class RPCLoadException(DbtRuntimeError): CODE = 10011 MESSAGE = ( 'RPC server failed to compile project, call the "status" method for' " compile status" ) def __init__(self, cause: Dict[str, Any]): self.cause = cause self.msg = f'{self.MESSAGE}: {self.cause["message"]}' super().__init__(self.msg) def data(self): return {"cause": self.cause, "message": self.msg} ================================================ FILE: core/dbt/flags.py ================================================ # Do not import the os package because we expose this package in jinja from argparse import Namespace from pathlib import Path # this roughly follows the patten of EVENT_MANAGER in dbt/common/events/functions.py # During de-globlization, we'll need to handle both similarly # Match USE_COLORS default with default in dbt.cli.params.use_colors for use in --version GLOBAL_FLAGS = Namespace(USE_COLORS=True) # type: ignore def set_flags(flags): global GLOBAL_FLAGS GLOBAL_FLAGS = flags def get_flags(): return GLOBAL_FLAGS def set_from_args(args: Namespace, project_flags): global GLOBAL_FLAGS from dbt.cli.flags import Flags, convert_config from dbt.cli.main import cli # we set attributes of args after initialize the flags, but project_flags # is being read in the Flags constructor, so we need to read it here and pass in # to make sure we use the correct project_flags profiles_dir = getattr(args, "PROFILES_DIR", None) or getattr(args, "profiles_dir", None) project_dir = getattr(args, "PROJECT_DIR", None) or getattr(args, "project_dir", None) if profiles_dir and project_dir: from dbt.config.project import read_project_flags project_flags = read_project_flags(project_dir, profiles_dir) # make a dummy context to get the flags, totally arbitrary ctx = cli.make_context("run", ["run"]) flags = Flags(ctx, project_flags) for arg_name, args_param_value in vars(args).items(): args_param_value = convert_config(arg_name, args_param_value) object.__setattr__(flags, arg_name.upper(), args_param_value) object.__setattr__(flags, arg_name.lower(), args_param_value) flags.set_common_global_flags() GLOBAL_FLAGS = flags # type: ignore def get_flag_dict(): flag_attr = { "use_experimental_parser", "static_parser", "warn_error", "warn_error_options", "write_json", "partial_parse", "use_colors", "profiles_dir", "debug", "log_format", "version_check", "fail_fast", "send_anonymous_usage_stats", "printer_width", "indirect_selection", "log_cache_events", "quiet", "no_print", "cache_selected_only", "introspect", "target_path", "log_path", "invocation_command", "empty", } return {key: getattr(GLOBAL_FLAGS, key.upper(), None) for key in flag_attr} # This is used by core/dbt/context/base.py to return a flag object # in Jinja. def get_flag_obj(): new_flags = Namespace() for key, val in get_flag_dict().items(): if isinstance(val, Path): val = str(val) setattr(new_flags, key.upper(), val) # The following 3 are CLI arguments only so they're not full-fledged flags, # but we put in flags for users. setattr(new_flags, "FULL_REFRESH", getattr(GLOBAL_FLAGS, "FULL_REFRESH", None)) setattr(new_flags, "STORE_FAILURES", getattr(GLOBAL_FLAGS, "STORE_FAILURES", None)) setattr(new_flags, "WHICH", getattr(GLOBAL_FLAGS, "WHICH", None)) # Project-level behavior change flag, exposed to Jinja for gating in adapter macros. setattr( new_flags, "REQUIRE_SQL_HEADER_IN_TEST_CONFIGS", getattr(GLOBAL_FLAGS, "REQUIRE_SQL_HEADER_IN_TEST_CONFIGS", False), ) return new_flags ================================================ FILE: core/dbt/graph/README.md ================================================ # Graph README ## Graph Selector Creation ### Selector Loading During dbt execution, the `@requires.project` decorator creates the final selector objects used in the graph. The `SelectorConfig` class loads selectors from the project configuration, while the `selector_config_from_data` function parses these selectors. #### Indirect Selection Default Value In `@requires.preflight`, dbt reads CLI flags, environment variables, and the parameter's default value. It resolves these inputs based on their precedence order and stores the resolved value in global flags. When loading selectors, the [`selection_criteria_from_dict`](https://github.com/dbt-labs/dbt-core/blob/b316c5f18021fef3d7fd6ec255427054b7d2205e/core/dbt/graph/selector_spec.py#L111) function resolves the indirect selection value to the global flags value if not set. This ensures correct resolution of the indirect selection value. ================================================ FILE: core/dbt/graph/__init__.py ================================================ from .cli import parse_difference, parse_from_selectors_definition # noqa: F401 from .graph import Graph, UniqueId # noqa: F401 from .queue import GraphQueue # noqa: F401 from .selector import NodeSelector, ResourceTypeSelector # noqa: F401 from .selector_spec import ( # noqa: F401 SelectionCriteria, SelectionDifference, SelectionIntersection, SelectionSpec, SelectionUnion, ) ================================================ FILE: core/dbt/graph/cli.py ================================================ # special support for CLI argument parsing. # TODO: Remove as part of https://github.com/dbt-labs/dbt-core/issues/6701 import itertools from copy import deepcopy from typing import Any, Dict, List, Optional, Tuple, Union from dbt.clients.yaml_helper import Dumper, Loader, yaml # noqa: F401 from dbt.contracts.selection import SelectorDefinition, SelectorFile from dbt.flags import get_flags from dbt_common.exceptions import DbtInternalError, DbtValidationError from .selector_spec import ( IndirectSelection, SelectionCriteria, SelectionDifference, SelectionIntersection, SelectionSpec, SelectionUnion, ) INTERSECTION_DELIMITER = "," DEFAULT_INCLUDES: List[str] = ["fqn:*", "source:*", "exposure:*", "metric:*", "semantic_model:*"] DEFAULT_EXCLUDES: List[str] = [] def parse_union( components: List[str], expect_exists: bool, ) -> SelectionUnion: # turn ['a b', 'c'] -> ['a', 'b', 'c'] raw_specs = itertools.chain.from_iterable(r.split(" ") for r in components) union_components: List[SelectionSpec] = [] flags = get_flags() # ['a', 'b', 'c,d'] -> union('a', 'b', intersection('c', 'd')) for raw_spec in raw_specs: intersection_components: List[SelectionSpec] = [ SelectionCriteria.from_single_spec(part) for part in raw_spec.split(INTERSECTION_DELIMITER) ] union_components.append( SelectionIntersection( components=intersection_components, expect_exists=expect_exists, raw=raw_spec, indirect_selection=IndirectSelection(flags.INDIRECT_SELECTION), ) ) return SelectionUnion( components=union_components, expect_exists=False, raw=components, indirect_selection=IndirectSelection(flags.INDIRECT_SELECTION), ) def parse_union_from_default(raw: Optional[List[str]], default: List[str]) -> SelectionUnion: components: List[str] expect_exists: bool if raw is None: return parse_union(components=default, expect_exists=False) else: return parse_union(components=raw, expect_exists=True) def parse_difference( include: Optional[List[str]], exclude: Optional[List[str]] ) -> SelectionDifference: if include == (): include = None included = parse_union_from_default(include, DEFAULT_INCLUDES) excluded = parse_union_from_default(exclude, DEFAULT_EXCLUDES) return SelectionDifference(components=[included, excluded]) RawDefinition = Union[str, Dict[str, Any]] def _get_list_dicts(dct: Dict[str, Any], key: str) -> List[RawDefinition]: result: List[RawDefinition] = [] if key not in dct: raise DbtInternalError(f"Expected to find key {key} in dict, only found {list(dct)}") values = dct[key] if not isinstance(values, list): raise DbtValidationError(f'Invalid value for key "{key}". Expected a list.') for value in values: if isinstance(value, dict): for value_key in value: if not isinstance(value_key, str): raise DbtValidationError( f'Expected all keys to "{key}" dict to be strings, ' f'but "{value_key}" is a "{type(value_key)}"' ) result.append(value) elif isinstance(value, str): result.append(value) else: raise DbtValidationError( f'Invalid value type {type(value)} in key "{key}", expected ' f"dict or str (value: {value})." ) return result def _parse_exclusions(definition, result={}) -> Optional[SelectionSpec]: exclusions = _get_list_dicts(definition, "exclude") parsed_exclusions = [parse_from_definition(excl, result=result) for excl in exclusions] if len(parsed_exclusions) == 1: return parsed_exclusions[0] elif len(parsed_exclusions) > 1: return SelectionUnion(components=parsed_exclusions, raw=exclusions) else: return None def _parse_include_exclude_subdefs( definitions: List[RawDefinition], result={} ) -> Tuple[List[SelectionSpec], Optional[SelectionSpec]]: include_parts: List[SelectionSpec] = [] diff_arg: Optional[SelectionSpec] = None for definition in definitions: if isinstance(definition, dict) and "exclude" in definition: # do not allow multiple exclude: defs at the same level if diff_arg is not None: yaml_sel_cfg = yaml.dump(definition) raise DbtValidationError( f"You cannot provide multiple exclude arguments to the " f"same selector set operator:\n{yaml_sel_cfg}" ) diff_arg = _parse_exclusions(definition, result=result) else: include_parts.append(parse_from_definition(definition, result=result)) return (include_parts, diff_arg) def parse_union_definition(definition: Dict[str, Any], result={}) -> SelectionSpec: union_def_parts = _get_list_dicts(definition, "union") include, exclude = _parse_include_exclude_subdefs(union_def_parts, result=result) union = SelectionUnion(components=include) if exclude is None: union.raw = definition return union else: return SelectionDifference(components=[union, exclude], raw=definition) def parse_intersection_definition(definition: Dict[str, Any], result={}) -> SelectionSpec: intersection_def_parts = _get_list_dicts(definition, "intersection") include, exclude = _parse_include_exclude_subdefs(intersection_def_parts, result=result) intersection = SelectionIntersection(components=include) if exclude is None: intersection.raw = definition return intersection else: return SelectionDifference(components=[intersection, exclude], raw=definition) def parse_dict_definition(definition: Dict[str, Any], result={}) -> SelectionSpec: diff_arg: Optional[SelectionSpec] = None if len(definition) == 1: key = list(definition)[0] value = definition[key] if not isinstance(key, str): raise DbtValidationError( f'Expected definition key to be a "str", got one of type ' f'"{type(key)}" ({key})' ) dct = { "method": key, "value": value, } elif "method" in definition and "value" in definition: dct = definition if "exclude" in definition: diff_arg = _parse_exclusions(definition, result=result) dct = {k: v for k, v in dct.items() if k != "exclude"} else: raise DbtValidationError( f'Expected either 1 key or else "method" ' f'and "value" keys, but got {list(definition)}' ) # if key isn't a valid method name, this will raise base = SelectionCriteria.selection_criteria_from_dict(definition, dct) if diff_arg is None: return base else: return SelectionDifference(components=[base, diff_arg]) def parse_from_definition( definition: RawDefinition, rootlevel=False, result: Dict[str, Dict[str, Union[SelectionSpec, bool]]] = {}, ) -> SelectionSpec: if ( isinstance(definition, dict) and ("union" in definition or "intersection" in definition) and rootlevel and len(definition) > 1 ): keys = ",".join(definition.keys()) raise DbtValidationError( f"Only a single 'union' or 'intersection' key is allowed " f"in a root level selector definition; found {keys}." ) if isinstance(definition, str): return SelectionCriteria.from_single_spec(definition) elif "union" in definition: return parse_union_definition(definition, result=result) elif "intersection" in definition: return parse_intersection_definition(definition, result=result) elif isinstance(definition, dict): return parse_dict_definition(definition, result=result) else: raise DbtValidationError( f"Expected to find union, intersection, str or dict, instead " f"found {type(definition)}: {definition}" ) def parse_from_selectors_definition( source: SelectorFile, ) -> Dict[str, Dict[str, Union[SelectionSpec, bool]]]: result: Dict[str, Dict[str, Union[SelectionSpec, bool]]] = {} selector: SelectorDefinition for selector in source.selectors: result[selector.name] = { "default": selector.default, "definition": parse_from_definition( selector.definition, rootlevel=True, result=deepcopy(result) ), } return result ================================================ FILE: core/dbt/graph/graph.py ================================================ from functools import partial from itertools import product from typing import Iterable, Iterator, NewType, Optional, Set import networkx as nx # type: ignore from dbt_common.exceptions import DbtInternalError UniqueId = NewType("UniqueId", str) class Graph: """A wrapper around the networkx graph that understands SelectionCriteria and how they interact with the graph. """ def __init__(self, graph) -> None: self.graph: nx.DiGraph = graph def nodes(self) -> Set[UniqueId]: return set(self.graph.nodes()) def edges(self): return self.graph.edges() def __iter__(self) -> Iterator[UniqueId]: return iter(self.graph.nodes()) def ancestors(self, node: UniqueId, max_depth: Optional[int] = None) -> Set[UniqueId]: """Returns all nodes having a path to `node` in `graph`""" if not self.graph.has_node(node): raise DbtInternalError(f"Node {node} not found in the graph!") filtered_graph = self.exclude_edge_type("parent_test") return { child for _, child in nx.bfs_edges(filtered_graph, node, reverse=True, depth_limit=max_depth) } def descendants(self, node: UniqueId, max_depth: Optional[int] = None) -> Set[UniqueId]: """Returns all nodes reachable from `node` in `graph`""" if not self.graph.has_node(node): raise DbtInternalError(f"Node {node} not found in the graph!") filtered_graph = self.exclude_edge_type("parent_test") return {child for _, child in nx.bfs_edges(filtered_graph, node, depth_limit=max_depth)} def exclude_edge_type(self, edge_type_to_exclude): return nx.subgraph_view( self.graph, filter_edge=partial(self.filter_edges_by_type, edge_type=edge_type_to_exclude), ) def filter_edges_by_type(self, first_node, second_node, edge_type): return self.graph.get_edge_data(first_node, second_node).get("edge_type") != edge_type def select_childrens_parents(self, selected: Set[UniqueId]) -> Set[UniqueId]: ancestors_for = self.select_children(selected) | selected return self.select_parents(ancestors_for) | ancestors_for def select_children( self, selected: Set[UniqueId], max_depth: Optional[int] = None ) -> Set[UniqueId]: """Returns all nodes which are descendants of the 'selected' set. Nodes in the 'selected' set are counted as children only if they are descendants of other nodes in the 'selected' set.""" children: Set[UniqueId] = set() i = 0 while len(selected) > 0 and (max_depth is None or i < max_depth): next_layer: Set[UniqueId] = set() for node in selected: next_layer.update( iter( e[1] for e in self.graph.out_edges(node) if e[1] not in children and self.filter_edges_by_type(e[0], e[1], "parent_test") ) ) children.update(next_layer) selected = next_layer i += 1 return children def select_parents( self, selected: Set[UniqueId], max_depth: Optional[int] = None ) -> Set[UniqueId]: """Returns all nodes which are ancestors of the 'selected' set. Nodes in the 'selected' set are counted as parents only if they are ancestors of other nodes in the 'selected' set.""" parents: Set[UniqueId] = set() i = 0 while len(selected) > 0 and (max_depth is None or i < max_depth): next_layer: Set[UniqueId] = set() for node in selected: next_layer.update( iter( e[0] for e in self.graph.in_edges(node) if e[0] not in parents and self.filter_edges_by_type(e[0], e[1], "parent_test") ) ) parents.update(next_layer) selected = next_layer i += 1 return parents def select_successors(self, selected: Set[UniqueId]) -> Set[UniqueId]: successors: Set[UniqueId] = set() for node in selected: successors.update(self.graph.successors(node)) return successors def get_subset_graph(self, selected: Iterable[UniqueId]) -> "Graph": """Create and return a new graph that is a shallow copy of the graph, but with only the nodes in include_nodes. Transitive edges across removed nodes are preserved as explicit new edges. """ new_graph: nx.DiGraph = self.graph.copy() include_nodes: Set[UniqueId] = set(selected) still_removing: bool = True while still_removing: nodes_to_remove = list( node for node in new_graph if node not in include_nodes and (new_graph.in_degree(node) * new_graph.out_degree(node)) == 0 ) if len(nodes_to_remove) == 0: still_removing = False else: new_graph.remove_nodes_from(nodes_to_remove) # sort remaining nodes by degree remaining_nodes = list(new_graph.nodes()) remaining_nodes.sort( key=lambda node: new_graph.in_degree(node) * new_graph.out_degree(node) ) for node in remaining_nodes: if node not in include_nodes: source_nodes = [x for x, _ in new_graph.in_edges(node)] target_nodes = [x for _, x in new_graph.out_edges(node)] new_edges = product(source_nodes, target_nodes) non_cyclic_new_edges = [ (source, target) for source, target in new_edges if source != target and not new_graph.has_edge(source, target) ] # removes cyclic refs and edges already existing in new graph new_graph.add_edges_from(non_cyclic_new_edges) new_graph.remove_node(node) for node in include_nodes: if node not in new_graph: raise ValueError( "Couldn't find model '{}' -- does it exist or is it disabled?".format(node) ) return Graph(new_graph) def subgraph(self, nodes: Iterable[UniqueId]) -> "Graph": # Take the original networkx graph and return a subgraph containing only # the selected unique_id nodes. return Graph(self.graph.subgraph(nodes)) def get_dependent_nodes(self, node: UniqueId): return nx.descendants(self.graph, node) ================================================ FILE: core/dbt/graph/queue.py ================================================ import threading from queue import PriorityQueue from typing import Dict, Generator, List, Optional, Set import networkx as nx # type: ignore from dbt.contracts.graph.manifest import Manifest from dbt.contracts.graph.nodes import ( Exposure, GraphMemberNode, Metric, ModelNode, SourceDefinition, ) from dbt.node_types import NodeType from .graph import UniqueId class GraphQueue: """A fancy queue that is backed by the dependency graph. Note: this will mutate input! This queue is thread-safe for `mark_done` calls, though you must ensure that separate threads do not call `.empty()` or `__len__()` and `.get()` at the same time, as there is an unlocked race! """ def __init__( self, graph: nx.DiGraph, manifest: Manifest, selected: Set[UniqueId], preserve_edges: bool = True, ) -> None: # 'create_empty_copy' returns a copy of the graph G with all of the edges removed, and leaves nodes intact. self.graph = graph if preserve_edges else nx.classes.function.create_empty_copy(graph) self.manifest = manifest self._selected = selected # store the queue as a priority queue. self.inner: PriorityQueue = PriorityQueue() # things that have been popped off the queue but not finished # and worker thread reservations self.in_progress: Set[UniqueId] = set() # microbatch nodes that have been popped off the queue but not finished self.in_progress_microbatch: Set[UniqueId] = set() # things that are in the queue self.queued: Set[UniqueId] = set() # this lock controls most things self.lock = threading.Lock() # store the 'score' of each node as a number. Lower is higher priority. self._scores = self._get_scores(self.graph) # populate the initial queue self._find_new_additions(list(self.graph.nodes())) # awaits after task end self.some_task_done = threading.Condition(self.lock) def get_selected_nodes(self) -> Set[UniqueId]: return self._selected.copy() def _include_in_cost(self, node_id: UniqueId) -> bool: node = self.manifest.expect(node_id) if node.resource_type != NodeType.Model: return False # must be a Model - tell mypy this won't be a Source or Exposure or Metric assert not isinstance(node, (SourceDefinition, Exposure, Metric)) if node.is_ephemeral: return False return True @staticmethod def _grouped_topological_sort( graph: nx.DiGraph, ) -> Generator[List[str], None, None]: """Topological sort of given graph that groups ties. Adapted from `nx.topological_sort`, this function returns a topo sort of a graph however instead of arbitrarily ordering ties in the sort order, ties are grouped together in lists. Args: graph: The graph to be sorted. Returns: A generator that yields lists of nodes, one list per graph depth level. """ indegree_map = {v: d for v, d in graph.in_degree() if d > 0} zero_indegree = [v for v, d in graph.in_degree() if d == 0] while zero_indegree: yield zero_indegree new_zero_indegree = [] for v in zero_indegree: for _, child in graph.edges(v): indegree_map[child] -= 1 if not indegree_map[child]: new_zero_indegree.append(child) zero_indegree = new_zero_indegree def _get_scores(self, graph: nx.DiGraph) -> Dict[str, int]: """Scoring nodes for processing order. Scores are calculated by the graph depth level. Lowest score (0) should be processed first. Args: graph: The graph to be scored. Returns: A dictionary consisting of `node name`:`score` pairs. """ # split graph by connected subgraphs subgraphs = (graph.subgraph(x) for x in nx.connected_components(nx.Graph(graph))) # score all nodes in all subgraphs scores = {} for subgraph in subgraphs: grouped_nodes = self._grouped_topological_sort(subgraph) for level, group in enumerate(grouped_nodes): for node in group: scores[node] = level return scores def get(self, block: bool = True, timeout: Optional[float] = None) -> GraphMemberNode: """Get a node off the inner priority queue. By default, this blocks. This takes the lock, but only for part of it. :param block: If True, block until the inner queue has data :param timeout: If set, block for timeout seconds waiting for data. :return: The node as present in the manifest. See `queue.PriorityQueue` for more information on `get()` behavior and exceptions. """ _, node_id = self.inner.get(block=block, timeout=timeout) node = self.manifest.expect(node_id) is_microbatch = ( isinstance(node, ModelNode) and node.config.incremental_strategy == "microbatch" ) with self.lock: self._mark_in_progress(node_id, is_microbatch=is_microbatch) return node def __len__(self) -> int: """The length of the queue is the number of tasks left for the queue to give out, regardless of where they are. Incomplete tasks are not part of the length. This takes the lock. """ with self.lock: return len(self.graph) - len(self.in_progress) def empty(self) -> bool: """The graph queue is 'empty' if it all remaining nodes in the graph are in progress. This takes the lock. """ return len(self) == 0 def _already_known(self, node: UniqueId) -> bool: """Decide if a node is already known (either handed out as a task, or in the queue). Callers must hold the lock. :param str node: The node ID to check :returns bool: If the node is in progress/queued. """ return node in self.in_progress or node in self.queued def _find_new_additions(self, candidates) -> None: """Find any nodes in the graph that need to be added to the internal queue and add them. """ for node in candidates: if self.graph.in_degree(node) == 0 and not self._already_known(node): self.inner.put((self._scores[node], node)) self.queued.add(node) def mark_done(self, node_id: UniqueId) -> None: """Given a node's unique ID, mark it as done. This method takes the lock. :param str node_id: The node ID to mark as complete. """ with self.lock: self.in_progress.remove(node_id) if node_id in self.in_progress_microbatch: self.in_progress_microbatch.remove(node_id) successors = list(self.graph.successors(node_id)) self.graph.remove_node(node_id) self._find_new_additions(successors) self.inner.task_done() self.some_task_done.notify_all() def _mark_in_progress(self, node_id: UniqueId, is_microbatch: bool = False) -> None: """Mark the node as 'in progress'. Callers must hold the lock. :param str node_id: The node ID to mark as in progress. :param bool is_microbatch: Whether the node is a microbatch model. """ self.queued.remove(node_id) self.in_progress.add(node_id) if is_microbatch: self.in_progress_microbatch.add(node_id) def join(self) -> None: """Join the queue. Blocks until all tasks are marked as done. Make sure not to call this before the queue reports that it is empty. """ self.inner.join() def wait_until_something_was_done(self) -> int: """Block until a task is done, then return the number of unfinished tasks. """ with self.lock: self.some_task_done.wait() return self.inner.unfinished_tasks ================================================ FILE: core/dbt/graph/selector.py ================================================ from typing import Any, Dict, List, Optional, Set, Tuple from dbt import selected_resources from dbt.contracts.graph.manifest import Manifest from dbt.contracts.graph.nodes import GraphMemberNode from dbt.contracts.state import PreviousState from dbt.events.types import NoNodesForSelectionCriteria, SelectorReportInvalidSelector from dbt.exceptions import DbtInternalError, InvalidSelectorError from dbt.node_types import NodeType from dbt_common.events.functions import fire_event, warn_or_error from .graph import Graph, UniqueId from .queue import GraphQueue from .selector_methods import MethodManager, MethodName from .selector_spec import IndirectSelection, SelectionCriteria, SelectionSpec def get_package_names(nodes): return set([node.split(".")[1] for node in nodes]) def can_select_indirectly(node): """If a node is not selected itself, but its parent(s) are, it may qualify for indirect selection. Today, only Test nodes can be indirectly selected. In the future, other node types or invocation flags might qualify. """ if node.resource_type == NodeType.Test: return True elif node.resource_type == NodeType.Unit: return True else: return False class NodeSelector(MethodManager): """The node selector is aware of the graph and manifest""" def __init__( self, graph: Graph, manifest: Manifest, previous_state: Optional[PreviousState] = None, include_empty_nodes: bool = False, selectors: Optional[Dict[str, Any]] = None, ) -> None: super().__init__(manifest, previous_state) self.full_graph: Graph = graph self.include_empty_nodes: bool = include_empty_nodes self._selectors: Optional[Dict[str, Any]] = selectors # build a subgraph containing only non-empty, enabled nodes and enabled # sources. graph_members = { unique_id for unique_id in self.full_graph.nodes() if self._is_graph_member(unique_id) } self.graph = self.full_graph.subgraph(graph_members) def get_method(self, method: MethodName, method_arguments: List[str], **kwargs: Any): return super().get_method( method, method_arguments, selectors=self._selectors, get_selected_callback=self.get_selected, ) def select_included( self, included_nodes: Set[UniqueId], spec: SelectionCriteria, ) -> Set[UniqueId]: """Select the explicitly included nodes, using the given spec. Return the selected set of unique IDs. """ method = self.get_method( spec.method, spec.method_arguments, selectors=self._selectors, get_selected_callback=self.get_selected, ) return set(method.search(included_nodes, spec.value)) def get_nodes_from_criteria( self, spec: SelectionCriteria ) -> Tuple[Set[UniqueId], Set[UniqueId]]: """Get all nodes specified by the single selection criteria. - collect the directly included nodes - find their specified relatives - perform any selector-specific expansion """ nodes = self.graph.nodes() try: collected = self.select_included(nodes, spec) except InvalidSelectorError: valid_selectors = ", ".join(self.SELECTOR_METHODS) fire_event( SelectorReportInvalidSelector( valid_selectors=valid_selectors, spec_method=spec.method, raw_spec=spec.raw ) ) return set(), set() neighbors = self.collect_specified_neighbors(spec, collected) selected = collected | neighbors # if --indirect-selection EMPTY, do not expand to adjacent tests if spec.indirect_selection == IndirectSelection.Empty: return selected, set() else: direct_nodes, indirect_nodes = self.expand_selection( selected=selected, indirect_selection=spec.indirect_selection ) return direct_nodes, indirect_nodes def collect_specified_neighbors( self, spec: SelectionCriteria, selected: Set[UniqueId] ) -> Set[UniqueId]: """Given the set of models selected by the explicit part of the selector (like "tag:foo"), apply the modifiers on the spec ("+"/"@"). Return the set of additional nodes that should be collected (which may overlap with the selected set). """ additional: Set[UniqueId] = set() if spec.childrens_parents: additional.update(self.graph.select_childrens_parents(selected)) if spec.parents: depth = spec.parents_depth additional.update(self.graph.select_parents(selected, depth)) if spec.children: depth = spec.children_depth additional.update(self.graph.select_children(selected, depth)) return additional def select_nodes_recursively( self, spec: SelectionSpec, warn_on_no_nodes: bool = True ) -> Tuple[Set[UniqueId], Set[UniqueId]]: """If the spec is a composite spec (a union, difference, or intersection), recurse into its selections and combine them. If the spec is a concrete selection criteria, resolve that using the given graph. """ if isinstance(spec, SelectionCriteria): direct_nodes, indirect_nodes = self.get_nodes_from_criteria(spec) else: bundles = [ self.select_nodes_recursively(spec=component, warn_on_no_nodes=warn_on_no_nodes) for component in spec.components ] direct_sets = [] indirect_sets = [] for direct, indirect in bundles: direct_sets.append(direct) indirect_sets.append(direct | indirect) initial_direct = spec.combined(direct_sets) indirect_nodes = spec.combined(indirect_sets) direct_nodes = self.incorporate_indirect_nodes( initial_direct, indirect_nodes, spec.indirect_selection ) if spec.expect_exists and len(direct_nodes) == 0 and warn_on_no_nodes: warn_or_error(NoNodesForSelectionCriteria(spec_raw=str(spec.raw))) return direct_nodes, indirect_nodes def select_nodes( self, spec: SelectionSpec, warn_on_no_nodes: bool = True ) -> Tuple[Set[UniqueId], Set[UniqueId]]: """Select the nodes in the graph according to the spec. This is the main point of entry for turning a spec into a set of nodes: - Recurse through spec, select by criteria, combine by set operation - Return final (unfiltered) selection set """ direct_nodes, indirect_nodes = self.select_nodes_recursively( spec=spec, warn_on_no_nodes=warn_on_no_nodes ) indirect_only = indirect_nodes.difference(direct_nodes) return direct_nodes, indirect_only def _is_graph_member(self, unique_id: UniqueId) -> bool: if unique_id in self.manifest.sources: source = self.manifest.sources[unique_id] return source.config.enabled elif unique_id in self.manifest.exposures: return True elif unique_id in self.manifest.functions: function = self.manifest.functions[unique_id] return function.config.enabled elif unique_id in self.manifest.metrics: metric = self.manifest.metrics[unique_id] return metric.config.enabled elif unique_id in self.manifest.semantic_models: semantic_model = self.manifest.semantic_models[unique_id] return semantic_model.config.enabled elif unique_id in self.manifest.unit_tests: unit_test = self.manifest.unit_tests[unique_id] return unit_test.config.enabled elif unique_id in self.manifest.saved_queries: saved_query = self.manifest.saved_queries[unique_id] return saved_query.config.enabled elif unique_id in self.manifest.exposures: exposure = self.manifest.exposures[unique_id] return exposure.config.enabled else: node = self.manifest.nodes[unique_id] return node.config.enabled def _is_empty_node(self, unique_id: UniqueId) -> bool: if unique_id in self.manifest.nodes: node = self.manifest.nodes[unique_id] return node.empty else: return False def node_is_match(self, node: GraphMemberNode) -> bool: """Determine if a node is a match for the selector. Non-match nodes will be excluded from results during filtering. """ return True def _is_match(self, unique_id: UniqueId) -> bool: node: GraphMemberNode if unique_id in self.manifest.nodes: node = self.manifest.nodes[unique_id] elif unique_id in self.manifest.sources: node = self.manifest.sources[unique_id] elif unique_id in self.manifest.exposures: node = self.manifest.exposures[unique_id] elif unique_id in self.manifest.functions: node = self.manifest.functions[unique_id] elif unique_id in self.manifest.metrics: node = self.manifest.metrics[unique_id] elif unique_id in self.manifest.semantic_models: node = self.manifest.semantic_models[unique_id] elif unique_id in self.manifest.unit_tests: node = self.manifest.unit_tests[unique_id] elif unique_id in self.manifest.saved_queries: node = self.manifest.saved_queries[unique_id] else: raise DbtInternalError(f"Node {unique_id} not found in the manifest!") return self.node_is_match(node) def filter_selection(self, selected: Set[UniqueId]) -> Set[UniqueId]: """Return the subset of selected nodes that is a match for this selector. """ return { unique_id for unique_id in selected if self._is_match(unique_id) and (self.include_empty_nodes or not self._is_empty_node(unique_id)) } def expand_selection( self, selected: Set[UniqueId], indirect_selection: IndirectSelection = IndirectSelection.Eager, ) -> Tuple[Set[UniqueId], Set[UniqueId]]: # Test selection by default expands to include an implicitly/indirectly selected tests. # `dbt test -m model_a` also includes tests that directly depend on `model_a`. # Expansion has four modes, EAGER, CAUTIOUS and BUILDABLE, EMPTY. # # EAGER mode: If ANY parent is selected, select the test. # # CAUTIOUS mode: # - If ALL parents are selected, select the test. # - If ANY parent is missing, return it separately. We'll keep it around # for later and see if its other parents show up. # # BUILDABLE mode: # - If ALL parents are selected, or the parents of the test are themselves parents of the selected, select the test. # - If ANY parent is missing, return it separately. We'll keep it around # for later and see if its other parents show up. # # EMPTY mode: Only select the given node and ignore attached nodes (i.e. ignore tests attached to a model) # # Users can opt out of inclusive EAGER mode by passing --indirect-selection cautious # CLI argument or by specifying `indirect_selection: true` in a yaml selector direct_nodes = set(selected) indirect_nodes = set() selected_and_parents = set() if indirect_selection == IndirectSelection.Buildable: selected_and_parents = selected.union(self.graph.select_parents(selected)).union( self.manifest.sources ) for unique_id in self.graph.select_successors(selected): if unique_id in self.manifest.nodes or unique_id in self.manifest.unit_tests: if unique_id in self.manifest.nodes: node = self.manifest.nodes[unique_id] elif unique_id in self.manifest.unit_tests: node = self.manifest.unit_tests[unique_id] # type: ignore # Test nodes that are not selected themselves, but whose parents are selected. # (Does not include unit tests because they can only have one parent.) if can_select_indirectly(node): # should we add it in directly? if indirect_selection == IndirectSelection.Eager or set( node.depends_on_nodes ) <= set(selected): direct_nodes.add(unique_id) elif indirect_selection == IndirectSelection.Buildable and set( node.depends_on_nodes ) <= set(selected_and_parents): direct_nodes.add(unique_id) elif indirect_selection == IndirectSelection.Empty: pass else: indirect_nodes.add(unique_id) return direct_nodes, indirect_nodes def incorporate_indirect_nodes( self, direct_nodes: Set[UniqueId], indirect_nodes: Set[UniqueId] = set(), indirect_selection: IndirectSelection = IndirectSelection.Eager, ) -> Set[UniqueId]: # Check tests previously selected indirectly to see if ALL their # parents are now present. # performance: if identical, skip the processing below if set(direct_nodes) == set(indirect_nodes): return direct_nodes selected = set(direct_nodes) if indirect_selection == IndirectSelection.Cautious: for unique_id in indirect_nodes: if unique_id in self.manifest.nodes: node = self.manifest.nodes[unique_id] if set(node.depends_on_nodes) <= set(selected): selected.add(unique_id) elif indirect_selection == IndirectSelection.Buildable: selected_and_parents = selected.union(self.graph.select_parents(selected)) for unique_id in indirect_nodes: if unique_id in self.manifest.nodes: node = self.manifest.nodes[unique_id] if set(node.depends_on_nodes) <= set(selected_and_parents): selected.add(unique_id) return selected def get_selected(self, spec: SelectionSpec, warn_on_no_nodes: bool = True) -> Set[UniqueId]: """get_selected runs through the node selection process: - node selection. Based on the include/exclude sets, the set of matched unique IDs is returned - includes direct + indirect selection (for tests) - filtering: - selectors can filter the nodes after all of them have been selected """ selected_nodes, indirect_only = self.select_nodes( spec=spec, warn_on_no_nodes=warn_on_no_nodes ) filtered_nodes = self.filter_selection(selected_nodes) return filtered_nodes def get_graph_queue(self, spec: SelectionSpec, preserve_edges: bool = True) -> GraphQueue: """Returns a queue over nodes in the graph that tracks progress of dependencies. """ # Filtering happens in get_selected selected_nodes = self.get_selected(spec) # Save to global variable selected_resources.set_selected_resources(selected_nodes) # Construct a new graph using the selected_nodes new_graph = self.full_graph.get_subset_graph(selected_nodes) # should we give a way here for consumers to mutate the graph? return GraphQueue(new_graph.graph, self.manifest, selected_nodes, preserve_edges) class ResourceTypeSelector(NodeSelector): def __init__( self, graph: Graph, manifest: Manifest, previous_state: Optional[PreviousState], resource_types: List[NodeType], include_empty_nodes: bool = False, selectors: Optional[Dict[str, Any]] = None, ) -> None: super().__init__( graph=graph, manifest=manifest, previous_state=previous_state, include_empty_nodes=include_empty_nodes, selectors=selectors, ) self.resource_types: Set[NodeType] = set(resource_types) def node_is_match(self, node): return node.resource_type in self.resource_types ================================================ FILE: core/dbt/graph/selector_methods.py ================================================ import abc from fnmatch import fnmatch from itertools import chain from pathlib import Path from typing import ( TYPE_CHECKING, Any, Callable, Dict, Iterator, List, Optional, Set, Tuple, Type, Union, ) from dbt.contracts.graph.manifest import Manifest from dbt.contracts.graph.nodes import ( Exposure, FunctionNode, GenericTestNode, ManifestNode, Metric, ModelNode, ResultNode, SavedQuery, SemanticModel, SingularTestNode, SourceDefinition, UnitTestDefinition, ) from dbt.contracts.graph.unparsed import UnparsedVersion from dbt.contracts.state import PreviousState from dbt.exceptions import DbtSelectorsError from dbt.node_types import NodeType from dbt_common.dataclass_schema import StrEnum from dbt_common.events.contextvars import get_project_root from dbt_common.exceptions import CompilationError, DbtInternalError, DbtRuntimeError from dbt_common.exceptions import RecursionError as DbtRecursionError from .graph import UniqueId if TYPE_CHECKING: from .selector_spec import SelectionSpec SELECTOR_GLOB = "*" SELECTOR_DELIMITER = ":" class MethodName(StrEnum): FQN = "fqn" Tag = "tag" Group = "group" Access = "access" Source = "source" Path = "path" File = "file" Package = "package" Config = "config" TestName = "test_name" TestType = "test_type" ResourceType = "resource_type" State = "state" Exposure = "exposure" Metric = "metric" Result = "result" SourceStatus = "source_status" Version = "version" SemanticModel = "semantic_model" SavedQuery = "saved_query" UnitTest = "unit_test" Selector = "selector" def is_selected_node(fqn: List[str], node_selector: str, is_versioned: bool) -> bool: # If qualified_name exactly matches model name (fqn's leaf), return True if is_versioned: flat_node_selector = node_selector.split(".") if fqn[-2] == node_selector: return True # If this is a versioned model, then the last two segments should be allowed to exactly match on either the '.' or '_' delimiter elif "_".join(fqn[-2:]) == "_".join(flat_node_selector[-2:]): return True else: if fqn[-1] == node_selector: return True # Flatten node parts. Dots in model names act as namespace separators flat_fqn = [item for segment in fqn for item in segment.split(".")] # Selector components cannot be more than fqn's if len(flat_fqn) < len(node_selector.split(".")): return False slurp_from_ix: Optional[int] = None for i, selector_part in enumerate(node_selector.split(".")): if any(wildcard in selector_part for wildcard in ("*", "?", "[", "]")): slurp_from_ix = i break elif flat_fqn[i] == selector_part: continue else: return False if slurp_from_ix is not None: # If we have a wildcard, we need to make sure that the selector matches the # rest of the fqn, this is 100% backwards compatible with the old behavior of # encountering a wildcard but more expressive in naturally allowing you to # match the rest of the fqn with more advanced patterns return fnmatch( ".".join(flat_fqn[slurp_from_ix:]), ".".join(node_selector.split(".")[slurp_from_ix:]), ) # if we get all the way down here, then the node is a match return True SelectorTarget = Union[ SourceDefinition, ManifestNode, Exposure, Metric, SemanticModel, UnitTestDefinition, SavedQuery ] class SelectorMethod(metaclass=abc.ABCMeta): def __init__( self, manifest: Manifest, previous_state: Optional[PreviousState], arguments: List[str], **kwargs: Any, ) -> None: self.manifest: Manifest = manifest self.previous_state = previous_state self.arguments: List[str] = arguments def parsed_nodes( self, included_nodes: Set[UniqueId] ) -> Iterator[Tuple[UniqueId, ManifestNode]]: for key, node in self.manifest.nodes.items(): unique_id = UniqueId(key) if unique_id not in included_nodes: continue yield unique_id, node def source_nodes( self, included_nodes: Set[UniqueId] ) -> Iterator[Tuple[UniqueId, SourceDefinition]]: for key, source in self.manifest.sources.items(): unique_id = UniqueId(key) if unique_id not in included_nodes: continue yield unique_id, source def exposure_nodes(self, included_nodes: Set[UniqueId]) -> Iterator[Tuple[UniqueId, Exposure]]: for key, exposure in self.manifest.exposures.items(): unique_id = UniqueId(key) if unique_id not in included_nodes: continue yield unique_id, exposure def metric_nodes(self, included_nodes: Set[UniqueId]) -> Iterator[Tuple[UniqueId, Metric]]: for key, metric in self.manifest.metrics.items(): unique_id = UniqueId(key) if unique_id not in included_nodes: continue yield unique_id, metric def unit_tests( self, included_nodes: Set[UniqueId] ) -> Iterator[Tuple[UniqueId, UnitTestDefinition]]: for unique_id, unit_test in self.manifest.unit_tests.items(): unique_id = UniqueId(unique_id) if unique_id not in included_nodes: continue yield unique_id, unit_test def parsed_and_unit_nodes(self, included_nodes: Set[UniqueId]): yield from chain( self.parsed_nodes(included_nodes), self.unit_tests(included_nodes), ) def semantic_model_nodes( self, included_nodes: Set[UniqueId] ) -> Iterator[Tuple[UniqueId, SemanticModel]]: for key, semantic_model in self.manifest.semantic_models.items(): unique_id = UniqueId(key) if unique_id not in included_nodes: continue yield unique_id, semantic_model def saved_query_nodes( self, included_nodes: Set[UniqueId] ) -> Iterator[Tuple[UniqueId, SavedQuery]]: for key, saved_query in self.manifest.saved_queries.items(): unique_id = UniqueId(key) if unique_id not in included_nodes: continue yield unique_id, saved_query def function_nodes( self, included_nodes: Set[UniqueId] ) -> Iterator[Tuple[UniqueId, FunctionNode]]: for key, function in self.manifest.functions.items(): unique_id = UniqueId(key) if unique_id not in included_nodes: continue yield unique_id, function def all_nodes( self, included_nodes: Set[UniqueId] ) -> Iterator[Tuple[UniqueId, SelectorTarget]]: yield from chain( self.parsed_nodes(included_nodes), self.source_nodes(included_nodes), self.exposure_nodes(included_nodes), self.metric_nodes(included_nodes), self.unit_tests(included_nodes), self.semantic_model_nodes(included_nodes), self.saved_query_nodes(included_nodes), self.function_nodes(included_nodes), ) def configurable_nodes( self, included_nodes: Set[UniqueId] ) -> Iterator[Tuple[UniqueId, ResultNode]]: yield from chain(self.parsed_nodes(included_nodes), self.source_nodes(included_nodes)) def non_source_nodes( self, included_nodes: Set[UniqueId], ) -> Iterator[Tuple[UniqueId, Union[Exposure, ManifestNode, Metric]]]: yield from chain( self.parsed_nodes(included_nodes), self.exposure_nodes(included_nodes), self.metric_nodes(included_nodes), self.unit_tests(included_nodes), self.semantic_model_nodes(included_nodes), self.saved_query_nodes(included_nodes), self.function_nodes(included_nodes), ) def groupable_nodes( self, included_nodes: Set[UniqueId], ) -> Iterator[Tuple[UniqueId, Union[ManifestNode, Metric]]]: yield from chain( self.parsed_nodes(included_nodes), self.metric_nodes(included_nodes), ) @abc.abstractmethod def search( self, included_nodes: Set[UniqueId], selector: str, ) -> Iterator[UniqueId]: raise NotImplementedError("subclasses should implement this") class QualifiedNameSelectorMethod(SelectorMethod): def node_is_match(self, qualified_name: str, fqn: List[str], is_versioned: bool) -> bool: """Determine if a qualified name matches an fqn for all package names in the graph. :param str qualified_name: The qualified name to match the nodes with :param List[str] fqn: The node's fully qualified name in the graph. """ unscoped_fqn = fqn[1:] if is_selected_node(fqn, qualified_name, is_versioned): return True # Match nodes across different packages elif is_selected_node(unscoped_fqn, qualified_name, is_versioned): return True return False def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]: """Yield all nodes in the graph that match the selector. :param str selector: The selector or node name """ non_source_nodes = list(self.non_source_nodes(included_nodes)) for unique_id, node in non_source_nodes: if self.node_is_match(selector, node.fqn, node.is_versioned): yield unique_id class TagSelectorMethod(SelectorMethod): def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]: """yields nodes from included that have the specified tag""" for unique_id, node in self.all_nodes(included_nodes): if hasattr(node, "tags") and any(fnmatch(tag, selector) for tag in node.tags): yield unique_id class GroupSelectorMethod(SelectorMethod): def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]: """yields nodes from included in the specified group""" for unique_id, node in self.groupable_nodes(included_nodes): node_group = node.config.get("group") if node_group and fnmatch(node_group, selector): yield unique_id class AccessSelectorMethod(SelectorMethod): def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]: """yields model nodes matching the specified access level""" for unique_id, node in self.parsed_nodes(included_nodes): if not isinstance(node, ModelNode): continue if selector == node.access: yield unique_id class SourceSelectorMethod(SelectorMethod): def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]: """yields nodes from included are the specified source.""" parts = selector.split(".") target_package = SELECTOR_GLOB if len(parts) == 1: target_source, target_table = parts[0], SELECTOR_GLOB elif len(parts) == 2: target_source, target_table = parts elif len(parts) == 3: target_package, target_source, target_table = parts else: # len(parts) > 3 or len(parts) == 0 msg = ( 'Invalid source selector value "{}". Sources must be of the ' "form `${{source_name}}`, " "`${{source_name}}.${{target_name}}`, or " "`${{package_name}}.${{source_name}}.${{target_name}}" ).format(selector) raise DbtRuntimeError(msg) for unique_id, node in self.source_nodes(included_nodes): if not fnmatch(node.package_name, target_package): continue if not fnmatch(node.source_name, target_source): continue if not fnmatch(node.name, target_table): continue yield unique_id class ExposureSelectorMethod(SelectorMethod): def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]: parts = selector.split(".") target_package = SELECTOR_GLOB if len(parts) == 1: target_name = parts[0] elif len(parts) == 2: target_package, target_name = parts else: msg = ( 'Invalid exposure selector value "{}". Exposures must be of ' "the form ${{exposure_name}} or " "${{exposure_package.exposure_name}}" ).format(selector) raise DbtRuntimeError(msg) for unique_id, node in self.exposure_nodes(included_nodes): if not fnmatch(node.package_name, target_package): continue if not fnmatch(node.name, target_name): continue yield unique_id class MetricSelectorMethod(SelectorMethod): def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]: parts = selector.split(".") target_package = SELECTOR_GLOB if len(parts) == 1: target_name = parts[0] elif len(parts) == 2: target_package, target_name = parts else: msg = ( 'Invalid metric selector value "{}". Metrics must be of ' "the form ${{metric_name}} or " "${{metric_package.metric_name}}" ).format(selector) raise DbtRuntimeError(msg) for unique_id, node in self.metric_nodes(included_nodes): if not fnmatch(node.package_name, target_package): continue if not fnmatch(node.name, target_name): continue yield unique_id class SemanticModelSelectorMethod(SelectorMethod): def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]: parts = selector.split(".") target_package = SELECTOR_GLOB if len(parts) == 1: target_name = parts[0] elif len(parts) == 2: target_package, target_name = parts else: msg = ( 'Invalid semantic model selector value "{}". Semantic models must be of ' "the form ${{semantic_model_name}} or " "${{semantic_model_package.semantic_model_name}}" ).format(selector) raise DbtRuntimeError(msg) for unique_id, node in self.semantic_model_nodes(included_nodes): if not fnmatch(node.package_name, target_package): continue if not fnmatch(node.name, target_name): continue yield unique_id class SavedQuerySelectorMethod(SelectorMethod): def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]: parts = selector.split(".") target_package = SELECTOR_GLOB if len(parts) == 1: target_name = parts[0] elif len(parts) == 2: target_package, target_name = parts else: msg = ( 'Invalid saved query selector value "{}". Saved queries must be of ' "the form ${{saved_query_name}} or " "${{saved_query_package.saved_query_name}}" ).format(selector) raise DbtRuntimeError(msg) for unique_id, node in self.saved_query_nodes(included_nodes): if not fnmatch(node.package_name, target_package): continue if not fnmatch(node.name, target_name): continue yield unique_id class UnitTestSelectorMethod(SelectorMethod): def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]: parts = selector.split(".") target_package = SELECTOR_GLOB if len(parts) == 1: target_name = parts[0] elif len(parts) == 2: target_package, target_name = parts else: msg = ( 'Invalid unit test selector value "{}". Saved queries must be of ' "the form ${{unit_test_name}} or " "${{unit_test_package_name.unit_test_name}}" ).format(selector) raise DbtRuntimeError(msg) for unique_id, node in self.unit_tests(included_nodes): if not fnmatch(node.package_name, target_package): continue if not fnmatch(node.name, target_name): continue yield unique_id class PathSelectorMethod(SelectorMethod): def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]: """Yields nodes from included that match the given path.""" # get project root from contextvar project_root = get_project_root() if project_root: root = Path(project_root) else: root = Path.cwd() paths = set(p.relative_to(root) for p in root.glob(selector)) for unique_id, node in self.all_nodes(included_nodes): ofp = Path(node.original_file_path) if ofp in paths: yield unique_id if hasattr(node, "patch_path") and node.patch_path: # type: ignore pfp = node.patch_path.split("://")[1] # type: ignore ymlfp = Path(pfp) if ymlfp in paths: yield unique_id if any(parent in paths for parent in ofp.parents): yield unique_id class FileSelectorMethod(SelectorMethod): def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]: """Yields nodes from included that match the given file name.""" for unique_id, node in self.all_nodes(included_nodes): if fnmatch(Path(node.original_file_path).name, selector): yield unique_id elif fnmatch(Path(node.original_file_path).stem, selector): yield unique_id class PackageSelectorMethod(SelectorMethod): def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]: """Yields nodes from included that have the specified package""" # `this` is an alias for the current dbt project name if selector == "this" and self.manifest.metadata.project_name is not None: selector = self.manifest.metadata.project_name for unique_id, node in self.all_nodes(included_nodes): if fnmatch(node.package_name, selector): yield unique_id def _getattr_descend(obj: Any, attrs: List[str]) -> Any: value = obj for attr in attrs: try: value = getattr(value, attr) except AttributeError: # if it implements getitem (dict, list, ...), use that. On failure, # raise an attribute error instead of the KeyError, TypeError, etc. # that arbitrary getitem calls might raise try: value = value[attr] except Exception as exc: raise AttributeError(f"'{type(value)}' object has no attribute '{attr}'") from exc return value class CaseInsensitive(str): def __eq__(self, other): if isinstance(other, str): return self.upper() == other.upper() else: return self.upper() == other class ConfigSelectorMethod(SelectorMethod): def search( self, included_nodes: Set[UniqueId], selector: Any, ) -> Iterator[UniqueId]: parts = self.arguments # special case: if the user wanted to compare test severity, # make the comparison case-insensitive if parts == ["severity"]: selector = CaseInsensitive(selector) # search sources is kind of useless now source configs only have # 'enabled', which you can't really filter on anyway, but maybe we'll # add more someday, so search them anyway. for unique_id, node in self.configurable_nodes(included_nodes): try: value = _getattr_descend(node.config, parts) except AttributeError: continue else: if isinstance(value, list): if ( (selector in value) or (CaseInsensitive(selector) == "true" and True in value) or (CaseInsensitive(selector) == "false" and False in value) ): yield unique_id else: if ( (selector == value) or (CaseInsensitive(selector) == "true" and value is True) or (CaseInsensitive(selector) == "false") and value is False ): yield unique_id class ResourceTypeSelectorMethod(SelectorMethod): def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]: try: resource_type = NodeType(selector) except ValueError as exc: raise DbtRuntimeError(f'Invalid resource_type selector "{selector}"') from exc for unique_id, node in self.all_nodes(included_nodes): if node.resource_type == resource_type: yield unique_id class TestNameSelectorMethod(SelectorMethod): __test__ = False def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]: for unique_id, node in self.parsed_and_unit_nodes(included_nodes): if node.resource_type == NodeType.Test and hasattr(node, "test_metadata"): if fnmatch(node.test_metadata.name, selector): # type: ignore[union-attr] yield unique_id elif node.resource_type == NodeType.Unit: if fnmatch(node.name, selector): yield unique_id class TestTypeSelectorMethod(SelectorMethod): __test__ = False def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]: search_types: List[Any] # continue supporting 'schema' + 'data' for backwards compatibility if selector in ("generic", "schema"): search_types = [GenericTestNode] elif selector in ("data"): search_types = [GenericTestNode, SingularTestNode] elif selector in ("singular"): search_types = [SingularTestNode] elif selector in ("unit"): search_types = [UnitTestDefinition] else: raise DbtRuntimeError( f'Invalid test type selector {selector}: expected "generic", "singular", "unit", or "data"' ) for unique_id, node in self.parsed_and_unit_nodes(included_nodes): if isinstance(node, tuple(search_types)): yield unique_id class StateSelectorMethod(SelectorMethod): def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self.modified_macros: Optional[List[str]] = None def _macros_modified(self) -> List[str]: # we checked in the caller! if self.previous_state is None or self.previous_state.manifest is None: raise DbtInternalError("No comparison manifest in _macros_modified") old_macros = self.previous_state.manifest.macros new_macros = self.manifest.macros modified = [] for uid, macro in new_macros.items(): if uid in old_macros: old_macro = old_macros[uid] if macro.macro_sql != old_macro.macro_sql: modified.append(uid) else: modified.append(uid) for uid, _ in old_macros.items(): if uid not in new_macros: modified.append(uid) return modified def recursively_check_macros_modified(self, node, visited_macros): if not hasattr(node, "depends_on"): return False for macro_uid in node.depends_on.macros: if macro_uid in visited_macros: continue visited_macros.append(macro_uid) # If macro_uid is None, it means the macro/test was removed but is still referenced. # Raise a clear error to match the behavior of regular dbt run. if macro_uid is None: raise CompilationError( f"Node '{node.name}' (in {node.original_file_path}) depends on a macro or test " f"that does not exist. This can happen when a macro or generic test is removed " f"but is still referenced. Check for typos and/or install package dependencies " f"with 'dbt deps'." ) if macro_uid in self.modified_macros: return True # this macro hasn't been modified, but depends on other # macros which each need to be tested for modification macro_node = self.manifest.macros[macro_uid] if len(macro_node.depends_on.macros) > 0: upstream_macros_changed = self.recursively_check_macros_modified( macro_node, visited_macros ) if upstream_macros_changed: return True continue # this macro hasn't been modified, but we haven't checked # the other macros the node depends on, so keep looking if len(node.depends_on.macros) > len(visited_macros): continue return False def check_macros_modified(self, node): # check if there are any changes in macros the first time if self.modified_macros is None: self.modified_macros = self._macros_modified() # no macros have been modified, skip looping entirely if not self.modified_macros: return False # recursively loop through upstream macros to see if any is modified else: visited_macros = [] return self.recursively_check_macros_modified(node, visited_macros) # TODO check modifed_content and check_modified macro seems a bit redundent def check_modified_content( self, old: Optional[SelectorTarget], new: SelectorTarget, adapter_type: str ) -> bool: different_contents = False if isinstance( new, (SourceDefinition, Exposure, Metric, SemanticModel, UnitTestDefinition, SavedQuery), ): # these all overwrite `same_contents` different_contents = not new.same_contents(old) # type: ignore elif new: # because we also pull in deleted/disabled nodes, this could be None different_contents = not new.same_contents(old, adapter_type) # type: ignore upstream_macro_change = self.check_macros_modified(new) check_modified_contract = False if isinstance(old, ModelNode): func = self.check_modified_contract("same_contract", adapter_type) check_modified_contract = func(old, new) return different_contents or upstream_macro_change or check_modified_contract def check_unmodified_content( self, old: Optional[SelectorTarget], new: SelectorTarget, adapter_type: str ) -> bool: return not self.check_modified_content(old, new, adapter_type) def check_modified_macros(self, old, new: SelectorTarget) -> bool: return self.check_macros_modified(new) @staticmethod def check_modified_factory( compare_method: str, ) -> Callable[[Optional[SelectorTarget], SelectorTarget], bool]: # get a function that compares two selector target based on compare method provided def check_modified_things(old: Optional[SelectorTarget], new: SelectorTarget) -> bool: if hasattr(new, compare_method): # when old body does not exist or old and new are not the same return not old or not getattr(new, compare_method)(old) # type: ignore else: return False return check_modified_things @staticmethod def check_modified_contract( compare_method: str, adapter_type: Optional[str], ) -> Callable[[Optional[SelectorTarget], SelectorTarget], bool]: # get a function that compares two selector target based on compare method provided def check_modified_contract(old: Optional[SelectorTarget], new: SelectorTarget) -> bool: if new is None and hasattr(old, compare_method + "_removed"): return getattr(old, compare_method + "_removed")() elif hasattr(new, compare_method): # when old body does not exist or old and new are not the same return not old or not getattr(new, compare_method)(old, adapter_type) # type: ignore else: return False return check_modified_contract def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]: if self.previous_state is None or self.previous_state.manifest is None: raise DbtRuntimeError("Got a state selector method, but no comparison manifest") adapter_type = self.manifest.metadata.adapter_type state_checks = { # it's new if there is no old version "new": lambda old, new: old is None, "old": lambda old, new: old is not None, # use methods defined above to compare properties of old + new "modified": self.check_modified_content, "unmodified": self.check_unmodified_content, "modified.body": self.check_modified_factory("same_body"), "modified.configs": self.check_modified_factory("same_config"), "modified.persisted_descriptions": self.check_modified_factory( "same_persisted_description" ), "modified.relation": self.check_modified_factory("same_database_representation"), "modified.macros": self.check_modified_macros, "modified.contract": self.check_modified_contract("same_contract", adapter_type), } if selector in state_checks: checker = state_checks[selector] else: raise DbtRuntimeError( f'Got an invalid selector "{selector}", expected one of ' f'"{list(state_checks)}"' ) manifest: Manifest = self.previous_state.manifest keyword_args = {} # initialize here to handle disabled node check below for unique_id, node in self.all_nodes(included_nodes): previous_node: Optional[SelectorTarget] = None if unique_id in manifest.nodes: previous_node = manifest.nodes[unique_id] elif unique_id in manifest.sources: previous_node = SourceDefinition.from_resource(manifest.sources[unique_id]) elif unique_id in manifest.exposures: previous_node = Exposure.from_resource(manifest.exposures[unique_id]) elif unique_id in manifest.metrics: previous_node = Metric.from_resource(manifest.metrics[unique_id]) elif unique_id in manifest.semantic_models: previous_node = SemanticModel.from_resource(manifest.semantic_models[unique_id]) elif unique_id in manifest.unit_tests: previous_node = UnitTestDefinition.from_resource(manifest.unit_tests[unique_id]) elif unique_id in manifest.saved_queries: previous_node = SavedQuery.from_resource(manifest.saved_queries[unique_id]) elif unique_id in manifest.functions: previous_node = FunctionNode.from_resource(manifest.functions[unique_id]) if checker.__name__ in [ "same_contract", "check_modified_content", "check_unmodified_content", ]: keyword_args["adapter_type"] = adapter_type # type: ignore if checker(previous_node, node, **keyword_args): # type: ignore yield unique_id # checkers that can handle removed nodes if checker.__name__ in [ "check_modified_contract", "check_modified_content", "check_unmodified_content", ]: # ignore included_nodes, since those cannot contain removed nodes for previous_unique_id, previous_node in manifest.nodes.items(): # detect removed (deleted, renamed, or disabled) nodes removed_node = None if previous_unique_id in self.manifest.disabled.keys(): removed_node = self.manifest.disabled[previous_unique_id][0] elif previous_unique_id not in self.manifest.nodes.keys(): removed_node = previous_node if removed_node: # do not yield -- removed nodes should never be selected for downstream execution # as they are not part of the current project's manifest.nodes checker(removed_node, None, **keyword_args) # type: ignore class ResultSelectorMethod(SelectorMethod): def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]: if self.previous_state is None or self.previous_state.results is None: raise DbtInternalError("No comparison run_results") matches = set( result.unique_id for result in self.previous_state.results if result.status == selector ) for unique_id, node in self.all_nodes(included_nodes): if unique_id in matches: yield unique_id class SourceStatusSelectorMethod(SelectorMethod): def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]: if self.previous_state is None or self.previous_state.sources is None: raise DbtInternalError( "No previous state comparison freshness results in sources.json" ) elif self.previous_state.sources_current is None: raise DbtInternalError("No current state comparison freshness results in sources.json") current_state_sources = { result.unique_id: getattr(result, "max_loaded_at", 0) for result in self.previous_state.sources_current.results if hasattr(result, "max_loaded_at") } current_state_sources_runtime_error = { result.unique_id for result in self.previous_state.sources_current.results if not hasattr(result, "max_loaded_at") } previous_state_sources = { result.unique_id: getattr(result, "max_loaded_at", 0) for result in self.previous_state.sources.results if hasattr(result, "max_loaded_at") } previous_state_sources_runtime_error = { result.unique_id for result in self.previous_state.sources_current.results if not hasattr(result, "max_loaded_at") } matches = set() if selector == "fresher": for unique_id in current_state_sources: if unique_id not in previous_state_sources: matches.add(unique_id) elif current_state_sources[unique_id] > previous_state_sources[unique_id]: matches.add(unique_id) for unique_id in matches: if ( unique_id in previous_state_sources_runtime_error or unique_id in current_state_sources_runtime_error ): matches.remove(unique_id) for unique_id, node in self.all_nodes(included_nodes): if unique_id in matches: yield unique_id class VersionSelectorMethod(SelectorMethod): def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]: for unique_id, node in self.parsed_nodes(included_nodes): if isinstance(node, ModelNode): if selector == "latest": if node.is_latest_version: yield unique_id elif selector == "prerelease": if ( node.version and node.latest_version and UnparsedVersion(v=node.version) > UnparsedVersion(v=node.latest_version) ): yield unique_id elif selector == "old": if ( node.version and node.latest_version and UnparsedVersion(v=node.version) < UnparsedVersion(v=node.latest_version) ): yield unique_id elif selector == "none": if node.version is None: yield unique_id else: raise DbtRuntimeError( f'Invalid version type selector {selector}: expected one of: "latest", "prerelease", "old", or "none"' ) class SelectorSelectorMethod(SelectorMethod): def __init__( self, manifest: Manifest, previous_state: Optional[PreviousState], arguments: List[str], *, get_selected_callback: Callable[..., Set[UniqueId]], selectors: Optional[Dict[str, Any]] = None, ) -> None: super().__init__(manifest, previous_state, arguments) self._selectors = selectors self._get_selected_callback = get_selected_callback def _search_for_matched_selector( self, included_nodes: Set[UniqueId], selection_spec: "SelectionSpec" ) -> Iterator[UniqueId]: selected = self._get_selected_callback(selection_spec) for unique_id in selected: if unique_id in included_nodes: yield unique_id def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]: if self._selectors is None or self._get_selected_callback is None: raise DbtSelectorsError("Cannot use selector: method if selectors.yml is not provided") matched_selector_dfns = [] # ensure that selectors are sorted to make the execution deterministic # this will ensure that when circular depdencies are detected # the same selector is used as the "head" of the circular dependency on every run sorted_selectors = sorted(self._selectors.items(), key=lambda x: x[0]) for s_name, s_value in sorted_selectors: if fnmatch(s_name, selector): matched_selector_dfns.append(s_value["definition"]) if not matched_selector_dfns: raise DbtSelectorsError( f"Selector '{selector}' did not match any selector in selectors.yml" ) for matched_selector_dfn in matched_selector_dfns: try: yield from self._search_for_matched_selector(included_nodes, matched_selector_dfn) except RecursionError as e: raise DbtRecursionError( f"Circular dependency detected in selector: {matched_selector_dfn.raw}" ) from e class MethodManager: SELECTOR_METHODS: Dict[MethodName, Type[SelectorMethod]] = { MethodName.FQN: QualifiedNameSelectorMethod, MethodName.Tag: TagSelectorMethod, MethodName.Group: GroupSelectorMethod, MethodName.Access: AccessSelectorMethod, MethodName.Source: SourceSelectorMethod, MethodName.Path: PathSelectorMethod, MethodName.File: FileSelectorMethod, MethodName.Package: PackageSelectorMethod, MethodName.Config: ConfigSelectorMethod, MethodName.TestName: TestNameSelectorMethod, MethodName.TestType: TestTypeSelectorMethod, MethodName.ResourceType: ResourceTypeSelectorMethod, MethodName.State: StateSelectorMethod, MethodName.Exposure: ExposureSelectorMethod, MethodName.Metric: MetricSelectorMethod, MethodName.Result: ResultSelectorMethod, MethodName.SourceStatus: SourceStatusSelectorMethod, MethodName.Version: VersionSelectorMethod, MethodName.SemanticModel: SemanticModelSelectorMethod, MethodName.SavedQuery: SavedQuerySelectorMethod, MethodName.UnitTest: UnitTestSelectorMethod, MethodName.Selector: SelectorSelectorMethod, } def __init__( self, manifest: Manifest, previous_state: Optional[PreviousState], ) -> None: self.manifest = manifest self.previous_state = previous_state def get_method( self, method: MethodName, method_arguments: List[str], **kwargs: Any ) -> SelectorMethod: if method not in self.SELECTOR_METHODS: raise DbtInternalError( f'Method name "{method}" is a valid node selection ' f"method name, but it is not handled" ) cls: Type[SelectorMethod] = self.SELECTOR_METHODS[method] return cls(self.manifest, self.previous_state, method_arguments, **kwargs) ================================================ FILE: core/dbt/graph/selector_spec.py ================================================ import os import re from abc import ABCMeta, abstractmethod from dataclasses import dataclass from typing import Any, Dict, Iterable, Iterator, List, Optional, Set, Tuple, Union from dbt.exceptions import InvalidSelectorError from dbt.flags import get_flags from dbt_common.dataclass_schema import StrEnum, dbtClassMixin from dbt_common.exceptions import DbtRuntimeError from .graph import UniqueId from .selector_methods import MethodName RAW_SELECTOR_PATTERN = re.compile( r"\A" r"(?P(\@))?" r"(?P((?P(\d*))\+))?" r"((?P([\w.]+)):)?(?P(.*?))" r"(?P(\+(?P(\d*))))?" r"\Z" ) SELECTOR_METHOD_SEPARATOR = "." class IndirectSelection(StrEnum): Eager = "eager" Cautious = "cautious" Buildable = "buildable" Empty = "empty" def _probably_path(value: str): """Decide if the value is probably a path. Windows has two path separators, so we should check both sep ('\\') and altsep ('/') there. """ if os.path.sep in value: return True elif os.path.altsep is not None and os.path.altsep in value: return True else: return False def _match_to_int(match: Dict[str, str], key: str) -> Optional[int]: raw = match.get(key) # turn the empty string into None, too. if not raw: return None try: return int(raw) except ValueError as exc: raise DbtRuntimeError(f"Invalid node spec - could not handle parent depth {raw}") from exc SelectionSpec = Union[ "SelectionCriteria", "SelectionIntersection", "SelectionDifference", "SelectionUnion", ] @dataclass class SelectionCriteria: raw: Any method: MethodName method_arguments: List[str] value: Any childrens_parents: bool parents: bool parents_depth: Optional[int] children: bool children_depth: Optional[int] indirect_selection: IndirectSelection = IndirectSelection.Eager def __post_init__(self): if self.children and self.childrens_parents: raise DbtRuntimeError( f'Invalid node spec {self.raw} - "@" prefix and "+" suffix ' "are incompatible" ) @classmethod def default_method(cls, value: str) -> MethodName: if _probably_path(value): return MethodName.Path elif value.lower().endswith((".sql", ".py", ".csv")): return MethodName.File else: return MethodName.FQN @classmethod def parse_method(cls, groupdict: Dict[str, Any]) -> Tuple[MethodName, List[str]]: raw_method = groupdict.get("method") if raw_method is None: return cls.default_method(groupdict["value"]), [] method_parts: List[str] = raw_method.split(SELECTOR_METHOD_SEPARATOR) try: method_name = MethodName(method_parts[0]) except ValueError as exc: raise InvalidSelectorError(f"'{method_parts[0]}' is not a valid method name") from exc # Following is for cases like config.severity and config.materialized method_arguments: List[str] = method_parts[1:] return method_name, method_arguments @classmethod def selection_criteria_from_dict( cls, raw: Any, dct: Dict[str, Any], ) -> "SelectionCriteria": if "value" not in dct: raise DbtRuntimeError(f'Invalid node spec "{raw}" - no search value!') method_name, method_arguments = cls.parse_method(dct) parents_depth = _match_to_int(dct, "parents_depth") children_depth = _match_to_int(dct, "children_depth") # If defined field in selector, override CLI flag indirect_selection = IndirectSelection( dct.get("indirect_selection", get_flags().INDIRECT_SELECTION) ) return cls( raw=raw, method=method_name, method_arguments=method_arguments, value=dct["value"], childrens_parents=bool(dct.get("childrens_parents")), parents=bool(dct.get("parents")), parents_depth=parents_depth, children=bool(dct.get("children")), children_depth=children_depth, indirect_selection=indirect_selection, ) @classmethod def dict_from_single_spec(cls, raw: str): result = RAW_SELECTOR_PATTERN.match(raw) if result is None: return {"error": "Invalid selector spec"} dct: Dict[str, Any] = result.groupdict() method_name, method_arguments = cls.parse_method(dct) meth_name = str(method_name) if method_arguments: meth_name += "." + ".".join(method_arguments) dct["method"] = meth_name dct = {k: v for k, v in dct.items() if (v is not None and v != "")} if "childrens_parents" in dct: dct["childrens_parents"] = bool(dct.get("childrens_parents")) if "parents" in dct: dct["parents"] = bool(dct.get("parents")) if "children" in dct: dct["children"] = bool(dct.get("children")) return dct @classmethod def from_single_spec(cls, raw: str) -> "SelectionCriteria": result = RAW_SELECTOR_PATTERN.match(raw) if result is None: # bad spec! raise DbtRuntimeError(f'Invalid selector spec "{raw}"') return cls.selection_criteria_from_dict(raw, result.groupdict()) class BaseSelectionGroup(dbtClassMixin, Iterable[SelectionSpec], metaclass=ABCMeta): def __init__( self, components: Iterable[SelectionSpec], indirect_selection: IndirectSelection = IndirectSelection.Eager, expect_exists: bool = False, raw: Any = None, ) -> None: self.components: List[SelectionSpec] = list(components) self.expect_exists = expect_exists self.raw = raw self.indirect_selection = indirect_selection def __iter__(self) -> Iterator[SelectionSpec]: for component in self.components: yield component @abstractmethod def combine_selections( self, selections: List[Set[UniqueId]], ) -> Set[UniqueId]: raise NotImplementedError("_combine_selections not implemented!") def combined(self, selections: List[Set[UniqueId]]) -> Set[UniqueId]: if not selections: return set() return self.combine_selections(selections) class SelectionIntersection(BaseSelectionGroup): def combine_selections( self, selections: List[Set[UniqueId]], ) -> Set[UniqueId]: return set.intersection(*selections) class SelectionDifference(BaseSelectionGroup): def combine_selections( self, selections: List[Set[UniqueId]], ) -> Set[UniqueId]: return set.difference(*selections) class SelectionUnion(BaseSelectionGroup): def combine_selections( self, selections: List[Set[UniqueId]], ) -> Set[UniqueId]: return set.union(*selections) ================================================ FILE: core/dbt/graph/thread_pool.py ================================================ from __future__ import annotations from math import floor from multiprocessing.pool import ThreadPool class DbtThreadPool(ThreadPool): """A ThreadPool that tracks whether or not it's been closed""" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.closed = False # The maximum number of threads that can be used by the pool # Used for determining how MicrobatchBatchRungners should be run self.max_threads = kwargs.get("processes") if kwargs.get("processes") else args[0] # The maximum number of microbatch models that can be run concurrently # Used for determining if a MicrobatchModelRunner can be submitted to the pool self.max_microbatch_models = max(1, floor(self.max_threads / 2)) def close(self): self.closed = True super().close() def is_closed(self): return self.closed ================================================ FILE: core/dbt/hooks.py ================================================ import json from typing import Any, Dict, Union from dbt_common.dataclass_schema import StrEnum class ModelHookType(StrEnum): PreHook = "pre-hook" PostHook = "post-hook" def get_hook_dict(source: Union[str, Dict[str, Any]]) -> Dict[str, Any]: """From a source string-or-dict, get a dictionary that can be passed to Hook.from_dict """ if isinstance(source, dict): return source try: return json.loads(source) except ValueError: return {"sql": source} ================================================ FILE: core/dbt/include/README.md ================================================ # Include Module The Include module is responsible for the starter project scaffold. # Directories ## `starter_project` Produces the default project after running the `dbt init` command for the CLI. `dbt-cloud` initializes the project by using [dbt-starter-project](https://github.com/dbt-labs/dbt-starter-project). # adapter.dispatch Packages (e.g. `include` directories of adapters, any [hub](https://hub.getdbt.com/)-hosted package) can be interpreted as namespaces of functions a.k.a macros. In `dbt`'s macrospace, we take advantage of the multiple dispatch programming language concept. In short, multiple dispatch supports dynamic searching for a function across several namespaces—usually in a manually specified manner/order. Adapters can have their own implementation of the same macro X. For example, a macro executed by `dbt-redshift` may need a specific implementation different from `dbt-snowflake`'s macro. We use multiple dispatch via `adapter.dispatch`, a Jinja function, which enables polymorphic macro invocations. The chosen implementation is selected according to what the `adapter` object is set to at runtime (it could be for redshift, postgres, and so on). For more on this object, check out the dbt docs [here](https://docs.getdbt.com/reference/dbt-jinja-functions/adapter). # dbt and database adapter python package interop Let’s say we have a fictional python app named `dbt-core` with this structure ``` dbt ├── adapters │   └── base.py ├── cli.py └── main.py ``` `pip install dbt-core` will install this application in my python environment, maintaining the same structure. Note that `dbt.adapters` only contains a `base.py`. In this example, we can assume that base.py includes an abstract class for creating connections. Let’s say we wanted to create an postgres adapter that this app could use, and can be installed independently. We can create a python package with the following structure called `dbt-postgres` ``` dbt └── adapters └── postgres └── impl.py ``` `pip install dbt-postgres` will install this package in the python environment, maintaining the same structure again. Let’s say `impl.py` imports `dbt.adapters.base` and implements a concrete class inheriting from the abstract class in `base.py` from the `dbt-core` package. Since our top level package is named the same in both packages, `pip` will put this in the same place. We end up with this installed in our python environment. ``` dbt ├── adapters │   ├── base.py │   └── postgres │   └── impl.py ├── cli.py └── main.py ``` `dbt.adapters` now has a postgres module that dbt can easily find and call directly. dbt and its adapters follows the same type of file structure convention. This is the magic that allows you to import `dbt.*` in database adapters, and using a factory pattern in dbt-core, we can create instances of concrete classes defined in the database adapter packages (for creating connections, defining database configuration, defining credentials, etc.) ================================================ FILE: core/dbt/include/__init__.py ================================================ from pkgutil import extend_path __path__ = extend_path(__path__, __name__) ================================================ FILE: core/dbt/include/starter_project/.gitignore ================================================ target/ dbt_packages/ logs/ ================================================ FILE: core/dbt/include/starter_project/README.md ================================================ Welcome to your new dbt project! ### Using the starter project Try running the following commands: - dbt run - dbt test ### Resources: - Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) - Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers - Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support - Find [dbt events](https://events.getdbt.com) near you - Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices ================================================ FILE: core/dbt/include/starter_project/__init__.py ================================================ import os PACKAGE_PATH = os.path.dirname(__file__) ================================================ FILE: core/dbt/include/starter_project/analyses/.gitkeep ================================================ ================================================ FILE: core/dbt/include/starter_project/dbt_project.yml ================================================ # Name your project! Project names should contain only lowercase characters # and underscores. A good package name should reflect your organization's # name or the intended use of these models name: '{project_name}' version: '1.0.0' # This setting configures which "profile" dbt uses for this project. profile: '{profile_name}' # These configurations specify where dbt should look for different types of files. # The `model-paths` config, for example, states that models in this project can be # found in the "models/" directory. You probably won't need to change these! model-paths: ["models"] analysis-paths: ["analyses"] test-paths: ["tests"] seed-paths: ["seeds"] macro-paths: ["macros"] snapshot-paths: ["snapshots"] clean-targets: # directories to be removed by `dbt clean` - "target" - "dbt_packages" # Configuring models # Full documentation: https://docs.getdbt.com/docs/configuring-models # In this example config, we tell dbt to build all models in the example/ # directory as views. These settings can be overridden in the individual model # files using the `{{{{ config(...) }}}}` macro. models: {project_name}: # Config indicated by + and applies to all files under models/example/ example: +materialized: view ================================================ FILE: core/dbt/include/starter_project/macros/.gitkeep ================================================ ================================================ FILE: core/dbt/include/starter_project/models/example/my_first_dbt_model.sql ================================================ /* Welcome to your first dbt model! Did you know that you can also configure models directly within SQL files? This will override configurations stated in dbt_project.yml Try changing "table" to "view" below */ {{ config(materialized='table') }} with source_data as ( select 1 as id union all select null as id ) select * from source_data /* Uncomment the line below to remove records with null `id` values */ -- where id is not null ================================================ FILE: core/dbt/include/starter_project/models/example/my_second_dbt_model.sql ================================================ -- Use the `ref` function to select from other models select * from {{ ref('my_first_dbt_model') }} where id = 1 ================================================ FILE: core/dbt/include/starter_project/models/example/schema.yml ================================================ version: 2 models: - name: my_first_dbt_model description: "A starter dbt model" columns: - name: id description: "The primary key for this table" data_tests: - unique - not_null - name: my_second_dbt_model description: "A starter dbt model" columns: - name: id description: "The primary key for this table" data_tests: - unique - not_null ================================================ FILE: core/dbt/include/starter_project/seeds/.gitkeep ================================================ ================================================ FILE: core/dbt/include/starter_project/snapshots/.gitkeep ================================================ ================================================ FILE: core/dbt/include/starter_project/tests/.gitkeep ================================================ ================================================ FILE: core/dbt/internal_deprecations.py ================================================ import functools from typing import Optional from dbt.events.types import InternalDeprecation from dbt_common.events.functions import warn_or_error def deprecated(suggested_action: str, version: str, reason: Optional[str]): def inner(func): @functools.wraps(func) def wrapped(*args, **kwargs): name = func.__name__ warn_or_error( InternalDeprecation( name=name, suggested_action=suggested_action, version=version, reason=reason, ) ) # TODO: pass in event? return func(*args, **kwargs) return wrapped return inner ================================================ FILE: core/dbt/jsonschemas/__init__.py ================================================ import os JSONSCHEMAS_PATH = os.path.dirname(__file__) ================================================ FILE: core/dbt/jsonschemas/jsonschemas.py ================================================ import json import re from datetime import date, datetime from pathlib import Path from typing import Any, Dict, Iterator, List, Optional, Union import jsonschema from jsonschema import ValidationError from jsonschema._keywords import type as type_rule from jsonschema.validators import Draft7Validator, extend from dbt import deprecations from dbt.jsonschemas import JSONSCHEMAS_PATH from dbt_common.context import get_invocation_context _PROJECT_SCHEMA: Optional[Dict[str, Any]] = None _RESOURCES_SCHEMA: Optional[Dict[str, Any]] = None _JSONSCHEMA_SUPPORTED_ADAPTERS = { "bigquery", "databricks", "redshift", "snowflake", } _HIERARCHICAL_CONFIG_KEYS = { "seeds", "sources", "models", "snapshots", "tests", "exposures", "data_tests", "metrics", "saved_queries", "semantic_models", "unit_tests", } _ADAPTER_TO_CONFIG_ALIASES = { "bigquery": ["dataset", "project"], } def load_json_from_package(jsonschema_type: str, filename: str) -> Dict[str, Any]: """Loads a JSON file from within a package.""" path = Path(JSONSCHEMAS_PATH).joinpath(jsonschema_type, filename) data = path.read_bytes() return json.loads(data) def project_schema() -> Dict[str, Any]: global _PROJECT_SCHEMA if _PROJECT_SCHEMA is None: _PROJECT_SCHEMA = load_json_from_package( jsonschema_type="project", filename="0.0.110.json" ) return _PROJECT_SCHEMA def resources_schema() -> Dict[str, Any]: global _RESOURCES_SCHEMA if _RESOURCES_SCHEMA is None: _RESOURCES_SCHEMA = load_json_from_package( jsonschema_type="resources", filename="latest.json" ) return _RESOURCES_SCHEMA def custom_type_rule(validator, types, instance, schema): """This is necessary because PyYAML loads things that look like dates or datetimes as those python objects. Then jsonschema.validate() fails because it expects strings. """ if "string" in types and (isinstance(instance, datetime) or isinstance(instance, date)): return else: return type_rule(validator, types, instance, schema) CustomDraft7Validator = extend(Draft7Validator, validators={"type": custom_type_rule}) def error_path_to_string(error: jsonschema.ValidationError) -> str: if len(error.path) == 0: return "" else: path = str(error.path.popleft()) for part in error.path: if isinstance(part, int): path += f"[{part}]" else: path += f".{part}" return path def _additional_properties_violation_keys(error: ValidationError) -> List[str]: found_keys = re.findall(r"'\S+'", error.message) return [key.strip("'") for key in found_keys] def _validate_with_schema( schema: Dict[str, Any], json: Dict[str, Any] ) -> Iterator[ValidationError]: validator = CustomDraft7Validator(schema) return validator.iter_errors(json) def _get_allowed_config_key_aliases() -> List[str]: config_aliases = [] invocation_context = get_invocation_context() for adapter in invocation_context.adapter_types: if adapter in _ADAPTER_TO_CONFIG_ALIASES: config_aliases.extend(_ADAPTER_TO_CONFIG_ALIASES[adapter]) return config_aliases def _get_allowed_config_fields_for_project_property(schema, property_field_name) -> List[str]: property_defn = schema["properties"].get(property_field_name) property_defn_name = None if property_defn and "anyOf" in property_defn: for any_of_item in property_defn["anyOf"]: if "$ref" in any_of_item: property_defn_name = any_of_item["$ref"].split("/")[-1] break if property_defn_name is None: return [] allowed_config_fields = set(schema["definitions"][property_defn_name]["properties"]) allowed_config_fields.update(_get_allowed_config_key_aliases()) return list(allowed_config_fields) def _get_allowed_config_fields_from_error_path( yml_schema: Dict[str, Any], error_path: List[Union[str, int]] ) -> Optional[List[str]]: property_field_name = None node_schema = yml_schema["properties"] for part in error_path: if isinstance(part, str): if part in node_schema: if "items" not in node_schema[part]: break # Update property field name property_field_name = node_schema[part]["items"]["$ref"].split("/")[-1] # Jump to the next level of the schema item_definition = node_schema[part]["items"]["$ref"].split("/")[-1] node_schema = yml_schema["definitions"][item_definition]["properties"] if not property_field_name: return None if "config" not in yml_schema["definitions"][property_field_name]["properties"]: return None config_field_name = yml_schema["definitions"][property_field_name]["properties"]["config"][ "anyOf" ][0]["$ref"].split("/")[-1] allowed_config_fields = list(set(yml_schema["definitions"][config_field_name]["properties"])) allowed_config_fields.extend(_get_allowed_config_key_aliases()) return allowed_config_fields def _can_run_validations() -> bool: invocation_context = get_invocation_context() return invocation_context.adapter_types.issubset(_JSONSCHEMA_SUPPORTED_ADAPTERS) def jsonschema_validate(schema: Dict[str, Any], json: Dict[str, Any], file_path: str) -> None: if not _can_run_validations(): return errors = _validate_with_schema(schema, json) for error in errors: # Listify the error path to make it easier to work with (it's a deque in the ValidationError object) error_path = list(error.path) if error.validator == "additionalProperties": keys = _additional_properties_violation_keys(error) if len(error.path) == 0: for key in keys: deprecations.warn( "custom-top-level-key-deprecation", msg="Unexpected top-level key" + (" " + key if key else ""), file=file_path, ) else: key_path = error_path_to_string(error) for key in keys: # Type params are not in the metrics v2 jsonschema from fusion, but dbt-core continues to maintain support for them in v1. if key == "type_params": continue # 'dataset' and 'project' are valid top-level source properties for BigQuery if ( len(error_path) == 2 and error_path[0] == "sources" and isinstance(error_path[1], int) and key in _get_allowed_config_key_aliases() ): continue if key == "overrides" and key_path.startswith("sources"): deprecations.warn( "source-override-deprecation", source_name=key_path.split(".")[-1], file=file_path, ) else: allowed_config_fields = _get_allowed_config_fields_from_error_path( schema, error_path ) if allowed_config_fields and key in allowed_config_fields: deprecations.warn( "property-moved-to-config-deprecation", key=key, file=file_path, key_path=key_path, ) else: deprecations.warn( "custom-key-in-object-deprecation", key=key, file=file_path, key_path=key_path, ) elif error.validator == "anyOf" and len(error_path) > 0: sub_errors = error.context or [] # schema yaml resource configs if error_path[-1] == "config": for sub_error in sub_errors: if ( isinstance(sub_error, ValidationError) and sub_error.validator == "additionalProperties" ): keys = _additional_properties_violation_keys(sub_error) key_path = error_path_to_string(error) for key in keys: if key in _get_allowed_config_key_aliases(): continue deprecations.warn( "custom-key-in-config-deprecation", key=key, file=file_path, key_path=key_path, ) # dbt_project.yml configs elif "dbt_project.yml" in file_path and error_path[0] in _HIERARCHICAL_CONFIG_KEYS: for sub_error in sub_errors: if isinstance(sub_error, ValidationError) and sub_error.validator == "type": allowed_config_fields = _get_allowed_config_fields_for_project_property( schema, property_field_name=error_path[0] ) is_missing_plus_prefix = ( len(sub_error.path) > 0 and isinstance(sub_error.path[-1], str) and not sub_error.path[-1].startswith("+") ) had_valid_config_key_in_path = any( k in allowed_config_fields for k in sub_error.path ) is_maybe_config_key = ( len(sub_error.path) > 0 and isinstance(sub_error.path[-1], str) and f"+{sub_error.path[-1]}" in allowed_config_fields ) # if its missing a plus prefix, does not have valid config key in path # and the last part of the error path might be a valid config key if ( is_missing_plus_prefix and is_maybe_config_key and not had_valid_config_key_in_path ): deprecations.warn( "missing-plus-prefix-in-config-deprecation", key=sub_error.path[-1], file=file_path, key_path=error_path_to_string(sub_error), ) elif error.validator == "type": # Not deprecating invalid types yet pass else: deprecations.warn( "generic-json-schema-validation-deprecation", violation=error.message, file=file_path, key_path=error_path_to_string(error), ) def validate_model_config( config: Dict[str, Any], file_path: str, is_python_model: bool = False ) -> None: if not _can_run_validations(): return resources_jsonschema = resources_schema() nested_definition_name = "ModelConfig" model_config_schema = { "$schema": "http://json-schema.org/draft-07/schema#", "title": nested_definition_name, **resources_jsonschema["definitions"][nested_definition_name], "definitions": { k: v for k, v in resources_jsonschema["definitions"].items() if k != nested_definition_name }, } errors = _validate_with_schema(model_config_schema, config) for error in errors: error_path = list(error.path) if error.validator == "additionalProperties": keys = _additional_properties_violation_keys(error) if len(error.path) == 0: key_path = error_path_to_string(error) for key in keys: # Special case for pre/post hook keys as they are updated during config parsing # from the user-provided pre_hook/post_hook to pre-hook/post-hook keys. # Avoids false positives as described in https://github.com/dbt-labs/dbt-core/issues/12087 if key in ("post-hook", "pre-hook"): continue # Special case for python model internal key additions # These keys are added during python model parsing and are not user-provided python_model_internal_keys = ( "config_keys_used", "config_keys_defaults", "meta_keys_used", "meta_keys_defaults", ) if key in python_model_internal_keys and is_python_model: continue # Dont raise deprecation warnings for adapter specific config key aliases if key in _get_allowed_config_key_aliases(): continue # For everything else, emit deprecation warning deprecations.warn( "custom-key-in-config-deprecation", key=key, file=file_path, key_path=key_path, ) else: error.path.appendleft("config") key_path = error_path_to_string(error) for key in keys: deprecations.warn( "custom-key-in-object-deprecation", key=key, file=file_path, key_path=key_path, ) elif error.validator == "type": # Not deprecating invalid types yet, except for pre-existing deprecation_date deprecation pass elif error.validator == "anyOf" and len(error_path) > 0: for sub_error in error.context or []: if ( isinstance(sub_error, ValidationError) and sub_error.validator == "additionalProperties" ): error.path.appendleft("config") keys = _additional_properties_violation_keys(sub_error) key_path = error_path_to_string(error) for key in keys: deprecations.warn( "custom-key-in-object-deprecation", key=key, file=file_path, key_path=key_path, ) else: deprecations.warn( "generic-json-schema-validation-deprecation", violation=error.message, file=file_path, key_path=error_path_to_string(error), ) ================================================ FILE: core/dbt/jsonschemas/project/0.0.110.json ================================================ { "$schema": "http://json-schema.org/draft-07/schema#", "title": "DbtProject", "type": "object", "required": [ "name" ], "properties": { "analyses": { "anyOf": [ { "$ref": "#/definitions/ProjectAnalysisConfig" }, { "type": "null" } ] }, "analysis-paths": { "type": [ "array", "null" ], "items": { "type": "string" } }, "asset-paths": { "type": [ "array", "null" ], "items": { "type": "string" } }, "clean-targets": { "type": [ "array", "null" ], "items": { "type": "string" } }, "config-version": { "type": [ "integer", "null" ], "format": "int32" }, "data_tests": { "anyOf": [ { "$ref": "#/definitions/ProjectDataTestConfig" }, { "type": "null" } ] }, "dbt-cloud": { "anyOf": [ { "$ref": "#/definitions/ProjectDbtCloudConfig" }, { "type": "null" } ] }, "dispatch": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/_Dispatch" } }, "docs-paths": { "type": [ "array", "null" ], "items": { "type": "string" } }, "exposures": { "anyOf": [ { "$ref": "#/definitions/ProjectExposureConfig" }, { "type": "null" } ] }, "flags": { "anyOf": [ { "$ref": "#/definitions/AnyValue" }, { "type": "null" } ] }, "function-paths": { "type": [ "array", "null" ], "items": { "type": "string" } }, "functions": { "anyOf": [ { "$ref": "#/definitions/ProjectFunctionConfig" }, { "type": "null" } ] }, "log-path": { "anyOf": [ { "$ref": "#/definitions/LogPath" }, { "type": "null" } ] }, "macro-paths": { "type": [ "array", "null" ], "items": { "type": "string" } }, "metrics": { "anyOf": [ { "$ref": "#/definitions/ProjectMetricConfigs" }, { "type": "null" } ] }, "model-paths": { "type": [ "array", "null" ], "items": { "type": "string" } }, "models": { "anyOf": [ { "$ref": "#/definitions/ProjectModelConfig" }, { "type": "null" } ] }, "name": { "type": "string" }, "on-run-end": { "anyOf": [ { "$ref": "#/definitions/SpannedStringOrArrayOfStrings" }, { "type": "null" } ] }, "on-run-start": { "anyOf": [ { "$ref": "#/definitions/SpannedStringOrArrayOfStrings" }, { "type": "null" } ] }, "packages-install-path": { "type": [ "string", "null" ] }, "profile": { "type": [ "string", "null" ] }, "query-comment": { "anyOf": [ { "$ref": "#/definitions/QueryComment" }, { "type": "null" } ] }, "quoting": { "anyOf": [ { "$ref": "#/definitions/DbtQuoting" }, { "type": "null" } ] }, "require-dbt-version": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "restrict-access": { "type": [ "boolean", "null" ] }, "saved-queries": { "anyOf": [ { "$ref": "#/definitions/ProjectSavedQueryConfig" }, { "type": "null" } ] }, "seed-paths": { "type": [ "array", "null" ], "items": { "type": "string" } }, "seeds": { "anyOf": [ { "$ref": "#/definitions/ProjectSeedConfig" }, { "type": "null" } ] }, "semantic-models": { "anyOf": [ { "$ref": "#/definitions/ProjectSemanticModelConfig" }, { "type": "null" } ] }, "snapshot-paths": { "type": [ "array", "null" ], "items": { "type": "string" } }, "snapshots": { "anyOf": [ { "$ref": "#/definitions/ProjectSnapshotConfig" }, { "type": "null" } ] }, "sources": { "anyOf": [ { "$ref": "#/definitions/ProjectSourceConfig" }, { "type": "null" } ] }, "sync": { "anyOf": [ { "$ref": "#/definitions/SyncConfig" }, { "type": "null" } ] }, "target-path": { "anyOf": [ { "$ref": "#/definitions/TargetPath" }, { "type": "null" } ] }, "test-paths": { "type": [ "array", "null" ], "items": { "type": "string" } }, "tests": { "anyOf": [ { "$ref": "#/definitions/ProjectDataTestConfig" }, { "type": "null" } ] }, "unit_tests": { "anyOf": [ { "$ref": "#/definitions/ProjectUnitTestConfig" }, { "type": "null" } ] }, "vars": { "anyOf": [ { "$ref": "#/definitions/AnyValue" }, { "type": "null" } ] }, "version": { "anyOf": [ { "$ref": "#/definitions/FloatOrString" }, { "type": "null" } ] } }, "additionalProperties": false, "definitions": { "Access": { "type": "string", "enum": [ "private", "protected", "public" ] }, "AnyValue": true, "BigqueryPartitionConfig": { "description": "reference: https://github.com/dbt-labs/dbt-adapters/blob/main/dbt-bigquery/src/dbt/adapters/bigquery/relation_configs/_partition.py#L12-L13", "type": "object", "anyOf": [ { "$ref": "#/definitions/RangeConfig" }, { "$ref": "#/definitions/TimeConfig" } ], "required": [ "field" ], "properties": { "copy_partitions": { "default": false, "type": "boolean" }, "data_type": { "default": "date", "type": "string" }, "field": { "type": "string" } }, "additionalProperties": false }, "ClusterConfig": { "description": "Configuration for cluster by columns.\n\ndbt-core allows either of the variants for the `cluster_by` to allow cluster on a single column or on multiple columns", "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } } ] }, "DataLakeObjectCategory": { "description": "See `category` from https://developer.salesforce.com/docs/data/connectapi/references/spec?meta=postDataLakeObject", "type": "string", "enum": [ "Profile", "Engagement", "Directory_Table", "Insights", "Other" ] }, "DbtBatchSize": { "type": "string", "enum": [ "hour", "day", "month", "year" ] }, "DbtContract": { "type": "object", "properties": { "alias_types": { "default": true, "type": "boolean" }, "checksum": { "anyOf": [ { "$ref": "#/definitions/AnyValue" }, { "type": "null" } ] }, "enforced": { "default": false, "type": "boolean" } }, "additionalProperties": false }, "DbtIncrementalStrategy": { "oneOf": [ { "type": "string", "enum": [ "append", "merge", "delete+insert", "insert_overwrite", "microbatch" ] }, { "description": "replace_where (Databricks only) see https://docs.getdbt.com/reference/resource-configs/databricks-configs", "type": "string", "enum": [ "replace_where" ] }, { "type": "object", "required": [ "custom" ], "properties": { "custom": { "type": "string" } }, "additionalProperties": false } ] }, "DbtMaterialization": { "oneOf": [ { "type": "string", "enum": [ "snapshot", "seed", "view", "table", "incremental", "materialized_view", "external", "test", "ephemeral", "unit", "analysis", "function" ] }, { "description": "only for databricks", "type": "string", "enum": [ "streaming_table" ] }, { "description": "only for snowflake", "type": "string", "enum": [ "dynamic_table" ] }, { "description": "for inline SQL compilation", "type": "string", "enum": [ "inline" ] }, { "type": "object", "required": [ "unknown" ], "properties": { "unknown": { "type": "string" } }, "additionalProperties": false } ] }, "DbtQuoting": { "type": "object", "properties": { "database": { "default": null, "type": [ "boolean", "null" ] }, "identifier": { "default": null, "type": [ "boolean", "null" ] }, "schema": { "default": null, "type": [ "boolean", "null" ] }, "snowflake_ignore_case": { "type": [ "boolean", "null" ] } }, "additionalProperties": false }, "DbtUniqueKey": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } } ] }, "DocsConfig": { "type": "object", "properties": { "node_color": { "type": [ "string", "null" ] }, "show": { "default": true, "type": "boolean" } }, "additionalProperties": false }, "ExportConfigExportAs": { "type": "string", "enum": [ "table", "view", "cache" ] }, "FloatOrString": { "anyOf": [ { "type": "number", "format": "float" }, { "type": "string" } ] }, "FreshnessDefinition": { "type": "object", "properties": { "error_after": { "default": { "count": null, "period": null }, "anyOf": [ { "$ref": "#/definitions/FreshnessRules" }, { "type": "null" } ] }, "filter": { "type": [ "string", "null" ] }, "warn_after": { "default": { "count": null, "period": null }, "anyOf": [ { "$ref": "#/definitions/FreshnessRules" }, { "type": "null" } ] } }, "additionalProperties": false }, "FreshnessPeriod": { "type": "string", "enum": [ "minute", "hour", "day" ] }, "FreshnessRules": { "type": "object", "properties": { "count": { "type": [ "integer", "null" ], "format": "int64" }, "period": { "anyOf": [ { "$ref": "#/definitions/FreshnessPeriod" }, { "type": "null" } ] } }, "additionalProperties": false }, "FunctionKind": { "description": "Function kind enum with same values as UDFKind", "type": "string", "enum": [ "scalar", "aggregate", "table" ] }, "GrantAccessToTarget": { "type": "object", "properties": { "dataset": { "type": [ "string", "null" ] }, "project": { "type": [ "string", "null" ] } }, "additionalProperties": false }, "GrantConfig": { "description": "Wrapper type for grant configurations that normalizes values to arrays during serialization.\n\nThis type handles grant configurations with the following behavior: - Normalizes string values to arrays during serialization - Preserves insertion order using IndexMap", "type": "object", "additionalProperties": { "$ref": "#/definitions/StringOrArrayOfStrings" } }, "HardDeletes": { "type": "string", "enum": [ "ignore", "invalidate", "new_record" ] }, "HookConfig": { "type": "object", "properties": { "index": { "type": [ "string", "null" ] }, "sql": { "type": [ "string", "null" ] }, "transaction": { "type": [ "boolean", "null" ] } }, "additionalProperties": false }, "Hooks": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "$ref": "#/definitions/HookConfig" }, { "type": "array", "items": { "$ref": "#/definitions/HookConfig" } } ] }, "IndexesConfig": { "description": "A wrapper type for the `indexes` config field that handles flexible deserialization.\n\ndbt-core accepts both list and dictionary formats for indexes. This type accepts either format on input but always serializes as a list.\n\n# Example\n\n```ignore // Input (list): [{columns: [\"id\"], unique: true}] // Serializes as: [{columns: [\"id\"], unique: true}]\n\n// Input (dict): {\"my_index\": {columns: [\"id\"], unique: true}} // Serializes as: [{columns: [\"id\"], unique: true}] (keys are discarded) ```", "type": [ "array", "null" ], "items": { "$ref": "#/definitions/PostgresIndex" } }, "LogPath": { "type": "string", "enum": [ "logs" ] }, "ModelFreshness": { "type": "object", "properties": { "build_after": { "anyOf": [ { "$ref": "#/definitions/ModelFreshnessRules" }, { "type": "null" } ] } }, "additionalProperties": false }, "ModelFreshnessRules": { "type": "object", "properties": { "count": { "type": [ "integer", "null" ], "format": "int64" }, "period": { "anyOf": [ { "$ref": "#/definitions/FreshnessPeriod" }, { "type": "null" } ] }, "updates_on": { "anyOf": [ { "$ref": "#/definitions/UpdatesOn" }, { "type": "null" } ] } }, "additionalProperties": false }, "OnConfigurationChange": { "type": "string", "enum": [ "apply", "continue", "fail", "unknown" ] }, "OnSchemaChange": { "type": "string", "enum": [ "ignore", "append_new_columns", "fail", "sync_all_columns", "unknown" ] }, "PartitionConfig": { "description": "Configuration for partition by columns.\n\ndbt-core allows either of the variants for the `partition_by` in the model config but the bigquery-adapter throws RunTime error the behaviors are tested from the latest dbt-core + bigquery-adapter as this is written we're conformant to this behavior via here and via the `into_bigquery()` method", "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "$ref": "#/definitions/BigqueryPartitionConfig" } ] }, "PersistDocsConfig": { "type": "object", "properties": { "columns": { "type": [ "boolean", "null" ] }, "relation": { "type": [ "boolean", "null" ] } }, "additionalProperties": false }, "PostgresIndex": { "type": "object", "required": [ "columns" ], "properties": { "columns": { "type": "array", "items": { "type": "string" } }, "type": { "type": [ "string", "null" ] }, "unique": { "type": [ "boolean", "null" ] } }, "additionalProperties": false }, "PrimaryKeyConfig": { "description": "A wrapper type for the `primary_key` config field that normalizes serialization.\n\nIn dbt-core, primary_key values are \"listified\" - single strings are converted to single-element arrays. This type accepts either format on input but always serializes as arrays.\n\n# Example\n\n```ignore // Input: \"id\" // Serializes as: [\"id\"]\n\n// Input: [\"id\", \"tenant_id\"] // Serializes as: [\"id\", \"tenant_id\"] ```", "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "ProjectAnalysisConfig": { "type": "object", "properties": { "+docs": { "anyOf": [ { "$ref": "#/definitions/DocsConfig" }, { "type": "null" } ] }, "+enabled": { "default": null, "type": [ "boolean", "null" ] }, "+group": { "type": [ "string", "null" ] }, "+meta": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "+static_analysis": { "anyOf": [ { "$ref": "#/definitions/StaticAnalysisKind" }, { "type": "null" } ] }, "+tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] } }, "additionalProperties": { "$ref": "#/definitions/ProjectAnalysisConfig" } }, "ProjectDataTestConfig": { "type": "object", "properties": { "+adapter_properties": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "+alias": { "type": [ "string", "null" ] }, "+as_columnstore": { "default": null, "type": [ "boolean", "null" ] }, "+auto_liquid_cluster": { "default": null, "type": [ "boolean", "null" ] }, "+auto_refresh": { "default": null, "type": [ "boolean", "null" ] }, "+automatic_clustering": { "default": null, "type": [ "boolean", "null" ] }, "+backup": { "default": null, "type": [ "boolean", "null" ] }, "+base_location_root": { "type": [ "string", "null" ] }, "+base_location_subpath": { "type": [ "string", "null" ] }, "+bind": { "default": null, "type": [ "boolean", "null" ] }, "+buckets": { "type": [ "integer", "null" ], "format": "int64" }, "+catalog": { "type": [ "string", "null" ] }, "+catalog_name": { "type": [ "string", "null" ] }, "+cluster_by": { "anyOf": [ { "$ref": "#/definitions/ClusterConfig" }, { "type": "null" } ] }, "+clustered_by": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "+compression": { "type": [ "string", "null" ] }, "+copy_grants": { "default": null, "type": [ "boolean", "null" ] }, "+database": { "type": [ "string", "null" ] }, "+databricks_compute": { "type": [ "string", "null" ] }, "+databricks_tags": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "+dist": { "type": [ "string", "null" ] }, "+enable_refresh": { "default": null, "type": [ "boolean", "null" ] }, "+enabled": { "default": null, "type": [ "boolean", "null" ] }, "+error_if": { "type": [ "string", "null" ] }, "+external_volume": { "type": [ "string", "null" ] }, "+fail_calc": { "type": [ "string", "null" ] }, "+file_format": { "type": [ "string", "null" ] }, "+full_refresh": { "default": null, "type": [ "boolean", "null" ] }, "+grant_access_to": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/GrantAccessToTarget" } }, "+group": { "type": [ "string", "null" ] }, "+hours_to_expiration": { "default": null, "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "+include_full_name_in_path": { "default": null, "type": [ "boolean", "null" ] }, "+indexes": { "default": null, "allOf": [ { "$ref": "#/definitions/IndexesConfig" } ] }, "+initialize": { "type": [ "string", "null" ] }, "+job_execution_timeout_seconds": { "default": null, "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "+kms_key_name": { "type": [ "string", "null" ] }, "+labels": { "type": [ "object", "null" ], "additionalProperties": { "type": "string" } }, "+labels_from_meta": { "default": null, "type": [ "boolean", "null" ] }, "+limit": { "type": [ "integer", "null" ], "format": "int32" }, "+liquid_clustered_by": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "+location_root": { "type": [ "string", "null" ] }, "+matched_condition": { "type": [ "string", "null" ] }, "+max_staleness": { "type": [ "string", "null" ] }, "+merge_with_schema_evolution": { "default": null, "type": [ "boolean", "null" ] }, "+meta": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "+not_matched_by_source_action": { "type": [ "string", "null" ] }, "+not_matched_by_source_condition": { "type": [ "string", "null" ] }, "+not_matched_condition": { "type": [ "string", "null" ] }, "+partition_by": { "anyOf": [ { "$ref": "#/definitions/PartitionConfig" }, { "type": "null" } ] }, "+partition_expiration_days": { "default": null, "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "+partitions": { "type": [ "array", "null" ], "items": { "type": "string" } }, "+query_tag": { "anyOf": [ { "$ref": "#/definitions/QueryTag" }, { "type": "null" } ] }, "+quoting": { "anyOf": [ { "$ref": "#/definitions/DbtQuoting" }, { "type": "null" } ] }, "+refresh_interval_minutes": { "default": null, "type": [ "number", "null" ], "format": "double" }, "+refresh_mode": { "type": [ "string", "null" ] }, "+require_partition_filter": { "default": null, "type": [ "boolean", "null" ] }, "+row_access_policy": { "type": [ "string", "null" ] }, "+schedule": { "anyOf": [ { "$ref": "#/definitions/Schedule" }, { "type": "null" } ] }, "+schema": { "type": [ "string", "null" ] }, "+secure": { "default": null, "type": [ "boolean", "null" ] }, "+severity": { "anyOf": [ { "$ref": "#/definitions/Severity" }, { "type": "null" } ] }, "+skip_matched_step": { "default": null, "type": [ "boolean", "null" ] }, "+skip_not_matched_step": { "default": null, "type": [ "boolean", "null" ] }, "+snowflake_warehouse": { "type": [ "string", "null" ] }, "+sort": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "+sort_type": { "type": [ "string", "null" ] }, "+source_alias": { "type": [ "string", "null" ] }, "+sql_header": { "type": [ "string", "null" ] }, "+static_analysis": { "anyOf": [ { "$ref": "#/definitions/StaticAnalysisKind" }, { "type": "null" } ] }, "+store_failures": { "default": null, "type": [ "boolean", "null" ] }, "+store_failures_as": { "anyOf": [ { "$ref": "#/definitions/StoreFailuresAs" }, { "type": "null" } ] }, "+table_tag": { "type": [ "string", "null" ] }, "+table_type": { "default": null, "type": [ "string", "null" ] }, "+tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "+target_alias": { "type": [ "string", "null" ] }, "+target_lag": { "type": [ "string", "null" ] }, "+tblproperties": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "+tmp_relation_type": { "type": [ "string", "null" ] }, "+transient": { "default": null, "type": [ "boolean", "null" ] }, "+warn_if": { "type": [ "string", "null" ] }, "+where": { "type": [ "string", "null" ] } }, "additionalProperties": { "$ref": "#/definitions/ProjectDataTestConfig" } }, "ProjectDbtCloudConfig": { "type": "object", "properties": { "account-host": { "type": [ "string", "null" ] }, "account_id": { "anyOf": [ { "$ref": "#/definitions/StringOrInteger" }, { "type": "null" } ] }, "api_key": { "anyOf": [ { "$ref": "#/definitions/StringOrInteger" }, { "type": "null" } ] }, "application": { "anyOf": [ { "$ref": "#/definitions/StringOrInteger" }, { "type": "null" } ] }, "defer-env-id": { "anyOf": [ { "$ref": "#/definitions/StringOrInteger" }, { "type": "null" } ] }, "environment": { "anyOf": [ { "$ref": "#/definitions/StringOrInteger" }, { "type": "null" } ] }, "job-id": { "anyOf": [ { "$ref": "#/definitions/StringOrInteger" }, { "type": "null" } ] }, "project-id": { "anyOf": [ { "$ref": "#/definitions/StringOrInteger" }, { "type": "null" } ] }, "run-id": { "anyOf": [ { "$ref": "#/definitions/StringOrInteger" }, { "type": "null" } ] }, "tenant_hostname": { "type": [ "string", "null" ] } }, "additionalProperties": false }, "ProjectExposureConfig": { "type": "object", "properties": { "+enabled": { "default": null, "type": [ "boolean", "null" ] }, "+meta": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "+tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] } }, "additionalProperties": { "$ref": "#/definitions/ProjectExposureConfig" } }, "ProjectFunctionConfig": { "type": "object", "properties": { "+access": { "anyOf": [ { "$ref": "#/definitions/Access" }, { "type": "null" } ] }, "+alias": { "type": [ "string", "null" ] }, "+database": { "type": [ "string", "null" ] }, "+description": { "type": [ "string", "null" ] }, "+docs": { "anyOf": [ { "$ref": "#/definitions/DocsConfig" }, { "type": "null" } ] }, "+enabled": { "type": [ "boolean", "null" ] }, "+entry_point": { "type": [ "string", "null" ] }, "+grants": { "$ref": "#/definitions/GrantConfig" }, "+group": { "type": [ "string", "null" ] }, "+meta": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "+on_configuration_change": { "type": [ "string", "null" ] }, "+quoting": { "anyOf": [ { "$ref": "#/definitions/DbtQuoting" }, { "type": "null" } ] }, "+runtime_version": { "type": [ "string", "null" ] }, "+schema": { "type": [ "string", "null" ] }, "+static_analysis": { "anyOf": [ { "$ref": "#/definitions/StaticAnalysisKind" }, { "type": "null" } ] }, "+tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "+type": { "anyOf": [ { "$ref": "#/definitions/FunctionKind" }, { "type": "null" } ] }, "+volatility": { "anyOf": [ { "$ref": "#/definitions/Volatility" }, { "type": "null" } ] } }, "additionalProperties": { "$ref": "#/definitions/ProjectFunctionConfig" } }, "ProjectMetricConfigs": { "type": "object", "properties": { "+enabled": { "type": [ "boolean", "null" ] }, "+group": { "type": [ "string", "null" ] }, "+meta": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "+tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] } }, "additionalProperties": { "$ref": "#/definitions/ProjectMetricConfigs" } }, "ProjectModelConfig": { "type": "object", "properties": { "+access": { "anyOf": [ { "$ref": "#/definitions/Access" }, { "type": "null" } ] }, "+adapter_properties": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "+additional_libs": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/AnyValue" } }, "+alias": { "type": [ "string", "null" ] }, "+as_columnstore": { "default": null, "type": [ "boolean", "null" ] }, "+auto_liquid_cluster": { "default": null, "type": [ "boolean", "null" ] }, "+auto_refresh": { "default": null, "type": [ "boolean", "null" ] }, "+automatic_clustering": { "default": null, "type": [ "boolean", "null" ] }, "+backup": { "default": null, "type": [ "boolean", "null" ] }, "+base_location_root": { "type": [ "string", "null" ] }, "+base_location_subpath": { "type": [ "string", "null" ] }, "+batch_id": { "type": [ "string", "null" ] }, "+batch_size": { "anyOf": [ { "$ref": "#/definitions/DbtBatchSize" }, { "type": "null" } ] }, "+begin": { "type": [ "string", "null" ] }, "+bind": { "default": null, "type": [ "boolean", "null" ] }, "+buckets": { "type": [ "integer", "null" ], "format": "int64" }, "+catalog": { "type": [ "string", "null" ] }, "+catalog_name": { "type": [ "string", "null" ] }, "+category": { "anyOf": [ { "$ref": "#/definitions/DataLakeObjectCategory" }, { "type": "null" } ] }, "+cluster_by": { "anyOf": [ { "$ref": "#/definitions/ClusterConfig" }, { "type": "null" } ] }, "+cluster_id": { "type": [ "string", "null" ] }, "+clustered_by": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "+column_types": { "type": [ "object", "null" ], "additionalProperties": { "type": "string" } }, "+compression": { "type": [ "string", "null" ] }, "+concurrent_batches": { "default": null, "type": [ "boolean", "null" ] }, "+contract": { "anyOf": [ { "$ref": "#/definitions/DbtContract" }, { "type": "null" } ] }, "+copy_grants": { "default": null, "type": [ "boolean", "null" ] }, "+create_notebook": { "default": null, "type": [ "boolean", "null" ] }, "+database": { "type": [ "string", "null" ] }, "+databricks_compute": { "type": [ "string", "null" ] }, "+databricks_tags": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "+dataproc_cluster_name": { "type": [ "string", "null" ] }, "+description": { "type": [ "string", "null" ] }, "+dist": { "type": [ "string", "null" ] }, "+docs": { "anyOf": [ { "$ref": "#/definitions/DocsConfig" }, { "type": "null" } ] }, "+enable_list_inference": { "default": null, "type": [ "boolean", "null" ] }, "+enable_refresh": { "default": null, "type": [ "boolean", "null" ] }, "+enabled": { "default": null, "type": [ "boolean", "null" ] }, "+event_time": { "type": [ "string", "null" ] }, "+external_access_integrations": { "description": "Snowflake Python model config: external access integrations for network access", "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "+external_volume": { "type": [ "string", "null" ] }, "+file_format": { "type": [ "string", "null" ] }, "+freshness": { "anyOf": [ { "$ref": "#/definitions/ModelFreshness" }, { "type": "null" } ] }, "+full_refresh": { "default": null, "type": [ "boolean", "null" ] }, "+grant_access_to": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/GrantAccessToTarget" } }, "+grants": { "$ref": "#/definitions/GrantConfig" }, "+group": { "type": [ "string", "null" ] }, "+hours_to_expiration": { "default": null, "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "+http_path": { "type": [ "string", "null" ] }, "+imports": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "+include_full_name_in_path": { "default": null, "type": [ "boolean", "null" ] }, "+incremental_predicates": { "type": [ "array", "null" ], "items": { "type": "string" } }, "+incremental_strategy": { "anyOf": [ { "$ref": "#/definitions/DbtIncrementalStrategy" }, { "type": "null" } ] }, "+index_url": { "type": [ "string", "null" ] }, "+indexes": { "default": null, "allOf": [ { "$ref": "#/definitions/IndexesConfig" } ] }, "+initialize": { "type": [ "string", "null" ] }, "+intermediate_format": { "type": [ "string", "null" ] }, "+jar_file_uri": { "type": [ "string", "null" ] }, "+job_cluster_config": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "+job_execution_timeout_seconds": { "default": null, "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "+kms_key_name": { "type": [ "string", "null" ] }, "+labels": { "type": [ "object", "null" ], "additionalProperties": { "type": "string" } }, "+labels_from_meta": { "default": null, "type": [ "boolean", "null" ] }, "+liquid_clustered_by": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "+location": { "type": [ "string", "null" ] }, "+location_root": { "type": [ "string", "null" ] }, "+lookback": { "type": [ "integer", "null" ], "format": "int32" }, "+matched_condition": { "type": [ "string", "null" ] }, "+materialized": { "anyOf": [ { "$ref": "#/definitions/DbtMaterialization" }, { "type": "null" } ] }, "+max_staleness": { "type": [ "string", "null" ] }, "+merge_exclude_columns": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "+merge_update_columns": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "+merge_with_schema_evolution": { "default": null, "type": [ "boolean", "null" ] }, "+meta": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "+not_matched_by_source_action": { "type": [ "string", "null" ] }, "+not_matched_by_source_condition": { "type": [ "string", "null" ] }, "+not_matched_condition": { "type": [ "string", "null" ] }, "+notebook_template_id": { "default": null, "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "+on_configuration_change": { "anyOf": [ { "$ref": "#/definitions/OnConfigurationChange" }, { "type": "null" } ] }, "+on_schema_change": { "anyOf": [ { "$ref": "#/definitions/OnSchemaChange" }, { "type": "null" } ] }, "+packages": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "+partition_by": { "anyOf": [ { "$ref": "#/definitions/PartitionConfig" }, { "type": "null" } ] }, "+partition_expiration_days": { "default": null, "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "+partitions": { "type": [ "array", "null" ], "items": { "type": "string" } }, "+persist_docs": { "anyOf": [ { "$ref": "#/definitions/PersistDocsConfig" }, { "type": "null" } ] }, "+post-hook": { "anyOf": [ { "$ref": "#/definitions/Hooks" }, { "type": "null" } ] }, "+pre-hook": { "anyOf": [ { "$ref": "#/definitions/Hooks" }, { "type": "null" } ] }, "+predicates": { "type": [ "array", "null" ], "items": { "type": "string" } }, "+primary_key": { "default": null, "allOf": [ { "$ref": "#/definitions/PrimaryKeyConfig" } ] }, "+python_job_config": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "+python_version": { "type": [ "string", "null" ] }, "+query_tag": { "anyOf": [ { "$ref": "#/definitions/QueryTag" }, { "type": "null" } ] }, "+quoting": { "anyOf": [ { "$ref": "#/definitions/DbtQuoting" }, { "type": "null" } ] }, "+refresh_interval_minutes": { "default": null, "type": [ "number", "null" ], "format": "double" }, "+refresh_mode": { "type": [ "string", "null" ] }, "+require_partition_filter": { "default": null, "type": [ "boolean", "null" ] }, "+resource_tags": { "type": [ "object", "null" ], "additionalProperties": { "type": "string" } }, "+row_access_policy": { "type": [ "string", "null" ] }, "+schedule": { "anyOf": [ { "$ref": "#/definitions/Schedule" }, { "type": "null" } ] }, "+schema": { "type": [ "string", "null" ] }, "+secrets": { "description": "Snowflake Python model config: secrets to pass to stored procedure", "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "+secure": { "default": null, "type": [ "boolean", "null" ] }, "+skip_matched_step": { "default": null, "type": [ "boolean", "null" ] }, "+skip_not_matched_step": { "default": null, "type": [ "boolean", "null" ] }, "+snowflake_warehouse": { "type": [ "string", "null" ] }, "+sort": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "+sort_type": { "type": [ "string", "null" ] }, "+source_alias": { "type": [ "string", "null" ] }, "+sql_header": { "type": [ "string", "null" ] }, "+static_analysis": { "anyOf": [ { "$ref": "#/definitions/StaticAnalysisKind" }, { "type": "null" } ] }, "+submission_method": { "type": [ "string", "null" ] }, "+sync": { "description": "Schema synchronization configuration", "anyOf": [ { "$ref": "#/definitions/SyncConfig" }, { "type": "null" } ] }, "+table_format": { "type": [ "string", "null" ] }, "+table_tag": { "type": [ "string", "null" ] }, "+table_type": { "default": null, "type": [ "string", "null" ] }, "+tags": { "$ref": "#/definitions/StringOrArrayOfStrings" }, "+target_alias": { "type": [ "string", "null" ] }, "+target_lag": { "type": [ "string", "null" ] }, "+tblproperties": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "+timeout": { "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "+tmp_relation_type": { "type": [ "string", "null" ] }, "+transient": { "default": null, "type": [ "boolean", "null" ] }, "+unique_key": { "anyOf": [ { "$ref": "#/definitions/DbtUniqueKey" }, { "type": "null" } ] }, "+use_anonymous_sproc": { "description": "Snowflake Python model config: use anonymous stored procedure (default: true)", "default": null, "type": [ "boolean", "null" ] }, "+user_folder_for_python": { "default": null, "type": [ "boolean", "null" ] } }, "additionalProperties": { "$ref": "#/definitions/ProjectModelConfig" } }, "ProjectSavedQueryConfig": { "type": "object", "properties": { "+cache": { "anyOf": [ { "$ref": "#/definitions/SavedQueryCache" }, { "type": "null" } ] }, "+enabled": { "type": [ "boolean", "null" ] }, "+export_as": { "anyOf": [ { "$ref": "#/definitions/ExportConfigExportAs" }, { "type": "null" } ] }, "+group": { "type": [ "string", "null" ] }, "+meta": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "+schema": { "type": [ "string", "null" ] }, "+tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] } }, "additionalProperties": { "$ref": "#/definitions/ProjectSavedQueryConfig" } }, "ProjectSeedConfig": { "type": "object", "properties": { "+adapter_properties": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "+alias": { "type": [ "string", "null" ] }, "+as_columnstore": { "type": [ "boolean", "null" ] }, "+auto_liquid_cluster": { "type": [ "boolean", "null" ] }, "+auto_refresh": { "type": [ "boolean", "null" ] }, "+automatic_clustering": { "type": [ "boolean", "null" ] }, "+backup": { "type": [ "boolean", "null" ] }, "+base_location_root": { "type": [ "string", "null" ] }, "+base_location_subpath": { "type": [ "string", "null" ] }, "+bind": { "type": [ "boolean", "null" ] }, "+buckets": { "type": [ "integer", "null" ], "format": "int64" }, "+catalog": { "type": [ "string", "null" ] }, "+catalog_name": { "type": [ "string", "null" ] }, "+cluster_by": { "anyOf": [ { "$ref": "#/definitions/ClusterConfig" }, { "type": "null" } ] }, "+clustered_by": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "+column_types": { "type": [ "object", "null" ], "additionalProperties": { "type": "string" } }, "+compression": { "type": [ "string", "null" ] }, "+copy_grants": { "type": [ "boolean", "null" ] }, "+database": { "type": [ "string", "null" ] }, "+databricks_compute": { "type": [ "string", "null" ] }, "+databricks_tags": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "+delimiter": { "type": [ "string", "null" ] }, "+dist": { "type": [ "string", "null" ] }, "+docs": { "anyOf": [ { "$ref": "#/definitions/DocsConfig" }, { "type": "null" } ] }, "+enable_refresh": { "type": [ "boolean", "null" ] }, "+enabled": { "type": [ "boolean", "null" ] }, "+event_time": { "type": [ "string", "null" ] }, "+external_volume": { "type": [ "string", "null" ] }, "+file_format": { "type": [ "string", "null" ] }, "+full_refresh": { "type": [ "boolean", "null" ] }, "+grant_access_to": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/GrantAccessToTarget" } }, "+grants": { "$ref": "#/definitions/GrantConfig" }, "+group": { "type": [ "string", "null" ] }, "+hours_to_expiration": { "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "+include_full_name_in_path": { "type": [ "boolean", "null" ] }, "+indexes": { "default": null, "allOf": [ { "$ref": "#/definitions/IndexesConfig" } ] }, "+initialize": { "type": [ "string", "null" ] }, "+job_execution_timeout_seconds": { "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "+kms_key_name": { "type": [ "string", "null" ] }, "+labels": { "type": [ "object", "null" ], "additionalProperties": { "type": "string" } }, "+labels_from_meta": { "type": [ "boolean", "null" ] }, "+liquid_clustered_by": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "+location_root": { "type": [ "string", "null" ] }, "+matched_condition": { "type": [ "string", "null" ] }, "+max_staleness": { "type": [ "string", "null" ] }, "+merge_with_schema_evolution": { "type": [ "boolean", "null" ] }, "+meta": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "+not_matched_by_source_action": { "type": [ "string", "null" ] }, "+not_matched_by_source_condition": { "type": [ "string", "null" ] }, "+not_matched_condition": { "type": [ "string", "null" ] }, "+partition_by": { "anyOf": [ { "$ref": "#/definitions/PartitionConfig" }, { "type": "null" } ] }, "+partition_expiration_days": { "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "+partitions": { "type": [ "array", "null" ], "items": { "type": "string" } }, "+persist_docs": { "anyOf": [ { "$ref": "#/definitions/PersistDocsConfig" }, { "type": "null" } ] }, "+post-hook": { "anyOf": [ { "$ref": "#/definitions/Hooks" }, { "type": "null" } ] }, "+pre-hook": { "anyOf": [ { "$ref": "#/definitions/Hooks" }, { "type": "null" } ] }, "+query_tag": { "anyOf": [ { "$ref": "#/definitions/QueryTag" }, { "type": "null" } ] }, "+quote_columns": { "type": [ "boolean", "null" ] }, "+quoting": { "anyOf": [ { "$ref": "#/definitions/DbtQuoting" }, { "type": "null" } ] }, "+refresh_interval_minutes": { "type": [ "number", "null" ], "format": "double" }, "+refresh_mode": { "type": [ "string", "null" ] }, "+require_partition_filter": { "type": [ "boolean", "null" ] }, "+row_access_policy": { "type": [ "string", "null" ] }, "+schedule": { "anyOf": [ { "$ref": "#/definitions/Schedule" }, { "type": "null" } ] }, "+schema": { "type": [ "string", "null" ] }, "+secure": { "type": [ "boolean", "null" ] }, "+skip_matched_step": { "type": [ "boolean", "null" ] }, "+skip_not_matched_step": { "type": [ "boolean", "null" ] }, "+snowflake_warehouse": { "type": [ "string", "null" ] }, "+sort": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "+sort_type": { "type": [ "string", "null" ] }, "+source_alias": { "type": [ "string", "null" ] }, "+static_analysis": { "anyOf": [ { "$ref": "#/definitions/StaticAnalysisKind" }, { "type": "null" } ] }, "+table_tag": { "type": [ "string", "null" ] }, "+table_type": { "type": [ "string", "null" ] }, "+tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "+target_alias": { "type": [ "string", "null" ] }, "+target_lag": { "type": [ "string", "null" ] }, "+tblproperties": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "+tmp_relation_type": { "type": [ "string", "null" ] }, "+transient": { "type": [ "boolean", "null" ] } }, "additionalProperties": { "$ref": "#/definitions/ProjectSeedConfig" } }, "ProjectSemanticModelConfig": { "type": "object", "properties": { "+enabled": { "default": null, "type": [ "boolean", "null" ] }, "+group": { "type": [ "string", "null" ] }, "+meta": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "+tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] } }, "additionalProperties": { "$ref": "#/definitions/ProjectSemanticModelConfig" } }, "ProjectSnapshotConfig": { "type": "object", "properties": { "+adapter_properties": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "+alias": { "type": [ "string", "null" ] }, "+as_columnstore": { "default": null, "type": [ "boolean", "null" ] }, "+auto_liquid_cluster": { "default": null, "type": [ "boolean", "null" ] }, "+auto_refresh": { "default": null, "type": [ "boolean", "null" ] }, "+automatic_clustering": { "default": null, "type": [ "boolean", "null" ] }, "+backup": { "default": null, "type": [ "boolean", "null" ] }, "+base_location_root": { "type": [ "string", "null" ] }, "+base_location_subpath": { "type": [ "string", "null" ] }, "+bind": { "default": null, "type": [ "boolean", "null" ] }, "+buckets": { "type": [ "integer", "null" ], "format": "int64" }, "+catalog": { "type": [ "string", "null" ] }, "+catalog_name": { "type": [ "string", "null" ] }, "+check_cols": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "+cluster_by": { "anyOf": [ { "$ref": "#/definitions/ClusterConfig" }, { "type": "null" } ] }, "+clustered_by": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "+compression": { "type": [ "string", "null" ] }, "+copy_grants": { "default": null, "type": [ "boolean", "null" ] }, "+database": { "type": [ "string", "null" ] }, "+databricks_compute": { "type": [ "string", "null" ] }, "+databricks_tags": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "+dbt_valid_to_current": { "type": [ "string", "null" ] }, "+dist": { "type": [ "string", "null" ] }, "+docs": { "anyOf": [ { "$ref": "#/definitions/DocsConfig" }, { "type": "null" } ] }, "+enable_refresh": { "default": null, "type": [ "boolean", "null" ] }, "+enabled": { "default": null, "type": [ "boolean", "null" ] }, "+event_time": { "type": [ "string", "null" ] }, "+external_volume": { "type": [ "string", "null" ] }, "+file_format": { "type": [ "string", "null" ] }, "+full_refresh": { "default": null, "type": [ "boolean", "null" ] }, "+grant_access_to": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/GrantAccessToTarget" } }, "+grants": { "$ref": "#/definitions/GrantConfig" }, "+group": { "type": [ "string", "null" ] }, "+hard_deletes": { "anyOf": [ { "$ref": "#/definitions/HardDeletes" }, { "type": "null" } ] }, "+hours_to_expiration": { "default": null, "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "+include_full_name_in_path": { "default": null, "type": [ "boolean", "null" ] }, "+indexes": { "default": null, "allOf": [ { "$ref": "#/definitions/IndexesConfig" } ] }, "+initialize": { "type": [ "string", "null" ] }, "+invalidate_hard_deletes": { "default": null, "type": [ "boolean", "null" ] }, "+job_execution_timeout_seconds": { "default": null, "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "+kms_key_name": { "type": [ "string", "null" ] }, "+labels": { "type": [ "object", "null" ], "additionalProperties": { "type": "string" } }, "+labels_from_meta": { "default": null, "type": [ "boolean", "null" ] }, "+liquid_clustered_by": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "+location_root": { "type": [ "string", "null" ] }, "+matched_condition": { "type": [ "string", "null" ] }, "+materialized": { "anyOf": [ { "$ref": "#/definitions/DbtMaterialization" }, { "type": "null" } ] }, "+max_staleness": { "type": [ "string", "null" ] }, "+merge_with_schema_evolution": { "default": null, "type": [ "boolean", "null" ] }, "+meta": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "+not_matched_by_source_action": { "type": [ "string", "null" ] }, "+not_matched_by_source_condition": { "type": [ "string", "null" ] }, "+not_matched_condition": { "type": [ "string", "null" ] }, "+partition_by": { "anyOf": [ { "$ref": "#/definitions/PartitionConfig" }, { "type": "null" } ] }, "+partition_expiration_days": { "default": null, "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "+partitions": { "type": [ "array", "null" ], "items": { "type": "string" } }, "+persist_docs": { "anyOf": [ { "$ref": "#/definitions/PersistDocsConfig" }, { "type": "null" } ] }, "+post-hook": { "anyOf": [ { "$ref": "#/definitions/Hooks" }, { "type": "null" } ] }, "+pre-hook": { "anyOf": [ { "$ref": "#/definitions/Hooks" }, { "type": "null" } ] }, "+query_tag": { "anyOf": [ { "$ref": "#/definitions/QueryTag" }, { "type": "null" } ] }, "+quote_columns": { "default": null, "type": [ "boolean", "null" ] }, "+quoting": { "anyOf": [ { "$ref": "#/definitions/DbtQuoting" }, { "type": "null" } ] }, "+refresh_interval_minutes": { "default": null, "type": [ "number", "null" ], "format": "double" }, "+refresh_mode": { "type": [ "string", "null" ] }, "+require_partition_filter": { "default": null, "type": [ "boolean", "null" ] }, "+row_access_policy": { "type": [ "string", "null" ] }, "+schedule": { "anyOf": [ { "$ref": "#/definitions/Schedule" }, { "type": "null" } ] }, "+schema": { "type": [ "string", "null" ] }, "+secure": { "default": null, "type": [ "boolean", "null" ] }, "+skip_matched_step": { "default": null, "type": [ "boolean", "null" ] }, "+skip_not_matched_step": { "default": null, "type": [ "boolean", "null" ] }, "+snapshot_meta_column_names": { "anyOf": [ { "$ref": "#/definitions/SnapshotMetaColumnNames" }, { "type": "null" } ] }, "+snowflake_warehouse": { "type": [ "string", "null" ] }, "+sort": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "+sort_type": { "type": [ "string", "null" ] }, "+source_alias": { "type": [ "string", "null" ] }, "+static_analysis": { "anyOf": [ { "$ref": "#/definitions/StaticAnalysisKind" }, { "type": "null" } ] }, "+strategy": { "type": [ "string", "null" ] }, "+sync": { "description": "Schema synchronization configuration", "anyOf": [ { "$ref": "#/definitions/SyncConfig" }, { "type": "null" } ] }, "+table_tag": { "type": [ "string", "null" ] }, "+table_type": { "default": null, "type": [ "string", "null" ] }, "+tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "+target_alias": { "type": [ "string", "null" ] }, "+target_database": { "type": [ "string", "null" ] }, "+target_lag": { "type": [ "string", "null" ] }, "+target_schema": { "type": [ "string", "null" ] }, "+tblproperties": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "+tmp_relation_type": { "type": [ "string", "null" ] }, "+transient": { "default": null, "type": [ "boolean", "null" ] }, "+unique_key": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "+updated_at": { "type": [ "string", "null" ] } }, "additionalProperties": { "$ref": "#/definitions/ProjectSnapshotConfig" } }, "ProjectSourceConfig": { "type": "object", "properties": { "+adapter_properties": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "+as_columnstore": { "default": null, "type": [ "boolean", "null" ] }, "+auto_liquid_cluster": { "default": null, "type": [ "boolean", "null" ] }, "+auto_refresh": { "default": null, "type": [ "boolean", "null" ] }, "+automatic_clustering": { "default": null, "type": [ "boolean", "null" ] }, "+backup": { "default": null, "type": [ "boolean", "null" ] }, "+base_location_root": { "type": [ "string", "null" ] }, "+base_location_subpath": { "type": [ "string", "null" ] }, "+bind": { "default": null, "type": [ "boolean", "null" ] }, "+buckets": { "type": [ "integer", "null" ], "format": "int64" }, "+catalog": { "type": [ "string", "null" ] }, "+catalog_name": { "type": [ "string", "null" ] }, "+cluster_by": { "anyOf": [ { "$ref": "#/definitions/ClusterConfig" }, { "type": "null" } ] }, "+clustered_by": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "+compression": { "type": [ "string", "null" ] }, "+copy_grants": { "default": null, "type": [ "boolean", "null" ] }, "+databricks_compute": { "type": [ "string", "null" ] }, "+databricks_tags": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "+dist": { "type": [ "string", "null" ] }, "+enable_refresh": { "default": null, "type": [ "boolean", "null" ] }, "+enabled": { "default": null, "type": [ "boolean", "null" ] }, "+event_time": { "type": [ "string", "null" ] }, "+external_volume": { "type": [ "string", "null" ] }, "+file_format": { "type": [ "string", "null" ] }, "+freshness": { "anyOf": [ { "$ref": "#/definitions/FreshnessDefinition" }, { "type": "null" } ] }, "+grant_access_to": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/GrantAccessToTarget" } }, "+hours_to_expiration": { "default": null, "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "+include_full_name_in_path": { "default": null, "type": [ "boolean", "null" ] }, "+indexes": { "default": null, "allOf": [ { "$ref": "#/definitions/IndexesConfig" } ] }, "+initialize": { "type": [ "string", "null" ] }, "+job_execution_timeout_seconds": { "default": null, "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "+kms_key_name": { "type": [ "string", "null" ] }, "+labels": { "type": [ "object", "null" ], "additionalProperties": { "type": "string" } }, "+labels_from_meta": { "default": null, "type": [ "boolean", "null" ] }, "+liquid_clustered_by": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "+loaded_at_field": { "type": [ "string", "null" ] }, "+loaded_at_query": { "type": [ "string", "null" ] }, "+location_root": { "type": [ "string", "null" ] }, "+matched_condition": { "type": [ "string", "null" ] }, "+max_staleness": { "type": [ "string", "null" ] }, "+merge_with_schema_evolution": { "default": null, "type": [ "boolean", "null" ] }, "+meta": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "+not_matched_by_source_action": { "type": [ "string", "null" ] }, "+not_matched_by_source_condition": { "type": [ "string", "null" ] }, "+not_matched_condition": { "type": [ "string", "null" ] }, "+partition_by": { "anyOf": [ { "$ref": "#/definitions/PartitionConfig" }, { "type": "null" } ] }, "+partition_expiration_days": { "default": null, "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "+partitions": { "type": [ "array", "null" ], "items": { "type": "string" } }, "+query_tag": { "anyOf": [ { "$ref": "#/definitions/QueryTag" }, { "type": "null" } ] }, "+quoting": { "anyOf": [ { "$ref": "#/definitions/DbtQuoting" }, { "type": "null" } ] }, "+refresh_interval_minutes": { "default": null, "type": [ "number", "null" ], "format": "double" }, "+refresh_mode": { "type": [ "string", "null" ] }, "+require_partition_filter": { "default": null, "type": [ "boolean", "null" ] }, "+row_access_policy": { "type": [ "string", "null" ] }, "+schedule": { "anyOf": [ { "$ref": "#/definitions/Schedule" }, { "type": "null" } ] }, "+schema_origin": { "description": "Specifies where the schema metadata originates: 'remote' (default) or 'local'", "anyOf": [ { "$ref": "#/definitions/SchemaOrigin" }, { "type": "null" } ] }, "+secure": { "default": null, "type": [ "boolean", "null" ] }, "+skip_matched_step": { "default": null, "type": [ "boolean", "null" ] }, "+skip_not_matched_step": { "default": null, "type": [ "boolean", "null" ] }, "+snowflake_warehouse": { "type": [ "string", "null" ] }, "+sort": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "+sort_type": { "type": [ "string", "null" ] }, "+source_alias": { "type": [ "string", "null" ] }, "+static_analysis": { "anyOf": [ { "$ref": "#/definitions/StaticAnalysisKind" }, { "type": "null" } ] }, "+sync": { "description": "Schema synchronization configuration", "anyOf": [ { "$ref": "#/definitions/SyncConfig" }, { "type": "null" } ] }, "+table_tag": { "type": [ "string", "null" ] }, "+table_type": { "default": null, "type": [ "string", "null" ] }, "+tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "+target_alias": { "type": [ "string", "null" ] }, "+target_lag": { "type": [ "string", "null" ] }, "+tblproperties": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "+tmp_relation_type": { "type": [ "string", "null" ] }, "+transient": { "default": null, "type": [ "boolean", "null" ] } }, "additionalProperties": { "$ref": "#/definitions/ProjectSourceConfig" } }, "ProjectUnitTestConfig": { "type": "object", "properties": { "+adapter_properties": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "+as_columnstore": { "default": null, "type": [ "boolean", "null" ] }, "+auto_liquid_cluster": { "default": null, "type": [ "boolean", "null" ] }, "+auto_refresh": { "default": null, "type": [ "boolean", "null" ] }, "+automatic_clustering": { "default": null, "type": [ "boolean", "null" ] }, "+backup": { "default": null, "type": [ "boolean", "null" ] }, "+base_location_root": { "type": [ "string", "null" ] }, "+base_location_subpath": { "type": [ "string", "null" ] }, "+bind": { "default": null, "type": [ "boolean", "null" ] }, "+buckets": { "type": [ "integer", "null" ], "format": "int64" }, "+catalog": { "type": [ "string", "null" ] }, "+catalog_name": { "type": [ "string", "null" ] }, "+cluster_by": { "anyOf": [ { "$ref": "#/definitions/ClusterConfig" }, { "type": "null" } ] }, "+clustered_by": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "+compression": { "type": [ "string", "null" ] }, "+copy_grants": { "default": null, "type": [ "boolean", "null" ] }, "+databricks_compute": { "type": [ "string", "null" ] }, "+databricks_tags": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "+dist": { "type": [ "string", "null" ] }, "+enable_refresh": { "default": null, "type": [ "boolean", "null" ] }, "+enabled": { "default": null, "type": [ "boolean", "null" ] }, "+external_volume": { "type": [ "string", "null" ] }, "+file_format": { "type": [ "string", "null" ] }, "+grant_access_to": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/GrantAccessToTarget" } }, "+hours_to_expiration": { "default": null, "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "+include_full_name_in_path": { "default": null, "type": [ "boolean", "null" ] }, "+indexes": { "default": null, "allOf": [ { "$ref": "#/definitions/IndexesConfig" } ] }, "+initialize": { "type": [ "string", "null" ] }, "+job_execution_timeout_seconds": { "default": null, "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "+kms_key_name": { "type": [ "string", "null" ] }, "+labels": { "type": [ "object", "null" ], "additionalProperties": { "type": "string" } }, "+labels_from_meta": { "default": null, "type": [ "boolean", "null" ] }, "+liquid_clustered_by": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "+location_root": { "type": [ "string", "null" ] }, "+matched_condition": { "type": [ "string", "null" ] }, "+max_staleness": { "type": [ "string", "null" ] }, "+merge_with_schema_evolution": { "default": null, "type": [ "boolean", "null" ] }, "+meta": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "+not_matched_by_source_action": { "type": [ "string", "null" ] }, "+not_matched_by_source_condition": { "type": [ "string", "null" ] }, "+not_matched_condition": { "type": [ "string", "null" ] }, "+partition_by": { "anyOf": [ { "$ref": "#/definitions/PartitionConfig" }, { "type": "null" } ] }, "+partition_expiration_days": { "default": null, "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "+partitions": { "type": [ "array", "null" ], "items": { "type": "string" } }, "+query_tag": { "anyOf": [ { "$ref": "#/definitions/QueryTag" }, { "type": "null" } ] }, "+refresh_interval_minutes": { "default": null, "type": [ "number", "null" ], "format": "double" }, "+refresh_mode": { "type": [ "string", "null" ] }, "+require_partition_filter": { "default": null, "type": [ "boolean", "null" ] }, "+row_access_policy": { "type": [ "string", "null" ] }, "+schedule": { "anyOf": [ { "$ref": "#/definitions/Schedule" }, { "type": "null" } ] }, "+secure": { "default": null, "type": [ "boolean", "null" ] }, "+skip_matched_step": { "default": null, "type": [ "boolean", "null" ] }, "+skip_not_matched_step": { "default": null, "type": [ "boolean", "null" ] }, "+snowflake_warehouse": { "type": [ "string", "null" ] }, "+sort": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "+sort_type": { "type": [ "string", "null" ] }, "+source_alias": { "type": [ "string", "null" ] }, "+static_analysis": { "anyOf": [ { "$ref": "#/definitions/StaticAnalysisKind" }, { "type": "null" } ] }, "+table_tag": { "type": [ "string", "null" ] }, "+table_type": { "default": null, "type": [ "string", "null" ] }, "+tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "+target_alias": { "type": [ "string", "null" ] }, "+target_lag": { "type": [ "string", "null" ] }, "+tblproperties": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "+tmp_relation_type": { "type": [ "string", "null" ] }, "+transient": { "default": null, "type": [ "boolean", "null" ] } }, "additionalProperties": { "$ref": "#/definitions/ProjectUnitTestConfig" } }, "QueryComment": { "anyOf": [ { "type": "string" }, { "$ref": "#/definitions/AnyValue" } ] }, "QueryTag": { "description": "A wrapper type for the `query_tag` config field that handles flexible deserialization.\n\nAccepts strings, dictionaries, or sequences for query_tag. Maps and sequences are JSON-serialized to strings.\n\n# Example\n\n```ignore // Input: \"my-tag\" // Stores as: \"my-tag\"\n\n// Input: {\"project\": \"foo\", \"env\": \"prod\"} // Stores as: \"{\\\"project\\\":\\\"foo\\\",\\\"env\\\":\\\"prod\\\"}\" ```", "type": "string" }, "Range": { "type": "object", "required": [ "end", "interval", "start" ], "properties": { "end": { "type": "integer", "format": "int64" }, "interval": { "type": "integer", "format": "int64" }, "start": { "type": "integer", "format": "int64" } }, "additionalProperties": false }, "RangeConfig": { "type": "object", "required": [ "range" ], "properties": { "range": { "$ref": "#/definitions/Range" } }, "additionalProperties": false }, "SavedQueryCache": { "type": "object", "properties": { "enabled": { "type": [ "boolean", "null" ] } }, "additionalProperties": false }, "Schedule": { "description": "Schedule configuration that accepts both string and structured formats. This allows users to specify schedule as either: - A string: `schedule: \"USING CRON 0,15,30,45 * * * * UTC\"` - A structured config: `schedule: { cron: \"0 * * * *\", time_zone_value: \"UTC\" }`", "anyOf": [ { "type": "string" }, { "$ref": "#/definitions/ScheduleConfig" } ] }, "ScheduleConfig": { "type": "object", "properties": { "cron": { "type": [ "string", "null" ] }, "time_zone_value": { "type": [ "string", "null" ] } }, "additionalProperties": false }, "SchemaOrigin": { "description": "Indicates where schema metadata originates from.\n\n- `Remote` (default): Schema is fetched from the remote warehouse - `Local`: Schema is derived from YAML column definitions", "oneOf": [ { "description": "Schema metadata comes from the remote warehouse (default)", "type": "string", "enum": [ "remote" ] }, { "description": "Schema metadata is derived from YAML column definitions", "type": "string", "enum": [ "local" ] } ] }, "SchemaRefreshInterval": { "description": "Schema refresh interval: '30m', '2h', '1d', or 'never'", "type": "string" }, "Severity": { "type": "string", "enum": [ "Error", "Warn" ] }, "SnapshotMetaColumnNames": { "type": "object", "properties": { "dbt_is_deleted": { "type": [ "string", "null" ] }, "dbt_scd_id": { "type": [ "string", "null" ] }, "dbt_updated_at": { "type": [ "string", "null" ] }, "dbt_valid_from": { "type": [ "string", "null" ] }, "dbt_valid_to": { "type": [ "string", "null" ] } }, "additionalProperties": false }, "SpannedStringOrArrayOfStrings": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } } ] }, "StaticAnalysisKind": { "type": "string", "enum": [ "unsafe", "off", "strict", "baseline", "on" ] }, "StoreFailuresAs": { "type": "string", "enum": [ "ephemeral", "table", "view" ] }, "StringOrArrayOfStrings": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } } ] }, "StringOrInteger": { "anyOf": [ { "type": "string" }, { "type": "integer", "format": "int64" } ] }, "SyncConfig": { "description": "Configuration for schema synchronization behavior.\n\nThis can be specified at source level (default for all tables) or at individual table level (overrides source-level settings).", "type": "object", "properties": { "schema_refresh_interval": { "description": "How often to refresh the schema from the warehouse. Examples: \"30m\", \"2h\", \"1d\", \"never\"", "anyOf": [ { "$ref": "#/definitions/SchemaRefreshInterval" }, { "type": "null" } ] } }, "additionalProperties": false }, "TargetPath": { "type": "string", "enum": [ "target" ] }, "TimeConfig": { "type": "object", "properties": { "granularity": { "default": "day", "type": "string" }, "time_ingestion_partitioning": { "description": "When this is true, the [`BigqueryPartitionConfig::field`] will be used as the `_PARTITIONTIME` pseudo column _PARTITIONTIME: https://cloud.google.com/bigquery/docs/partitioned-tables#ingestion_time https://docs.getdbt.com/reference/resource-configs/bigquery-configs#partitioning-by-an-ingestion-date-or-timestamp", "default": false, "type": "boolean" } }, "additionalProperties": false }, "UpdatesOn": { "type": "string", "enum": [ "any", "all" ] }, "Volatility": { "description": "Function volatility enum - defines the function's eligibility for certain optimizations Matches the Python Volatility enum from dbt-core", "oneOf": [ { "type": "string", "enum": [ "stable" ] }, { "description": "Deterministic - An deterministic function will always return the same output when given the same input.", "type": "string", "enum": [ "deterministic" ] }, { "description": "NonDeterministic - A non-deterministic function may change the return value from evaluation to evaluation. Multiple invocations of a non-deterministic function may return different results when used in the same query.", "type": "string", "enum": [ "non-deterministic" ] } ] }, "_Dispatch": { "type": "object", "required": [ "macro_namespace", "search_order" ], "properties": { "macro_namespace": { "type": "string" }, "search_order": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false } } } ================================================ FILE: core/dbt/jsonschemas/project/0.0.85.json ================================================ { "$schema": "http://json-schema.org/draft-07/schema#", "title": "DbtProject", "type": "object", "required": [ "name" ], "properties": { "analysis-paths": { "type": [ "array", "null" ], "items": { "type": "string" } }, "asset-paths": { "type": [ "array", "null" ], "items": { "type": "string" } }, "clean-targets": { "type": [ "array", "null" ], "items": { "type": "string" } }, "config-version": { "type": [ "integer", "null" ], "format": "int32" }, "data_tests": { "anyOf": [ { "$ref": "#/definitions/DataTestConfigs" }, { "type": "null" } ] }, "dbt-cloud": true, "dispatch": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/_Dispatch" } }, "docs-paths": { "type": [ "array", "null" ], "items": { "type": "string" } }, "flags": true, "log-path": { "type": [ "string", "null" ] }, "macro-paths": { "type": [ "array", "null" ], "items": { "type": "string" } }, "metrics": { "anyOf": [ { "$ref": "#/definitions/MetricConfigs" }, { "type": "null" } ] }, "model-paths": { "type": [ "array", "null" ], "items": { "type": "string" } }, "models": { "anyOf": [ { "$ref": "#/definitions/ModelConfigs" }, { "type": "null" } ] }, "name": { "type": "string" }, "on-run-end": { "anyOf": [ { "$ref": "#/definitions/StringOrVecString" }, { "type": "null" } ] }, "on-run-start": { "anyOf": [ { "$ref": "#/definitions/StringOrVecString" }, { "type": "null" } ] }, "packages-install-path": { "type": [ "string", "null" ] }, "profile": { "type": [ "string", "null" ] }, "query-comment": { "anyOf": [ { "$ref": "#/definitions/QueryComment" }, { "type": "null" } ] }, "quoting": true, "require-dbt-version": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "restrict-access": { "type": [ "boolean", "null" ] }, "saved-queries": { "anyOf": [ { "$ref": "#/definitions/SavedQueriesConfigs" }, { "type": "null" } ] }, "seed-paths": { "type": [ "array", "null" ], "items": { "type": "string" } }, "seeds": { "anyOf": [ { "$ref": "#/definitions/SeedConfigs" }, { "type": "null" } ] }, "semantic-models": { "anyOf": [ { "$ref": "#/definitions/ModelConfigs" }, { "type": "null" } ] }, "snapshot-paths": { "type": [ "array", "null" ], "items": { "type": "string" } }, "snapshots": { "anyOf": [ { "$ref": "#/definitions/SnapshotConfigs" }, { "type": "null" } ] }, "sources": { "anyOf": [ { "$ref": "#/definitions/SourceConfigs" }, { "type": "null" } ] }, "target-path": { "type": [ "string", "null" ] }, "test-paths": { "type": [ "array", "null" ], "items": { "type": "string" } }, "tests": { "anyOf": [ { "$ref": "#/definitions/DataTestConfigs" }, { "type": "null" } ] }, "unit_tests": { "anyOf": [ { "$ref": "#/definitions/UnitTestConfigs" }, { "type": "null" } ] }, "vars": { "anyOf": [ { "$ref": "#/definitions/AnyValue" }, { "type": "null" } ] }, "version": { "anyOf": [ { "$ref": "#/definitions/FloatOrString" }, { "type": "null" } ] } }, "definitions": { "Access": { "type": "string", "enum": [ "private", "protected", "public" ] }, "AnyValue": true, "BooleanOrJinjaString": { "anyOf": [ { "type": "string" }, { "type": "boolean" } ] }, "Contract": { "type": "object", "required": [ "enforced" ], "properties": { "alias_types": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "enforced": { "$ref": "#/definitions/BooleanOrJinjaString" } } }, "DataTestConfigs": { "type": "object", "properties": { "+alias": { "type": [ "string", "null" ] }, "+database": { "type": [ "string", "null" ] }, "+enabled": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "+error_if": { "type": [ "string", "null" ] }, "+fail_calc": { "type": [ "string", "null" ] }, "+group": { "type": [ "string", "null" ] }, "+limit": { "type": [ "integer", "null" ], "format": "int32" }, "+meta": true, "+schema": { "type": [ "string", "null" ] }, "+severity": { "anyOf": [ { "$ref": "#/definitions/Severity" }, { "type": "null" } ] }, "+store_failures": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "+tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "+warn_if": { "type": [ "string", "null" ] }, "__additional_properties__": { "type": "object", "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "alias": { "type": [ "string", "null" ] }, "database": { "type": [ "string", "null" ] }, "enabled": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "error_if": { "type": [ "string", "null" ] }, "fail_calc": { "type": [ "string", "null" ] }, "group": { "type": [ "string", "null" ] }, "limit": { "type": [ "integer", "null" ], "format": "int32" }, "meta": true, "schema": { "type": [ "string", "null" ] }, "severity": { "anyOf": [ { "$ref": "#/definitions/Severity" }, { "type": "null" } ] }, "store_failures": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "warn_if": { "type": [ "string", "null" ] } } }, "DocsConfig": { "type": "object", "properties": { "node_color": { "type": [ "string", "null" ] }, "show": { "type": [ "boolean", "null" ] } } }, "FloatOrString": { "anyOf": [ { "type": "number", "format": "float" }, { "type": "string" } ] }, "HookConfig": { "type": "object", "properties": { "sql": { "type": [ "string", "null" ] }, "transaction": { "type": [ "boolean", "null" ] } } }, "Hooks": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "$ref": "#/definitions/HookConfig" }, { "type": "array", "items": true } ] }, "MetricConfigs": { "type": "object", "properties": { "+meta": true, "enabled": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] } }, "additionalProperties": true }, "ModelConfigs": { "type": "object", "properties": { "+access": { "anyOf": [ { "$ref": "#/definitions/Access" }, { "type": "null" } ] }, "+alias": { "type": [ "string", "null" ] }, "+auto_refresh": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "+backup": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "+bind": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "+contract": { "anyOf": [ { "$ref": "#/definitions/Contract" }, { "type": "null" } ] }, "+copy_grants": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "+database": { "type": [ "string", "null" ] }, "+docs": { "anyOf": [ { "$ref": "#/definitions/DocsConfig" }, { "type": "null" } ] }, "+enabled": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "+file_format": { "type": [ "string", "null" ] }, "+full_refresh": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "+grant_access_to": { "type": [ "array", "null" ], "items": true }, "+grants": true, "+group": { "type": [ "string", "null" ] }, "+hours_to_expiration": { "type": [ "number", "null" ], "format": "float" }, "+include_full_name_in_path": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "+incremental_predicates": { "type": [ "array", "null" ], "items": { "type": "string" } }, "+incremental_strategy": { "type": [ "string", "null" ] }, "+kms_key_name": { "type": [ "string", "null" ] }, "+labels": true, "+location": { "type": [ "string", "null" ] }, "+location_root": { "type": [ "string", "null" ] }, "+materialized": { "type": [ "string", "null" ] }, "+meta": true, "+on_configuration_change": { "anyOf": [ { "$ref": "#/definitions/OnConfigurationChange" }, { "type": "null" } ] }, "+on_schema_change": { "anyOf": [ { "$ref": "#/definitions/OnSchemaChange" }, { "type": "null" } ] }, "+persist_docs": { "anyOf": [ { "$ref": "#/definitions/PersistDocsConfig" }, { "type": "null" } ] }, "+post-hook": { "anyOf": [ { "$ref": "#/definitions/Hooks" }, { "type": "null" } ] }, "+pre-hook": { "anyOf": [ { "$ref": "#/definitions/Hooks" }, { "type": "null" } ] }, "+predicates": { "type": [ "array", "null" ], "items": { "type": "string" } }, "+query_tag": { "type": [ "string", "null" ] }, "+schema": { "type": [ "string", "null" ] }, "+secure": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "+snowflake_warehouse": { "type": [ "string", "null" ] }, "+sql_header": { "type": [ "string", "null" ] }, "+table_format": { "type": [ "string", "null" ] }, "+tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "+target_lag": { "type": [ "string", "null" ] }, "+tblproperties": true, "+transient": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "__additional_properties__": { "type": "object", "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "access": { "anyOf": [ { "$ref": "#/definitions/Access" }, { "type": "null" } ] }, "alias": { "type": [ "string", "null" ] }, "auto_refresh": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "backup": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "bind": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "contract": { "anyOf": [ { "$ref": "#/definitions/Contract" }, { "type": "null" } ] }, "copy_grants": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "database": { "type": [ "string", "null" ] }, "docs": { "anyOf": [ { "$ref": "#/definitions/DocsConfig" }, { "type": "null" } ] }, "enabled": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "file_format": { "type": [ "string", "null" ] }, "full_refresh": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "grant_access_to": { "type": [ "array", "null" ], "items": true }, "grants": true, "group": { "type": [ "string", "null" ] }, "hours_to_expiration": { "type": [ "number", "null" ], "format": "float" }, "include_full_name_in_path": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "incremental_strategy": { "type": [ "string", "null" ] }, "kms_key_name": { "type": [ "string", "null" ] }, "labels": true, "location": { "type": [ "string", "null" ] }, "location_root": { "type": [ "string", "null" ] }, "materialized": { "type": [ "string", "null" ] }, "meta": true, "on_configuration_change": { "anyOf": [ { "$ref": "#/definitions/OnConfigurationChange" }, { "type": "null" } ] }, "on_schema_change": { "anyOf": [ { "$ref": "#/definitions/OnSchemaChange" }, { "type": "null" } ] }, "persist_docs": { "anyOf": [ { "$ref": "#/definitions/PersistDocsConfig" }, { "type": "null" } ] }, "post-hook": { "anyOf": [ { "$ref": "#/definitions/Hooks" }, { "type": "null" } ] }, "pre-hook": { "anyOf": [ { "$ref": "#/definitions/Hooks" }, { "type": "null" } ] }, "query_tag": { "type": [ "string", "null" ] }, "schema": { "type": [ "string", "null" ] }, "secure": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "snowflake_warehouse": { "type": [ "string", "null" ] }, "sql_header": { "type": [ "string", "null" ] }, "table_format": { "type": [ "string", "null" ] }, "tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "target_lag": { "type": [ "string", "null" ] }, "tblproperties": true, "transient": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] } } }, "OnConfigurationChange": { "type": "string", "enum": [ "apply", "continue", "fail" ] }, "OnSchemaChange": { "type": "string", "enum": [ "append_new_columns", "fail", "ignore", "sync_all_columns" ] }, "PersistDocsConfig": { "type": "object", "properties": { "columns": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "relation": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] } } }, "QueryComment": { "anyOf": [ { "type": "string" }, true ] }, "SavedQueriesConfigs": { "type": "object", "properties": { "+schema": { "type": [ "string", "null" ] } } }, "SeedConfigs": { "type": "object", "properties": { "+column_types": true, "+copy_grants": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "+database": { "type": [ "string", "null" ] }, "+docs": { "anyOf": [ { "$ref": "#/definitions/DocsConfig" }, { "type": "null" } ] }, "+enabled": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "+full_refresh": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "+grants": true, "+group": { "type": [ "string", "null" ] }, "+meta": true, "+persist_docs": { "anyOf": [ { "$ref": "#/definitions/PersistDocsConfig" }, { "type": "null" } ] }, "+post-hook": { "anyOf": [ { "$ref": "#/definitions/Hooks" }, { "type": "null" } ] }, "+pre-hook": { "anyOf": [ { "$ref": "#/definitions/Hooks" }, { "type": "null" } ] }, "+quote_columns": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "+schema": { "type": [ "string", "null" ] }, "+snowflake_warehouse": { "type": [ "string", "null" ] }, "+tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "+transient": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "column_types": true, "copy_grants": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "database": { "type": [ "string", "null" ] }, "delimiter": { "type": [ "string", "null" ] }, "docs": { "anyOf": [ { "$ref": "#/definitions/DocsConfig" }, { "type": "null" } ] }, "enabled": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "full_refresh": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "grants": true, "group": { "type": [ "string", "null" ] }, "meta": true, "persist_docs": { "anyOf": [ { "$ref": "#/definitions/PersistDocsConfig" }, { "type": "null" } ] }, "post-hook": { "anyOf": [ { "$ref": "#/definitions/Hooks" }, { "type": "null" } ] }, "pre-hook": { "anyOf": [ { "$ref": "#/definitions/Hooks" }, { "type": "null" } ] }, "quote_columns": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "schema": { "type": [ "string", "null" ] }, "snowflake_warehouse": { "type": [ "string", "null" ] }, "tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "transient": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] } }, "additionalProperties": true }, "Severity": { "anyOf": [ { "type": "string" }, { "type": "string" } ] }, "SnapshotConfigs": { "type": "object", "properties": { "+alias": { "type": [ "string", "null" ] }, "+check_cols": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "+docs": { "anyOf": [ { "$ref": "#/definitions/DocsConfig" }, { "type": "null" } ] }, "+enabled": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "+grants": true, "+group": { "type": [ "string", "null" ] }, "+invalidate_hard_deletes": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "+meta": true, "+persist_docs": { "anyOf": [ { "$ref": "#/definitions/PersistDocsConfig" }, { "type": "null" } ] }, "+post-hook": { "anyOf": [ { "$ref": "#/definitions/Hooks" }, { "type": "null" } ] }, "+pre-hook": { "anyOf": [ { "$ref": "#/definitions/Hooks" }, { "type": "null" } ] }, "+quote_columns": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "+snapshot_meta_column_names": true, "+strategy": { "type": [ "string", "null" ] }, "+tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "+target_database": { "type": [ "string", "null" ] }, "+target_schema": { "type": [ "string", "null" ] }, "+transient": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "+unique_key": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "+updated_at": { "type": [ "string", "null" ] }, "__additional_properties__": { "type": "object", "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "alias": { "type": [ "string", "null" ] }, "check_cols": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "docs": { "anyOf": [ { "$ref": "#/definitions/DocsConfig" }, { "type": "null" } ] }, "enabled": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "grants": true, "group": { "type": [ "string", "null" ] }, "invalidate_hard_deletes": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "meta": true, "persist_docs": { "anyOf": [ { "$ref": "#/definitions/PersistDocsConfig" }, { "type": "null" } ] }, "post-hook": { "anyOf": [ { "$ref": "#/definitions/Hooks" }, { "type": "null" } ] }, "pre-hook": { "anyOf": [ { "$ref": "#/definitions/Hooks" }, { "type": "null" } ] }, "quote_columns": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "schema": { "type": [ "string", "null" ] }, "snapshot_meta_column_names": true, "strategy": { "type": [ "string", "null" ] }, "tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "target_database": { "type": [ "string", "null" ] }, "target_schema": { "type": [ "string", "null" ] }, "transient": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "unique_key": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "updated_at": { "type": [ "string", "null" ] } } }, "SourceConfigs": { "type": "object", "properties": { "+enabled": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "+meta": true, "+tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "enabled": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "meta": true, "tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] } }, "additionalProperties": true }, "StringOrArrayOfStrings": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } } ] }, "StringOrVecString": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } } ] }, "UnitTestConfigs": { "type": "object", "properties": { "+enabled": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "+meta": true, "+tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "enabled": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "meta": true, "tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] } }, "additionalProperties": true }, "_Dispatch": { "type": "object", "required": [ "macro_namespace", "search_order" ], "properties": { "macro_namespace": { "type": "string" }, "search_order": { "type": "array", "items": { "type": "string" } } } } } } ================================================ FILE: core/dbt/jsonschemas/resources/0.0.110.json ================================================ { "$schema": "http://json-schema.org/draft-07/schema#", "title": "DbtPropertiesFile", "type": "object", "additionalProperties": false, "properties": { "analyses": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/_Analyses" } }, "exposures": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/_Exposures" } }, "groups": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/_Groups" } }, "macros": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/_Macros" } }, "metrics": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/_Metrics" } }, "models": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/_Models" } }, "saved_queries": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/_SavedQueries" } }, "seeds": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/_Seeds" } }, "semantic_models": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/_SemanticModels" } }, "snapshots": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/_Snapshots" } }, "sources": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/_Sources" } }, "unit_tests": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/_UnitTests" } }, "version": { "anyOf": [ { "$ref": "#/definitions/FloatOrString" }, { "type": "null" } ] } }, "definitions": { "AcceptedValuesTest": { "type": "object", "required": [ "accepted_values" ], "properties": { "accepted_values": true } }, "AggregationTypeParams": { "type": "object", "properties": { "percentile": { "type": [ "number", "null" ], "format": "float" }, "use_approximate_percentile": { "type": [ "boolean", "null" ] }, "use_discrete_percentile": { "type": [ "boolean", "null" ] } } }, "AnyValue": true, "BooleanOrJinjaString": { "anyOf": [ { "type": "string" }, { "type": "boolean" } ] }, "ColumnProperties": { "type": "object", "required": [ "name" ], "properties": { "constraints": { "type": [ "array", "null" ], "items": true }, "data_tests": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/DataTests" } }, "data_type": { "type": [ "string", "null" ] }, "description": { "type": [ "string", "null" ] }, "granularity": { "anyOf": [ { "$ref": "#/definitions/_ColumnPropertiesGranularity" }, { "type": "null" } ] }, "meta": true, "name": { "type": "string" }, "policy_tags": { "type": [ "array", "null" ], "items": { "type": "string" } }, "quote": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "tests": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/DataTests" } } } }, "CustomGranularity": { "type": "object", "required": [ "name" ], "properties": { "column_name": { "type": [ "string", "null" ] }, "name": { "type": "string" } } }, "DataTests": { "anyOf": [ { "type": "string" }, { "$ref": "#/definitions/UniqueTest" }, { "$ref": "#/definitions/NotNullTest" }, { "$ref": "#/definitions/RelationshipsTest" }, { "$ref": "#/definitions/AcceptedValuesTest" }, true ] }, "Dimension": { "type": "object", "required": [ "name", "type" ], "properties": { "config": true, "description": { "type": [ "string", "null" ] }, "expr": { "anyOf": [ { "$ref": "#/definitions/_DimensionExpr" }, { "type": "null" } ] }, "is_partition": { "type": [ "boolean", "null" ] }, "label": { "type": [ "string", "null" ] }, "name": { "type": "string" }, "type": { "$ref": "#/definitions/_DimensionType_" }, "type_params": { "anyOf": [ { "$ref": "#/definitions/DimensionTypeParams" }, { "type": "null" } ] } } }, "DimensionTypeParams": { "type": "object", "required": [ "time_granularity" ], "properties": { "time_granularity": { "type": "string" }, "validity_params": { "anyOf": [ { "$ref": "#/definitions/ValidityParams" }, { "type": "null" } ] } } }, "DocsConfig": { "type": "object", "properties": { "node_color": { "type": [ "string", "null" ] }, "show": { "type": [ "boolean", "null" ] } } }, "Entity": { "type": "object", "required": [ "name", "type" ], "properties": { "config": true, "description": { "type": [ "string", "null" ] }, "entity": { "type": [ "string", "null" ] }, "expr": { "anyOf": [ { "$ref": "#/definitions/_EntityExpr" }, { "type": "null" } ] }, "label": { "type": [ "string", "null" ] }, "name": { "type": "string" }, "type": { "$ref": "#/definitions/_EntityType_" } } }, "Export": { "type": "object", "required": [ "name" ], "properties": { "config": { "anyOf": [ { "$ref": "#/definitions/_ExportConfig" }, { "type": "null" } ] }, "name": { "type": "string" } } }, "ExposurePropertiesConfigs": { "type": "object", "additionalProperties": false, "properties": { "enabled": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "meta": true } }, "FloatOrString": { "anyOf": [ { "type": "number", "format": "float" }, { "type": "string" } ] }, "FreshnessDefinition": { "type": "object", "properties": { "error_after": { "anyOf": [ { "$ref": "#/definitions/FreshnessRules" }, { "type": "null" } ] }, "warn_after": { "anyOf": [ { "$ref": "#/definitions/FreshnessRules" }, { "type": "null" } ] } } }, "FreshnessPeriod": { "type": "string", "enum": [ "minute", "hour", "day" ] }, "FreshnessRules": { "type": "object", "properties": { "count": { "anyOf": [ { "$ref": "#/definitions/NumberOrJinjaString" }, { "type": "null" } ] }, "period": { "anyOf": [ { "$ref": "#/definitions/FreshnessPeriod" }, { "type": "null" } ] } } }, "HookConfig": { "type": "object", "properties": { "sql": { "type": [ "string", "null" ] }, "transaction": { "type": [ "boolean", "null" ] } } }, "Hooks": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "$ref": "#/definitions/HookConfig" }, { "type": "array", "items": true } ] }, "Measure": { "type": "object", "required": [ "agg", "name" ], "properties": { "agg": { "$ref": "#/definitions/_MeasureAgg" }, "agg_params": { "anyOf": [ { "$ref": "#/definitions/AggregationTypeParams" }, { "type": "null" } ] }, "agg_time_dimension": { "type": [ "string", "null" ] }, "config": true, "create_metric": { "type": [ "boolean", "null" ] }, "create_metric_display_name": { "type": [ "string", "null" ] }, "description": { "type": [ "string", "null" ] }, "expr": { "anyOf": [ { "$ref": "#/definitions/_MeasureExpr" }, { "type": "null" } ] }, "label": { "type": [ "string", "null" ] }, "name": { "type": "string" }, "non_additive_dimension": { "anyOf": [ { "$ref": "#/definitions/NonAdditiveDimension" }, { "type": "null" } ] } } }, "ModelPropertiesConfigs": { "type": "object", "additionalProperties": false, "properties": { "auto_refresh": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "backup": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "batch_size": { "type": [ "string", "null" ] }, "begin": { "type": [ "string", "null" ] }, "contract": { "anyOf": [ { "$ref": "#/definitions/_ModelPropertiesConfigsContract" }, { "type": "null" } ] }, "docs": { "anyOf": [ { "$ref": "#/definitions/DocsConfig" }, { "type": "null" } ] }, "enabled": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "event_time": { "type": [ "string", "null" ] }, "file_format": { "type": [ "string", "null" ] }, "grant_access_to": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/_GrantAccessTo" } }, "grants": true, "group": { "type": [ "string", "null" ] }, "hours_to_expiration": { "type": [ "number", "null" ], "format": "float" }, "include_full_name_in_path": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "incremental_strategy": { "type": [ "string", "null" ] }, "kms_key_name": { "type": [ "string", "null" ] }, "labels": true, "location": { "type": [ "string", "null" ] }, "location_root": { "type": [ "string", "null" ] }, "lookback": { "type": [ "number", "null" ], "format": "float" }, "materialized": { "type": [ "string", "null" ] }, "meta": true, "on_configuration_change": { "type": [ "string", "null" ] }, "on_schema_change": { "type": [ "string", "null" ] }, "snowflake_warehouse": { "type": [ "string", "null" ] }, "sql_header": { "type": [ "string", "null" ] }, "table_format": { "type": [ "string", "null" ] }, "tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "target_lag": { "type": [ "string", "null" ] }, "tblproperties": true, "unique_key": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] } } }, "NonAdditiveDimension": { "type": "object", "required": [ "name" ], "properties": { "name": { "type": "string" }, "window_choice": { "anyOf": [ { "$ref": "#/definitions/_NonAdditiveDimensionWindowChoice" }, { "type": "null" } ] }, "window_groupings": { "type": [ "array", "null" ], "items": { "type": "string" } } } }, "NotNullTest": { "type": "object", "required": [ "not_null" ], "properties": { "not_null": true } }, "NumberOrJinjaString": { "anyOf": [ { "type": "string" }, { "type": "integer", "format": "int32" } ] }, "PersistDocsConfig": { "type": "object", "properties": { "columns": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "relation": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] } } }, "RelationshipsTest": { "type": "object", "required": [ "relationships" ], "properties": { "relationships": true } }, "StringOrArrayOfStrings": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } } ] }, "UniqueTest": { "type": "object", "required": [ "unique" ], "properties": { "unique": true } }, "ValidityParams": { "type": "object", "properties": { "is_end": { "type": [ "boolean", "null" ] }, "is_start": { "type": [ "boolean", "null" ] } } }, "_Analyses": { "type": "object", "required": [ "name" ], "properties": { "columns": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/_Columns" } }, "config": { "anyOf": [ { "$ref": "#/definitions/__AnalysesConfig" }, { "type": "null" } ] }, "description": { "type": [ "string", "null" ] }, "docs": { "anyOf": [ { "$ref": "#/definitions/DocsConfig" }, { "type": "null" } ] }, "group": { "type": [ "string", "null" ] }, "name": { "type": "string" } } }, "_Arguments": { "type": "object", "required": [ "name" ], "properties": { "description": { "type": [ "string", "null" ] }, "name": { "type": "string" }, "type": { "type": [ "string", "null" ] } } }, "_ColumnPropertiesGranularity": { "type": "string", "enum": [ "nanosecond", "microsecond", "millisecond", "second", "minute", "hour", "day", "week", "month", "quarter", "year" ] }, "_Columns": { "type": "object", "required": [ "name" ], "properties": { "data_type": { "type": [ "string", "null" ] }, "description": { "type": [ "string", "null" ] }, "name": { "type": "string" } } }, "_DimensionExpr": { "anyOf": [ { "type": "string" }, { "type": "boolean" } ] }, "_DimensionType_": { "type": "string", "enum": [ "CATEGORICAL", "TIME", "categorical", "time" ] }, "_EntityExpr": { "anyOf": [ { "type": "string" }, { "type": "boolean" } ] }, "_EntityType_": { "type": "string", "enum": [ "PRIMARY", "UNIQUE", "FOREIGN", "NATURAL", "primary", "unique", "foreign", "natural" ] }, "_ExportConfig": { "type": "object", "properties": { "alias": { "type": [ "string", "null" ] }, "export_as": { "anyOf": [ { "$ref": "#/definitions/__ExportConfigExportAs" }, { "type": "null" } ] }, "schema": { "type": [ "string", "null" ] } }, "additionalProperties": true }, "_Exposures": { "type": "object", "required": [ "name", "owner", "type" ], "additionalProperties": false, "properties": { "config": { "anyOf": [ { "$ref": "#/definitions/ExposurePropertiesConfigs" }, { "type": "null" } ] }, "depends_on": { "type": [ "array", "null" ], "items": { "type": "string" } }, "description": { "type": [ "string", "null" ] }, "label": { "type": [ "string", "null" ] }, "maturity": { "type": [ "string", "null" ] }, "meta": true, "name": { "type": "string" }, "owner": { "$ref": "#/definitions/__ExposuresOwner" }, "tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "type": { "type": "string" }, "url": { "type": [ "string", "null" ] } } }, "_Given": { "type": "object", "properties": { "fixture": { "type": [ "string", "null" ] }, "format": { "type": [ "string", "null" ] }, "input": { "type": [ "string", "null" ] }, "rows": true } }, "_GrantAccessTo": { "type": "object", "properties": { "dataset": { "type": [ "string", "null" ] }, "project": { "type": [ "string", "null" ] } } }, "_Groups": { "type": "object", "required": [ "name", "owner" ], "properties": { "name": { "type": "string" }, "owner": { "$ref": "#/definitions/__GroupsOwner" } } }, "_Macros": { "type": "object", "required": [ "name" ], "additionalProperties": false, "properties": { "arguments": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/_Arguments" } }, "description": { "type": [ "string", "null" ] }, "docs": { "anyOf": [ { "$ref": "#/definitions/DocsConfig" }, { "type": "null" } ] }, "name": { "type": "string" } } }, "_MeasureAgg": { "type": "string", "enum": [ "SUM", "MIN", "MAX", "AVERAGE", "COUNT_DISTINCT", "SUM_BOOLEAN", "COUNT", "PERCENTILE", "MEDIAN", "sum", "min", "max", "average", "count_distinct", "sum_boolean", "count", "percentile", "median" ] }, "_MeasureExpr": { "anyOf": [ { "type": "string" }, { "type": "integer", "format": "int32" }, { "type": "boolean" } ] }, "_Metrics": { "type": "object" }, "_ModelPropertiesConfigsContract": { "type": "object", "properties": { "alias_types": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "enforced": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] } } }, "_Models": { "type": "object", "required": [ "name" ], "additionalProperties": false, "properties": { "access": { "type": [ "string", "null" ] }, "columns": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/ColumnProperties" } }, "config": { "anyOf": [ { "$ref": "#/definitions/ModelPropertiesConfigs" }, { "type": "null" } ] }, "constraints": { "type": [ "array", "null" ], "items": true }, "data_tests": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/DataTests" } }, "deprecation_date": { "type": [ "string", "null" ] }, "description": { "type": [ "string", "null" ] }, "docs": { "anyOf": [ { "$ref": "#/definitions/DocsConfig" }, { "type": "null" } ] }, "group": { "type": [ "string", "null" ] }, "identifier": { "type": [ "string", "null" ] }, "latest_version": { "anyOf": [ { "$ref": "#/definitions/FloatOrString" }, { "type": "null" } ] }, "meta": true, "name": { "type": "string" }, "tests": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/DataTests" } }, "time_spine": { "anyOf": [ { "$ref": "#/definitions/__ModelsTimeSpine" }, { "type": "null" } ] }, "versions": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/_Versions" } } } }, "_NonAdditiveDimensionWindowChoice": { "type": "string", "enum": [ "MIN", "MAX", "min", "max" ] }, "_SavedQueries": { "type": "object", "required": [ "name", "query_params" ], "additionalProperties": false, "properties": { "config": { "anyOf": [ { "$ref": "#/definitions/__SavedQueriesConfig" }, { "type": "null" } ] }, "description": { "type": [ "string", "null" ] }, "exports": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/Export" } }, "label": { "type": [ "string", "null" ] }, "name": { "type": "string" }, "query_params": { "$ref": "#/definitions/__SavedQueriesQueryParams" } } }, "_Seeds": { "type": "object", "required": [ "name" ], "additionalProperties": false, "properties": { "columns": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/ColumnProperties" } }, "config": { "anyOf": [ { "$ref": "#/definitions/__SeedsConfig" }, { "type": "null" } ] }, "data_tests": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/DataTests" } }, "description": { "type": [ "string", "null" ] }, "docs": { "anyOf": [ { "$ref": "#/definitions/DocsConfig" }, { "type": "null" } ] }, "group": { "type": [ "string", "null" ] }, "name": { "type": "string" }, "tests": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/DataTests" } } } }, "_SemanticModels": { "type": "object", "required": [ "model", "name" ], "additionalProperties": false, "properties": { "config": { "anyOf": [ { "$ref": "#/definitions/ModelPropertiesConfigs" }, { "type": "null" } ] }, "defaults": { "anyOf": [ { "$ref": "#/definitions/__SemanticModelsDefaults" }, { "type": "null" } ] }, "description": { "type": [ "string", "null" ] }, "dimensions": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/Dimension" } }, "entities": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/Entity" } }, "label": { "type": [ "string", "null" ] }, "measures": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/Measure" } }, "model": { "type": "string" }, "name": { "type": "string" }, "primary_entity": { "type": [ "string", "null" ] } } }, "_Snapshots": { "type": "object", "required": [ "name" ], "additionalProperties": false, "properties": { "columns": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/ColumnProperties" } }, "config": { "anyOf": [ { "$ref": "#/definitions/__SnapshotsConfig" }, { "type": "null" } ] }, "data_tests": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/DataTests" } }, "description": { "type": [ "string", "null" ] }, "docs": { "anyOf": [ { "$ref": "#/definitions/DocsConfig" }, { "type": "null" } ] }, "group": { "type": [ "string", "null" ] }, "meta": true, "name": { "type": "string" }, "relation": { "type": [ "string", "null" ] }, "tests": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/DataTests" } } } }, "_Sources": { "type": "object", "required": [ "name" ], "additionalProperties": false, "properties": { "config": { "anyOf": [ { "$ref": "#/definitions/__SourcesConfig" }, { "type": "null" } ] }, "database": { "type": [ "string", "null" ] }, "description": { "type": [ "string", "null" ] }, "freshness": { "anyOf": [ { "$ref": "#/definitions/FreshnessDefinition" }, { "type": "null" } ] }, "loaded_at_field": { "type": [ "string", "null" ] }, "loaded_at_query": { "type": [ "string", "null" ] }, "loader": { "type": [ "string", "null" ] }, "meta": true, "name": { "type": "string" }, "overrides": { "type": [ "string", "null" ] }, "quoting": { "anyOf": [ { "$ref": "#/definitions/__SourcesQuoting" }, { "type": "null" } ] }, "schema": { "type": [ "string", "null" ] }, "tables": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/_Tables" } }, "tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] } } }, "_Tables": { "type": "object", "required": [ "name" ], "additionalProperties": false, "properties": { "columns": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/ColumnProperties" } }, "config": { "anyOf": [ { "$ref": "#/definitions/__TablesConfig" }, { "type": "null" } ] }, "data_tests": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/DataTests" } }, "description": { "type": [ "string", "null" ] }, "external": true, "freshness": { "anyOf": [ { "$ref": "#/definitions/FreshnessDefinition" }, { "type": "null" } ] }, "identifier": { "type": [ "string", "null" ] }, "loaded_at_field": { "type": [ "string", "null" ] }, "loaded_at_query": { "type": [ "string", "null" ] }, "loader": { "type": [ "string", "null" ] }, "meta": true, "name": { "type": "string" }, "quoting": { "anyOf": [ { "$ref": "#/definitions/__TablesQuoting" }, { "type": "null" } ] }, "tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "tests": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/DataTests" } } } }, "_UnitTests": { "type": "object", "required": [ "expect", "model", "name" ], "additionalProperties": false, "properties": { "config": { "anyOf": [ { "$ref": "#/definitions/__UnitTestsConfig" }, { "type": "null" } ] }, "description": { "type": [ "string", "null" ] }, "expect": { "$ref": "#/definitions/__UnitTestsExpect" }, "given": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/_Given" } }, "model": { "type": "string" }, "name": { "type": "string" }, "overrides": { "anyOf": [ { "$ref": "#/definitions/__UnitTestsOverrides" }, { "type": "null" } ] }, "versions": true } }, "_Versions": { "type": "object", "required": [ "v" ], "properties": { "v": true }, "additionalProperties": true }, "__AnalysesConfig": { "type": "object", "properties": { "tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] } } }, "__ExportConfigExportAs": { "type": "string", "enum": [ "table", "view", "cache" ] }, "__ExposuresOwner": { "type": "object", "properties": { "email": { "type": [ "string", "null" ] }, "name": { "type": [ "string", "null" ] } } }, "__GroupsOwner": { "type": "object", "properties": { "email": { "type": [ "string", "null" ] }, "name": { "type": [ "string", "null" ] } }, "additionalProperties": true }, "__ModelsTimeSpine": { "type": "object", "required": [ "standard_granularity_column" ], "properties": { "custom_granularities": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/CustomGranularity" } }, "standard_granularity_column": { "type": "string" } } }, "__SavedQueriesConfig": { "type": "object", "additionalProperties": false, "properties": { "cache": { "anyOf": [ { "$ref": "#/definitions/___SavedQueriesConfigCache" }, { "type": "null" } ] }, "enabled": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "meta": true } }, "__SavedQueriesQueryParams": { "type": "object", "properties": { "dimensions": { "type": [ "array", "null" ], "items": { "type": "string" } }, "group_by": { "type": [ "array", "null" ], "items": { "type": "string" } }, "metrics": { "type": [ "array", "null" ], "items": { "type": "string" } }, "where": { "type": [ "array", "null" ], "items": { "type": "string" } } } }, "__SeedsConfig": { "type": "object", "additionalProperties": false, "properties": { "column_types": true, "copy_grants": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "database": { "type": [ "string", "null" ] }, "enabled": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "grants": true, "quote_columns": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "schema": { "type": [ "string", "null" ] } } }, "__SemanticModelsDefaults": { "type": "object", "properties": { "agg_time_dimension": { "type": [ "string", "null" ] } } }, "__SnapshotsConfig": { "type": "object", "additionalProperties": false, "properties": { "alias": { "type": [ "string", "null" ] }, "check_cols": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "dbt_valid_to_current": { "type": [ "string", "null" ] }, "enabled": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "grants": true, "meta": true, "persist_docs": { "anyOf": [ { "$ref": "#/definitions/PersistDocsConfig" }, { "type": "null" } ] }, "post-hook": { "anyOf": [ { "$ref": "#/definitions/Hooks" }, { "type": "null" } ] }, "pre-hook": { "anyOf": [ { "$ref": "#/definitions/Hooks" }, { "type": "null" } ] }, "quote_columns": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "snapshot_meta_column_names": { "anyOf": [ { "$ref": "#/definitions/___SnapshotsConfigSnapshotMetaColumnNames" }, { "type": "null" } ] }, "strategy": { "type": [ "string", "null" ] }, "tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "target_database": { "type": [ "string", "null" ] }, "target_schema": { "type": [ "string", "null" ] }, "unique_key": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "updated_at": { "type": [ "string", "null" ] } } }, "__SourcesConfig": { "type": "object", "additionalProperties": false, "properties": { "enabled": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "event_time": { "type": [ "string", "null" ] } } }, "__SourcesQuoting": { "type": "object", "properties": { "database": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "identifier": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "schema": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] } } }, "__TablesConfig": { "type": "object", "properties": { "event_time": { "type": [ "string", "null" ] } }, "additionalProperties": true }, "__TablesQuoting": { "type": "object", "properties": { "database": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "identifier": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "schema": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] } } }, "__UnitTestsConfig": { "type": "object", "additionalProperties": false, "properties": { "enabled": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "meta": true, "tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] } } }, "__UnitTestsExpect": { "type": "object", "properties": { "fixture": { "type": [ "string", "null" ] }, "format": { "type": [ "string", "null" ] }, "rows": true } }, "__UnitTestsOverrides": { "type": "object", "properties": { "env_vars": true, "macros": true, "vars": true } }, "___SavedQueriesConfigCache": { "type": "object", "properties": { "enabled": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] } } }, "___SnapshotsConfigSnapshotMetaColumnNames": { "type": "object", "properties": { "dbt_scd_id": { "type": [ "string", "null" ] }, "dbt_updated_at": { "type": [ "string", "null" ] }, "dbt_valid_from": { "type": [ "string", "null" ] }, "dbt_valid_to": { "type": [ "string", "null" ] } } } } } ================================================ FILE: core/dbt/jsonschemas/resources/0.0.85.json ================================================ { "$schema": "http://json-schema.org/draft-07/schema#", "title": "DbtSchemaFiles", "type": "object", "additionalProperties": false, "properties": { "analyses": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/_Analyses" } }, "exposures": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/_Exposures" } }, "groups": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/_Groups" } }, "macros": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/_Macros" } }, "metrics": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/_Metrics" } }, "models": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/_Models" } }, "saved_queries": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/_SavedQueries" } }, "seeds": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/_Seeds" } }, "semantic_models": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/_SemanticModels" } }, "snapshots": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/_Snapshots" } }, "sources": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/_Sources" } }, "unit_tests": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/_UnitTests" } }, "version": { "anyOf": [ { "$ref": "#/definitions/FloatOrString" }, { "type": "null" } ] } }, "definitions": { "AcceptedValuesTest": { "type": "object", "required": [ "accepted_values" ], "properties": { "accepted_values": true } }, "AggregationTypeParams": { "type": "object", "properties": { "percentile": { "type": [ "number", "null" ], "format": "float" }, "use_approximate_percentile": { "type": [ "boolean", "null" ] }, "use_discrete_percentile": { "type": [ "boolean", "null" ] } } }, "AnyValue": true, "BooleanOrJinjaString": { "anyOf": [ { "type": "string" }, { "type": "boolean" } ] }, "ColumnProperties": { "type": "object", "required": [ "name" ], "properties": { "constraints": { "type": [ "array", "null" ], "items": true }, "data_tests": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/DataTests" } }, "data_type": { "type": [ "string", "null" ] }, "description": { "type": [ "string", "null" ] }, "granularity": { "anyOf": [ { "$ref": "#/definitions/_ColumnPropertiesGranularity" }, { "type": "null" } ] }, "meta": true, "name": { "type": "string" }, "policy_tags": { "type": [ "array", "null" ], "items": { "type": "string" } }, "quote": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "tests": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/DataTests" } } } }, "CustomGranularity": { "type": "object", "required": [ "name" ], "properties": { "column_name": { "type": [ "string", "null" ] }, "name": { "type": "string" } } }, "DataTests": { "anyOf": [ { "type": "string" }, { "$ref": "#/definitions/UniqueTest" }, { "$ref": "#/definitions/NotNullTest" }, { "$ref": "#/definitions/RelationshipsTest" }, { "$ref": "#/definitions/AcceptedValuesTest" }, true ] }, "Dimension": { "type": "object", "required": [ "name", "type" ], "properties": { "config": true, "description": { "type": [ "string", "null" ] }, "expr": { "anyOf": [ { "$ref": "#/definitions/_DimensionExpr" }, { "type": "null" } ] }, "is_partition": { "type": [ "boolean", "null" ] }, "label": { "type": [ "string", "null" ] }, "name": { "type": "string" }, "type": { "$ref": "#/definitions/_DimensionType_" }, "type_params": { "anyOf": [ { "$ref": "#/definitions/DimensionTypeParams" }, { "type": "null" } ] } } }, "DimensionTypeParams": { "type": "object", "required": [ "time_granularity" ], "properties": { "time_granularity": { "type": "string" }, "validity_params": { "anyOf": [ { "$ref": "#/definitions/ValidityParams" }, { "type": "null" } ] } } }, "DocsConfig": { "type": "object", "properties": { "node_color": { "type": [ "string", "null" ] }, "show": { "type": [ "boolean", "null" ] } } }, "Entity": { "type": "object", "required": [ "name", "type" ], "properties": { "config": true, "description": { "type": [ "string", "null" ] }, "entity": { "type": [ "string", "null" ] }, "expr": { "anyOf": [ { "$ref": "#/definitions/_EntityExpr" }, { "type": "null" } ] }, "label": { "type": [ "string", "null" ] }, "name": { "type": "string" }, "type": { "$ref": "#/definitions/_EntityType_" } } }, "Export": { "type": "object", "required": [ "name" ], "properties": { "config": { "anyOf": [ { "$ref": "#/definitions/_ExportConfig" }, { "type": "null" } ] }, "name": { "type": "string" } } }, "ExposurePropertiesConfigs": { "type": "object", "additionalProperties": false, "properties": { "enabled": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "meta": true } }, "FloatOrString": { "anyOf": [ { "type": "number", "format": "float" }, { "type": "string" } ] }, "FreshnessDefinition": { "anyOf": [ true ] }, "HookConfig": { "type": "object", "properties": { "sql": { "type": [ "string", "null" ] }, "transaction": { "type": [ "boolean", "null" ] } } }, "Hooks": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "$ref": "#/definitions/HookConfig" }, { "type": "array", "items": true } ] }, "Measure": { "type": "object", "required": [ "agg", "name" ], "properties": { "agg": { "$ref": "#/definitions/_MeasureAgg" }, "agg_params": { "anyOf": [ { "$ref": "#/definitions/AggregationTypeParams" }, { "type": "null" } ] }, "agg_time_dimension": { "type": [ "string", "null" ] }, "config": true, "create_metric": { "type": [ "boolean", "null" ] }, "create_metric_display_name": { "type": [ "string", "null" ] }, "description": { "type": [ "string", "null" ] }, "expr": { "anyOf": [ { "$ref": "#/definitions/_MeasureExpr" }, { "type": "null" } ] }, "label": { "type": [ "string", "null" ] }, "name": { "type": "string" }, "non_additive_dimension": { "anyOf": [ { "$ref": "#/definitions/NonAdditiveDimension" }, { "type": "null" } ] } } }, "ModelPropertiesConfigs": { "type": "object", "additionalProperties": false, "properties": { "auto_refresh": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "backup": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "batch_size": { "type": [ "string", "null" ] }, "begin": { "type": [ "string", "null" ] }, "contract": { "anyOf": [ { "$ref": "#/definitions/_ModelPropertiesConfigsContract" }, { "type": "null" } ] }, "docs": { "anyOf": [ { "$ref": "#/definitions/DocsConfig" }, { "type": "null" } ] }, "enabled": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "event_time": { "type": [ "string", "null" ] }, "file_format": { "type": [ "string", "null" ] }, "grant_access_to": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/_GrantAccessTo" } }, "grants": true, "group": { "type": [ "string", "null" ] }, "hours_to_expiration": { "type": [ "number", "null" ], "format": "float" }, "include_full_name_in_path": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "incremental_strategy": { "type": [ "string", "null" ] }, "kms_key_name": { "type": [ "string", "null" ] }, "labels": true, "location": { "type": [ "string", "null" ] }, "location_root": { "type": [ "string", "null" ] }, "lookback": { "type": [ "number", "null" ], "format": "float" }, "materialized": { "type": [ "string", "null" ] }, "meta": true, "on_configuration_change": { "type": [ "string", "null" ] }, "on_schema_change": { "type": [ "string", "null" ] }, "snowflake_warehouse": { "type": [ "string", "null" ] }, "sql_header": { "type": [ "string", "null" ] }, "table_format": { "type": [ "string", "null" ] }, "tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "target_lag": { "type": [ "string", "null" ] }, "tblproperties": true, "unique_key": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] } } }, "NonAdditiveDimension": { "type": "object", "required": [ "name" ], "properties": { "name": { "type": "string" }, "window_choice": { "anyOf": [ { "$ref": "#/definitions/_NonAdditiveDimensionWindowChoice" }, { "type": "null" } ] }, "window_groupings": { "type": [ "array", "null" ], "items": { "type": "string" } } } }, "NotNullTest": { "type": "object", "required": [ "not_null" ], "properties": { "not_null": true } }, "PersistDocsConfig": { "type": "object", "properties": { "columns": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "relation": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] } } }, "RelationshipsTest": { "type": "object", "required": [ "relationships" ], "properties": { "relationships": true } }, "StringOrArrayOfStrings": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } } ] }, "UniqueTest": { "type": "object", "required": [ "unique" ], "properties": { "unique": true } }, "ValidityParams": { "type": "object", "properties": { "is_end": { "type": [ "boolean", "null" ] }, "is_start": { "type": [ "boolean", "null" ] } } }, "_Analyses": { "type": "object", "required": [ "name" ], "properties": { "columns": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/_Columns" } }, "config": { "anyOf": [ { "$ref": "#/definitions/__AnalysesConfig" }, { "type": "null" } ] }, "description": { "type": [ "string", "null" ] }, "docs": { "anyOf": [ { "$ref": "#/definitions/DocsConfig" }, { "type": "null" } ] }, "group": { "type": [ "string", "null" ] }, "name": { "type": "string" } } }, "_Arguments": { "type": "object", "required": [ "name" ], "properties": { "description": { "type": [ "string", "null" ] }, "name": { "type": "string" }, "type": { "type": [ "string", "null" ] } } }, "_ColumnPropertiesGranularity": { "type": "string", "enum": [ "nanosecond", "microsecond", "millisecond", "second", "minute", "hour", "day", "week", "month", "quarter", "year" ] }, "_Columns": { "type": "object", "required": [ "name" ], "properties": { "data_type": { "type": [ "string", "null" ] }, "description": { "type": [ "string", "null" ] }, "name": { "type": "string" } } }, "_DimensionExpr": { "anyOf": [ { "type": "string" }, { "type": "boolean" } ] }, "_DimensionType_": { "type": "string", "enum": [ "CATEGORICAL", "TIME", "categorical", "time" ] }, "_EntityExpr": { "anyOf": [ { "type": "string" }, { "type": "boolean" } ] }, "_EntityType_": { "type": "string", "enum": [ "PRIMARY", "UNIQUE", "FOREIGN", "NATURAL", "primary", "unique", "foreign", "natural" ] }, "_ExportConfig": { "type": "object", "properties": { "alias": { "type": [ "string", "null" ] }, "export_as": { "anyOf": [ { "$ref": "#/definitions/__ExportConfigExportAs" }, { "type": "null" } ] }, "schema": { "type": [ "string", "null" ] } }, "additionalProperties": true }, "_Exposures": { "type": "object", "additionalProperties": false, "required": [ "name", "owner", "type" ], "properties": { "config": { "anyOf": [ { "$ref": "#/definitions/ExposurePropertiesConfigs" }, { "type": "null" } ] }, "depends_on": { "type": [ "array", "null" ], "items": { "type": "string" } }, "description": { "type": [ "string", "null" ] }, "label": { "type": [ "string", "null" ] }, "maturity": { "type": [ "string", "null" ] }, "meta": true, "name": { "type": "string" }, "owner": { "$ref": "#/definitions/__ExposuresOwner" }, "tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "type": { "type": "string" }, "url": { "type": [ "string", "null" ] } } }, "_Given": { "type": "object", "properties": { "fixture": { "type": [ "string", "null" ] }, "format": { "type": [ "string", "null" ] }, "input": { "type": [ "string", "null" ] }, "rows": true } }, "_GrantAccessTo": { "type": "object", "properties": { "dataset": { "type": [ "string", "null" ] }, "project": { "type": [ "string", "null" ] } } }, "_Groups": { "type": "object", "required": [ "name", "owner" ], "properties": { "name": { "type": "string" }, "owner": { "$ref": "#/definitions/__GroupsOwner" } } }, "_Macros": { "type": "object", "additionalProperties": false, "required": [ "name" ], "properties": { "arguments": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/_Arguments" } }, "description": { "type": [ "string", "null" ] }, "docs": { "anyOf": [ { "$ref": "#/definitions/DocsConfig" }, { "type": "null" } ] }, "name": { "type": "string" } } }, "_MeasureAgg": { "type": "string", "enum": [ "SUM", "MIN", "MAX", "AVERAGE", "COUNT_DISTINCT", "SUM_BOOLEAN", "COUNT", "PERCENTILE", "MEDIAN", "sum", "min", "max", "average", "count_distinct", "sum_boolean", "count", "percentile", "median" ] }, "_MeasureExpr": { "anyOf": [ { "type": "string" }, { "type": "integer", "format": "int32" }, { "type": "boolean" } ] }, "_Metrics": { "type": "object", "properties": { "__additional_properties__": { "type": "object", "additionalProperties": { "$ref": "#/definitions/AnyValue" } } } }, "_ModelPropertiesConfigsContract": { "type": "object", "properties": { "alias_types": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "enforced": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] } } }, "_Models": { "type": "object", "additionalProperties": false, "required": [ "name" ], "properties": { "access": { "type": [ "string", "null" ] }, "columns": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/ColumnProperties" } }, "config": { "anyOf": [ { "$ref": "#/definitions/ModelPropertiesConfigs" }, { "type": "null" } ] }, "constraints": { "type": [ "array", "null" ], "items": true }, "data_tests": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/DataTests" } }, "deprecation_date": { "type": [ "string", "null" ] }, "description": { "type": [ "string", "null" ] }, "docs": { "anyOf": [ { "$ref": "#/definitions/DocsConfig" }, { "type": "null" } ] }, "group": { "type": [ "string", "null" ] }, "identifier": { "type": [ "string", "null" ] }, "latest_version": { "type": [ "number", "null" ], "format": "float" }, "meta": true, "name": { "type": "string" }, "tests": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/DataTests" } }, "time_spine": { "anyOf": [ { "$ref": "#/definitions/__ModelsTimeSpine" }, { "type": "null" } ] }, "versions": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/_Versions" } } } }, "_NonAdditiveDimensionWindowChoice": { "type": "string", "enum": [ "MIN", "MAX", "min", "max" ] }, "_SavedQueries": { "type": "object", "additionalProperties": false, "required": [ "name", "query_params" ], "properties": { "config": { "anyOf": [ { "$ref": "#/definitions/__SavedQueriesConfig" }, { "type": "null" } ] }, "description": { "type": [ "string", "null" ] }, "exports": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/Export" } }, "label": { "type": [ "string", "null" ] }, "name": { "type": "string" }, "query_params": { "type": "object", "properties": { "dimensions": { "type": [ "array", "null" ], "items": { "type": "string" } }, "group_by": { "type": [ "array", "null" ], "items": { "type": "string" } }, "metrics": { "type": [ "array", "null" ], "items": { "type": "string" } }, "where": { "type": [ "array", "null" ], "items": { "type": "string" } } } } } }, "_Seeds": { "type": "object", "additionalProperties": false, "required": [ "name" ], "properties": { "columns": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/ColumnProperties" } }, "config": { "anyOf": [ { "$ref": "#/definitions/__SeedsConfig" }, { "type": "null" } ] }, "data_tests": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/DataTests" } }, "description": { "type": [ "string", "null" ] }, "docs": { "anyOf": [ { "$ref": "#/definitions/DocsConfig" }, { "type": "null" } ] }, "group": { "type": [ "string", "null" ] }, "name": { "type": "string" }, "tests": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/DataTests" } } } }, "_SemanticModels": { "type": "object", "additionalProperties": false, "required": [ "model", "name" ], "properties": { "config": { "anyOf": [ { "$ref": "#/definitions/ModelPropertiesConfigs" }, { "type": "null" } ] }, "defaults": { "anyOf": [ { "$ref": "#/definitions/__SemanticModelsDefaults" }, { "type": "null" } ] }, "description": { "type": [ "string", "null" ] }, "dimensions": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/Dimension" } }, "entities": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/Entity" } }, "label": { "type": [ "string", "null" ] }, "measures": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/Measure" } }, "model": { "type": "string" }, "name": { "type": "string" }, "primary_entity": { "type": [ "string", "null" ] } } }, "_Snapshots": { "type": "object", "additionalProperties": false, "required": [ "name" ], "properties": { "columns": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/ColumnProperties" } }, "config": { "anyOf": [ { "$ref": "#/definitions/__SnapshotsConfig" }, { "type": "null" } ] }, "data_tests": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/DataTests" } }, "description": { "type": [ "string", "null" ] }, "docs": { "anyOf": [ { "$ref": "#/definitions/DocsConfig" }, { "type": "null" } ] }, "group": { "type": [ "string", "null" ] }, "meta": true, "name": { "type": "string" }, "relation": { "type": [ "string", "null" ] }, "tests": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/DataTests" } } } }, "_Sources": { "type": "object", "additionalProperties": false, "required": [ "name" ], "properties": { "config": true, "database": { "type": [ "string", "null" ] }, "description": { "type": [ "string", "null" ] }, "freshness": { "anyOf": [ { "$ref": "#/definitions/FreshnessDefinition" }, { "type": "null" } ] }, "loaded_at_field": { "type": [ "string", "null" ] }, "loaded_at_query": { "type": [ "string", "null" ] }, "loader": { "type": [ "string", "null" ] }, "meta": true, "name": { "type": "string" }, "overrides": { "type": [ "string", "null" ] }, "quoting": { "anyOf": [ { "$ref": "#/definitions/__SourcesQuoting" }, { "type": "null" } ] }, "schema": { "type": [ "string", "null" ] }, "tables": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/_Tables" } }, "tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] } } }, "_Tables": { "type": "object", "additionalProperties": false, "required": [ "name" ], "properties": { "columns": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/ColumnProperties" } }, "config": { "anyOf": [ { "$ref": "#/definitions/__TablesConfig" }, { "type": "null" } ] }, "data_tests": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/DataTests" } }, "description": { "type": [ "string", "null" ] }, "external": true, "freshness": { "anyOf": [ { "$ref": "#/definitions/FreshnessDefinition" }, { "type": "null" } ] }, "identifier": { "type": [ "string", "null" ] }, "loaded_at_field": { "type": [ "string", "null" ] }, "loaded_at_query": { "type": [ "string", "null" ] }, "loader": { "type": [ "string", "null" ] }, "meta": true, "name": { "type": "string" }, "quoting": { "anyOf": [ { "$ref": "#/definitions/__TablesQuoting" }, { "type": "null" } ] }, "tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "tests": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/DataTests" } } } }, "_UnitTests": { "type": "object", "additionalProperties": false, "required": [ "expect", "model", "name" ], "properties": { "config": { "anyOf": [ { "$ref": "#/definitions/__UnitTestsConfig" }, { "type": "null" } ] }, "description": { "type": [ "string", "null" ] }, "expect": { "$ref": "#/definitions/__UnitTestsExpect" }, "given": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/_Given" } }, "model": { "type": "string" }, "name": { "type": "string" }, "overrides": { "anyOf": [ { "$ref": "#/definitions/__UnitTestsOverrides" }, { "type": "null" } ] }, "versions": true } }, "_Versions": { "type": "object", "additionalProperties": true }, "__AnalysesConfig": { "type": "object", "properties": { "tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] } } }, "__ExportConfigExportAs": { "type": "string", "enum": [ "table", "view", "cache" ] }, "__ExposuresOwner": { "type": "object", "properties": { "email": { "type": [ "string", "null" ] }, "name": { "type": [ "string", "null" ] } } }, "__GroupsOwner": { "type": "object", "properties": { "email": { "type": [ "string", "null" ] }, "name": { "type": [ "string", "null" ] } }, "additionalProperties": true }, "__ModelsTimeSpine": { "type": "object", "required": [ "standard_granularity_column" ], "properties": { "custom_granularities": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/CustomGranularity" } }, "standard_granularity_column": { "type": "string" } } }, "__SavedQueriesConfig": { "type": "object", "properties": { "cache": { "anyOf": [ { "$ref": "#/definitions/___SavedQueriesConfigCache" }, { "type": "null" } ] }, "enabled": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "meta": true }, "additionalProperties": false }, "__SeedsConfig": { "type": "object", "properties": { "column_types": true, "copy_grants": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "database": { "type": [ "string", "null" ] }, "enabled": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "grants": true, "quote_columns": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "schema": { "type": [ "string", "null" ] } } }, "__SemanticModelsDefaults": { "type": "object", "properties": { "agg_time_dimension": { "type": [ "string", "null" ] } } }, "__SnapshotsConfig": { "type": "object", "additionalProperties": false, "properties": { "alias": { "type": [ "string", "null" ] }, "check_cols": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "dbt_valid_to_current": { "type": [ "string", "null" ] }, "enabled": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "grants": true, "meta": true, "persist_docs": { "anyOf": [ { "$ref": "#/definitions/PersistDocsConfig" }, { "type": "null" } ] }, "post-hook": { "anyOf": [ { "$ref": "#/definitions/Hooks" }, { "type": "null" } ] }, "pre-hook": { "anyOf": [ { "$ref": "#/definitions/Hooks" }, { "type": "null" } ] }, "quote_columns": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "snapshot_meta_column_names": { "anyOf": [ { "$ref": "#/definitions/___SnapshotsConfigSnapshotMetaColumnNames" }, { "type": "null" } ] }, "strategy": { "type": [ "string", "null" ] }, "tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "target_database": { "type": [ "string", "null" ] }, "target_schema": { "type": [ "string", "null" ] }, "unique_key": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "updated_at": { "type": [ "string", "null" ] } } }, "__SourcesQuoting": { "type": "object", "properties": { "database": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "identifier": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "schema": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] } } }, "__TablesConfig": { "type": "object", "properties": { "event_time": { "type": [ "string", "null" ] } }, "additionalProperties": true }, "__TablesQuoting": { "type": "object", "properties": { "database": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "identifier": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "schema": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] } } }, "__UnitTestsConfig": { "type": "object", "additionalProperties": false, "properties": { "enabled": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] }, "meta": true, "tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] } } }, "__UnitTestsExpect": { "type": "object", "properties": { "fixture": { "type": [ "string", "null" ] }, "format": { "type": [ "string", "null" ] }, "rows": true } }, "__UnitTestsOverrides": { "type": "object", "properties": { "env_vars": true, "macros": true, "vars": true } }, "___SavedQueriesConfigCache": { "type": "object", "properties": { "enabled": { "anyOf": [ { "$ref": "#/definitions/BooleanOrJinjaString" }, { "type": "null" } ] } } }, "___SnapshotsConfigSnapshotMetaColumnNames": { "type": "object", "properties": { "dbt_scd_id": { "type": [ "string", "null" ] }, "dbt_updated_at": { "type": [ "string", "null" ] }, "dbt_valid_from": { "type": [ "string", "null" ] }, "dbt_valid_to": { "type": [ "string", "null" ] } } } } } ================================================ FILE: core/dbt/jsonschemas/resources/latest.json ================================================ { "$schema": "http://json-schema.org/draft-07/schema#", "title": "DbtPropertiesFile", "type": "object", "properties": { "analyses": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/AnalysesProperties" } }, "anchors": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/AnyValue" } }, "data_tests": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/DataTestProperties" } }, "exposures": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/ExposureProperties" } }, "functions": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/FunctionProperties" } }, "groups": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/GroupProperties" } }, "macros": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/MacrosProperties" } }, "metrics": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/MetricsProperties" } }, "models": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/ModelProperties" } }, "saved_queries": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/SavedQueriesProperties" } }, "seeds": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/SeedProperties" } }, "semantic_models": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/AnyValue" } }, "snapshots": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/SnapshotProperties" } }, "sources": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/SourceProperties" } }, "tests": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/DataTestProperties" } }, "unit_tests": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/UnitTestProperties" } }, "version": { "anyOf": [ { "$ref": "#/definitions/FloatOrString" }, { "type": "null" } ] } }, "additionalProperties": false, "definitions": { "Access": { "type": "string", "enum": [ "private", "protected", "public" ] }, "AggregationType": { "type": "string", "enum": [ "sum", "min", "max", "count_distinct", "sum_boolean", "average", "percentile", "median", "count" ] }, "AnalysesConfig": { "type": "object", "properties": { "description": { "type": [ "string", "null" ] }, "docs": { "anyOf": [ { "$ref": "#/definitions/DocsConfig" }, { "type": "null" } ] }, "enabled": { "type": [ "boolean", "null" ] }, "group": { "type": [ "string", "null" ] }, "meta": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "static_analysis": { "anyOf": [ { "$ref": "#/definitions/StaticAnalysisKind" }, { "type": "null" } ] }, "tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] } }, "additionalProperties": false }, "AnalysesProperties": { "type": "object", "required": [ "name" ], "properties": { "columns": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/ColumnProperties" } }, "config": { "anyOf": [ { "$ref": "#/definitions/AnalysesConfig" }, { "type": "null" } ] }, "description": { "type": [ "string", "null" ] }, "name": { "type": "string" }, "static_analysis_off_reason": { "readOnly": true, "anyOf": [ { "$ref": "#/definitions/StaticAnalysisOffReason" }, { "type": "null" } ] } }, "additionalProperties": false }, "AnyValue": true, "BigqueryPartitionConfig": { "description": "reference: https://github.com/dbt-labs/dbt-adapters/blob/main/dbt-bigquery/src/dbt/adapters/bigquery/relation_configs/_partition.py#L12-L13", "type": "object", "anyOf": [ { "$ref": "#/definitions/RangeConfig" }, { "$ref": "#/definitions/TimeConfig" } ], "required": [ "field" ], "properties": { "copy_partitions": { "default": false, "type": "boolean" }, "data_type": { "default": "date", "type": "string" }, "field": { "type": "string" } }, "additionalProperties": false }, "ClusterConfig": { "description": "Configuration for cluster by columns.\n\ndbt-core allows either of the variants for the `cluster_by` to allow cluster on a single column or on multiple columns", "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } } ] }, "ColumnConfig": { "type": "object", "properties": { "databricks_tags": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "meta": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] } }, "additionalProperties": false }, "ColumnMask": { "type": "object", "required": [ "function" ], "properties": { "function": { "type": "string" }, "using_columns": { "type": [ "string", "null" ] } }, "additionalProperties": false }, "ColumnProperties": { "type": "object", "required": [ "name" ], "properties": { "column_mask": { "anyOf": [ { "$ref": "#/definitions/ColumnMask" }, { "type": "null" } ] }, "config": { "anyOf": [ { "$ref": "#/definitions/ColumnConfig" }, { "type": "null" } ] }, "constraints": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/Constraint" } }, "data_tests": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/DataTests" } }, "data_type": { "type": [ "string", "null" ] }, "databricks_tags": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "description": { "type": [ "string", "null" ] }, "dimension": { "anyOf": [ { "$ref": "#/definitions/ColumnPropertiesDimension" }, { "type": "null" } ] }, "entity": { "anyOf": [ { "$ref": "#/definitions/Entity" }, { "type": "null" } ] }, "granularity": { "anyOf": [ { "$ref": "#/definitions/Granularity" }, { "type": "null" } ] }, "name": { "type": "string" }, "policy_tags": { "type": [ "array", "null" ], "items": { "type": "string" } }, "quote": { "type": [ "boolean", "null" ] }, "tests": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/DataTests" } } }, "additionalProperties": false }, "ColumnPropertiesDimension": { "anyOf": [ { "$ref": "#/definitions/ColumnPropertiesDimensionConfig" }, { "$ref": "#/definitions/ColumnPropertiesDimensionType" } ] }, "ColumnPropertiesDimensionConfig": { "type": "object", "required": [ "type" ], "properties": { "config": { "anyOf": [ { "$ref": "#/definitions/SemanticLayerElementConfig" }, { "type": "null" } ] }, "description": { "type": [ "string", "null" ] }, "is_partition": { "type": [ "boolean", "null" ] }, "label": { "type": [ "string", "null" ] }, "name": { "type": [ "string", "null" ] }, "type": { "$ref": "#/definitions/ColumnPropertiesDimensionType" }, "validity_params": { "anyOf": [ { "$ref": "#/definitions/DimensionValidityParams" }, { "type": "null" } ] } }, "additionalProperties": false }, "ColumnPropertiesDimensionType": { "type": "string", "enum": [ "categorical", "time" ] }, "ColumnPropertiesEntityType": { "type": "string", "enum": [ "foreign", "natural", "primary", "unique" ] }, "ConstantProperty": { "type": "object", "required": [ "base_property", "conversion_property" ], "properties": { "base_property": { "type": "string" }, "conversion_property": { "type": "string" } }, "additionalProperties": false }, "Constraint": { "description": "Constraints (model level or column level)", "type": "object", "required": [ "type" ], "properties": { "expression": { "type": [ "string", "null" ] }, "name": { "type": [ "string", "null" ] }, "to": { "type": [ "string", "null" ] }, "to_columns": { "description": "Only ForeignKey constraints accept: a list columns in that table containing the corresponding primary or unique key.", "type": [ "array", "null" ], "items": { "type": "string" } }, "type": { "$ref": "#/definitions/ConstraintType" }, "warn_unenforced": { "type": [ "boolean", "null" ] }, "warn_unsupported": { "type": [ "boolean", "null" ] } }, "additionalProperties": false }, "ConstraintType": { "type": "string", "enum": [ "not_null", "unique", "primary_key", "foreign_key", "check", "custom" ] }, "ConversionCalculationType": { "type": "string", "enum": [ "conversions", "conversion_rate" ] }, "CustomTest": { "anyOf": [ { "$ref": "#/definitions/CustomTestMultiKey" }, { "type": "object", "additionalProperties": { "$ref": "#/definitions/CustomTestInner" } } ] }, "CustomTestInner": { "type": "object", "properties": { "arguments": { "anyOf": [ { "$ref": "#/definitions/AnyValue" }, { "type": "null" } ] }, "column_name": { "type": [ "string", "null" ] }, "config": { "anyOf": [ { "$ref": "#/definitions/DataTestConfig" }, { "type": "null" } ] }, "description": { "type": [ "string", "null" ] }, "name": { "type": [ "string", "null" ] } }, "additionalProperties": false }, "CustomTestMultiKey": { "type": "object", "required": [ "test_name" ], "properties": { "arguments": { "anyOf": [ { "$ref": "#/definitions/AnyValue" }, { "type": "null" } ] }, "column_name": { "type": [ "string", "null" ] }, "config": { "anyOf": [ { "$ref": "#/definitions/DataTestConfig" }, { "type": "null" } ] }, "description": { "type": [ "string", "null" ] }, "name": { "type": [ "string", "null" ] }, "test_name": { "type": "string" } }, "additionalProperties": false }, "DataLakeObjectCategory": { "description": "See `category` from https://developer.salesforce.com/docs/data/connectapi/references/spec?meta=postDataLakeObject", "type": "string", "enum": [ "Profile", "Engagement", "Directory_Table", "Insights", "Other" ] }, "DataTestConfig": { "description": "This configuration is a superset of all warehouse specific configurations that users can set", "type": "object", "properties": { "adapter_properties": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "alias": { "type": [ "string", "null" ] }, "as_columnstore": { "type": [ "boolean", "null" ] }, "auto_liquid_cluster": { "type": [ "boolean", "null" ] }, "auto_refresh": { "type": [ "boolean", "null" ] }, "automatic_clustering": { "type": [ "boolean", "null" ] }, "backup": { "type": [ "boolean", "null" ] }, "base_location_root": { "type": [ "string", "null" ] }, "base_location_subpath": { "type": [ "string", "null" ] }, "batch_id": { "type": [ "string", "null" ] }, "bind": { "type": [ "boolean", "null" ] }, "buckets": { "type": [ "integer", "null" ], "format": "int64" }, "catalog": { "type": [ "string", "null" ] }, "catalog_name": { "type": [ "string", "null" ] }, "category": { "anyOf": [ { "$ref": "#/definitions/DataLakeObjectCategory" }, { "type": "null" } ] }, "cluster_by": { "anyOf": [ { "$ref": "#/definitions/ClusterConfig" }, { "type": "null" } ] }, "clustered_by": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "compression": { "type": [ "string", "null" ] }, "copy_grants": { "type": [ "boolean", "null" ] }, "database": { "type": [ "string", "null" ] }, "databricks_compute": { "type": [ "string", "null" ] }, "databricks_tags": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "dataproc_cluster_name": { "type": [ "string", "null" ] }, "description": { "type": [ "string", "null" ] }, "dist": { "type": [ "string", "null" ] }, "enable_list_inference": { "type": [ "boolean", "null" ] }, "enable_refresh": { "type": [ "boolean", "null" ] }, "enabled": { "default": null, "type": [ "boolean", "null" ] }, "error_if": { "type": [ "string", "null" ] }, "external_volume": { "type": [ "string", "null" ] }, "fail_calc": { "type": [ "string", "null" ] }, "file_format": { "type": [ "string", "null" ] }, "full_refresh": { "default": null, "type": [ "boolean", "null" ] }, "grant_access_to": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/GrantAccessToTarget" } }, "group": { "type": [ "string", "null" ] }, "hours_to_expiration": { "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "include_full_name_in_path": { "type": [ "boolean", "null" ] }, "indexes": { "default": null, "allOf": [ { "$ref": "#/definitions/IndexesConfig" } ] }, "initialize": { "type": [ "string", "null" ] }, "intermediate_format": { "type": [ "string", "null" ] }, "jar_file_uri": { "type": [ "string", "null" ] }, "job_execution_timeout_seconds": { "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "kms_key_name": { "type": [ "string", "null" ] }, "labels": { "type": [ "object", "null" ], "additionalProperties": { "type": "string" } }, "labels_from_meta": { "type": [ "boolean", "null" ] }, "limit": { "type": [ "integer", "null" ], "format": "int32" }, "liquid_clustered_by": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "location_root": { "type": [ "string", "null" ] }, "matched_condition": { "type": [ "string", "null" ] }, "materialized": { "anyOf": [ { "$ref": "#/definitions/DbtMaterialization" }, { "type": "null" } ] }, "max_staleness": { "type": [ "string", "null" ] }, "merge_with_schema_evolution": { "type": [ "boolean", "null" ] }, "meta": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "not_matched_by_source_action": { "type": [ "string", "null" ] }, "not_matched_by_source_condition": { "type": [ "string", "null" ] }, "not_matched_condition": { "type": [ "string", "null" ] }, "notebook_template_id": { "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "partition_by": { "anyOf": [ { "$ref": "#/definitions/PartitionConfig" }, { "type": "null" } ] }, "partition_expiration_days": { "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "partitions": { "type": [ "array", "null" ], "items": { "type": "string" } }, "primary_key": { "default": null, "allOf": [ { "$ref": "#/definitions/PrimaryKeyConfig" } ] }, "query_tag": { "anyOf": [ { "$ref": "#/definitions/QueryTag" }, { "type": "null" } ] }, "quoting": { "anyOf": [ { "$ref": "#/definitions/DbtQuoting" }, { "type": "null" } ] }, "refresh_interval_minutes": { "type": [ "number", "null" ], "format": "double" }, "refresh_mode": { "type": [ "string", "null" ] }, "require_partition_filter": { "type": [ "boolean", "null" ] }, "resource_tags": { "type": [ "object", "null" ], "additionalProperties": { "type": "string" } }, "row_access_policy": { "type": [ "string", "null" ] }, "schedule": { "anyOf": [ { "$ref": "#/definitions/Schedule" }, { "type": "null" } ] }, "schema": { "type": [ "string", "null" ] }, "secure": { "type": [ "boolean", "null" ] }, "severity": { "anyOf": [ { "$ref": "#/definitions/Severity" }, { "type": "null" } ] }, "skip_matched_step": { "type": [ "boolean", "null" ] }, "skip_not_matched_step": { "type": [ "boolean", "null" ] }, "snowflake_warehouse": { "type": [ "string", "null" ] }, "sort": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "sort_type": { "type": [ "string", "null" ] }, "source_alias": { "type": [ "string", "null" ] }, "sql_header": { "type": [ "string", "null" ] }, "static_analysis": { "anyOf": [ { "$ref": "#/definitions/StaticAnalysisKind" }, { "type": "null" } ] }, "store_failures": { "default": null, "type": [ "boolean", "null" ] }, "store_failures_as": { "anyOf": [ { "$ref": "#/definitions/StoreFailuresAs" }, { "type": "null" } ] }, "table_tag": { "type": [ "string", "null" ] }, "table_type": { "type": [ "string", "null" ] }, "tags": { "default": [], "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "target_alias": { "type": [ "string", "null" ] }, "target_lag": { "type": [ "string", "null" ] }, "tblproperties": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "timeout": { "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "tmp_relation_type": { "type": [ "string", "null" ] }, "transient": { "type": [ "boolean", "null" ] }, "warn_if": { "type": [ "string", "null" ] }, "where": { "type": [ "string", "null" ] } }, "additionalProperties": false }, "DataTestProperties": { "type": "object", "required": [ "name" ], "properties": { "config": { "anyOf": [ { "$ref": "#/definitions/DataTestConfig" }, { "type": "null" } ] }, "description": { "type": [ "string", "null" ] }, "name": { "type": "string" }, "static_analysis_off_reason": { "readOnly": true, "anyOf": [ { "$ref": "#/definitions/StaticAnalysisOffReason" }, { "type": "null" } ] } }, "additionalProperties": false }, "DataTests": { "anyOf": [ { "type": "string" }, { "$ref": "#/definitions/CustomTest" } ] }, "DbtBatchSize": { "type": "string", "enum": [ "hour", "day", "month", "year" ] }, "DbtContract": { "type": "object", "properties": { "alias_types": { "default": true, "type": "boolean" }, "checksum": { "anyOf": [ { "$ref": "#/definitions/AnyValue" }, { "type": "null" } ] }, "enforced": { "default": false, "type": "boolean" } }, "additionalProperties": false }, "DbtIncrementalStrategy": { "oneOf": [ { "type": "string", "enum": [ "append", "merge", "delete+insert", "insert_overwrite", "microbatch" ] }, { "description": "replace_where (Databricks only) see https://docs.getdbt.com/reference/resource-configs/databricks-configs", "type": "string", "enum": [ "replace_where" ] }, { "type": "object", "required": [ "custom" ], "properties": { "custom": { "type": "string" } }, "additionalProperties": false } ] }, "DbtMaterialization": { "oneOf": [ { "type": "string", "enum": [ "snapshot", "seed", "view", "table", "incremental", "materialized_view", "external", "test", "ephemeral", "unit", "analysis", "function" ] }, { "description": "only for databricks", "type": "string", "enum": [ "streaming_table" ] }, { "description": "only for snowflake", "type": "string", "enum": [ "dynamic_table" ] }, { "description": "for inline SQL compilation", "type": "string", "enum": [ "inline" ] }, { "type": "object", "required": [ "unknown" ], "properties": { "unknown": { "type": "string" } }, "additionalProperties": false } ] }, "DbtOwner": { "type": "object", "properties": { "email": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "name": { "type": [ "string", "null" ] } }, "additionalProperties": false }, "DbtQuoting": { "type": "object", "properties": { "database": { "default": null, "type": [ "boolean", "null" ] }, "identifier": { "default": null, "type": [ "boolean", "null" ] }, "schema": { "default": null, "type": [ "boolean", "null" ] }, "snowflake_ignore_case": { "type": [ "boolean", "null" ] } }, "additionalProperties": false }, "DbtUniqueKey": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } } ] }, "DerivedDimension": { "type": "object", "required": [ "expr", "name", "type" ], "properties": { "config": { "anyOf": [ { "$ref": "#/definitions/SemanticLayerElementConfig" }, { "type": "null" } ] }, "description": { "type": [ "string", "null" ] }, "expr": { "type": "string" }, "granularity": { "anyOf": [ { "$ref": "#/definitions/Granularity" }, { "type": "null" } ] }, "is_partition": { "type": [ "boolean", "null" ] }, "label": { "type": [ "string", "null" ] }, "name": { "type": "string" }, "type": { "$ref": "#/definitions/ColumnPropertiesDimensionType" }, "validity_params": { "anyOf": [ { "$ref": "#/definitions/DimensionValidityParams" }, { "type": "null" } ] } }, "additionalProperties": false }, "DerivedEntity": { "type": "object", "required": [ "expr", "name", "type" ], "properties": { "config": { "anyOf": [ { "$ref": "#/definitions/SemanticLayerElementConfig" }, { "type": "null" } ] }, "description": { "type": [ "string", "null" ] }, "expr": { "type": "string" }, "label": { "type": [ "string", "null" ] }, "name": { "type": "string" }, "type": { "$ref": "#/definitions/ColumnPropertiesEntityType" } }, "additionalProperties": false }, "DerivedSemantics": { "type": "object", "properties": { "dimensions": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/DerivedDimension" } }, "entities": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/DerivedEntity" } } }, "additionalProperties": false }, "DimensionValidityParams": { "type": "object", "properties": { "is_end": { "default": false, "type": "boolean" }, "is_start": { "default": false, "type": "boolean" } }, "additionalProperties": false }, "DocsConfig": { "type": "object", "properties": { "node_color": { "type": [ "string", "null" ] }, "show": { "default": true, "type": "boolean" } }, "additionalProperties": false }, "Entity": { "anyOf": [ { "$ref": "#/definitions/EntityConfig" }, { "$ref": "#/definitions/ColumnPropertiesEntityType" } ] }, "EntityConfig": { "type": "object", "required": [ "type" ], "properties": { "config": { "anyOf": [ { "$ref": "#/definitions/SemanticLayerElementConfig" }, { "type": "null" } ] }, "description": { "type": [ "string", "null" ] }, "label": { "type": [ "string", "null" ] }, "name": { "type": [ "string", "null" ] }, "type": { "$ref": "#/definitions/ColumnPropertiesEntityType" } }, "additionalProperties": false }, "Expect": { "type": "object", "properties": { "fixture": { "type": [ "string", "null" ] }, "format": { "default": "dict", "allOf": [ { "$ref": "#/definitions/Formats" } ] }, "rows": { "anyOf": [ { "$ref": "#/definitions/Rows" }, { "type": "null" } ] } }, "additionalProperties": false }, "Export": { "type": "object", "required": [ "name" ], "properties": { "config": { "anyOf": [ { "$ref": "#/definitions/ExportConfig" }, { "type": "null" } ] }, "name": { "type": "string" } }, "additionalProperties": false }, "ExportConfig": { "type": "object", "properties": { "alias": { "type": [ "string", "null" ] }, "export_as": { "anyOf": [ { "$ref": "#/definitions/ExportConfigExportAs" }, { "type": "null" } ] }, "schema": { "type": [ "string", "null" ] } }, "additionalProperties": false }, "ExportConfigExportAs": { "type": "string", "enum": [ "table", "view", "cache" ] }, "ExposureConfig": { "type": "object", "properties": { "enabled": { "default": null, "type": [ "boolean", "null" ] }, "meta": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "tags": { "default": [], "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] } }, "additionalProperties": false }, "ExposureProperties": { "type": "object", "required": [ "name", "owner", "type" ], "properties": { "config": { "anyOf": [ { "$ref": "#/definitions/ExposureConfig" }, { "type": "null" } ] }, "depends_on": { "type": [ "array", "null" ], "items": { "type": "string" } }, "description": { "type": [ "string", "null" ] }, "label": { "type": [ "string", "null" ] }, "maturity": { "type": [ "string", "null" ] }, "name": { "type": "string" }, "owner": { "$ref": "#/definitions/DbtOwner" }, "type": { "$ref": "#/definitions/ExposureType" }, "url": { "type": [ "string", "null" ] } }, "additionalProperties": false }, "ExposureType": { "type": "string", "enum": [ "dashboard", "notebook", "analysis", "ml", "application" ] }, "FloatOrString": { "anyOf": [ { "type": "number", "format": "float" }, { "type": "string" } ] }, "Formats": { "type": "string", "enum": [ "dict", "csv", "sql" ] }, "FreshnessDefinition": { "type": "object", "properties": { "error_after": { "default": { "count": null, "period": null }, "anyOf": [ { "$ref": "#/definitions/FreshnessRules" }, { "type": "null" } ] }, "filter": { "type": [ "string", "null" ] }, "warn_after": { "default": { "count": null, "period": null }, "anyOf": [ { "$ref": "#/definitions/FreshnessRules" }, { "type": "null" } ] } }, "additionalProperties": false }, "FreshnessPeriod": { "type": "string", "enum": [ "minute", "hour", "day" ] }, "FreshnessRules": { "type": "object", "properties": { "count": { "type": [ "integer", "null" ], "format": "int64" }, "period": { "anyOf": [ { "$ref": "#/definitions/FreshnessPeriod" }, { "type": "null" } ] } }, "additionalProperties": false }, "FunctionArgument": { "type": "object", "properties": { "data_type": { "type": [ "string", "null" ] }, "default_value": { "type": [ "string", "null" ] }, "description": { "type": [ "string", "null" ] }, "name": { "type": [ "string", "null" ] } }, "additionalProperties": false }, "FunctionConfig": { "description": "This configuration is a superset of all warehouse specific configurations that users can set", "type": "object", "properties": { "access": { "anyOf": [ { "$ref": "#/definitions/Access" }, { "type": "null" } ] }, "adapter_properties": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "alias": { "type": [ "string", "null" ] }, "as_columnstore": { "type": [ "boolean", "null" ] }, "auto_liquid_cluster": { "type": [ "boolean", "null" ] }, "auto_refresh": { "type": [ "boolean", "null" ] }, "automatic_clustering": { "type": [ "boolean", "null" ] }, "backup": { "type": [ "boolean", "null" ] }, "base_location_root": { "type": [ "string", "null" ] }, "base_location_subpath": { "type": [ "string", "null" ] }, "batch_id": { "type": [ "string", "null" ] }, "bind": { "type": [ "boolean", "null" ] }, "buckets": { "type": [ "integer", "null" ], "format": "int64" }, "catalog": { "type": [ "string", "null" ] }, "catalog_name": { "type": [ "string", "null" ] }, "category": { "anyOf": [ { "$ref": "#/definitions/DataLakeObjectCategory" }, { "type": "null" } ] }, "cluster_by": { "anyOf": [ { "$ref": "#/definitions/ClusterConfig" }, { "type": "null" } ] }, "clustered_by": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "compression": { "type": [ "string", "null" ] }, "copy_grants": { "type": [ "boolean", "null" ] }, "database": { "type": [ "string", "null" ] }, "databricks_compute": { "type": [ "string", "null" ] }, "databricks_tags": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "dataproc_cluster_name": { "type": [ "string", "null" ] }, "description": { "type": [ "string", "null" ] }, "dist": { "type": [ "string", "null" ] }, "docs": { "anyOf": [ { "$ref": "#/definitions/DocsConfig" }, { "type": "null" } ] }, "enable_list_inference": { "type": [ "boolean", "null" ] }, "enable_refresh": { "type": [ "boolean", "null" ] }, "enabled": { "type": [ "boolean", "null" ] }, "entry_point": { "type": [ "string", "null" ] }, "external_volume": { "type": [ "string", "null" ] }, "file_format": { "type": [ "string", "null" ] }, "grant_access_to": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/GrantAccessToTarget" } }, "grants": { "$ref": "#/definitions/GrantConfig" }, "group": { "type": [ "string", "null" ] }, "hours_to_expiration": { "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "include_full_name_in_path": { "type": [ "boolean", "null" ] }, "indexes": { "default": null, "allOf": [ { "$ref": "#/definitions/IndexesConfig" } ] }, "initialize": { "type": [ "string", "null" ] }, "intermediate_format": { "type": [ "string", "null" ] }, "jar_file_uri": { "type": [ "string", "null" ] }, "job_execution_timeout_seconds": { "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "kms_key_name": { "type": [ "string", "null" ] }, "labels": { "type": [ "object", "null" ], "additionalProperties": { "type": "string" } }, "labels_from_meta": { "type": [ "boolean", "null" ] }, "liquid_clustered_by": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "location_root": { "type": [ "string", "null" ] }, "matched_condition": { "type": [ "string", "null" ] }, "max_staleness": { "type": [ "string", "null" ] }, "merge_with_schema_evolution": { "type": [ "boolean", "null" ] }, "meta": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "not_matched_by_source_action": { "type": [ "string", "null" ] }, "not_matched_by_source_condition": { "type": [ "string", "null" ] }, "not_matched_condition": { "type": [ "string", "null" ] }, "notebook_template_id": { "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "on_configuration_change": { "type": [ "string", "null" ] }, "partition_by": { "anyOf": [ { "$ref": "#/definitions/PartitionConfig" }, { "type": "null" } ] }, "partition_expiration_days": { "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "partitions": { "type": [ "array", "null" ], "items": { "type": "string" } }, "primary_key": { "default": null, "allOf": [ { "$ref": "#/definitions/PrimaryKeyConfig" } ] }, "query_tag": { "anyOf": [ { "$ref": "#/definitions/QueryTag" }, { "type": "null" } ] }, "quoting": { "anyOf": [ { "$ref": "#/definitions/DbtQuoting" }, { "type": "null" } ] }, "refresh_interval_minutes": { "type": [ "number", "null" ], "format": "double" }, "refresh_mode": { "type": [ "string", "null" ] }, "require_partition_filter": { "type": [ "boolean", "null" ] }, "resource_tags": { "type": [ "object", "null" ], "additionalProperties": { "type": "string" } }, "row_access_policy": { "type": [ "string", "null" ] }, "runtime_version": { "type": [ "string", "null" ] }, "schedule": { "anyOf": [ { "$ref": "#/definitions/Schedule" }, { "type": "null" } ] }, "schema": { "type": [ "string", "null" ] }, "secure": { "type": [ "boolean", "null" ] }, "skip_matched_step": { "type": [ "boolean", "null" ] }, "skip_not_matched_step": { "type": [ "boolean", "null" ] }, "snowflake_warehouse": { "type": [ "string", "null" ] }, "sort": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "sort_type": { "type": [ "string", "null" ] }, "source_alias": { "type": [ "string", "null" ] }, "static_analysis": { "anyOf": [ { "$ref": "#/definitions/StaticAnalysisKind" }, { "type": "null" } ] }, "table_tag": { "type": [ "string", "null" ] }, "table_type": { "type": [ "string", "null" ] }, "tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "target_alias": { "type": [ "string", "null" ] }, "target_lag": { "type": [ "string", "null" ] }, "tblproperties": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "timeout": { "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "tmp_relation_type": { "type": [ "string", "null" ] }, "transient": { "type": [ "boolean", "null" ] }, "type": { "default": "scalar", "anyOf": [ { "$ref": "#/definitions/FunctionKind" }, { "type": "null" } ] }, "volatility": { "anyOf": [ { "$ref": "#/definitions/Volatility" }, { "type": "null" } ] } }, "additionalProperties": false }, "FunctionKind": { "description": "Function kind enum with same values as UDFKind", "type": "string", "enum": [ "scalar", "aggregate", "table" ] }, "FunctionProperties": { "type": "object", "required": [ "name" ], "properties": { "arguments": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/FunctionArgument" } }, "config": { "anyOf": [ { "$ref": "#/definitions/FunctionConfig" }, { "type": "null" } ] }, "data_tests": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/DataTests" } }, "description": { "type": [ "string", "null" ] }, "identifier": { "type": [ "string", "null" ] }, "language": { "default": "sql", "type": [ "string", "null" ] }, "name": { "type": "string" }, "returns": { "anyOf": [ { "$ref": "#/definitions/FunctionReturnType" }, { "type": "null" } ] }, "static_analysis_off_reason": { "readOnly": true, "anyOf": [ { "$ref": "#/definitions/StaticAnalysisOffReason" }, { "type": "null" } ] }, "tests": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/DataTests" } } }, "additionalProperties": false }, "FunctionReturnType": { "type": "object", "properties": { "data_type": { "type": [ "string", "null" ] }, "description": { "type": [ "string", "null" ] } }, "additionalProperties": false }, "Given": { "type": "object", "required": [ "input" ], "properties": { "fixture": { "type": [ "string", "null" ] }, "format": { "default": "dict", "allOf": [ { "$ref": "#/definitions/Formats" } ] }, "input": { "type": "string" }, "rows": { "anyOf": [ { "$ref": "#/definitions/Rows" }, { "type": "null" } ] } }, "additionalProperties": false }, "GrantAccessToTarget": { "type": "object", "properties": { "dataset": { "type": [ "string", "null" ] }, "project": { "type": [ "string", "null" ] } }, "additionalProperties": false }, "GrantConfig": { "description": "Wrapper type for grant configurations that normalizes values to arrays during serialization.\n\nThis type handles grant configurations with the following behavior: - Normalizes string values to arrays during serialization - Preserves insertion order using IndexMap", "type": "object", "additionalProperties": { "$ref": "#/definitions/StringOrArrayOfStrings" } }, "Granularity": { "type": "string", "enum": [ "nanosecond", "microsecond", "millisecond", "second", "minute", "hour", "day", "week", "month", "quarter", "year" ] }, "GroupConfig": { "type": "object", "properties": { "meta": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } } }, "additionalProperties": false }, "GroupProperties": { "type": "object", "required": [ "name", "owner" ], "properties": { "config": { "anyOf": [ { "$ref": "#/definitions/GroupConfig" }, { "type": "null" } ] }, "description": { "type": [ "string", "null" ] }, "name": { "type": "string" }, "owner": { "$ref": "#/definitions/DbtOwner" } }, "additionalProperties": false }, "HardDeletes": { "type": "string", "enum": [ "ignore", "invalidate", "new_record" ] }, "HookConfig": { "type": "object", "properties": { "index": { "type": [ "string", "null" ] }, "sql": { "type": [ "string", "null" ] }, "transaction": { "type": [ "boolean", "null" ] } }, "additionalProperties": false }, "Hooks": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "$ref": "#/definitions/HookConfig" }, { "type": "array", "items": { "$ref": "#/definitions/HookConfig" } } ] }, "IndexesConfig": { "description": "A wrapper type for the `indexes` config field that handles flexible deserialization.\n\ndbt-core accepts both list and dictionary formats for indexes. This type accepts either format on input but always serializes as a list.\n\n# Example\n\n```ignore // Input (list): [{columns: [\"id\"], unique: true}] // Serializes as: [{columns: [\"id\"], unique: true}]\n\n// Input (dict): {\"my_index\": {columns: [\"id\"], unique: true}} // Serializes as: [{columns: [\"id\"], unique: true}] (keys are discarded) ```", "type": [ "array", "null" ], "items": { "$ref": "#/definitions/PostgresIndex" } }, "MacrosArguments": { "type": "object", "required": [ "name" ], "properties": { "description": { "type": [ "string", "null" ] }, "name": { "type": "string" }, "type": { "type": [ "string", "null" ] } }, "additionalProperties": false }, "MacrosConfig": { "type": "object", "properties": { "docs": { "anyOf": [ { "$ref": "#/definitions/DocsConfig" }, { "type": "null" } ] }, "meta": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } } }, "additionalProperties": false }, "MacrosProperties": { "type": "object", "required": [ "name" ], "properties": { "arguments": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/MacrosArguments" } }, "config": { "anyOf": [ { "$ref": "#/definitions/MacrosConfig" }, { "type": "null" } ] }, "description": { "type": [ "string", "null" ] }, "docs": { "anyOf": [ { "$ref": "#/definitions/DocsConfig" }, { "type": "null" } ] }, "meta": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "name": { "type": "string" } }, "additionalProperties": false }, "MetricConfig": { "type": "object", "properties": { "enabled": { "default": null, "type": [ "boolean", "null" ] }, "group": { "type": [ "string", "null" ] }, "meta": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "tags": { "default": [], "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] } }, "additionalProperties": false }, "MetricExpr": { "anyOf": [ { "type": "string" }, { "type": "integer", "format": "int32" } ] }, "MetricPropertiesMetricInput": { "type": "object", "required": [ "name" ], "properties": { "alias": { "type": [ "string", "null" ] }, "filter": { "type": [ "string", "null" ] }, "name": { "type": "string" }, "offset_to_grain": { "type": [ "string", "null" ] }, "offset_window": { "type": [ "string", "null" ] } }, "additionalProperties": false }, "MetricPropertiesNonAdditiveDimension": { "type": "object", "required": [ "name", "window_agg" ], "properties": { "group_by": { "type": [ "array", "null" ], "items": { "type": "string" } }, "name": { "type": "string" }, "window_agg": { "$ref": "#/definitions/WindowChoice" } }, "additionalProperties": false }, "MetricType": { "type": "string", "enum": [ "simple", "ratio", "cumulative", "derived", "conversion" ] }, "MetricsProperties": { "type": "object", "required": [ "name" ], "properties": { "agg": { "anyOf": [ { "$ref": "#/definitions/AggregationType" }, { "type": "null" } ] }, "agg_time_dimension": { "type": [ "string", "null" ] }, "base_metric": { "anyOf": [ { "$ref": "#/definitions/StringOrMetricPropertiesMetricInput" }, { "type": "null" } ] }, "calculation": { "anyOf": [ { "$ref": "#/definitions/ConversionCalculationType" }, { "type": "null" } ] }, "config": { "anyOf": [ { "$ref": "#/definitions/MetricConfig" }, { "type": "null" } ] }, "constant_properties": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/ConstantProperty" } }, "conversion_metric": { "anyOf": [ { "$ref": "#/definitions/StringOrMetricPropertiesMetricInput" }, { "type": "null" } ] }, "denominator": { "anyOf": [ { "$ref": "#/definitions/StringOrMetricPropertiesMetricInput" }, { "type": "null" } ] }, "description": { "type": [ "string", "null" ] }, "entity": { "type": [ "string", "null" ] }, "expr": { "anyOf": [ { "$ref": "#/definitions/MetricExpr" }, { "type": "null" } ] }, "fill_nulls_with": { "type": [ "integer", "null" ], "format": "int32" }, "filter": { "type": [ "string", "null" ] }, "grain_to_date": { "anyOf": [ { "$ref": "#/definitions/Granularity" }, { "type": "null" } ] }, "hidden": { "default": false, "type": [ "boolean", "null" ] }, "input_metric": { "anyOf": [ { "$ref": "#/definitions/StringOrMetricPropertiesMetricInput" }, { "type": "null" } ] }, "input_metrics": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/MetricPropertiesMetricInput" } }, "join_to_timespine": { "type": [ "boolean", "null" ] }, "label": { "type": [ "string", "null" ] }, "name": { "type": "string" }, "non_additive_dimension": { "anyOf": [ { "$ref": "#/definitions/MetricPropertiesNonAdditiveDimension" }, { "type": "null" } ] }, "numerator": { "anyOf": [ { "$ref": "#/definitions/StringOrMetricPropertiesMetricInput" }, { "type": "null" } ] }, "percentile": { "type": [ "number", "null" ], "format": "float" }, "percentile_type": { "anyOf": [ { "$ref": "#/definitions/PercentileType" }, { "type": "null" } ] }, "period_agg": { "anyOf": [ { "$ref": "#/definitions/PeriodAggregationType" }, { "type": "null" } ] }, "time_granularity": { "anyOf": [ { "$ref": "#/definitions/Granularity" }, { "type": "null" } ] }, "type": { "anyOf": [ { "$ref": "#/definitions/MetricType" }, { "type": "null" } ] }, "window": { "type": [ "string", "null" ] } }, "additionalProperties": false }, "ModelConfig": { "description": "This configuration is a superset of all warehouse specific configurations that users can set", "type": "object", "properties": { "access": { "anyOf": [ { "$ref": "#/definitions/Access" }, { "type": "null" } ] }, "adapter_properties": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "additional_libs": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/AnyValue" } }, "alias": { "type": [ "string", "null" ] }, "as_columnstore": { "type": [ "boolean", "null" ] }, "auto_liquid_cluster": { "type": [ "boolean", "null" ] }, "auto_refresh": { "type": [ "boolean", "null" ] }, "automatic_clustering": { "type": [ "boolean", "null" ] }, "backup": { "type": [ "boolean", "null" ] }, "base_location_root": { "type": [ "string", "null" ] }, "base_location_subpath": { "type": [ "string", "null" ] }, "batch_id": { "type": [ "string", "null" ] }, "batch_size": { "anyOf": [ { "$ref": "#/definitions/DbtBatchSize" }, { "type": "null" } ] }, "begin": { "type": [ "string", "null" ] }, "bind": { "type": [ "boolean", "null" ] }, "buckets": { "type": [ "integer", "null" ], "format": "int64" }, "catalog": { "type": [ "string", "null" ] }, "catalog_name": { "type": [ "string", "null" ] }, "category": { "anyOf": [ { "$ref": "#/definitions/DataLakeObjectCategory" }, { "type": "null" } ] }, "cluster_by": { "anyOf": [ { "$ref": "#/definitions/ClusterConfig" }, { "type": "null" } ] }, "cluster_id": { "type": [ "string", "null" ] }, "clustered_by": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "column_types": { "type": [ "object", "null" ], "additionalProperties": { "type": "string" } }, "compression": { "type": [ "string", "null" ] }, "concurrent_batches": { "default": null, "type": [ "boolean", "null" ] }, "contract": { "anyOf": [ { "$ref": "#/definitions/DbtContract" }, { "type": "null" } ] }, "copy_grants": { "type": [ "boolean", "null" ] }, "create_notebook": { "type": [ "boolean", "null" ] }, "database": { "type": [ "string", "null" ] }, "databricks_compute": { "type": [ "string", "null" ] }, "databricks_tags": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "dataproc_cluster_name": { "type": [ "string", "null" ] }, "description": { "type": [ "string", "null" ] }, "dist": { "type": [ "string", "null" ] }, "docs": { "anyOf": [ { "$ref": "#/definitions/DocsConfig" }, { "type": "null" } ] }, "enable_list_inference": { "type": [ "boolean", "null" ] }, "enable_refresh": { "type": [ "boolean", "null" ] }, "enabled": { "default": null, "type": [ "boolean", "null" ] }, "event_time": { "type": [ "string", "null" ] }, "external_access_integrations": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "external_volume": { "type": [ "string", "null" ] }, "file_format": { "type": [ "string", "null" ] }, "freshness": { "anyOf": [ { "$ref": "#/definitions/ModelFreshness" }, { "type": "null" } ] }, "full_refresh": { "default": null, "type": [ "boolean", "null" ] }, "grant_access_to": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/GrantAccessToTarget" } }, "grants": { "$ref": "#/definitions/GrantConfig" }, "group": { "type": [ "string", "null" ] }, "hours_to_expiration": { "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "http_path": { "type": [ "string", "null" ] }, "imports": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "include_full_name_in_path": { "type": [ "boolean", "null" ] }, "incremental_predicates": { "type": [ "array", "null" ], "items": { "type": "string" } }, "incremental_strategy": { "anyOf": [ { "$ref": "#/definitions/DbtIncrementalStrategy" }, { "type": "null" } ] }, "index_url": { "type": [ "string", "null" ] }, "indexes": { "default": null, "allOf": [ { "$ref": "#/definitions/IndexesConfig" } ] }, "initialize": { "type": [ "string", "null" ] }, "intermediate_format": { "type": [ "string", "null" ] }, "jar_file_uri": { "type": [ "string", "null" ] }, "job_cluster_config": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "job_execution_timeout_seconds": { "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "kms_key_name": { "type": [ "string", "null" ] }, "labels": { "type": [ "object", "null" ], "additionalProperties": { "type": "string" } }, "labels_from_meta": { "type": [ "boolean", "null" ] }, "liquid_clustered_by": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "location": { "type": [ "string", "null" ] }, "location_root": { "type": [ "string", "null" ] }, "lookback": { "type": [ "integer", "null" ], "format": "int32" }, "matched_condition": { "type": [ "string", "null" ] }, "materialized": { "anyOf": [ { "$ref": "#/definitions/DbtMaterialization" }, { "type": "null" } ] }, "max_staleness": { "type": [ "string", "null" ] }, "merge_exclude_columns": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "merge_update_columns": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "merge_with_schema_evolution": { "type": [ "boolean", "null" ] }, "meta": { "default": {}, "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "not_matched_by_source_action": { "type": [ "string", "null" ] }, "not_matched_by_source_condition": { "type": [ "string", "null" ] }, "not_matched_condition": { "type": [ "string", "null" ] }, "notebook_template_id": { "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "on_configuration_change": { "anyOf": [ { "$ref": "#/definitions/OnConfigurationChange" }, { "type": "null" } ] }, "on_schema_change": { "anyOf": [ { "$ref": "#/definitions/OnSchemaChange" }, { "type": "null" } ] }, "packages": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "partition_by": { "anyOf": [ { "$ref": "#/definitions/PartitionConfig" }, { "type": "null" } ] }, "partition_expiration_days": { "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "partitions": { "type": [ "array", "null" ], "items": { "type": "string" } }, "persist_docs": { "anyOf": [ { "$ref": "#/definitions/PersistDocsConfig" }, { "type": "null" } ] }, "post_hook": { "anyOf": [ { "$ref": "#/definitions/Hooks" }, { "type": "null" } ] }, "pre_hook": { "anyOf": [ { "$ref": "#/definitions/Hooks" }, { "type": "null" } ] }, "predicates": { "type": [ "array", "null" ], "items": { "type": "string" } }, "primary_key": { "default": null, "allOf": [ { "$ref": "#/definitions/PrimaryKeyConfig" } ] }, "python_job_config": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "python_version": { "type": [ "string", "null" ] }, "query_tag": { "anyOf": [ { "$ref": "#/definitions/QueryTag" }, { "type": "null" } ] }, "quoting": { "anyOf": [ { "$ref": "#/definitions/DbtQuoting" }, { "type": "null" } ] }, "refresh_interval_minutes": { "type": [ "number", "null" ], "format": "double" }, "refresh_mode": { "type": [ "string", "null" ] }, "require_partition_filter": { "type": [ "boolean", "null" ] }, "resource_tags": { "type": [ "object", "null" ], "additionalProperties": { "type": "string" } }, "row_access_policy": { "type": [ "string", "null" ] }, "schedule": { "anyOf": [ { "$ref": "#/definitions/Schedule" }, { "type": "null" } ] }, "schema": { "type": [ "string", "null" ] }, "secrets": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "secure": { "type": [ "boolean", "null" ] }, "skip_matched_step": { "type": [ "boolean", "null" ] }, "skip_not_matched_step": { "type": [ "boolean", "null" ] }, "snowflake_warehouse": { "type": [ "string", "null" ] }, "sort": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "sort_type": { "type": [ "string", "null" ] }, "source_alias": { "type": [ "string", "null" ] }, "sql_header": { "type": [ "string", "null" ] }, "static_analysis": { "anyOf": [ { "$ref": "#/definitions/StaticAnalysisKind" }, { "type": "null" } ] }, "submission_method": { "type": [ "string", "null" ] }, "sync": { "description": "Schema synchronization configuration", "anyOf": [ { "$ref": "#/definitions/SyncConfig" }, { "type": "null" } ] }, "table_format": { "type": [ "string", "null" ] }, "table_tag": { "type": [ "string", "null" ] }, "table_type": { "type": [ "string", "null" ] }, "tags": { "default": [], "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "target_alias": { "type": [ "string", "null" ] }, "target_lag": { "type": [ "string", "null" ] }, "tblproperties": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "timeout": { "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "tmp_relation_type": { "type": [ "string", "null" ] }, "transient": { "type": [ "boolean", "null" ] }, "unique_key": { "anyOf": [ { "$ref": "#/definitions/DbtUniqueKey" }, { "type": "null" } ] }, "use_anonymous_sproc": { "default": null, "type": [ "boolean", "null" ] }, "user_folder_for_python": { "type": [ "boolean", "null" ] } }, "additionalProperties": false }, "ModelConstraint": { "description": "Model level contraint", "type": "object", "required": [ "type" ], "properties": { "columns": { "default": null, "type": [ "array", "null" ], "items": { "type": "string" } }, "expression": { "type": [ "string", "null" ] }, "name": { "type": [ "string", "null" ] }, "to": { "type": [ "string", "null" ] }, "to_columns": { "description": "Only ForeignKey constraints accept: a list columns in that table containing the corresponding primary or unique key.", "default": null, "type": [ "array", "null" ], "items": { "type": "string" } }, "type": { "$ref": "#/definitions/ConstraintType" }, "warn_unenforced": { "type": [ "boolean", "null" ] }, "warn_unsupported": { "type": [ "boolean", "null" ] } }, "additionalProperties": false }, "ModelFreshness": { "type": "object", "properties": { "build_after": { "anyOf": [ { "$ref": "#/definitions/ModelFreshnessRules" }, { "type": "null" } ] } }, "additionalProperties": false }, "ModelFreshnessRules": { "type": "object", "properties": { "count": { "type": [ "integer", "null" ], "format": "int64" }, "period": { "anyOf": [ { "$ref": "#/definitions/FreshnessPeriod" }, { "type": "null" } ] }, "updates_on": { "anyOf": [ { "$ref": "#/definitions/UpdatesOn" }, { "type": "null" } ] } }, "additionalProperties": false }, "ModelProperties": { "type": "object", "required": [ "name" ], "properties": { "agg_time_dimension": { "type": [ "string", "null" ] }, "columns": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/ColumnProperties" } }, "config": { "anyOf": [ { "$ref": "#/definitions/ModelConfig" }, { "type": "null" } ] }, "constraints": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/ModelConstraint" } }, "data_tests": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/DataTests" } }, "deprecation_date": { "type": [ "string", "null" ] }, "derived_semantics": { "anyOf": [ { "$ref": "#/definitions/DerivedSemantics" }, { "type": "null" } ] }, "description": { "type": [ "string", "null" ] }, "identifier": { "type": [ "string", "null" ] }, "latest_version": { "anyOf": [ { "$ref": "#/definitions/FloatOrString" }, { "type": "null" } ] }, "metrics": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/MetricsProperties" } }, "name": { "type": "string" }, "primary_entity": { "type": [ "string", "null" ] }, "semantic_model": { "anyOf": [ { "$ref": "#/definitions/ModelPropertiesSemanticModelConfig" }, { "type": "null" } ] }, "static_analysis_off_reason": { "readOnly": true, "anyOf": [ { "$ref": "#/definitions/StaticAnalysisOffReason" }, { "type": "null" } ] }, "tests": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/DataTests" } }, "time_spine": { "anyOf": [ { "$ref": "#/definitions/ModelPropertiesTimeSpine" }, { "type": "null" } ] }, "versions": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/Versions" } } }, "additionalProperties": false }, "ModelPropertiesSemanticModelConfig": { "type": "object", "required": [ "enabled" ], "properties": { "config": { "anyOf": [ { "$ref": "#/definitions/SemanticLayerElementConfig" }, { "type": "null" } ] }, "enabled": { "type": "boolean" }, "group": { "type": [ "string", "null" ] }, "name": { "type": [ "string", "null" ] } }, "additionalProperties": false }, "ModelPropertiesTimeSpine": { "type": "object", "required": [ "standard_granularity_column" ], "properties": { "custom_granularities": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/TimeSpineCustomGranularity" } }, "standard_granularity_column": { "type": "string" } }, "additionalProperties": false }, "OnConfigurationChange": { "type": "string", "enum": [ "apply", "continue", "fail", "unknown" ] }, "OnSchemaChange": { "type": "string", "enum": [ "ignore", "append_new_columns", "fail", "sync_all_columns", "unknown" ] }, "PartitionConfig": { "description": "Configuration for partition by columns.\n\ndbt-core allows either of the variants for the `partition_by` in the model config but the bigquery-adapter throws RunTime error the behaviors are tested from the latest dbt-core + bigquery-adapter as this is written we're conformant to this behavior via here and via the `into_bigquery()` method", "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "$ref": "#/definitions/BigqueryPartitionConfig" } ] }, "PercentileType": { "type": "string", "enum": [ "discrete", "continuous" ] }, "PeriodAggregationType": { "type": "string", "enum": [ "first", "last", "average" ] }, "PersistDocsConfig": { "type": "object", "properties": { "columns": { "type": [ "boolean", "null" ] }, "relation": { "type": [ "boolean", "null" ] } }, "additionalProperties": false }, "PostgresIndex": { "type": "object", "required": [ "columns" ], "properties": { "columns": { "type": "array", "items": { "type": "string" } }, "type": { "type": [ "string", "null" ] }, "unique": { "type": [ "boolean", "null" ] } }, "additionalProperties": false }, "PrimaryKeyConfig": { "description": "A wrapper type for the `primary_key` config field that normalizes serialization.\n\nIn dbt-core, primary_key values are \"listified\" - single strings are converted to single-element arrays. This type accepts either format on input but always serializes as arrays.\n\n# Example\n\n```ignore // Input: \"id\" // Serializes as: [\"id\"]\n\n// Input: [\"id\", \"tenant_id\"] // Serializes as: [\"id\", \"tenant_id\"] ```", "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "QueryTag": { "description": "A wrapper type for the `query_tag` config field that handles flexible deserialization.\n\nAccepts strings, dictionaries, or sequences for query_tag. Maps and sequences are JSON-serialized to strings.\n\n# Example\n\n```ignore // Input: \"my-tag\" // Stores as: \"my-tag\"\n\n// Input: {\"project\": \"foo\", \"env\": \"prod\"} // Stores as: \"{\\\"project\\\":\\\"foo\\\",\\\"env\\\":\\\"prod\\\"}\" ```", "type": "string" }, "Range": { "type": "object", "required": [ "end", "interval", "start" ], "properties": { "end": { "type": "integer", "format": "int64" }, "interval": { "type": "integer", "format": "int64" }, "start": { "type": "integer", "format": "int64" } }, "additionalProperties": false }, "RangeConfig": { "type": "object", "required": [ "range" ], "properties": { "range": { "$ref": "#/definitions/Range" } }, "additionalProperties": false }, "Rows": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "object", "additionalProperties": { "$ref": "#/definitions/AnyValue" } } } ] }, "SavedQueriesProperties": { "type": "object", "required": [ "name", "query_params" ], "properties": { "config": { "anyOf": [ { "$ref": "#/definitions/SavedQueryConfig" }, { "type": "null" } ] }, "description": { "type": [ "string", "null" ] }, "exports": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/Export" } }, "label": { "type": [ "string", "null" ] }, "name": { "type": "string" }, "query_params": { "$ref": "#/definitions/SavedQueriesQueryParams" } }, "additionalProperties": false }, "SavedQueriesQueryParams": { "type": "object", "properties": { "group_by": { "type": [ "array", "null" ], "items": { "type": "string" } }, "metrics": { "type": [ "array", "null" ], "items": { "type": "string" } }, "where": { "type": [ "array", "null" ], "items": { "type": "string" } } }, "additionalProperties": false }, "SavedQueryCache": { "type": "object", "properties": { "enabled": { "type": [ "boolean", "null" ] } }, "additionalProperties": false }, "SavedQueryConfig": { "type": "object", "properties": { "cache": { "anyOf": [ { "$ref": "#/definitions/SavedQueryCache" }, { "type": "null" } ] }, "enabled": { "default": null, "type": [ "boolean", "null" ] }, "export_as": { "anyOf": [ { "$ref": "#/definitions/ExportConfigExportAs" }, { "type": "null" } ] }, "group": { "type": [ "string", "null" ] }, "meta": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "schema": { "type": [ "string", "null" ] }, "tags": { "default": [], "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] } }, "additionalProperties": false }, "Schedule": { "description": "Schedule configuration that accepts both string and structured formats. This allows users to specify schedule as either: - A string: `schedule: \"USING CRON 0,15,30,45 * * * * UTC\"` - A structured config: `schedule: { cron: \"0 * * * *\", time_zone_value: \"UTC\" }`", "anyOf": [ { "type": "string" }, { "$ref": "#/definitions/ScheduleConfig" } ] }, "ScheduleConfig": { "type": "object", "properties": { "cron": { "type": [ "string", "null" ] }, "time_zone_value": { "type": [ "string", "null" ] } }, "additionalProperties": false }, "SchemaOrigin": { "description": "Indicates where schema metadata originates from.\n\n- `Remote` (default): Schema is fetched from the remote warehouse - `Local`: Schema is derived from YAML column definitions", "oneOf": [ { "description": "Schema metadata comes from the remote warehouse (default)", "type": "string", "enum": [ "remote" ] }, { "description": "Schema metadata is derived from YAML column definitions", "type": "string", "enum": [ "local" ] } ] }, "SchemaRefreshInterval": { "description": "Schema refresh interval: '30m', '2h', '1d', or 'never'", "type": "string" }, "SeedConfig": { "description": "This configuration is a superset of all warehouse specific configurations that users can set", "type": "object", "properties": { "adapter_properties": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "alias": { "type": [ "string", "null" ] }, "as_columnstore": { "type": [ "boolean", "null" ] }, "auto_liquid_cluster": { "type": [ "boolean", "null" ] }, "auto_refresh": { "type": [ "boolean", "null" ] }, "automatic_clustering": { "type": [ "boolean", "null" ] }, "backup": { "type": [ "boolean", "null" ] }, "base_location_root": { "type": [ "string", "null" ] }, "base_location_subpath": { "type": [ "string", "null" ] }, "batch_id": { "type": [ "string", "null" ] }, "bind": { "type": [ "boolean", "null" ] }, "buckets": { "type": [ "integer", "null" ], "format": "int64" }, "catalog": { "type": [ "string", "null" ] }, "catalog_name": { "type": [ "string", "null" ] }, "category": { "anyOf": [ { "$ref": "#/definitions/DataLakeObjectCategory" }, { "type": "null" } ] }, "cluster_by": { "anyOf": [ { "$ref": "#/definitions/ClusterConfig" }, { "type": "null" } ] }, "clustered_by": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "column_types": { "type": [ "object", "null" ], "additionalProperties": { "type": "string" } }, "compression": { "type": [ "string", "null" ] }, "copy_grants": { "type": [ "boolean", "null" ] }, "database": { "type": [ "string", "null" ] }, "databricks_compute": { "type": [ "string", "null" ] }, "databricks_tags": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "dataproc_cluster_name": { "type": [ "string", "null" ] }, "delimiter": { "type": [ "string", "null" ] }, "description": { "type": [ "string", "null" ] }, "dist": { "type": [ "string", "null" ] }, "docs": { "anyOf": [ { "$ref": "#/definitions/DocsConfig" }, { "type": "null" } ] }, "enable_list_inference": { "type": [ "boolean", "null" ] }, "enable_refresh": { "type": [ "boolean", "null" ] }, "enabled": { "default": null, "type": [ "boolean", "null" ] }, "event_time": { "type": [ "string", "null" ] }, "external_volume": { "type": [ "string", "null" ] }, "file_format": { "type": [ "string", "null" ] }, "full_refresh": { "type": [ "boolean", "null" ] }, "grant_access_to": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/GrantAccessToTarget" } }, "grants": { "default": {}, "allOf": [ { "$ref": "#/definitions/GrantConfig" } ] }, "group": { "type": [ "string", "null" ] }, "hours_to_expiration": { "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "include_full_name_in_path": { "type": [ "boolean", "null" ] }, "indexes": { "default": null, "allOf": [ { "$ref": "#/definitions/IndexesConfig" } ] }, "initialize": { "type": [ "string", "null" ] }, "intermediate_format": { "type": [ "string", "null" ] }, "jar_file_uri": { "type": [ "string", "null" ] }, "job_execution_timeout_seconds": { "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "kms_key_name": { "type": [ "string", "null" ] }, "labels": { "type": [ "object", "null" ], "additionalProperties": { "type": "string" } }, "labels_from_meta": { "type": [ "boolean", "null" ] }, "liquid_clustered_by": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "location_root": { "type": [ "string", "null" ] }, "matched_condition": { "type": [ "string", "null" ] }, "materialized": { "anyOf": [ { "$ref": "#/definitions/DbtMaterialization" }, { "type": "null" } ] }, "max_staleness": { "type": [ "string", "null" ] }, "merge_with_schema_evolution": { "type": [ "boolean", "null" ] }, "meta": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "not_matched_by_source_action": { "type": [ "string", "null" ] }, "not_matched_by_source_condition": { "type": [ "string", "null" ] }, "not_matched_condition": { "type": [ "string", "null" ] }, "notebook_template_id": { "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "partition_by": { "anyOf": [ { "$ref": "#/definitions/PartitionConfig" }, { "type": "null" } ] }, "partition_expiration_days": { "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "partitions": { "type": [ "array", "null" ], "items": { "type": "string" } }, "persist_docs": { "anyOf": [ { "$ref": "#/definitions/PersistDocsConfig" }, { "type": "null" } ] }, "post_hook": { "anyOf": [ { "$ref": "#/definitions/Hooks" }, { "type": "null" } ] }, "pre_hook": { "anyOf": [ { "$ref": "#/definitions/Hooks" }, { "type": "null" } ] }, "primary_key": { "default": null, "allOf": [ { "$ref": "#/definitions/PrimaryKeyConfig" } ] }, "query_tag": { "anyOf": [ { "$ref": "#/definitions/QueryTag" }, { "type": "null" } ] }, "quote_columns": { "default": null, "type": [ "boolean", "null" ] }, "quoting": { "anyOf": [ { "$ref": "#/definitions/DbtQuoting" }, { "type": "null" } ] }, "refresh_interval_minutes": { "type": [ "number", "null" ], "format": "double" }, "refresh_mode": { "type": [ "string", "null" ] }, "require_partition_filter": { "type": [ "boolean", "null" ] }, "resource_tags": { "type": [ "object", "null" ], "additionalProperties": { "type": "string" } }, "row_access_policy": { "type": [ "string", "null" ] }, "schedule": { "anyOf": [ { "$ref": "#/definitions/Schedule" }, { "type": "null" } ] }, "schema": { "type": [ "string", "null" ] }, "secure": { "type": [ "boolean", "null" ] }, "skip_matched_step": { "type": [ "boolean", "null" ] }, "skip_not_matched_step": { "type": [ "boolean", "null" ] }, "snowflake_warehouse": { "type": [ "string", "null" ] }, "sort": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "sort_type": { "type": [ "string", "null" ] }, "source_alias": { "type": [ "string", "null" ] }, "static_analysis": { "anyOf": [ { "$ref": "#/definitions/StaticAnalysisKind" }, { "type": "null" } ] }, "table_tag": { "type": [ "string", "null" ] }, "table_type": { "type": [ "string", "null" ] }, "tags": { "default": [], "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "target_alias": { "type": [ "string", "null" ] }, "target_lag": { "type": [ "string", "null" ] }, "tblproperties": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "timeout": { "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "tmp_relation_type": { "type": [ "string", "null" ] }, "transient": { "type": [ "boolean", "null" ] } }, "additionalProperties": false }, "SeedProperties": { "type": "object", "required": [ "name" ], "properties": { "columns": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/ColumnProperties" } }, "config": { "anyOf": [ { "$ref": "#/definitions/SeedConfig" }, { "type": "null" } ] }, "data_tests": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/DataTests" } }, "description": { "type": [ "string", "null" ] }, "name": { "type": "string" }, "static_analysis_off_reason": { "readOnly": true, "anyOf": [ { "$ref": "#/definitions/StaticAnalysisOffReason" }, { "type": "null" } ] }, "tests": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/DataTests" } } }, "additionalProperties": false }, "SemanticLayerElementConfig": { "type": "object", "properties": { "meta": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } } }, "additionalProperties": false }, "Severity": { "type": "string", "enum": [ "Error", "Warn" ] }, "SnapshotConfig": { "description": "This configuration is a superset of all warehouse specific configurations that users can set", "type": "object", "properties": { "adapter_properties": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "alias": { "type": [ "string", "null" ] }, "as_columnstore": { "type": [ "boolean", "null" ] }, "auto_liquid_cluster": { "type": [ "boolean", "null" ] }, "auto_refresh": { "type": [ "boolean", "null" ] }, "automatic_clustering": { "type": [ "boolean", "null" ] }, "backup": { "type": [ "boolean", "null" ] }, "base_location_root": { "type": [ "string", "null" ] }, "base_location_subpath": { "type": [ "string", "null" ] }, "batch_id": { "type": [ "string", "null" ] }, "bind": { "type": [ "boolean", "null" ] }, "buckets": { "type": [ "integer", "null" ], "format": "int64" }, "catalog": { "type": [ "string", "null" ] }, "catalog_name": { "type": [ "string", "null" ] }, "category": { "anyOf": [ { "$ref": "#/definitions/DataLakeObjectCategory" }, { "type": "null" } ] }, "check_cols": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "cluster_by": { "anyOf": [ { "$ref": "#/definitions/ClusterConfig" }, { "type": "null" } ] }, "clustered_by": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "compression": { "type": [ "string", "null" ] }, "copy_grants": { "type": [ "boolean", "null" ] }, "database": { "type": [ "string", "null" ] }, "databricks_compute": { "type": [ "string", "null" ] }, "databricks_tags": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "dataproc_cluster_name": { "type": [ "string", "null" ] }, "dbt_valid_to_current": { "type": [ "string", "null" ] }, "description": { "type": [ "string", "null" ] }, "dist": { "type": [ "string", "null" ] }, "docs": { "anyOf": [ { "$ref": "#/definitions/DocsConfig" }, { "type": "null" } ] }, "enable_list_inference": { "type": [ "boolean", "null" ] }, "enable_refresh": { "type": [ "boolean", "null" ] }, "enabled": { "default": null, "type": [ "boolean", "null" ] }, "event_time": { "type": [ "string", "null" ] }, "external_volume": { "type": [ "string", "null" ] }, "file_format": { "type": [ "string", "null" ] }, "full_refresh": { "default": null, "type": [ "boolean", "null" ] }, "grant_access_to": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/GrantAccessToTarget" } }, "grants": { "default": {}, "allOf": [ { "$ref": "#/definitions/GrantConfig" } ] }, "group": { "type": [ "string", "null" ] }, "hard_deletes": { "anyOf": [ { "$ref": "#/definitions/HardDeletes" }, { "type": "null" } ] }, "hours_to_expiration": { "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "include_full_name_in_path": { "type": [ "boolean", "null" ] }, "indexes": { "default": null, "allOf": [ { "$ref": "#/definitions/IndexesConfig" } ] }, "initialize": { "type": [ "string", "null" ] }, "intermediate_format": { "type": [ "string", "null" ] }, "invalidate_hard_deletes": { "default": null, "type": [ "boolean", "null" ] }, "jar_file_uri": { "type": [ "string", "null" ] }, "job_execution_timeout_seconds": { "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "kms_key_name": { "type": [ "string", "null" ] }, "labels": { "type": [ "object", "null" ], "additionalProperties": { "type": "string" } }, "labels_from_meta": { "type": [ "boolean", "null" ] }, "liquid_clustered_by": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "location_root": { "type": [ "string", "null" ] }, "matched_condition": { "type": [ "string", "null" ] }, "materialized": { "anyOf": [ { "$ref": "#/definitions/DbtMaterialization" }, { "type": "null" } ] }, "max_staleness": { "type": [ "string", "null" ] }, "merge_with_schema_evolution": { "type": [ "boolean", "null" ] }, "meta": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "not_matched_by_source_action": { "type": [ "string", "null" ] }, "not_matched_by_source_condition": { "type": [ "string", "null" ] }, "not_matched_condition": { "type": [ "string", "null" ] }, "notebook_template_id": { "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "partition_by": { "anyOf": [ { "$ref": "#/definitions/PartitionConfig" }, { "type": "null" } ] }, "partition_expiration_days": { "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "partitions": { "type": [ "array", "null" ], "items": { "type": "string" } }, "persist_docs": { "anyOf": [ { "$ref": "#/definitions/PersistDocsConfig" }, { "type": "null" } ] }, "post_hook": { "anyOf": [ { "$ref": "#/definitions/Hooks" }, { "type": "null" } ] }, "pre_hook": { "anyOf": [ { "$ref": "#/definitions/Hooks" }, { "type": "null" } ] }, "primary_key": { "default": null, "allOf": [ { "$ref": "#/definitions/PrimaryKeyConfig" } ] }, "query_tag": { "anyOf": [ { "$ref": "#/definitions/QueryTag" }, { "type": "null" } ] }, "quote_columns": { "default": null, "type": [ "boolean", "null" ] }, "quoting": { "anyOf": [ { "$ref": "#/definitions/DbtQuoting" }, { "type": "null" } ] }, "refresh_interval_minutes": { "type": [ "number", "null" ], "format": "double" }, "refresh_mode": { "type": [ "string", "null" ] }, "require_partition_filter": { "type": [ "boolean", "null" ] }, "resource_tags": { "type": [ "object", "null" ], "additionalProperties": { "type": "string" } }, "row_access_policy": { "type": [ "string", "null" ] }, "schedule": { "anyOf": [ { "$ref": "#/definitions/Schedule" }, { "type": "null" } ] }, "schema": { "type": [ "string", "null" ] }, "secure": { "type": [ "boolean", "null" ] }, "skip_matched_step": { "type": [ "boolean", "null" ] }, "skip_not_matched_step": { "type": [ "boolean", "null" ] }, "snapshot_meta_column_names": { "anyOf": [ { "$ref": "#/definitions/SnapshotMetaColumnNames" }, { "type": "null" } ] }, "snowflake_warehouse": { "type": [ "string", "null" ] }, "sort": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "sort_type": { "type": [ "string", "null" ] }, "source_alias": { "type": [ "string", "null" ] }, "static_analysis": { "anyOf": [ { "$ref": "#/definitions/StaticAnalysisKind" }, { "type": "null" } ] }, "strategy": { "type": [ "string", "null" ] }, "sync": { "description": "Schema synchronization configuration", "anyOf": [ { "$ref": "#/definitions/SyncConfig" }, { "type": "null" } ] }, "table_tag": { "type": [ "string", "null" ] }, "table_type": { "type": [ "string", "null" ] }, "tags": { "default": [], "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "target_alias": { "type": [ "string", "null" ] }, "target_database": { "type": [ "string", "null" ] }, "target_lag": { "type": [ "string", "null" ] }, "target_schema": { "type": [ "string", "null" ] }, "tblproperties": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "timeout": { "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "tmp_relation_type": { "type": [ "string", "null" ] }, "transient": { "type": [ "boolean", "null" ] }, "unique_key": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "updated_at": { "type": [ "string", "null" ] } }, "additionalProperties": false }, "SnapshotMetaColumnNames": { "type": "object", "properties": { "dbt_is_deleted": { "type": [ "string", "null" ] }, "dbt_scd_id": { "type": [ "string", "null" ] }, "dbt_updated_at": { "type": [ "string", "null" ] }, "dbt_valid_from": { "type": [ "string", "null" ] }, "dbt_valid_to": { "type": [ "string", "null" ] } }, "additionalProperties": false }, "SnapshotProperties": { "type": "object", "required": [ "name" ], "properties": { "columns": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/ColumnProperties" } }, "config": { "anyOf": [ { "$ref": "#/definitions/SnapshotConfig" }, { "type": "null" } ] }, "data_tests": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/DataTests" } }, "description": { "type": [ "string", "null" ] }, "name": { "type": "string" }, "relation": { "type": [ "string", "null" ] }, "static_analysis_off_reason": { "readOnly": true, "anyOf": [ { "$ref": "#/definitions/StaticAnalysisOffReason" }, { "type": "null" } ] }, "tests": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/DataTests" } } }, "additionalProperties": false }, "SourceConfig": { "description": "This configuration is a superset of all warehouse specific configurations that users can set", "type": "object", "properties": { "adapter_properties": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "as_columnstore": { "type": [ "boolean", "null" ] }, "auto_liquid_cluster": { "type": [ "boolean", "null" ] }, "auto_refresh": { "type": [ "boolean", "null" ] }, "automatic_clustering": { "type": [ "boolean", "null" ] }, "backup": { "type": [ "boolean", "null" ] }, "base_location_root": { "type": [ "string", "null" ] }, "base_location_subpath": { "type": [ "string", "null" ] }, "batch_id": { "type": [ "string", "null" ] }, "bind": { "type": [ "boolean", "null" ] }, "buckets": { "type": [ "integer", "null" ], "format": "int64" }, "catalog": { "type": [ "string", "null" ] }, "catalog_name": { "type": [ "string", "null" ] }, "category": { "anyOf": [ { "$ref": "#/definitions/DataLakeObjectCategory" }, { "type": "null" } ] }, "cluster_by": { "anyOf": [ { "$ref": "#/definitions/ClusterConfig" }, { "type": "null" } ] }, "clustered_by": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "compression": { "type": [ "string", "null" ] }, "copy_grants": { "type": [ "boolean", "null" ] }, "databricks_compute": { "type": [ "string", "null" ] }, "databricks_tags": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "dataproc_cluster_name": { "type": [ "string", "null" ] }, "description": { "type": [ "string", "null" ] }, "dist": { "type": [ "string", "null" ] }, "enable_list_inference": { "type": [ "boolean", "null" ] }, "enable_refresh": { "type": [ "boolean", "null" ] }, "enabled": { "default": null, "type": [ "boolean", "null" ] }, "event_time": { "type": [ "string", "null" ] }, "external_volume": { "type": [ "string", "null" ] }, "file_format": { "type": [ "string", "null" ] }, "freshness": { "anyOf": [ { "$ref": "#/definitions/FreshnessDefinition" }, { "type": "null" } ] }, "grant_access_to": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/GrantAccessToTarget" } }, "hours_to_expiration": { "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "include_full_name_in_path": { "type": [ "boolean", "null" ] }, "indexes": { "default": null, "allOf": [ { "$ref": "#/definitions/IndexesConfig" } ] }, "initialize": { "type": [ "string", "null" ] }, "intermediate_format": { "type": [ "string", "null" ] }, "jar_file_uri": { "type": [ "string", "null" ] }, "job_execution_timeout_seconds": { "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "kms_key_name": { "type": [ "string", "null" ] }, "labels": { "type": [ "object", "null" ], "additionalProperties": { "type": "string" } }, "labels_from_meta": { "type": [ "boolean", "null" ] }, "liquid_clustered_by": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "loaded_at_field": { "type": [ "string", "null" ] }, "loaded_at_query": { "type": [ "string", "null" ] }, "location_root": { "type": [ "string", "null" ] }, "matched_condition": { "type": [ "string", "null" ] }, "max_staleness": { "type": [ "string", "null" ] }, "merge_with_schema_evolution": { "type": [ "boolean", "null" ] }, "meta": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "not_matched_by_source_action": { "type": [ "string", "null" ] }, "not_matched_by_source_condition": { "type": [ "string", "null" ] }, "not_matched_condition": { "type": [ "string", "null" ] }, "notebook_template_id": { "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "partition_by": { "anyOf": [ { "$ref": "#/definitions/PartitionConfig" }, { "type": "null" } ] }, "partition_expiration_days": { "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "partitions": { "type": [ "array", "null" ], "items": { "type": "string" } }, "primary_key": { "default": null, "allOf": [ { "$ref": "#/definitions/PrimaryKeyConfig" } ] }, "query_tag": { "anyOf": [ { "$ref": "#/definitions/QueryTag" }, { "type": "null" } ] }, "quoting": { "anyOf": [ { "$ref": "#/definitions/DbtQuoting" }, { "type": "null" } ] }, "refresh_interval_minutes": { "type": [ "number", "null" ], "format": "double" }, "refresh_mode": { "type": [ "string", "null" ] }, "require_partition_filter": { "type": [ "boolean", "null" ] }, "resource_tags": { "type": [ "object", "null" ], "additionalProperties": { "type": "string" } }, "row_access_policy": { "type": [ "string", "null" ] }, "schedule": { "anyOf": [ { "$ref": "#/definitions/Schedule" }, { "type": "null" } ] }, "schema_origin": { "description": "Specifies where the schema metadata originates: 'remote' (default) or 'local'", "anyOf": [ { "$ref": "#/definitions/SchemaOrigin" }, { "type": "null" } ] }, "secure": { "type": [ "boolean", "null" ] }, "skip_matched_step": { "type": [ "boolean", "null" ] }, "skip_not_matched_step": { "type": [ "boolean", "null" ] }, "snowflake_warehouse": { "type": [ "string", "null" ] }, "sort": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "sort_type": { "type": [ "string", "null" ] }, "source_alias": { "type": [ "string", "null" ] }, "static_analysis": { "anyOf": [ { "$ref": "#/definitions/StaticAnalysisKind" }, { "type": "null" } ] }, "sync": { "description": "Schema synchronization configuration", "anyOf": [ { "$ref": "#/definitions/SyncConfig" }, { "type": "null" } ] }, "table_tag": { "type": [ "string", "null" ] }, "table_type": { "type": [ "string", "null" ] }, "tags": { "default": [], "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "target_alias": { "type": [ "string", "null" ] }, "target_lag": { "type": [ "string", "null" ] }, "tblproperties": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "timeout": { "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "tmp_relation_type": { "type": [ "string", "null" ] }, "transient": { "type": [ "boolean", "null" ] } }, "additionalProperties": false }, "SourceProperties": { "type": "object", "required": [ "name" ], "properties": { "catalog": { "type": [ "string", "null" ] }, "config": { "anyOf": [ { "$ref": "#/definitions/SourceConfig" }, { "type": "null" } ] }, "database": { "type": [ "string", "null" ] }, "description": { "type": [ "string", "null" ] }, "loader": { "type": [ "string", "null" ] }, "name": { "type": "string" }, "quoting": { "anyOf": [ { "$ref": "#/definitions/DbtQuoting" }, { "type": "null" } ] }, "schema": { "type": [ "string", "null" ] }, "tables": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/Tables" } } }, "additionalProperties": false }, "StaticAnalysisKind": { "type": "string", "enum": [ "unsafe", "off", "strict", "baseline", "on" ] }, "StaticAnalysisOffReason": { "type": "string", "enum": [ "configuredoff", "unabletofetchschema", "nodownstream" ] }, "StoreFailuresAs": { "type": "string", "enum": [ "ephemeral", "table", "view" ] }, "StringOrArrayOfStrings": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } } ] }, "StringOrMetricPropertiesMetricInput": { "anyOf": [ { "type": "string" }, { "$ref": "#/definitions/MetricPropertiesMetricInput" } ] }, "SyncConfig": { "description": "Configuration for schema synchronization behavior.\n\nThis can be specified at source level (default for all tables) or at individual table level (overrides source-level settings).", "type": "object", "properties": { "schema_refresh_interval": { "description": "How often to refresh the schema from the warehouse. Examples: \"30m\", \"2h\", \"1d\", \"never\"", "anyOf": [ { "$ref": "#/definitions/SchemaRefreshInterval" }, { "type": "null" } ] } }, "additionalProperties": false }, "Tables": { "type": "object", "required": [ "name" ], "properties": { "columns": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/ColumnProperties" } }, "config": { "anyOf": [ { "$ref": "#/definitions/TablesConfig" }, { "type": "null" } ] }, "data_tests": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/DataTests" } }, "description": { "type": [ "string", "null" ] }, "external": { "anyOf": [ { "$ref": "#/definitions/AnyValue" }, { "type": "null" } ] }, "identifier": { "type": [ "string", "null" ] }, "loader": { "type": [ "string", "null" ] }, "name": { "type": "string" }, "quoting": { "anyOf": [ { "$ref": "#/definitions/DbtQuoting" }, { "type": "null" } ] }, "tests": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/DataTests" } } }, "additionalProperties": false }, "TablesConfig": { "type": "object", "properties": { "enabled": { "type": [ "boolean", "null" ] }, "event_time": { "type": [ "string", "null" ] }, "freshness": { "anyOf": [ { "$ref": "#/definitions/FreshnessDefinition" }, { "type": "null" } ] }, "loaded_at_field": { "type": [ "string", "null" ] }, "loaded_at_query": { "type": [ "string", "null" ] }, "meta": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "schema_origin": { "anyOf": [ { "$ref": "#/definitions/SchemaOrigin" }, { "type": "null" } ] }, "sync": { "anyOf": [ { "$ref": "#/definitions/SyncConfig" }, { "type": "null" } ] }, "tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] } }, "additionalProperties": false }, "TimeConfig": { "type": "object", "properties": { "granularity": { "default": "day", "type": "string" }, "time_ingestion_partitioning": { "description": "When this is true, the [`BigqueryPartitionConfig::field`] will be used as the `_PARTITIONTIME` pseudo column _PARTITIONTIME: https://cloud.google.com/bigquery/docs/partitioned-tables#ingestion_time https://docs.getdbt.com/reference/resource-configs/bigquery-configs#partitioning-by-an-ingestion-date-or-timestamp", "default": false, "type": "boolean" } }, "additionalProperties": false }, "TimeSpineCustomGranularity": { "type": "object", "required": [ "name" ], "properties": { "column_name": { "type": [ "string", "null" ] }, "name": { "type": "string" } }, "additionalProperties": false }, "UnitTestConfig": { "description": "This configuration is a superset of all warehouse specific configurations that users can set", "type": "object", "properties": { "adapter_properties": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "as_columnstore": { "type": [ "boolean", "null" ] }, "auto_liquid_cluster": { "type": [ "boolean", "null" ] }, "auto_refresh": { "type": [ "boolean", "null" ] }, "automatic_clustering": { "type": [ "boolean", "null" ] }, "backup": { "type": [ "boolean", "null" ] }, "base_location_root": { "type": [ "string", "null" ] }, "base_location_subpath": { "type": [ "string", "null" ] }, "batch_id": { "type": [ "string", "null" ] }, "bind": { "type": [ "boolean", "null" ] }, "buckets": { "type": [ "integer", "null" ], "format": "int64" }, "catalog": { "type": [ "string", "null" ] }, "catalog_name": { "type": [ "string", "null" ] }, "category": { "anyOf": [ { "$ref": "#/definitions/DataLakeObjectCategory" }, { "type": "null" } ] }, "cluster_by": { "anyOf": [ { "$ref": "#/definitions/ClusterConfig" }, { "type": "null" } ] }, "clustered_by": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "compression": { "type": [ "string", "null" ] }, "copy_grants": { "type": [ "boolean", "null" ] }, "databricks_compute": { "type": [ "string", "null" ] }, "databricks_tags": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "dataproc_cluster_name": { "type": [ "string", "null" ] }, "description": { "type": [ "string", "null" ] }, "dist": { "type": [ "string", "null" ] }, "enable_list_inference": { "type": [ "boolean", "null" ] }, "enable_refresh": { "type": [ "boolean", "null" ] }, "enabled": { "type": [ "boolean", "null" ] }, "external_volume": { "type": [ "string", "null" ] }, "file_format": { "type": [ "string", "null" ] }, "grant_access_to": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/GrantAccessToTarget" } }, "hours_to_expiration": { "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "include_full_name_in_path": { "type": [ "boolean", "null" ] }, "indexes": { "default": null, "allOf": [ { "$ref": "#/definitions/IndexesConfig" } ] }, "initialize": { "type": [ "string", "null" ] }, "intermediate_format": { "type": [ "string", "null" ] }, "jar_file_uri": { "type": [ "string", "null" ] }, "job_execution_timeout_seconds": { "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "kms_key_name": { "type": [ "string", "null" ] }, "labels": { "type": [ "object", "null" ], "additionalProperties": { "type": "string" } }, "labels_from_meta": { "type": [ "boolean", "null" ] }, "liquid_clustered_by": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "location_root": { "type": [ "string", "null" ] }, "matched_condition": { "type": [ "string", "null" ] }, "max_staleness": { "type": [ "string", "null" ] }, "merge_with_schema_evolution": { "type": [ "boolean", "null" ] }, "meta": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "not_matched_by_source_action": { "type": [ "string", "null" ] }, "not_matched_by_source_condition": { "type": [ "string", "null" ] }, "not_matched_condition": { "type": [ "string", "null" ] }, "notebook_template_id": { "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "partition_by": { "anyOf": [ { "$ref": "#/definitions/PartitionConfig" }, { "type": "null" } ] }, "partition_expiration_days": { "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "partitions": { "type": [ "array", "null" ], "items": { "type": "string" } }, "primary_key": { "default": null, "allOf": [ { "$ref": "#/definitions/PrimaryKeyConfig" } ] }, "query_tag": { "anyOf": [ { "$ref": "#/definitions/QueryTag" }, { "type": "null" } ] }, "refresh_interval_minutes": { "type": [ "number", "null" ], "format": "double" }, "refresh_mode": { "type": [ "string", "null" ] }, "require_partition_filter": { "type": [ "boolean", "null" ] }, "resource_tags": { "type": [ "object", "null" ], "additionalProperties": { "type": "string" } }, "row_access_policy": { "type": [ "string", "null" ] }, "schedule": { "anyOf": [ { "$ref": "#/definitions/Schedule" }, { "type": "null" } ] }, "secure": { "type": [ "boolean", "null" ] }, "skip_matched_step": { "type": [ "boolean", "null" ] }, "skip_not_matched_step": { "type": [ "boolean", "null" ] }, "snowflake_warehouse": { "type": [ "string", "null" ] }, "sort": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "sort_type": { "type": [ "string", "null" ] }, "source_alias": { "type": [ "string", "null" ] }, "static_analysis": { "anyOf": [ { "$ref": "#/definitions/StaticAnalysisKind" }, { "type": "null" } ] }, "table_tag": { "type": [ "string", "null" ] }, "table_type": { "type": [ "string", "null" ] }, "tags": { "anyOf": [ { "$ref": "#/definitions/StringOrArrayOfStrings" }, { "type": "null" } ] }, "target_alias": { "type": [ "string", "null" ] }, "target_lag": { "type": [ "string", "null" ] }, "tblproperties": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "timeout": { "type": [ "integer", "null" ], "format": "uint64", "minimum": 0.0 }, "tmp_relation_type": { "type": [ "string", "null" ] }, "transient": { "type": [ "boolean", "null" ] } }, "additionalProperties": false }, "UnitTestOverrides": { "type": "object", "properties": { "env_vars": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "macros": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } }, "vars": { "type": [ "object", "null" ], "additionalProperties": { "$ref": "#/definitions/AnyValue" } } }, "additionalProperties": false }, "UnitTestProperties": { "type": "object", "required": [ "expect", "model", "name" ], "properties": { "config": { "anyOf": [ { "$ref": "#/definitions/UnitTestConfig" }, { "type": "null" } ] }, "description": { "type": [ "string", "null" ] }, "expect": { "$ref": "#/definitions/Expect" }, "given": { "type": [ "array", "null" ], "items": { "$ref": "#/definitions/Given" } }, "model": { "type": "string" }, "name": { "type": "string" }, "overrides": { "anyOf": [ { "$ref": "#/definitions/UnitTestOverrides" }, { "type": "null" } ] }, "static_analysis_off_reason": { "readOnly": true, "anyOf": [ { "$ref": "#/definitions/StaticAnalysisOffReason" }, { "type": "null" } ] }, "versions": { "anyOf": [ { "$ref": "#/definitions/AnyValue" }, { "type": "null" } ] } }, "additionalProperties": false }, "UpdatesOn": { "type": "string", "enum": [ "any", "all" ] }, "Versions": { "type": "object", "required": [ "v" ], "properties": { "config": { "anyOf": [ { "$ref": "#/definitions/AnyValue" }, { "type": "null" } ] }, "deprecation_date": { "type": [ "string", "null" ] }, "v": { "$ref": "#/definitions/AnyValue" } }, "additionalProperties": { "type": "object", "additionalProperties": { "$ref": "#/definitions/AnyValue" } } }, "Volatility": { "description": "Function volatility enum - defines the function's eligibility for certain optimizations Matches the Python Volatility enum from dbt-core", "oneOf": [ { "type": "string", "enum": [ "stable" ] }, { "description": "Deterministic - An deterministic function will always return the same output when given the same input.", "type": "string", "enum": [ "deterministic" ] }, { "description": "NonDeterministic - A non-deterministic function may change the return value from evaluation to evaluation. Multiple invocations of a non-deterministic function may return different results when used in the same query.", "type": "string", "enum": [ "non-deterministic" ] } ] }, "WindowChoice": { "type": "string", "enum": [ "min", "max" ] } } } ================================================ FILE: core/dbt/links.py ================================================ ProfileConfigDocs = "https://docs.getdbt.com/docs/configure-your-profile" SnowflakeQuotingDocs = "https://docs.getdbt.com/v0.10/docs/configuring-quoting" IncrementalDocs = "https://docs.getdbt.com/docs/configuring-incremental-models" BigQueryNewPartitionBy = "https://docs.getdbt.com/docs/upgrading-to-0-16-0" ================================================ FILE: core/dbt/materializations/__init__.py ================================================ ================================================ FILE: core/dbt/materializations/incremental/__init__.py ================================================ ================================================ FILE: core/dbt/materializations/incremental/microbatch.py ================================================ from datetime import datetime, timedelta from typing import Any, Dict, List, Optional import pytz from dbt.artifacts.resources.types import BatchSize from dbt.artifacts.schemas.batch_results import BatchType from dbt.contracts.graph.nodes import ModelNode, NodeConfig from dbt.exceptions import DbtInternalError, DbtRuntimeError class MicrobatchBuilder: """A utility class for building microbatch definitions associated with a specific model""" def __init__( self, model: ModelNode, is_incremental: bool, event_time_start: Optional[datetime], event_time_end: Optional[datetime], default_end_time: Optional[datetime] = None, ): if model.config.incremental_strategy != "microbatch": raise DbtInternalError( f"Model '{model.name}' does not use 'microbatch' incremental_strategy." ) self.model = model if self.model.config.batch_size is None: raise DbtRuntimeError( f"Microbatch model '{self.model.name}' does not have a 'batch_size' config (one of {[batch_size.value for batch_size in BatchSize]}) specificed." ) self.is_incremental = is_incremental self.event_time_start = ( event_time_start.replace(tzinfo=pytz.UTC) if event_time_start else None ) self.event_time_end = event_time_end.replace(tzinfo=pytz.UTC) if event_time_end else None self.default_end_time = default_end_time or datetime.now(pytz.UTC) def build_end_time(self): """Defaults the end_time to the current time in UTC unless a non `None` event_time_end was provided""" end_time = self.event_time_end or self.default_end_time return MicrobatchBuilder.ceiling_timestamp(end_time, self.model.config.batch_size) def build_start_time(self, checkpoint: Optional[datetime]): """Create a start time based off the passed in checkpoint. If the checkpoint is `None`, or this is the first run of a microbatch model, then the model's configured `begin` value will be returned as a checkpoint is necessary to build a start time. This is because we build the start time relative to the checkpoint via the batchsize and offset, and we cannot offset a checkpoint if there is no checkpoint. """ assert isinstance(self.model.config, NodeConfig) batch_size = self.model.config.batch_size # Use event_time_start if it is provided. if self.event_time_start: return MicrobatchBuilder.truncate_timestamp(self.event_time_start, batch_size) # First run, use model's configured 'begin' as start. if not self.is_incremental or checkpoint is None: if not self.model.config.begin: raise DbtRuntimeError( f"Microbatch model '{self.model.name}' requires a 'begin' configuration." ) return MicrobatchBuilder.truncate_timestamp(self.model.config.begin, batch_size) lookback = self.model.config.lookback # If the checkpoint is equivalent to itself truncated then the checkpoint stradles # the batch line. In this case the last batch will end with the checkpoint, but start # should be the previous hour/day/month/year. Thus we need to increase the lookback by # 1 to get this affect properly. if checkpoint == MicrobatchBuilder.truncate_timestamp(checkpoint, batch_size): lookback += 1 return MicrobatchBuilder.offset_timestamp(checkpoint, batch_size, -1 * lookback) def build_batches(self, start: datetime, end: datetime) -> List[BatchType]: """ Given a start and end datetime, builds a list of batches where each batch is the size of the model's batch_size. """ batch_size = self.model.config.batch_size curr_batch_start: datetime = start curr_batch_end: datetime = MicrobatchBuilder.offset_timestamp( curr_batch_start, batch_size, 1 ) batches: List[BatchType] = [(curr_batch_start, curr_batch_end)] while curr_batch_end < end: curr_batch_start = curr_batch_end curr_batch_end = MicrobatchBuilder.offset_timestamp(curr_batch_start, batch_size, 1) batches.append((curr_batch_start, curr_batch_end)) # use exact end value as stop batches[-1] = (batches[-1][0], end) return batches @staticmethod def build_jinja_context_for_batch(model: ModelNode, incremental_batch: bool) -> Dict[str, Any]: """ Create context with entries that reflect microbatch model + incremental execution state Assumes self.model has been (re)-compiled with necessary batch filters applied. """ jinja_context: Dict[str, Any] = {} # Microbatch model properties jinja_context["model"] = model.to_dict() jinja_context["sql"] = model.compiled_code jinja_context["compiled_code"] = model.compiled_code # Add incremental context variables for batches running incrementally if incremental_batch: jinja_context["is_incremental"] = lambda: True jinja_context["should_full_refresh"] = lambda: False return jinja_context @staticmethod def offset_timestamp(timestamp: datetime, batch_size: BatchSize, offset: int) -> datetime: """Truncates the passed in timestamp based on the batch_size and then applies the offset by the batch_size. Note: It's important to understand that the offset applies to the truncated timestamp, not the origin timestamp. Thus being offset by a day isn't relative to the any given hour that day, but relative to the start of the day. So if the timestamp is the very end of a day, 2024-09-17 23:59:59, you have a batch size of a day, and an offset of +1, then the returned value ends up being only one second later, 2024-09-18 00:00:00. 2024-09-17 16:06:00 + Batchsize.hour -1 -> 2024-09-17 15:00:00 2024-09-17 16:06:00 + Batchsize.hour +1 -> 2024-09-17 17:00:00 2024-09-17 16:06:00 + Batchsize.day -1 -> 2024-09-16 00:00:00 2024-09-17 16:06:00 + Batchsize.day +1 -> 2024-09-18 00:00:00 2024-09-17 16:06:00 + Batchsize.month -1 -> 2024-08-01 00:00:00 2024-09-17 16:06:00 + Batchsize.month +1 -> 2024-10-01 00:00:00 2024-09-17 16:06:00 + Batchsize.year -1 -> 2023-01-01 00:00:00 2024-09-17 16:06:00 + Batchsize.year +1 -> 2025-01-01 00:00:00 """ truncated = MicrobatchBuilder.truncate_timestamp(timestamp, batch_size) offset_timestamp: datetime if batch_size == BatchSize.hour: offset_timestamp = truncated + timedelta(hours=offset) elif batch_size == BatchSize.day: offset_timestamp = truncated + timedelta(days=offset) elif batch_size == BatchSize.month: offset_timestamp = truncated for _ in range(abs(offset)): if offset < 0: offset_timestamp = offset_timestamp - timedelta(days=1) else: offset_timestamp = offset_timestamp + timedelta(days=31) offset_timestamp = MicrobatchBuilder.truncate_timestamp( offset_timestamp, batch_size ) elif batch_size == BatchSize.year: offset_timestamp = truncated.replace(year=truncated.year + offset) return offset_timestamp @staticmethod def truncate_timestamp(timestamp: datetime, batch_size: BatchSize) -> datetime: """Truncates the passed in timestamp based on the batch_size. 2024-09-17 16:06:00 + Batchsize.hour -> 2024-09-17 16:00:00 2024-09-17 16:06:00 + Batchsize.day -> 2024-09-17 00:00:00 2024-09-17 16:06:00 + Batchsize.month -> 2024-09-01 00:00:00 2024-09-17 16:06:00 + Batchsize.year -> 2024-01-01 00:00:00 """ if batch_size == BatchSize.hour: truncated = datetime( timestamp.year, timestamp.month, timestamp.day, timestamp.hour, 0, 0, 0, pytz.utc, ) elif batch_size == BatchSize.day: truncated = datetime( timestamp.year, timestamp.month, timestamp.day, 0, 0, 0, 0, pytz.utc ) elif batch_size == BatchSize.month: truncated = datetime(timestamp.year, timestamp.month, 1, 0, 0, 0, 0, pytz.utc) elif batch_size == BatchSize.year: truncated = datetime(timestamp.year, 1, 1, 0, 0, 0, 0, pytz.utc) return truncated @staticmethod def batch_id(start_time: datetime, batch_size: BatchSize) -> str: return MicrobatchBuilder.format_batch_start(start_time, batch_size).replace("-", "") @staticmethod def format_batch_start(batch_start: datetime, batch_size: BatchSize) -> str: """Format the passed in datetime based on the batch_size. 2024-09-17 16:06:00 + Batchsize.hour -> 2024-09-17T16 2024-09-17 16:06:00 + Batchsize.day -> 2024-09-17 2024-09-17 16:06:00 + Batchsize.month -> 2024-09 2024-09-17 16:06:00 + Batchsize.year -> 2024 """ if batch_size == BatchSize.year: return batch_start.strftime("%Y") elif batch_size == BatchSize.month: return batch_start.strftime("%Y-%m") elif batch_size == BatchSize.day: return batch_start.strftime("%Y-%m-%d") else: # batch_size == BatchSize.hour return batch_start.strftime("%Y-%m-%dT%H") @staticmethod def ceiling_timestamp(timestamp: datetime, batch_size: BatchSize) -> datetime: """Takes the given timestamp and moves it to the ceiling for the given batch size Note, if the timestamp is already the batch size ceiling, that is returned 2024-09-17 16:06:00 + BatchSize.hour -> 2024-09-17 17:00:00 2024-09-17 16:00:00 + BatchSize.hour -> 2024-09-17 16:00:00 2024-09-17 16:06:00 + BatchSize.day -> 2024-09-18 00:00:00 2024-09-17 00:00:00 + BatchSize.day -> 2024-09-17 00:00:00 2024-09-17 16:06:00 + BatchSize.month -> 2024-10-01 00:00:00 2024-09-01 00:00:00 + BatchSize.month -> 2024-09-01 00:00:00 2024-09-17 16:06:00 + BatchSize.year -> 2025-01-01 00:00:00 2024-01-01 00:00:00 + BatchSize.year -> 2024-01-01 00:00:00 """ ceiling = truncated = MicrobatchBuilder.truncate_timestamp(timestamp, batch_size) if truncated != timestamp: ceiling = MicrobatchBuilder.offset_timestamp(truncated, batch_size, 1) return ceiling ================================================ FILE: core/dbt/mp_context.py ================================================ from multiprocessing import get_context from multiprocessing.context import SpawnContext _MP_CONTEXT = get_context("spawn") def get_mp_context() -> SpawnContext: return _MP_CONTEXT ================================================ FILE: core/dbt/node_types.py ================================================ from typing import List # preserving import path during dbt/artifacts refactor from dbt.artifacts.resources.types import ( # noqa AccessType, ModelLanguage, NodeType, RunHookType, ) EXECUTABLE_NODE_TYPES: List["NodeType"] = [ NodeType.Model, NodeType.Test, NodeType.Snapshot, NodeType.Analysis, NodeType.Operation, NodeType.Seed, NodeType.Documentation, NodeType.RPCCall, NodeType.SqlOperation, NodeType.Function, ] REFABLE_NODE_TYPES: List["NodeType"] = [ NodeType.Model, NodeType.Seed, NodeType.Snapshot, ] TEST_NODE_TYPES: List["NodeType"] = [ NodeType.Test, NodeType.Unit, ] VERSIONED_NODE_TYPES: List["NodeType"] = [ NodeType.Model, ] ================================================ FILE: core/dbt/parser/README.md ================================================ # Parser README The parsing stage of dbt produces: * the 'manifest', a Python structure in the code * the project `target/manifest.json` file ## Top Level Parsing steps 1. Load the macro manifest (MacroManifest): * Inputs: SQL files in the 'macros' directory * Outputs: the MacroManifest encapsulated in a BaseAdapter; a pared down Manifest-type file which contains only macros and files (to be copied into the full Manifest) The macro manifest takes the place of the full-fledged Manifest and contains only macros, which are used for resolving tests during parsing. The 'adapter' code retains a MacroManifest, while the mainline parsing code will have a full manifest. The class MacroParser assists in this process. 2. Loads all internal 'projects' (i.e. has a `dbt_project.yml`) and all projects in the project path. 3. Create a ManifestLoader object from the project dependencies and the current config. 4. Read and parse the project files. Results are loaded into the ManifestLoader. ManifestLoader loads the MacroManifest object into a BaseAdapter object. 5. Write the partial parse results (the pickle file). This writes out the 'results' from the ManifestLoader, so the "create the manifest" step has not occured yet. Things yet to happen include patching the nodes, patching the macros, and processing refs, sources, and docs. 6. Sources are patched. First, source tests are parsed. Nodes, sources, macros, docs, exposures, metadata, files, and selectors are copied into the Manifest by the ManifestLoader. And finally, nodes (from `results.patches`) are "patched" and macros too (from `results.macro_patches`). 7. Process the manifest (refs, sources, docs). * Loops through all nodes, for each node find the node that matches * the sources refs, and adds the unique id of the source to each * node's 'depends_on' 8. Check the Manifest and check for resource uniqueness. 9. Build the flat graph 10. Write out the target/manifest.json file. ## Parse the project files There are several parser-type objects. Each "parser" gets a list of of matching files specified by directory and ('dbt_project.yml', '*.sql', '*.yml', *.csv, or *.md) ### ModelParser code: core/dbt/parser/models.py. Most of the code is in SimpleSQLParser. paths: source\_paths + `*.sql` Manifest: nodes ### SnapshotParser code: core/dbt/parser/snapshots.py paths: snapshot\_paths + `*.sql` ### AnalysisParser code: core/dbt/parser/analysis.py paths: analysis\_paths + `*.sql` Manifest: nodes ### DataTestParser code: core/dbt/parser/data\_test.py paths: test\_paths + `*.sql` Manifest: nodes ### HookParser code: core/dbt/parser/hooks.py paths: 'dbt\_project.yml' ### SeedParser code: core/dbt/parser/seeds.py paths: data\_paths + `*.csv` ### DocumentationParser code: core/dbt/parser/docs.py paths: docs\_paths, `*.md` Manifest: docs ### SchemaParser * Input: yaml files * Output: various nodes in the Manifest and 'patch' files in the ParseResult/Manifest code: core/dbt/parser/schemas.py paths: all\_source\_paths = source\_paths, data\_paths, snapshot\_paths, analysis\_paths, macro\_paths, `*.yml` This "parses" the `.yml` (property) files in the dbt project. It loops through each yaml file and pulls out the tests in the various config sections and jinja renders them. A different sub-parser is called for each main dictionary key in the yaml. * 'models' - TestablePatchParser * plus 'patches' at create manifest time * Manifest: nodes * 'seeds' - TestablePatchParser * 'snapshots' - TestablePatchParser * 'sources' - SourceParser * plus 'patch\_sources' at create manifest times * Manifest: sources * 'macros' - MacroPatchParser * plus 'macro\_patches' at create manifest time * Manifest: macros * 'analyses' - AnalysisPatchParser * 'exposures' - SchemaParser.parse\_exposures * no 'patches' * Manifest: exposures * 'groups' - SchemaParser.parse\_groups * no 'patches' * Manifest: groups # dbt Manifest ### nodes These have executable SQL attached. Models - Are generated from SQL files in the 'models' directory - have a unique_id starting with 'model.' - Final object is a ModelNode Singular Tests - Are generated from SQL files in 'tests' directory - have a unique_id starting with 'test.' - Final object is a SingularTestNode Generic Tests - Are generated from 'tests' in schema yaml files, which ultimately derive from tests in the 'macros' directory - Have a unique_id starting with 'test.' - Final object is a GenericTestNode - fqn is .schema_test. Hooks - come from 'on-run-start' and 'on-run-end' config attributes. - have a unique_id starting with 'operation.' - FQN is of the form: ["dbt_labs_internal_analytics","hooks","dbt_labs_internal_analytics-on-run-end-0"] Analysis - comes from SQL files in 'analysis' directory - Final object is a AnalysisNode RPC Node - This is a "node" representing the bit of Jinja-SQL that gets passed into the run_sql or compile_sql methods. When you're using the Cloud IDE, and you're working in a scratch tab, and you just want to compile/run what you have there: it needs to be parsed and executed, but it's not actually a model/node in the project, so it's this special thing. This is a temporary addition to the running manifest. - Object is a RPCNode ### sources - comes from 'sources' sections in yaml files - Final object is a SourceDefinition node - have a unique_id starting with 'source.' ### macros - comes from SQL files in 'macros' directory - Final object is a Macro node - have a unique_id starting with 'macro.' - Test macros are used in schema tests ### docs - comes from .md files in 'docs' directory - Final object is a Documentation ### exposures - comes from 'exposures' sections in yaml files - Final object is a Exposure node ## Temporary patch files The information in these structures is stored here by the schema parser, but should be resolved before the final manifest is written and do not show up in the written manifest. Ideally we'd like to skip this step and apply the changes directly to the nodes, macros, and sources instead. With the current staged parser we have to save this with the manifest information. ### patches ### macro_patches ### source_patches ## Other ### selectors Selectors are set in config yaml files and can be used to determine which nodes should be 'compiled' and run. Selectors can also be done on the command line and will be in cli args. ### disabled Models, sources, or other nodes that the user has disabled by setting enabled: false. They should be completely ignored by dbt, as if they don't exist. In its simplest/silliest form, it's a way to keep code around without deleting it. Some folks do cleverer things, like dynamically enabling/disabling certain models based on the database adapter type or the value of --vars. ### files This contains a list of all of the files that were processed with links to the nodes, docs, macros, sources, exposures, patches, macro_patches, source_patches, i.e. all of the other places that data is stored in the manifest. It also has a checksum of the contents. The 'files' structure is in the saved manifest, but not in the manifest.json file that is written out. It is used in partial parsing to determine whether to use previously generated nodes. ### metadata From the ManifestMetadata class. Contains dbt_schema_version, project_id, user_id, send_anonymous_usage_stats, adapter_type ### flat_graph Used during execution in context.common (?). Builds dictionaries of nodes and sources. Not sure why this is used instead of the original nodes and sources. Not in the written manifest. ### state_check This used to be in ParseResults (not committed yet). The saved version of this is compared against the current version to see if we can use the saved Manifest. Contains var_hash, profile_hash, and project_hashes, to compare to the saved Manifest to see if things have changed that would invalidate it. ## Written Manifest only ### child_map ### parent_map ================================================ FILE: core/dbt/parser/__init__.py ================================================ from . import ( # noqa analysis, base, docs, generic_test, hooks, macros, models, schemas, singular_test, snapshots, ) from .analysis import AnalysisParser # noqa from .base import ConfiguredParser, Parser # noqa from .docs import DocumentationParser # noqa from .generic_test import GenericTestParser # noqa from .hooks import HookParser # noqa from .macros import MacroParser # noqa from .models import ModelParser # noqa from .schemas import SchemaParser # noqa from .seeds import SeedParser # noqa from .singular_test import SingularTestParser # noqa from .snapshots import SnapshotParser # noqa ================================================ FILE: core/dbt/parser/analysis.py ================================================ import os from dbt.contracts.graph.nodes import AnalysisNode from dbt.node_types import NodeType from dbt.parser.base import SimpleSQLParser from dbt.parser.search import FileBlock class AnalysisParser(SimpleSQLParser[AnalysisNode]): def parse_from_dict(self, dct, validate=True) -> AnalysisNode: if validate: AnalysisNode.validate(dct) return AnalysisNode.from_dict(dct) @property def resource_type(self) -> NodeType: return NodeType.Analysis @classmethod def get_compiled_path(cls, block: FileBlock): return os.path.join("analysis", block.path.relative_path) ================================================ FILE: core/dbt/parser/base.py ================================================ import abc import itertools import os from typing import Any, Dict, Generic, List, Optional, TypeVar from dbt import deprecations, hooks, utils from dbt.adapters.factory import get_adapter # noqa: F401 from dbt.artifacts.resources import Contract, Export from dbt.clients.jinja import MacroGenerator, get_rendered from dbt.config import RuntimeConfig from dbt.context.context_config import ContextConfig from dbt.context.providers import ( generate_generate_name_macro_context, generate_parser_model_context, ) from dbt.contracts.files import SchemaSourceFile from dbt.contracts.graph.manifest import Manifest from dbt.contracts.graph.nodes import BaseNode, ManifestNode from dbt.contracts.graph.unparsed import Docs, UnparsedNode from dbt.exceptions import ( ConfigUpdateError, DbtInternalError, DictParseError, InvalidAccessTypeError, ParsingError, ) from dbt.flags import get_flags from dbt.jsonschemas.jsonschemas import validate_model_config from dbt.node_types import AccessType, ModelLanguage, NodeType from dbt.parser.common import resource_types_to_schema_file_keys from dbt.parser.search import FileBlock from dbt_common.clients._jinja_blocks import ExtractWarning from dbt_common.dataclass_schema import ValidationError from dbt_common.utils import deep_merge # internally, the parser may store a less-restrictive type that will be # transformed into the final type. But it will have to be derived from # ParsedNode to be operable. FinalValue = TypeVar("FinalValue", bound=BaseNode) IntermediateValue = TypeVar("IntermediateValue", bound=BaseNode) FinalNode = TypeVar("FinalNode", bound=ManifestNode) ConfiguredBlockType = TypeVar("ConfiguredBlockType", bound=FileBlock) class BaseParser(Generic[FinalValue]): def __init__(self, project: RuntimeConfig, manifest: Manifest) -> None: self.project: RuntimeConfig = project self.manifest: Manifest = manifest @abc.abstractmethod def parse_file(self, block: FileBlock) -> None: pass @abc.abstractproperty def resource_type(self) -> NodeType: pass def generate_unique_id(self, resource_name: str, hash: Optional[str] = None) -> str: """Returns a unique identifier for a resource An optional hash may be passed in to ensure uniqueness for edge cases""" return ".".join( filter(None, [self.resource_type, self.project.project_name, resource_name, hash]) ) def _handle_extract_warning(self, warning: ExtractWarning, file: str) -> None: deprecations.warn("unexpected-jinja-block-deprecation", msg=warning.msg, file=file) class Parser(BaseParser[FinalValue], Generic[FinalValue]): def __init__( self, project: RuntimeConfig, manifest: Manifest, root_project: RuntimeConfig, ) -> None: super().__init__(project, manifest) self.root_project = root_project class RelationUpdate: # "component" is database, schema or alias def __init__(self, config: RuntimeConfig, manifest: Manifest, component: str) -> None: default_macro = manifest.find_generate_macro_by_name( component=component, root_project_name=config.project_name, ) if default_macro is None: raise DbtInternalError(f"No macro with name generate_{component}_name found") default_macro_context = generate_generate_name_macro_context( default_macro, config, manifest ) self.default_updater = MacroGenerator(default_macro, default_macro_context) package_names = config.dependencies.keys() if config.dependencies else {} package_updaters = {} for package_name in package_names: package_macro = manifest.find_generate_macro_by_name( component=component, root_project_name=config.project_name, imported_package=package_name, ) if package_macro: imported_macro_context = generate_generate_name_macro_context( package_macro, config, manifest ) package_updaters[package_macro.package_name] = MacroGenerator( package_macro, imported_macro_context ) self.package_updaters = package_updaters self.component = component def __call__(self, parsed_node: Any, override: Optional[str]) -> None: if getattr(parsed_node, "package_name", None) in self.package_updaters: new_value = self.package_updaters[parsed_node.package_name](override, parsed_node) else: new_value = self.default_updater(override, parsed_node) if isinstance(new_value, str): new_value = new_value.strip() setattr(parsed_node, self.component, new_value) class ConfiguredParser( Parser[FinalNode], Generic[ConfiguredBlockType, FinalNode], ): def __init__( self, project: RuntimeConfig, manifest: Manifest, root_project: RuntimeConfig, ) -> None: super().__init__(project, manifest, root_project) # this sets callables from RelationUpdate self._update_node_database = RelationUpdate( manifest=manifest, config=root_project, component="database" ) self._update_node_schema = RelationUpdate( manifest=manifest, config=root_project, component="schema" ) self._update_node_alias = RelationUpdate( manifest=manifest, config=root_project, component="alias" ) @classmethod @abc.abstractmethod def get_compiled_path(cls, block: ConfiguredBlockType) -> str: pass @abc.abstractmethod def parse_from_dict(self, dict, validate=True) -> FinalNode: pass @abc.abstractproperty def resource_type(self) -> NodeType: pass @property def default_schema(self): return self.root_project.credentials.schema @property def default_database(self): return self.root_project.credentials.database def get_fqn_prefix(self, path: str) -> List[str]: no_ext = os.path.splitext(path)[0] fqn = [self.project.project_name] fqn.extend(utils.split_path(no_ext)[:-1]) return fqn def get_fqn(self, path: str, name: str) -> List[str]: """Get the FQN for the node. This impacts node selection and config application. """ fqn = self.get_fqn_prefix(path) fqn.append(name) return fqn def _mangle_hooks(self, config): """Given a config dict that may have `pre-hook`/`post-hook` keys, convert it from the yucky maybe-a-string, maybe-a-dict to a dict. """ # Like most of parsing, this is a horrible hack :( for key in hooks.ModelHookType: if key in config: config[key] = [hooks.get_hook_dict(h) for h in config[key]] def _create_error_node( self, name: str, path: str, original_file_path: str, raw_code: str, language: str = "sql" ) -> UnparsedNode: """If we hit an error before we've actually parsed a node, provide some level of useful information by attaching this to the exception. """ # this is a bit silly, but build an UnparsedNode just for error # message reasons return UnparsedNode( name=name, resource_type=self.resource_type, path=path, original_file_path=original_file_path, package_name=self.project.project_name, raw_code=raw_code, language=language, ) def _create_parsetime_node( self, block: ConfiguredBlockType, path: str, config: ContextConfig, fqn: List[str], name=None, **kwargs, ) -> FinalNode: """Create the node that will be passed in to the parser context for "rendering". Some information may be partial, as it'll be updated by config() and any ref()/source() calls discovered during rendering. """ if name is None: name = block.name if block.path.relative_path.endswith(".py"): language = ModelLanguage.python else: # this is not ideal but we have a lot of tests to adjust if don't do it language = ModelLanguage.sql dct = { "alias": name, "schema": self.default_schema, "database": self.default_database, "fqn": fqn, "name": name, "resource_type": self.resource_type, "path": path, "original_file_path": block.path.original_file_path, "package_name": self.project.project_name, "raw_code": block.contents or "", "language": language, "unique_id": self.generate_unique_id(name), "config": self.config_dict(config), "checksum": block.file.checksum.to_dict(omit_none=True), } dct.update(kwargs) # TODO: we're doing this becaus return type is _required_ for the FunctionNode # but we don't get the return type until we patch the node with the yml definition # so we need to set it to a default value here. if self.resource_type == NodeType.Function: dct["returns"] = {"data_type": "INVALID_TYPE"} try: return self.parse_from_dict(dct, validate=True) except ValidationError as exc: # this is a bit silly, but build an UnparsedNode just for error # message reasons node = self._create_error_node( name=block.name, path=path, original_file_path=block.path.original_file_path, raw_code=block.contents, ) raise DictParseError(exc, node=node) def _context_for(self, parsed_node: FinalNode, config: ContextConfig) -> Dict[str, Any]: return generate_parser_model_context(parsed_node, self.root_project, self.manifest, config) def render_with_context(self, parsed_node: FinalNode, config: ContextConfig): # Given the parsed node and a ContextConfig to use during parsing, # render the node's sql with macro capture enabled. # Note: this mutates the config object when config calls are rendered. context = self._context_for(parsed_node, config) # this goes through the process of rendering, but just throws away # the rendered result. The "macro capture" is the point? get_rendered(parsed_node.raw_code, context, parsed_node, capture_macros=True) return context # This is taking the original config for the node, converting it to a dict, # updating the config with new config passed in, then re-creating the # config from the dict in the node. def update_parsed_node_config_dict( self, parsed_node: FinalNode, config_dict: Dict[str, Any] ) -> None: # Overwrite node config final_config_dict = parsed_node.config.to_dict(omit_none=True) final_config_dict.update({k.strip(): v for (k, v) in config_dict.items()}) # re-mangle hooks, in case we got new ones self._mangle_hooks(final_config_dict) parsed_node.config = parsed_node.config.from_dict(final_config_dict) def update_parsed_node_relation_names( self, parsed_node: FinalNode, config_dict: Dict[str, Any] ) -> None: # These call the RelationUpdate callable to go through generate_name macros self._update_node_database(parsed_node, config_dict.get("database")) self._update_node_schema(parsed_node, config_dict.get("schema")) if not isinstance(parsed_node, Export) and parsed_node.schema is None: if not get_flags().require_valid_schema_from_generate_schema_name: deprecations.warn( "generate-schema-name-null-value-deprecation", resource_unique_id=parsed_node.unique_id, ) else: raise ParsingError( f"Node '{parsed_node.unique_id}' has a schema set to None as a result of a generate_schema_name call.\nPlease set a valid schema name, or preserve the legacy behavior by setting the behavior flag 'require_valid_schema_from_generate_schema_name' to True." ) self._update_node_alias(parsed_node, config_dict.get("alias")) # Snapshot nodes use special "target_database" and "target_schema" fields # for backward compatibility # We have to do getattr here because saved_query parser calls this method with # Export object instead of a node. if getattr(parsed_node, "resource_type", None) == NodeType.Snapshot: if "target_database" in config_dict and config_dict["target_database"]: parsed_node.database = config_dict["target_database"] if "target_schema" in config_dict and config_dict["target_schema"]: parsed_node.schema = config_dict["target_schema"] self._update_node_relation_name(parsed_node) def update_parsed_node_config( self, parsed_node: FinalNode, config: ContextConfig, context=None, patch_config_dict=None, patch_file_id=None, validate_config_call_dict: bool = False, ) -> None: """Given the ContextConfig used for parsing and the parsed node, generate and set the true values to use, overriding the temporary parse values set in _build_intermediate_parsed_node. """ # build_config_dict takes the config_call_dict in the ContextConfig object # and calls calculate_node_config to combine dbt_project configs and # config calls from SQL files, plus patch configs (from schema files) # This normalize the config for a model node due #8520; should be improved latter if not patch_config_dict: patch_config_dict = {} if ( parsed_node.resource_type == NodeType.Model and parsed_node.language == ModelLanguage.python ): if "materialized" not in patch_config_dict: patch_config_dict["materialized"] = "table" config_dict = config.build_config_dict(patch_config_dict=patch_config_dict) # Set tags on node provided in config blocks. Tags are additive, so even if # config has been built before, we don't have to reset tags in the parsed_node. model_tags = config_dict.get("tags", []) for tag in model_tags: if tag not in parsed_node.tags: parsed_node.tags.append(tag) # If we have meta in the config, copy to node level, for backwards # compatibility with earlier node-only config. if "meta" in config_dict and config_dict["meta"]: parsed_node.meta = config_dict["meta"] # If we have group in the config, copy to node level if "group" in config_dict and config_dict["group"]: parsed_node.group = config_dict["group"] # If we have access in the config, copy to node level if parsed_node.resource_type == NodeType.Model and config_dict.get("access", None): if AccessType.is_valid(config_dict["access"]): assert hasattr(parsed_node, "access") parsed_node.access = AccessType(config_dict["access"]) else: raise InvalidAccessTypeError( unique_id=parsed_node.unique_id, field_value=config_dict["access"] ) # If we have docs in the config, merge with the node level, for backwards # compatibility with earlier node-only config. if "docs" in config_dict and config_dict["docs"]: # we set show at the value of the config if it is set, otherwise, inherit the value docs_show = ( config_dict["docs"]["show"] if "show" in config_dict["docs"] else parsed_node.docs.show ) if "node_color" in config_dict["docs"]: parsed_node.docs = Docs( show=docs_show, node_color=config_dict["docs"]["node_color"] ) else: parsed_node.docs = Docs(show=docs_show) # If we have contract in the config, copy to node level if "contract" in config_dict and config_dict["contract"]: contract_dct = config_dict["contract"] Contract.validate(contract_dct) # Seed node has contract config (from NodeConfig) but no contract in SeedNode if hasattr(parsed_node, "contract"): parsed_node.contract = Contract.from_dict(contract_dct) if get_flags().state_modified_compare_more_unrendered_values: # Use the patch_file.unrendered_configs if available to update patch_dict_config, # as provided patch_config_dict may actually already be rendered and thus sensitive to jinja evaluations if patch_file_id: patch_file = self.manifest.files.get(patch_file_id, None) if patch_file and isinstance(patch_file, SchemaSourceFile): schema_key = resource_types_to_schema_file_keys.get(parsed_node.resource_type) if schema_key: if unrendered_patch_config := patch_file.get_unrendered_config( schema_key, parsed_node.name, getattr(parsed_node, "version", None) ): patch_config_dict = deep_merge( patch_config_dict, unrendered_patch_config ) # unrendered_config is used to compare the original database/schema/alias # values and to handle 'same_config' and 'same_contents' calls parsed_node.unrendered_config = config.build_config_dict( rendered=False, patch_config_dict=patch_config_dict ) # We validate the _config_call_dict here because there is more than # one way that the _config_call_dict can be set and also, later it gets # read multiple times. Doing the validation here ensures that the config # is only validated once. if parsed_node.resource_type == NodeType.Model and validate_config_call_dict: python_model = parsed_node.language == ModelLanguage.python validate_model_config( config._config_call_dict, parsed_node.original_file_path, is_python_model=python_model, ) parsed_node.config_call_dict = config._config_call_dict parsed_node.unrendered_config_call_dict = config._unrendered_config_call_dict # do this once before we parse the node database/schema/alias, so # parsed_node.config is what it would be if they did nothing self.update_parsed_node_config_dict(parsed_node, config_dict) # This updates the node database/schema/alias/relation_name self.update_parsed_node_relation_names(parsed_node, config_dict) # tests don't have hooks if parsed_node.resource_type == NodeType.Test: return # at this point, we've collected our hooks. Use the node context to # render each hook and collect refs/sources assert hasattr(parsed_node.config, "pre_hook") and hasattr(parsed_node.config, "post_hook") hooks = list(itertools.chain(parsed_node.config.pre_hook, parsed_node.config.post_hook)) # skip context rebuilding if there aren't any hooks if not hooks: return if not context: context = self._context_for(parsed_node, config) for hook in hooks: get_rendered(hook.sql, context, parsed_node, capture_macros=True) def initial_config(self, fqn: List[str]) -> ContextConfig: config_version = min([self.project.config_version, self.root_project.config_version]) if config_version == 2: return ContextConfig( self.root_project, fqn, self.resource_type, self.project.project_name, ) else: raise DbtInternalError( f"Got an unexpected project version={config_version}, expected 2" ) def config_dict( self, config: ContextConfig, ) -> Dict[str, Any]: config_dict = config.build_config_dict(base=True) self._mangle_hooks(config_dict) return config_dict def render_update( self, node: FinalNode, config: ContextConfig, validate_config_call_dict: bool = False ) -> None: try: context = self.render_with_context(node, config) self.update_parsed_node_config( node, config, context=context, validate_config_call_dict=validate_config_call_dict ) except ValidationError as exc: # we got a ValidationError - probably bad types in config() raise ConfigUpdateError(exc, node=node) from exc def add_result_node(self, block: FileBlock, node: ManifestNode): if node.config.enabled: self.manifest.add_node(block.file, node) else: self.manifest.add_disabled(block.file, node) def parse_node(self, block: ConfiguredBlockType) -> FinalNode: compiled_path: str = self.get_compiled_path(block) fqn = self.get_fqn(compiled_path, block.name) config: ContextConfig = self.initial_config(fqn) node = self._create_parsetime_node( block=block, path=compiled_path, config=config, fqn=fqn, ) self.render_update(node, config) self.add_result_node(block, node) return node def _update_node_relation_name(self, node: ManifestNode): # Seed and Snapshot nodes and Models that are not ephemeral, # and TestNodes that store_failures. # TestNodes do not get a relation_name without store failures # because no schema is created. if getattr(node, "is_relational", None) and not getattr(node, "is_ephemeral_model", None): adapter = get_adapter(self.root_project) relation_cls = adapter.Relation node.relation_name = str(relation_cls.create_from(self.root_project, node)) else: # Set it to None in case it changed with a config update node.relation_name = None @abc.abstractmethod def parse_file(self, file_block: FileBlock) -> None: pass class SimpleParser( ConfiguredParser[ConfiguredBlockType, FinalNode], Generic[ConfiguredBlockType, FinalNode], ): pass class SQLParser(ConfiguredParser[FileBlock, FinalNode], Generic[FinalNode]): def parse_file(self, file_block: FileBlock) -> None: self.parse_node(file_block) class SimpleSQLParser(SQLParser[FinalNode]): pass ================================================ FILE: core/dbt/parser/common.py ================================================ from dataclasses import dataclass from typing import Any, Dict, Generic, List, Optional, TypeVar, Union from dbt.artifacts.resources import ( ColumnConfig, ColumnDimension, ColumnEntity, ColumnInfo, NodeVersion, ) from dbt.contracts.graph.nodes import UnpatchedSourceDefinition from dbt.contracts.graph.unparsed import ( HasColumnDocs, HasColumnProps, HasColumnTests, UnparsedAnalysisUpdate, UnparsedColumn, UnparsedColumnEntityV2, UnparsedDimensionV2, UnparsedExposure, UnparsedFunctionUpdate, UnparsedMacroUpdate, UnparsedModelUpdate, UnparsedNodeUpdate, UnparsedSingularTestUpdate, ) from dbt.exceptions import ParsingError from dbt.node_types import NodeType from dbt.parser.search import FileBlock from dbt_common.contracts.constraints import ColumnLevelConstraint, ConstraintType from dbt_common.exceptions import DbtInternalError from dbt_semantic_interfaces.type_enums import ( DimensionType, EntityType, TimeGranularity, ) schema_file_keys_to_resource_types = { "models": NodeType.Model, "seeds": NodeType.Seed, "snapshots": NodeType.Snapshot, "sources": NodeType.Source, "macros": NodeType.Macro, "analyses": NodeType.Analysis, "exposures": NodeType.Exposure, "metrics": NodeType.Metric, "semantic_models": NodeType.SemanticModel, "saved_queries": NodeType.SavedQuery, "functions": NodeType.Function, } resource_types_to_schema_file_keys = { v: k for (k, v) in schema_file_keys_to_resource_types.items() } schema_file_keys = list(schema_file_keys_to_resource_types.keys()) def trimmed(inp: str) -> str: if len(inp) < 50: return inp return inp[:44] + "..." + inp[-3:] TestDef = Union[str, Dict[str, Any]] Target = TypeVar( "Target", UnparsedNodeUpdate, UnparsedMacroUpdate, UnparsedAnalysisUpdate, UnpatchedSourceDefinition, UnparsedExposure, UnparsedModelUpdate, UnparsedFunctionUpdate, UnparsedSingularTestUpdate, ) ColumnTarget = TypeVar( "ColumnTarget", UnparsedModelUpdate, UnparsedNodeUpdate, UnparsedAnalysisUpdate, UnpatchedSourceDefinition, ) Versioned = TypeVar("Versioned", bound=UnparsedModelUpdate) Testable = TypeVar("Testable", UnparsedNodeUpdate, UnpatchedSourceDefinition, UnparsedModelUpdate) @dataclass class YamlBlock(FileBlock): data: Dict[str, Any] @classmethod def from_file_block(cls, src: FileBlock, data: Dict[str, Any]): return cls( file=src.file, data=data, ) @dataclass class TargetBlock(YamlBlock, Generic[Target]): target: Target @property def name(self): return self.target.name @property def columns(self): return [] @property def data_tests(self) -> List[TestDef]: return [] @property def tests(self) -> List[TestDef]: return [] @classmethod def from_yaml_block(cls, src: YamlBlock, target: Target) -> "TargetBlock[Target]": return cls( file=src.file, data=src.data, target=target, ) @dataclass class TargetColumnsBlock(TargetBlock[ColumnTarget], Generic[ColumnTarget]): @property def columns(self): if self.target.columns is None: return [] else: return self.target.columns @dataclass class TestBlock(TargetColumnsBlock[Testable], Generic[Testable]): @property def data_tests(self) -> List[TestDef]: if self.target.data_tests is None: return [] else: return self.target.data_tests @property def quote_columns(self) -> Optional[bool]: return self.target.quote_columns @classmethod def from_yaml_block(cls, src: YamlBlock, target: Testable) -> "TestBlock[Testable]": return cls( file=src.file, data=src.data, target=target, ) @dataclass class VersionedTestBlock(TestBlock, Generic[Versioned]): @property def columns(self): if not self.target.versions: return super().columns else: raise DbtInternalError(".columns for VersionedTestBlock with versions") @property def data_tests(self) -> List[TestDef]: if not self.target.versions: return super().data_tests else: raise DbtInternalError(".data_tests for VersionedTestBlock with versions") @classmethod def from_yaml_block(cls, src: YamlBlock, target: Versioned) -> "VersionedTestBlock[Versioned]": return cls( file=src.file, data=src.data, target=target, ) @dataclass class GenericTestBlock(TestBlock[Testable], Generic[Testable]): data_test: Dict[str, Any] column_name: Optional[str] tags: List[str] version: Optional[NodeVersion] @classmethod def from_test_block( cls, src: TestBlock, data_test: Dict[str, Any], column_name: Optional[str], tags: List[str], version: Optional[NodeVersion], ) -> "GenericTestBlock": return cls( file=src.file, data=src.data, target=src.target, data_test=data_test, column_name=column_name, tags=tags, version=version, ) class ParserRef: """A helper object to hold parse-time references.""" def __init__(self) -> None: self.column_info: Dict[str, ColumnInfo] = {} def _add(self, column: HasColumnProps) -> None: tags: List[str] = getattr(column, "tags", []) quote: Optional[bool] = None granularity: Optional[TimeGranularity] = None dimension: Optional[Union[DimensionType, ColumnDimension]] = None entity: Optional[Union[EntityType, ColumnEntity]] = None if isinstance(column, UnparsedColumn): quote = column.quote granularity = TimeGranularity(column.granularity) if column.granularity else None if isinstance(column.dimension, UnparsedDimensionV2): dimension = ColumnDimension( name=column.dimension.name or column.name, type=(DimensionType(column.dimension.type)), description=column.dimension.description or column.description, label=column.dimension.label, is_partition=column.dimension.is_partition, config=column.dimension.config or {}, validity_params=( ColumnDimension.ColumnDimensionValidityParams( is_start=column.dimension.validity_params.is_start, is_end=column.dimension.validity_params.is_end, ) if column.dimension.validity_params else None ), ) elif isinstance(column.dimension, str): dimension = DimensionType(column.dimension) if isinstance(column.entity, UnparsedColumnEntityV2): entity = ColumnEntity( name=column.entity.name, type=EntityType(column.entity.type), description=column.entity.description or column.description, label=column.entity.label, config=column.entity.config or {}, ) elif isinstance(column.entity, str): entity = EntityType(column.entity) if any( c for c in column.constraints if "type" not in c or not ConstraintType.is_valid(c["type"]) ): raise ParsingError(f"Invalid constraint type on column {column.name}") # Merge meta and tags from column and config column_config_meta = ( column.config["meta"] if isinstance(column.config.get("meta"), dict) else {} ) column_config_tags = [] if "tags" in column.config: if isinstance(column.config["tags"], list): column_config_tags = column.config["tags"] elif isinstance(column.config["tags"], str): column_config_tags = [column.config["tags"]] column_meta = {**column.meta, **column_config_meta} column_tags = list(set(tags + column_config_tags)) self.column_info[column.name] = ColumnInfo( name=column.name, description=column.description, data_type=column.data_type, constraints=[ColumnLevelConstraint.from_dict(c) for c in column.constraints], meta=column_meta, tags=column_tags, quote=quote, _extra=column.extra, granularity=granularity, dimension=dimension, entity=entity, config=ColumnConfig(meta=column_meta, tags=column_tags), ) @classmethod def from_target(cls, target: Union[HasColumnDocs, HasColumnTests]) -> "ParserRef": refs = cls() for column in target.columns: refs._add(column) return refs @classmethod def from_versioned_target(cls, target: Versioned, version: NodeVersion) -> "ParserRef": refs = cls() for base_column in target.get_columns_for_version(version): refs._add(base_column) return refs ================================================ FILE: core/dbt/parser/docs.py ================================================ import re from typing import Iterable, Optional from dbt.clients.jinja import get_rendered from dbt.contracts.files import SourceFile from dbt.contracts.graph.nodes import Documentation from dbt.node_types import NodeType from dbt.parser.base import Parser from dbt.parser.search import BlockContents, BlockSearcher, FileBlock SHOULD_PARSE_RE = re.compile(r"{[{%]") class DocumentationParser(Parser[Documentation]): @property def resource_type(self) -> NodeType: return NodeType.Documentation @classmethod def get_compiled_path(cls, block: FileBlock): return block.path.relative_path def generate_unique_id(self, resource_name: str, _: Optional[str] = None) -> str: # For consistency, use the same format for doc unique_ids return f"doc.{self.project.project_name}.{resource_name}" def parse_block(self, block: BlockContents) -> Iterable[Documentation]: unique_id = self.generate_unique_id(block.name) contents = get_rendered(block.contents, {}).strip() doc = Documentation( path=block.file.path.relative_path, original_file_path=block.path.original_file_path, package_name=self.project.project_name, unique_id=unique_id, name=block.name, block_contents=contents, resource_type=NodeType.Documentation, ) return [doc] def parse_file(self, file_block: FileBlock): assert isinstance(file_block.file, SourceFile) searcher: Iterable[BlockContents] = BlockSearcher( source=[file_block], allowed_blocks={"docs"}, source_tag_factory=BlockContents, check_jinja=False, ) for block in searcher: for parsed in self.parse_block(block): self.manifest.add_doc(file_block.file, parsed) ================================================ FILE: core/dbt/parser/fixtures.py ================================================ import csv from io import StringIO from typing import Any, Dict, List, Optional from dbt.contracts.files import FixtureSourceFile from dbt.contracts.graph.nodes import UnitTestFileFixture from dbt.node_types import NodeType from dbt.parser.base import Parser from dbt.parser.search import FileBlock class FixtureParser(Parser[UnitTestFileFixture]): @property def resource_type(self) -> NodeType: return NodeType.Fixture @classmethod def get_compiled_path(cls, block: FileBlock): # Is this necessary? return block.path.relative_path def generate_unique_id(self, resource_name: str, _: Optional[str] = None) -> str: return f"fixture.{self.project.project_name}.{resource_name}" def parse_file(self, file_block: FileBlock): assert isinstance(file_block.file, FixtureSourceFile) unique_id = self.generate_unique_id(file_block.name) if file_block.file.path.relative_path.endswith(".sql"): rows = file_block.file.contents # type: ignore else: # endswith('.csv') rows = self.get_rows(file_block.file.contents) # type: ignore fixture = UnitTestFileFixture( name=file_block.name, path=file_block.file.path.relative_path, original_file_path=file_block.path.original_file_path, package_name=self.project.project_name, unique_id=unique_id, resource_type=NodeType.Fixture, rows=rows, ) self.manifest.add_fixture(file_block.file, fixture) def get_rows(self, contents) -> List[Dict[str, Any]]: rows = [] dummy_file = StringIO(contents) reader = csv.DictReader(dummy_file) for row in reader: rows.append(row) return rows ================================================ FILE: core/dbt/parser/functions.py ================================================ from dbt.artifacts.resources.types import NodeType from dbt.contracts.files import SourceFile from dbt.contracts.graph.nodes import FunctionNode, ManifestNode from dbt.parser.base import SimpleParser from dbt.parser.search import FileBlock class FunctionParser(SimpleParser[FileBlock, FunctionNode]): def parse_from_dict(self, dct, validate=True) -> FunctionNode: if validate: FunctionNode.validate(dct) return FunctionNode.from_dict(dct) @property def resource_type(self) -> NodeType: return NodeType.Function @classmethod def get_compiled_path(cls, block: FileBlock): return block.path.relative_path # overrides SimpleSQLParser.add_result_node def add_result_node(self, block: FileBlock, node: ManifestNode): assert isinstance(node, FunctionNode), "Got non FunctionNode in FunctionParser" file = block.file assert isinstance(file, SourceFile) if node.config.enabled: self.manifest.add_function(file, node) else: self.manifest.add_disabled(file, node) def parse_file(self, file_block: FileBlock) -> None: self.parse_node(file_block) ================================================ FILE: core/dbt/parser/generic_test.py ================================================ from typing import Iterable, List import jinja2 from dbt.contracts.files import SourceFile from dbt.contracts.graph.nodes import GenericTestNode, Macro from dbt.contracts.graph.unparsed import UnparsedMacro from dbt.exceptions import ParsingError from dbt.node_types import NodeType from dbt.parser.base import BaseParser from dbt.parser.search import FileBlock from dbt_common.clients import jinja from dbt_common.utils import MACRO_PREFIX class GenericTestParser(BaseParser[GenericTestNode]): @property def resource_type(self) -> NodeType: return NodeType.Macro @classmethod def get_compiled_path(cls, block: FileBlock): return block.path.relative_path def create_generic_test_macro( self, block: jinja.BlockTag, base_node: UnparsedMacro, name: str ) -> Macro: unique_id = self.generate_unique_id(name) macro_sql = block.full_block or "" return Macro( path=base_node.path, macro_sql=macro_sql, original_file_path=base_node.original_file_path, package_name=base_node.package_name, resource_type=base_node.resource_type, name=name, unique_id=unique_id, ) def parse_unparsed_generic_test(self, base_node: UnparsedMacro) -> Iterable[Macro]: try: blocks: List[jinja.BlockTag] = [ t for t in jinja.extract_toplevel_blocks( base_node.raw_code, allowed_blocks={"test", "data_test"}, collect_raw_data=False, ) if isinstance(t, jinja.BlockTag) ] except ParsingError as exc: exc.add_node(base_node) raise for block in blocks: try: ast = jinja.parse(block.full_block) except ParsingError as e: e.add_node(base_node) raise # generic tests are structured as macros so we want to count the number of macro blocks generic_test_nodes = list(ast.find_all(jinja2.nodes.Macro)) if len(generic_test_nodes) != 1: # things have gone disastrously wrong, we thought we only # parsed one block! raise ParsingError( f"Found multiple generic tests in {block.full_block}, expected 1", node=base_node, ) generic_test_name = generic_test_nodes[0].name if not generic_test_name.startswith(MACRO_PREFIX): continue name: str = generic_test_name.replace(MACRO_PREFIX, "") node = self.create_generic_test_macro(block, base_node, name) yield node def parse_file(self, block: FileBlock): assert isinstance(block.file, SourceFile) source_file = block.file assert isinstance(source_file.contents, str) original_file_path = source_file.path.original_file_path # this is really only used for error messages base_node = UnparsedMacro( path=original_file_path, original_file_path=original_file_path, package_name=self.project.project_name, raw_code=source_file.contents, resource_type=NodeType.Macro, language="sql", ) for node in self.parse_unparsed_generic_test(base_node): self.manifest.add_macro(block.file, node) ================================================ FILE: core/dbt/parser/generic_test_builders.py ================================================ import re from copy import deepcopy from typing import Any, Dict, Generic, List, Optional, Tuple from dbt import deprecations from dbt.artifacts.resources import NodeVersion from dbt.clients.jinja import GENERIC_TEST_KWARGS_NAME, get_rendered from dbt.contracts.graph.nodes import UnpatchedSourceDefinition from dbt.contracts.graph.unparsed import UnparsedModelUpdate, UnparsedNodeUpdate from dbt.exceptions import ( CustomMacroPopulatingConfigValueError, SameKeyNestedError, TagNotStringError, TagsNotListOfStringsError, TestArgIncludesModelError, TestArgsNotDictError, TestConfigNotDictError, TestDefinitionDictLengthError, TestNameNotStringError, TestTypeError, UnexpectedTestNamePatternError, ) from dbt.flags import get_flags from dbt.parser.common import Testable from dbt.utils import md5 from dbt_common.exceptions.macros import UndefinedMacroError def synthesize_generic_test_names( test_type: str, test_name: str, args: Dict[str, Any] ) -> Tuple[str, str]: # Using the type, name, and arguments to this generic test, synthesize a (hopefully) unique name # Will not be unique if multiple tests have same name + arguments, and only configs differ # Returns a shorter version (hashed/truncated, for the compiled file) # as well as the full name (for the unique_id + FQN) flat_args = [] for arg_name in sorted(args): # the model is already embedded in the name, so skip it if arg_name == "model": continue arg_val = args[arg_name] if isinstance(arg_val, dict): parts = list(arg_val.values()) elif isinstance(arg_val, (list, tuple)): parts = list(arg_val) else: parts = [arg_val] flat_args.extend([str(part) for part in parts]) clean_flat_args = [re.sub("[^0-9a-zA-Z_]+", "_", arg) for arg in flat_args] unique = "__".join(clean_flat_args) # for the file path + alias, the name must be <64 characters # if the full name is too long, include the first 30 identifying chars plus # a 32-character hash of the full contents test_identifier = "{}_{}".format(test_type, test_name) full_name = "{}_{}".format(test_identifier, unique) if len(full_name) >= 64: test_trunc_identifier = test_identifier[:30] label = md5(full_name) short_name = "{}_{}".format(test_trunc_identifier, label) else: short_name = full_name return short_name, full_name class TestBuilder(Generic[Testable]): """An object to hold assorted test settings and perform basic parsing Test names have the following pattern: - the test name itself may be namespaced (package.test) - or it may not be namespaced (test) """ # The 'test_name' is used to find the 'macro' that implements the test TEST_NAME_PATTERN = re.compile( r"((?P([a-zA-Z_][0-9a-zA-Z_]*))\.)?" r"(?P([a-zA-Z_][0-9a-zA-Z_]*))" ) # args in the test entry representing test configs CONFIG_ARGS = ( "severity", "tags", "enabled", "where", "limit", "warn_if", "error_if", "fail_calc", "store_failures", "store_failures_as", "sql_header", "meta", "database", "schema", "alias", ) def __init__( self, data_test: Dict[str, Any], target: Testable, package_name: str, render_ctx: Dict[str, Any], column_name: Optional[str] = None, version: Optional[NodeVersion] = None, ) -> None: test_name, test_args = self.extract_test_args( data_test, target.original_file_path, target.name, column_name, package_name ) self.args: Dict[str, Any] = test_args if "model" in self.args: raise TestArgIncludesModelError() self.package_name: str = package_name self.target: Testable = target self.version: Optional[NodeVersion] = version self.render_ctx: Dict[str, Any] = render_ctx self.column_name: Optional[str] = column_name self.args["model"] = self.build_model_str() match = self.TEST_NAME_PATTERN.match(test_name) if match is None: raise UnexpectedTestNamePatternError(test_name) groups = match.groupdict() self.name: str = groups["test_name"] self.namespace: str = groups["test_namespace"] self.config: Dict[str, Any] = {} # Process legacy args self.config.update(self._process_legacy_args()) # Process config args if present if "config" in self.args: self.config.update(self._render_values(self.args.pop("config", {}))) # Gate sql_header behind behavior change flag if ( self.config.get("sql_header") is not None and not get_flags().require_sql_header_in_test_configs ): deprecations.warn( "custom-key-in-config-deprecation", key="sql_header", file=target.original_file_path, key_path="models.config", ) if self.namespace is not None: self.package_name = self.namespace # If the user has provided a description for this generic test, use it # Then delete the "description" argument to: # 1. Avoid passing it into the test macro # 2. Avoid passing it into the test name synthesis # Otherwise, use an empty string self.description: str = "" if "description" in self.args: self.description = self.args["description"] del self.args["description"] # If the user has provided a custom name for this generic test, use it # Then delete the "name" argument to avoid passing it into the test macro # Otherwise, use an auto-generated name synthesized from test inputs self.compiled_name: str = "" self.fqn_name: str = "" if "name" in self.args: # Assign the user-defined name here, which will be checked for uniqueness later # we will raise an error if two tests have same name for same model + column combo self.compiled_name = self.args["name"] self.fqn_name = self.args["name"] del self.args["name"] else: short_name, full_name = self.get_synthetic_test_names() self.compiled_name = short_name self.fqn_name = full_name # use hashed name as alias if full name is too long if short_name != full_name and "alias" not in self.config: self.config["alias"] = short_name def _process_legacy_args(self): config = {} if "config" in self.args and not isinstance(self.args["config"], dict): raise TestConfigNotDictError(self.args["config"]) for key in self.CONFIG_ARGS: value = self.args.pop(key, None) if value and "config" in self.args and key in self.args["config"]: raise SameKeyNestedError() if not value and "config" in self.args: value = self.args["config"].pop(key, None) config[key] = value return self._render_values(config) def _render_values(self, config: Dict[str, Any]) -> Dict[str, Any]: rendered_config = {} for key, value in config.items(): if isinstance(value, str): try: value = get_rendered(value, self.render_ctx, native=True) except UndefinedMacroError as e: raise CustomMacroPopulatingConfigValueError( target_name=self.target.name, column_name=self.column_name, name=self.name, key=key, err_msg=e.msg, ) if value is not None: rendered_config[key] = value return rendered_config def _bad_type(self) -> TypeError: return TypeError('invalid target type "{}"'.format(type(self.target))) @staticmethod def extract_test_args( data_test, file_path, resource_name=None, column_name=None, package_name=None ) -> Tuple[str, Dict[str, Any]]: if not isinstance(data_test, dict): raise TestTypeError(data_test) # If the test is a dictionary with top-level keys, the test name is "test_name" # and the rest are arguments # {'name': 'my_favorite_test', 'test_name': 'unique', 'config': {'where': '1=1'}} if "test_name" in data_test.keys(): test_name = data_test.pop("test_name") test_args = data_test # If the test is a nested dictionary with one top-level key, the test name # is the dict name, and nested keys are arguments # {'unique': {'name': 'my_favorite_test', 'config': {'where': '1=1'}}} else: data_test = list(data_test.items()) if len(data_test) != 1: raise TestDefinitionDictLengthError(data_test) test_name, test_args = data_test[0] if not isinstance(test_args, dict): raise TestArgsNotDictError(test_args) if not isinstance(test_name, str): raise TestNameNotStringError(test_name) test_args = deepcopy(test_args) if column_name is not None: test_args["column_name"] = column_name # Extract kwargs when they are nested under new 'arguments' property separately from 'config' if require_generic_test_arguments_property is enabled if get_flags().require_generic_test_arguments_property: arguments = test_args.pop("arguments", {}) if not arguments and any( k not in ("config", "column_name", "description", "name") for k in test_args.keys() ): resource = ( f"'{resource_name}' in package '{package_name}'" if package_name else f"'{resource_name}'" ) deprecations.warn( "missing-arguments-property-in-generic-test-deprecation", test_name=f"`{test_name}` defined on {resource} ({file_path})", ) if isinstance(arguments, dict): test_args = {**test_args, **arguments} elif "arguments" in test_args: deprecations.warn( "arguments-property-in-generic-test-deprecation", test_name=f"`{test_name}` ({test_args['arguments']})", ) return test_name, test_args def tags(self) -> List[str]: tags = self.config.get("tags", []) if isinstance(tags, str): tags = [tags] if not isinstance(tags, list): raise TagsNotListOfStringsError(tags) for tag in tags: if not isinstance(tag, str): raise TagNotStringError(tag) return tags[:] def macro_name(self) -> str: macro_name = "test_{}".format(self.name) if self.namespace is not None: macro_name = "{}.{}".format(self.namespace, macro_name) return macro_name def get_synthetic_test_names(self) -> Tuple[str, str]: # Returns two names: shorter (for the compiled file), full (for the unique_id + FQN) target_name = self.target.name if isinstance(self.target, UnparsedModelUpdate): name = self.name if self.version: target_name = f"{self.target.name}_v{self.version}" elif isinstance(self.target, UnparsedNodeUpdate): name = self.name elif isinstance(self.target, UnpatchedSourceDefinition): name = "source_" + self.name else: raise self._bad_type() if self.namespace is not None: name = "{}_{}".format(self.namespace, name) return synthesize_generic_test_names(name, target_name, self.args) def construct_config(self) -> str: configs = ",".join( [ f"{key}=" + ( ('"' + value.replace('"', '\\"') + '"') if isinstance(value, str) else str(value) ) for key, value in self.config.items() ] ) if configs: return f"{{{{ config({configs}) }}}}" else: return "" # this is the 'raw_code' that's used in 'render_update' and execution # of the test macro def build_raw_code(self) -> str: return ("{{{{ {macro}(**{kwargs_name}) }}}}{config}").format( macro=self.macro_name(), config=self.construct_config(), kwargs_name=GENERIC_TEST_KWARGS_NAME, ) def build_model_str(self): targ = self.target if isinstance(self.target, UnparsedModelUpdate): if self.version: target_str = f"ref('{targ.name}', version='{self.version}')" else: target_str = f"ref('{targ.name}')" elif isinstance(self.target, UnparsedNodeUpdate): target_str = f"ref('{targ.name}')" elif isinstance(self.target, UnpatchedSourceDefinition): target_str = f"source('{targ.source.name}', '{targ.table.name}')" return f"{{{{ get_where_subquery({target_str}) }}}}" ================================================ FILE: core/dbt/parser/hooks.py ================================================ from dataclasses import dataclass from typing import Iterable, Iterator, List, Tuple, Union from dbt.context.context_config import ContextConfig from dbt.contracts.files import FilePath from dbt.contracts.graph.nodes import HookNode from dbt.node_types import NodeType, RunHookType from dbt.parser.base import SimpleParser from dbt.parser.search import FileBlock from dbt.utils import get_pseudo_hook_path from dbt_common.exceptions import DbtInternalError @dataclass class HookBlock(FileBlock): project: str value: str index: int hook_type: RunHookType @property def contents(self): return self.value @property def name(self): return "{}-{!s}-{!s}".format(self.project, self.hook_type, self.index) class HookSearcher(Iterable[HookBlock]): def __init__(self, project, source_file, hook_type) -> None: self.project = project self.source_file = source_file self.hook_type = hook_type def _hook_list(self, hooks: Union[str, List[str], Tuple[str, ...]]) -> List[str]: if isinstance(hooks, tuple): hooks = list(hooks) elif not isinstance(hooks, list): hooks = [hooks] return hooks def get_hook_defs(self) -> List[str]: if self.hook_type == RunHookType.Start: hooks = self.project.on_run_start elif self.hook_type == RunHookType.End: hooks = self.project.on_run_end else: raise DbtInternalError( 'hook_type must be one of "{}" or "{}" (got {})'.format( RunHookType.Start, RunHookType.End, self.hook_type ) ) return self._hook_list(hooks) def __iter__(self) -> Iterator[HookBlock]: hooks = self.get_hook_defs() for index, hook in enumerate(hooks): yield HookBlock( file=self.source_file, project=self.project.project_name, value=hook, index=index, hook_type=self.hook_type, ) class HookParser(SimpleParser[HookBlock, HookNode]): # Hooks are only in the dbt_project.yml file for the project def get_path(self) -> FilePath: # There ought to be an existing file object for this, but # until that is implemented use a dummy modification time path = FilePath( project_root=self.project.project_root, searched_path=".", relative_path="dbt_project.yml", modification_time=0.0, ) return path def parse_from_dict(self, dct, validate=True) -> HookNode: if validate: HookNode.validate(dct) return HookNode.from_dict(dct) @classmethod def get_compiled_path(cls, block: HookBlock): return get_pseudo_hook_path(block.name) def _create_parsetime_node( self, block: HookBlock, path: str, config: ContextConfig, fqn: List[str], name=None, **kwargs, ) -> HookNode: return super()._create_parsetime_node( block=block, path=path, config=config, fqn=fqn, index=block.index, name=name, tags=[str(block.hook_type)], ) @property def resource_type(self) -> NodeType: return NodeType.Operation def parse_file(self, block: FileBlock) -> None: for hook_type in RunHookType: for hook in HookSearcher(self.project, block.file, hook_type): self.parse_node(hook) ================================================ FILE: core/dbt/parser/macros.py ================================================ from typing import Iterable, List import jinja2 from dbt.artifacts.resources import MacroArgument from dbt.clients.jinja import get_supported_languages from dbt.contracts.files import FilePath, SourceFile from dbt.contracts.graph.nodes import Macro from dbt.contracts.graph.unparsed import UnparsedMacro from dbt.exceptions import ParsingError from dbt.flags import get_flags from dbt.node_types import NodeType from dbt.parser.base import BaseParser from dbt.parser.search import FileBlock, filesystem_search from dbt_common.clients import jinja from dbt_common.clients._jinja_blocks import ExtractWarning from dbt_common.utils import MACRO_PREFIX class MacroParser(BaseParser[Macro]): # This is only used when creating a MacroManifest separate # from the normal parsing flow. def get_paths(self) -> List[FilePath]: return filesystem_search( project=self.project, relative_dirs=self.project.macro_paths, extension=".sql" ) @property def resource_type(self) -> NodeType: return NodeType.Macro @classmethod def get_compiled_path(cls, block: FileBlock): return block.path.relative_path def parse_macro(self, block: jinja.BlockTag, base_node: UnparsedMacro, name: str) -> Macro: unique_id = self.generate_unique_id(name) macro_sql = block.full_block or "" return Macro( path=base_node.path, macro_sql=macro_sql, original_file_path=base_node.original_file_path, package_name=base_node.package_name, resource_type=base_node.resource_type, name=name, unique_id=unique_id, ) def parse_unparsed_macros(self, base_node: UnparsedMacro) -> Iterable[Macro]: # This is a bit of a hack to get the file path to the deprecation def wrap_handle_extract_warning(warning: ExtractWarning) -> None: self._handle_extract_warning(warning=warning, file=base_node.original_file_path) try: blocks: List[jinja.BlockTag] = [ t for t in jinja.extract_toplevel_blocks( base_node.raw_code, allowed_blocks={"macro", "materialization", "test", "data_test"}, collect_raw_data=False, warning_callback=wrap_handle_extract_warning, ) if isinstance(t, jinja.BlockTag) ] except ParsingError as exc: exc.add_node(base_node) raise for block in blocks: try: ast = jinja.parse(block.full_block) except ParsingError as e: e.add_node(base_node) raise if ( isinstance(ast, jinja2.nodes.Template) and hasattr(ast, "body") and len(ast.body) == 1 and isinstance(ast.body[0], jinja2.nodes.Macro) ): # If the top level node in the Template is a Macro, things look # good and this is much faster than traversing the full ast, as # in the following else clause. It's not clear if that traversal # is ever really needed. macro = ast.body[0] else: macro_nodes = list(ast.find_all(jinja2.nodes.Macro)) if len(macro_nodes) != 1: # things have gone disastrously wrong, we thought we only # parsed one block! raise ParsingError( f"Found multiple macros in {block.full_block}, expected 1", node=base_node ) macro = macro_nodes[0] if not macro.name.startswith(MACRO_PREFIX): continue name: str = macro.name.replace(MACRO_PREFIX, "") node = self.parse_macro(block, base_node, name) if getattr(get_flags(), "validate_macro_args", False): node.arguments = self._extract_args(macro) # get supported_languages for materialization macro if block.block_type_name == "materialization": node.supported_languages = get_supported_languages(macro) yield node def _extract_args(self, macro) -> List[MacroArgument]: try: return list([MacroArgument(name=arg.name) for arg in macro.args]) except Exception: return [] def parse_file(self, block: FileBlock): assert isinstance(block.file, SourceFile) source_file = block.file assert isinstance(source_file.contents, str) original_file_path = source_file.path.original_file_path # this is really only used for error messages base_node = UnparsedMacro( path=original_file_path, original_file_path=original_file_path, package_name=self.project.project_name, raw_code=source_file.contents, resource_type=NodeType.Macro, language="sql", ) for node in self.parse_unparsed_macros(base_node): self.manifest.add_macro(block.file, node) ================================================ FILE: core/dbt/parser/manifest.py ================================================ import json import os import pprint import time import traceback from copy import deepcopy from dataclasses import dataclass, field from datetime import date, datetime, timezone from itertools import chain from typing import Any, Callable, Dict, List, Mapping, Optional, Set, Tuple, Type, Union import jinja2 import msgpack from jinja2.nodes import Call, Const import dbt.deprecations import dbt.exceptions import dbt.tracking import dbt.utils import dbt_common.utils from dbt import plugins from dbt.adapters.capability import Capability from dbt.adapters.factory import ( get_adapter, get_adapter_package_names, get_relation_class_by_name, register_adapter, ) from dbt.artifacts.resources import ( CatalogWriteIntegrationConfig, FileHash, MetricInput, NodeRelation, NodeVersion, ) from dbt.artifacts.resources.types import BatchSize from dbt.artifacts.schemas.base import Writable from dbt.clients.jinja import MacroStack, get_rendered from dbt.clients.jinja_static import statically_extract_macro_calls from dbt.config import Project, RuntimeConfig from dbt.constants import ( MANIFEST_FILE_NAME, PARTIAL_PARSE_FILE_NAME, SEMANTIC_MANIFEST_FILE_NAME, ) from dbt.context.configured import generate_macro_context from dbt.context.docs import generate_runtime_docs_context from dbt.context.macro_resolver import MacroResolver, TestMacroNamespace from dbt.context.providers import ParseProvider, generate_runtime_macro_context from dbt.context.query_header import generate_query_header_context from dbt.contracts.files import ParseFileType, SchemaSourceFile from dbt.contracts.graph.manifest import ( Disabled, MacroManifest, Manifest, ManifestStateCheck, ParsingInfo, ) from dbt.contracts.graph.nodes import ( Exposure, GenericTestNode, Macro, ManifestNode, Metric, ModelNode, ResultNode, SavedQuery, SeedNode, SemanticManifestNode, SemanticModel, SourceDefinition, ) from dbt.contracts.graph.semantic_manifest import SemanticManifest from dbt.events.types import ( ArtifactWritten, DeprecatedModel, DeprecatedReference, InvalidConcurrentBatchesConfig, InvalidDisabledTargetInTestNode, MicrobatchModelNoEventTimeInputs, NodeNotFoundOrDisabled, PackageNodeDependsOnRootProjectNode, ParsedFileLoadFailed, ParsePerfInfoPath, PartialParsingError, PartialParsingErrorProcessingFile, PartialParsingNotEnabled, PartialParsingSkipParsing, SpacesInResourceNameDeprecation, StateCheckVarsHash, UnableToPartialParse, UpcomingReferenceDeprecation, ) from dbt.exceptions import ( AmbiguousAliasError, DuplicateResourceNameError, InvalidAccessTypeError, TargetNotFoundError, scrub_secrets, ) from dbt.flags import get_flags from dbt.mp_context import get_mp_context from dbt.node_types import AccessType, NodeType from dbt.parser.analysis import AnalysisParser from dbt.parser.base import Parser from dbt.parser.docs import DocumentationParser from dbt.parser.fixtures import FixtureParser from dbt.parser.functions import FunctionParser from dbt.parser.generic_test import GenericTestParser from dbt.parser.hooks import HookParser from dbt.parser.macros import MacroParser from dbt.parser.models import ModelParser from dbt.parser.partial import PartialParsing, special_override_macros from dbt.parser.read_files import ( FileDiff, ReadFiles, ReadFilesFromDiff, ReadFilesFromFileSystem, load_source_file, ) from dbt.parser.schemas import SchemaParser from dbt.parser.search import FileBlock from dbt.parser.seeds import SeedParser from dbt.parser.singular_test import SingularTestParser from dbt.parser.snapshots import SnapshotParser from dbt.parser.sources import SourcePatcher from dbt.parser.unit_tests import process_models_for_unit_test from dbt.utils.artifact_upload import add_artifact_produced from dbt.version import __version__ from dbt_common.clients.jinja import parse from dbt_common.clients.system import make_directory, path_exists, read_json, write_file from dbt_common.constants import SECRET_ENV_PREFIX from dbt_common.dataclass_schema import StrEnum, dbtClassMixin from dbt_common.events.base_types import EventLevel from dbt_common.events.functions import fire_event, get_invocation_id, warn_or_error from dbt_common.events.types import Note from dbt_common.exceptions.base import DbtValidationError from dbt_common.helper_types import PathSet from dbt_semantic_interfaces.enum_extension import assert_values_exhausted from dbt_semantic_interfaces.type_enums import MetricType PERF_INFO_FILE_NAME = "perf_info.json" def extended_mashumaro_encoder(data): return msgpack.packb(data, default=extended_msgpack_encoder, use_bin_type=True) def extended_msgpack_encoder(obj): if type(obj) is date: date_bytes = msgpack.ExtType(1, obj.isoformat().encode()) return date_bytes elif type(obj) is datetime: datetime_bytes = msgpack.ExtType(2, obj.isoformat().encode()) return datetime_bytes elif isinstance(obj, jinja2.Undefined): return None return obj def extended_mashumuro_decoder(data): return msgpack.unpackb(data, ext_hook=extended_msgpack_decoder, raw=False) def extended_msgpack_decoder(code, data): if code == 1: d = date.fromisoformat(data.decode()) return d elif code == 2: dt = datetime.fromisoformat(data.decode()) return dt else: return msgpack.ExtType(code, data) def version_to_str(version: Optional[Union[str, int]]) -> str: if isinstance(version, int): return str(version) elif isinstance(version, str): return version return "" class ReparseReason(StrEnum): version_mismatch = "01_version_mismatch" file_not_found = "02_file_not_found" vars_changed = "03_vars_changed" profile_changed = "04_profile_changed" deps_changed = "05_deps_changed" project_config_changed = "06_project_config_changed" load_file_failure = "07_load_file_failure" exception = "08_exception" proj_env_vars_changed = "09_project_env_vars_changed" prof_env_vars_changed = "10_profile_env_vars_changed" # Part of saved performance info @dataclass class ParserInfo(dbtClassMixin): parser: str elapsed: float parsed_path_count: int = 0 # Part of saved performance info @dataclass class ProjectLoaderInfo(dbtClassMixin): project_name: str elapsed: float parsers: List[ParserInfo] = field(default_factory=list) parsed_path_count: int = 0 # Part of saved performance info @dataclass class ManifestLoaderInfo(dbtClassMixin, Writable): path_count: int = 0 parsed_path_count: int = 0 static_analysis_path_count: int = 0 static_analysis_parsed_path_count: int = 0 is_partial_parse_enabled: Optional[bool] = None is_static_analysis_enabled: Optional[bool] = None read_files_elapsed: Optional[float] = None load_macros_elapsed: Optional[float] = None parse_project_elapsed: Optional[float] = None patch_sources_elapsed: Optional[float] = None process_manifest_elapsed: Optional[float] = None load_all_elapsed: Optional[float] = None projects: List[ProjectLoaderInfo] = field(default_factory=list) _project_index: Dict[str, ProjectLoaderInfo] = field(default_factory=dict) def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None): del dct["_project_index"] return dct # The ManifestLoader loads the manifest. The standard way to use the # ManifestLoader is using the 'get_full_manifest' class method, but # many tests use abbreviated processes. class ManifestLoader: def __init__( self, root_project: RuntimeConfig, all_projects: Mapping[str, RuntimeConfig], macro_hook: Optional[Callable[[Manifest], Any]] = None, file_diff: Optional[FileDiff] = None, ) -> None: self.root_project: RuntimeConfig = root_project self.all_projects: Mapping[str, RuntimeConfig] = all_projects self.file_diff = file_diff self.manifest: Manifest = Manifest() self.new_manifest = self.manifest self.manifest.metadata = root_project.get_metadata() self.macro_resolver = None # built after macros are loaded self.started_at = time.time() # This is a MacroQueryStringSetter callable, which is called # later after we set the MacroManifest in the adapter. It sets # up the query headers. self.macro_hook: Callable[[Manifest], Any] if macro_hook is None: self.macro_hook = lambda m: None else: self.macro_hook = macro_hook self._perf_info = self.build_perf_info() # State check determines whether the saved_manifest and the current # manifest match well enough to do partial parsing self.manifest.state_check = self.build_manifest_state_check() # We need to know if we're actually partially parsing. It could # have been enabled, but not happening because of some issue. self.partially_parsing = False self.partial_parser: Optional[PartialParsing] = None self.skip_parsing = False # This is a saved manifest from a previous run that's used for partial parsing self.saved_manifest: Optional[Manifest] = self.read_manifest_for_partial_parse() # This is the method that builds a complete manifest. We sometimes # use an abbreviated process in tests. @classmethod def get_full_manifest( cls, config: RuntimeConfig, *, file_diff: Optional[FileDiff] = None, reset: bool = False, write_perf_info=False, ) -> Manifest: adapter = get_adapter(config) # type: ignore # reset is set in a TaskManager load_manifest call, since # the config and adapter may be persistent. if reset: config.clear_dependencies() adapter.clear_macro_resolver() macro_hook = adapter.connections.set_query_header flags = get_flags() if not flags.PARTIAL_PARSE_FILE_DIFF: file_diff = FileDiff.from_dict( { "deleted": [], "changed": [], "added": [], } ) # Hack to test file_diffs elif os.environ.get("DBT_PP_FILE_DIFF_TEST"): file_diff_path = "file_diff.json" if path_exists(file_diff_path): file_diff_dct = read_json(file_diff_path) file_diff = FileDiff.from_dict(file_diff_dct) # Start performance counting start_load_all = time.perf_counter() projects = config.load_dependencies() loader = cls( config, projects, macro_hook=macro_hook, file_diff=file_diff, ) manifest = loader.load() _check_manifest(manifest, config) manifest.build_flat_graph() # This needs to happen after loading from a partial parse, # so that the adapter has the query headers from the macro_hook. loader.save_macros_to_adapter(adapter) # Save performance info loader._perf_info.load_all_elapsed = time.perf_counter() - start_load_all loader.track_project_load() if write_perf_info: loader.write_perf_info(config.project_target_path) return manifest # This is where the main action happens def load(self) -> Manifest: start_read_files = time.perf_counter() # This updates the "files" dictionary in self.manifest, and creates # the partial_parser_files dictionary (see read_files.py), # which is a dictionary of projects to a dictionary # of parsers to lists of file strings. The file strings are # used to get the SourceFiles from the manifest files. saved_files = self.saved_manifest.files if self.saved_manifest else {} file_reader: Optional[ReadFiles] = None if self.file_diff: # We're getting files from a file diff file_reader = ReadFilesFromDiff( all_projects=self.all_projects, files=self.manifest.files, saved_files=saved_files, root_project_name=self.root_project.project_name, file_diff=self.file_diff, ) else: # We're getting files from the file system file_reader = ReadFilesFromFileSystem( all_projects=self.all_projects, files=self.manifest.files, saved_files=saved_files, ) # Set the files in the manifest and save the project_parser_files file_reader.read_files() self.manifest.files = file_reader.files project_parser_files = orig_project_parser_files = file_reader.project_parser_files self._perf_info.path_count = len(self.manifest.files) self._perf_info.read_files_elapsed = time.perf_counter() - start_read_files self.skip_parsing = False project_parser_files = self.safe_update_project_parser_files_partially( project_parser_files ) if self.manifest._parsing_info is None: self.manifest._parsing_info = ParsingInfo() if self.skip_parsing: fire_event(PartialParsingSkipParsing()) else: # Load Macros and tests # We need to parse the macros first, so they're resolvable when # the other files are loaded. Also need to parse tests, specifically # generic tests start_load_macros = time.perf_counter() self.load_and_parse_macros(project_parser_files) # If we're partially parsing check that certain macros have not been changed if self.partially_parsing and self.skip_partial_parsing_because_of_macros(): fire_event( UnableToPartialParse( reason="change detected to override macro. Starting full parse." ) ) # Get new Manifest with original file records and move over the macros self.manifest = self.new_manifest # contains newly read files project_parser_files = orig_project_parser_files self.partially_parsing = False self.load_and_parse_macros(project_parser_files) self._perf_info.load_macros_elapsed = time.perf_counter() - start_load_macros # Now that the macros are parsed, parse the rest of the files. # This is currently done on a per project basis. start_parse_projects = time.perf_counter() # Load the rest of the files except for schema yaml files parser_types: List[Type[Parser]] = [ ModelParser, SnapshotParser, AnalysisParser, SingularTestParser, SeedParser, DocumentationParser, HookParser, FixtureParser, FunctionParser, ] for project in self.all_projects.values(): if project.project_name not in project_parser_files: continue self.parse_project( project, project_parser_files[project.project_name], parser_types ) # Now that we've loaded most of the nodes (except for schema tests, sources, metrics) # load up the Lookup objects to resolve them by name, so the SourceFiles store # the unique_id instead of the name. Sources are loaded from yaml files, so # aren't in place yet self.manifest.rebuild_ref_lookup() self.manifest.rebuild_doc_lookup() self.manifest.rebuild_disabled_lookup() # Load yaml files parser_types = [SchemaParser] # type: ignore for project in self.all_projects.values(): if project.project_name not in project_parser_files: continue self.parse_project( project, project_parser_files[project.project_name], parser_types ) self.cleanup_disabled() self._perf_info.parse_project_elapsed = time.perf_counter() - start_parse_projects # patch_sources converts the UnparsedSourceDefinitions in the # Manifest.sources to SourceDefinition via 'patch_source' # in SourcePatcher start_patch = time.perf_counter() patcher = SourcePatcher(self.root_project, self.manifest) patcher.construct_sources() self.manifest.sources = patcher.sources self._perf_info.patch_sources_elapsed = time.perf_counter() - start_patch # We need to rebuild disabled in order to include disabled sources self.manifest.rebuild_disabled_lookup() # copy the selectors from the root_project to the manifest self.manifest.selectors = self.root_project.manifest_selectors # inject any available external nodes self.manifest.build_parent_and_child_maps() external_nodes_modified = self.inject_external_nodes() if external_nodes_modified: self.manifest.rebuild_ref_lookup() # update the refs, sources, docs and metrics depends_on.nodes # These check the created_at time on the nodes to # determine whether they need processing. start_process = time.perf_counter() self.process_sources(self.root_project.project_name) self.process_refs(self.root_project.project_name, self.root_project.dependencies) self.process_unit_tests(self.root_project.project_name) self.process_docs(self.root_project) self.process_metrics(self.root_project) self.process_saved_queries(self.root_project) self.process_model_inferred_primary_keys() self.process_functions(self.root_project.project_name) self.check_valid_group_config() self.check_valid_access_property() self.check_valid_snapshot_config() self.check_valid_microbatch_config() semantic_manifest = SemanticManifest(self.manifest) if not semantic_manifest.validate(): raise dbt.exceptions.ParsingError("Semantic Manifest validation failed.") # update tracking data self._perf_info.process_manifest_elapsed = time.perf_counter() - start_process self._perf_info.static_analysis_parsed_path_count = ( self.manifest._parsing_info.static_analysis_parsed_path_count ) self._perf_info.static_analysis_path_count = ( self.manifest._parsing_info.static_analysis_path_count ) # Inject any available external nodes, reprocess refs if changes to the manifest were made. external_nodes_modified = False if self.skip_parsing: # If we didn't skip parsing, this will have already run because it must run # before process_refs. If we did skip parsing, then it's possible that only # external nodes have changed and we need to run this to capture that. self.manifest.build_parent_and_child_maps() external_nodes_modified = self.inject_external_nodes() if external_nodes_modified: self.manifest.rebuild_ref_lookup() self.process_refs( self.root_project.project_name, self.root_project.dependencies, ) # parent and child maps will be rebuilt by write_manifest if not self.skip_parsing or external_nodes_modified: # write out the fully parsed manifest self.write_manifest_for_partial_parse() self.check_for_model_deprecations() self.check_for_spaces_in_resource_names() self.check_for_microbatch_deprecations() self.check_forcing_batch_concurrency() self.check_microbatch_model_has_a_filtered_input() self.check_function_default_arguments_ordering() return self.manifest def safe_update_project_parser_files_partially(self, project_parser_files: Dict) -> Dict: if self.saved_manifest is None: return project_parser_files self.partial_parser = PartialParsing(self.saved_manifest, self.manifest.files) # type: ignore[arg-type] self.skip_parsing = self.partial_parser.skip_parsing() if self.skip_parsing: # nothing changed, so we don't need to generate project_parser_files fire_event( Note(msg="Nothing changed, skipping partial parsing."), level=EventLevel.DEBUG ) self.manifest = self.saved_manifest # type: ignore[assignment] else: # create child_map and parent_map self.saved_manifest.build_parent_and_child_maps() # type: ignore[union-attr] # create group_map self.saved_manifest.build_group_map() # type: ignore[union-attr] # files are different, we need to create a new set of # project_parser_files. try: project_parser_files = self.partial_parser.get_parsing_files() self.partially_parsing = True self.manifest = self.saved_manifest # type: ignore[assignment] except Exception as exc: # pp_files should still be the full set and manifest is new manifest, # since get_parsing_files failed fire_event( UnableToPartialParse(reason="an error occurred. Switching to full reparse.") ) # Get traceback info tb_info = traceback.format_exc() # index last stack frame in traceback (i.e. lastest exception and its context) tb_last_frame = traceback.extract_tb(exc.__traceback__)[-1] exc_info = { "traceback": tb_info, "exception": tb_info.splitlines()[-1], "code": tb_last_frame.line, # if the source is not available, it is None "location": f"line {tb_last_frame.lineno} in {tb_last_frame.name}", } # get file info for local logs parse_file_type: str = "" file_id = self.partial_parser.processing_file if file_id: source_file = None if file_id in self.saved_manifest.files: source_file = self.saved_manifest.files[file_id] elif file_id in self.manifest.files: source_file = self.manifest.files[file_id] if source_file: parse_file_type = source_file.parse_file_type fire_event(PartialParsingErrorProcessingFile(file=file_id)) exc_info["parse_file_type"] = parse_file_type fire_event(PartialParsingError(exc_info=exc_info)) # Send event if dbt.tracking.active_user is not None: exc_info["full_reparse_reason"] = ReparseReason.exception dbt.tracking.track_partial_parser(exc_info) if os.environ.get("DBT_PP_TEST"): raise exc return project_parser_files def check_for_model_deprecations(self): # build parent and child_maps self.manifest.build_parent_and_child_maps() for node in self.manifest.nodes.values(): if isinstance(node, ModelNode) and node.deprecation_date: if node.is_past_deprecation_date: warn_or_error( DeprecatedModel( model_name=node.name, model_version=version_to_str(node.version), deprecation_date=node.deprecation_date.isoformat(), ) ) # At this point _process_refs should already have been called, and # we just rebuilt the parent and child maps. # Get the child_nodes and check for deprecations. child_nodes = self.manifest.child_map[node.unique_id] for child_unique_id in child_nodes: child_node = self.manifest.nodes.get(child_unique_id) if not isinstance(child_node, ModelNode): continue if node.is_past_deprecation_date: event_cls = DeprecatedReference else: event_cls = UpcomingReferenceDeprecation warn_or_error( event_cls( model_name=child_node.name, ref_model_package=node.package_name, ref_model_name=node.name, ref_model_version=version_to_str(node.version), ref_model_latest_version=str(node.latest_version), ref_model_deprecation_date=node.deprecation_date.isoformat(), ) ) def check_for_spaces_in_resource_names(self): """Validates that resource names do not contain spaces If `DEBUG` flag is `False`, logs only first bad model name, unless `REQUIRE_RESOURCE_NAMES_WITHOUT_SPACES` is `True` as error will indicate all bad model names If `DEBUG` flag is `True`, logs every bad model name If `REQUIRE_RESOURCE_NAMES_WITHOUT_SPACES` is `True`, logs are `ERROR` level and an exception is raised if any names are bad If `REQUIRE_RESOURCE_NAMES_WITHOUT_SPACES` is `False`, logs are `WARN` level """ improper_resource_names_unique_ids = set() error_on_invalid_resource_name = ( self.root_project.args.REQUIRE_RESOURCE_NAMES_WITHOUT_SPACES ) level = EventLevel.ERROR if error_on_invalid_resource_name else EventLevel.WARN flags = get_flags() for node in self.manifest.nodes.values(): if " " in node.name: if ( not improper_resource_names_unique_ids and not error_on_invalid_resource_name ) or flags.DEBUG: fire_event( SpacesInResourceNameDeprecation( unique_id=node.unique_id, level=level.value, ), level=level, ) improper_resource_names_unique_ids.add(node.unique_id) if improper_resource_names_unique_ids: if level == EventLevel.WARN: dbt.deprecations.warn( "resource-names-with-spaces", count_invalid_names=len(improper_resource_names_unique_ids), show_debug_hint=(not flags.DEBUG), ) else: # ERROR level formatted_resources_with_spaces = "\n".join( f" * '{unique_id}' ({self.manifest.nodes[unique_id].original_file_path})" for unique_id in improper_resource_names_unique_ids ) raise DbtValidationError( f"Resource names cannot contain spaces:\n{formatted_resources_with_spaces}\nPlease rename the invalid model(s) so that their name(s) do not contain any spaces." ) def check_for_microbatch_deprecations(self) -> None: if not get_flags().require_batched_execution_for_custom_microbatch_strategy: has_microbatch_model = False for _, node in self.manifest.nodes.items(): if ( isinstance(node, ModelNode) and node.config.materialized == "incremental" and node.config.incremental_strategy == "microbatch" ): has_microbatch_model = True break if has_microbatch_model and not self.manifest._microbatch_macro_is_core( self.root_project.project_name ): dbt.deprecations.warn("microbatch-macro-outside-of-batches-deprecation") def load_and_parse_macros(self, project_parser_files): for project in self.all_projects.values(): if project.project_name not in project_parser_files: continue parser_files = project_parser_files[project.project_name] if "MacroParser" in parser_files: parser = MacroParser(project, self.manifest) for file_id in parser_files["MacroParser"]: block = FileBlock(self.manifest.files[file_id]) parser.parse_file(block) # increment parsed path count for performance tracking self._perf_info.parsed_path_count += 1 # generic tests hisotrically lived in the macros directoy but can now be nested # in a /generic directory under /tests so we want to process them here as well if "GenericTestParser" in parser_files: parser = GenericTestParser(project, self.manifest) for file_id in parser_files["GenericTestParser"]: block = FileBlock(self.manifest.files[file_id]) parser.parse_file(block) # increment parsed path count for performance tracking self._perf_info.parsed_path_count += 1 self.build_macro_resolver() # Look at changed macros and update the macro.depends_on.macros self.macro_depends_on() # Parse the files in the 'parser_files' dictionary, for parsers listed in # 'parser_types' def parse_project( self, project: RuntimeConfig, parser_files, parser_types: List[Type[Parser]], ) -> None: project_loader_info = self._perf_info._project_index[project.project_name] start_timer = time.perf_counter() total_parsed_path_count = 0 # Loop through parsers with loaded files. for parser_cls in parser_types: parser_name = parser_cls.__name__ # No point in creating a parser if we don't have files for it if parser_name not in parser_files or not parser_files[parser_name]: continue # Initialize timing info project_parsed_path_count = 0 parser_start_timer = time.perf_counter() # Parse the project files for this parser parser: Parser = parser_cls(project, self.manifest, self.root_project) for file_id in parser_files[parser_name]: block = FileBlock(self.manifest.files[file_id]) if isinstance(parser, SchemaParser): assert isinstance(block.file, SchemaSourceFile) if self.partially_parsing: dct = block.file.pp_dict else: dct = block.file.dict_from_yaml # this is where the schema file gets parsed parser.parse_file(block, dct=dct) # Came out of here with UnpatchedSourceDefinition containing configs at the source level # and not configs at the table level (as expected) else: parser.parse_file(block) project_parsed_path_count += 1 # Save timing info project_loader_info.parsers.append( ParserInfo( parser=parser.resource_type, parsed_path_count=project_parsed_path_count, elapsed=time.perf_counter() - parser_start_timer, ) ) total_parsed_path_count += project_parsed_path_count # HookParser doesn't run from loaded files, just dbt_project.yml, # so do separately # This shouldn't need to be parsed again if we're starting from # a saved manifest, because that won't be allowed if dbt_project.yml # changed, but leave for now. if not self.partially_parsing and HookParser in parser_types: hook_parser = HookParser(project, self.manifest, self.root_project) path = hook_parser.get_path() file = load_source_file(path, ParseFileType.Hook, project.project_name, {}) if file: file_block = FileBlock(file) hook_parser.parse_file(file_block) # Store the performance info elapsed = time.perf_counter() - start_timer project_loader_info.parsed_path_count = ( project_loader_info.parsed_path_count + total_parsed_path_count ) project_loader_info.elapsed += elapsed self._perf_info.parsed_path_count = ( self._perf_info.parsed_path_count + total_parsed_path_count ) # This should only be called after the macros have been loaded def build_macro_resolver(self): internal_package_names = get_adapter_package_names(self.root_project.credentials.type) self.macro_resolver = MacroResolver( self.manifest.macros, self.root_project.project_name, internal_package_names ) # Loop through macros in the manifest and statically parse # the 'macro_sql' to find depends_on.macros def macro_depends_on(self): macro_ctx = generate_macro_context(self.root_project) macro_namespace = TestMacroNamespace(self.macro_resolver, {}, None, MacroStack(), []) adapter = get_adapter(self.root_project) db_wrapper = ParseProvider().DatabaseWrapper(adapter, macro_namespace) for macro in self.manifest.macros.values(): if macro.created_at < self.started_at: continue possible_macro_calls = statically_extract_macro_calls( macro.macro_sql, macro_ctx, db_wrapper ) for macro_name in possible_macro_calls: # adapter.dispatch calls can generate a call with the same name as the macro # it ought to be an adapter prefix (postgres_) or default_ if macro_name == macro.name: continue package_name = macro.package_name if "." in macro_name: package_name, macro_name = macro_name.split(".") dep_macro_id = self.macro_resolver.get_macro_id(package_name, macro_name) if dep_macro_id: macro.depends_on.add_macro(dep_macro_id) # will check for dupes def write_manifest_for_partial_parse(self): path = os.path.join(self.root_project.project_target_path, PARTIAL_PARSE_FILE_NAME) try: # This shouldn't be necessary, but we have gotten bug reports (#3757) of the # saved manifest not matching the code version. if self.manifest.metadata.dbt_version != __version__: fire_event( UnableToPartialParse(reason="saved manifest contained the wrong version") ) self.manifest.metadata.dbt_version = __version__ manifest_msgpack = self.manifest.to_msgpack(extended_mashumaro_encoder) make_directory(os.path.dirname(path)) with open(path, "wb") as fp: fp.write(manifest_msgpack) except Exception: raise def inject_external_nodes(self) -> bool: # Remove previously existing external nodes since we are regenerating them manifest_nodes_modified = False # Remove all dependent nodes before removing referencing nodes for unique_id in self.manifest.external_node_unique_ids: remove_dependent_project_references(self.manifest, unique_id) manifest_nodes_modified = True for unique_id in self.manifest.external_node_unique_ids: # remove external nodes from manifest only after dependent project references safely removed self.manifest.nodes.pop(unique_id) # Inject any newly-available external nodes pm = plugins.get_plugin_manager(self.root_project.project_name) plugin_model_nodes = pm.get_nodes().models for node_arg in plugin_model_nodes.values(): node = ModelNode.from_args(node_arg) # node may already exist from package or running project (even if it is disabled), # in which case we should avoid clobbering it with an external node if ( node.unique_id not in self.manifest.nodes and node.unique_id not in self.manifest.disabled ): self.manifest.add_node_nofile(node) manifest_nodes_modified = True return manifest_nodes_modified def is_partial_parsable(self, manifest: Manifest) -> Tuple[bool, Optional[str]]: """Compare the global hashes of the read-in parse results' values to the known ones, and return if it is ok to re-use the results. """ valid = True reparse_reason = None if manifest.metadata.dbt_version != __version__: # #3757 log both versions because of reports of invalid cases of mismatch. fire_event(UnableToPartialParse(reason="of a version mismatch")) # If the version is wrong, the other checks might not work return False, ReparseReason.version_mismatch if self.manifest.state_check.vars_hash != manifest.state_check.vars_hash: fire_event( UnableToPartialParse( reason="config vars, config profile, or config target have changed" ) ) fire_event( Note( msg=f"previous checksum: {self.manifest.state_check.vars_hash.checksum}, current checksum: {manifest.state_check.vars_hash.checksum}" ), level=EventLevel.DEBUG, ) valid = False reparse_reason = ReparseReason.vars_changed if self.manifest.state_check.profile_hash != manifest.state_check.profile_hash: # Note: This should be made more granular. We shouldn't need to invalidate # partial parsing if a non-used profile section has changed. fire_event(UnableToPartialParse(reason="profile has changed")) valid = False reparse_reason = ReparseReason.profile_changed if ( self.manifest.state_check.project_env_vars_hash != manifest.state_check.project_env_vars_hash ): fire_event( UnableToPartialParse(reason="env vars used in dbt_project.yml have changed") ) valid = False reparse_reason = ReparseReason.proj_env_vars_changed missing_keys = { k for k in self.manifest.state_check.project_hashes if k not in manifest.state_check.project_hashes } if missing_keys: fire_event(UnableToPartialParse(reason="a project dependency has been added")) valid = False reparse_reason = ReparseReason.deps_changed for key, new_value in self.manifest.state_check.project_hashes.items(): if key in manifest.state_check.project_hashes: old_value = manifest.state_check.project_hashes[key] if new_value != old_value: fire_event(UnableToPartialParse(reason="a project config has changed")) valid = False reparse_reason = ReparseReason.project_config_changed return valid, reparse_reason def skip_partial_parsing_because_of_macros(self): if not self.partial_parser: return False if self.partial_parser.deleted_special_override_macro: return True # Check for custom versions of these special macros for macro_name in special_override_macros: macro = self.macro_resolver.get_macro(None, macro_name) if macro and macro.package_name != "dbt": if ( macro.file_id in self.partial_parser.file_diff["changed"] or macro.file_id in self.partial_parser.file_diff["added"] ): # The file with the macro in it has changed return True return False def read_manifest_for_partial_parse(self) -> Optional[Manifest]: flags = get_flags() if not flags.PARTIAL_PARSE: fire_event(PartialParsingNotEnabled()) return None path = flags.PARTIAL_PARSE_FILE_PATH or os.path.join( self.root_project.project_target_path, PARTIAL_PARSE_FILE_NAME ) reparse_reason = None if os.path.exists(path): try: with open(path, "rb") as fp: manifest_mp = fp.read() manifest: Manifest = Manifest.from_msgpack(manifest_mp, decoder=extended_mashumuro_decoder) # type: ignore # keep this check inside the try/except in case something about # the file has changed in weird ways, perhaps due to being a # different version of dbt is_partial_parsable, reparse_reason = self.is_partial_parsable(manifest) if is_partial_parsable: # We don't want to have stale generated_at dates manifest.metadata.generated_at = datetime.now(timezone.utc).replace( tzinfo=None ) # or invocation_ids manifest.metadata.invocation_id = get_invocation_id() return manifest except Exception as exc: fire_event( ParsedFileLoadFailed(path=path, exc=str(exc), exc_info=traceback.format_exc()) ) reparse_reason = ReparseReason.load_file_failure else: fire_event( UnableToPartialParse(reason="saved manifest not found. Starting full parse.") ) reparse_reason = ReparseReason.file_not_found # this event is only fired if a full reparse is needed if dbt.tracking.active_user is not None: # no active_user if doing load_macros dbt.tracking.track_partial_parser({"full_reparse_reason": reparse_reason}) return None def build_perf_info(self): flags = get_flags() mli = ManifestLoaderInfo( is_partial_parse_enabled=flags.PARTIAL_PARSE, is_static_analysis_enabled=flags.STATIC_PARSER, ) for project in self.all_projects.values(): project_info = ProjectLoaderInfo( project_name=project.project_name, elapsed=0, ) mli.projects.append(project_info) mli._project_index[project.project_name] = project_info return mli # TODO: handle --vars in the same way we handle env_var # https://github.com/dbt-labs/dbt-core/issues/6323 def build_manifest_state_check(self): config = self.root_project all_projects = self.all_projects # if any of these change, we need to reject the parser # Create a FileHash of vars string, profile name and target name # This does not capture vars in dbt_project, just the command line # arg vars, but since any changes to that file will cause state_check # to not pass, it doesn't matter. If we move to more granular checking # of env_vars, that would need to change. # We are using the parsed cli_vars instead of config.args.vars, in order # to sort them and avoid reparsing because of ordering issues. secret_vars = [ v for k, v in config.cli_vars.items() if k.startswith(SECRET_ENV_PREFIX) and v.strip() ] stringified_cli_vars = pprint.pformat(config.cli_vars) vars_hash_contents = [ stringified_cli_vars, config.profile_name, config.target_name, __version__, ] # We only add vars from `vars.yml` if it's present to prevent affecting users not using vars.yml from getting fully parsed. if config.vars_from_file: vars_hash_contents.append(pprint.pformat(config.vars_from_file)) vars_hash = FileHash.from_contents("\x00".join(vars_hash_contents)) fire_event( StateCheckVarsHash( checksum=vars_hash.checksum, vars=scrub_secrets(stringified_cli_vars, secret_vars), profile=config.profile_name, target=config.target_name, version=__version__, ) ) # Create a FileHash of the env_vars in the project key_list = list(config.project_env_vars.keys()) key_list.sort() env_var_str = "" for key in key_list: env_var_str += f"{key}:{config.project_env_vars[key]}|" project_env_vars_hash = FileHash.from_contents(env_var_str) # Create a hash of the connection_info, which user has access to in # jinja context. Thus attributes here may affect the parsing result. # Ideally we should not expose all of the connection info to the jinja. # Renaming this variable mean that we will have to do a whole lot more # change to make sure the previous manifest can be loaded correctly. # This is an example of naming should be chosen based on the functionality # rather than the implementation details. connection_keys = list(config.credentials.connection_info()) # avoid reparsing because of ordering issues connection_keys.sort() profile_hash = FileHash.from_contents(pprint.pformat(connection_keys)) # Create a FileHashes for dbt_project for all dependencies project_hashes = {} for name, project in all_projects.items(): path = os.path.join(project.project_root, "dbt_project.yml") with open(path) as fp: project_hashes[name] = FileHash.from_contents(fp.read()) # Create the ManifestStateCheck object state_check = ManifestStateCheck( project_env_vars_hash=project_env_vars_hash, vars_hash=vars_hash, profile_hash=profile_hash, project_hashes=project_hashes, ) return state_check def save_macros_to_adapter(self, adapter): adapter.set_macro_resolver(self.manifest) # This executes the callable macro_hook and sets the # query headers # This executes the callable macro_hook and sets the query headers query_header_context = generate_query_header_context(adapter.config, self.manifest) self.macro_hook(query_header_context) # This creates a MacroManifest which contains the macros in # the adapter. Only called by the load_macros call from the # adapter. def create_macro_manifest(self): for project in self.all_projects.values(): # what is the manifest passed in actually used for? macro_parser = MacroParser(project, self.manifest) for path in macro_parser.get_paths(): source_file = load_source_file(path, ParseFileType.Macro, project.project_name, {}) block = FileBlock(source_file) # This does not add the file to the manifest.files, # but that shouldn't be necessary here. macro_parser.parse_file(block) macro_manifest = MacroManifest(self.manifest.macros) return macro_manifest # This is called by the adapter code only, to create the # MacroManifest that's stored in the adapter. # 'get_full_manifest' uses a persistent ManifestLoader while this # creates a temporary ManifestLoader and throws it away. # Not sure when this would actually get used except in tests. # The ManifestLoader loads macros with other files, then copies # into the adapter MacroManifest. @classmethod def load_macros( cls, root_config: RuntimeConfig, macro_hook: Callable[[Manifest], Any], base_macros_only=False, ) -> Manifest: # base_only/base_macros_only: for testing only, # allows loading macros without running 'dbt deps' first projects = root_config.load_dependencies(base_only=base_macros_only) # This creates a loader object, including result, # and then throws it away, returning only the # manifest loader = cls(root_config, projects, macro_hook) return loader.create_macro_manifest() # Create tracking event for saving performance info def track_project_load(self): invocation_id = get_invocation_id() dbt.tracking.track_project_load( { "invocation_id": invocation_id, "project_id": self.root_project.hashed_name(), "path_count": self._perf_info.path_count, "parsed_path_count": self._perf_info.parsed_path_count, "read_files_elapsed": self._perf_info.read_files_elapsed, "load_macros_elapsed": self._perf_info.load_macros_elapsed, "parse_project_elapsed": self._perf_info.parse_project_elapsed, "patch_sources_elapsed": self._perf_info.patch_sources_elapsed, "process_manifest_elapsed": (self._perf_info.process_manifest_elapsed), "load_all_elapsed": self._perf_info.load_all_elapsed, "is_partial_parse_enabled": (self._perf_info.is_partial_parse_enabled), "is_static_analysis_enabled": self._perf_info.is_static_analysis_enabled, "static_analysis_path_count": self._perf_info.static_analysis_path_count, "static_analysis_parsed_path_count": self._perf_info.static_analysis_parsed_path_count, # noqa: E501 } ) # Takes references in 'refs' array of nodes and exposures, finds the target # node, and updates 'depends_on.nodes' with the unique id def process_refs(self, current_project: str, dependencies: Optional[Mapping[str, Project]]): for node in self.manifest.nodes.values(): if node.created_at < self.started_at: continue _process_refs(self.manifest, current_project, node, dependencies) for exposure in self.manifest.exposures.values(): if exposure.created_at < self.started_at: continue _process_refs(self.manifest, current_project, exposure, dependencies) for metric in self.manifest.metrics.values(): if metric.created_at < self.started_at: continue _process_refs(self.manifest, current_project, metric, dependencies) for semantic_model in self.manifest.semantic_models.values(): if semantic_model.created_at < self.started_at: continue _process_refs(self.manifest, current_project, semantic_model, dependencies) self.update_semantic_model(semantic_model) for function in self.manifest.functions.values(): if function.created_at < self.started_at: continue _process_refs(self.manifest, current_project, function, dependencies) # Takes references in 'metrics' array of nodes and exposures, finds the target # node, and updates 'depends_on.nodes' with the unique id def process_metrics(self, config: RuntimeConfig): current_project = config.project_name for metric in self.manifest.metrics.values(): if metric.created_at < self.started_at: continue _process_metric_node(self.manifest, current_project, metric) _process_metrics_for_node(self.manifest, current_project, metric) for node in self.manifest.nodes.values(): if node.created_at < self.started_at: continue _process_metrics_for_node(self.manifest, current_project, node) for exposure in self.manifest.exposures.values(): if exposure.created_at < self.started_at: continue _process_metrics_for_node(self.manifest, current_project, exposure) def process_saved_queries(self, config: RuntimeConfig): """Processes SavedQuery nodes to populate their `depends_on`.""" # Note: This will also capture various nodes which have been re-parsed # because they refer to some other changed node, so there will be # false positives. Ideally we would compare actual changes. semantic_manifest_changed = False semantic_manifest_nodes: chain[SemanticManifestNode] = chain( self.manifest.saved_queries.values(), self.manifest.semantic_models.values(), self.manifest.metrics.values(), ) for node in semantic_manifest_nodes: # Check if this node has been modified in this parsing run if node.created_at > self.started_at: semantic_manifest_changed = True break # as soon as we run into one changed node we can stop if semantic_manifest_changed is False: return current_project = config.project_name for saved_query in self.manifest.saved_queries.values(): # TODO: # 1. process `where` of SavedQuery for `depends_on`s # 2. process `group_by` of SavedQuery for `depends_on`` _process_metrics_for_node(self.manifest, current_project, saved_query) def process_model_inferred_primary_keys(self): """Processes Model nodes to populate their `primary_key`.""" model_to_generic_test_map: Dict[str, List[GenericTestNode]] = {} for node in self.manifest.nodes.values(): if not isinstance(node, ModelNode): continue if node.created_at < self.started_at: continue if not model_to_generic_test_map: model_to_generic_test_map = self.build_model_to_generic_tests_map() generic_tests: List[GenericTestNode] = [] if node.unique_id in model_to_generic_test_map: generic_tests = model_to_generic_test_map[node.unique_id] primary_key = node.infer_primary_key(generic_tests) node.primary_key = sorted(primary_key) def update_semantic_model(self, semantic_model) -> None: # This has to be done at the end of parsing because the referenced model # might have alias/schema/database fields that are updated by yaml config. if semantic_model.depends_on_nodes: refd_node = self.manifest.nodes[semantic_model.depends_on_nodes[0]] semantic_model.node_relation = NodeRelation( relation_name=refd_node.relation_name, alias=refd_node.alias, schema_name=refd_node.schema, database=refd_node.database, ) # nodes: node and column descriptions, version columns descriptions # sources: source and table descriptions, column descriptions # macros: macro argument descriptions # exposures: exposure descriptions # metrics: metric descriptions # semantic_models: semantic model descriptions def process_docs(self, config: RuntimeConfig): for node in self.manifest.nodes.values(): if node.created_at < self.started_at: continue ctx = generate_runtime_docs_context( config, node, self.manifest, config.project_name, ) _process_docs_for_node(ctx, node, self.manifest) for source in self.manifest.sources.values(): if source.created_at < self.started_at: continue ctx = generate_runtime_docs_context( config, source, self.manifest, config.project_name, ) _process_docs_for_source(ctx, source, self.manifest) for macro in self.manifest.macros.values(): if macro.created_at < self.started_at: continue ctx = generate_runtime_docs_context( config, macro, self.manifest, config.project_name, ) _process_docs_for_macro(ctx, macro) for exposure in self.manifest.exposures.values(): if exposure.created_at < self.started_at: continue ctx = generate_runtime_docs_context( config, exposure, self.manifest, config.project_name, ) _process_docs_for_exposure(ctx, exposure) for metric in self.manifest.metrics.values(): if metric.created_at < self.started_at: continue ctx = generate_runtime_docs_context( config, metric, self.manifest, config.project_name, ) _process_docs_for_metrics(ctx, metric) for semantic_model in self.manifest.semantic_models.values(): if semantic_model.created_at < self.started_at: continue ctx = generate_runtime_docs_context( config, semantic_model, self.manifest, config.project_name, ) _process_docs_for_semantic_model(ctx, semantic_model) for saved_query in self.manifest.saved_queries.values(): if saved_query.created_at < self.started_at: continue ctx = generate_runtime_docs_context( config, saved_query, self.manifest, config.project_name ) _process_docs_for_saved_query(ctx, saved_query) # Loops through all nodes and exposures, for each element in # 'sources' array finds the source node and updates the # 'depends_on.nodes' array with the unique id def process_sources(self, current_project: str): for node in self.manifest.nodes.values(): if node.resource_type == NodeType.Source: continue assert not isinstance(node, SourceDefinition) if node.created_at < self.started_at: continue _process_sources_for_node(self.manifest, current_project, node) for exposure in self.manifest.exposures.values(): if exposure.created_at < self.started_at: continue _process_sources_for_exposure(self.manifest, current_project, exposure) # Loops through all nodes, for each element in # 'unit_test' array finds the node and updates the # 'depends_on.nodes' array with the unique id def process_unit_tests(self, current_project: str): models_to_versions = None unit_test_unique_ids = list(self.manifest.unit_tests.keys()) for unit_test_unique_id in unit_test_unique_ids: # This is because some unit tests will be removed when processing # and the list of unit_test_unique_ids won't have changed if unit_test_unique_id in self.manifest.unit_tests: unit_test = self.manifest.unit_tests[unit_test_unique_id] else: continue if unit_test.created_at < self.started_at: continue if not models_to_versions: models_to_versions = _build_model_names_to_versions(self.manifest) process_models_for_unit_test( self.manifest, current_project, unit_test, models_to_versions ) # Loops through all nodes, for each element in # 'functions' array finds the node and updates the # 'depends_on.nodes' array with the unique id def process_functions(self, current_project: str): for node in self.manifest.nodes.values(): if node.created_at < self.started_at: continue _process_functions_for_node(self.manifest, current_project, node) for function in self.manifest.functions.values(): if function.created_at < self.started_at: continue _process_functions_for_node(self.manifest, current_project, function) def cleanup_disabled(self): # make sure the nodes are in the manifest.nodes or the disabled dict, # correctly now that the schema files are also parsed disabled_nodes = [] for node in self.manifest.nodes.values(): if not node.config.enabled: disabled_nodes.append(node.unique_id) self.manifest.add_disabled_nofile(node) for unique_id in disabled_nodes: self.manifest.nodes.pop(unique_id) disabled_copy = deepcopy(self.manifest.disabled) for disabled in disabled_copy.values(): for node in disabled: if node.config.enabled: for dis_index, dis_node in enumerate(disabled): # Remove node from disabled and unique_id from disabled dict if necessary del self.manifest.disabled[node.unique_id][dis_index] if not self.manifest.disabled[node.unique_id]: self.manifest.disabled.pop(node.unique_id) self.manifest.add_node_nofile(node) self.manifest.rebuild_ref_lookup() def check_valid_group_config(self): manifest = self.manifest group_names = {group.name for group in manifest.groups.values()} for metric in manifest.metrics.values(): self.check_valid_group_config_node(metric, group_names) for semantic_model in manifest.semantic_models.values(): self.check_valid_group_config_node(semantic_model, group_names) for saved_query in manifest.saved_queries.values(): self.check_valid_group_config_node(saved_query, group_names) for node in manifest.nodes.values(): self.check_valid_group_config_node(node, group_names) def check_valid_group_config_node( self, groupable_node: Union[Metric, SavedQuery, SemanticModel, ManifestNode], valid_group_names: Set[str], ): groupable_node_group = groupable_node.group if groupable_node_group and groupable_node_group not in valid_group_names: raise dbt.exceptions.ParsingError( f"Invalid group '{groupable_node_group}', expected one of {sorted(list(valid_group_names))}", node=groupable_node, ) def check_valid_access_property(self): for node in self.manifest.nodes.values(): if ( isinstance(node, ModelNode) and node.access == AccessType.Public and node.get_materialization() == "ephemeral" ): raise InvalidAccessTypeError( unique_id=node.unique_id, field_value=node.access, materialization=node.get_materialization(), ) def check_valid_snapshot_config(self): # Snapshot config can be set in either SQL files or yaml files, # so we need to validate afterward. for node in self.manifest.nodes.values(): if node.resource_type != NodeType.Snapshot: continue if node.created_at < self.started_at: continue node.config.final_validate() def check_valid_microbatch_config(self): if self.manifest.use_microbatch_batches(project_name=self.root_project.project_name): for node in self.manifest.nodes.values(): if ( node.config.materialized == "incremental" and node.config.incremental_strategy == "microbatch" ): # Required configs: event_time, batch_size, begin event_time = node.config.event_time if event_time is None: raise dbt.exceptions.ParsingError( f"Microbatch model '{node.name}' must provide an 'event_time' (string) config that indicates the name of the event time column." ) if not isinstance(event_time, str): raise dbt.exceptions.ParsingError( f"Microbatch model '{node.name}' must provide an 'event_time' config of type string, but got: {type(event_time)}." ) begin = node.config.begin if begin is None: raise dbt.exceptions.ParsingError( f"Microbatch model '{node.name}' must provide a 'begin' (datetime) config that indicates the earliest timestamp the microbatch model should be built from." ) # Try to cast begin to a datetime using same format as mashumaro for consistency with other yaml-provided datetimes # Mashumaro default: https://github.com/Fatal1ty/mashumaro/blob/4ac16fd060a6c651053475597b58b48f958e8c5c/README.md?plain=1#L1186 if isinstance(begin, str): try: begin = datetime.fromisoformat(begin) node.config.begin = begin except Exception: raise dbt.exceptions.ParsingError( f"Microbatch model '{node.name}' must provide a 'begin' config of valid datetime (ISO format), but got: {begin}." ) if not isinstance(begin, datetime): raise dbt.exceptions.ParsingError( f"Microbatch model '{node.name}' must provide a 'begin' config of type datetime, but got: {type(begin)}." ) batch_size = node.config.batch_size valid_batch_sizes = [size.value for size in BatchSize] if batch_size not in valid_batch_sizes: raise dbt.exceptions.ParsingError( f"Microbatch model '{node.name}' must provide a 'batch_size' config that is one of {valid_batch_sizes}, but got: {batch_size}." ) # Optional config: lookback (int) lookback = node.config.lookback if not isinstance(lookback, int) and lookback is not None: raise dbt.exceptions.ParsingError( f"Microbatch model '{node.name}' must provide the optional 'lookback' config as type int, but got: {type(lookback)})." ) # optional config: concurrent_batches (bool) concurrent_batches = node.config.concurrent_batches if not isinstance(concurrent_batches, bool) and concurrent_batches is not None: raise dbt.exceptions.ParsingError( f"Microbatch model '{node.name}' optional 'concurrent_batches' config must be of type `bool` if specified, but got: {type(concurrent_batches)})." ) def check_forcing_batch_concurrency(self) -> None: if self.manifest.use_microbatch_batches(project_name=self.root_project.project_name): adapter = get_adapter(self.root_project) if not adapter.supports(Capability.MicrobatchConcurrency): models_forcing_concurrent_batches = 0 for node in self.manifest.nodes.values(): if ( hasattr(node.config, "concurrent_batches") and node.config.concurrent_batches is True ): models_forcing_concurrent_batches += 1 if models_forcing_concurrent_batches > 0: warn_or_error( InvalidConcurrentBatchesConfig( num_models=models_forcing_concurrent_batches, adapter_type=adapter.type(), ) ) def check_microbatch_model_has_a_filtered_input(self): if self.manifest.use_microbatch_batches(project_name=self.root_project.project_name): for node in self.manifest.nodes.values(): if ( node.config.materialized == "incremental" and node.config.incremental_strategy == "microbatch" ): # Validate upstream node event_time (if configured) has_input_with_event_time_config = False for input_unique_id in node.depends_on.nodes: input_node = self.manifest.expect(unique_id=input_unique_id) input_event_time = input_node.config.event_time if input_event_time: if not isinstance(input_event_time, str): raise dbt.exceptions.ParsingError( f"Microbatch model '{node.name}' depends on an input node '{input_node.name}' with an 'event_time' config of invalid (non-string) type: {type(input_event_time)}." ) has_input_with_event_time_config = True if not has_input_with_event_time_config: fire_event(MicrobatchModelNoEventTimeInputs(model_name=node.name)) def check_function_default_arguments_ordering(self): for function in self.manifest.functions.values(): found_default_value = False for argument in function.arguments: if not found_default_value and argument.default_value is not None: found_default_value = True elif found_default_value and argument.default_value is None: raise dbt.exceptions.ParsingError( f"Non-defaulted argument '{argument.name}' of function '{function.name}' comes after a defaulted argument. Non-defaulted arguments cannot come after defaulted arguments. " ) def write_perf_info(self, target_path: str): path = os.path.join(target_path, PERF_INFO_FILE_NAME) write_file(path, json.dumps(self._perf_info, cls=dbt.utils.JSONEncoder, indent=4)) fire_event(ParsePerfInfoPath(path=path)) def build_model_to_generic_tests_map(self) -> Dict[str, List[GenericTestNode]]: """Return a list of generic tests that are attached to the given model, including disabled tests""" model_to_generic_tests_map: Dict[str, List[GenericTestNode]] = {} for _, node in self.manifest.nodes.items(): if isinstance(node, GenericTestNode) and node.attached_node: if node.attached_node not in model_to_generic_tests_map: model_to_generic_tests_map[node.attached_node] = [] model_to_generic_tests_map[node.attached_node].append(node) for _, nodes in self.manifest.disabled.items(): for disabled_node in nodes: if isinstance(disabled_node, GenericTestNode) and disabled_node.attached_node: if disabled_node.attached_node not in model_to_generic_tests_map: model_to_generic_tests_map[disabled_node.attached_node] = [] model_to_generic_tests_map[disabled_node.attached_node].append(disabled_node) return model_to_generic_tests_map def invalid_target_fail_unless_test( node, target_name: str, target_kind: str, target_package: Optional[str] = None, target_version: Optional[NodeVersion] = None, disabled: Optional[bool] = None, should_warn_if_disabled: bool = True, ): if node.resource_type == NodeType.Test: if disabled: event = InvalidDisabledTargetInTestNode( resource_type_title=node.resource_type.title(), unique_id=node.unique_id, original_file_path=node.original_file_path, target_kind=target_kind, target_name=target_name, target_package=target_package if target_package else "", ) fire_event(event, EventLevel.WARN if should_warn_if_disabled else None) else: warn_or_error( NodeNotFoundOrDisabled( original_file_path=node.original_file_path, unique_id=node.unique_id, resource_type_title=node.resource_type.title(), target_name=target_name, target_kind=target_kind, target_package=target_package if target_package else "", disabled=str(disabled), ) ) else: raise TargetNotFoundError( node=node, target_name=target_name, target_kind=target_kind, target_package=target_package, target_version=target_version, disabled=disabled, ) def warn_if_package_node_depends_on_root_project_node( node: ManifestNode, target_model: ManifestNode, ref_package_name: Optional[str], current_project: str, ) -> None: """ Args: node: The node that specifies the ref target_model: The node that is being ref'd to ref_package_name: The package name specified in the ref current_project: The root project """ if ( node.package_name != current_project and target_model.package_name == current_project and ref_package_name != current_project ): warn_or_error( PackageNodeDependsOnRootProjectNode( node_name=node.name, package_name=node.package_name, root_project_unique_id=target_model.unique_id, ) ) def _build_model_names_to_versions(manifest: Manifest) -> Dict[str, Dict]: model_names_to_versions: Dict[str, Dict] = {} for node in manifest.nodes.values(): if node.resource_type != NodeType.Model: continue if not node.is_versioned: continue if node.package_name not in model_names_to_versions: model_names_to_versions[node.package_name] = {} if node.name not in model_names_to_versions[node.package_name]: model_names_to_versions[node.package_name][node.name] = [] model_names_to_versions[node.package_name][node.name].append(node.unique_id) return model_names_to_versions def _check_resource_uniqueness( manifest: Manifest, config: RuntimeConfig, ) -> None: alias_resources: Dict[str, ManifestNode] = {} name_resources: Dict[str, Dict] = {} for _, node in manifest.nodes.items(): if not node.is_relational: continue if node.package_name not in name_resources: name_resources[node.package_name] = {"ver": {}, "unver": {}} existing_unversioned_node = name_resources[node.package_name]["unver"].get(node.name) if existing_unversioned_node is not None and not node.is_versioned: if get_flags().require_unique_project_resource_names: raise DuplicateResourceNameError(existing_unversioned_node, node) else: dbt.deprecations.warn( "duplicate-name-distinct-node-types-deprecation", resource_name=node.name, unique_id1=existing_unversioned_node.unique_id, unique_id2=node.unique_id, package_name=node.package_name, ) if node.is_versioned: name_resources[node.package_name]["ver"][node.name] = node else: name_resources[node.package_name]["unver"][node.name] = node # the full node name is really defined by the adapter's relation relation_cls = get_relation_class_by_name(config.credentials.type) relation = relation_cls.create_from(quoting=config, relation_config=node) # type: ignore[arg-type] full_node_name = str(relation) existing_alias = alias_resources.get(full_node_name) if existing_alias is not None: raise AmbiguousAliasError( node_1=existing_alias, node_2=node, duped_name=full_node_name ) alias_resources[full_node_name] = node for ver_unver_dict in name_resources.values(): versioned_names = ver_unver_dict["ver"].keys() unversioned_names = ver_unver_dict["unver"].keys() intersection_versioned = set(versioned_names).intersection(set(unversioned_names)) if intersection_versioned: for name in intersection_versioned: versioned_node = ver_unver_dict["ver"][name] unversioned_node = ver_unver_dict["unver"][name] raise dbt.exceptions.DuplicateVersionedUnversionedError( versioned_node, unversioned_node ) def _warn_for_unused_resource_config_paths(manifest: Manifest, config: RuntimeConfig) -> None: resource_fqns: Mapping[str, PathSet] = manifest.get_resource_fqns() disabled_fqns: PathSet = frozenset( tuple(n.fqn) for n in list(chain.from_iterable(manifest.disabled.values())) ) config.warn_for_unused_resource_config_paths(resource_fqns, disabled_fqns) def _check_manifest(manifest: Manifest, config: RuntimeConfig) -> None: _check_resource_uniqueness(manifest, config) _warn_for_unused_resource_config_paths(manifest, config) DocsContextCallback = Callable[[ResultNode], Dict[str, Any]] def _get_doc_blocks(description: str, manifest: Manifest, node_package: str) -> List[str]: ast = parse(description) doc_blocks: List[str] = [] if not hasattr(ast, "body"): return doc_blocks for statement in ast.body: for node in statement.nodes: if ( isinstance(node, Call) and hasattr(node, "node") and hasattr(node, "args") and hasattr(node.node, "name") and node.node.name == "doc" ): # Only Const is statically resolvable to a block name at parse time. # * It does have a "value" attribute but mypy is unconvinced so the hasattr is to make it extra happy. # Filter out Const values to avoid raising an unhandled exception attempting # to statically parseother jinja expression nodes (ie Concat, CondExpr) doc_args = [ arg.value for arg in node.args if isinstance(arg, Const) and hasattr(arg, "value") ] if len(doc_args) == 1: package, name = None, doc_args[0] elif len(doc_args) == 2: package, name = doc_args else: continue if not manifest.metadata.project_name: continue resolved_doc = manifest.resolve_doc( name, package, manifest.metadata.project_name, node_package ) if resolved_doc: doc_blocks.append(resolved_doc.unique_id) return doc_blocks # node and column descriptions def _process_docs_for_node( context: Dict[str, Any], node: ManifestNode, manifest: Manifest, ): node.doc_blocks = _get_doc_blocks(node.description, manifest, node.package_name) node.description = get_rendered(node.description, context) for column_name, column in node.columns.items(): column.doc_blocks = _get_doc_blocks(column.description, manifest, node.package_name) column.description = get_rendered(column.description, context) # source and table descriptions, column descriptions def _process_docs_for_source( context: Dict[str, Any], source: SourceDefinition, manifest: Manifest, ): source.doc_blocks = _get_doc_blocks(source.description, manifest, source.package_name) source.description = get_rendered(source.description, context) source.source_description = get_rendered(source.source_description, context) for column in source.columns.values(): column.doc_blocks = _get_doc_blocks(column.description, manifest, source.package_name) column.description = get_rendered(column.description, context) # macro argument descriptions def _process_docs_for_macro(context: Dict[str, Any], macro: Macro) -> None: macro.description = get_rendered(macro.description, context) for arg in macro.arguments: arg.description = get_rendered(arg.description, context) # exposure descriptions def _process_docs_for_exposure(context: Dict[str, Any], exposure: Exposure) -> None: exposure.description = get_rendered(exposure.description, context) def _process_docs_for_metrics(context: Dict[str, Any], metric: Metric) -> None: metric.description = get_rendered(metric.description, context) def _process_docs_for_semantic_model( context: Dict[str, Any], semantic_model: SemanticModel ) -> None: if semantic_model.description: semantic_model.description = get_rendered(semantic_model.description, context) for dimension in semantic_model.dimensions: if dimension.description: dimension.description = get_rendered(dimension.description, context) for measure in semantic_model.measures: if measure.description: measure.description = get_rendered(measure.description, context) for entity in semantic_model.entities: if entity.description: entity.description = get_rendered(entity.description, context) def _process_docs_for_saved_query(context: Dict[str, Any], saved_query: SavedQuery) -> None: if saved_query.description: saved_query.description = get_rendered(saved_query.description, context) def _process_refs( manifest: Manifest, current_project: str, node, dependencies: Optional[Mapping[str, Project]] ) -> None: """Given a manifest and node in that manifest, process its refs""" dependencies = dependencies or {} if isinstance(node, SeedNode): return for ref in node.refs: target_model: Optional[Union[Disabled, ManifestNode]] = None target_model_name: str = ref.name target_model_package: Optional[str] = ref.package target_model_version: Optional[NodeVersion] = ref.version if len(ref.positional_args) < 1 or len(ref.positional_args) > 2: raise dbt.exceptions.DbtInternalError( f"Refs should always be 1 or 2 arguments - got {len(ref.positional_args)}" ) target_model = manifest.resolve_ref( node, target_model_name, target_model_package, target_model_version, current_project, node.package_name, ) if target_model is None or isinstance(target_model, Disabled): # This may raise. Even if it doesn't, we don't want to add # this exposure to the graph b/c there is no destination exposure node.config.enabled = False invalid_target_fail_unless_test( node=node, target_name=target_model_name, target_kind="node", target_package=target_model_package, target_version=target_model_version, disabled=(isinstance(target_model, Disabled)), should_warn_if_disabled=False, ) continue elif manifest.is_invalid_private_ref(node, target_model, dependencies): raise dbt.exceptions.DbtReferenceError( unique_id=node.unique_id, ref_unique_id=target_model.unique_id, access=AccessType.Private, scope=dbt_common.utils.cast_to_str(target_model.group), ) elif manifest.is_invalid_protected_ref(node, target_model, dependencies): raise dbt.exceptions.DbtReferenceError( unique_id=node.unique_id, ref_unique_id=target_model.unique_id, access=AccessType.Protected, scope=target_model.package_name, ) if not get_flags().require_ref_searches_node_package_before_root: warn_if_package_node_depends_on_root_project_node( node, target_model, ref.package, current_project ) target_model_id = target_model.unique_id node.depends_on.add_node(target_model_id) def _add_depends_on_metrics_to_v2_metric( metric: Metric, input_metrics: List[MetricInput], manifest: Manifest, current_project: str, ) -> None: """Set the depends_on property for a v2 metric that depends on other metrics""" for input_metric in input_metrics: target_metric = manifest.resolve_metric( target_metric_name=input_metric.name, target_metric_package=None, current_project=current_project, node_package=metric.package_name, ) if target_metric is None: raise dbt.exceptions.ParsingError( f"The metric `{input_metric.name}` does not exist but was referenced.", node=metric, ) elif isinstance(target_metric, Disabled): raise dbt.exceptions.ParsingError( f"The metric `{input_metric.name}` is disabled and thus cannot be referenced.", node=metric, ) _process_metric_node( manifest=manifest, current_project=current_project, metric=target_metric, ) metric.depends_on.add_node(target_metric.unique_id) def _process_metric_depends_on_semantic_models_for_measures( manifest: Manifest, current_project: str, metric: Metric, ) -> None: """For a given v1 metric, set the `depends_on` property""" assert ( len(metric.type_params.input_measures) > 0 or metric.type_params.metric_aggregation_params is not None ), f"{metric} should have a measure or agg type defined, but it does not." for input_measure in metric.type_params.input_measures: target_semantic_model = manifest.resolve_semantic_model_for_measure( target_measure_name=input_measure.name, current_project=current_project, node_package=metric.package_name, ) if target_semantic_model is None: raise dbt.exceptions.ParsingError( f"A semantic model having a measure `{input_measure.name}` does not exist but was referenced.", node=metric, ) if target_semantic_model.config.enabled is False: raise dbt.exceptions.ParsingError( f"The measure `{input_measure.name}` is referenced on disabled semantic model `{target_semantic_model.name}`.", node=metric, ) metric.depends_on.add_node(target_semantic_model.unique_id) def _process_multiple_metric_inputs( manifest: Manifest, current_project: str, metric: Metric, metric_inputs: List[MetricInput], ) -> None: for input_metric in metric_inputs: target_metric = manifest.resolve_metric( target_metric_name=input_metric.name, target_metric_package=None, current_project=current_project, node_package=metric.package_name, ) if target_metric is None: raise dbt.exceptions.ParsingError( f"The metric `{input_metric.name}` does not exist but was referenced.", node=metric, ) elif isinstance(target_metric, Disabled): raise dbt.exceptions.ParsingError( f"The metric `{input_metric.name}` is disabled and thus cannot be referenced.", node=metric, ) _process_metric_node( manifest=manifest, current_project=current_project, metric=target_metric ) for input_measure in target_metric.type_params.input_measures: metric.add_input_measure(input_measure) metric.depends_on.add_node(target_metric.unique_id) def _process_metric_node( manifest: Manifest, current_project: str, metric: Metric, ) -> None: """Sets a metric's `input_measures` and `depends_on` properties""" # This ensures that if this metrics input_measures have already been set # we skip the work. This could happen either due to recursion or if multiple # metrics derive from another given metric. # NOTE: This does not protect against infinite loops if len(metric.type_params.input_measures) > 0: return # TODO DI-4613: we need a v2 equivalent to avoid unnecessary work! (This will # probably require passing through / maintaining a "processed" set of metric names) if metric.type is MetricType.SIMPLE or metric.type is MetricType.CUMULATIVE: if metric.type is MetricType.SIMPLE and ( metric.type_params.measure is None and metric.type_params.metric_aggregation_params is None ): # This should be caught earlier, but just in case, we assert here to avoid # any unexpected behaviors. raise dbt.exceptions.ParsingError( f"Simple metric {metric} should have a measure or agg type defined, but it does not.", node=metric, ) elif metric.type is MetricType.CUMULATIVE and ( metric.type_params.measure is None and ( metric.type_params.cumulative_type_params is None or metric.type_params.cumulative_type_params.metric is None ) ): raise dbt.exceptions.ParsingError( f"Cumulative metric {metric} should have a measure or input_metric defined, but it does not.", node=metric, ) if metric.type_params.measure is not None: # v1 dependencies metric.add_input_measure(metric.type_params.measure) _process_metric_depends_on_semantic_models_for_measures( manifest=manifest, current_project=current_project, metric=metric ) else: # v2 dependencies if metric.type is MetricType.SIMPLE: semantic_model_dependency = metric.type_params.get_semantic_model_name() if semantic_model_dependency is None: raise dbt.exceptions.ParsingError( f"Simple metric `{metric.name}` must be attached to a semantic model.", node=metric, ) unique_id = ( f"{NodeType.SemanticModel}.{current_project}.{semantic_model_dependency}" ) metric.depends_on.add_node(unique_id) if metric.type is MetricType.CUMULATIVE: cumulative_type_params = metric.type_params.cumulative_type_params input_metric = ( cumulative_type_params.metric if cumulative_type_params is not None else None ) assert ( input_metric is not None ), f"Cumulative metric `{metric.name}` must have a metric as an input." _add_depends_on_metrics_to_v2_metric( metric, input_metrics=[input_metric], manifest=manifest, current_project=current_project, ) elif metric.type is MetricType.CONVERSION: conversion_type_params = metric.type_params.conversion_type_params assert ( conversion_type_params ), f"{metric.name} is a conversion metric and must have conversion_type_params defined." # Handle old-style YAML measure inputs base_measure = conversion_type_params.base_measure conversion_measure = conversion_type_params.conversion_measure base_metric = conversion_type_params.base_metric conversion_metric = conversion_type_params.conversion_metric if base_measure is not None or conversion_measure is not None: # v1 dependencies assert base_measure is not None and conversion_measure is not None, ( f"Conversion metric `{metric.name}` cannot have only one of base measure " + "and conversion measure defined." ) metric.add_input_measure(base_measure) metric.add_input_measure(conversion_measure) _process_metric_depends_on_semantic_models_for_measures( manifest=manifest, current_project=current_project, metric=metric, ) elif base_metric is not None or conversion_metric is not None: # v2 dependencies assert base_metric is not None and conversion_metric is not None, ( f"Conversion metric `{metric.name}` cannot have only one of base metric " + "and conversion metric defined." ) _add_depends_on_metrics_to_v2_metric( metric, input_metrics=[base_metric, conversion_metric], manifest=manifest, current_project=current_project, ) else: raise dbt.exceptions.ParsingError( f"Depending the version of YAML being used, conversion metric `{metric.name}` " + "must have base and conversion measures or base and conversion metrics defined.", node=metric, ) elif metric.type is MetricType.DERIVED or metric.type is MetricType.RATIO: input_metrics = metric.input_metrics if metric.type is MetricType.RATIO: if metric.type_params.numerator is None or metric.type_params.denominator is None: raise dbt.exceptions.ParsingError( "Invalid ratio metric. Both a numerator and denominator must be specified", node=metric, ) input_metrics = [metric.type_params.numerator, metric.type_params.denominator] for input_metric in input_metrics: target_metric = manifest.resolve_metric( target_metric_name=input_metric.name, target_metric_package=None, current_project=current_project, node_package=metric.package_name, ) if target_metric is None: raise dbt.exceptions.ParsingError( f"The metric `{input_metric.name}` does not exist but was referenced by metric `{metric.name}`.", node=metric, ) elif isinstance(target_metric, Disabled): raise dbt.exceptions.ParsingError( f"The metric `{input_metric.name}` is disabled and thus cannot be referenced.", node=metric, ) _process_metric_node( manifest=manifest, current_project=current_project, metric=target_metric ) for input_measure in target_metric.type_params.input_measures: metric.add_input_measure(input_measure) metric.depends_on.add_node(target_metric.unique_id) else: assert_values_exhausted(metric.type) def _process_metrics_for_node( manifest: Manifest, current_project: str, node: Union[ManifestNode, Metric, Exposure, SavedQuery], ): """Given a manifest and a node in that manifest, process its metrics""" metrics: List[List[str]] if isinstance(node, SeedNode): return elif isinstance(node, SavedQuery): metrics = [[metric] for metric in node.metrics] else: metrics = node.metrics for metric in metrics: target_metric: Optional[Union[Disabled, Metric]] = None target_metric_name: str target_metric_package: Optional[str] = None if len(metric) == 1: target_metric_name = metric[0] elif len(metric) == 2: target_metric_package, target_metric_name = metric else: raise dbt.exceptions.DbtInternalError( f"Metric references should always be 1 or 2 arguments - got {len(metric)}" ) target_metric = manifest.resolve_metric( target_metric_name, target_metric_package, current_project, node.package_name, ) if target_metric is None or isinstance(target_metric, Disabled): # This may raise. Even if it doesn't, we don't want to add # this node to the graph b/c there is no destination node node.config.enabled = False invalid_target_fail_unless_test( node=node, target_name=target_metric_name, target_kind="metric", target_package=target_metric_package, disabled=(isinstance(target_metric, Disabled)), ) continue target_metric_id = target_metric.unique_id node.depends_on.add_node(target_metric_id) def remove_dependent_project_references(manifest, external_node_unique_id): for child_id in manifest.child_map[external_node_unique_id]: node = manifest.expect(child_id) # child node may have been modified and already recreated its depends_on.nodes list if external_node_unique_id in node.depends_on_nodes: node.depends_on_nodes.remove(external_node_unique_id) node.created_at = time.time() def _process_sources_for_exposure(manifest: Manifest, current_project: str, exposure: Exposure): target_source: Optional[Union[Disabled, SourceDefinition]] = None for source_name, table_name in exposure.sources: target_source = manifest.resolve_source( source_name, table_name, current_project, exposure.package_name, ) if target_source is None or isinstance(target_source, Disabled): exposure.config.enabled = False invalid_target_fail_unless_test( node=exposure, target_name=f"{source_name}.{table_name}", target_kind="source", disabled=(isinstance(target_source, Disabled)), ) continue target_source_id = target_source.unique_id exposure.depends_on.add_node(target_source_id) def _process_sources_for_metric(manifest: Manifest, current_project: str, metric: Metric): target_source: Optional[Union[Disabled, SourceDefinition]] = None for source_name, table_name in metric.sources: target_source = manifest.resolve_source( source_name, table_name, current_project, metric.package_name, ) if target_source is None or isinstance(target_source, Disabled): metric.config.enabled = False invalid_target_fail_unless_test( node=metric, target_name=f"{source_name}.{table_name}", target_kind="source", disabled=(isinstance(target_source, Disabled)), ) continue target_source_id = target_source.unique_id metric.depends_on.add_node(target_source_id) def _process_sources_for_node(manifest: Manifest, current_project: str, node: ManifestNode): if isinstance(node, SeedNode): return target_source: Optional[Union[Disabled, SourceDefinition]] = None for source_name, table_name in node.sources: target_source = manifest.resolve_source( source_name, table_name, current_project, node.package_name, ) if target_source is None or isinstance(target_source, Disabled): # this follows the same pattern as refs node.config.enabled = False invalid_target_fail_unless_test( node=node, target_name=f"{source_name}.{table_name}", target_kind="source", disabled=(isinstance(target_source, Disabled)), ) continue target_source_id = target_source.unique_id node.depends_on.add_node(target_source_id) def _process_functions_for_node( manifest: Manifest, current_project: str, node: ManifestNode ) -> None: """Given a manifest and node in that manifest, process its functions""" if isinstance(node, SeedNode): return for function_args in node.functions: target_function_name: str target_function_package: Optional[str] = None if len(function_args) == 1: target_function_name = function_args[0] elif len(function_args) == 2: target_function_package, target_function_name = function_args else: raise dbt.exceptions.DbtInternalError( f"Functions should always be 1 or 2 arguments - got {len(function_args)}" ) target_function = manifest.resolve_function( target_function_name, target_function_package, current_project, node.package_name, ) if target_function is None or isinstance(target_function, Disabled): node.config.enabled = False invalid_target_fail_unless_test( node=node, target_name=target_function_name, target_kind="function", target_package=target_function_package, disabled=(isinstance(target_function, Disabled)), should_warn_if_disabled=False, ) continue node.depends_on.add_node(target_function.unique_id) # This is called in task.rpc.sql_commands when a "dynamic" node is # created in the manifest, in 'add_refs' def process_macro(config: RuntimeConfig, manifest: Manifest, macro: Macro) -> None: ctx = generate_runtime_docs_context( config, macro, manifest, config.project_name, ) _process_docs_for_macro(ctx, macro) # This is called in task.rpc.sql_commands when a "dynamic" node is # created in the manifest, in 'add_refs' def process_node(config: RuntimeConfig, manifest: Manifest, node: ManifestNode): _process_sources_for_node(manifest, config.project_name, node) _process_refs(manifest, config.project_name, node, config.dependencies) ctx = generate_runtime_docs_context(config, node, manifest, config.project_name) _process_docs_for_node(ctx, node, manifest) def write_semantic_manifest(manifest: Manifest, target_path: str) -> None: path = os.path.join(target_path, SEMANTIC_MANIFEST_FILE_NAME) semantic_manifest = SemanticManifest(manifest) semantic_manifest.write_json_to_file(path) def write_manifest(manifest: Manifest, target_path: str, which: Optional[str] = None): file_name = MANIFEST_FILE_NAME path = os.path.join(target_path, file_name) manifest.write(path) add_artifact_produced(path) write_semantic_manifest(manifest=manifest, target_path=target_path) def parse_manifest( runtime_config: RuntimeConfig, write_perf_info: bool, write: bool, write_json: bool, active_integrations: List[Optional[CatalogWriteIntegrationConfig]], ) -> Manifest: register_adapter(runtime_config, get_mp_context()) adapter = get_adapter(runtime_config) adapter.set_macro_context_generator(generate_runtime_macro_context) for integration in active_integrations: adapter.add_catalog_integration(integration) manifest = ManifestLoader.get_full_manifest( runtime_config, write_perf_info=write_perf_info, ) # If we should (over)write the manifest in the target path, do that now if write and write_json: write_manifest(manifest, runtime_config.project_target_path) pm = plugins.get_plugin_manager(runtime_config.project_name) plugin_artifacts = pm.get_manifest_artifacts(manifest) for path, plugin_artifact in plugin_artifacts.items(): plugin_artifact.write(path) fire_event( ArtifactWritten( artifact_type=plugin_artifact.__class__.__name__, artifact_path=path ) ) return manifest ================================================ FILE: core/dbt/parser/models.py ================================================ # New for Python models :p import ast import random from copy import deepcopy from functools import reduce from itertools import chain from typing import Any, Dict, Iterator, List, Optional, Tuple, Union import dbt.tracking as tracking from dbt import utils from dbt.artifacts.resources import RefArgs from dbt.clients.jinja import get_rendered from dbt.context.context_config import ContextConfig from dbt.contracts.graph.nodes import ModelNode from dbt.exceptions import ( ModelConfigError, ParsingError, PythonLiteralEvalError, PythonParsingError, ) from dbt.flags import get_flags from dbt.node_types import ModelLanguage, NodeType from dbt.parser.base import SimpleSQLParser from dbt.parser.search import FileBlock from dbt_common.contracts.config.base import merge_config_dicts from dbt_common.dataclass_schema import ValidationError from dbt_common.exceptions.macros import UndefinedMacroError from dbt_extractor import ExtractionError, py_extract_from_source # type: ignore dbt_function_key_words = set(["ref", "source", "config", "get", "meta_get"]) dbt_function_full_names = set( ["dbt.ref", "dbt.source", "dbt.config", "dbt.config.get", "dbt.config.meta_get"] ) class PythonValidationVisitor(ast.NodeVisitor): def __init__(self) -> None: super().__init__() self.dbt_errors: List[str] = [] self.num_model_def = 0 def visit_FunctionDef(self, node: ast.FunctionDef) -> None: if node.name == "model": self.num_model_def += 1 if node.args.args and not node.args.args[0].arg == "dbt": self.dbt_errors.append("'dbt' not provided for model as the first argument") if len(node.args.args) != 2: self.dbt_errors.append( "model function should have two args, `dbt` and a session to current warehouse" ) # check we have a return and only one if not isinstance(node.body[-1], ast.Return) or isinstance( node.body[-1].value, ast.Tuple ): self.dbt_errors.append( "In current version, model function should return only one dataframe object" ) def check_error(self, node): if self.num_model_def != 1: raise ParsingError( f"dbt allows exactly one model defined per python file, found {self.num_model_def}", node=node, ) if len(self.dbt_errors) != 0: raise ParsingError("\n".join(self.dbt_errors), node=node) class PythonParseVisitor(ast.NodeVisitor): def __init__(self, dbt_node): super().__init__() self.dbt_node = dbt_node self.dbt_function_calls = [] self.packages = [] @classmethod def _flatten_attr(cls, node): if isinstance(node, ast.Attribute): return str(cls._flatten_attr(node.value)) + "." + node.attr elif isinstance(node, ast.Name): return str(node.id) else: pass def _safe_eval(self, node): try: return ast.literal_eval(node) except (SyntaxError, ValueError, TypeError, MemoryError, RecursionError) as exc: raise PythonLiteralEvalError(exc, node=self.dbt_node) from exc def _get_call_literals(self, node): # List of literals arg_literals = [] kwarg_literals = {} # TODO : Make sure this throws (and that we catch it) # for non-literal inputs for arg in node.args: rendered = self._safe_eval(arg) arg_literals.append(rendered) for keyword in node.keywords: key = keyword.arg rendered = self._safe_eval(keyword.value) kwarg_literals[key] = rendered return arg_literals, kwarg_literals def visit_Call(self, node: ast.Call) -> None: # check weather the current call could be a dbt function call if isinstance(node.func, ast.Attribute) and node.func.attr in dbt_function_key_words: func_name = self._flatten_attr(node.func) # check weather the current call really is a dbt function call if func_name in dbt_function_full_names: # drop the dot-dbt prefix func_name = func_name.split(".")[-1] args, kwargs = self._get_call_literals(node) self.dbt_function_calls.append((func_name, args, kwargs)) # no matter what happened above, we should keep visiting the rest of the tree # visit args and kwargs to see if there's call in it for obj in node.args + [kwarg.value for kwarg in node.keywords]: if isinstance(obj, ast.Call): self.visit_Call(obj) # support dbt.ref in list args, kwargs elif isinstance(obj, ast.List) or isinstance(obj, ast.Tuple): for el in obj.elts: if isinstance(el, ast.Call): self.visit_Call(el) # support dbt.ref in dict args, kwargs elif isinstance(obj, ast.Dict): for value in obj.values: if isinstance(value, ast.Call): self.visit_Call(value) # support dbt function calls in f-strings elif isinstance(obj, ast.JoinedStr): for value in obj.values: if isinstance(value, ast.FormattedValue) and isinstance(value.value, ast.Call): self.visit_Call(value.value) # visit node.func.value if we are at an call attr if isinstance(node.func, ast.Attribute): self.attribute_helper(node.func) def attribute_helper(self, node: ast.Attribute) -> None: while isinstance(node, ast.Attribute): node = node.value # type: ignore if isinstance(node, ast.Call): self.visit_Call(node) def visit_Import(self, node: ast.Import) -> None: for n in node.names: self.packages.append(n.name.split(".")[0]) def visit_ImportFrom(self, node: ast.ImportFrom) -> None: if node.module: self.packages.append(node.module.split(".")[0]) def verify_python_model_code(node): # TODO: add a test for this try: rendered_python = get_rendered( node.raw_code, {}, node, ) if rendered_python != node.raw_code: raise ParsingError("") except (UndefinedMacroError, ParsingError): raise ParsingError("No jinja in python model code is allowed", node=node) class ModelParser(SimpleSQLParser[ModelNode]): def parse_from_dict(self, dct, validate=True) -> ModelNode: if validate: ModelNode.validate(dct) return ModelNode.from_dict(dct) @property def resource_type(self) -> NodeType: return NodeType.Model @classmethod def get_compiled_path(cls, block: FileBlock): return block.path.relative_path def parse_python_model(self, node, config, context): config_keys_used = [] config_keys_defaults = [] meta_keys_used = [] meta_keys_defaults = [] try: tree = ast.parse(node.raw_code, filename=node.original_file_path) except SyntaxError as exc: raise PythonParsingError(exc, node=node) from exc # Only parse if AST tree has instructions in body if tree.body: # We are doing a validator and a parser because visit_FunctionDef in parser # would actually make the parser not doing the visit_Calls any more dbt_validator = PythonValidationVisitor() dbt_validator.visit(tree) dbt_validator.check_error(node) dbt_parser = PythonParseVisitor(node) dbt_parser.visit(tree) for func, args, kwargs in dbt_parser.dbt_function_calls: if func == "get": num_args = len(args) if num_args == 0: raise ParsingError( "dbt.config.get() requires at least one argument", node=node, ) if num_args > 2: raise ParsingError( f"dbt.config.get() takes at most 2 arguments ({num_args} given)", node=node, ) key = args[0] default_value = args[1] if num_args == 2 else None config_keys_used.append(key) config_keys_defaults.append(default_value) continue if func == "meta_get": num_args = len(args) if num_args == 0: raise ParsingError( "dbt.config.meta_get() requires at least one argument", node=node, ) if num_args > 2: raise ParsingError( f"dbt.config.meta_get() takes at most 2 arguments ({num_args} given)", node=node, ) key = args[0] default_value = args[1] if num_args == 2 else None meta_keys_used.append(key) meta_keys_defaults.append(default_value) continue context[func](*args, **kwargs) if config_keys_used or meta_keys_used: # this is being used in macro build_config_dict context["config"]( config_keys_used=config_keys_used, config_keys_defaults=config_keys_defaults, meta_keys_used=meta_keys_used, meta_keys_defaults=meta_keys_defaults, ) def render_update( self, node: ModelNode, config: ContextConfig, validate_config_call_dict: bool = False ) -> None: self.manifest._parsing_info.static_analysis_path_count += 1 flags = get_flags() if node.language == ModelLanguage.python: try: verify_python_model_code(node) context = self._context_for(node, config) self.parse_python_model(node, config, context) self.update_parsed_node_config( node, config, context=context, validate_config_call_dict=True ) except ValidationError as exc: # we got a ValidationError - probably bad types in config() raise ModelConfigError(exc, node=node) from exc return elif not flags.STATIC_PARSER: # jinja rendering super().render_update(node, config) return # only sample for experimental parser correctness on normal runs, # not when the experimental parser flag is on. exp_sample: bool = False # sampling the stable static parser against jinja is significantly # more expensive and therefore done far less frequently. stable_sample: bool = False # there are two samples above, and it is perfectly fine if both happen # at the same time. If that happens, the experimental parser, stable # parser, and jinja rendering will run on the same model file and # send back codes for experimental v stable, and stable v jinja. if not flags.USE_EXPERIMENTAL_PARSER: # `True` roughly 1/5000 times this function is called # sample = random.randint(1, 5001) == 5000 stable_sample = random.randint(1, 5001) == 5000 # sampling the experimental parser is explicitly disabled here, but use the following # commented code to sample a fraction of the time when new # experimental features are added. # `True` roughly 1/100 times this function is called # exp_sample = random.randint(1, 101) == 100 # top-level declaration of variables statically_parsed: Optional[Union[str, Dict[str, List[Any]]]] = None experimental_sample: Optional[Union[str, Dict[str, List[Any]]]] = None exp_sample_node: Optional[ModelNode] = None exp_sample_config: Optional[ContextConfig] = None jinja_sample_node: Optional[ModelNode] = None jinja_sample_config: Optional[ContextConfig] = None result: List[str] = [] # sample the experimental parser only during a normal run if exp_sample and not flags.USE_EXPERIMENTAL_PARSER: experimental_sample = self.run_experimental_parser(node) # if the experimental parser succeeded, make a full copy of model parser # and populate _everything_ into it so it can be compared apples-to-apples # with a fully jinja-rendered project. This is necessary because the experimental # parser will likely add features that the existing static parser will fail on # so comparing those directly would give us bad results. The comparison will be # conducted after this model has been fully rendered either by the static parser # or by full jinja rendering if isinstance(experimental_sample, dict): model_parser_copy = self.partial_deepcopy() exp_sample_node = deepcopy(node) exp_sample_config = deepcopy(config) model_parser_copy.populate(exp_sample_node, exp_sample_config, experimental_sample) # use the experimental parser exclusively if the flag is on if flags.USE_EXPERIMENTAL_PARSER: statically_parsed = self.run_experimental_parser(node) # run the stable static parser unless it is explicitly turned off else: statically_parsed = self.run_static_parser(node) # if the static parser succeeded, extract some data in easy-to-compare formats if isinstance(statically_parsed, dict): # only sample jinja for the purpose of comparing with the stable static parser # if we know we don't need to fall back to jinja (i.e. - nothing to compare # with jinja v jinja). # This means we skip sampling for 40% of the 1/5000 samples. We could run the # sampling rng here, but the effect would be the same since we would only roll # it 40% of the time. So I've opted to keep all the rng code colocated above. if stable_sample and not flags.USE_EXPERIMENTAL_PARSER: # if this will _never_ mutate anything `self` we could avoid these deep copies, # but we can't really guarantee that going forward. model_parser_copy = self.partial_deepcopy() jinja_sample_node = deepcopy(node) jinja_sample_config = deepcopy(config) # rendering mutates the node and the config super(ModelParser, model_parser_copy).render_update( jinja_sample_node, jinja_sample_config ) # update the unrendered config with values from the static parser. # values from yaml files are in there already self.populate(node, config, statically_parsed) # if we took a jinja sample, compare now that the base node has been populated if jinja_sample_node is not None and jinja_sample_config is not None: result = _get_stable_sample_result( jinja_sample_node, jinja_sample_config, node, config ) # if we took an experimental sample, compare now that the base node has been populated if exp_sample_node is not None and exp_sample_config is not None: result = _get_exp_sample_result( exp_sample_node, exp_sample_config, node, config, ) self.manifest._parsing_info.static_analysis_parsed_path_count += 1 # if the static parser didn't succeed, fall back to jinja else: # jinja rendering super().render_update(node, config, validate_config_call_dict=True) # if sampling, add the correct messages for tracking if exp_sample and isinstance(experimental_sample, str): if experimental_sample == "cannot_parse": result += ["01_experimental_parser_cannot_parse"] elif experimental_sample == "has_banned_macro": result += ["08_has_banned_macro"] elif stable_sample and isinstance(statically_parsed, str): if statically_parsed == "cannot_parse": result += ["81_stable_parser_cannot_parse"] elif statically_parsed == "has_banned_macro": result += ["88_has_banned_macro"] # only send the tracking event if there is at least one result code if result: # fire a tracking event. this fires one event for every sample # so that we have data on a per file basis. Not only can we expect # no false positives or misses, we can expect the number model # files parseable by the experimental parser to match our internal # testing. if tracking.active_user is not None: # None in some tests tracking.track_experimental_parser_sample( { "project_id": self.root_project.hashed_name(), "file_id": utils.get_hash(node), "status": result, } ) def run_static_parser(self, node: ModelNode) -> Optional[Union[str, Dict[str, List[Any]]]]: # if any banned macros have been overridden by the user, we cannot use the static parser. if self._has_banned_macro(node): return "has_banned_macro" # run the stable static parser and return the results try: statically_parsed = py_extract_from_source(node.raw_code) return _shift_sources(statically_parsed) # if we want information on what features are barring the static # parser from reading model files, this is where we would add that # since that information is stored in the `ExtractionError`. except ExtractionError: return "cannot_parse" def run_experimental_parser( self, node: ModelNode ) -> Optional[Union[str, Dict[str, List[Any]]]]: # if any banned macros have been overridden by the user, we cannot use the static parser. if self._has_banned_macro(node): return "has_banned_macro" # run the experimental parser and return the results try: # for now, this line calls the stable static parser since there are no # experimental features. Change `py_extract_from_source` to the new # experimental call when we add additional features. experimentally_parsed = py_extract_from_source(node.raw_code) return _shift_sources(experimentally_parsed) # if we want information on what features are barring the experimental # parser from reading model files, this is where we would add that # since that information is stored in the `ExtractionError`. except ExtractionError: return "cannot_parse" # checks for banned macros def _has_banned_macro(self, node: ModelNode) -> bool: # first check if there is a banned macro defined in scope for this model file root_project_name = self.root_project.project_name project_name = node.package_name banned_macros = ["ref", "source", "config"] all_banned_macro_keys: Iterator[str] = chain.from_iterable( map( lambda name: [f"macro.{project_name}.{name}", f"macro.{root_project_name}.{name}"], banned_macros, ) ) return reduce( lambda z, key: z or (key in self.manifest.macros), all_banned_macro_keys, False ) # this method updates the model node rendered and unrendered config as well # as the node object. Used to populate these values when circumventing jinja # rendering like the static parser. def populate(self, node: ModelNode, config: ContextConfig, statically_parsed: Dict[str, Any]): # manually fit configs in config._config_call_dict = _get_config_call_dict(statically_parsed) # if there are hooks present this, it WILL render jinja. Will need to change # when the experimental parser supports hooks self.update_parsed_node_config(node, config, validate_config_call_dict=True) # update the unrendered config with values from the file. # values from yaml files are in there already node.unrendered_config.update(dict(statically_parsed["configs"])) # set refs and sources on the node object refs: List[RefArgs] = [] for ref in statically_parsed["refs"]: name = ref.get("name") package = ref.get("package") version = ref.get("version") refs.append(RefArgs(name, package, version)) node.refs += refs node.sources += statically_parsed["sources"] # configs don't need to be merged into the node because they # are read from config._config_call_dict # the manifest is often huge so this method avoids deepcopying it def partial_deepcopy(self): return ModelParser(deepcopy(self.project), self.manifest, deepcopy(self.root_project)) # pure function. safe to use elsewhere, but unlikely to be useful outside this file. def _get_config_call_dict(static_parser_result: Dict[str, Any]) -> Dict[str, Any]: config_call_dict: Dict[str, Any] = {} for c in static_parser_result["configs"]: merge_config_dicts(config_call_dict, {c[0]: c[1]}) return config_call_dict # TODO if we format sources in the extractor to match this type, we won't need this function. def _shift_sources(static_parser_result: Dict[str, List[Any]]) -> Dict[str, List[Any]]: shifted_result = deepcopy(static_parser_result) source_calls = [] for s in static_parser_result["sources"]: source_calls.append([s[0], s[1]]) shifted_result["sources"] = source_calls return shifted_result # returns a list of string codes to be sent as a tracking event def _get_exp_sample_result( sample_node: ModelNode, sample_config: ContextConfig, node: ModelNode, config: ContextConfig, ) -> List[str]: result: List[Tuple[int, str]] = _get_sample_result(sample_node, sample_config, node, config) def process(codemsg): code, msg = codemsg return f"0{code}_experimental_{msg}" return list(map(process, result)) # returns a list of string codes to be sent as a tracking event def _get_stable_sample_result( sample_node: ModelNode, sample_config: ContextConfig, node: ModelNode, config: ContextConfig, ) -> List[str]: result: List[Tuple[int, str]] = _get_sample_result(sample_node, sample_config, node, config) def process(codemsg): code, msg = codemsg return f"8{code}_stable_{msg}" return list(map(process, result)) # returns a list of string codes that need a single digit prefix to be prepended # before being sent as a tracking event def _get_sample_result( sample_node: ModelNode, sample_config: ContextConfig, node: ModelNode, config: ContextConfig, ) -> List[Tuple[int, str]]: result: List[Tuple[int, str]] = [] # look for false positive configs for k in sample_config._config_call_dict.keys(): if k not in config._config_call_dict.keys(): result += [(2, "false_positive_config_value")] break # look for missed configs for k in config._config_call_dict.keys(): if k not in sample_config._config_call_dict.keys(): result += [(3, "missed_config_value")] break # look for false positive sources for s in sample_node.sources: if s not in node.sources: result += [(4, "false_positive_source_value")] break # look for missed sources for s in node.sources: if s not in sample_node.sources: result += [(5, "missed_source_value")] break # look for false positive refs for r in sample_node.refs: if r not in node.refs: result += [(6, "false_positive_ref_value")] break # look for missed refs for r in node.refs: if r not in sample_node.refs: result += [(7, "missed_ref_value")] break # if there are no errors, return a success value if not result: result = [(0, "exact_match")] return result ================================================ FILE: core/dbt/parser/partial.py ================================================ import os from copy import deepcopy from typing import Callable, Dict, List, MutableMapping, Union from dbt.constants import DEFAULT_ENV_PLACEHOLDER from dbt.contracts.files import ( AnySourceFile, ParseFileType, SchemaSourceFile, SourceFile, parse_file_type_to_parser, ) from dbt.contracts.graph.manifest import Manifest from dbt.contracts.graph.nodes import ( AnalysisNode, GenericTestNode, ModelNode, SeedNode, SnapshotNode, ) from dbt.events.types import PartialParsingEnabled, PartialParsingFile from dbt.node_types import NodeType from dbt_common.context import get_invocation_context from dbt_common.events.base_types import EventLevel from dbt_common.events.functions import fire_event mssat_files = ( ParseFileType.Model, ParseFileType.Seed, ParseFileType.Snapshot, ParseFileType.Analysis, ParseFileType.SingularTest, ) mg_files = ( ParseFileType.Macro, ParseFileType.GenericTest, ) key_to_prefix = { "models": "model", "seeds": "seed", "snapshots": "snapshot", "analyses": "analysis", "sources": "source", } parse_file_type_to_key = { ParseFileType.Model: "models", ParseFileType.Seed: "seeds", ParseFileType.Snapshot: "snapshots", ParseFileType.Analysis: "analyses", } # These macro names have special treatment in the ManifestLoader and # partial parsing. If they have changed we will skip partial parsing special_override_macros = [ "ref", "source", "config", "generate_schema_name", "generate_database_name", "generate_alias_name", "function", ] # Partial parsing. Create a diff of files from saved manifest and current # files and produce a project_parser_file dictionary to drive parsing of # only the necessary changes. # Will produce a 'skip_parsing' method, and a project_parser_file dictionary # All file objects from the new manifest are deepcopied, because we need # to preserve an unchanged file object in case we need to drop back to a # a full parse (such as for certain macro changes) class PartialParsing: def __init__( self, saved_manifest: Manifest, new_files: MutableMapping[str, AnySourceFile] ) -> None: self.saved_manifest = saved_manifest self.new_files = new_files self.project_parser_files: Dict = {} self.saved_files = self.saved_manifest.files self.project_parser_files = {} self.macro_child_map: Dict[str, List[str]] = {} ( self.env_vars_changed_source_files, self.env_vars_changed_schema_files, ) = self.build_env_vars_to_files() self.build_file_diff() self.processing_file = None self.deleted_special_override_macro = False self.disabled_by_file_id = self.saved_manifest.build_disabled_by_file_id() def skip_parsing(self): return ( not self.file_diff["deleted"] and not self.file_diff["added"] and not self.file_diff["changed"] and not self.file_diff["changed_schema_files"] and not self.file_diff["deleted_schema_files"] ) # Compare the previously saved manifest files and the just-loaded manifest # files to see if anything changed def build_file_diff(self): saved_file_ids = set(self.saved_files.keys()) new_file_ids = set(self.new_files.keys()) deleted_all_files = saved_file_ids.difference(new_file_ids) added = new_file_ids.difference(saved_file_ids) common = saved_file_ids.intersection(new_file_ids) changed_or_deleted_macro_file = False # separate out deleted schema files deleted_schema_files = [] deleted = [] for file_id in deleted_all_files: if self.saved_files[file_id].parse_file_type == ParseFileType.Schema: deleted_schema_files.append(file_id) else: if self.saved_files[file_id].parse_file_type in mg_files: changed_or_deleted_macro_file = True deleted.append(file_id) changed = [] changed_schema_files = [] unchanged = [] for file_id in common: if self.saved_files[file_id].checksum == self.new_files[file_id].checksum: unchanged.append(file_id) else: # separate out changed schema files if self.saved_files[file_id].parse_file_type == ParseFileType.Schema: sf = self.saved_files[file_id] if type(sf).__name__ != "SchemaSourceFile": raise Exception(f"Serialization failure for {file_id}") changed_schema_files.append(file_id) else: if self.saved_files[file_id].parse_file_type in mg_files: changed_or_deleted_macro_file = True changed.append(file_id) # handle changed env_vars for non-schema-files for file_id in self.env_vars_changed_source_files: if file_id in deleted or file_id in changed: continue changed.append(file_id) # handle changed env_vars for schema files for file_id in self.env_vars_changed_schema_files.keys(): if file_id in deleted_schema_files or file_id in changed_schema_files: continue changed_schema_files.append(file_id) file_diff = { "deleted": deleted, "deleted_schema_files": deleted_schema_files, "added": added, "changed": changed, "changed_schema_files": changed_schema_files, "unchanged": unchanged, } if changed_or_deleted_macro_file: self.macro_child_map = self.saved_manifest.build_macro_child_map() deleted = len(deleted) + len(deleted_schema_files) changed = len(changed) + len(changed_schema_files) event = PartialParsingEnabled(deleted=deleted, added=len(added), changed=changed) if get_invocation_context().env.get("DBT_PP_TEST"): fire_event(event, level=EventLevel.INFO) else: fire_event(event) self.file_diff = file_diff # generate the list of files that need parsing # uses self.manifest.files generated by 'read_files' def get_parsing_files(self): if self.skip_parsing(): return {} # Need to add new files first, because changes in schema files # might refer to them for file_id in self.file_diff["added"]: self.processing_file = file_id self.add_to_saved(file_id) # Need to process schema files next, because the dictionaries # need to be in place for handling SQL file changes # The reverse sort here is just to ensure that the schema file # processing order test case works, because otherwise the order # of processing the schema files is not guaranteed. self.file_diff["changed_schema_files"].sort(reverse=True) for file_id in self.file_diff["changed_schema_files"]: self.processing_file = file_id self.change_schema_file(file_id) for file_id in self.file_diff["deleted_schema_files"]: self.processing_file = file_id self.delete_schema_file(file_id) for file_id in self.file_diff["deleted"]: self.processing_file = file_id self.delete_from_saved(file_id) for file_id in self.file_diff["changed"]: self.processing_file = file_id self.update_in_saved(file_id) return self.project_parser_files # Add the file to the project parser dictionaries to schedule parsing def add_to_pp_files(self, source_file): file_id = source_file.file_id parser_name = parse_file_type_to_parser[source_file.parse_file_type] project_name = source_file.project_name if not parser_name or not project_name: raise Exception( f"Did not find parse_file_type or project_name " f"in SourceFile for {source_file.file_id}" ) if project_name not in self.project_parser_files: self.project_parser_files[project_name] = {} if parser_name not in self.project_parser_files[project_name]: self.project_parser_files[project_name][parser_name] = [] if ( file_id not in self.project_parser_files[project_name][parser_name] and file_id not in self.file_diff["deleted"] and file_id not in self.file_diff["deleted_schema_files"] ): self.project_parser_files[project_name][parser_name].append(file_id) def already_scheduled_for_parsing(self, source_file): file_id = source_file.file_id project_name = source_file.project_name if project_name not in self.project_parser_files: return False parser_name = parse_file_type_to_parser[source_file.parse_file_type] if parser_name not in self.project_parser_files[project_name]: return False if file_id not in self.project_parser_files[project_name][parser_name]: return False return True # Add new files, including schema files def add_to_saved(self, file_id): # add file object to saved manifest.files source_file = deepcopy(self.new_files[file_id]) if source_file.parse_file_type == ParseFileType.Schema: self.handle_added_schema_file(source_file) self.saved_files[file_id] = source_file # update pp_files to parse self.add_to_pp_files(source_file) fire_event(PartialParsingFile(operation="added", file_id=file_id)) def handle_added_schema_file(self, source_file): source_file.pp_dict = source_file.dict_from_yaml.copy() if "sources" in source_file.pp_dict: for source in source_file.pp_dict["sources"]: # We need to remove the original source, so it can # be properly patched if "overrides" in source: self.remove_source_override_target(source) if "models" in source_file.pp_dict: for model in source_file.pp_dict["models"]: if "versions" in model: self.versioned_model_delete_schema_mssa_links(source_file, "models", model) def delete_disabled(self, unique_id, file_id): # This node/metric/exposure is disabled. Find it and remove it from disabled dictionary. for dis_index, dis_node in enumerate(self.saved_manifest.disabled[unique_id]): if dis_node.file_id == file_id: node = dis_node index = dis_index break # Remove node from disabled del self.saved_manifest.disabled[unique_id][index] # if all nodes were removed for the unique id, delete the unique_id # from the disabled dict if not self.saved_manifest.disabled[unique_id]: self.saved_manifest.disabled.pop(unique_id) return node # Deletes for all non-schema files def delete_from_saved(self, file_id): # Look at all things touched by file, remove those # nodes, and update pp_files to parse unless the # file creating those nodes has also been deleted saved_source_file = self.saved_files[file_id] # SQL file: models, seeds, snapshots, analyses, tests: SQL files, except # macros/tests if saved_source_file.parse_file_type in mssat_files: self.remove_mssat_file(saved_source_file) self.saved_manifest.files.pop(file_id) # macros if saved_source_file.parse_file_type in mg_files: self.delete_macro_file(saved_source_file, follow_references=True) # docs if saved_source_file.parse_file_type == ParseFileType.Documentation: self.delete_doc_node(saved_source_file) # fixtures if saved_source_file.parse_file_type == ParseFileType.Fixture: self.delete_fixture_node(saved_source_file) # functions if saved_source_file.parse_file_type == ParseFileType.Function: self.delete_function_node(saved_source_file) fire_event(PartialParsingFile(operation="deleted", file_id=file_id)) # Updates for non-schema files def update_in_saved(self, file_id): new_source_file = deepcopy(self.new_files[file_id]) old_source_file = self.saved_files[file_id] if new_source_file.parse_file_type in mssat_files: self.update_mssat_in_saved(new_source_file, old_source_file) elif new_source_file.parse_file_type in mg_files: self.update_macro_in_saved(new_source_file, old_source_file) elif new_source_file.parse_file_type == ParseFileType.Documentation: self.update_doc_in_saved(new_source_file, old_source_file) elif new_source_file.parse_file_type == ParseFileType.Fixture: self.update_fixture_in_saved(new_source_file, old_source_file) elif new_source_file.parse_file_type == ParseFileType.Function: self.update_function_in_saved(new_source_file, old_source_file) else: raise Exception(f"Invalid parse_file_type in source_file {file_id}") fire_event(PartialParsingFile(operation="updated", file_id=file_id)) # Models, seeds, snapshots: patches and tests # analyses: patches, no tests # tests: not touched by schema files (no patches, no tests) # Updated schema files should have been processed already. def update_mssat_in_saved(self, new_source_file, old_source_file): if self.already_scheduled_for_parsing(old_source_file): return # These files only have one node except for snapshots unique_ids = [] if old_source_file.nodes: unique_ids = old_source_file.nodes # replace source_file in saved and add to parsing list file_id = new_source_file.file_id self.saved_files[file_id] = deepcopy(new_source_file) self.add_to_pp_files(new_source_file) for unique_id in unique_ids: self.remove_node_in_saved(new_source_file, unique_id) def remove_node_in_saved(self, source_file, unique_id): if unique_id in self.saved_manifest.nodes: # delete node in saved node = self.saved_manifest.nodes.pop(unique_id) elif ( source_file.file_id in self.disabled_by_file_id and unique_id in self.saved_manifest.disabled ): # This node is disabled. Find the node and remove it from disabled dictionary. node = self.delete_disabled(unique_id, source_file.file_id) else: # Has already been deleted by another action return # look at patch_path in model node to see if we need # to reapply a patch from a schema_file. if node.patch_path: file_id = node.patch_path # it might be changed... then what? if ( file_id not in self.file_diff["deleted"] and file_id in self.saved_files and source_file.parse_file_type in parse_file_type_to_key ): # Schema files should already be updated if this comes from a node, # but this code is also called when updating groups and exposures. # This might save the old schema file element, so when the schema file # is processed, it should overwrite it by passing True to "merge_patch" schema_file = self.saved_files[file_id] dict_key = parse_file_type_to_key[source_file.parse_file_type] # look for a matching list dictionary elem_patch = None if dict_key in schema_file.dict_from_yaml: for elem in schema_file.dict_from_yaml[dict_key]: if elem["name"] == node.name: elem_patch = elem break if elem_patch: self.delete_schema_mssa_links(schema_file, dict_key, elem_patch) self.merge_patch(schema_file, dict_key, elem_patch) if unique_id in schema_file.node_patches: schema_file.node_patches.remove(unique_id) if unique_id in self.saved_manifest.disabled: # We have a patch_path in disabled nodes with a patch so # that we can connect the patch to the node for node in self.saved_manifest.disabled[unique_id]: node.patch_path = None def update_macro_in_saved(self, new_source_file, old_source_file): if self.already_scheduled_for_parsing(old_source_file): return self.handle_macro_file_links(old_source_file, follow_references=True) file_id = new_source_file.file_id self.saved_files[file_id] = deepcopy(new_source_file) self.add_to_pp_files(new_source_file) def update_doc_in_saved(self, new_source_file, old_source_file): if self.already_scheduled_for_parsing(old_source_file): return self.delete_doc_node(old_source_file) self.saved_files[new_source_file.file_id] = deepcopy(new_source_file) self.add_to_pp_files(new_source_file) def update_fixture_in_saved(self, new_source_file, old_source_file): if self.already_scheduled_for_parsing(old_source_file): return self.delete_fixture_node(old_source_file) self.saved_files[new_source_file.file_id] = deepcopy(new_source_file) self.add_to_pp_files(new_source_file) def update_function_in_saved( self, new_source_file: SourceFile, old_source_file: SourceFile ) -> None: if self.already_scheduled_for_parsing(old_source_file): return self.delete_function_node(old_source_file) self.saved_files[new_source_file.file_id] = deepcopy(new_source_file) self.add_to_pp_files(new_source_file) def remove_mssat_file(self, source_file: AnySourceFile): # nodes [unique_ids] -- SQL files # There should always be a node for a SQL file if not isinstance(source_file, SourceFile) or not source_file.nodes: return # There is generally only 1 node for SQL files, except for macros and snapshots for unique_id in source_file.nodes: self.remove_node_in_saved(source_file, unique_id) self.schedule_referencing_nodes_for_parsing(unique_id) # We need to re-parse nodes that reference another removed node def schedule_referencing_nodes_for_parsing(self, unique_id): # Look at "children", i.e. nodes that reference this node if unique_id in self.saved_manifest.child_map: self.schedule_nodes_for_parsing(self.saved_manifest.child_map[unique_id]) def schedule_nodes_for_parsing(self, unique_ids): for unique_id in unique_ids: if unique_id in self.saved_manifest.nodes: node = self.saved_manifest.nodes[unique_id] if node.resource_type == NodeType.Test and node.test_node_type == "generic": # test nodes are handled separately. Must be removed from schema file continue file_id = node.file_id if file_id in self.saved_files and file_id not in self.file_diff["deleted"]: source_file = self.saved_files[file_id] self.remove_mssat_file(source_file) # content of non-schema files is only in new files self.saved_files[file_id] = deepcopy(self.new_files[file_id]) self.add_to_pp_files(self.saved_files[file_id]) elif unique_id in self.saved_manifest.sources: source = self.saved_manifest.sources[unique_id] self._schedule_for_parsing( "sources", source, source.source_name, self.delete_schema_source ) elif unique_id in self.saved_manifest.exposures: exposure = self.saved_manifest.exposures[unique_id] self._schedule_for_parsing( "exposures", exposure, exposure.name, self.delete_schema_exposure ) elif unique_id in self.saved_manifest.metrics: metric = self.saved_manifest.metrics[unique_id] self._schedule_for_parsing( "metrics", metric, metric.name, self.delete_schema_metric ) elif unique_id in self.saved_manifest.semantic_models: semantic_model = self.saved_manifest.semantic_models[unique_id] self._schedule_for_parsing( "semantic_models", semantic_model, semantic_model.name, self.delete_schema_semantic_model, ) elif unique_id in self.saved_manifest.saved_queries: saved_query = self.saved_manifest.saved_queries[unique_id] self._schedule_for_parsing( "saved_queries", saved_query, saved_query.name, self.delete_schema_saved_query ) elif unique_id in self.saved_manifest.macros: macro = self.saved_manifest.macros[unique_id] file_id = macro.file_id if file_id in self.saved_files and file_id not in self.file_diff["deleted"]: source_file = self.saved_files[file_id] self.delete_macro_file(source_file) self.saved_files[file_id] = deepcopy(self.new_files[file_id]) self.add_to_pp_files(self.saved_files[file_id]) elif unique_id in self.saved_manifest.unit_tests: unit_test = self.saved_manifest.unit_tests[unique_id] self._schedule_for_parsing( "unit_tests", unit_test, unit_test.name, self.delete_schema_unit_test ) def _schedule_for_parsing(self, dict_key: str, element, name, delete: Callable) -> None: file_id = element.file_id if ( file_id in self.saved_files and file_id not in self.file_diff["deleted"] and file_id not in self.file_diff["deleted_schema_files"] ): schema_file = self.saved_files[file_id] elements = [] assert isinstance(schema_file, SchemaSourceFile) if dict_key in schema_file.dict_from_yaml: elements = schema_file.dict_from_yaml[dict_key] schema_element = self.get_schema_element(elements, name) if schema_element: delete(schema_file, schema_element) self.merge_patch(schema_file, dict_key, schema_element) def delete_macro_file(self, source_file, follow_references=False): self.check_for_special_deleted_macros(source_file) self.handle_macro_file_links(source_file, follow_references) file_id = source_file.file_id # It's not clear when this file_id would not exist in saved_files if file_id in self.saved_files: self.saved_files.pop(file_id) def check_for_special_deleted_macros(self, source_file): for unique_id in source_file.macros: if unique_id in self.saved_manifest.macros: package_name = unique_id.split(".")[1] if package_name == "dbt": continue macro = self.saved_manifest.macros[unique_id] if macro.name in special_override_macros: self.deleted_special_override_macro = True def recursively_gather_macro_references(self, macro_unique_id, referencing_nodes): for unique_id in self.macro_child_map[macro_unique_id]: if unique_id in referencing_nodes: continue referencing_nodes.append(unique_id) if unique_id.startswith("macro."): self.recursively_gather_macro_references(unique_id, referencing_nodes) def handle_macro_file_links(self, source_file, follow_references=False): # remove the macros in the 'macros' dictionary macros = source_file.macros.copy() for unique_id in macros: if unique_id not in self.saved_manifest.macros: # This happens when a macro has already been removed if unique_id in source_file.macros: source_file.macros.remove(unique_id) continue base_macro = self.saved_manifest.macros.pop(unique_id) # Recursively check children of this macro # The macro_child_map might not exist if a macro is removed by # schedule_nodes_for parsing. We only want to follow # references if the macro file itself has been updated or # deleted, not if we're just updating referenced nodes. if self.macro_child_map and follow_references: referencing_nodes = [] self.recursively_gather_macro_references(unique_id, referencing_nodes) self.schedule_macro_nodes_for_parsing(referencing_nodes) if base_macro.patch_path: file_id = base_macro.patch_path if file_id in self.saved_files: schema_file = self.saved_files[file_id] macro_patches = [] if "macros" in schema_file.dict_from_yaml: macro_patches = schema_file.dict_from_yaml["macros"] macro_patch = self.get_schema_element(macro_patches, base_macro.name) self.delete_schema_macro_patch(schema_file, macro_patch) self.merge_patch(schema_file, "macros", macro_patch) # The macro may have already been removed by handling macro children if unique_id in source_file.macros: source_file.macros.remove(unique_id) # similar to schedule_nodes_for_parsing but doesn't do sources and exposures # and handles schema tests def schedule_macro_nodes_for_parsing(self, unique_ids): for unique_id in unique_ids: if unique_id in self.saved_manifest.nodes: node = self.saved_manifest.nodes[unique_id] # Both generic tests from yaml files and singular tests have NodeType.Test # so check for generic test. if node.resource_type == NodeType.Test and node.test_node_type == "generic": schema_file_id = node.file_id schema_file = self.saved_manifest.files[schema_file_id] (key, name) = schema_file.get_key_and_name_for_test(node.unique_id) if key and name: patch_list = [] if key in schema_file.dict_from_yaml: patch_list = schema_file.dict_from_yaml[key] patch = self.get_schema_element(patch_list, name) if patch: if key in ["models", "seeds", "snapshots"]: self.delete_schema_mssa_links(schema_file, key, patch) self.merge_patch(schema_file, key, patch) if unique_id in schema_file.node_patches: schema_file.node_patches.remove(unique_id) elif key == "sources": # re-schedule source if "overrides" in patch: # This is a source patch; need to re-parse orig source self.remove_source_override_target(patch) self.delete_schema_source(schema_file, patch) self.merge_patch(schema_file, "sources", patch) else: file_id = node.file_id if file_id in self.saved_files and file_id not in self.file_diff["deleted"]: source_file = self.saved_files[file_id] self.remove_mssat_file(source_file) # content of non-schema files is only in new files self.saved_files[file_id] = deepcopy(self.new_files[file_id]) self.add_to_pp_files(self.saved_files[file_id]) elif unique_id in self.saved_manifest.macros: macro = self.saved_manifest.macros[unique_id] file_id = macro.file_id if file_id in self.saved_files and file_id not in self.file_diff["deleted"]: source_file = self.saved_files[file_id] self.delete_macro_file(source_file) self.saved_files[file_id] = deepcopy(self.new_files[file_id]) self.add_to_pp_files(self.saved_files[file_id]) def delete_doc_node(self, source_file): # remove the nodes in the 'docs' dictionary docs = source_file.docs.copy() for unique_id in docs: self.saved_manifest.docs.pop(unique_id) source_file.docs.remove(unique_id) # The unique_id of objects that contain a doc call are stored in the # doc source_file.nodes self.schedule_nodes_for_parsing(source_file.nodes) source_file.nodes = [] # Remove the file object self.saved_manifest.files.pop(source_file.file_id) def delete_fixture_node(self, source_file): # remove fixtures from the "fixtures" dictionary fixture_unique_id = source_file.fixture self.saved_manifest.fixtures.pop(fixture_unique_id) unit_tests = source_file.unit_tests.copy() for unique_id in unit_tests: unit_test = self.saved_manifest.unit_tests.pop(unique_id) # schedule unit_test for parsing self._schedule_for_parsing( "unit_tests", unit_test, unit_test.name, self.delete_schema_unit_test ) source_file.unit_tests.remove(unique_id) self.saved_manifest.files.pop(source_file.file_id) def delete_function_node(self, source_file: SourceFile) -> None: # There should always be a node for a Function file if not isinstance(source_file, SourceFile) or not source_file.functions: return # There can only be one node of a function function_unique_id = source_file.functions[0] # Remove the function node from the saved manifest function_node = self.saved_manifest.functions.pop(function_unique_id) # Remove the function node from the source file so that it's not viewed as a # duplicate when it's re-added source_file.functions.remove(function_unique_id) # If this function had a schema patch, schedule that schema element to be reapplied. patch_path = function_node.patch_path if ( patch_path is not None and patch_path in self.saved_files and patch_path not in self.file_diff["deleted_schema_files"] ): schema_file = self.saved_files[patch_path] # Only proceed if this is a schema file if isinstance(schema_file, SchemaSourceFile): elements = schema_file.dict_from_yaml.get("functions", []) schema_element = self.get_schema_element(elements, function_node.name) if schema_element: # Remove any previous links and re-merge the patch to pp_dict so it gets reparsed self.delete_schema_function(schema_file, schema_element) self.merge_patch(schema_file, "functions", schema_element) # Finally, remove the deleted function file from saved files if source_file.file_id in self.saved_manifest.files: self.saved_manifest.files.pop(source_file.file_id) # Schema files ----------------------- # Changed schema files def change_schema_file(self, file_id): saved_schema_file = self.saved_files[file_id] new_schema_file = deepcopy(self.new_files[file_id]) saved_yaml_dict = saved_schema_file.dict_from_yaml new_yaml_dict = new_schema_file.dict_from_yaml if saved_schema_file.pp_dict is None: saved_schema_file.pp_dict = {} self.handle_schema_file_changes(saved_schema_file, saved_yaml_dict, new_yaml_dict) # copy from new schema_file to saved_schema_file to preserve references # that weren't removed saved_schema_file.contents = new_schema_file.contents saved_schema_file.checksum = new_schema_file.checksum saved_schema_file.dfy = new_schema_file.dfy # schedule parsing self.add_to_pp_files(saved_schema_file) # schema_file pp_dict should have been generated already fire_event(PartialParsingFile(operation="updated", file_id=file_id)) # Delete schema files -- a variation on change_schema_file def delete_schema_file(self, file_id): saved_schema_file = self.saved_files[file_id] saved_yaml_dict = saved_schema_file.dict_from_yaml new_yaml_dict = {} self.handle_schema_file_changes(saved_schema_file, saved_yaml_dict, new_yaml_dict) self.saved_manifest.files.pop(file_id) # For each key in a schema file dictionary, process the changed, deleted, and added # elements for the key lists def handle_schema_file_changes(self, schema_file, saved_yaml_dict, new_yaml_dict): # loop through comparing previous dict_from_yaml with current dict_from_yaml # Need to do the deleted/added/changed thing, just like the files lists env_var_changes = {} if schema_file.file_id in self.env_vars_changed_schema_files: env_var_changes = self.env_vars_changed_schema_files[schema_file.file_id] # models, seeds, snapshots, analyses for dict_key in ["models", "seeds", "snapshots", "analyses"]: key_diff = self.get_diff_for(dict_key, saved_yaml_dict, new_yaml_dict) if key_diff["changed"]: for elem in key_diff["changed"]: if dict_key == "snapshots" and "relation" in elem: self.delete_yaml_snapshot(schema_file, elem) self.delete_schema_mssa_links(schema_file, dict_key, elem) self.merge_patch(schema_file, dict_key, elem, True) if key_diff["deleted"]: for elem in key_diff["deleted"]: if dict_key == "snapshots" and "relation" in elem: self.delete_yaml_snapshot(schema_file, elem) self.delete_schema_mssa_links(schema_file, dict_key, elem) if key_diff["added"]: for elem in key_diff["added"]: if dict_key == "models" and "versions" in elem: self.versioned_model_delete_schema_mssa_links(schema_file, dict_key, elem) self.merge_patch(schema_file, dict_key, elem, True) # Handle schema file updates due to env_var changes if dict_key in env_var_changes and dict_key in new_yaml_dict: for name in env_var_changes[dict_key]: if name in key_diff["changed_or_deleted_names"]: continue elem = self.get_schema_element(new_yaml_dict[dict_key], name) if elem: if dict_key == "snapshots" and "relation" in elem: self.delete_yaml_snapshot(schema_file, elem) self.delete_schema_mssa_links(schema_file, dict_key, elem) self.merge_patch(schema_file, dict_key, elem, True) # sources dict_key = "sources" source_diff = self.get_diff_for(dict_key, saved_yaml_dict, new_yaml_dict) if source_diff["changed"]: for source in source_diff["changed"]: if "overrides" in source: # This is a source patch; need to re-parse orig source self.remove_source_override_target(source) self.delete_schema_source(schema_file, source) self.merge_patch(schema_file, dict_key, source, True) if source_diff["deleted"]: for source in source_diff["deleted"]: if "overrides" in source: # This is a source patch; need to re-parse orig source self.remove_source_override_target(source) self.delete_schema_source(schema_file, source) if source_diff["added"]: for source in source_diff["added"]: if "overrides" in source: # This is a source patch; need to re-parse orig source self.remove_source_override_target(source) self.merge_patch(schema_file, dict_key, source, True) # Handle schema file updates due to env_var changes if dict_key in env_var_changes and dict_key in new_yaml_dict: for name in env_var_changes[dict_key]: if name in source_diff["changed_or_deleted_names"]: continue source = self.get_schema_element(new_yaml_dict[dict_key], name) if source: if "overrides" in source: self.remove_source_override_target(source) self.delete_schema_source(schema_file, source) self.merge_patch(schema_file, dict_key, source, True) def handle_change(key: str, delete: Callable): self._handle_element_change( schema_file, saved_yaml_dict, new_yaml_dict, env_var_changes, key, delete ) handle_change("macros", self.delete_schema_macro_patch) handle_change("exposures", self.delete_schema_exposure) handle_change("metrics", self.delete_schema_metric) handle_change("groups", self.delete_schema_group) handle_change("semantic_models", self.delete_schema_semantic_model) handle_change("unit_tests", self.delete_schema_unit_test) handle_change("saved_queries", self.delete_schema_saved_query) handle_change("data_tests", self.delete_schema_data_test_patch) handle_change("functions", self.delete_schema_function) def _handle_element_change( self, schema_file, saved_yaml_dict, new_yaml_dict, env_var_changes, dict_key: str, delete ): element_diff = self.get_diff_for(dict_key, saved_yaml_dict, new_yaml_dict) if element_diff["changed"]: for element in element_diff["changed"]: delete(schema_file, element) self.merge_patch(schema_file, dict_key, element, True) if element_diff["deleted"]: for element in element_diff["deleted"]: delete(schema_file, element) if element_diff["added"]: for element in element_diff["added"]: self.merge_patch(schema_file, dict_key, element, True) # Handle schema file updates due to env_var changes if dict_key in env_var_changes and dict_key in new_yaml_dict: for name in env_var_changes[dict_key]: if name in element_diff["changed_or_deleted_names"]: continue elem = self.get_schema_element(new_yaml_dict[dict_key], name) if elem: delete(schema_file, elem) self.merge_patch(schema_file, dict_key, elem, True) # Take a "section" of the schema file yaml dictionary from saved and new schema files # and determine which parts have changed def get_diff_for(self, key, saved_yaml_dict, new_yaml_dict): if key in saved_yaml_dict or key in new_yaml_dict: saved_elements = saved_yaml_dict[key] if key in saved_yaml_dict else [] new_elements = new_yaml_dict[key] if key in new_yaml_dict else [] else: return {"deleted": [], "added": [], "changed": []} # for each set of keys, need to create a dictionary of names pointing to entry saved_elements_by_name = {} new_elements_by_name = {} # sources have two part names? for element in saved_elements: saved_elements_by_name[element["name"]] = element for element in new_elements: new_elements_by_name[element["name"]] = element # now determine which elements, by name, are added, deleted or changed saved_element_names = set(saved_elements_by_name.keys()) new_element_names = set(new_elements_by_name.keys()) deleted = saved_element_names.difference(new_element_names) added = new_element_names.difference(saved_element_names) common = saved_element_names.intersection(new_element_names) changed = [] for element_name in common: if saved_elements_by_name[element_name] != new_elements_by_name[element_name]: changed.append(element_name) # make lists of yaml elements to return as diffs deleted_elements = [saved_elements_by_name[name].copy() for name in deleted] added_elements = [new_elements_by_name[name].copy() for name in added] changed_elements = [new_elements_by_name[name].copy() for name in changed] diff = { "deleted": deleted_elements, "added": added_elements, "changed": changed_elements, "changed_or_deleted_names": list(changed) + list(deleted), } return diff # Merge a patch file into the pp_dict in a schema file. The "new_patch" # flag indicates that we're processing a schema file, so if a matching # patch has already been scheduled, replace it. def merge_patch(self, schema_file, key, patch, new_patch=False): if schema_file.pp_dict is None: schema_file.pp_dict = {} pp_dict = schema_file.pp_dict if key not in pp_dict: pp_dict[key] = [patch] else: # check that this patch hasn't already been saved found_elem = None for elem in pp_dict[key]: if elem["name"] == patch["name"]: found_elem = elem if not found_elem: pp_dict[key].append(patch) elif found_elem and new_patch: # remove patch and replace with new one pp_dict[key].remove(found_elem) pp_dict[key].append(patch) schema_file.delete_from_env_vars(key, patch["name"]) schema_file.delete_from_unrendered_configs(key, patch["name"]) self.add_to_pp_files(schema_file) # For model, seed, snapshot, analysis schema dictionary keys, # delete the patches and tests from the patch def delete_schema_mssa_links(self, schema_file, dict_key, elem) -> None: # find elem node unique_id in node_patches prefix = key_to_prefix[dict_key] elem_unique_ids = [] for unique_id in schema_file.node_patches: if not unique_id.startswith(prefix): continue parts = unique_id.split(".") elem_name = parts[2] if elem_name == elem["name"]: elem_unique_ids.append(unique_id) self._delete_schema_mssa_links(schema_file, dict_key, elem, elem_unique_ids) def versioned_model_delete_schema_mssa_links(self, schema_file, dict_key, elem) -> None: elem_unique_ids = [] # We need to look up possible existing models that this new or modified patch applies to unique_id = f"model.{schema_file.project_name}.{elem['name']}" if unique_id in self.saved_manifest.nodes: elem_unique_ids.append(unique_id) if not elem_unique_ids: return self._delete_schema_mssa_links(schema_file, dict_key, elem, elem_unique_ids) def _delete_schema_mssa_links(self, schema_file, dict_key, elem, elem_unique_ids): # remove elem node and remove unique_id from node_patches for elem_unique_id in elem_unique_ids: # might have been already removed # For all-yaml snapshots, we don't do this, since the node # should have already been removed. if ( elem_unique_id in self.saved_manifest.nodes or elem_unique_id in self.saved_manifest.disabled ): nodes: List[Union[ModelNode, SeedNode, SnapshotNode, AnalysisNode]] = [] if elem_unique_id in self.saved_manifest.nodes: nodes = [self.saved_manifest.nodes.pop(elem_unique_id)] # type: ignore[list-item] else: # The value of disabled items is a list of nodes nodes = self.saved_manifest.disabled.pop(elem_unique_id) # type: ignore[assignment] # need to add the node source_file to pp_files for node in nodes: file_id = node.file_id # need to copy new file to saved files in order to get content if file_id in self.new_files: self.saved_files[file_id] = deepcopy(self.new_files[file_id]) if self.saved_files[file_id]: source_file = self.saved_files[file_id] self.add_to_pp_files(source_file) # if the node's group has changed - need to reparse all referencing nodes to ensure valid ref access if node.group != elem.get("group"): self.schedule_referencing_nodes_for_parsing(node.unique_id) # If the latest version has changed, a version has been removed, or a version has been added, # we need to reparse referencing nodes. if node.is_versioned or elem.get("versions"): self.schedule_referencing_nodes_for_parsing(node.unique_id) # remove from patches # For versioned models, the schedule_referencing_nodes_for_parsing call above # could have caused a recursive visit to this file. if elem_unique_id in schema_file.node_patches: schema_file.node_patches.remove(elem_unique_id) # for models, seeds, snapshots (not analyses) if dict_key in ["models", "seeds", "snapshots"]: # find related tests and remove them self.remove_tests(schema_file, dict_key, elem["name"]) def remove_tests(self, schema_file, dict_key, name): tests = schema_file.get_tests(dict_key, name) for test_unique_id in tests: if test_unique_id in self.saved_manifest.nodes: self.saved_manifest.nodes.pop(test_unique_id) schema_file.remove_tests(dict_key, name) # We also need to remove tests in other schema files that # reference this node. unique_id = f"{key_to_prefix[dict_key]}.{schema_file.project_name}.{name}" if unique_id in self.saved_manifest.child_map: for child_id in self.saved_manifest.child_map[unique_id]: if child_id.startswith("test") and child_id in self.saved_manifest.nodes: child_test = self.saved_manifest.nodes[child_id] if isinstance(child_test, GenericTestNode) and child_test.attached_node: if child_test.attached_node in self.saved_manifest.nodes: attached_node = self.saved_manifest.nodes[child_test.attached_node] self.update_in_saved(attached_node.file_id) def delete_yaml_snapshot(self, schema_file, snapshot_dict): snapshot_name = snapshot_dict["name"] snapshots = schema_file.snapshots.copy() for unique_id in snapshots: if unique_id in self.saved_manifest.nodes: snapshot = self.saved_manifest.nodes[unique_id] if snapshot.name == snapshot_name: self.saved_manifest.nodes.pop(unique_id) schema_file.snapshots.remove(unique_id) elif unique_id in self.saved_manifest.disabled: self.delete_disabled(unique_id, schema_file.file_id) schema_file.snapshots.remove(unique_id) def delete_schema_source(self, schema_file, source_dict): # both patches, tests, and source nodes source_name = source_dict["name"] # There may be multiple sources for each source dict, since # there will be a separate source node for each table. # SourceDefinition name = table name, dict name is source_name sources = schema_file.sources.copy() for unique_id in sources: if unique_id in self.saved_manifest.sources: source = self.saved_manifest.sources[unique_id] if source.source_name == source_name: source = self.saved_manifest.sources.pop(unique_id) schema_file.sources.remove(unique_id) self.schedule_referencing_nodes_for_parsing(unique_id) self.remove_tests(schema_file, "sources", source_name) def delete_schema_macro_patch(self, schema_file, macro): # This is just macro patches that need to be reapplied macro_unique_id = None if macro["name"] in schema_file.macro_patches: macro_unique_id = schema_file.macro_patches[macro["name"]] del schema_file.macro_patches[macro["name"]] # Need to delete all macros in the same file # and then reapply all schema file updates for those macros if macro_unique_id and macro_unique_id in self.saved_manifest.macros: macro = self.saved_manifest.macros.pop(macro_unique_id) macro_file_id = macro.file_id if macro_file_id in self.new_files: source_file = self.saved_files[macro_file_id] self.delete_macro_file(source_file) self.saved_files[macro_file_id] = deepcopy(self.new_files[macro_file_id]) self.add_to_pp_files(self.saved_files[macro_file_id]) def delete_schema_data_test_patch(self, schema_file, data_test): data_test_unique_id = None for unique_id in schema_file.node_patches: if not unique_id.startswith("test"): continue parts = unique_id.split(".") elem_name = parts[2] if elem_name == data_test["name"]: data_test_unique_id = unique_id break if data_test_unique_id and data_test_unique_id in self.saved_manifest.nodes: singular_data_test = self.saved_manifest.nodes.pop(data_test_unique_id) file_id = singular_data_test.file_id if file_id in self.new_files: self.saved_files[file_id] = deepcopy(self.new_files[file_id]) self.add_to_pp_files(self.saved_files[file_id]) # exposures are created only from schema files, so just delete # the exposure or the disabled exposure. def delete_schema_exposure(self, schema_file, exposure_dict): exposure_name = exposure_dict["name"] exposures = schema_file.exposures.copy() for unique_id in exposures: if unique_id in self.saved_manifest.exposures: exposure = self.saved_manifest.exposures[unique_id] if exposure.name == exposure_name: self.saved_manifest.exposures.pop(unique_id) schema_file.exposures.remove(unique_id) elif unique_id in self.saved_manifest.disabled: self.delete_disabled(unique_id, schema_file.file_id) # groups are created only from schema files, so just delete the group def delete_schema_group(self, schema_file, group_dict): group_name = group_dict["name"] groups = schema_file.groups.copy() for unique_id in groups: if unique_id in self.saved_manifest.groups: group = self.saved_manifest.groups[unique_id] if group.name == group_name: self.schedule_nodes_for_parsing(self.saved_manifest.group_map[group.name]) self.saved_manifest.groups.pop(unique_id) schema_file.groups.remove(unique_id) # metrics are created only from schema files, but also can be referred to by other nodes def delete_schema_metric(self, schema_file, metric_dict): metric_name = metric_dict["name"] metrics = schema_file.metrics.copy() for unique_id in metrics: if unique_id in self.saved_manifest.metrics: metric = self.saved_manifest.metrics[unique_id] if metric.name == metric_name: # Need to find everything that referenced this metric and schedule for parsing if unique_id in self.saved_manifest.child_map: self.schedule_nodes_for_parsing(self.saved_manifest.child_map[unique_id]) self.saved_manifest.metrics.pop(unique_id) schema_file.metrics.remove(unique_id) elif unique_id in self.saved_manifest.disabled: self.delete_disabled(unique_id, schema_file.file_id) def delete_schema_saved_query(self, schema_file, saved_query_dict): saved_query_name = saved_query_dict["name"] saved_queries = schema_file.saved_queries.copy() for unique_id in saved_queries: if unique_id in self.saved_manifest.saved_queries: saved_query = self.saved_manifest.saved_queries[unique_id] if saved_query.name == saved_query_name: # Need to find everything that referenced this saved_query and schedule for parsing if unique_id in self.saved_manifest.child_map: self.schedule_nodes_for_parsing(self.saved_manifest.child_map[unique_id]) self.saved_manifest.saved_queries.pop(unique_id) elif unique_id in self.saved_manifest.disabled: self.delete_disabled(unique_id, schema_file.file_id) def delete_schema_semantic_model(self, schema_file, semantic_model_dict): semantic_model_name = semantic_model_dict["name"] semantic_models = schema_file.semantic_models.copy() for unique_id in semantic_models: if unique_id in self.saved_manifest.semantic_models: semantic_model = self.saved_manifest.semantic_models[unique_id] if semantic_model.name == semantic_model_name: # Need to find everything that referenced this semantic model and schedule for parsing if unique_id in self.saved_manifest.child_map: self.schedule_nodes_for_parsing(self.saved_manifest.child_map[unique_id]) self.saved_manifest.semantic_models.pop(unique_id) schema_file.semantic_models.remove(unique_id) elif unique_id in self.saved_manifest.disabled: self.delete_disabled(unique_id, schema_file.file_id) if schema_file.generated_metrics: # If this partial parse file has an old "generated_metrics" list, # call code to fix it up before processing. schema_file.fix_metrics_from_measures() if semantic_model_name in schema_file.metrics_from_measures: for unique_id in schema_file.metrics_from_measures[semantic_model_name]: if unique_id in self.saved_manifest.metrics: self.saved_manifest.metrics.pop(unique_id) elif unique_id in self.saved_manifest.disabled: self.delete_disabled(unique_id, schema_file.file_id) del schema_file.metrics_from_measures[semantic_model_name] def delete_schema_unit_test(self, schema_file, unit_test_dict): unit_test_name = unit_test_dict["name"] unit_tests = schema_file.unit_tests.copy() for unique_id in unit_tests: if unique_id in self.saved_manifest.unit_tests: unit_test = self.saved_manifest.unit_tests[unique_id] if unit_test.name == unit_test_name: self.saved_manifest.unit_tests.pop(unique_id) schema_file.unit_tests.remove(unique_id) # No disabled unit tests yet def delete_schema_function(self, schema_file: SchemaSourceFile, function_dict: dict) -> None: function_name = function_dict["name"] functions = schema_file.node_patches.copy() for unique_id in functions: if unique_id in self.saved_manifest.functions: function = self.saved_manifest.functions[unique_id] if function.name == function_name: removed_function = self.saved_manifest.functions.pop(unique_id) # For schema patches, recorded unique_ids live in node_patches (ndp) if unique_id in schema_file.node_patches: schema_file.node_patches.remove(unique_id) # Schedule the function's SQL file for reparsing so the node is re-added file_id = removed_function.file_id if file_id and file_id in self.new_files: self.saved_files[file_id] = deepcopy(self.new_files[file_id]) if file_id and file_id in self.saved_files: self.add_to_pp_files(self.saved_files[file_id]) def get_schema_element(self, elem_list, elem_name): for element in elem_list: if "name" in element and element["name"] == elem_name: return element return None def get_schema_file_for_source(self, package_name, source_name): schema_file = None for source in self.saved_manifest.sources.values(): if source.package_name == package_name and source.source_name == source_name: file_id = source.file_id if file_id in self.saved_files: schema_file = self.saved_files[file_id] break return schema_file def get_source_override_file_and_dict(self, source): package = source["overrides"] source_name = source["name"] orig_source_schema_file = self.get_schema_file_for_source(package, source_name) orig_sources = orig_source_schema_file.dict_from_yaml["sources"] orig_source = self.get_schema_element(orig_sources, source_name) return (orig_source_schema_file, orig_source) def remove_source_override_target(self, source_dict): (orig_file, orig_source) = self.get_source_override_file_and_dict(source_dict) if orig_source: self.delete_schema_source(orig_file, orig_source) self.merge_patch(orig_file, "sources", orig_source) self.add_to_pp_files(orig_file) # This builds a dictionary of files that need to be scheduled for parsing # because the env var has changed. # source_files # env_vars_changed_source_files: [file_id, file_id...] # schema_files # env_vars_changed_schema_files: {file_id: {"yaml_key": [name, ..]}} def build_env_vars_to_files(self): unchanged_vars = [] changed_vars = [] delete_vars = [] # Check whether the env_var has changed and add it to # an unchanged or changed list for env_var in self.saved_manifest.env_vars: prev_value = self.saved_manifest.env_vars[env_var] current_value = os.getenv(env_var) if current_value is None: # This will be true when depending on the default value. # We store env vars set by defaults as a static string so we can recognize they have # defaults. We depend on default changes triggering reparsing by file change. If # the file has not changed we can assume the default has not changed. if prev_value == DEFAULT_ENV_PLACEHOLDER: unchanged_vars.append(env_var) continue # env_var no longer set, remove from manifest delete_vars.append(env_var) if prev_value == current_value: unchanged_vars.append(env_var) else: # prev_value != current_value changed_vars.append(env_var) for env_var in delete_vars: del self.saved_manifest.env_vars[env_var] env_vars_changed_source_files = [] env_vars_changed_schema_files = {} # The SourceFiles contain a list of env_vars that were used in the file. # The SchemaSourceFiles contain a dictionary of yaml_key to schema entry names to # a list of vars. # Create a list of file_ids for source_files that need to be reparsed, and # a dictionary of file_ids to yaml_keys to names. for source_file in self.saved_files.values(): if source_file.parse_file_type == ParseFileType.Fixture: continue file_id = source_file.file_id if not source_file.env_vars: continue if source_file.parse_file_type == ParseFileType.Schema: for yaml_key in source_file.env_vars.keys(): for name in source_file.env_vars[yaml_key].keys(): for env_var in source_file.env_vars[yaml_key][name]: if env_var in changed_vars: if file_id not in env_vars_changed_schema_files: env_vars_changed_schema_files[file_id] = {} if yaml_key not in env_vars_changed_schema_files[file_id]: env_vars_changed_schema_files[file_id][yaml_key] = [] if name not in env_vars_changed_schema_files[file_id][yaml_key]: env_vars_changed_schema_files[file_id][yaml_key].append(name) break # if one env_var is changed we can stop else: for env_var in source_file.env_vars: if env_var in changed_vars: env_vars_changed_source_files.append(file_id) break # if one env_var is changed we can stop return (env_vars_changed_source_files, env_vars_changed_schema_files) ================================================ FILE: core/dbt/parser/read_files.py ================================================ import os import pathlib from dataclasses import dataclass, field from typing import Dict, List, Mapping, MutableMapping, Optional, Protocol import pathspec # type: ignore from dbt.config import Project from dbt.contracts.files import ( AnySourceFile, FileHash, FilePath, FixtureSourceFile, ParseFileType, SchemaSourceFile, SourceFile, ) from dbt.events.types import InputFileDiffError from dbt.exceptions import ParsingError from dbt.parser.common import schema_file_keys from dbt.parser.schemas import yaml_from_file from dbt.parser.search import filesystem_search from dbt_common.clients.system import load_file_contents from dbt_common.dataclass_schema import dbtClassMixin from dbt_common.events.functions import fire_event @dataclass class InputFile(dbtClassMixin): path: str content: str modification_time: float = 0.0 @dataclass class FileDiff(dbtClassMixin): deleted: List[str] # Note: it would be possible to not distinguish between # added and changed files, but we would lose some error handling. changed: List[InputFile] added: List[InputFile] def normalize_file_contents(contents: str) -> str: """Normalize file contents by compacting whitespace and newlines to a single whitespace. This ensures consistent checksums regardless of formatting differences. """ return " ".join(contents.split()) # This loads the files contents and creates the SourceFile object def load_source_file( path: FilePath, parse_file_type: ParseFileType, project_name: str, saved_files, ) -> Optional[AnySourceFile]: if parse_file_type == ParseFileType.Schema: sf_cls = SchemaSourceFile elif parse_file_type == ParseFileType.Fixture: sf_cls = FixtureSourceFile # type:ignore[assignment] else: sf_cls = SourceFile # type:ignore[assignment] source_file = sf_cls( path=path, checksum=FileHash.empty(), parse_file_type=parse_file_type, project_name=project_name, ) skip_loading_schema_file = False if ( parse_file_type == ParseFileType.Schema and saved_files and source_file.file_id in saved_files ): old_source_file = saved_files[source_file.file_id] if ( source_file.path.modification_time != 0.0 and old_source_file.path.modification_time == source_file.path.modification_time ): source_file.checksum = old_source_file.checksum source_file.dfy = old_source_file.dfy skip_loading_schema_file = True if not skip_loading_schema_file: # We strip the file_contents before generating the checksum because we want # the checksum to match the stored file contents file_contents = load_file_contents(path.absolute_path, strip=True) source_file.contents = file_contents normalized_contents = normalize_file_contents(file_contents) source_file.checksum = FileHash.from_contents(normalized_contents) if parse_file_type == ParseFileType.Schema and source_file.contents: dfy = yaml_from_file(source_file=source_file, validate=True) if dfy: validate_yaml(source_file.path.original_file_path, dfy) source_file.dfy = dfy return source_file # Do some minimal validation of the yaml in a schema file. # Check version, that key values are lists and that each element in # the lists has a 'name' key def validate_yaml(file_path, dct): for key in schema_file_keys: if key in dct: if not isinstance(dct[key], list): msg = ( f"The schema file at {file_path} is " f"invalid because the value of '{key}' is not a list" ) raise ParsingError(msg) for element in dct[key]: if not isinstance(element, dict): msg = ( f"The schema file at {file_path} is " f"invalid because a list element for '{key}' is not a dictionary" ) raise ParsingError(msg) if "name" not in element: msg = ( f"The schema file at {file_path} is " f"invalid because a list element for '{key}' does not have a " "name attribute." ) raise ParsingError(msg) # Special processing for big seed files def load_seed_source_file(match: FilePath, project_name) -> SourceFile: if match.seed_too_large(): # We don't want to calculate a hash of this file. Use the path. source_file = SourceFile.big_seed(match) else: file_contents = load_file_contents(match.absolute_path, strip=True) checksum = FileHash.from_contents(file_contents) source_file = SourceFile(path=match, checksum=checksum) source_file.contents = "" source_file.parse_file_type = ParseFileType.Seed source_file.project_name = project_name return source_file # Use the FilesystemSearcher to get a bunch of FilePaths, then turn # them into a bunch of FileSource objects def get_source_files(project, paths, extension, parse_file_type, saved_files, ignore_spec): # file path list fp_list = filesystem_search(project, paths, extension, ignore_spec) # file block list fb_list = [] for fp in fp_list: if parse_file_type == ParseFileType.Seed: fb_list.append(load_seed_source_file(fp, project.project_name)) # singular tests live in /tests but only generic tests live # in /tests/generic and fixtures in /tests/fixture so we want to skip those else: if parse_file_type == ParseFileType.SingularTest: path = pathlib.Path(fp.relative_path) if path.parts[0] in ["generic", "fixtures"]: continue file = load_source_file(fp, parse_file_type, project.project_name, saved_files) # only append the list if it has contents. added to fix #3568 if file: fb_list.append(file) return fb_list def read_files_for_parser(project, files, parse_ft, file_type_info, saved_files, ignore_spec): dirs = file_type_info["paths"] parser_files = [] for extension in file_type_info["extensions"]: source_files = get_source_files( project, dirs, extension, parse_ft, saved_files, ignore_spec ) for sf in source_files: files[sf.file_id] = sf parser_files.append(sf.file_id) return parser_files def generate_dbt_ignore_spec(project_root): ignore_file_path = os.path.join(project_root, ".dbtignore") ignore_spec = None if os.path.exists(ignore_file_path): with open(ignore_file_path) as f: ignore_spec = pathspec.PathSpec.from_lines(pathspec.patterns.GitWildMatchPattern, f) return ignore_spec # Protocol for the ReadFiles... classes class ReadFiles(Protocol): files: MutableMapping[str, AnySourceFile] project_parser_files: Dict def read_files(self): pass @dataclass class ReadFilesFromFileSystem: all_projects: Mapping[str, Project] files: MutableMapping[str, AnySourceFile] = field(default_factory=dict) # saved_files is only used to compare schema files saved_files: MutableMapping[str, AnySourceFile] = field(default_factory=dict) # project_parser_files = { # "my_project": { # "ModelParser": ["my_project://models/my_model.sql"] # } # } # project_parser_files: Dict = field(default_factory=dict) def read_files(self): for project in self.all_projects.values(): file_types = get_file_types_for_project(project) self.read_files_for_project(project, file_types) def read_files_for_project(self, project, file_types): dbt_ignore_spec = generate_dbt_ignore_spec(project.project_root) project_files = self.project_parser_files[project.project_name] = {} for parse_ft, file_type_info in file_types.items(): project_files[file_type_info["parser"]] = read_files_for_parser( project, self.files, parse_ft, file_type_info, self.saved_files, dbt_ignore_spec, ) @dataclass class ReadFilesFromDiff: root_project_name: str all_projects: Mapping[str, Project] file_diff: FileDiff files: MutableMapping[str, AnySourceFile] = field(default_factory=dict) # saved_files is used to construct a fresh copy of files, without # additional information from parsing saved_files: MutableMapping[str, AnySourceFile] = field(default_factory=dict) project_parser_files: Dict = field(default_factory=dict) project_file_types: Dict = field(default_factory=dict) local_package_dirs: Optional[List[str]] = None def read_files(self): # Copy the base file information from the existing manifest. # We will do deletions, adds, changes from the file_diff to emulate # a complete read of the project file system. for file_id, source_file in self.saved_files.items(): if isinstance(source_file, SchemaSourceFile): file_cls = SchemaSourceFile else: file_cls = SourceFile new_source_file = file_cls( path=source_file.path, checksum=source_file.checksum, project_name=source_file.project_name, parse_file_type=source_file.parse_file_type, contents=source_file.contents, ) self.files[file_id] = new_source_file # Now that we have a copy of the files, remove deleted files # For now, we assume that all files are in the root_project, until # we've determined whether project name will be provided or deduced # from the directory. for input_file_path in self.file_diff.deleted: project_name = self.get_project_name(input_file_path) file_id = f"{project_name}://{input_file_path}" if file_id in self.files: self.files.pop(file_id) else: fire_event(InputFileDiffError(category="deleted file not found", file_id=file_id)) # Now we do the changes for input_file in self.file_diff.changed: project_name = self.get_project_name(input_file.path) file_id = f"{project_name}://{input_file.path}" if file_id in self.files: # Get the existing source_file object and update the contents and mod time source_file = self.files[file_id] source_file.contents = input_file.content source_file.checksum = FileHash.from_contents(input_file.content) source_file.path.modification_time = input_file.modification_time # Handle creation of dictionary version of schema file content if isinstance(source_file, SchemaSourceFile) and source_file.contents: dfy = yaml_from_file(source_file) if dfy: validate_yaml(source_file.path.original_file_path, dfy) source_file.dfy = dfy # TODO: ensure we have a file object even for empty files, such as schema files # Now the new files for input_file in self.file_diff.added: project_name = self.get_project_name(input_file.path) # FilePath # searched_path i.e. "models" # relative_path i.e. the part after searched_path, or "model.sql" # modification_time float, default 0.0... # project_root # We use PurePath because there's no actual filesystem to look at input_file_path = pathlib.PurePath(input_file.path) extension = input_file_path.suffix searched_path = input_file_path.parts[0] # check what happens with generic tests... searched_path/relative_path relative_path_parts = input_file_path.parts[1:] relative_path = pathlib.PurePath("").joinpath(*relative_path_parts) # Create FilePath object input_file_path = FilePath( searched_path=searched_path, relative_path=str(relative_path), modification_time=input_file.modification_time, project_root=self.all_projects[project_name].project_root, ) # Now use the extension and "searched_path" to determine which file_type (file_types, file_type_lookup) = self.get_project_file_types(project_name) parse_ft_for_extension = set() parse_ft_for_path = set() if extension in file_type_lookup["extensions"]: parse_ft_for_extension = file_type_lookup["extensions"][extension] if searched_path in file_type_lookup["paths"]: parse_ft_for_path = file_type_lookup["paths"][searched_path] if len(parse_ft_for_extension) == 0 or len(parse_ft_for_path) == 0: fire_event(InputFileDiffError(category="not a project file", file_id=file_id)) continue parse_ft_set = parse_ft_for_extension.intersection(parse_ft_for_path) if ( len(parse_ft_set) != 1 ): # There should only be one result for a path/extension combination fire_event( InputFileDiffError( category="unable to resolve diff file location", file_id=file_id ) ) continue parse_ft = parse_ft_set.pop() source_file_cls = SourceFile if parse_ft == ParseFileType.Schema: source_file_cls = SchemaSourceFile source_file = source_file_cls( path=input_file_path, contents=input_file.content, checksum=FileHash.from_contents(input_file.content), project_name=project_name, parse_file_type=parse_ft, ) if source_file_cls == SchemaSourceFile: dfy = yaml_from_file(source_file) if dfy: validate_yaml(source_file.path.original_file_path, dfy) source_file.dfy = dfy else: # don't include in files because no content continue self.files[source_file.file_id] = source_file def get_project_name(self, path): # It's not currently possible to recognize any other project files, # and it's an open issue how to handle deps. return self.root_project_name def get_project_file_types(self, project_name): if project_name not in self.project_file_types: file_types = get_file_types_for_project(self.all_projects[project_name]) file_type_lookup = self.get_file_type_lookup(file_types) self.project_file_types[project_name] = { "file_types": file_types, "file_type_lookup": file_type_lookup, } file_types = self.project_file_types[project_name]["file_types"] file_type_lookup = self.project_file_types[project_name]["file_type_lookup"] return (file_types, file_type_lookup) def get_file_type_lookup(self, file_types): file_type_lookup = {"paths": {}, "extensions": {}} for parse_ft, file_type in file_types.items(): for path in file_type["paths"]: if path not in file_type_lookup["paths"]: file_type_lookup["paths"][path] = set() file_type_lookup["paths"][path].add(parse_ft) for extension in file_type["extensions"]: if extension not in file_type_lookup["extensions"]: file_type_lookup["extensions"][extension] = set() file_type_lookup["extensions"][extension].add(parse_ft) return file_type_lookup def get_file_types_for_project(project): file_types = { ParseFileType.Macro: { "paths": project.macro_paths, "extensions": [".sql"], "parser": "MacroParser", }, ParseFileType.Model: { "paths": project.model_paths, "extensions": [".sql", ".py"], "parser": "ModelParser", }, ParseFileType.Snapshot: { "paths": project.snapshot_paths, "extensions": [".sql"], "parser": "SnapshotParser", }, ParseFileType.Analysis: { "paths": project.analysis_paths, "extensions": [".sql"], "parser": "AnalysisParser", }, ParseFileType.SingularTest: { "paths": project.test_paths, "extensions": [".sql"], "parser": "SingularTestParser", }, ParseFileType.GenericTest: { "paths": project.generic_test_paths, "extensions": [".sql"], "parser": "GenericTestParser", }, ParseFileType.Seed: { "paths": project.seed_paths, "extensions": [".csv"], "parser": "SeedParser", }, ParseFileType.Documentation: { "paths": project.docs_paths, "extensions": [".md"], "parser": "DocumentationParser", }, ParseFileType.Schema: { "paths": project.all_source_paths, "extensions": [".yml", ".yaml"], "parser": "SchemaParser", }, ParseFileType.Fixture: { "paths": project.fixture_paths, "extensions": [".csv", ".sql"], "parser": "FixtureParser", }, ParseFileType.Function: { "paths": project.function_paths, "extensions": [".sql", ".py"], "parser": "FunctionParser", }, } return file_types ================================================ FILE: core/dbt/parser/schema_generic_tests.py ================================================ import itertools import os import pathlib from typing import Any, Dict, List, Optional, Union from dbt.adapters.factory import get_adapter, get_adapter_package_names from dbt.artifacts.resources import NodeVersion, RefArgs from dbt.clients.jinja import add_rendered_test_kwargs, get_rendered from dbt.context.configured import SchemaYamlVars, generate_schema_yml_context from dbt.context.context_config import ContextConfig from dbt.context.macro_resolver import MacroResolver from dbt.context.providers import generate_test_context from dbt.contracts.files import FileHash from dbt.contracts.graph.nodes import ( GenericTestNode, GraphMemberNode, ManifestNode, UnpatchedSourceDefinition, ) from dbt.contracts.graph.unparsed import UnparsedColumn, UnparsedNodeUpdate from dbt.exceptions import ( CompilationError, ParsingError, SchemaConfigError, TestConfigError, ) from dbt.node_types import NodeType from dbt.parser.base import SimpleParser from dbt.parser.common import ( GenericTestBlock, Testable, TestBlock, TestDef, VersionedTestBlock, trimmed, ) from dbt.parser.generic_test_builders import TestBuilder from dbt.parser.search import FileBlock from dbt.utils import get_pseudo_test_path, md5 from dbt_common.dataclass_schema import ValidationError # This parser handles the tests that are defined in "schema" (yaml) files, on models, # sources, etc. The base generic test is handled by the GenericTestParser class SchemaGenericTestParser(SimpleParser): def __init__( self, project, manifest, root_project, ) -> None: super().__init__(project, manifest, root_project) self.schema_yaml_vars = SchemaYamlVars() self.render_ctx = generate_schema_yml_context( self.root_project, self.project.project_name, self.schema_yaml_vars ) internal_package_names = get_adapter_package_names(self.root_project.credentials.type) self.macro_resolver = MacroResolver( self.manifest.macros, self.root_project.project_name, internal_package_names ) @property def resource_type(self) -> NodeType: return NodeType.Test @classmethod def get_compiled_path(cls, block: FileBlock) -> str: return block.path.relative_path def parse_file(self, block: FileBlock, dct: Optional[Dict] = None) -> None: pass def parse_from_dict(self, dct, validate=True) -> GenericTestNode: if validate: GenericTestNode.validate(dct) return GenericTestNode.from_dict(dct) def parse_column_tests( self, block: TestBlock, column: UnparsedColumn, version: Optional[NodeVersion] ) -> None: if not column.data_tests: return for data_test in column.data_tests: self.parse_test(block, data_test, column, version) def create_test_node( self, target: Union[UnpatchedSourceDefinition, UnparsedNodeUpdate], path: str, config: ContextConfig, tags: List[str], fqn: List[str], name: str, raw_code: str, test_metadata: Dict[str, Any], file_key_name: str, column_name: Optional[str], description: str, ) -> GenericTestNode: HASH_LENGTH = 10 # N.B: This function builds a hashable string from any given test_metadata dict. # it's a bit fragile for general use (only supports str, int, float, List, Dict) # but it gets the job done here without the overhead of complete ser(de). def get_hashable_md(data: Union[str, int, float, List, Dict]) -> Union[str, List, Dict]: if type(data) == dict: return {k: get_hashable_md(data[k]) for k in sorted(data.keys())} # type: ignore elif type(data) == list: return [get_hashable_md(val) for val in data] # type: ignore else: return str(data) hashable_metadata = repr(get_hashable_md(test_metadata)) hash_string = "".join([name, hashable_metadata]) test_hash = md5(hash_string)[-HASH_LENGTH:] dct = { "alias": name, "schema": self.default_schema, "database": self.default_database, "fqn": fqn, "name": name, "resource_type": self.resource_type, "tags": tags, "path": path, "original_file_path": target.original_file_path, "package_name": self.project.project_name, "raw_code": raw_code, "language": "sql", "unique_id": self.generate_unique_id(name, test_hash), "config": self.config_dict(config), "test_metadata": test_metadata, "column_name": column_name, "checksum": FileHash.empty().to_dict(omit_none=True), "file_key_name": file_key_name, "description": description, } try: GenericTestNode.validate(dct) return GenericTestNode.from_dict(dct) except ValidationError as exc: # this is a bit silly, but build an UnparsedNode just for error # message reasons node = self._create_error_node( name=target.name, path=path, original_file_path=target.original_file_path, raw_code=raw_code, ) raise TestConfigError(exc, node) # This is called directly in the SourcePatcher and by the "parse_node" # command which is called by the SchemaParser. def parse_generic_test( self, target: Testable, data_test: Dict[str, Any], tags: List[str], column_name: Optional[str], schema_file_id: str, version: Optional[NodeVersion], ) -> GenericTestNode: try: builder = TestBuilder( data_test=data_test, target=target, column_name=column_name, version=version, package_name=target.package_name, render_ctx=self.render_ctx, ) if self.schema_yaml_vars.env_vars: self.store_env_vars(target, schema_file_id, self.schema_yaml_vars.env_vars) self.schema_yaml_vars.env_vars = {} except ParsingError as exc: context = trimmed(str(target)) msg = "Invalid test config given in {}:\n\t{}\n\t@: {}".format( target.original_file_path, exc.msg, context ) raise ParsingError(msg) from exc except CompilationError as exc: context = trimmed(str(target)) msg = ( "Invalid generic test configuration given in " f"{target.original_file_path}: \n{exc.msg}\n\t@: {context}" ) raise CompilationError(msg) from exc original_name = os.path.basename(target.original_file_path) compiled_path = get_pseudo_test_path(builder.compiled_name, original_name) # fqn is the relative path of the yaml file where this generic test is defined, # minus the project-level directory and the file name itself # TODO pass a consistent path object from both UnparsedNode and UnpatchedSourceDefinition path = pathlib.Path(target.original_file_path) relative_path = str(path.relative_to(*path.parts[:1])) fqn = self.get_fqn(relative_path, builder.fqn_name) # this is the ContextConfig that is used in render_update config: ContextConfig = self.initial_config(fqn) # Adding the builder's config to the ContextConfig # is needed to ensure the config makes it to the pre_model hook which dbt-snowflake needs config.add_config_call(builder.config) # builder.args contains keyword args for the test macro, # not configs which have been separated out in the builder. # The keyword args are not completely rendered until compilation. metadata = { "namespace": builder.namespace, "name": builder.name, "kwargs": builder.args, } tags = sorted(set(itertools.chain(tags, builder.tags()))) if isinstance(target, UnpatchedSourceDefinition): file_key_name = f"{target.source.yaml_key}.{target.source.name}" else: file_key_name = f"{target.yaml_key}.{target.name}" node = self.create_test_node( target=target, path=compiled_path, config=config, fqn=fqn, tags=tags, name=builder.fqn_name, raw_code=builder.build_raw_code(), column_name=column_name, test_metadata=metadata, file_key_name=file_key_name, description=builder.description, ) self.render_test_update(node, config, builder, schema_file_id) return node def _lookup_attached_node( self, target: Testable, version: Optional[NodeVersion] ) -> Optional[Union[ManifestNode, GraphMemberNode]]: """Look up attached node for Testable target nodes other than sources. Can be None if generic test attached to SQL node with no corresponding .sql file.""" attached_node = None # type: Optional[Union[ManifestNode, GraphMemberNode]] if not isinstance(target, UnpatchedSourceDefinition): attached_node_unique_id = self.manifest.ref_lookup.get_unique_id( target.name, target.package_name, version ) if attached_node_unique_id: attached_node = self.manifest.nodes[attached_node_unique_id] else: disabled_node = self.manifest.disabled_lookup.find( target.name, None ) or self.manifest.disabled_lookup.find(target.name.upper(), None) if disabled_node: attached_node = self.manifest.disabled[disabled_node[0].unique_id][0] return attached_node def store_env_vars(self, target, schema_file_id, env_vars): self.manifest.env_vars.update(env_vars) if schema_file_id in self.manifest.files: schema_file = self.manifest.files[schema_file_id] if isinstance(target, UnpatchedSourceDefinition): search_name = target.source.name yaml_key = target.source.yaml_key if "." in search_name: # source file definitions (search_name, _) = search_name.split(".") else: search_name = target.name yaml_key = target.yaml_key for var in env_vars.keys(): schema_file.add_env_var(var, yaml_key, search_name) # This does special shortcut processing for the two # most common internal macros, not_null and unique, # which avoids the jinja rendering to resolve config # and variables, etc, which might be in the macro. # In the future we will look at generalizing this # more to handle additional macros or to use static # parsing to avoid jinja overhead. def render_test_update(self, node, config, builder, schema_file_id): macro_unique_id = self.macro_resolver.get_macro_id( node.package_name, "test_" + builder.name ) # Add the depends_on here so we can limit the macros added # to the context in rendering processing node.depends_on.add_macro(macro_unique_id) if macro_unique_id in ["macro.dbt.test_not_null", "macro.dbt.test_unique"]: config_call_dict = builder.config config._config_call_dict = config_call_dict # This sets the config from dbt_project self.update_parsed_node_config(node, config) # source node tests are processed at patch_source time if isinstance(builder.target, UnpatchedSourceDefinition): sources = [builder.target.fqn[-2], builder.target.fqn[-1]] node.sources.append(sources) else: # all other nodes node.refs.append(RefArgs(name=builder.target.name, version=builder.version)) else: try: # make a base context that doesn't have the magic kwargs field context = generate_test_context( node, self.root_project, self.manifest, config, self.macro_resolver, ) # update with rendered test kwargs (which collects any refs) # Note: This does not actually update the kwargs with the rendered # values. That happens in compilation. add_rendered_test_kwargs(context, node, capture_macros=True) # the parsed node is not rendered in the native context. get_rendered(node.raw_code, context, node, capture_macros=True) self.update_parsed_node_config(node, config) # env_vars should have been updated in the context env_var method except ValidationError as exc: # we got a ValidationError - probably bad types in config() raise SchemaConfigError(exc, node=node) from exc # Set attached_node for generic test nodes, if available. # Generic test node inherits attached node's group config value. attached_node = self._lookup_attached_node(builder.target, builder.version) if attached_node: node.attached_node = attached_node.unique_id node.group, node.group = attached_node.group, attached_node.group def parse_node(self, block: GenericTestBlock) -> GenericTestNode: """In schema parsing, we rewrite most of the part of parse_node that builds the initial node to be parsed, but rendering is basically the same """ node = self.parse_generic_test( target=block.target, data_test=block.data_test, tags=block.tags, column_name=block.column_name, schema_file_id=block.file.file_id, version=block.version, ) self.add_test_node(block, node) return node def add_test_node(self, block: GenericTestBlock, node: GenericTestNode): test_from = {"key": block.target.yaml_key, "name": block.target.name} if node.config.enabled: self.manifest.add_node(block.file, node, test_from) else: self.manifest.add_disabled(block.file, node, test_from) def render_with_context( self, node: GenericTestNode, config: ContextConfig, ) -> None: """Given the parsed node and a ContextConfig to use during parsing, collect all the refs that might be squirreled away in the test arguments. This includes the implicit "model" argument. """ # make a base context that doesn't have the magic kwargs field context = self._context_for(node, config) # update it with the rendered test kwargs (which collects any refs) add_rendered_test_kwargs(context, node, capture_macros=True) # the parsed node is not rendered in the native context. get_rendered(node.raw_code, context, node, capture_macros=True) def parse_test( self, target_block: TestBlock, data_test: TestDef, column: Optional[UnparsedColumn], version: Optional[NodeVersion], ) -> None: if isinstance(data_test, str): data_test = {data_test: {}} if column is None: column_name: Optional[str] = None column_tags: List[str] = [] else: column_name = column.name should_quote = column.quote or (column.quote is None and target_block.quote_columns) if should_quote: column_name = get_adapter(self.root_project).quote(column_name) column_config_tags = column.config.get("tags", []) if isinstance(column_config_tags, str): column_config_tags = [column_config_tags] column_tags = list(set(column.tags + column_config_tags)) block = GenericTestBlock.from_test_block( src=target_block, data_test=data_test, column_name=column_name, tags=column_tags, version=version, ) self.parse_node(block) def parse_tests(self, block: TestBlock) -> None: for column in block.columns: self.parse_column_tests(block, column, None) for data_test in block.data_tests: self.parse_test(block, data_test, None, None) def parse_versioned_tests(self, block: VersionedTestBlock) -> None: if not block.target.versions: self.parse_tests(block) else: for version in block.target.versions: for column in block.target.get_columns_for_version(version.v): self.parse_column_tests(block, column, version.v) for test in block.target.get_tests_for_version(version.v): self.parse_test(block, test, None, version.v) def generate_unique_id(self, resource_name: str, hash: Optional[str] = None) -> str: return ".".join( filter(None, [self.resource_type, self.project.project_name, resource_name, hash]) ) ================================================ FILE: core/dbt/parser/schema_renderer.py ================================================ from typing import Any, Dict from dbt.config.renderer import BaseRenderer, Keypath # This class renders dictionaries derived from "schema" yaml files. # It calls Jinja on strings (in deep_map_render), except for certain # keys which are skipped because they need to be rendered later # (tests and description). Test configs are rendered in the # generic test builder code, but skips the keyword args. The test # keyword args are rendered to capture refs in render_test_update. # Keyword args are finally rendered at compilation time. # Descriptions are not rendered until 'process_docs'. # Pre- and post-hooks in configs are late-rendered. class SchemaYamlRenderer(BaseRenderer): def __init__(self, context: Dict[str, Any], key: str) -> None: super().__init__(context) self.key = key @property def name(self): return "Rendering yaml" def _is_norender_key(self, keypath: Keypath) -> bool: """ models: - name: blah description: blah data_tests: ... columns: - name: description: blah data_tests: ... Return True if it's tests, data_tests or description - those aren't rendered now because they're rendered later in parse_generic_tests or process_docs. "tests" and "data_tests" are both currently supported but "tests" has been deprecated """ # top level descriptions and data_tests if len(keypath) >= 1 and keypath[0] in ( "tests", "data_tests", "description", "loaded_at_query", ): return True # columns descriptions and data_tests if len(keypath) == 2 and keypath[1] in ( "tests", "data_tests", "description", "loaded_at_query", ): return True # config: pre- and post-hooks, and loaded_at_query if ( len(keypath) >= 2 and keypath[0] == "config" and keypath[1] in ("pre_hook", "post_hook", "loaded_at_query") ): return True # versions if len(keypath) == 5 and keypath[4] == "description": return True if ( len(keypath) >= 3 and keypath[0] in ("columns", "dimensions", "measures", "entities", "metrics") and keypath[2] in ("tests", "data_tests", "description") ): return True # derived_semantics descriptions (v2 semantic layer) if ( len(keypath) >= 4 and keypath[0] == "derived_semantics" and keypath[1] in ("dimensions", "entities") and keypath[3] in ("tests", "data_tests", "description") ): return True return False # don't render descriptions or test keyword arguments def should_render_keypath(self, keypath: Keypath) -> bool: if len(keypath) < 1: return True if self.key == "sources": if keypath[0] in ("description", "loaded_at_query"): return False if len(keypath) >= 2 and keypath[0] == "config" and keypath[1] == "loaded_at_query": return False if keypath[0] == "tables": if self._is_norender_key(keypath[2:]): return False elif self.key == "macros": if keypath[0] == "arguments": if self._is_norender_key(keypath[1:]): return False elif self._is_norender_key(keypath[0:]): return False elif self.key == "metrics": # This ensures that metric filters are skipped if keypath[-1] == "filter" or len(keypath) > 1 and keypath[-2] == "filter": return False elif self._is_norender_key(keypath[0:]): return False elif self.key == "saved_queries": # This ensures that saved query filters are skipped if keypath[0] == "query_params" and len(keypath) > 1 and keypath[1] == "where": return False elif self._is_norender_key(keypath[0:]): return False else: # models, seeds, snapshots, analyses # Skip metric filters — consistent with the "metrics" branch above, # using positional checks to avoid over-matching. # metrics is always at keypath[0] in this branch (model-relative path), # and filter is always at [-1] (string) or [-2] (list item). if keypath[0] == "metrics" and ( keypath[-1] == "filter" or (len(keypath) > 1 and keypath[-2] == "filter") ): return False if self._is_norender_key(keypath[0:]): return False return True ================================================ FILE: core/dbt/parser/schema_yaml_readers.py ================================================ from collections.abc import Sequence from typing import Any, Dict, List, Optional, Union from dbt.artifacts.resources import ( ColumnDimension, ColumnEntity, ColumnInfo, ConversionTypeParams, CumulativeTypeParams, Defaults, Dimension, DimensionTypeParams, DimensionValidityParams, Entity, Export, ExportConfig, ExposureConfig, Measure, MeasureAggregationParameters, MetricAggregationParams, MetricConfig, MetricInput, MetricInputMeasure, MetricTimeWindow, MetricTypeParams, NonAdditiveDimension, QueryParams, SavedQueryConfig, SemanticLayerElementConfig, WhereFilter, WhereFilterIntersection, ) from dbt.clients.jinja import get_rendered from dbt.context.context_config import ( BaseContextConfigGenerator, ContextConfigGenerator, UnrenderedConfigGenerator, ) from dbt.context.providers import ( generate_parse_exposure, generate_parse_semantic_models, ) from dbt.contracts.files import SchemaSourceFile from dbt.contracts.graph.nodes import ( Exposure, Group, Metric, ModelNode, ParsedNodePatch, SavedQuery, SemanticModel, ) from dbt.contracts.graph.unparsed import ( PercentileType, UnparsedConversionTypeParams, UnparsedCumulativeTypeParams, UnparsedDerivedDimensionV2, UnparsedDerivedSemantics, UnparsedDimension, UnparsedDimensionTypeParams, UnparsedEntity, UnparsedExport, UnparsedExposure, UnparsedGroup, UnparsedMeasure, UnparsedMetric, UnparsedMetricBase, UnparsedMetricInput, UnparsedMetricInputMeasure, UnparsedMetricTypeParams, UnparsedMetricV2, UnparsedNonAdditiveDimension, UnparsedNonAdditiveDimensionV2, UnparsedQueryParams, UnparsedSavedQuery, UnparsedSemanticModel, UnparsedSemanticModelConfig, UnparsedSemanticResourceConfig, ) from dbt.exceptions import JSONValidationError, YamlParseDictError from dbt.node_types import NodeType from dbt.parser.common import YamlBlock from dbt.parser.schemas import ParseResult, SchemaParser, YamlReader from dbt_common.dataclass_schema import ValidationError from dbt_common.exceptions import DbtInternalError from dbt_semantic_interfaces.type_enums import ( AggregationType, ConversionCalculationType, DimensionType, EntityType, MetricType, PeriodAggregation, TimeGranularity, ) def parse_where_filter( where: Optional[Union[List[str], str]], ) -> Optional[WhereFilterIntersection]: if where is None: return None elif isinstance(where, str): return WhereFilterIntersection([WhereFilter(where)]) else: return WhereFilterIntersection([WhereFilter(where_str) for where_str in where]) class ExposureParser(YamlReader): def __init__(self, schema_parser: SchemaParser, yaml: YamlBlock) -> None: super().__init__(schema_parser, yaml, NodeType.Exposure.pluralize()) self.schema_parser = schema_parser self.yaml = yaml def parse_exposure(self, unparsed: UnparsedExposure) -> None: package_name = self.project.project_name unique_id = f"{NodeType.Exposure}.{package_name}.{unparsed.name}" path = self.yaml.path.relative_path fqn = self.schema_parser.get_fqn_prefix(path) fqn.append(unparsed.name) config = self._generate_exposure_config( target=unparsed, fqn=fqn, package_name=package_name, rendered=True, ) config = config.finalize_and_validate() unrendered_config = self._generate_exposure_config( target=unparsed, fqn=fqn, package_name=package_name, rendered=False, ) if not isinstance(config, ExposureConfig): raise DbtInternalError( f"Calculated a {type(config)} for an exposure, but expected an ExposureConfig" ) tags = sorted(set(self.project.exposures.get("tags", []) + unparsed.tags + config.tags)) meta = {**self.project.exposures.get("meta", {}), **unparsed.meta, **config.meta} config.tags = tags config.meta = meta parsed = Exposure( resource_type=NodeType.Exposure, package_name=package_name, path=path, original_file_path=self.yaml.path.original_file_path, unique_id=unique_id, fqn=fqn, name=unparsed.name, type=unparsed.type, url=unparsed.url, meta=meta, tags=tags, description=unparsed.description, label=unparsed.label, owner=unparsed.owner, maturity=unparsed.maturity, config=config, unrendered_config=unrendered_config, ) ctx = generate_parse_exposure( parsed, self.root_project, self.schema_parser.manifest, package_name, ) depends_on_jinja = "\n".join("{{ " + line + "}}" for line in unparsed.depends_on) get_rendered(depends_on_jinja, ctx, parsed, capture_macros=True) # parsed now has a populated refs/sources/metrics assert isinstance(self.yaml.file, SchemaSourceFile) if parsed.config.enabled: self.manifest.add_exposure(self.yaml.file, parsed) else: self.manifest.add_disabled(self.yaml.file, parsed) def _generate_exposure_config( self, target: UnparsedExposure, fqn: List[str], package_name: str, rendered: bool ): generator: BaseContextConfigGenerator if rendered: generator = ContextConfigGenerator(self.root_project) else: generator = UnrenderedConfigGenerator(self.root_project) # configs with precendence set precedence_configs = dict() # apply exposure configs precedence_configs.update(target.config) return generator.calculate_node_config( config_call_dict={}, fqn=fqn, resource_type=NodeType.Exposure, project_name=package_name, base=False, patch_config_dict=precedence_configs, ) def parse(self) -> None: for data in self.get_key_dicts(): try: UnparsedExposure.validate(data) unparsed = UnparsedExposure.from_dict(data) except (ValidationError, JSONValidationError) as exc: raise YamlParseDictError(self.yaml.path, self.key, data, exc) self.parse_exposure(unparsed) class MetricParser(YamlReader): def __init__(self, schema_parser: SchemaParser, yaml: YamlBlock) -> None: super().__init__(schema_parser, yaml, NodeType.Metric.pluralize()) self.schema_parser = schema_parser self.yaml = yaml def _get_input_measure( self, unparsed_input_measure: Union[UnparsedMetricInputMeasure, str], ) -> MetricInputMeasure: if isinstance(unparsed_input_measure, str): return MetricInputMeasure(name=unparsed_input_measure) else: return MetricInputMeasure( name=unparsed_input_measure.name, filter=parse_where_filter(unparsed_input_measure.filter), alias=unparsed_input_measure.alias, join_to_timespine=unparsed_input_measure.join_to_timespine, fill_nulls_with=unparsed_input_measure.fill_nulls_with, ) def _get_optional_input_measure( self, unparsed_input_measure: Optional[Union[UnparsedMetricInputMeasure, str]], ) -> Optional[MetricInputMeasure]: if unparsed_input_measure is not None: return self._get_input_measure(unparsed_input_measure) else: return None def _get_input_measures( self, unparsed_input_measures: Optional[List[Union[UnparsedMetricInputMeasure, str]]], ) -> List[MetricInputMeasure]: input_measures: List[MetricInputMeasure] = [] if unparsed_input_measures is not None: for unparsed_input_measure in unparsed_input_measures: input_measures.append(self._get_input_measure(unparsed_input_measure)) return input_measures def _get_period_agg(self, unparsed_period_agg: str) -> PeriodAggregation: return PeriodAggregation(unparsed_period_agg) def _get_optional_time_window( self, unparsed_window: Optional[str] ) -> Optional[MetricTimeWindow]: if unparsed_window is not None: parts = unparsed_window.lower().split(" ") if len(parts) != 2: raise YamlParseDictError( self.yaml.path, "window", {"window": unparsed_window}, f"Invalid window ({unparsed_window}) in cumulative/conversion metric. Should be of the form ` `, " "e.g., `28 days`", ) granularity = parts[1] # once we drop python 3.8 this could just be `granularity = parts[0].removesuffix('s') if granularity.endswith("s") and granularity[:-1] in [ item.value for item in TimeGranularity ]: # Can only remove the `s` if it's a standard grain, months -> month granularity = granularity[:-1] count = parts[0] if not count.isdigit(): raise YamlParseDictError( self.yaml.path, "window", {"window": unparsed_window}, f"Invalid count ({count}) in cumulative/conversion metric window string: ({unparsed_window})", ) return MetricTimeWindow( count=int(count), granularity=granularity, ) else: return None def _get_metric_input(self, unparsed: Union[UnparsedMetricInput, str]) -> MetricInput: if isinstance(unparsed, str): return MetricInput(name=unparsed) else: return MetricInput( name=unparsed.name, filter=parse_where_filter(unparsed.filter), alias=unparsed.alias, offset_window=self._get_optional_time_window(unparsed.offset_window), offset_to_grain=unparsed.offset_to_grain, ) def _get_optional_metric_input( self, unparsed: Optional[Union[UnparsedMetricInput, str]], ) -> Optional[MetricInput]: if unparsed is not None: return self._get_metric_input(unparsed) else: return None def _get_metric_inputs( self, unparsed_metric_inputs: Optional[List[Union[UnparsedMetricInput, str]]], ) -> List[MetricInput]: metric_inputs: List[MetricInput] = [] if unparsed_metric_inputs is not None: for unparsed_metric_input in unparsed_metric_inputs: metric_inputs.append(self._get_metric_input(unparsed=unparsed_metric_input)) return metric_inputs def _get_optional_v1_conversion_type_params( self, unparsed: Optional[UnparsedConversionTypeParams] ) -> Optional[ConversionTypeParams]: if unparsed is None: return None if unparsed.base_measure is None: raise ValidationError( "base_measure is required for conversion metrics that use type_params." ) if unparsed.conversion_measure is None: raise ValidationError( "conversion_measure is required for conversion metrics that use type_params." ) return ConversionTypeParams( base_measure=self._get_input_measure(unparsed.base_measure), conversion_measure=self._get_input_measure(unparsed.conversion_measure), entity=unparsed.entity, calculation=ConversionCalculationType(unparsed.calculation), window=self._get_optional_time_window(unparsed.window), constant_properties=unparsed.constant_properties, ) def _get_optional_v2_conversion_type_params( self, unparsed_metric: UnparsedMetricV2, ) -> Optional[ConversionTypeParams]: if MetricType(unparsed_metric.type) is not MetricType.CONVERSION: return None if unparsed_metric.base_metric is None: raise ValidationError("base_metric is required for conversion metrics.") if unparsed_metric.conversion_metric is None: raise ValidationError("conversion_metric is required for conversion metrics.") if unparsed_metric.entity is None: raise ValidationError("entity is required for conversion metrics.") return ConversionTypeParams( base_metric=self._get_metric_input(unparsed_metric.base_metric), conversion_metric=self._get_metric_input(unparsed_metric.conversion_metric), entity=unparsed_metric.entity, calculation=ConversionCalculationType(unparsed_metric.calculation), window=self._get_optional_time_window(unparsed_metric.window), constant_properties=unparsed_metric.constant_properties, ) def _get_optional_v1_cumulative_type_params( self, unparsed_metric: UnparsedMetric ) -> Optional[CumulativeTypeParams]: unparsed_type_params = unparsed_metric.type_params if unparsed_metric.type.lower() == MetricType.CUMULATIVE.value: if not unparsed_type_params.cumulative_type_params: unparsed_type_params.cumulative_type_params = UnparsedCumulativeTypeParams() if ( unparsed_type_params.window and not unparsed_type_params.cumulative_type_params.window ): unparsed_type_params.cumulative_type_params.window = unparsed_type_params.window if ( unparsed_type_params.grain_to_date and not unparsed_type_params.cumulative_type_params.grain_to_date ): unparsed_type_params.cumulative_type_params.grain_to_date = ( unparsed_type_params.grain_to_date ) return CumulativeTypeParams( window=self._get_optional_time_window( unparsed_type_params.cumulative_type_params.window ), grain_to_date=unparsed_type_params.cumulative_type_params.grain_to_date, period_agg=self._get_period_agg( unparsed_type_params.cumulative_type_params.period_agg ), ) return None def _get_optional_v2_cumulative_type_params( self, unparsed_metric: UnparsedMetricV2, ) -> Optional[CumulativeTypeParams]: if MetricType(unparsed_metric.type) is not MetricType.CUMULATIVE: return None input_metric = unparsed_metric.input_metric if input_metric is None: raise ValidationError("input_metric is required for cumulative metrics.") return CumulativeTypeParams( window=self._get_optional_time_window(unparsed_metric.window), grain_to_date=unparsed_metric.grain_to_date, period_agg=self._get_period_agg(unparsed_metric.period_agg), metric=self._get_metric_input(input_metric), ) def _get_v2_non_additive_dimension( self, unparsed_non_additive_dimension: Optional[UnparsedNonAdditiveDimensionV2], ) -> Optional[NonAdditiveDimension]: if unparsed_non_additive_dimension is None: return None return NonAdditiveDimension( name=unparsed_non_additive_dimension.name, window_choice=AggregationType(unparsed_non_additive_dimension.window_agg), window_groupings=unparsed_non_additive_dimension.group_by, ) def _get_metric_type_params( self, unparsed_metric: UnparsedMetricBase, generated_from: Optional[str] = None, default_agg_time_dimension: Optional[str] = None, ) -> MetricTypeParams: if isinstance(unparsed_metric, UnparsedMetric): type_params = unparsed_metric.type_params grain_to_date: Optional[TimeGranularity] = None if type_params.grain_to_date is not None: # This should've been changed to a string (to support custom grain), but since this # is a legacy field waiting to be deprecated, we will not support custom grain here # in order to force customers off of using this field. The field to use should be # `cumulative_type_params.grain_to_date` grain_to_date = TimeGranularity(type_params.grain_to_date) return MetricTypeParams( measure=self._get_optional_input_measure(type_params.measure), numerator=self._get_optional_metric_input(type_params.numerator), denominator=self._get_optional_metric_input(type_params.denominator), expr=str(type_params.expr) if type_params.expr is not None else None, window=self._get_optional_time_window(type_params.window), grain_to_date=grain_to_date, metrics=self._get_metric_inputs(type_params.metrics), conversion_type_params=self._get_optional_v1_conversion_type_params( type_params.conversion_type_params ), cumulative_type_params=self._get_optional_v1_cumulative_type_params( unparsed_metric=unparsed_metric, ), # input measures are calculated via metric processing post parsing # input_measures=?, ) elif isinstance(unparsed_metric, UnparsedMetricV2): if unparsed_metric.agg is not None: if generated_from is None: raise YamlParseDictError( self.yaml.path, self.key, yaml_data=unparsed_metric.to_dict(), cause="simple metrics in v2 YAML must be attached to semantic_model", ) metric_aggregation_params = MetricAggregationParams( semantic_model=generated_from, agg=AggregationType(unparsed_metric.agg), agg_params=MeasureAggregationParameters( percentile=unparsed_metric.percentile, use_discrete_percentile=(unparsed_metric.percentile_type or "").lower() == PercentileType.DISCRETE, use_approximate_percentile=(unparsed_metric.percentile_type or "").lower() == PercentileType.CONTINUOUS, ), agg_time_dimension=unparsed_metric.agg_time_dimension or default_agg_time_dimension, non_additive_dimension=self._get_v2_non_additive_dimension( unparsed_non_additive_dimension=unparsed_metric.non_additive_dimension, ), ) else: metric_aggregation_params = None return MetricTypeParams( numerator=self._get_optional_metric_input(unparsed_metric.numerator), denominator=self._get_optional_metric_input(unparsed_metric.denominator), expr=str(unparsed_metric.expr) if unparsed_metric.expr is not None else None, window=self._get_optional_time_window(unparsed_metric.window), metrics=self._get_metric_inputs(unparsed_metric.input_metrics), conversion_type_params=self._get_optional_v2_conversion_type_params( unparsed_metric=unparsed_metric, ), cumulative_type_params=self._get_optional_v2_cumulative_type_params( unparsed_metric=unparsed_metric, ), metric_aggregation_params=metric_aggregation_params, join_to_timespine=unparsed_metric.join_to_timespine or False, is_private=unparsed_metric.hidden, ) else: raise DbtInternalError( f"Tried to parse type params for a {type(unparsed_metric)}, but expected " "an UnparsedMetric or UnparsedMetricV2", ) def parse_metric( self, unparsed: UnparsedMetricBase, generated_from: Optional[str] = None, default_agg_time_dimension: Optional[str] = None, ) -> None: package_name = self.project.project_name unique_id = f"{NodeType.Metric}.{package_name}.{unparsed.name}" path = self.yaml.path.relative_path fqn = self.schema_parser.get_fqn_prefix(path) fqn.append(unparsed.name) config = self._generate_metric_config( target=unparsed, fqn=fqn, package_name=package_name, rendered=True, ) config = config.finalize_and_validate() unrendered_config = self._generate_metric_config( target=unparsed, fqn=fqn, package_name=package_name, rendered=False, ) if not isinstance(config, MetricConfig): raise DbtInternalError( f"Calculated a {type(config)} for a metric, but expected a MetricConfig" ) if isinstance(unparsed, UnparsedMetric): # If we have meta in the config, copy to node level, for backwards # compatibility with earlier node-only config. if "meta" in config and config["meta"]: unparsed.meta = config["meta"] meta = unparsed.meta tags = unparsed.tags elif isinstance(unparsed, UnparsedMetricV2): # V2 Metrics do not have a top-level meta field; this should be part of # the config. meta = {} tags = [] else: raise DbtInternalError( f"Tried to parse a {type(unparsed)} into a metric, but expected " "an UnparsedMetric or UnparsedMetricV2", ) parsed = Metric( resource_type=NodeType.Metric, package_name=package_name, path=path, original_file_path=self.yaml.path.original_file_path, unique_id=unique_id, fqn=fqn, name=unparsed.name, description=unparsed.description, label=unparsed.label or unparsed.name, type=MetricType(unparsed.type), type_params=self._get_metric_type_params( unparsed, generated_from=generated_from, default_agg_time_dimension=default_agg_time_dimension, ), time_granularity=unparsed.time_granularity, filter=parse_where_filter(unparsed.filter), meta=meta, tags=tags, config=config, unrendered_config=unrendered_config, group=config.group, ) # if the metric is disabled we do not want it included in the manifest, only in the disabled dict assert isinstance(self.yaml.file, SchemaSourceFile) if parsed.config.enabled: self.manifest.add_metric(self.yaml.file, parsed, generated_from) else: self.manifest.add_disabled(self.yaml.file, parsed) def _generate_metric_config( self, target: UnparsedMetricBase, fqn: List[str], package_name: str, rendered: bool ): generator: BaseContextConfigGenerator if rendered: generator = ContextConfigGenerator(self.root_project) else: generator = UnrenderedConfigGenerator(self.root_project) # configs with precendence set precedence_configs = dict() # first apply metric configs precedence_configs.update(target.config) config = generator.calculate_node_config( config_call_dict={}, fqn=fqn, resource_type=NodeType.Metric, project_name=package_name, base=False, patch_config_dict=precedence_configs, ) return config def _parse_v2_metric( self, data: dict[str, Any], semantic_model_name: Optional[str] = None ) -> None: try: UnparsedMetricV2.validate(data) unparsed = UnparsedMetricV2.from_dict(data) except (ValidationError, JSONValidationError) as exc: raise YamlParseDictError(self.yaml.path, self.key, data, exc) self.parse_metric(unparsed=unparsed) def parse_v2_metrics_from_dbt_model_patch(self, model_patch: ParsedNodePatch) -> None: if model_patch.metrics is None: return # Resolve the semantic model name, respecting custom name overrides semantic_model_name = model_patch.name if isinstance(model_patch.semantic_model, UnparsedSemanticModelConfig): if model_patch.semantic_model.name is not None: semantic_model_name = model_patch.semantic_model.name for metric in model_patch.metrics: is_simple = MetricType(metric.type) == MetricType.SIMPLE semantic_model = semantic_model_name if is_simple else None self.parse_metric( metric, generated_from=semantic_model, default_agg_time_dimension=model_patch.agg_time_dimension if is_simple else None, ) def parse(self) -> None: for data in self.get_key_dicts(): # The main differentiator of old-style yaml and new-style is "type_params", # so if that is missing, we'll assume you're using the newer yaml. if "type_params" in data: try: UnparsedMetric.validate(data) unparsed = UnparsedMetric.from_dict(data) except (ValidationError, JSONValidationError) as exc: raise YamlParseDictError(self.yaml.path, self.key, data, exc) self.parse_metric(unparsed) else: self._parse_v2_metric(data) class GroupParser(YamlReader): def __init__(self, schema_parser: SchemaParser, yaml: YamlBlock) -> None: super().__init__(schema_parser, yaml, NodeType.Group.pluralize()) self.schema_parser = schema_parser self.yaml = yaml def parse_group(self, unparsed: UnparsedGroup) -> None: package_name = self.project.project_name unique_id = f"{NodeType.Group}.{package_name}.{unparsed.name}" path = self.yaml.path.relative_path fqn = self.schema_parser.get_fqn_prefix(path) fqn.append(unparsed.name) config = self._generate_group_config(unparsed, fqn, package_name, True) parsed = Group( resource_type=NodeType.Group, package_name=package_name, path=path, original_file_path=self.yaml.path.original_file_path, unique_id=unique_id, name=unparsed.name, owner=unparsed.owner, description=unparsed.description, config=config, ) assert isinstance(self.yaml.file, SchemaSourceFile) self.manifest.add_group(self.yaml.file, parsed) def parse(self): for data in self.get_key_dicts(): try: UnparsedGroup.validate(data) unparsed = UnparsedGroup.from_dict(data) except (ValidationError, JSONValidationError) as exc: raise YamlParseDictError(self.yaml.path, self.key, data, exc) self.parse_group(unparsed) def _generate_group_config( self, target: UnparsedGroup, fqn: List[str], package_name: str, rendered: bool ): generator: BaseContextConfigGenerator if rendered: generator = ContextConfigGenerator(self.root_project) else: generator = UnrenderedConfigGenerator(self.root_project) # configs with precendence set precedence_configs = dict() # first apply metric configs precedence_configs.update(target.config) config = generator.calculate_node_config( config_call_dict={}, fqn=fqn, resource_type=NodeType.Group, project_name=package_name, base=False, patch_config_dict=precedence_configs, ) return config class SemanticModelParser(YamlReader): def __init__(self, schema_parser: SchemaParser, yaml: YamlBlock) -> None: super().__init__(schema_parser, yaml, "semantic_models") self.schema_parser = schema_parser self.yaml = yaml def _get_dimension_type_params( self, unparsed: Optional[UnparsedDimensionTypeParams] ) -> Optional[DimensionTypeParams]: if unparsed is not None: return DimensionTypeParams( time_granularity=TimeGranularity(unparsed.time_granularity), validity_params=unparsed.validity_params, ) else: return None def _get_dimensions(self, unparsed_dimensions: List[UnparsedDimension]) -> List[Dimension]: dimensions: List[Dimension] = [] for unparsed in unparsed_dimensions: dimensions.append( Dimension( name=unparsed.name, type=DimensionType(unparsed.type), description=unparsed.description, label=unparsed.label, is_partition=unparsed.is_partition, type_params=self._get_dimension_type_params(unparsed=unparsed.type_params), expr=unparsed.expr, metadata=None, # TODO: requires a fair bit of parsing context config=SemanticLayerElementConfig(meta=unparsed.config.get("meta", {})), ) ) return dimensions def _get_entities(self, unparsed_entities: List[UnparsedEntity]) -> List[Entity]: entities: List[Entity] = [] for unparsed in unparsed_entities: entities.append( Entity( name=unparsed.name, type=EntityType(unparsed.type), description=unparsed.description, label=unparsed.label, role=unparsed.role, expr=unparsed.expr, config=SemanticLayerElementConfig(meta=unparsed.config.get("meta", {})), ) ) return entities def _get_non_additive_dimension( self, unparsed: Optional[UnparsedNonAdditiveDimension] ) -> Optional[NonAdditiveDimension]: if unparsed is not None: return NonAdditiveDimension( name=unparsed.name, window_choice=AggregationType(unparsed.window_choice), window_groupings=unparsed.window_groupings, ) else: return None def _get_measures(self, unparsed_measures: List[UnparsedMeasure]) -> List[Measure]: measures: List[Measure] = [] for unparsed in unparsed_measures: measures.append( Measure( name=unparsed.name, agg=AggregationType(unparsed.agg), description=unparsed.description, label=unparsed.label, expr=str(unparsed.expr) if unparsed.expr is not None else None, agg_params=unparsed.agg_params, non_additive_dimension=self._get_non_additive_dimension( unparsed.non_additive_dimension ), agg_time_dimension=unparsed.agg_time_dimension, config=SemanticLayerElementConfig(meta=unparsed.config.get("meta", {})), ) ) return measures def _create_metric( self, measure: UnparsedMeasure, enabled: bool, semantic_model_name: str, meta: Optional[Dict[str, Any]] = None, ) -> None: config: Dict[str, Any] = {"enabled": enabled} if meta is not None: # Need to propagate meta to metric from measure during create_metric: True config["meta"] = meta unparsed_metric = UnparsedMetric( name=measure.name, label=measure.label or measure.name, type="simple", type_params=UnparsedMetricTypeParams( measure=measure.name, expr=measure.expr or measure.name # type: ignore ), description=measure.description or f"Metric created from measure {measure.name}", config=config, ) parser = MetricParser(self.schema_parser, yaml=self.yaml) parser.parse_metric(unparsed=unparsed_metric, generated_from=semantic_model_name) def _generate_semantic_model_config( self, target_config: Dict[str, Any], fqn: List[str], package_name: str, rendered: bool, ): generator: BaseContextConfigGenerator if rendered: generator = ContextConfigGenerator(self.root_project) else: generator = UnrenderedConfigGenerator(self.root_project) # configs with precendence set precedence_configs = dict() # first apply semantic model configs precedence_configs.update(target_config) config = generator.calculate_node_config( config_call_dict={}, fqn=fqn, resource_type=NodeType.SemanticModel, project_name=package_name, base=False, patch_config_dict=precedence_configs, ) return config def _parse_semantic_model_v1(self, unparsed: UnparsedSemanticModel) -> None: entities = self._get_entities(unparsed.entities) measures = self._get_measures(unparsed.measures) dimensions = self._get_dimensions(unparsed.dimensions) self._parse_semantic_model_helper( semantic_model_name=unparsed.name, semantic_model_config=unparsed.config, description=unparsed.description, label=unparsed.label, model=unparsed.model, defaults=unparsed.defaults, primary_entity=unparsed.primary_entity, entities=entities, measures=measures, unparsed_measures=unparsed.measures, dimensions=dimensions, ) def _parse_v2_column_dimensions(self, columns: Dict[str, ColumnInfo]) -> List[Dimension]: dimensions: List[Dimension] = [] for column in columns.values(): if column.dimension is None: continue elif isinstance(column.dimension, DimensionType): dimensions.append( Dimension( name=column.name, type=column.dimension, description=column.description, metadata=None, # Not yet supported in v1 or v2 YAML config=SemanticLayerElementConfig(meta=column.config.get("meta", {})), ) ) elif isinstance(column.dimension, ColumnDimension): type_params = ( ( DimensionTypeParams( time_granularity=column.granularity, validity_params=( DimensionValidityParams( is_start=column.dimension.validity_params.is_start, is_end=column.dimension.validity_params.is_end, ) if column.dimension.validity_params is not None else None ), ) if column.granularity is not None else None ) if column.granularity is not None else None ) meta = dict(column.config.get("meta", {})) meta.update((column.dimension.config or {}).get("meta", {})) config = SemanticLayerElementConfig(meta=meta) dimension_name = column.dimension.name or column.name dimensions.append( Dimension( # required type=DimensionType(column.dimension.type), # fields that use column's values as fallback values name=dimension_name, description=column.dimension.description or column.description, config=config, # optional fields label=column.dimension.label, is_partition=column.dimension.is_partition, type_params=type_params, metadata=None, # Not yet supported in v1 or v2 YAML # When the dimension name differs from the column name, set expr # to the column name so MetricFlow queries the correct warehouse column. expr=column.name if dimension_name != column.name else None, ) ) return dimensions def _parse_v2_derived_dimensions( self, derived_dimensions: List[UnparsedDerivedDimensionV2], ) -> List[Dimension]: dimensions: List[Dimension] = [] for derived_dimension in derived_dimensions: type_params = None if derived_dimension.granularity is not None: type_params = DimensionTypeParams( time_granularity=TimeGranularity(derived_dimension.granularity), validity_params=( DimensionValidityParams( is_start=derived_dimension.validity_params.is_start, is_end=derived_dimension.validity_params.is_end, ) if derived_dimension.validity_params is not None else None ), ) dimensions.append( Dimension( type=DimensionType(derived_dimension.type), name=derived_dimension.name, description=derived_dimension.description, label=derived_dimension.label, is_partition=derived_dimension.is_partition, config=SemanticLayerElementConfig( meta=derived_dimension.config.get("meta", {}) ), type_params=type_params, # fields unique to derived dimensions expr=derived_dimension.expr, ) ) return dimensions def _parse_v2_column_entities(self, columns: Dict[str, ColumnInfo]) -> List[Entity]: entities: List[Entity] = [] for column in columns.values(): if column.entity is None: continue elif isinstance(column.entity, ColumnEntity): entities.append( Entity( name=column.entity.name, type=column.entity.type, description=column.entity.description, label=column.entity.label, # When the entity name differs from the column name, set expr # to the column name so MetricFlow queries the correct warehouse column. expr=column.name if column.entity.name != column.name else None, config=SemanticLayerElementConfig( meta=column.entity.config.get("meta", column.config.get("meta", {})) ), ) ) elif isinstance(column.entity, EntityType): entities.append( Entity( name=column.name, type=column.entity, description=column.description, label=None, # there's no label to carry through from columns config=SemanticLayerElementConfig(meta=column.config.get("meta", {})), ) ) return entities def _parse_v2_derived_semantics_entities( self, derived_semantics: UnparsedDerivedSemantics ) -> List[Entity]: entities: List[Entity] = [] for unparsed_entity in derived_semantics.entities: entities.append( Entity( name=unparsed_entity.name, type=EntityType(unparsed_entity.type), description=unparsed_entity.description, label=unparsed_entity.label, expr=unparsed_entity.expr, config=SemanticLayerElementConfig(meta=unparsed_entity.config.get("meta", {})), ) ) return entities def parse_v2_semantic_model_from_dbt_model_patch( self, node: ModelNode, patch: ParsedNodePatch, ) -> None: if patch.semantic_model is None: # We shouldn't be calling this method in this case, but for safety # and typechecking, we'll return early here. return dimensions = self._parse_v2_column_dimensions(patch.columns) if patch.derived_semantics is not None: dimensions.extend( self._parse_v2_derived_dimensions(patch.derived_semantics.dimensions) ) entities = self._parse_v2_column_entities(patch.columns) if patch.derived_semantics is not None: entities.extend(self._parse_v2_derived_semantics_entities(patch.derived_semantics)) name = node.name config: Dict[str, Any] = {} if isinstance(patch.semantic_model, UnparsedSemanticModelConfig): if patch.semantic_model.name is not None: name = patch.semantic_model.name if patch.semantic_model.config is not None: unparsed_sub_config = patch.semantic_model.config if isinstance(unparsed_sub_config, UnparsedSemanticResourceConfig): if unparsed_sub_config.meta is not None: config["meta"] = unparsed_sub_config.meta if patch.semantic_model.enabled is not None: config["enabled"] = patch.semantic_model.enabled if patch.semantic_model.group is not None: config["group"] = patch.semantic_model.group elif isinstance(patch.semantic_model, bool): # boolean value just indicates that the model has a semantic model, # so nothing to do here. pass else: # this should be unreachable, but just in case raise ValueError(f"Invalid semantic model config: {patch.semantic_model}") self._parse_semantic_model_helper( semantic_model_name=name, semantic_model_config=config, description=node.description, label=None, # does not seem to be available in v2 YAML, unless it is part of the semantic model config's 'group'? model=f"ref('{patch.name}')", defaults=Defaults(agg_time_dimension=patch.agg_time_dimension), primary_entity=patch.primary_entity, entities=entities, dimensions=dimensions, # Measures are not part of the v2 YAML design. measures=[], unparsed_measures=[], ) def _parse_semantic_model_helper( self, semantic_model_name: str, semantic_model_config: Dict[str, Any], description: Optional[str], label: Optional[str], model: str, defaults, primary_entity, entities: List[Entity], dimensions: List[Dimension], measures: List[Measure], # v1 only unparsed_measures: List[UnparsedMeasure] = [], # v1 only ) -> None: package_name = self.project.project_name unique_id = f"{NodeType.SemanticModel}.{package_name}.{semantic_model_name}" path = self.yaml.path.relative_path fqn = self.schema_parser.get_fqn_prefix(path) fqn.append(semantic_model_name) config = self._generate_semantic_model_config( target_config=semantic_model_config, fqn=fqn, package_name=package_name, rendered=True, ) # Combine configs according to the behavior documented here https://docs.getdbt.com/reference/configs-and-properties#combining-configs elements: Sequence[Union[Dimension, Entity, Measure]] = [ *dimensions, *entities, *measures, ] for element in elements: if config is not None: if element.config is None: element.config = SemanticLayerElementConfig(meta=config.meta) else: element.config.meta = {**config.get("meta", {}), **element.config.meta} config = config.finalize_and_validate() unrendered_config = self._generate_semantic_model_config( target_config=semantic_model_config, fqn=fqn, package_name=package_name, rendered=False, ) parsed = SemanticModel( description=description, label=label, fqn=fqn, model=model, name=semantic_model_name, node_relation=None, # Resolved from the value of "model" after parsing original_file_path=self.yaml.path.original_file_path, package_name=package_name, path=path, resource_type=NodeType.SemanticModel, unique_id=unique_id, entities=entities, measures=measures, dimensions=dimensions, defaults=defaults, primary_entity=primary_entity, config=config, unrendered_config=unrendered_config, group=config.group, ) ctx = generate_parse_semantic_models( parsed, self.root_project, self.schema_parser.manifest, package_name, ) if parsed.model is not None: model_ref = "{{ " + parsed.model + " }}" # This sets the "refs" in the SemanticModel from the SemanticModelRefResolver in context/providers.py get_rendered(model_ref, ctx, parsed) # if the semantic model is disabled we do not want it included in the manifest, # only in the disabled dict assert isinstance(self.yaml.file, SchemaSourceFile) if parsed.config.enabled: self.manifest.add_semantic_model(self.yaml.file, parsed) else: self.manifest.add_disabled(self.yaml.file, parsed) # Create a metric for each measure with `create_metric = True` # This is only relevant for v1 SL YAML; v2 does not include measures at all. for measure in unparsed_measures: if measure.create_metric is True: self._create_metric( measure=measure, enabled=parsed.config.enabled, semantic_model_name=parsed.name, meta=config.meta if config is not None else None, ) def parse(self) -> None: for data in self.get_key_dicts(): try: UnparsedSemanticModel.validate(data) unparsed = UnparsedSemanticModel.from_dict(data) except (ValidationError, JSONValidationError) as exc: raise YamlParseDictError(self.yaml.path, self.key, data, exc) self._parse_semantic_model_v1(unparsed) class SavedQueryParser(YamlReader): def __init__(self, schema_parser: SchemaParser, yaml: YamlBlock) -> None: super().__init__(schema_parser, yaml, "saved_queries") self.schema_parser = schema_parser self.yaml = yaml def _generate_saved_query_config( self, target: UnparsedSavedQuery, fqn: List[str], package_name: str, rendered: bool ): generator: BaseContextConfigGenerator if rendered: generator = ContextConfigGenerator(self.root_project) else: generator = UnrenderedConfigGenerator(self.root_project) # configs with precendence set precedence_configs = dict() # first apply semantic model configs precedence_configs.update(target.config) config = generator.calculate_node_config( config_call_dict={}, fqn=fqn, resource_type=NodeType.SavedQuery, project_name=package_name, base=False, patch_config_dict=precedence_configs, ) return config def _get_export_config( self, unparsed_export_config: Dict[str, Any], saved_query_config: SavedQueryConfig ) -> ExportConfig: # Combine the two dictionaries using dictionary unpacking # the second dictionary is the one whose keys take priority combined = {**saved_query_config.__dict__, **unparsed_export_config} # `schema` is the user facing attribute, but for DSI protocol purposes we track it as `schema_name` if combined.get("schema") is not None and combined.get("schema_name") is None: combined["schema_name"] = combined["schema"] return ExportConfig.from_dict(combined) def _get_export( self, unparsed: UnparsedExport, saved_query_config: SavedQueryConfig ) -> Export: return Export( name=unparsed.name, config=self._get_export_config(unparsed.config, saved_query_config), unrendered_config=unparsed.config, ) def _get_query_params(self, unparsed: UnparsedQueryParams) -> QueryParams: return QueryParams( group_by=unparsed.group_by, metrics=unparsed.metrics, where=parse_where_filter(unparsed.where), order_by=unparsed.order_by, limit=unparsed.limit, ) def parse_saved_query(self, unparsed: UnparsedSavedQuery) -> None: package_name = self.project.project_name unique_id = f"{NodeType.SavedQuery}.{package_name}.{unparsed.name}" path = self.yaml.path.relative_path fqn = self.schema_parser.get_fqn_prefix(path) fqn.append(unparsed.name) config = self._generate_saved_query_config( target=unparsed, fqn=fqn, package_name=package_name, rendered=True, ) config = config.finalize_and_validate() unrendered_config = self._generate_saved_query_config( target=unparsed, fqn=fqn, package_name=package_name, rendered=False, ) # The parser handles plain strings just fine, but we need to be able # to join two lists, remove duplicates, and sort, so we have to wrap things here. def wrap_tags(s: Union[List[str], str]) -> List[str]: if s is None: return [] return [s] if isinstance(s, str) else s config_tags = wrap_tags(config.get("tags")) unparsed_tags = wrap_tags(unparsed.tags) tags = list(set([*unparsed_tags, *config_tags])) tags.sort() parsed = SavedQuery( description=unparsed.description, label=unparsed.label, fqn=fqn, name=unparsed.name, original_file_path=self.yaml.path.original_file_path, package_name=package_name, path=path, resource_type=NodeType.SavedQuery, unique_id=unique_id, query_params=self._get_query_params(unparsed.query_params), exports=[self._get_export(export, config) for export in unparsed.exports], config=config, unrendered_config=unrendered_config, group=config.group, tags=tags, ) for export in parsed.exports: self.schema_parser.update_parsed_node_relation_names(export, export.config.to_dict()) # type: ignore if not export.config.schema_name: export.config.schema_name = getattr(export, "schema", None) delattr(export, "schema") export.config.database = getattr(export, "database", None) or export.config.database delattr(export, "database") if not export.config.alias: export.config.alias = getattr(export, "alias", None) delattr(export, "alias") delattr(export, "relation_name") # Only add thes saved query if it's enabled, otherwise we track it with other diabled nodes assert isinstance(self.yaml.file, SchemaSourceFile) if parsed.config.enabled: self.manifest.add_saved_query(self.yaml.file, parsed) else: self.manifest.add_disabled(self.yaml.file, parsed) def parse(self) -> ParseResult: for data in self.get_key_dicts(): try: UnparsedSavedQuery.validate(data) unparsed = UnparsedSavedQuery.from_dict(data) except (ValidationError, JSONValidationError) as exc: raise YamlParseDictError(self.yaml.path, self.key, data, exc) self.parse_saved_query(unparsed) # The supertype (YamlReader) requires `parse` to return a ParseResult, so # we return an empty one because we don't have one to actually return. return ParseResult() ================================================ FILE: core/dbt/parser/schemas.py ================================================ import datetime import re import time from abc import ABCMeta, abstractmethod from dataclasses import dataclass, field from typing import ( Any, Callable, Dict, Generic, Iterable, List, Optional, Tuple, Type, TypeVar, ) from dbt.artifacts.resources import CustomGranularity, Docs, RefArgs, TimeSpine from dbt.clients.checked_load import ( checked_load, issue_deprecation_warnings_for_failures, ) from dbt.clients.jinja_static import statically_parse_ref_or_source from dbt.clients.yaml_helper import load_yaml_text from dbt.config import RuntimeConfig from dbt.context.configured import SchemaYamlVars, generate_schema_yml_context from dbt.context.context_config import ContextConfig from dbt.contracts.files import SchemaSourceFile, SourceFile from dbt.contracts.graph.manifest import Manifest from dbt.contracts.graph.nodes import ( FunctionNode, Macro, ModelNode, ParsedFunctionPatch, ParsedMacroPatch, ParsedNodePatch, ParsedSingularTestPatch, UnpatchedSourceDefinition, ) from dbt.contracts.graph.unparsed import ( HasColumnDocs, HasColumnTests, SourcePatch, UnparsedAnalysisUpdate, UnparsedFunctionUpdate, UnparsedMacroUpdate, UnparsedModelUpdate, UnparsedNodeUpdate, UnparsedSingularTestUpdate, UnparsedSourceDefinition, ) from dbt.events.types import ( InvalidMacroAnnotation, MacroNotFoundForPatch, NoNodeForYamlKey, UnsupportedConstraintMaterialization, ValidationWarning, WrongResourceSchemaFile, ) from dbt.exceptions import ( DbtInternalError, DuplicateMacroPatchNameError, DuplicatePatchPathError, DuplicateSourcePatchNameError, InvalidAccessTypeError, JSONValidationError, ParsingError, YamlLoadError, YamlParseDictError, YamlParseListError, ) from dbt.flags import get_flags from dbt.node_types import AccessType, NodeType from dbt.parser.base import SimpleParser from dbt.parser.common import ( ParserRef, TargetBlock, TestBlock, VersionedTestBlock, YamlBlock, schema_file_keys_to_resource_types, trimmed, ) from dbt.parser.schema_generic_tests import SchemaGenericTestParser from dbt.parser.schema_renderer import SchemaYamlRenderer from dbt.parser.search import FileBlock from dbt.utils import coerce_dict_str from dbt_common.contracts.constraints import ConstraintType, ModelLevelConstraint from dbt_common.dataclass_schema import ValidationError, dbtClassMixin from dbt_common.events import EventLevel from dbt_common.events.functions import fire_event, warn_or_error from dbt_common.events.types import Note from dbt_common.exceptions import DbtValidationError from dbt_common.utils import deep_merge # =============================================================================== # Schema Parser classes # # The SchemaParser is a subclass of the SimpleParser from base.py, as is # the SchemaGenericTestParser. The schema sub-parsers are all subclasses of # the YamlReader parsing class. Most of the action in creating SourceDefinition # nodes actually happens in the SourcePatcher class, in sources.py, which is # called as a late-stage parsing step in manifest.py. # # The "patch" parsers read yaml config and properties and apply them to # nodes that were already created from sql files. # # The SchemaParser and SourcePatcher both use the SchemaGenericTestParser # (in schema_generic_tests.py) to create generic test nodes. # # YamlReader # MetricParser (metrics) [schema_yaml_readers.py] # ExposureParser (exposures) [schema_yaml_readers.py] # GroupParser (groups) [schema_yaml_readers.py] # SourceParser (sources) # PatchParser # MacroPatchParser (macros) # NodePatchParser # ModelPatchParser (models) # AnalysisPatchParser (analyses) # TestablePatchParser (seeds, snapshots) # # =============================================================================== def yaml_from_file( source_file: SchemaSourceFile, validate: bool = False ) -> Optional[Dict[str, Any]]: """If loading the yaml fails, raise an exception.""" try: # source_file.contents can sometimes be None to_load = source_file.contents or "" if validate: contents, failures = checked_load(to_load) issue_deprecation_warnings_for_failures( failures=failures, file=source_file.path.original_file_path ) if contents is not None: from dbt.jsonschemas.jsonschemas import ( jsonschema_validate, resources_schema, ) # Validate the yaml against the jsonschema to raise deprecation warnings # for invalid fields. jsonschema_validate( schema=resources_schema(), json=contents, file_path=source_file.path.original_file_path, ) else: contents = load_yaml_text(to_load, source_file.path) if contents is None: return contents if not isinstance(contents, dict): raise DbtValidationError( f"Contents of file '{source_file.original_file_path}' are not valid. Dictionary expected." ) # When loaded_at_field is defined as None or null, it shows up in # the dict but when it is not defined, it does not show up in the dict # We need to capture this to be able to override source level settings later. for source in contents.get("sources", []): for table in source.get("tables", []): if "loaded_at_field" in table or ( "config" in table and table["config"] is not None and table["config"].get("loaded_at_field") ): table["loaded_at_field_present"] = True return contents except DbtValidationError as e: raise YamlLoadError( project_name=source_file.project_name, path=source_file.path.relative_path, exc=e ) # This is the main schema file parser, but almost everything happens in the # the schema sub-parsers. class SchemaParser(SimpleParser[YamlBlock, ModelNode]): def __init__( self, project: RuntimeConfig, manifest: Manifest, root_project: RuntimeConfig, ) -> None: super().__init__(project, manifest, root_project) self.generic_test_parser = SchemaGenericTestParser(project, manifest, root_project) self.schema_yaml_vars = SchemaYamlVars() self.render_ctx = generate_schema_yml_context( self.root_project, self.project.project_name, self.schema_yaml_vars ) # This is unnecessary, but mypy was requiring it. Clean up parser code so # we don't have to do this. def parse_from_dict(self, dct): pass @classmethod def get_compiled_path(cls, block: FileBlock) -> str: # should this raise an error? return block.path.relative_path @property def resource_type(self) -> NodeType: return NodeType.Test def parse_file(self, block: FileBlock, dct: Optional[Dict] = None) -> None: assert isinstance(block.file, SchemaSourceFile) # If partially parsing, dct should be from pp_dict, otherwise # dict_from_yaml if dct: # contains the FileBlock and the data (dictionary) yaml_block = YamlBlock.from_file_block(block, dct) parser: YamlReader # There are 9 different yaml lists which are parsed by different parsers: # Model, Seed, Snapshot, Source, Macro, Analysis, Exposure, Metric, Group # ModelPatchParser.parse() if "models" in dct: # the models are already in the manifest as nodes when we reach this code, # even if they are disabled in the schema file model_parse_result = ModelPatchParser(self, yaml_block, "models").parse() for versioned_test_block in model_parse_result.versioned_test_blocks: self.generic_test_parser.parse_versioned_tests(versioned_test_block) # PatchParser.parse() if "seeds" in dct: seed_parse_result = TestablePatchParser(self, yaml_block, "seeds").parse() for test_block in seed_parse_result.test_blocks: self.generic_test_parser.parse_tests(test_block) # PatchParser.parse() if "snapshots" in dct: self._add_yaml_snapshot_nodes_to_manifest(dct["snapshots"], block) snapshot_parse_result = TestablePatchParser(self, yaml_block, "snapshots").parse() for test_block in snapshot_parse_result.test_blocks: self.generic_test_parser.parse_tests(test_block) # This parser uses SourceParser.parse() which doesn't return # any test blocks. Source tests are handled at a later point # in the process. if "sources" in dct: parser = SourceParser(self, yaml_block, "sources") parser.parse() # PatchParser.parse() (but never test_blocks) if "macros" in dct: parser = MacroPatchParser(self, yaml_block, "macros") parser.parse() if "data_tests" in dct: parser = SingularTestPatchParser(self, yaml_block, "data_tests") try: parser.parse() except ParsingError as e: fire_event( Note( msg=f"Unable to parse 'data_tests' section of file '{block.path.original_file_path}'\n{e}", ), EventLevel.WARN, ) # PatchParser.parse() (but never test_blocks) if "analyses" in dct: parser = AnalysisPatchParser(self, yaml_block, "analyses") parser.parse() # ExposureParser.parse() if "exposures" in dct: from dbt.parser.schema_yaml_readers import ExposureParser exp_parser = ExposureParser(self, yaml_block) exp_parser.parse() # FunctionPatchParser.parse() if "functions" in dct: function_parser = FunctionPatchParser(self, yaml_block, "functions") function_parser.parse() # MetricParser.parse() if "metrics" in dct: from dbt.parser.schema_yaml_readers import MetricParser metric_parser = MetricParser(self, yaml_block) metric_parser.parse() # GroupParser.parse() if "groups" in dct: from dbt.parser.schema_yaml_readers import GroupParser group_parser = GroupParser(self, yaml_block) group_parser.parse() if "semantic_models" in dct: from dbt.parser.schema_yaml_readers import SemanticModelParser semantic_model_parser = SemanticModelParser(self, yaml_block) semantic_model_parser.parse() if "unit_tests" in dct: from dbt.parser.unit_tests import UnitTestParser unit_test_parser = UnitTestParser(self, yaml_block) unit_test_parser.parse() if "saved_queries" in dct: from dbt.parser.schema_yaml_readers import SavedQueryParser saved_query_parser = SavedQueryParser(self, yaml_block) saved_query_parser.parse() def _add_yaml_snapshot_nodes_to_manifest( self, snapshots: List[Dict[str, Any]], block: FileBlock ) -> None: """We support the creation of simple snapshots in yaml, without an accompanying SQL definition. For such snapshots, the user must supply a 'relation' property to indicate the target of the snapshot. This function looks for such snapshots and adds a node to manifest for each one we find, since they were not added during SQL parsing.""" rebuild_refs = False for snapshot in snapshots: if "relation" in snapshot: from dbt.parser import SnapshotParser if "name" not in snapshot: raise ParsingError("A snapshot must define the 'name' property. ") # Reuse the logic of SnapshotParser as far as possible to create # a new node we can add to the manifest. parser = SnapshotParser(self.project, self.manifest, self.root_project) fqn = parser.get_fqn_prefix(block.path.relative_path) fqn.append(snapshot["name"]) compiled_path = snapshot["name"] + ".sql" snapshot_node = parser._create_parsetime_node( block, compiled_path, parser.initial_config(fqn), fqn, snapshot["name"], ) # Parse the expected ref() or source() expression given by # 'relation' so that we know what we are snapshotting. source_or_ref = statically_parse_ref_or_source(snapshot["relation"]) if isinstance(source_or_ref, RefArgs): snapshot_node.refs.append(source_or_ref) else: snapshot_node.sources.append(source_or_ref) # Implement the snapshot SQL as a simple select * snapshot_node.raw_code = "select * from {{ " + snapshot["relation"] + " }}" # Add our new node to the manifest, and note that ref lookup collections # will need to be rebuilt. This adds the node unique_id to the "snapshots" # list in the SchemaSourceFile. self.manifest.add_node(block.file, snapshot_node) rebuild_refs = True if rebuild_refs: self.manifest.rebuild_ref_lookup() Parsed = TypeVar( "Parsed", UnpatchedSourceDefinition, ParsedNodePatch, ParsedMacroPatch, ParsedSingularTestPatch ) NodeTarget = TypeVar( "NodeTarget", UnparsedNodeUpdate, UnparsedAnalysisUpdate, UnparsedModelUpdate, UnparsedFunctionUpdate, ) NonSourceTarget = TypeVar( "NonSourceTarget", UnparsedNodeUpdate, UnparsedAnalysisUpdate, UnparsedMacroUpdate, UnparsedModelUpdate, UnparsedFunctionUpdate, UnparsedSingularTestUpdate, ) @dataclass class ParseResult: test_blocks: List[TestBlock] = field(default_factory=list) versioned_test_blocks: List[VersionedTestBlock] = field(default_factory=list) # abstract base class (ABCMeta) # Many subclasses: MetricParser, ExposureParser, GroupParser, SourceParser, # PatchParser, SemanticModelParser, SavedQueryParser, UnitTestParser class YamlReader(metaclass=ABCMeta): def __init__(self, schema_parser: SchemaParser, yaml: YamlBlock, key: str) -> None: self.schema_parser: SchemaParser = schema_parser # key: models, seeds, snapshots, sources, macros, # analyses, exposures, unit_tests self.key: str = key self.yaml: YamlBlock = yaml self.schema_yaml_vars: SchemaYamlVars = SchemaYamlVars() self.render_ctx = generate_schema_yml_context( self.schema_parser.root_project, self.schema_parser.project.project_name, self.schema_yaml_vars, ) self.renderer: SchemaYamlRenderer = SchemaYamlRenderer(self.render_ctx, self.key) @property def manifest(self) -> Manifest: return self.schema_parser.manifest @property def project(self) -> RuntimeConfig: return self.schema_parser.project @property def default_database(self) -> str: return self.schema_parser.default_database @property def root_project(self) -> RuntimeConfig: return self.schema_parser.root_project # for the different schema subparsers ('models', 'source', etc) # get the list of dicts pointed to by the key in the yaml config, # ensure that the dicts have string keys def get_key_dicts(self) -> Iterable[Dict[str, Any]]: data = self.yaml.data.get(self.key, []) if not isinstance(data, list): raise ParsingError( "{} must be a list, got {} instead: ({})".format( self.key, type(data), trimmed(str(data)) ) ) path = self.yaml.path.original_file_path # for each dict in the data (which is a list of dicts) for entry in data: # check that entry is a dict and that all dict values # are strings if coerce_dict_str(entry) is None: raise YamlParseListError(path, self.key, data, "expected a dict with string keys") if "name" not in entry and "model" not in entry: raise ParsingError("Entry did not contain a name") unrendered_config = {} if "config" in entry: unrendered_config = entry["config"] unrendered_version_configs = {} if "versions" in entry: for version in entry["versions"]: if "v" in version: unrendered_version_configs[version["v"]] = version.get("config", {}) # For sources unrendered_database = entry.get("database", None) unrendered_schema = entry.get("schema", None) # Render the data (except for tests, data_tests and descriptions). # See the SchemaYamlRenderer entry = self.render_entry(entry) schema_file = self.yaml.file assert isinstance(schema_file, SchemaSourceFile) if unrendered_config: schema_file.add_unrendered_config(unrendered_config, self.key, entry["name"]) for version, unrendered_version_config in unrendered_version_configs.items(): schema_file.add_unrendered_config( unrendered_version_config, self.key, entry["name"], version ) if unrendered_database: schema_file.add_unrendered_database(self.key, entry["name"], unrendered_database) if unrendered_schema: schema_file.add_unrendered_schema(self.key, entry["name"], unrendered_schema) if self.schema_yaml_vars.env_vars: self.schema_parser.manifest.env_vars.update(self.schema_yaml_vars.env_vars) for var in self.schema_yaml_vars.env_vars.keys(): schema_file.add_env_var(var, self.key, entry["name"]) self.schema_yaml_vars.env_vars = {} yield entry def render_entry(self, dct): try: # This does a deep_map which will fail if there are circular references dct = self.renderer.render_data(dct) except ParsingError as exc: raise ParsingError( f"Failed to render {self.yaml.file.path.original_file_path} from " f"project {self.project.project_name}: {exc}" ) from exc return dct @abstractmethod def parse(self) -> Optional[ParseResult]: raise NotImplementedError("parse is abstract") T = TypeVar("T", bound=dbtClassMixin) # This parses the 'sources' keys in yaml files. class SourceParser(YamlReader): def _target_from_dict(self, cls: Type[T], data: Dict[str, Any]) -> T: path = self.yaml.path.original_file_path try: cls.validate(data) return cls.from_dict(data) except (ValidationError, JSONValidationError) as exc: raise YamlParseDictError(path, self.key, data, exc) # This parse method takes the yaml dictionaries in 'sources' keys and uses them # to create UnparsedSourceDefinition objects. They are then turned # into UnpatchedSourceDefinition objects in 'add_source_definitions' # or SourcePatch objects in 'add_source_patch' def parse(self) -> ParseResult: # get a verified list of dicts for the key handled by this parser for data in self.get_key_dicts(): data = self.project.credentials.translate_aliases(data, recurse=True) is_override = "overrides" in data if is_override: data["path"] = self.yaml.path.original_file_path patch = self._target_from_dict(SourcePatch, data) assert isinstance(self.yaml.file, SchemaSourceFile) source_file = self.yaml.file # source patches must be unique key = (patch.overrides, patch.name) if key in self.manifest.source_patches: raise DuplicateSourcePatchNameError(patch, self.manifest.source_patches[key]) self.manifest.source_patches[key] = patch source_file.source_patches.append(key) else: source = self._target_from_dict(UnparsedSourceDefinition, data) # Store unrendered_database and unrendered_schema for state:modified comparisons if isinstance(self.yaml.file, SchemaSourceFile): source.unrendered_database = self.yaml.file.get_unrendered_database( "sources", source.name ) source.unrendered_schema = self.yaml.file.get_unrendered_schema( "sources", source.name ) self.add_source_definitions(source) return ParseResult() def add_source_definitions(self, source: UnparsedSourceDefinition) -> None: package_name = self.project.project_name original_file_path = self.yaml.path.original_file_path fqn_path = self.yaml.path.relative_path for table in source.tables: unique_id = ".".join([NodeType.Source, package_name, source.name, table.name]) # the FQN is project name / path elements /source_name /table_name fqn = self.schema_parser.get_fqn_prefix(fqn_path) fqn.extend([source.name, table.name]) source_def = UnpatchedSourceDefinition( source=source, table=table, path=original_file_path, original_file_path=original_file_path, package_name=package_name, unique_id=unique_id, resource_type=NodeType.Source, fqn=fqn, name=f"{source.name}_{table.name}", ) assert isinstance(self.yaml.file, SchemaSourceFile) source_file: SchemaSourceFile = self.yaml.file self.manifest.add_source(source_file, source_def) # This class has two subclasses: NodePatchParser and MacroPatchParser class PatchParser(YamlReader, Generic[NonSourceTarget, Parsed]): @abstractmethod def _target_type(self) -> Type[NonSourceTarget]: raise NotImplementedError("_target_type not implemented") @abstractmethod def get_block(self, node: NonSourceTarget) -> TargetBlock: raise NotImplementedError("get_block is abstract") @abstractmethod def parse_patch(self, block: TargetBlock[NonSourceTarget], refs: ParserRef) -> None: raise NotImplementedError("parse_patch is abstract") def parse(self) -> ParseResult: node: NonSourceTarget # This will always be empty if the node a macro or analysis test_blocks: List[TestBlock] = [] # This will always be empty if the node is _not_ a model versioned_test_blocks: List[VersionedTestBlock] = [] # get list of 'node' objects # UnparsedNodeUpdate (TestablePatchParser, models, seeds, snapshots) # = HasColumnTests, HasTests # UnparsedAnalysisUpdate (UnparsedAnalysisParser, analyses) # = HasColumnDocs, HasDocs # UnparsedMacroUpdate (MacroPatchParser, 'macros') # = HasDocs # correspond to this parser's 'key' for node in self.get_unparsed_target(): # node_block is a TargetBlock (Macro or Analysis) # or a TestBlock (all of the others) node_block = self.get_block(node) if isinstance(node_block, TestBlock): # TestablePatchParser = seeds, snapshots test_blocks.append(node_block) if isinstance(node_block, VersionedTestBlock): # models versioned_test_blocks.append(node_block) if isinstance(node, (HasColumnDocs, HasColumnTests)): # UnparsedNodeUpdate and UnparsedAnalysisUpdate refs: ParserRef = ParserRef.from_target(node) else: refs = ParserRef() # There's no unique_id on the node yet so cannot add to disabled dict self.parse_patch(node_block, refs) return ParseResult(test_blocks, versioned_test_blocks) def get_unparsed_target(self) -> Iterable[NonSourceTarget]: path = self.yaml.path.original_file_path # get verified list of dicts for the 'key' that this # parser handles key_dicts = self.get_key_dicts() for data in key_dicts: # add extra data to each dict. This updates the dicts # in the parser yaml data.update( { "original_file_path": path, "yaml_key": self.key, "package_name": self.project.project_name, } ) try: # target_type: UnparsedNodeUpdate, UnparsedAnalysisUpdate, # or UnparsedMacroUpdate, UnparsedFunctionUpdate self._target_type().validate(data) if self.key != "macros": # macros don't have the 'config' key support yet self.normalize_meta_attribute(data, path) self.normalize_docs_attribute(data, path) self.normalize_group_attribute(data, path) self.normalize_contract_attribute(data, path) self.normalize_access_attribute(data, path) # `tests` has been deprecated, convert to `data_tests` here if present self.validate_data_tests(data) node = self._target_type().from_dict(data) except (ValidationError, JSONValidationError) as exc: raise YamlParseDictError(path, self.key, data, exc) else: yield node # We want to raise an error if some attributes are in two places, and move them # from toplevel to config if necessary def normalize_attribute(self, data, path, attribute) -> None: if attribute in data: if "config" in data and attribute in data["config"]: raise ParsingError( f""" In {path}: found {attribute} dictionary in 'config' dictionary and as top-level key. Remove the top-level key and define it under 'config' dictionary only. """.strip() ) else: if "config" not in data: data["config"] = {} data["config"][attribute] = data.pop(attribute) def normalize_meta_attribute(self, data, path) -> None: return self.normalize_attribute(data, path, "meta") def normalize_docs_attribute(self, data, path) -> None: return self.normalize_attribute(data, path, "docs") def normalize_group_attribute(self, data, path) -> None: return self.normalize_attribute(data, path, "group") def normalize_contract_attribute(self, data, path) -> None: return self.normalize_attribute(data, path, "contract") def normalize_access_attribute(self, data, path) -> None: return self.normalize_attribute(data, path, "access") @property def is_root_project(self) -> bool: if self.root_project.project_name == self.project.project_name: return True return False def validate_data_tests(self, data) -> None: # Rename 'tests' -> 'data_tests' at both model-level and column-level # Raise a validation error if the user has defined both names def validate_and_rename(data, is_root_project: bool) -> None: if data.get("tests"): if "tests" in data and "data_tests" in data: raise ValidationError( "Invalid test config: cannot have both 'tests' and 'data_tests' defined" ) data["data_tests"] = data.pop("tests") # model-level tests validate_and_rename(data, self.is_root_project) # column-level tests if data.get("columns"): for column in data["columns"]: validate_and_rename(column, self.is_root_project) # versioned models if data.get("versions"): for version in data["versions"]: validate_and_rename(version, self.is_root_project) if version.get("columns"): for column in version["columns"]: validate_and_rename(column, self.is_root_project) def patch_node_config(self, node, patch) -> None: if "access" in patch.config: if AccessType.is_valid(patch.config["access"]): patch.config["access"] = AccessType(patch.config["access"]) else: raise InvalidAccessTypeError( unique_id=node.unique_id, field_value=patch.config["access"], ) # Get the ContextConfig that's used in calculating the config # This must match the model resource_type that's being patched config = ContextConfig( self.schema_parser.root_project, node.fqn, node.resource_type, self.schema_parser.project.project_name, ) # We need to re-apply the config_call_dict after the patch config config._config_call_dict = node.config_call_dict config._unrendered_config_call_dict = node.unrendered_config_call_dict self.schema_parser.update_parsed_node_config( node, config, patch_config_dict=patch.config, patch_file_id=patch.file_id, ) # Subclasses of NodePatchParser: TestablePatchParser, ModelPatchParser, AnalysisPatchParser, FunctionPatchParser # so models, seeds, snapshots, analyses, functions class NodePatchParser(PatchParser[NodeTarget, ParsedNodePatch], Generic[NodeTarget]): def _get_node_patch(self, block: TargetBlock[NodeTarget], refs: ParserRef) -> ParsedNodePatch: # We're not passing the ParsedNodePatch around anymore, so we # could possibly skip creating one. Leaving here for now for # code consistency. deprecation_date: Optional[datetime.datetime] = None time_spine: Optional[TimeSpine] = None semantic_model = None metrics = None derived_semantics = None agg_time_dimension = None primary_entity = None if isinstance(block.target, UnparsedModelUpdate): deprecation_date = block.target.deprecation_date time_spine = ( TimeSpine( standard_granularity_column=block.target.time_spine.standard_granularity_column, custom_granularities=[ CustomGranularity( name=custom_granularity.name, column_name=custom_granularity.column_name, ) for custom_granularity in block.target.time_spine.custom_granularities ], ) if block.target.time_spine else None ) semantic_model = block.target.semantic_model metrics = block.target.metrics derived_semantics = block.target.derived_semantics agg_time_dimension = block.target.agg_time_dimension primary_entity = block.target.primary_entity return ParsedNodePatch( name=block.target.name, original_file_path=block.target.original_file_path, yaml_key=block.target.yaml_key, package_name=block.target.package_name, description=block.target.description, columns=refs.column_info, meta=block.target.meta, docs=block.target.docs, config=block.target.config, access=block.target.access, version=None, latest_version=None, constraints=block.target.constraints, deprecation_date=deprecation_date, time_spine=time_spine, semantic_model=semantic_model, metrics=metrics, derived_semantics=derived_semantics, agg_time_dimension=agg_time_dimension, primary_entity=primary_entity, ) def parse_patch(self, block: TargetBlock[NodeTarget], refs: ParserRef) -> None: patch = self._get_node_patch(block, refs) assert isinstance(self.yaml.file, SchemaSourceFile) source_file: SchemaSourceFile = self.yaml.file # TODO: I'd like to refactor this out but the early return makes doing so a bit messy if patch.yaml_key in ["models", "seeds", "snapshots"]: unique_id = self.manifest.ref_lookup.get_unique_id( patch.name, self.project.project_name, None ) or self.manifest.ref_lookup.get_unique_id(patch.name, None, None) if unique_id: resource_type = NodeType(unique_id.split(".")[0]) if resource_type.pluralize() != patch.yaml_key: warn_or_error( WrongResourceSchemaFile( patch_name=patch.name, resource_type=resource_type, plural_resource_type=resource_type.pluralize(), yaml_key=patch.yaml_key, file_path=patch.original_file_path, ) ) return elif patch.yaml_key == "functions": unique_id = self.manifest.function_lookup.get_unique_id(patch.name, None) elif patch.yaml_key == "analyses": unique_id = self.manifest.analysis_lookup.get_unique_id(patch.name, None, None) else: raise DbtInternalError( f"Unexpected yaml_key {patch.yaml_key} for patch in " f"file {source_file.path.original_file_path}" ) # handle disabled nodes if unique_id is None: # Node might be disabled. Following call returns list of matching disabled nodes resource_type = schema_file_keys_to_resource_types[patch.yaml_key] found_nodes = self.manifest.disabled_lookup.find( patch.name, patch.package_name, resource_types=[resource_type] ) if found_nodes: if len(found_nodes) > 1 and patch.config.get("enabled"): # There are multiple disabled nodes for this model and the schema file wants to enable one. # We have no way to know which one to enable. resource_type = found_nodes[0].unique_id.split(".")[0] msg = ( f"Found {len(found_nodes)} matching disabled nodes for " f"{resource_type} '{patch.name}'. Multiple nodes for the same " "unique id cannot be enabled in the schema file. They must be enabled " "in `dbt_project.yml` or in the sql files." ) raise ParsingError(msg) # all nodes in the disabled dict have the same unique_id so just grab the first one # to append with the unique id source_file.append_patch(patch.yaml_key, found_nodes[0].unique_id) for node in found_nodes: node.patch_path = source_file.file_id # re-calculate the node config with the patch config. Always do this # for the case when no config is set to ensure the default of true gets captured if patch.config: self.patch_node_config(node, patch) self.patch_node_properties(node, patch) else: warn_or_error( NoNodeForYamlKey( patch_name=patch.name, yaml_key=patch.yaml_key, file_path=source_file.path.original_file_path, ) ) return # we only return early if no disabled early nodes are found. Why don't we return after patching the disabled nodes? if patch.yaml_key == "functions": node = self.manifest.functions.get(unique_id) else: node = self.manifest.nodes.get(unique_id) if node: # patches can't be overwritten if node.patch_path: package_name, existing_file_path = node.patch_path.split("://") raise DuplicatePatchPathError(patch, existing_file_path) source_file.append_patch(patch.yaml_key, node.unique_id) # re-calculate the node config with the patch config. Always do this # for the case when no config is set to ensure the default of true gets captured if patch.config: self.patch_node_config(node, patch) self.patch_node_properties(node, patch) def patch_node_properties(self, node, patch: "ParsedNodePatch") -> None: """Given a ParsedNodePatch, add the new information to the node.""" # explicitly pick out the parts to update so we don't inadvertently # step on the model name or anything # Note: config should already be updated node.patch_path = patch.file_id # update created_at so process_docs will run in partial parsing node.created_at = time.time() node.description = patch.description node.columns = patch.columns node.name = patch.name if not isinstance(node, ModelNode): for attr in ["latest_version", "access", "version", "constraints"]: if getattr(patch, attr): warn_or_error( ValidationWarning( field_name=attr, resource_type=node.resource_type.value, node_name=patch.name, ) ) # TestablePatchParser = seeds, snapshots class TestablePatchParser(NodePatchParser[UnparsedNodeUpdate]): __test__ = False def get_block(self, node: UnparsedNodeUpdate) -> TestBlock: return TestBlock.from_yaml_block(self.yaml, node) def _target_type(self) -> Type[UnparsedNodeUpdate]: return UnparsedNodeUpdate class ModelPatchParser(NodePatchParser[UnparsedModelUpdate]): def get_block(self, node: UnparsedModelUpdate) -> VersionedTestBlock: return VersionedTestBlock.from_yaml_block(self.yaml, node) def parse_patch(self, block: TargetBlock[UnparsedModelUpdate], refs: ParserRef) -> None: target = block.target if NodeType.Model.pluralize() != target.yaml_key: warn_or_error( WrongResourceSchemaFile( patch_name=target.name, resource_type=NodeType.Model, plural_resource_type=NodeType.Model.pluralize(), yaml_key=target.yaml_key, file_path=target.original_file_path, ) ) return versions = target.versions if not versions: super().parse_patch(block, refs) else: assert isinstance(self.yaml.file, SchemaSourceFile) source_file: SchemaSourceFile = self.yaml.file latest_version = ( target.latest_version if target.latest_version is not None else max(versions).v ) for unparsed_version in versions: versioned_model_name = ( unparsed_version.defined_in or f"{block.name}_{unparsed_version.formatted_v}" ) # ref lookup without version - version is not set yet versioned_model_unique_id = self.manifest.ref_lookup.get_unique_id( versioned_model_name, target.package_name, None ) versioned_model_node: Optional[ModelNode] = None add_node_nofile_fn: Callable # If this is the latest version, it's allowed to define itself in a model file name that doesn't have a suffix if versioned_model_unique_id is None and unparsed_version.v == latest_version: versioned_model_unique_id = self.manifest.ref_lookup.get_unique_id( block.name, target.package_name, None ) if versioned_model_unique_id is None: # Node might be disabled. Following call returns list of matching disabled nodes found_nodes = self.manifest.disabled_lookup.find( versioned_model_name, None, resource_types=[NodeType.Model] ) if found_nodes: if len(found_nodes) > 1 and target.config.get("enabled"): # There are multiple disabled nodes for this model and the schema file wants to enable one. # We have no way to know which one to enable. resource_type = found_nodes[0].unique_id.split(".")[0] msg = ( f"Found {len(found_nodes)} matching disabled nodes for " f"{resource_type} '{target.name}'. Multiple nodes for the same " "unique id cannot be enabled in the schema file. They must be enabled " "in `dbt_project.yml` or in the sql files." ) raise ParsingError(msg) # We know that there's only one node in the disabled list because # otherwise we would have raised the error above found_node = found_nodes[0] self.manifest.disabled.pop(found_node.unique_id) assert isinstance(found_node, ModelNode) versioned_model_node = found_node add_node_nofile_fn = self.manifest.add_disabled_nofile else: found_node = self.manifest.nodes.pop(versioned_model_unique_id) assert isinstance(found_node, ModelNode) versioned_model_node = found_node add_node_nofile_fn = self.manifest.add_node_nofile if versioned_model_node is None: warn_or_error( NoNodeForYamlKey( patch_name=versioned_model_name, yaml_key=target.yaml_key, file_path=source_file.path.original_file_path, ) ) continue # update versioned node unique_id versioned_model_node_unique_id_old = versioned_model_node.unique_id versioned_model_node.unique_id = ( f"model.{target.package_name}.{target.name}.{unparsed_version.formatted_v}" ) # update source file.nodes with new unique_id model_source_file = self.manifest.files[versioned_model_node.file_id] assert isinstance(model_source_file, SourceFile) # because of incomplete test setup, check before removing if versioned_model_node_unique_id_old in model_source_file.nodes: model_source_file.nodes.remove(versioned_model_node_unique_id_old) model_source_file.nodes.append(versioned_model_node.unique_id) # update versioned node fqn versioned_model_node.fqn[-1] = target.name versioned_model_node.fqn.append(unparsed_version.formatted_v) # add versioned node back to nodes/disabled add_node_nofile_fn(versioned_model_node) # flatten columns based on include/exclude version_refs: ParserRef = ParserRef.from_versioned_target( block.target, unparsed_version.v ) versioned_model_patch = ParsedNodePatch( name=target.name, original_file_path=target.original_file_path, yaml_key=target.yaml_key, package_name=target.package_name, description=unparsed_version.description or target.description, columns=version_refs.column_info, meta=target.meta, docs=unparsed_version.docs or target.docs, config=deep_merge(target.config, unparsed_version.config), access=unparsed_version.access or target.access, version=unparsed_version.v, latest_version=latest_version, constraints=unparsed_version.constraints or target.constraints, deprecation_date=unparsed_version.deprecation_date, ) # Node patched before config because config patching depends on model name, # which may have been updated in the version patch # versioned_model_node.patch(versioned_model_patch) self.patch_node_properties(versioned_model_node, versioned_model_patch) # Includes alias recomputation self.patch_node_config(versioned_model_node, versioned_model_patch) # Need to reapply setting constraints and contract checksum here, because # they depend on node.contract.enabled, which wouldn't be set when # patch_node_properties was called if it wasn't set in the model file. self.patch_constraints(versioned_model_node, versioned_model_patch.constraints) versioned_model_node.build_contract_checksum() source_file.append_patch( versioned_model_patch.yaml_key, versioned_model_node.unique_id ) self.manifest.rebuild_ref_lookup() self.manifest.rebuild_disabled_lookup() def _target_type(self) -> Type[UnparsedModelUpdate]: return UnparsedModelUpdate def patch_node_properties(self, node, patch: "ParsedNodePatch") -> None: super().patch_node_properties(node, patch) # Remaining patch properties are only relevant to ModelNode objects if not isinstance(node, ModelNode): return node.version = patch.version node.latest_version = patch.latest_version node.deprecation_date = patch.deprecation_date if patch.access: if AccessType.is_valid(patch.access): node.access = AccessType(patch.access) else: raise InvalidAccessTypeError( unique_id=node.unique_id, field_value=patch.access, ) # breaking out False and None here is wordy but extra clear semantic_model_enabled = patch.semantic_model is True or ( patch.semantic_model is not False and patch.semantic_model is not None and patch.semantic_model.enabled is not False ) if semantic_model_enabled: from dbt.parser.schema_yaml_readers import SemanticModelParser semantic_model_parser = SemanticModelParser(self.schema_parser, self.yaml) semantic_model_parser.parse_v2_semantic_model_from_dbt_model_patch( node=node, patch=patch, ) from dbt.parser.schema_yaml_readers import MetricParser MetricParser(self.schema_parser, self.yaml).parse_v2_metrics_from_dbt_model_patch( patch ) # These two will have to be reapplied after config is built for versioned models self.patch_constraints(node, patch.constraints) self.patch_time_spine(node, patch.time_spine) node.build_contract_checksum() def patch_constraints(self, node: ModelNode, constraints: List[Dict[str, Any]]) -> None: contract_config = node.config.get("contract") if contract_config.enforced is True: self._validate_constraint_prerequisites(node) if any( c for c in constraints if "type" not in c or not ConstraintType.is_valid(c["type"]) ): raise ParsingError( f"Invalid constraint type on model {node.name}: " f"Type must be one of {[ct.value for ct in ConstraintType]}" ) self._validate_pk_constraints(node, constraints) node.constraints = [ModelLevelConstraint.from_dict(c) for c in constraints] self._process_constraints_refs_and_sources(node) def _process_constraints_refs_and_sources(self, model_node: ModelNode) -> None: """ Populate model_node.refs and model_node.sources based on foreign-key constraint references, whether defined at the model-level or column-level. """ for constraint in model_node.all_constraints: if constraint.type == ConstraintType.foreign_key and constraint.to: try: ref_or_source = statically_parse_ref_or_source(constraint.to) except ParsingError: raise ParsingError( f"Invalid 'ref' or 'source' syntax on foreign key constraint 'to' on model {model_node.name}: {constraint.to}." ) if isinstance(ref_or_source, RefArgs): model_node.refs.append(ref_or_source) else: model_node.sources.append(ref_or_source) def patch_time_spine(self, node: ModelNode, time_spine: Optional[TimeSpine]) -> None: node.time_spine = time_spine def _validate_pk_constraints( self, model_node: ModelNode, constraints: List[Dict[str, Any]] ) -> None: errors = [] # check for primary key constraints defined at the column level pk_col: List[str] = [] for col in model_node.columns.values(): for constraint in col.constraints: if constraint.type == ConstraintType.primary_key: pk_col.append(col.name) if len(pk_col) > 1: errors.append( f"Found {len(pk_col)} columns ({pk_col}) with primary key constraints defined. " "Primary keys for multiple columns must be defined as a model level constraint." ) if len(pk_col) > 0 and ( any( constraint.type == ConstraintType.primary_key for constraint in model_node.constraints ) or any(constraint["type"] == ConstraintType.primary_key for constraint in constraints) ): errors.append( "Primary key constraints defined at the model level and the columns level. " "Primary keys can be defined at the model level or the column level, not both." ) if errors: raise ParsingError( f"Primary key constraint error: ({model_node.original_file_path})\n" + "\n".join(errors) ) def _validate_constraint_prerequisites(self, model_node: ModelNode) -> None: column_warn_unsupported = [ constraint.warn_unsupported for column in model_node.columns.values() for constraint in column.constraints ] model_warn_unsupported = [ constraint.warn_unsupported for constraint in model_node.constraints ] warn_unsupported = column_warn_unsupported + model_warn_unsupported # if any constraint has `warn_unsupported` as True then send the warning if any(warn_unsupported) and not model_node.materialization_enforces_constraints: warn_or_error( UnsupportedConstraintMaterialization(materialized=model_node.config.materialized), node=model_node, ) errors = [] if not model_node.columns: errors.append( "Constraints must be defined in a `yml` schema configuration file like `schema.yml`." ) if str(model_node.language) != "sql": errors.append(f"Language Error: Expected 'sql' but found '{model_node.language}'") if errors: raise ParsingError( f"Contract enforcement failed for: ({model_node.original_file_path})\n" + "\n".join(errors) ) class AnalysisPatchParser(NodePatchParser[UnparsedAnalysisUpdate]): def get_block(self, node: UnparsedAnalysisUpdate) -> TargetBlock: return TargetBlock.from_yaml_block(self.yaml, node) def _target_type(self) -> Type[UnparsedAnalysisUpdate]: return UnparsedAnalysisUpdate class SingularTestPatchParser(PatchParser[UnparsedSingularTestUpdate, ParsedSingularTestPatch]): def get_block(self, node: UnparsedSingularTestUpdate) -> TargetBlock: return TargetBlock.from_yaml_block(self.yaml, node) def _target_type(self) -> Type[UnparsedSingularTestUpdate]: return UnparsedSingularTestUpdate def parse_patch(self, block: TargetBlock[UnparsedSingularTestUpdate], refs: ParserRef) -> None: patch = ParsedSingularTestPatch( name=block.target.name, description=block.target.description, meta=block.target.meta, docs=block.target.docs, config=block.target.config, original_file_path=block.target.original_file_path, yaml_key=block.target.yaml_key, package_name=block.target.package_name, ) assert isinstance(self.yaml.file, SchemaSourceFile) source_file: SchemaSourceFile = self.yaml.file unique_id = self.manifest.singular_test_lookup.get_unique_id( block.name, block.target.package_name ) if not unique_id: warn_or_error( NoNodeForYamlKey( patch_name=patch.name, yaml_key=patch.yaml_key, file_path=source_file.path.original_file_path, ) ) return node = self.manifest.nodes.get(unique_id) assert node is not None source_file.append_patch(patch.yaml_key, unique_id) if patch.config: self.patch_node_config(node, patch) node.patch_path = patch.file_id node.description = patch.description node.created_at = time.time() class FunctionPatchParser(NodePatchParser[UnparsedFunctionUpdate]): def get_block(self, node: UnparsedFunctionUpdate) -> TargetBlock: return TargetBlock.from_yaml_block(self.yaml, node) def _target_type(self) -> Type[UnparsedFunctionUpdate]: return UnparsedFunctionUpdate def patch_node_properties(self, node, patch: "ParsedNodePatch") -> None: super().patch_node_properties(node, patch) assert isinstance(patch, ParsedFunctionPatch) assert isinstance(node, FunctionNode) node.arguments = patch.arguments node.returns = patch.returns def _get_node_patch(self, block: TargetBlock[NodeTarget], refs: ParserRef) -> ParsedNodePatch: target = block.target assert isinstance(target, UnparsedFunctionUpdate) return ParsedFunctionPatch( name=target.name, original_file_path=target.original_file_path, yaml_key=target.yaml_key, package_name=target.package_name, description=target.description, columns=refs.column_info, meta=target.meta, docs=target.docs, config=target.config, access=target.access, version=None, latest_version=None, constraints=target.constraints, deprecation_date=None, time_spine=None, arguments=target.arguments, returns=target.returns, ) class MacroPatchParser(PatchParser[UnparsedMacroUpdate, ParsedMacroPatch]): def get_block(self, node: UnparsedMacroUpdate) -> TargetBlock: return TargetBlock.from_yaml_block(self.yaml, node) def _target_type(self) -> Type[UnparsedMacroUpdate]: return UnparsedMacroUpdate def parse_patch(self, block: TargetBlock[UnparsedMacroUpdate], refs: ParserRef) -> None: patch = ParsedMacroPatch( name=block.target.name, original_file_path=block.target.original_file_path, yaml_key=block.target.yaml_key, package_name=block.target.package_name, arguments=block.target.arguments, description=block.target.description, meta=block.target.meta, docs=block.target.docs, config=block.target.config, ) assert isinstance(self.yaml.file, SchemaSourceFile) source_file = self.yaml.file # macros are fully namespaced unique_id = f"macro.{patch.package_name}.{patch.name}" macro = self.manifest.macros.get(unique_id) if not macro: warn_or_error(MacroNotFoundForPatch(patch_name=patch.name)) return if macro.patch_path: package_name, existing_file_path = macro.patch_path.split("://") raise DuplicateMacroPatchNameError(patch, existing_file_path) source_file.macro_patches[patch.name] = unique_id # former macro.patch code macro.patch_path = patch.file_id macro.description = patch.description macro.created_at = time.time() meta = {**(patch.meta or {}), **(patch.config.get("meta") or {})} docs = patch.config.get("docs") or patch.docs # config inherits from HasConfig which is a dict so we need to cast it to Docs if isinstance(docs, dict): docs = Docs(**docs) macro.meta = meta macro.docs = docs macro.config.meta = meta macro.config.docs = docs if getattr(get_flags(), "validate_macro_args", False): self._check_patch_arguments(macro, patch) macro.arguments = patch.arguments if patch.arguments else macro.arguments else: macro.arguments = patch.arguments def _check_patch_arguments(self, macro: Macro, patch: ParsedMacroPatch) -> None: if not patch.arguments: return for macro_arg, patch_arg in zip(macro.arguments, patch.arguments): if patch_arg.name != macro_arg.name: msg = f"Argument {patch_arg.name} in yaml for macro {macro.name} does not match the jinja definition." self._fire_macro_arg_warning(msg, macro) if len(patch.arguments) != len(macro.arguments): msg = f"The number of arguments in the yaml for macro {macro.name} does not match the jinja definition." self._fire_macro_arg_warning(msg, macro) for patch_arg in patch.arguments: arg_type = patch_arg.type if arg_type is not None and arg_type.strip() != "" and not is_valid_type(arg_type): msg = f"Argument {patch_arg.name} in the yaml for macro {macro.name} has an invalid type." self._fire_macro_arg_warning(msg, macro) def _fire_macro_arg_warning(self, msg: str, macro: Macro) -> None: warn_or_error( InvalidMacroAnnotation( msg=msg, macro_unique_id=macro.unique_id, macro_file_path=macro.original_file_path ) ) # valid type names, along with the number of parameters they require macro_types: Dict[str, int] = { "str": 0, "string": 0, "bool": 0, "int": 0, "integer": 0, "float": 0, "any": 0, "list": 1, "dict": 2, "optional": 1, "relation": 0, "column": 0, } def is_valid_type(buffer: str) -> bool: buffer = buffer.replace(" ", "").replace("\t", "") type_desc, remainder = match_type_desc(buffer) return type_desc is not None and remainder == "" def match_type_desc(buffer: str) -> Tuple[Optional[str], str]: """A matching buffer is a type name followed by an argument list with the correct number of arguments.""" type_name, remainder = match_type_name(buffer) if type_name is None: return None, buffer attr_list, remainder = match_arg_list(remainder, macro_types[type_name]) if attr_list is None: return None, buffer return type_name + attr_list, remainder alpha_pattern = re.compile(r"[a-z]+") def match_type_name(buffer: str) -> Tuple[Optional[str], str]: """A matching buffer starts with one of the valid type names from macro_types""" match = alpha_pattern.match(buffer) if match is not None and buffer[: match.end(0)] in macro_types: return buffer[: match.end(0)], buffer[match.end(0) :] else: return None, buffer def match_arg_list(buffer: str, arg_count: int) -> Tuple[Optional[str], str]: """A matching buffer must begin with '[', followed by exactly arg_count type specs, followed by ']'""" if arg_count == 0: return "", buffer if not buffer.startswith("["): return None, buffer remainder = buffer[1:] for i in range(arg_count): type_desc, remainder = match_type_desc(remainder) if type_desc is None: return None, buffer if i != arg_count - 1: if not remainder.startswith(","): return None, buffer remainder = remainder[1:] if not remainder.startswith("]"): return None, buffer else: return "", remainder[1:] ================================================ FILE: core/dbt/parser/search.py ================================================ import os from dataclasses import dataclass from typing import ( Callable, Generic, Iterable, Iterator, List, Optional, Set, TypeVar, Union, ) from pathspec import PathSpec # type: ignore from dbt import deprecations from dbt.config import Project from dbt.contracts.files import AnySourceFile, FilePath from dbt.exceptions import DbtInternalError, ParsingError from dbt_common.clients._jinja_blocks import ExtractWarning from dbt_common.clients.jinja import BlockTag, extract_toplevel_blocks from dbt_common.clients.system import find_matching # What's the point of wrapping a SourceFile with this class? # Could it be removed? @dataclass class FileBlock: file: AnySourceFile @property def name(self): base = os.path.basename(self.file.path.relative_path) name, _ = os.path.splitext(base) return name @property def contents(self): return self.file.contents @property def path(self): return self.file.path # The BlockTag is used in Jinja processing # Why do we have different classes where the only # difference is what 'contents' returns? @dataclass class BlockContents(FileBlock): file: AnySourceFile # if you remove this, mypy will get upset block: BlockTag @property def name(self): return self.block.block_name @property def contents(self): return self.block.contents @dataclass class FullBlock(FileBlock): file: AnySourceFile # if you remove this, mypy will get upset block: BlockTag @property def name(self): return self.block.block_name @property def contents(self): return self.block.full_block def filesystem_search( project: Project, relative_dirs: List[str], extension: str, ignore_spec: Optional[PathSpec] = None, ): ext = "[!.#~]*" + extension root = project.project_root file_path_list = [] for result in find_matching(root, relative_dirs, ext, ignore_spec): if "searched_path" not in result or "relative_path" not in result: raise DbtInternalError("Invalid result from find_matching: {}".format(result)) file_match = FilePath( searched_path=result["searched_path"], relative_path=result["relative_path"], modification_time=result["modification_time"], project_root=root, ) file_path_list.append(file_match) return file_path_list Block = Union[BlockContents, FullBlock] BlockSearchResult = TypeVar("BlockSearchResult", BlockContents, FullBlock) BlockSearchResultFactory = Callable[[AnySourceFile, BlockTag], BlockSearchResult] class BlockSearcher(Generic[BlockSearchResult], Iterable[BlockSearchResult]): def __init__( self, source: List[FileBlock], allowed_blocks: Set[str], source_tag_factory: BlockSearchResultFactory, check_jinja: bool = True, ) -> None: self.source = source self.allowed_blocks = allowed_blocks self.source_tag_factory: BlockSearchResultFactory = source_tag_factory self.check_jinja = check_jinja def extract_blocks(self, source_file: FileBlock) -> Iterable[BlockTag]: # This is a bit of a hack to get the file path to the deprecation def wrap_handle_extract_warning(warning: ExtractWarning) -> None: self._handle_extract_warning(warning=warning, file=source_file.path.relative_path) try: blocks = extract_toplevel_blocks( source_file.contents, allowed_blocks=self.allowed_blocks, collect_raw_data=False, warning_callback=wrap_handle_extract_warning if self.check_jinja else None, ) # this makes mypy happy, and this is an invariant we really need for block in blocks: assert isinstance(block, BlockTag) yield block except ParsingError as exc: if exc.node is None: exc.add_node(source_file) raise def _handle_extract_warning(self, warning: ExtractWarning, file: str) -> None: deprecations.warn("unexpected-jinja-block-deprecation", msg=warning.msg, file=file) def __iter__(self) -> Iterator[BlockSearchResult]: for entry in self.source: for block in self.extract_blocks(entry): yield self.source_tag_factory(entry.file, block) ================================================ FILE: core/dbt/parser/seeds.py ================================================ from dbt.context.context_config import ContextConfig from dbt.contracts.graph.nodes import SeedNode from dbt.node_types import NodeType from dbt.parser.base import SimpleSQLParser from dbt.parser.search import FileBlock class SeedParser(SimpleSQLParser[SeedNode]): def parse_from_dict(self, dct, validate=True) -> SeedNode: # seeds need the root_path because the contents are not loaded dct["root_path"] = self.project.project_root if "language" in dct: del dct["language"] # raw_code is not currently used, but it might be in the future if validate: SeedNode.validate(dct) return SeedNode.from_dict(dct) @property def resource_type(self) -> NodeType: return NodeType.Seed @classmethod def get_compiled_path(cls, block: FileBlock): return block.path.relative_path def render_with_context(self, parsed_node: SeedNode, config: ContextConfig) -> None: """Seeds don't need to do any rendering.""" ================================================ FILE: core/dbt/parser/singular_test.py ================================================ from dbt.contracts.graph.nodes import SingularTestNode from dbt.node_types import NodeType from dbt.parser.base import SimpleSQLParser from dbt.parser.search import FileBlock from dbt.utils import get_pseudo_test_path class SingularTestParser(SimpleSQLParser[SingularTestNode]): def parse_from_dict(self, dct, validate=True) -> SingularTestNode: if validate: SingularTestNode.validate(dct) return SingularTestNode.from_dict(dct) @property def resource_type(self) -> NodeType: return NodeType.Test @classmethod def get_compiled_path(cls, block: FileBlock): return get_pseudo_test_path(block.name, block.path.relative_path) ================================================ FILE: core/dbt/parser/snapshots.py ================================================ import os from typing import List from dbt.context.context_config import ContextConfig from dbt.contracts.graph.nodes import SnapshotNode from dbt.node_types import NodeType from dbt.parser.base import SQLParser from dbt.parser.search import BlockContents, BlockSearcher, FileBlock from dbt.utils import split_path class SnapshotParser(SQLParser[SnapshotNode]): def parse_from_dict(self, dct, validate=True) -> SnapshotNode: if validate: SnapshotNode.validate(dct) return SnapshotNode.from_dict(dct) @property def resource_type(self) -> NodeType: return NodeType.Snapshot @classmethod def get_compiled_path(cls, block: FileBlock): return block.name + ".sql" def get_fqn(self, path: str, name: str) -> List[str]: """Get the FQN for the node. This impacts node selection and config application. On snapshots, the fqn includes the filename. """ no_ext = os.path.splitext(path)[0] fqn = [self.project.project_name] fqn.extend(split_path(no_ext)) fqn.append(name) return fqn def parse_node(self, block: FileBlock) -> SnapshotNode: compiled_path: str = self.get_compiled_path(block) # Use the file's relative_path for FQN computation (not the compiled_path) # to maintain backward compatibility. The compiled_path is based on the # snapshot name for uniqueness (multiple snapshots can share one file), # but the FQN should still reflect the file structure. fqn = self.get_fqn(block.path.relative_path, block.name) config: ContextConfig = self.initial_config(fqn) node = self._create_parsetime_node( block=block, path=compiled_path, config=config, fqn=fqn, ) self.render_update(node, config) self.add_result_node(block, node) return node def parse_file(self, file_block: FileBlock) -> None: blocks = BlockSearcher( source=[file_block], allowed_blocks={"snapshot"}, source_tag_factory=BlockContents, ) for block in blocks: self.parse_node(block) ================================================ FILE: core/dbt/parser/sources.py ================================================ import itertools from dataclasses import replace from pathlib import Path from typing import Any, Dict, Iterable, List, Optional, Set, Tuple from dbt.adapters.capability import Capability from dbt.adapters.factory import get_adapter from dbt.artifacts.resources import FreshnessThreshold, SourceConfig, Time from dbt.config import RuntimeConfig from dbt.context.context_config import ( BaseContextConfigGenerator, ContextConfigGenerator, UnrenderedConfigGenerator, ) from dbt.contracts.graph.manifest import Manifest, SourceKey from dbt.contracts.graph.nodes import ( GenericTestNode, SourceDefinition, UnpatchedSourceDefinition, ) from dbt.contracts.graph.unparsed import ( SourcePatch, SourceTablePatch, UnparsedColumn, UnparsedSourceDefinition, UnparsedSourceTableDefinition, ) from dbt.events.types import FreshnessConfigProblem, UnusedTables, ValidationWarning from dbt.exceptions import ParsingError from dbt.node_types import NodeType from dbt.parser.common import ParserRef from dbt.parser.schema_generic_tests import SchemaGenericTestParser from dbt_common.events.functions import fire_event, warn_or_error from dbt_common.exceptions import DbtInternalError # An UnparsedSourceDefinition is taken directly from the yaml # file. It can affect multiple tables, all of which will eventually # have their own source node. An UnparsedSourceDefinition will # generate multiple UnpatchedSourceDefinition nodes (one per # table) in the SourceParser.add_source_definitions. The # SourcePatcher takes an UnparsedSourceDefinition and the # SourcePatch and produces a SourceDefinition. Each # SourcePatch can be applied to multiple UnpatchedSourceDefinitions. class SourcePatcher: def __init__( self, root_project: RuntimeConfig, manifest: Manifest, ) -> None: self.root_project = root_project self.manifest = manifest self.generic_test_parsers: Dict[str, SchemaGenericTestParser] = {} self.patches_used: Dict[SourceKey, Set[str]] = {} self.sources: Dict[str, SourceDefinition] = {} self._deprecations: Set[Any] = set() # This method calls the 'parse_source' method which takes # the UnpatchedSourceDefinitions in the manifest and combines them # with SourcePatches to produce SourceDefinitions. def construct_sources(self) -> None: for unique_id, unpatched in self.manifest.sources.items(): schema_file = self.manifest.files[unpatched.file_id] if isinstance(unpatched, SourceDefinition): # In partial parsing, there will be SourceDefinitions # which must be retained. self.sources[unpatched.unique_id] = unpatched continue # returns None if there is no patch patch = self.get_patch_for(unpatched) # returns unpatched if there is no patch patched = self.patch_source(unpatched, patch) # now use the patched UnpatchedSourceDefinition to extract test data. for test in self.get_source_tests(patched): if test.config.enabled: self.manifest.add_node_nofile(test) else: self.manifest.add_disabled_nofile(test) # save the test unique_id in the schema_file, so we can # process in partial parsing test_from = {"key": "sources", "name": patched.source.name} schema_file.add_test(test.unique_id, test_from) # Convert UnpatchedSourceDefinition to a SourceDefinition parsed = self.parse_source(patched) if parsed.config.enabled: self.sources[unique_id] = parsed else: self.manifest.add_disabled_nofile(parsed) self.warn_unused() def patch_source( self, unpatched: UnpatchedSourceDefinition, patch: Optional[SourcePatch], ) -> UnpatchedSourceDefinition: # This skips patching if no patch exists because of the # performance overhead of converting to and from dicts if patch is None: return unpatched source_dct = unpatched.source.to_dict(omit_none=True) table_dct = unpatched.table.to_dict(omit_none=True) patch_path: Optional[Path] = None source_table_patch: Optional[SourceTablePatch] = None if patch is not None: source_table_patch = patch.get_table_named(unpatched.table.name) source_dct.update(patch.to_patch_dict()) patch_path = patch.path if source_table_patch is not None: table_dct.update(source_table_patch.to_patch_dict()) source = UnparsedSourceDefinition.from_dict(source_dct) table = UnparsedSourceTableDefinition.from_dict(table_dct) return replace(unpatched, source=source, table=table, patch_path=patch_path) # This converts an UnpatchedSourceDefinition to a SourceDefinition def parse_source(self, target: UnpatchedSourceDefinition) -> SourceDefinition: source = target.source table = target.table refs = ParserRef.from_target(table) unique_id = target.unique_id description = table.description or "" source_description = source.description or "" quoting = source.quoting.merged(table.quoting) # Retain original source meta prior to merge with table meta source_meta = {**source.meta, **source.config.get("meta", {})} config = self._generate_source_config( target=target, rendered=True, ) config = config.finalize_and_validate() unrendered_config = self._generate_source_config( target=target, rendered=False, ) if not isinstance(config, SourceConfig): raise DbtInternalError( f"Calculated a {type(config)} for a source, but expected a SourceConfig" ) default_database = self.root_project.credentials.database parsed_source = SourceDefinition( package_name=target.package_name, database=(source.database or default_database), unrendered_database=source.unrendered_database, schema=(source.schema or source.name), unrendered_schema=source.unrendered_schema, identifier=(table.identifier or table.name), path=target.path, original_file_path=target.original_file_path, columns=refs.column_info, unique_id=unique_id, name=table.name, description=description, external=table.external, source_name=source.name, source_description=source_description, source_meta=source_meta, meta=config.meta, loader=source.loader, loaded_at_field=config.loaded_at_field, loaded_at_query=config.loaded_at_query, freshness=config.freshness, quoting=quoting, resource_type=NodeType.Source, fqn=target.fqn, tags=config.tags, config=config, unrendered_config=unrendered_config, ) if ( parsed_source.freshness and not parsed_source.loaded_at_field and not parsed_source.loaded_at_query and not get_adapter(self.root_project).supports(Capability.TableLastModifiedMetadata) ): # Metadata-based freshness is being used by default for this node, # but is not available through the configured adapter, so warn the # user that freshness info will not be collected for this node at # runtime. fire_event( FreshnessConfigProblem( msg=f"The configured adapter does not support metadata-based freshness. A loaded_at_field must be specified for source '{source.name}.{table.name}'." ) ) # relation name is added after instantiation because the adapter does # not provide the relation name for a UnpatchedSourceDefinition object parsed_source.relation_name = self._get_relation_name(parsed_source) return parsed_source # Use the SchemaGenericTestParser to parse the source tests def get_generic_test_parser_for(self, package_name: str) -> "SchemaGenericTestParser": if package_name in self.generic_test_parsers: generic_test_parser = self.generic_test_parsers[package_name] else: all_projects = self.root_project.load_dependencies() project = all_projects[package_name] generic_test_parser = SchemaGenericTestParser( project, self.manifest, self.root_project ) self.generic_test_parsers[package_name] = generic_test_parser return generic_test_parser def get_source_tests(self, target: UnpatchedSourceDefinition) -> Iterable[GenericTestNode]: is_root_project = True if self.root_project.project_name == target.package_name else False target.validate_data_tests(is_root_project) for data_test, column in target.get_tests(): yield self.parse_source_test( target=target, data_test=data_test, column=column, ) def get_patch_for( self, unpatched: UnpatchedSourceDefinition, ) -> Optional[SourcePatch]: if isinstance(unpatched, SourceDefinition): return None key = (unpatched.package_name, unpatched.source.name) patch: Optional[SourcePatch] = self.manifest.source_patches.get(key) if patch is None: return None if key not in self.patches_used: # mark the key as used self.patches_used[key] = set() if patch.get_table_named(unpatched.table.name) is not None: self.patches_used[key].add(unpatched.table.name) return patch # This calls parse_generic_test in the SchemaGenericTestParser def parse_source_test( self, target: UnpatchedSourceDefinition, data_test: Dict[str, Any], column: Optional[UnparsedColumn], ) -> GenericTestNode: column_name: Optional[str] if column is None: column_name = None else: column_name = column.name should_quote = column.quote or (column.quote is None and target.quote_columns) if should_quote: column_name = get_adapter(self.root_project).quote(column_name) tags_sources = [target.source.tags, target.table.tags] if column is not None: tags_sources.append(column.tags) if column_config_tags := column.config.get("tags", []): if isinstance(column_config_tags, list): tags_sources.append(column_config_tags) elif isinstance(column_config_tags, str): tags_sources.append([column_config_tags]) tags = list(itertools.chain.from_iterable(tags_sources)) generic_test_parser = self.get_generic_test_parser_for(target.package_name) node = generic_test_parser.parse_generic_test( target=target, data_test=data_test, tags=tags, column_name=column_name, schema_file_id=target.file_id, version=None, ) return node def _generate_source_config(self, target: UnpatchedSourceDefinition, rendered: bool): generator: BaseContextConfigGenerator if rendered: generator = ContextConfigGenerator(self.root_project) else: generator = UnrenderedConfigGenerator(self.root_project) # configs with precendence set precedence_configs = dict() # first apply source configs precedence_configs.update(target.source.config) # then overrite anything that is defined on source tables # this is not quite complex enough for configs that can be set as top-level node keys, but # it works while source configs can only include `enabled`. precedence_configs.update(target.table.config) precedence_freshness = self.calculate_freshness_from_raw_target(target) if precedence_freshness: precedence_configs["freshness"] = precedence_freshness.to_dict() elif precedence_freshness is None: precedence_configs["freshness"] = None else: # this means that the user did not set a freshness threshold in the source schema file, as such # there should be no freshness precedence precedence_configs.pop("freshness", None) precedence_loaded_at_field, precedence_loaded_at_query = ( self.calculate_loaded_at_field_query_from_raw_target(target) ) precedence_configs["loaded_at_field"] = precedence_loaded_at_field precedence_configs["loaded_at_query"] = precedence_loaded_at_query # Handle merges across source, table, and config for meta and tags precedence_meta = self.calculate_meta_from_raw_target(target) precedence_configs["meta"] = precedence_meta precedence_tags = self.calculate_tags_from_raw_target(target) precedence_configs["tags"] = precedence_tags # Because freshness is a "object" config, the freshness from the dbt_project.yml and the freshness # from the schema file _won't_ get merged by this process. The result will be that the freshness will # come from the schema file if provided, and if not, it'll fall back to the dbt_project.yml freshness. return generator.calculate_node_config( config_call_dict={}, fqn=target.fqn, resource_type=NodeType.Source, project_name=target.package_name, base=False, patch_config_dict=precedence_configs, ) def _get_relation_name(self, node: SourceDefinition): adapter = get_adapter(self.root_project) relation_cls = adapter.Relation return str(relation_cls.create_from(self.root_project, node)) def warn_unused(self) -> None: unused_tables: Dict[SourceKey, Optional[Set[str]]] = {} for patch in self.manifest.source_patches.values(): key = (patch.overrides, patch.name) if key not in self.patches_used: unused_tables[key] = None elif patch.tables is not None: table_patches = {t.name for t in patch.tables} unused = table_patches - self.patches_used[key] # don't add unused tables, the if unused: # because patches are required to be unique, we can safely # write without looking unused_tables[key] = unused if unused_tables: unused_tables_formatted = self.get_unused_msg(unused_tables) warn_or_error(UnusedTables(unused_tables=unused_tables_formatted)) self.manifest.source_patches = {} def get_unused_msg( self, unused_tables: Dict[SourceKey, Optional[Set[str]]], ) -> List: unused_tables_formatted = [] for key, table_names in unused_tables.items(): patch = self.manifest.source_patches[key] patch_name = f"{patch.overrides}.{patch.name}" if table_names is None: unused_tables_formatted.append(f" - Source {patch_name} (in {patch.path})") else: for table_name in sorted(table_names): unused_tables_formatted.append( f" - Source table {patch_name}.{table_name} " f"(in {patch.path})" ) return unused_tables_formatted def calculate_freshness_from_raw_target( self, target: UnpatchedSourceDefinition, ) -> Optional[FreshnessThreshold]: source: UnparsedSourceDefinition = target.source source_freshness = source.freshness source_config_freshness_raw: Optional[Dict] = source.config.get( "freshness", {} ) # Will only be None if the user explicitly set it to null source_config_freshness: Optional[FreshnessThreshold] = ( FreshnessThreshold.from_dict(source_config_freshness_raw) if source_config_freshness_raw is not None else None ) table: UnparsedSourceTableDefinition = target.table table_freshness = table.freshness table_config_freshness_raw: Optional[Dict] = table.config.get( "freshness", {} ) # Will only be None if the user explicitly set it to null table_config_freshness: Optional[FreshnessThreshold] = ( FreshnessThreshold.from_dict(table_config_freshness_raw) if table_config_freshness_raw is not None else None ) return merge_source_freshness( source_freshness, source_config_freshness, table_freshness, table_config_freshness, ) def calculate_loaded_at_field_query_from_raw_target( self, target: UnpatchedSourceDefinition ) -> Tuple[Optional[str], Optional[str]]: # We need to be able to tell the difference between explicitly setting the loaded_at_field to None/null # and when it's simply not set. This allows a user to override the source level loaded_at_field so that # specific table can default to metadata-based freshness. # loaded_at_field and loaded_at_query are supported both at top-level (deprecated) and config-level (preferred) on sources and tables. if target.table.loaded_at_field_present and ( target.table.loaded_at_query or target.table.config.get("loaded_at_query") ): raise ParsingError( "Cannot specify both loaded_at_field and loaded_at_query at table level." ) if (target.source.loaded_at_field or target.source.config.get("loaded_at_field")) and ( target.source.loaded_at_query or target.source.config.get("loaded_at_query") ): raise ParsingError( "Cannot specify both loaded_at_field and loaded_at_query at source level." ) if ( target.table.loaded_at_field_present or target.table.loaded_at_field is not None or target.table.config.get("loaded_at_field") is not None ): loaded_at_field = target.table.loaded_at_field or target.table.config.get( "loaded_at_field" ) else: loaded_at_field = target.source.loaded_at_field or target.source.config.get( "loaded_at_field" ) # may be None, that's okay loaded_at_query: Optional[str] if ( target.table.loaded_at_query is not None or target.table.config.get("loaded_at_query") is not None ): loaded_at_query = target.table.loaded_at_query or target.table.config.get( "loaded_at_query" ) else: if target.table.loaded_at_field_present: loaded_at_query = None else: loaded_at_query = target.source.loaded_at_query or target.source.config.get( "loaded_at_query" ) return loaded_at_field, loaded_at_query def calculate_meta_from_raw_target(self, target: UnpatchedSourceDefinition) -> Dict[str, Any]: source_meta = target.source.meta or {} source_config_meta = target.source.config.get("meta", {}) source_config_meta = source_config_meta if isinstance(source_config_meta, dict) else {} table_meta = target.table.meta or {} table_config_meta = target.table.config.get("meta", {}) table_config_meta = table_config_meta if isinstance(table_config_meta, dict) else {} return {**source_meta, **source_config_meta, **table_meta, **table_config_meta} def calculate_tags_from_raw_target(self, target: UnpatchedSourceDefinition) -> List[str]: source_tags = target.source.tags or [] source_config_tags = self._get_config_tags( target.source.config.get("tags", []), target.source.name ) table_tags = target.table.tags or [] table_config_tags = self._get_config_tags( target.table.config.get("tags", []), target.table.name ) return sorted( set(itertools.chain(source_tags, source_config_tags, table_tags, table_config_tags)) ) def _get_config_tags(self, tags: Any, source_name: str) -> List[str]: config_tags = tags if isinstance(tags, list) else [tags] config_tags_valid: List[str] = [] for tag in config_tags: if not isinstance(tag, str): warn_or_error( ValidationWarning( field_name=f"`config.tags`: {tags}", resource_type=NodeType.Source.value, node_name=source_name, ) ) else: config_tags_valid.append(tag) return config_tags_valid def merge_freshness_time_thresholds( base: Optional[Time], update: Optional[Time] ) -> Optional[Time]: if base and update: return base.merged(update) elif update is None: return None else: return update or base def merge_source_freshness( *thresholds: Optional[FreshnessThreshold], ) -> Optional[FreshnessThreshold]: if not thresholds: return None # Initialize with the first threshold. # If the first threshold is None, current_merged_value will be None, # and subsequent merges will correctly follow the original logic. current_merged_value: Optional[FreshnessThreshold] = thresholds[0] # Iterate through the rest of the thresholds, applying the original pairwise logic for i in range(1, len(thresholds)): base = current_merged_value update = thresholds[i] if base is not None and update is not None: merged_freshness_obj = base.merged(update) # merge one level deeper the error_after and warn_after thresholds merged_error_after = merge_freshness_time_thresholds( base.error_after, update.error_after ) merged_warn_after = merge_freshness_time_thresholds(base.warn_after, update.warn_after) merged_freshness_obj.error_after = merged_error_after merged_freshness_obj.warn_after = merged_warn_after current_merged_value = merged_freshness_obj elif base is None and bool(update): # If current_merged_value (base) is None, the update becomes the new value current_merged_value = update else: # This covers cases where 'update' is None, or both 'base' and 'update' are None. # Following original logic, if 'update' is None, the result of the pair-merge is None. current_merged_value = None return current_merged_value ================================================ FILE: core/dbt/parser/sql.py ================================================ import os from dataclasses import dataclass from typing import Iterable from dbt.contracts.graph.manifest import SourceFile from dbt.contracts.graph.nodes import Macro, SqlNode from dbt.contracts.graph.unparsed import UnparsedMacro from dbt.node_types import NodeType from dbt.parser.base import SimpleSQLParser from dbt.parser.macros import MacroParser from dbt.parser.search import FileBlock from dbt_common.exceptions import DbtInternalError @dataclass class SqlBlock(FileBlock): block_name: str @property def name(self): return self.block_name class SqlBlockParser(SimpleSQLParser[SqlNode]): def parse_from_dict(self, dct, validate=True) -> SqlNode: if validate: SqlNode.validate(dct) return SqlNode.from_dict(dct) @property def resource_type(self) -> NodeType: return NodeType.SqlOperation @staticmethod def get_compiled_path(block: FileBlock): # we do it this way to make mypy happy if not isinstance(block, SqlBlock): raise DbtInternalError( "While parsing SQL operation, got an actual file block instead of " "an SQL block: {}".format(block) ) return os.path.join("sql", block.name) def parse_remote(self, sql: str, name: str) -> SqlNode: source_file = SourceFile.remote(sql, self.project.project_name, "sql") contents = SqlBlock(block_name=name, file=source_file) return self.parse_node(contents) class SqlMacroParser(MacroParser): def parse_remote(self, contents) -> Iterable[Macro]: base = UnparsedMacro( path="from remote system", original_file_path="from remote system", package_name=self.project.project_name, raw_code=contents, language="sql", resource_type=NodeType.Macro, ) for node in self.parse_unparsed_macros(base): yield node ================================================ FILE: core/dbt/parser/unit_tests.py ================================================ import csv import os from copy import deepcopy from csv import DictReader from io import StringIO from pathlib import Path from typing import Any, Dict, List, Optional, Set from dbt import utils from dbt.artifacts.resources import ModelConfig, UnitTestConfig, UnitTestFormat from dbt.config import RuntimeConfig from dbt.context.context_config import ContextConfig from dbt.context.providers import generate_parser_unit_test_context, get_rendered from dbt.contracts.files import FileHash, SchemaSourceFile from dbt.contracts.graph.manifest import Manifest from dbt.contracts.graph.model_config import UnitTestNodeConfig from dbt.contracts.graph.nodes import ( DependsOn, ModelNode, UnitTestDefinition, UnitTestNode, UnitTestSourceDefinition, ) from dbt.contracts.graph.unparsed import UnparsedUnitTest from dbt.exceptions import InvalidUnitTestGivenInput, ParsingError from dbt.flags import get_flags from dbt.graph import UniqueId from dbt.node_types import NodeType from dbt.parser.schemas import ( JSONValidationError, ParseResult, SchemaParser, ValidationError, YamlBlock, YamlParseDictError, YamlReader, ) from dbt.utils import get_pseudo_test_path from dbt_common.events.functions import fire_event from dbt_common.events.types import SystemStdErr from dbt_extractor import ExtractionError, py_extract_from_source # type: ignore class UnitTestManifestLoader: def __init__(self, manifest, root_project, selected) -> None: self.manifest: Manifest = manifest self.root_project: RuntimeConfig = root_project # selected comes from the initial selection against a "regular" manifest self.selected: Set[UniqueId] = selected self.unit_test_manifest = Manifest(macros=manifest.macros) def load(self) -> Manifest: for unique_id in self.selected: if unique_id in self.manifest.unit_tests: unit_test_case: UnitTestDefinition = self.manifest.unit_tests[unique_id] if not unit_test_case.config.enabled: continue self.parse_unit_test_case(unit_test_case) return self.unit_test_manifest def parse_unit_test_case(self, test_case: UnitTestDefinition): # Create unit test node based on the node being tested # The tested_node has already been resolved and is in depends_on.nodes tested_node_unique_id = test_case.depends_on.nodes[0] tested_node = self.manifest.nodes[tested_node_unique_id] assert isinstance(tested_node, ModelNode) # Create UnitTestNode based on model being tested. Since selection has # already been done, we don't have to care about fields that are necessary # for selection. # Note: no depends_on, that's added later using input nodes name = test_case.name if tested_node.is_versioned: name = name + f"_v{tested_node.version}" expected_sql: Optional[str] = None if test_case.expect.format == UnitTestFormat.SQL: expected_rows: List[Dict[str, Any]] = [] expected_sql = test_case.expect.rows # type: ignore else: assert isinstance(test_case.expect.rows, List) expected_rows = deepcopy(test_case.expect.rows) assert isinstance(expected_rows, List) unit_test_node = UnitTestNode( name=name, resource_type=NodeType.Unit, package_name=test_case.package_name, path=get_pseudo_test_path(name, test_case.original_file_path), original_file_path=test_case.original_file_path, unique_id=test_case.unique_id, config=UnitTestNodeConfig( materialized="unit", expected_rows=expected_rows, expected_sql=expected_sql ), raw_code=tested_node.raw_code, database=tested_node.database, schema=tested_node.schema, alias=name, fqn=test_case.unique_id.split("."), checksum=FileHash.empty(), tested_node_unique_id=tested_node.unique_id, overrides=test_case.overrides, ) ctx = generate_parser_unit_test_context(unit_test_node, self.root_project, self.manifest) get_rendered(unit_test_node.raw_code, ctx, unit_test_node, capture_macros=True) # unit_test_node now has a populated refs/sources self.unit_test_manifest.nodes[unit_test_node.unique_id] = unit_test_node # Now create input_nodes for the test inputs """ given: - input: ref('my_model_a') rows: [] - input: ref('my_model_b') rows: - {id: 1, b: 2} - {id: 2, b: 2} """ # Add the model "input" nodes, consisting of all referenced models in the unit test. # This creates an ephemeral model for every input in every test, so there may be multiple # input models substituting for the same input ref'd model. Note that since these are # always "ephemeral" they just wrap the tested_node SQL in additional CTEs. No actual table # or view is created. for given in test_case.given: # extract the original_input_node from the ref in the "input" key of the given list original_input_node = self._get_original_input_node( given.input, tested_node, test_case.name, ctx, unit_test_node ) input_name = original_input_node.name common_fields = { "resource_type": NodeType.Model, # root directory for input and output fixtures "original_file_path": unit_test_node.original_file_path, "config": ModelConfig(materialized="ephemeral"), "database": original_input_node.database, "alias": original_input_node.identifier, "schema": original_input_node.schema, "fqn": original_input_node.fqn, "checksum": FileHash.empty(), "raw_code": self._build_fixture_raw_code(given.rows, None, given.format), "package_name": original_input_node.package_name, "unique_id": f"model.{original_input_node.package_name}.{input_name}", "name": input_name, "path": f"{input_name}.sql", } resource_type = original_input_node.resource_type if resource_type in ( NodeType.Model, NodeType.Seed, NodeType.Snapshot, ): input_node = ModelNode( **common_fields, defer_relation=original_input_node.defer_relation, ) if resource_type == NodeType.Model: if original_input_node.version: input_node.version = original_input_node.version if original_input_node.latest_version: input_node.latest_version = original_input_node.latest_version elif resource_type == NodeType.Source: # We are reusing the database/schema/identifier from the original source, # but that shouldn't matter since this acts as an ephemeral model which just # wraps a CTE around the unit test node. input_node = UnitTestSourceDefinition( **common_fields, source_name=original_input_node.source_name, # needed for source lookup ) # In the case of multiple sources with the same name, we add the source schema name to the unique id. # This additionally prevents duplicate CTE names during compilation. input_node.unique_id = f"model.{original_input_node.package_name}.{original_input_node.source_name}__{input_name}" # Sources need to go in the sources dictionary in order to create the right lookup self.unit_test_manifest.sources[input_node.unique_id] = input_node # type: ignore # Both ModelNode and UnitTestSourceDefinition need to go in nodes dictionary self.unit_test_manifest.nodes[input_node.unique_id] = input_node # Populate this_input_node_unique_id if input fixture represents node being tested if original_input_node == tested_node: unit_test_node.this_input_node_unique_id = input_node.unique_id # Add unique ids of input_nodes to depends_on unit_test_node.depends_on.nodes.append(input_node.unique_id) # Add functions to the manifest and depends_on for unique_id in tested_node.depends_on.nodes: if unique_id in self.manifest.functions: unit_test_node.depends_on.nodes.append(unique_id) self.unit_test_manifest.functions[unique_id] = self.manifest.functions[unique_id] def _build_fixture_raw_code(self, rows, column_name_to_data_types, fixture_format) -> str: # We're not currently using column_name_to_data_types, but leaving here for # possible future use. if fixture_format == UnitTestFormat.SQL: return rows else: return ("{{{{ get_fixture_sql({rows}, {column_name_to_data_types}) }}}}").format( rows=rows, column_name_to_data_types=column_name_to_data_types ) def _get_original_input_node( self, input: str, tested_node: ModelNode, test_case_name: str, ctx: Optional[Dict[str, Any]] = None, unit_test_node: Optional[UnitTestNode] = None, ): """ Returns the original input node as defined in the project given an input reference and the node being tested. input: str representing how input node is referenced in tested model sql * examples: - "ref('my_model_a')" - "source('my_source_schema', 'my_source_name')" - "this" tested_node: ModelNode of representing node being tested """ if input.strip() == "this": original_input_node = tested_node else: try: statically_parsed = py_extract_from_source(f"{{{{ {input} }}}}") except ExtractionError: if ( getattr(get_flags(), "SUPPORT_CUSTOM_REF_KWARGS", False) and ctx is not None and unit_test_node is not None ): statically_parsed = self._extract_ref_or_source_via_jinja( input, ctx, unit_test_node ) else: raise InvalidUnitTestGivenInput(input=input) if statically_parsed["refs"]: ref = list(statically_parsed["refs"])[0] name = ref.get("name") package = ref.get("package") version = ref.get("version") # TODO: disabled lookup, versioned lookup, public models original_input_node = self.manifest.ref_lookup.find( name, package, version, self.manifest ) elif statically_parsed["sources"]: source = list(statically_parsed["sources"])[0] input_source_name, input_name = source original_input_node = self.manifest.source_lookup.find( f"{input_source_name}.{input_name}", None, self.manifest, ) else: raise InvalidUnitTestGivenInput(input=input) if not original_input_node: msg = f"Unit test '{test_case_name}' had an input ({input}) which was not found in the manifest." raise ParsingError(msg) return original_input_node def _extract_ref_or_source_via_jinja( self, input_str: str, ctx: Dict[str, Any], unit_test_node: UnitTestNode, ) -> Dict[str, Any]: """Fall back to Jinja rendering to resolve ref() calls with custom kwargs. When py_extract_from_source cannot handle custom keyword arguments (e.g., ref('model', revision=2) via a custom ref macro), this method renders the input through Jinja so the custom macro can map custom kwargs to standard ones (like version). """ # Save current refs/sources to restore after rendering original_refs = list(unit_test_node.refs) original_sources = list(unit_test_node.sources) try: get_rendered(f"{{{{ {input_str} }}}}", ctx, unit_test_node, capture_macros=True) except Exception: raise InvalidUnitTestGivenInput(input=input_str) # Extract newly captured refs/sources new_refs = unit_test_node.refs[len(original_refs) :] new_sources = unit_test_node.sources[len(original_sources) :] # Restore original refs/sources unit_test_node.refs = original_refs unit_test_node.sources = original_sources result: Dict[str, Any] = {"refs": [], "sources": []} for ref_arg in new_refs: result["refs"].append( {"name": ref_arg.name, "package": ref_arg.package, "version": ref_arg.version} ) for source in new_sources: result["sources"].append(source) if not result["refs"] and not result["sources"]: raise InvalidUnitTestGivenInput(input=input_str) return result class UnitTestParser(YamlReader): def __init__(self, schema_parser: SchemaParser, yaml: YamlBlock) -> None: super().__init__(schema_parser, yaml, "unit_tests") self.schema_parser = schema_parser self.yaml = yaml def parse(self) -> ParseResult: for data in self.get_key_dicts(): unit_test: UnparsedUnitTest = self._get_unit_test(data) tested_model_node = find_tested_model_node( self.manifest, self.project.project_name, unit_test.model ) unit_test_case_unique_id = ( f"{NodeType.Unit}.{self.project.project_name}.{unit_test.model}.{unit_test.name}" ) unit_test_fqn = self._build_fqn( self.project.project_name, self.yaml.path.original_file_path, unit_test.model, unit_test.name, ) unit_test_config = self._build_unit_test_config(unit_test_fqn, unit_test.config) unit_test_definition = UnitTestDefinition( name=unit_test.name, model=unit_test.model, resource_type=NodeType.Unit, package_name=self.project.project_name, path=self.yaml.path.relative_path, original_file_path=self.yaml.path.original_file_path, unique_id=unit_test_case_unique_id, given=unit_test.given, expect=unit_test.expect, description=unit_test.description, overrides=unit_test.overrides, depends_on=DependsOn(), fqn=unit_test_fqn, config=unit_test_config, versions=unit_test.versions, ) if tested_model_node: if tested_model_node.config.enabled: unit_test_definition.depends_on.nodes.append(tested_model_node.unique_id) unit_test_definition.schema = tested_model_node.schema else: unit_test_definition.config.enabled = False # Check that format and type of rows matches for each given input, # convert rows to a list of dictionaries, and add the unique_id of # the unit_test_definition to the fixture source_file for partial parsing. self._validate_and_normalize_given(unit_test_definition) self._validate_and_normalize_expect(unit_test_definition) # for calculating state:modified unit_test_definition.build_unit_test_checksum() assert isinstance(self.yaml.file, SchemaSourceFile) if unit_test_definition.config.enabled: self.manifest.add_unit_test(self.yaml.file, unit_test_definition) else: self.manifest.add_disabled(self.yaml.file, unit_test_definition) return ParseResult() def _get_unit_test(self, data: Dict[str, Any]) -> UnparsedUnitTest: try: UnparsedUnitTest.validate(data) return UnparsedUnitTest.from_dict(data) except (ValidationError, JSONValidationError) as exc: raise YamlParseDictError(self.yaml.path, self.key, data, exc) def _build_unit_test_config( self, unit_test_fqn: List[str], config_dict: Dict[str, Any] ) -> UnitTestConfig: config = ContextConfig( self.schema_parser.root_project, unit_test_fqn, NodeType.Unit, self.schema_parser.project.project_name, ) unit_test_config_dict = config.build_config_dict(patch_config_dict=config_dict) unit_test_config_dict = self.render_entry(unit_test_config_dict) return UnitTestConfig.from_dict(unit_test_config_dict) def _build_fqn(self, package_name, original_file_path, model_name, test_name): # This code comes from "get_fqn" and "get_fqn_prefix" in the base parser. # We need to get the directories underneath the model-path. path = Path(original_file_path) relative_path = str(path.relative_to(*path.parts[:1])) no_ext = os.path.splitext(relative_path)[0] fqn = [package_name] fqn.extend(utils.split_path(no_ext)[:-1]) fqn.append(model_name) fqn.append(test_name) return fqn def _get_fixture(self, fixture_name: str, project_name: str): fixture_unique_id = f"{NodeType.Fixture}.{project_name}.{fixture_name}" if fixture_unique_id in self.manifest.fixtures: fixture = self.manifest.fixtures[fixture_unique_id] return fixture else: raise ParsingError( f"File not found for fixture '{fixture_name}' in unit tests in {self.yaml.path.original_file_path}" ) def _validate_and_normalize_given(self, unit_test_definition): for ut_input in unit_test_definition.given: self._validate_and_normalize_rows(ut_input, unit_test_definition, "input") def _validate_and_normalize_expect(self, unit_test_definition): self._validate_and_normalize_rows( unit_test_definition.expect, unit_test_definition, "expected" ) def _validate_and_normalize_rows(self, ut_fixture, unit_test_definition, fixture_type) -> None: if ut_fixture.format == UnitTestFormat.Dict: if ut_fixture.rows is None and ut_fixture.fixture is None: # This is a seed ut_fixture.rows = self._load_rows_from_seed(ut_fixture.input) if not isinstance(ut_fixture.rows, list): raise ParsingError( f"Unit test {unit_test_definition.name} has {fixture_type} rows " f"which do not match format {ut_fixture.format}" ) elif ut_fixture.format == UnitTestFormat.CSV: if not (isinstance(ut_fixture.rows, str) or isinstance(ut_fixture.fixture, str)): raise ParsingError( f"Unit test {unit_test_definition.name} has {fixture_type} rows or fixtures " f"which do not match format {ut_fixture.format}. Expected string." ) if ut_fixture.fixture: csv_rows = self.get_fixture_file_rows( ut_fixture.fixture, self.project.project_name, unit_test_definition.unique_id ) else: csv_rows = self._convert_csv_to_list_of_dicts(ut_fixture.rows) # Empty values (e.g. ,,) in a csv fixture should default to null, not "" ut_fixture.rows = [ {k: (None if v == "" else v) for k, v in row.items()} for row in csv_rows ] elif ut_fixture.format == UnitTestFormat.SQL: if not (isinstance(ut_fixture.rows, str) or isinstance(ut_fixture.fixture, str)): raise ParsingError( f"Unit test {unit_test_definition.name} has {fixture_type} rows or fixtures " f"which do not match format {ut_fixture.format}. Expected string." ) if ut_fixture.fixture: ut_fixture.rows = self.get_fixture_file_rows( ut_fixture.fixture, self.project.project_name, unit_test_definition.unique_id ) # sanitize order of input if ut_fixture.rows and ( ut_fixture.format == UnitTestFormat.Dict or ut_fixture.format == UnitTestFormat.CSV ): self._promote_first_non_none_row(ut_fixture) def _promote_first_non_none_row(self, ut_fixture): """ Promote the first row with no None values to the top of the ut_fixture.rows list. This function modifies the ut_fixture object in place. Needed for databases like Redshift which uses the first value in a column to determine the column type. If the first value is None, the type is assumed to be VARCHAR(1). This leads to obscure type mismatch errors centered on a unit test fixture's `expect`. See https://github.com/dbt-labs/dbt-redshift/issues/821 for more info. """ non_none_row_index = None # Iterate through each row and its index for index, row in enumerate(ut_fixture.rows): # Check if all values in the row are not None if all(value is not None for value in row.values()): non_none_row_index = index break if non_none_row_index is None: fire_event( SystemStdErr( bmsg="Unit Test fixtures benefit from having at least one row free of Null values to ensure consistent column types. Failure to meet this recommendation can result in type mismatch errors between unit test source models and `expected` fixtures." ) ) else: ut_fixture.rows[0], ut_fixture.rows[non_none_row_index] = ( ut_fixture.rows[non_none_row_index], ut_fixture.rows[0], ) def get_fixture_file_rows(self, fixture_name, project_name, utdef_unique_id): # find fixture file object and store unit_test_definition unique_id fixture = self._get_fixture(fixture_name, project_name) fixture_source_file = self.manifest.files[fixture.file_id] fixture_source_file.unit_tests.append(utdef_unique_id) return fixture.rows def _convert_csv_to_list_of_dicts(self, csv_string: str) -> List[Dict[str, Any]]: dummy_file = StringIO(csv_string) reader = csv.DictReader(dummy_file) rows = [] for row in reader: rows.append(row) return rows def _load_rows_from_seed(self, ref_str: str) -> List[Dict[str, Any]]: """Read rows from seed file on disk if not specified in YAML config. If seed file doesn't exist, return empty list.""" ref = py_extract_from_source("{{ " + ref_str + " }}")["refs"][0] rows: List[Dict[str, Any]] = [] seed_name = ref["name"] package_name = ref.get("package", self.project.project_name) seed_node = self.manifest.ref_lookup.find(seed_name, package_name, None, self.manifest) if not seed_node or seed_node.resource_type != NodeType.Seed: # Seed not found in custom package specified if package_name != self.project.project_name: raise ParsingError( f"Unable to find seed '{package_name}.{seed_name}' for unit tests in '{package_name}' package" ) else: raise ParsingError( f"Unable to find seed '{package_name}.{seed_name}' for unit tests in directories: {self.project.seed_paths}" ) seed_path = Path(self.project.project_root) / seed_node.original_file_path with open(seed_path, "r") as f: for row in DictReader(f): rows.append(row) return rows def find_tested_model_node( manifest: Manifest, current_project: str, unit_test_model: str ) -> Optional[ModelNode]: model_name_split = unit_test_model.split() model_name = model_name_split[0] model_version = model_name_split[1] if len(model_name_split) == 2 else None tested_node = manifest.ref_lookup.find(model_name, current_project, model_version, manifest) if not tested_node: disabled_node = manifest.disabled_lookup.find( model_name, current_project, model_version, [NodeType.Model] ) if disabled_node: tested_node = disabled_node[0] return tested_node # This is called by the ManifestLoader after other processing has been done, # so that model versions are available. def process_models_for_unit_test( manifest: Manifest, current_project: str, unit_test_def: UnitTestDefinition, models_to_versions ): # If the unit tests doesn't have a depends_on.nodes[0] then we weren't able to resolve # the model, either because versions hadn't been processed yet, or it's not a valid model name if not unit_test_def.depends_on.nodes: tested_node = find_tested_model_node(manifest, current_project, unit_test_def.model) if not tested_node: raise ParsingError( f"Unable to find model '{current_project}.{unit_test_def.model}' for " f"unit test '{unit_test_def.name}' in {unit_test_def.original_file_path}" ) if tested_node.config.enabled: unit_test_def.depends_on.nodes.append(tested_node.unique_id) unit_test_def.schema = tested_node.schema else: # If the model is disabled, the unit test should be disabled unit_test_def.config.enabled = False # The UnitTestDefinition should only have one "depends_on" at this point, # the one that's found by the "model" field. target_model_id = unit_test_def.depends_on.nodes[0] if target_model_id not in manifest.nodes: if target_model_id in manifest.disabled: # If the model is disabled, the unit test should be disabled unit_test_def.config.enabled = False else: # If we've reached here and the model is not disabled, throw an error raise ParsingError( f"Unit test '{unit_test_def.name}' references a model that does not exist: {target_model_id}" ) if not unit_test_def.config.enabled: # Ensure the unit test is disabled in the manifest if unit_test_def.unique_id in manifest.unit_tests: manifest.unit_tests.pop(unit_test_def.unique_id) if unit_test_def.unique_id not in manifest.disabled: manifest.add_disabled(manifest.files[unit_test_def.file_id], unit_test_def) # The unit test is disabled, so we don't need to do any further processing (#10540) return target_model = manifest.nodes[target_model_id] assert isinstance(target_model, ModelNode) target_model_is_incremental = "macro.dbt.is_incremental" in target_model.depends_on.macros unit_test_def_has_incremental_override = unit_test_def.overrides and isinstance( unit_test_def.overrides.macros.get("is_incremental"), bool ) if target_model_is_incremental and (not unit_test_def_has_incremental_override): raise ParsingError( f"Boolean override for 'is_incremental' must be provided for unit test '{unit_test_def.name}' in model '{target_model.name}'" ) unit_test_def_incremental_override_true = ( unit_test_def.overrides and unit_test_def.overrides.macros.get("is_incremental") ) unit_test_def_has_this_input = "this" in [i.input for i in unit_test_def.given] if ( target_model_is_incremental and unit_test_def_incremental_override_true and (not unit_test_def_has_this_input) ): raise ParsingError( f"Unit test '{unit_test_def.name}' for incremental model '{target_model.name}' must have a 'this' input" ) # unit_test_versions = unit_test_def.versions # We're setting up unit tests for versioned models, so if # the model isn't versioned, we don't need to do anything if not target_model.is_versioned: if unit_test_def.versions and ( unit_test_def.versions.include or unit_test_def.versions.exclude ): # If model is not versioned, we should not have an include or exclude msg = ( f"Unit test '{unit_test_def.name}' should not have a versions include or exclude " f"when referencing non-versioned model '{target_model.name}'" ) raise ParsingError(msg) else: return versioned_models = [] if ( target_model.package_name in models_to_versions and target_model.name in models_to_versions[target_model.package_name] ): versioned_models = models_to_versions[target_model.package_name][target_model.name] versions_to_test = [] if unit_test_def.versions is None: versions_to_test = versioned_models elif unit_test_def.versions.exclude: for model_unique_id in versioned_models: model = manifest.nodes[model_unique_id] assert isinstance(model, ModelNode) if model.version in unit_test_def.versions.exclude: continue else: versions_to_test.append(model.unique_id) elif unit_test_def.versions.include: for model_unique_id in versioned_models: model = manifest.nodes[model_unique_id] assert isinstance(model, ModelNode) if model.version in unit_test_def.versions.include: versions_to_test.append(model.unique_id) else: continue if not versions_to_test: msg = ( f"Unit test '{unit_test_def.name}' referenced a version of '{target_model.name}' " "which was not found." ) raise ParsingError(msg) else: # Create unit test definitions that match the model versions original_unit_test_def = manifest.unit_tests.pop(unit_test_def.unique_id) original_unit_test_dict = original_unit_test_def.to_dict() schema_file = manifest.files[original_unit_test_def.file_id] assert isinstance(schema_file, SchemaSourceFile) schema_file.unit_tests.remove(original_unit_test_def.unique_id) for versioned_model_unique_id in versions_to_test: versioned_model = manifest.nodes[versioned_model_unique_id] assert isinstance(versioned_model, ModelNode) versioned_unit_test_unique_id = f"{NodeType.Unit}.{unit_test_def.package_name}.{unit_test_def.model}.{unit_test_def.name}_v{versioned_model.version}" new_unit_test_def = UnitTestDefinition.from_dict(original_unit_test_dict) new_unit_test_def.unique_id = versioned_unit_test_unique_id new_unit_test_def.depends_on.nodes[0] = versioned_model_unique_id new_unit_test_def.version = versioned_model.version schema_file.unit_tests.append(versioned_unit_test_unique_id) # fqn? manifest.unit_tests[versioned_unit_test_unique_id] = new_unit_test_def ================================================ FILE: core/dbt/plugins/__init__.py ================================================ from typing import Optional # these are just exports, they need "noqa" so flake8 will not complain. from .manager import PluginManager, dbt_hook, dbtPlugin # noqa PLUGIN_MANAGER: Optional[PluginManager] = None def set_up_plugin_manager(project_name: str): global PLUGIN_MANAGER PLUGIN_MANAGER = PluginManager.from_modules(project_name) def get_plugin_manager(project_name: str) -> PluginManager: global PLUGIN_MANAGER if not PLUGIN_MANAGER: set_up_plugin_manager(project_name) assert PLUGIN_MANAGER return PLUGIN_MANAGER ================================================ FILE: core/dbt/plugins/contracts.py ================================================ from typing import Dict # just exports, they need "noqa" so flake8 will not complain. from dbt.artifacts.schemas.base import ArtifactMixin as PluginArtifact # noqa from dbt.artifacts.schemas.base import BaseArtifactMetadata # noqa from dbt.artifacts.schemas.base import schema_version # noqa from dbt_common.dataclass_schema import ExtensibleDbtClassMixin, dbtClassMixin # noqa PluginArtifacts = Dict[str, PluginArtifact] ================================================ FILE: core/dbt/plugins/exceptions.py ================================================ # just exports, they need "noqa" so flake8 will not complain. from dbt.exceptions import dbtPluginError # noqa ================================================ FILE: core/dbt/plugins/manager.py ================================================ import functools import importlib import pkgutil from types import ModuleType from typing import Callable, Dict, List, Mapping import dbt.tracking from dbt.contracts.graph.manifest import Manifest from dbt.plugins.contracts import PluginArtifacts from dbt.plugins.manifest import PluginNodes from dbt_common.exceptions import DbtRuntimeError from dbt_common.tests import test_caching_enabled def dbt_hook(func): def inner(*args, **kwargs): try: return func(*args, **kwargs) except Exception as e: raise DbtRuntimeError(f"{func.__name__}: {e}") setattr(inner, "is_dbt_hook", True) return inner class dbtPlugin: """ EXPERIMENTAL: dbtPlugin is the base class for creating plugins. Its interface is **not** stable and will likely change between dbt-core versions. """ def __init__(self, project_name: str) -> None: self.project_name = project_name try: self.initialize() except DbtRuntimeError as e: # Remove the first line of DbtRuntimeError to avoid redundant "Runtime Error" line raise DbtRuntimeError("\n".join(str(e).split("\n")[1:])) except Exception as e: raise DbtRuntimeError(str(e)) @property def name(self) -> str: return self.__class__.__name__ def initialize(self) -> None: """ Initialize the plugin. This function may be overridden by subclasses that have additional initialization steps. """ pass def get_nodes(self) -> PluginNodes: """ Provide PluginNodes to dbt for injection into dbt's DAG. Currently the only node types that are accepted are model nodes. """ raise NotImplementedError(f"get_nodes hook not implemented for {self.name}") def get_manifest_artifacts(self, manifest: Manifest) -> PluginArtifacts: """ Given a manifest, provide PluginArtifacts derived for writing by core. PluginArtifacts share the same lifecycle as the manifest.json file -- they will either be written or not depending on whether the manifest is written. """ raise NotImplementedError(f"get_manifest_artifacts hook not implemented for {self.name}") @functools.lru_cache(maxsize=None) def _get_dbt_modules() -> Mapping[str, ModuleType]: # This is an expensive function, especially in the context of testing, when # it is called repeatedly, so we break it out and cache the result globally. return { name: importlib.import_module(name) for _, name, _ in pkgutil.iter_modules() if name.startswith(PluginManager.PLUGIN_MODULE_PREFIX) } _MODULES_CACHE = None class PluginManager: PLUGIN_MODULE_PREFIX = "dbt_" PLUGIN_ATTR_NAME = "plugins" def __init__(self, plugins: List[dbtPlugin]) -> None: self._plugins = plugins self._valid_hook_names = set() # default hook implementations from dbtPlugin for hook_name in dir(dbtPlugin): if not hook_name.startswith("_"): self._valid_hook_names.add(hook_name) self.hooks: Dict[str, List[Callable]] = {} for plugin in self._plugins: for hook_name in dir(plugin): hook = getattr(plugin, hook_name) if ( callable(hook) and hasattr(hook, "is_dbt_hook") and hook_name in self._valid_hook_names ): if hook_name in self.hooks: self.hooks[hook_name].append(hook) else: self.hooks[hook_name] = [hook] @classmethod def from_modules(cls, project_name: str) -> "PluginManager": if test_caching_enabled(): global _MODULES_CACHE if _MODULES_CACHE is None: discovered_dbt_modules = cls.get_prefixed_modules() _MODULES_CACHE = discovered_dbt_modules else: discovered_dbt_modules = _MODULES_CACHE else: discovered_dbt_modules = cls.get_prefixed_modules() plugins = [] for name, module in discovered_dbt_modules.items(): if hasattr(module, cls.PLUGIN_ATTR_NAME): available_plugins = getattr(module, cls.PLUGIN_ATTR_NAME, []) for plugin_cls in available_plugins: assert issubclass( plugin_cls, dbtPlugin ), f"'plugin' in {name} must be subclass of dbtPlugin" plugin = plugin_cls(project_name=project_name) plugins.append(plugin) return cls(plugins=plugins) @classmethod def get_prefixed_modules(cls): return { name: importlib.import_module(name) for _, name, _ in pkgutil.iter_modules() if name.startswith(cls.PLUGIN_MODULE_PREFIX) } def get_manifest_artifacts(self, manifest: Manifest) -> PluginArtifacts: all_plugin_artifacts = {} for hook_method in self.hooks.get("get_manifest_artifacts", []): plugin_artifacts = hook_method(manifest) all_plugin_artifacts.update(plugin_artifacts) return all_plugin_artifacts def get_nodes(self) -> PluginNodes: all_plugin_nodes = PluginNodes() for hook_method in self.hooks.get("get_nodes", []): plugin_nodes = hook_method() dbt.tracking.track_plugin_get_nodes( { "plugin_name": hook_method.__self__.name, # type: ignore "num_model_nodes": len(plugin_nodes.models), "num_model_packages": len( {model.package_name for model in plugin_nodes.models.values()} ), } ) all_plugin_nodes.update(plugin_nodes) return all_plugin_nodes ================================================ FILE: core/dbt/plugins/manifest.py ================================================ from dataclasses import dataclass, field from typing import Dict from dbt.artifacts.resources import NodeVersion # noqa # all these are just exports, they need "noqa" so flake8 will not complain. from dbt.contracts.graph.manifest import Manifest # noqa from dbt.contracts.graph.node_args import ModelNodeArgs from dbt.graph.graph import UniqueId # noqa from dbt.node_types import AccessType, NodeType # noqa @dataclass class PluginNodes: models: Dict[str, ModelNodeArgs] = field(default_factory=dict) def add_model(self, model_args: ModelNodeArgs) -> None: self.models[model_args.unique_id] = model_args def update(self, other: "PluginNodes") -> None: self.models.update(other.models) ================================================ FILE: core/dbt/profiler.py ================================================ from contextlib import contextmanager from cProfile import Profile from pstats import Stats from typing import Any, Generator @contextmanager def profiler(enable: bool, outfile: str) -> Generator[Any, None, None]: try: if enable: profiler = Profile() profiler.enable() yield finally: if enable: profiler.disable() stats = Stats(profiler) stats.sort_stats("tottime") stats.dump_stats(str(outfile)) ================================================ FILE: core/dbt/py.typed ================================================ # dummy file, our types are defined inline ================================================ FILE: core/dbt/runners/__init__.py ================================================ from .exposure_runner import ExposureRunner from .saved_query_runner import SavedQueryRunner ================================================ FILE: core/dbt/runners/exposure_runner.py ================================================ from dbt.runners.no_op_runner import NoOpRunner class ExposureRunner(NoOpRunner): @property def description(self) -> str: return f"exposure {self.node.name}" ================================================ FILE: core/dbt/runners/no_op_runner.py ================================================ import threading from dbt.artifacts.schemas.results import RunStatus from dbt.artifacts.schemas.run import RunResult from dbt.contracts.graph.manifest import Manifest from dbt.events.types import LogNodeNoOpResult from dbt.task.base import BaseRunner from dbt_common.events.functions import fire_event class NoOpRunner(BaseRunner): @property def description(self) -> str: raise NotImplementedError("description not implemented") def before_execute(self) -> None: pass def compile(self, manifest: Manifest): return self.node def after_execute(self, result) -> None: fire_event( LogNodeNoOpResult( description=self.description, index=self.node_index, total=self.num_nodes, node_info=self.node.node_info, ) ) def execute(self, compiled_node, manifest): # no-op return RunResult( node=compiled_node, status=RunStatus.NoOp, timing=[], thread_id=threading.current_thread().name, execution_time=0, message="NO-OP", adapter_response={}, failures=0, batch_results=None, agate_table=None, ) ================================================ FILE: core/dbt/runners/saved_query_runner.py ================================================ from dbt.runners.no_op_runner import NoOpRunner class SavedQueryRunner(NoOpRunner): @property def description(self) -> str: return f"saved query {self.node.name}" ================================================ FILE: core/dbt/selected_resources.py ================================================ from typing import Any, Set SELECTED_RESOURCES = [] def set_selected_resources(selected_resources: Set[Any]) -> None: global SELECTED_RESOURCES SELECTED_RESOURCES = list(selected_resources) ================================================ FILE: core/dbt/task/README.md ================================================ # Task README ### Task Hierarchy ``` BaseTask ┣ CleanTask ┣ ConfiguredTask ┃ ┣ GraphRunnableTask ┃ ┃ ┣ CloneTask ┃ ┃ ┣ CompileTask ┃ ┃ ┃ ┣ GenerateTask ┃ ┃ ┃ ┣ RunTask ┃ ┃ ┃ ┃ ┣ BuildTask ┃ ┃ ┃ ┃ ┣ FreshnessTask ┃ ┃ ┃ ┃ ┣ SeedTask ┃ ┃ ┃ ┃ ┣ SnapshotTask ┃ ┃ ┃ ┃ ┗ TestTask ┃ ┃ ┃ ┗ ShowTask ┃ ┃ ┗ ListTask ┃ ┣ RetryTask ┃ ┣ RunOperationTask ┃ ┗ ServeTask ┣ DebugTask ┣ DepsTask ┗ InitTask ``` ### Runner Hierarchy ``` BaseRunner ┣ CloneRunner ┣ CompileRunner ┃ ┣ GenericSqlRunner ┃ ┃ ┣ SqlCompileRunner ┃ ┃ ┗ SqlExecuteRunner ┃ ┣ ModelRunner ┃ ┃ ┣ SeedRunner ┃ ┃ ┗ SnapshotRunner ┃ ┣ ShowRunner ┃ ┗ TestRunner ┣ FreshnessRunner ┗ SavedQueryRunner ``` ================================================ FILE: core/dbt/task/__init__.py ================================================ ================================================ FILE: core/dbt/task/base.py ================================================ import os import threading import time import traceback from abc import ABCMeta, abstractmethod from contextlib import nullcontext from datetime import datetime, timezone from pathlib import Path from typing import Any, Dict, List, Optional, Set import dbt.exceptions import dbt_common.exceptions.base from dbt import tracking from dbt.artifacts.resources.types import NodeType from dbt.artifacts.schemas.results import ( NodeStatus, RunningStatus, RunStatus, TimingInfo, collect_timing_info, ) from dbt.artifacts.schemas.run import RunResult from dbt.cli.flags import Flags from dbt.compilation import Compiler from dbt.config import RuntimeConfig from dbt.config.profile import read_profile from dbt.constants import DBT_PROJECT_FILE_NAME from dbt.contracts.graph.manifest import Manifest from dbt.events.types import ( CatchableExceptionOnRun, GenericExceptionOnRun, InternalErrorOnRun, LogDbtProfileError, LogDbtProjectError, LogDebugStackTrace, LogSkipBecauseError, NodeCompiling, NodeConnectionReleaseError, NodeExecuting, SkippingDetails, ) from dbt.flags import get_flags from dbt.graph import Graph from dbt.task import group_lookup from dbt.task.printer import print_run_result_error from dbt_common.events.contextvars import get_node_info from dbt_common.events.functions import fire_event from dbt_common.exceptions import DbtInternalError, DbtRuntimeError, NotImplementedError def read_profiles(profiles_dir: Optional[str] = None) -> Dict[str, Any]: """This is only used for some error handling""" if profiles_dir is None: profiles_dir = get_flags().PROFILES_DIR raw_profiles = read_profile(profiles_dir) if raw_profiles is None: profiles = {} else: profiles = {k: v for (k, v) in raw_profiles.items() if k != "config"} return profiles class BaseTask(metaclass=ABCMeta): def __init__(self, args: Flags) -> None: self.args = args def __enter__(self): self.orig_dir = os.getcwd() return self def __exit__(self, exc_type, exc_value, traceback): os.chdir(self.orig_dir) @abstractmethod def run(self): raise dbt_common.exceptions.base.NotImplementedError("Not Implemented") def interpret_results(self, results): return True def get_nearest_project_dir(project_dir: Optional[str]) -> Path: # If the user provides an explicit project directory, use that # but don't look at parent directories. if project_dir: cur_dir = Path(project_dir) project_file = Path(project_dir) / DBT_PROJECT_FILE_NAME if project_file.is_file(): return cur_dir else: raise dbt_common.exceptions.DbtRuntimeError( "fatal: Invalid --project-dir flag. Not a dbt project. " "Missing dbt_project.yml file" ) cur_dir = Path.cwd() project_file = cur_dir / DBT_PROJECT_FILE_NAME if project_file.is_file(): return cur_dir else: raise dbt_common.exceptions.DbtRuntimeError( "fatal: Not a dbt project (or any of the parent directories). " "Missing dbt_project.yml file" ) def move_to_nearest_project_dir(project_dir: Optional[str]) -> Path: nearest_project_dir = get_nearest_project_dir(project_dir) resolved_nearest_project_dir = nearest_project_dir.resolve() os.chdir(resolved_nearest_project_dir) return resolved_nearest_project_dir # TODO: look into deprecating this class in favor of several small functions that # produce the same behavior. currently this class only contains manifest compilation, # holding a manifest, and moving direcories. class ConfiguredTask(BaseTask): def __init__( self, args: Flags, config: RuntimeConfig, manifest: Optional[Manifest] = None ) -> None: super().__init__(args) self.config = config self.graph: Optional[Graph] = None self.manifest = manifest self.compiler = Compiler(self.config) def compile_manifest(self) -> None: if self.manifest is None: raise DbtInternalError("compile_manifest called before manifest was loaded") start_compile_manifest = time.perf_counter() self.graph = self.compiler.compile(self.manifest) compile_time = time.perf_counter() - start_compile_manifest if dbt.tracking.active_user is not None: dbt.tracking.track_runnable_timing({"graph_compilation_elapsed": compile_time}) @classmethod def from_args(cls, args: Flags, *pargs, **kwargs): move_to_nearest_project_dir(args.project_dir) try: # This is usually RuntimeConfig config = RuntimeConfig.from_args(args) except dbt.exceptions.DbtProjectError as exc: fire_event(LogDbtProjectError(exc=str(exc))) tracking.track_invalid_invocation(args=args, result_type=exc.result_type) raise dbt_common.exceptions.DbtRuntimeError("Could not run dbt") from exc except dbt.exceptions.DbtProfileError as exc: all_profile_names = list(read_profiles(get_flags().PROFILES_DIR).keys()) fire_event(LogDbtProfileError(exc=str(exc), profiles=all_profile_names)) tracking.track_invalid_invocation(args=args, result_type=exc.result_type) raise dbt_common.exceptions.DbtRuntimeError("Could not run dbt") from exc return cls(args, config, *pargs, **kwargs) class ExecutionContext: """During execution and error handling, dbt makes use of mutable state: timing information and the newest (compiled vs executed) form of the node. """ def __init__(self, node) -> None: self.timing: List[TimingInfo] = [] self.node = node class BaseRunner(metaclass=ABCMeta): def __init__(self, config, adapter, node, node_index: int, num_nodes: int) -> None: self.config = config self.compiler = Compiler(config) self.adapter = adapter self.node = node self.node_index = node_index self.num_nodes = num_nodes self.skip = False self.skip_cause: Optional[RunResult] = None self.run_ephemeral_models = False @abstractmethod def compile(self, manifest: Manifest) -> Any: pass def _node_build_path(self) -> Optional[str]: return self.node.build_path if hasattr(self.node, "build_path") else None def get_result_status(self, result) -> Dict[str, str]: if result.status == NodeStatus.Error: return {"node_status": "error", "node_error": str(result.message)} elif result.status == NodeStatus.Skipped: return {"node_status": "skipped"} elif result.status == NodeStatus.Fail: return {"node_status": "failed"} elif result.status == NodeStatus.Warn: return {"node_status": "warn"} else: return {"node_status": "passed"} def run_with_hooks(self, manifest): if self.skip: return self.on_skip() # no before/after printing for ephemeral mdoels if not self.node.is_ephemeral_model: self.before_execute() result = self.safe_run(manifest) self.node.update_event_status( node_status=result.status, finished_at=datetime.now(timezone.utc).replace(tzinfo=None).isoformat(), ) if not self.node.is_ephemeral_model: self.after_execute(result) return result def _build_run_result( self, node, start_time, status, timing_info, message, agate_table=None, adapter_response=None, failures=None, batch_results=None, ): execution_time = time.time() - start_time thread_id = threading.current_thread().name if adapter_response is None: adapter_response = {} return RunResult( status=status, thread_id=thread_id, execution_time=execution_time, timing=timing_info, message=message, node=node, agate_table=agate_table, adapter_response=adapter_response, failures=failures, batch_results=batch_results, ) def error_result(self, node, message, start_time, timing_info): return self._build_run_result( node=node, start_time=start_time, status=RunStatus.Error, timing_info=timing_info, message=message, ) def ephemeral_result(self, node, start_time, timing_info): return self._build_run_result( node=node, start_time=start_time, status=RunStatus.Success, timing_info=timing_info, message=None, ) def from_run_result(self, result, start_time, timing_info): return self._build_run_result( node=result.node, start_time=start_time, status=result.status, timing_info=timing_info, message=result.message, agate_table=result.agate_table, adapter_response=result.adapter_response, failures=result.failures, batch_results=result.batch_results, ) def compile_and_execute(self, manifest: Manifest, ctx: ExecutionContext): result = None with ( self.adapter.connection_named(self.node.unique_id, self.node) if get_flags().INTROSPECT else nullcontext() ): ctx.node.update_event_status(node_status=RunningStatus.Compiling) fire_event( NodeCompiling( node_info=ctx.node.node_info, ) ) with collect_timing_info("compile", ctx.timing.append): # if we fail here, we still have a compiled node to return # this has the benefit of showing a build path for the errant # model. This calls the 'compile' method in CompileTask ctx.node = self.compile(manifest) # for ephemeral nodes, we only want to compile, not run if not ctx.node.is_ephemeral_model or self.run_ephemeral_models: ctx.node.update_event_status(node_status=RunningStatus.Executing) fire_event( NodeExecuting( node_info=ctx.node.node_info, ) ) with collect_timing_info("execute", ctx.timing.append): result = self.run(ctx.node, manifest) ctx.node = result.node return result def _handle_catchable_exception(self, e: DbtRuntimeError, ctx: ExecutionContext) -> str: if e.node is None: e.add_node(ctx.node) fire_event( CatchableExceptionOnRun( exc=str(e), exc_info=traceback.format_exc(), node_info=get_node_info() ) ) return str(e) def _handle_internal_exception(self, e: DbtInternalError, ctx: ExecutionContext) -> str: fire_event( InternalErrorOnRun( build_path=self._node_build_path(), exc=str(e), node_info=get_node_info() ) ) return str(e) def _handle_generic_exception(self, e: Exception, ctx: ExecutionContext) -> str: fire_event( GenericExceptionOnRun( build_path=self._node_build_path(), unique_id=self.node.unique_id, exc=str(e), node_info=get_node_info(), ) ) fire_event(LogDebugStackTrace(exc_info=traceback.format_exc())) return str(e) def handle_exception(self, e: Exception, ctx: ExecutionContext) -> str: if isinstance(e, DbtRuntimeError): error = self._handle_catchable_exception(e, ctx) elif isinstance(e, DbtInternalError): error = self._handle_internal_exception(e, ctx) else: error = self._handle_generic_exception(e, ctx) return error def safe_run(self, manifest: Manifest): started = time.time() ctx = ExecutionContext(self.node) error = None result = None try: result = self.compile_and_execute(manifest, ctx) except Exception as e: error = self.handle_exception(e, ctx) finally: exc_str = self._safe_release_connection() # if releasing failed and the result doesn't have an error yet, set # an error if ( exc_str is not None and result is not None and result.status != NodeStatus.Error and error is None ): error = exc_str if error is not None: result = self.error_result(ctx.node, error, started, ctx.timing) elif result is not None: result = self.from_run_result(result, started, ctx.timing) else: result = self.ephemeral_result(ctx.node, started, ctx.timing) return result def _safe_release_connection(self): """Try to release a connection. If an exception is hit, log and return the error string. """ try: self.adapter.release_connection() except Exception as exc: fire_event( NodeConnectionReleaseError( node_name=self.node.name, exc=str(exc), exc_info=traceback.format_exc() ) ) return str(exc) return None def before_execute(self) -> None: raise NotImplementedError("before_execute is not implemented") def execute(self, compiled_node, manifest): raise NotImplementedError("execute is not implemented") def run(self, compiled_node, manifest): return self.execute(compiled_node, manifest) def after_execute(self, result) -> None: raise NotImplementedError("after_execute is not implemented") def _skip_caused_by_ephemeral_failure(self) -> bool: if self.skip_cause is None or self.skip_cause.node is None: return False return self.skip_cause.node.is_ephemeral_model def on_skip(self): schema_name = getattr(self.node, "schema", "") node_name = self.node.name error_message = None if not self.node.is_ephemeral_model: # if this model was skipped due to an upstream ephemeral model # failure, print a special 'error skip' message. # Include skip_cause NodeStatus group = group_lookup.get(self.node.unique_id) if self._skip_caused_by_ephemeral_failure(): fire_event( LogSkipBecauseError( schema=schema_name, relation=node_name, index=self.node_index, total=self.num_nodes, status=self.skip_cause.status, group=group, ) ) # skip_cause here should be the run_result from the ephemeral model print_run_result_error(result=self.skip_cause, newline=False) if self.skip_cause is None: # mypy appeasement raise DbtInternalError( "Skip cause not set but skip was somehow caused by an ephemeral failure" ) # set an error so dbt will exit with an error code error_message = ( "Compilation Error in {}, caused by compilation error " "in referenced ephemeral model {}".format( self.node.unique_id, self.skip_cause.node.unique_id ) ) else: # 'skipped' nodes should not have a value for 'node_finished_at' # they do have 'node_started_at', which is set in GraphRunnableTask.call_runner self.node.update_event_status(node_status=RunStatus.Skipped) fire_event( SkippingDetails( resource_type=self.node.resource_type, schema=schema_name, node_name=node_name, index=self.node_index, total=self.num_nodes, node_info=self.node.node_info, group=group, ) ) node_result = RunResult.from_node(self.node, RunStatus.Skipped, error_message) return node_result def do_skip(self, cause=None) -> None: self.skip = True self.skip_cause = cause def resource_types_from_args( args: Flags, all_resource_values: Set[NodeType], default_resource_values: Set[NodeType] ) -> Set[NodeType]: if not args.resource_types: resource_types = default_resource_values else: # This is a list of strings, not NodeTypes arg_resource_types = set(args.resource_types) if "all" in arg_resource_types: arg_resource_types.remove("all") arg_resource_types.update(all_resource_values) if "default" in arg_resource_types: arg_resource_types.remove("default") arg_resource_types.update(default_resource_values) # Convert to a set of NodeTypes now that the non-NodeType strings are gone resource_types = set([NodeType(rt) for rt in list(arg_resource_types)]) if args.exclude_resource_types: # Convert from a list of strings to a set of NodeTypes exclude_resource_types = set([NodeType(rt) for rt in args.exclude_resource_types]) resource_types = resource_types - exclude_resource_types return resource_types ================================================ FILE: core/dbt/task/build.py ================================================ from typing import Dict, Iterable, List, Optional, Set, Type from dbt.adapters.base import BaseRelation from dbt.artifacts.schemas.results import NodeStatus from dbt.artifacts.schemas.run import RunResult from dbt.cli.flags import Flags from dbt.config.runtime import RuntimeConfig from dbt.contracts.graph.manifest import Manifest from dbt.exceptions import DbtInternalError from dbt.graph import Graph, GraphQueue, ResourceTypeSelector from dbt.node_types import NodeType from dbt.runners import ExposureRunner as exposure_runner from dbt.runners import SavedQueryRunner as saved_query_runner from dbt.task.base import BaseRunner, resource_types_from_args from dbt.task.run import MicrobatchModelRunner from .function import FunctionRunner as function_runner from .run import ModelRunner as run_model_runner from .run import RunTask from .seed import SeedRunner as seed_runner from .snapshot import SnapshotRunner as snapshot_model_runner from .test import TestRunner as test_runner class BuildTask(RunTask): """The Build task processes all assets of a given process and attempts to 'build' them in an opinionated fashion. Every resource type outlined in RUNNER_MAP will be processed by the mapped runners class. I.E. a resource of type Model is handled by the ModelRunner which is imported as run_model_runner.""" MARK_DEPENDENT_ERRORS_STATUSES = [ NodeStatus.Error, NodeStatus.Fail, NodeStatus.Skipped, NodeStatus.PartialSuccess, ] RUNNER_MAP = { NodeType.Model: run_model_runner, NodeType.Snapshot: snapshot_model_runner, NodeType.Seed: seed_runner, NodeType.Test: test_runner, NodeType.Unit: test_runner, NodeType.SavedQuery: saved_query_runner, NodeType.Exposure: exposure_runner, NodeType.Function: function_runner, } ALL_RESOURCE_VALUES = frozenset({x for x in RUNNER_MAP.keys()}) def __init__(self, args: Flags, config: RuntimeConfig, manifest: Manifest) -> None: super().__init__(args, config, manifest) self.selected_unit_tests: Set = set() self.model_to_unit_test_map: Dict[str, List] = {} def resource_types(self, no_unit_tests: bool = False) -> List[NodeType]: resource_types = resource_types_from_args( self.args, set(self.ALL_RESOURCE_VALUES), set(self.ALL_RESOURCE_VALUES) ) # First we get selected_nodes including unit tests, then without, # and do a set difference. if no_unit_tests is True and NodeType.Unit in resource_types: resource_types.remove(NodeType.Unit) return list(resource_types) def get_model_schemas(self, adapter, selected_uids: Iterable[str]) -> Set[BaseRelation]: # Get model schemas as usual model_schemas = super().get_model_schemas(adapter, selected_uids) # Get function schemas function_schemas: Set[BaseRelation] = set() for function in ( self.manifest.functions.values() if self.manifest else [] ): # functionally the manifest will never be None as we do an assert in super().get_model_schemas(...) if function.unique_id in selected_uids: relation = adapter.Relation.create_from(self.config, function) function_schemas.add(relation.without_identifier()) return model_schemas.union(function_schemas) # overrides get_graph_queue in runnable.py def get_graph_queue(self) -> GraphQueue: # Following uses self.selection_arg and self.exclusion_arg spec = self.get_selection_spec() # selector including unit tests full_selector = self.get_node_selector(no_unit_tests=False) # selected node unique_ids with unit_tests full_selected_nodes = full_selector.get_selected(spec=spec, warn_on_no_nodes=False) # This selector removes the unit_tests from the selector selector_wo_unit_tests = self.get_node_selector(no_unit_tests=True) # selected node unique_ids without unit_tests selected_nodes_wo_unit_tests = selector_wo_unit_tests.get_selected( spec=spec, warn_on_no_nodes=False ) # Get the difference in the sets of nodes with and without unit tests and # save it selected_unit_tests = full_selected_nodes - selected_nodes_wo_unit_tests self.selected_unit_tests = selected_unit_tests self.build_model_to_unit_test_map(selected_unit_tests) # get_graph_queue in the selector will remove NodeTypes not specified # in the node_selector (filter_selection). return selector_wo_unit_tests.get_graph_queue(spec) # overrides handle_job_queue in runnable.py def handle_job_queue(self, pool, callback): if self.run_count == 0: self.num_nodes = self.num_nodes + len(self.selected_unit_tests) node = self.job_queue.get() if ( node.resource_type == NodeType.Model and self.model_to_unit_test_map and node.unique_id in self.model_to_unit_test_map ): self.handle_model_with_unit_tests_node(node, pool, callback) else: self.handle_job_queue_node(node, pool, callback) def handle_model_with_unit_tests_node(self, node, pool, callback): self._raise_set_error() args = [node, pool] if self.config.args.single_threaded: callback(self.call_model_and_unit_tests_runner(*args)) else: pool.apply_async(self.call_model_and_unit_tests_runner, args=args, callback=callback) def call_model_and_unit_tests_runner(self, node, pool) -> RunResult: assert self.manifest for unit_test_unique_id in self.model_to_unit_test_map[node.unique_id]: unit_test_node = self.manifest.unit_tests[unit_test_unique_id] unit_test_runner = self.get_runner(unit_test_node) # If the model is marked skip, also skip the unit tests if node.unique_id in self._skipped_children: # cause is only for ephemeral nodes unit_test_runner.do_skip(cause=None) result = self.call_runner(unit_test_runner) self._handle_result(result) if result.status in self.MARK_DEPENDENT_ERRORS_STATUSES: # The _skipped_children dictionary can contain a run_result for ephemeral nodes, # but that should never be the case here. self._skipped_children[node.unique_id] = None runner = self.get_runner(node) if runner.node.unique_id in self._skipped_children: cause = self._skipped_children.pop(runner.node.unique_id) runner.do_skip(cause=cause) if isinstance(runner, MicrobatchModelRunner): runner.set_parent_task(self) runner.set_pool(pool) return self.call_runner(runner) # handle non-model-plus-unit-tests nodes def handle_job_queue_node(self, node, pool, callback): self._raise_set_error() runner = self.get_runner(node) # we finally know what we're running! Make sure we haven't decided # to skip it due to upstream failures if runner.node.unique_id in self._skipped_children: cause = self._skipped_children.pop(runner.node.unique_id) runner.do_skip(cause=cause) if isinstance(runner, MicrobatchModelRunner): runner.set_parent_task(self) runner.set_pool(pool) args = [runner] self._submit(pool, args, callback) # Make a map of model unique_ids to selected unit test unique_ids, # for processing before the model. def build_model_to_unit_test_map(self, selected_unit_tests): dct = {} for unit_test_unique_id in selected_unit_tests: unit_test = self.manifest.unit_tests[unit_test_unique_id] model_unique_id = unit_test.depends_on.nodes[0] if model_unique_id not in dct: dct[model_unique_id] = [] dct[model_unique_id].append(unit_test.unique_id) self.model_to_unit_test_map = dct # We return two different kinds of selectors, one with unit tests and one without def get_node_selector(self, no_unit_tests=False) -> ResourceTypeSelector: if self.manifest is None or self.graph is None: raise DbtInternalError("manifest and graph must be set to get node selection") resource_types = self.resource_types(no_unit_tests) return ResourceTypeSelector( graph=self.graph, manifest=self.manifest, previous_state=self.previous_state, resource_types=resource_types, selectors=self.config.selectors, ) def get_runner_type(self, node) -> Optional[Type[BaseRunner]]: if ( node.resource_type == NodeType.Model and super().get_runner_type(node) == MicrobatchModelRunner ): return MicrobatchModelRunner return self.RUNNER_MAP.get(node.resource_type) # Special build compile_manifest method to pass add_test_edges to the compiler def compile_manifest(self) -> None: if self.manifest is None: raise DbtInternalError("compile_manifest called before manifest was loaded") self.graph: Graph = self.compiler.compile(self.manifest, add_test_edges=True) ================================================ FILE: core/dbt/task/clean.py ================================================ from pathlib import Path from shutil import rmtree from dbt import deprecations from dbt.cli.flags import Flags from dbt.config.project import Project from dbt.events.types import CheckCleanPath, ConfirmCleanPath, FinishedCleanPaths from dbt.task.base import BaseTask, move_to_nearest_project_dir from dbt_common.events.functions import fire_event from dbt_common.exceptions import DbtRuntimeError class CleanTask(BaseTask): def __init__(self, args: Flags, config: Project): super().__init__(args) self.config = config self.project = config def run(self) -> None: """ This function takes all the paths in the target file and cleans the project paths that are not protected. """ project_dir = move_to_nearest_project_dir(self.args.project_dir) potential_clean_paths = set(Path(p).resolve() for p in self.project.clean_targets) source_paths = set( Path(p).resolve() for p in (*self.project.all_source_paths, *self.project.test_paths) ) clean_paths = potential_clean_paths.difference(source_paths) if potential_clean_paths != clean_paths: raise DbtRuntimeError( f"dbt will not clean the following source paths: {[str(s) for s in source_paths.intersection(potential_clean_paths)]}" ) paths_outside_project = set( path for path in clean_paths if project_dir not in path.absolute().parents ) if paths_outside_project and self.args.clean_project_files_only: raise DbtRuntimeError( f"dbt will not clean the following directories outside the project: {[str(p) for p in paths_outside_project]}" ) if ( "dbt_modules" in self.project.clean_targets and self.config.packages_install_path not in self.config.clean_targets ): deprecations.warn("install-packages-path") for path in clean_paths: fire_event(CheckCleanPath(path=str(path))) rmtree(path, True) fire_event(ConfirmCleanPath(path=str(path))) fire_event(FinishedCleanPaths()) ================================================ FILE: core/dbt/task/clone.py ================================================ import threading from typing import AbstractSet, Any, Collection, Iterable, List, Optional, Set, Type from dbt.adapters.base import BaseAdapter, BaseRelation from dbt.artifacts.resources.types import NodeType from dbt.artifacts.schemas.run import RunResult, RunStatus from dbt.clients.jinja import MacroGenerator from dbt.context.providers import generate_runtime_model_context from dbt.contracts.graph.manifest import Manifest from dbt.graph import ResourceTypeSelector from dbt.node_types import REFABLE_NODE_TYPES from dbt.task.base import BaseRunner, resource_types_from_args from dbt.task.run import _validate_materialization_relations_dict from dbt.task.runnable import GraphRunnableMode, GraphRunnableTask from dbt_common.dataclass_schema import dbtClassMixin from dbt_common.exceptions import CompilationError, DbtInternalError class CloneRunner(BaseRunner): def before_execute(self) -> None: pass def after_execute(self, result) -> None: pass def _build_run_model_result(self, model, context): result = context["load_result"]("main") if result: status = RunStatus.Success message = str(result.response) else: status = RunStatus.Success message = "No-op" adapter_response = {} if result and isinstance(result.response, dbtClassMixin): adapter_response = result.response.to_dict(omit_none=True) return RunResult( node=model, status=status, timing=[], thread_id=threading.current_thread().name, execution_time=0, message=message, adapter_response=adapter_response, failures=None, batch_results=None, ) def compile(self, manifest: Manifest): # no-op return self.node def _materialization_relations(self, result: Any, model) -> List[BaseRelation]: if isinstance(result, str): msg = ( 'The materialization ("{}") did not explicitly return a ' "list of relations to add to the cache.".format(str(model.get_materialization())) ) raise CompilationError(msg, node=model) if isinstance(result, dict): return _validate_materialization_relations_dict(result, model) msg = ( "Invalid return value from materialization, expected a dict " 'with key "relations", got: {}'.format(str(result)) ) raise CompilationError(msg, node=model) def execute(self, model, manifest): context = generate_runtime_model_context(model, self.config, manifest) materialization_macro = manifest.find_materialization_macro_by_name( self.config.project_name, "clone", self.adapter.type() ) if "config" not in context: raise DbtInternalError( "Invalid materialization context generated, missing config: {}".format(context) ) context_config = context["config"] hook_ctx = self.adapter.pre_model_hook(context_config) try: result = MacroGenerator(materialization_macro, context)() finally: self.adapter.post_model_hook(context_config, hook_ctx) for relation in self._materialization_relations(result, model): self.adapter.cache_added(relation.incorporate(dbt_created=True)) return self._build_run_model_result(model, context) class CloneTask(GraphRunnableTask): def raise_on_first_error(self) -> bool: return False def get_run_mode(self) -> GraphRunnableMode: return GraphRunnableMode.Independent def _get_deferred_manifest(self) -> Optional[Manifest]: # Unlike other commands, 'clone' always requires a state manifest # Load previous state, regardless of whether --defer flag has been set return self._get_previous_state() def get_model_schemas(self, adapter, selected_uids: Iterable[str]) -> Set[BaseRelation]: if self.manifest is None: raise DbtInternalError("manifest was None in get_model_schemas") result: Set[BaseRelation] = set() for node in self.manifest.nodes.values(): if node.unique_id not in selected_uids: continue if node.is_relational and not node.is_ephemeral: relation = adapter.Relation.create_from(self.config, node) result.add(relation.without_identifier()) # cache the 'other' schemas too! if node.defer_relation: # type: ignore other_relation = adapter.Relation.create_from( self.config, node.defer_relation # type: ignore ) result.add(other_relation.without_identifier()) return result def before_run(self, adapter: BaseAdapter, selected_uids: AbstractSet[str]) -> RunStatus: with adapter.connection_named("master"): self.defer_to_manifest() # only create target schemas, but also cache defer_relation schemas schemas_to_create = super().get_model_schemas(adapter, selected_uids) self.create_schemas(adapter, schemas_to_create) schemas_to_cache = self.get_model_schemas(adapter, selected_uids) self.populate_adapter_cache(adapter, schemas_to_cache) return RunStatus.Success @property def resource_types(self) -> List[NodeType]: resource_types: Collection[NodeType] = resource_types_from_args( self.args, set(REFABLE_NODE_TYPES), set(REFABLE_NODE_TYPES) ) # filter out any non-refable node types resource_types = [rt for rt in resource_types if rt in REFABLE_NODE_TYPES] return list(resource_types) def get_node_selector(self) -> ResourceTypeSelector: resource_types = self.resource_types if self.manifest is None or self.graph is None: raise DbtInternalError("manifest and graph must be set to get perform node selection") return ResourceTypeSelector( graph=self.graph, manifest=self.manifest, previous_state=self.previous_state, resource_types=resource_types, selectors=self.config.selectors, ) def get_runner_type(self, _) -> Optional[Type[BaseRunner]]: return CloneRunner ================================================ FILE: core/dbt/task/compile.py ================================================ import threading from typing import Optional, Type from dbt.artifacts.schemas.run import RunResult, RunStatus from dbt.contracts.graph.manifest import Manifest from dbt.events.types import CompiledNode, ParseInlineNodeError from dbt.flags import get_flags from dbt.graph import ResourceTypeSelector from dbt.node_types import EXECUTABLE_NODE_TYPES, NodeType from dbt.parser.manifest import process_node from dbt.parser.sql import SqlBlockParser from dbt.task.base import BaseRunner from dbt.task.runnable import GraphRunnableTask from dbt_common.events.base_types import EventLevel from dbt_common.events.functions import fire_event from dbt_common.events.types import Note from dbt_common.exceptions import CompilationError from dbt_common.exceptions import DbtBaseException as DbtException from dbt_common.exceptions import DbtInternalError class CompileRunner(BaseRunner): def before_execute(self) -> None: pass def after_execute(self, result) -> None: pass def execute(self, compiled_node, manifest): return RunResult( node=compiled_node, status=RunStatus.Success, timing=[], thread_id=threading.current_thread().name, execution_time=0, message=None, adapter_response={}, failures=None, batch_results=None, ) def compile(self, manifest: Manifest): return self.compiler.compile_node(self.node, manifest, {}) def get_node_representation(self): display_quote_policy = {"database": False, "schema": False, "identifier": False} relation = self.adapter.Relation.create_from( self.config, self.node, quote_policy=display_quote_policy ) # exclude the database from output if it's the default if self.node.database == self.config.credentials.database: relation = relation.include(database=False) return str(relation) class CompileTask(GraphRunnableTask): # We add a new inline node to the manifest during initialization # it should be removed before the task is complete _inline_node_id = None def raise_on_first_error(self) -> bool: return True def get_node_selector(self) -> ResourceTypeSelector: if getattr(self.args, "inline", None): resource_types = [NodeType.SqlOperation] else: resource_types = EXECUTABLE_NODE_TYPES if self.manifest is None or self.graph is None: raise DbtInternalError("manifest and graph must be set to get perform node selection") return ResourceTypeSelector( graph=self.graph, manifest=self.manifest, previous_state=self.previous_state, resource_types=resource_types, selectors=self.config.selectors, ) def get_runner_type(self, _) -> Optional[Type[BaseRunner]]: return CompileRunner def task_end_messages(self, results) -> None: is_inline = bool(getattr(self.args, "inline", None)) output_format = getattr(self.args, "output", "text") if is_inline: matched_results = [result for result in results if result.node.name == "inline_query"] elif self.selection_arg: matched_results = [] for result in results: if result.node.name in self.selection_arg[0]: matched_results.append(result) else: fire_event( Note(msg=f"Excluded node '{result.node.name}' from results"), EventLevel.DEBUG, ) # No selector passed, compiling all nodes else: matched_results = [] for result in matched_results: fire_event( CompiledNode( node_name=result.node.name, compiled=result.node.compiled_code, is_inline=is_inline, output_format=output_format, unique_id=result.node.unique_id, quiet=get_flags().QUIET, ) ) def _runtime_initialize(self): if getattr(self.args, "inline", None): try: block_parser = SqlBlockParser( project=self.config, manifest=self.manifest, root_project=self.config ) sql_node = block_parser.parse_remote(self.args.inline, "inline_query") process_node(self.config, self.manifest, sql_node) # Special hack to remove disabled, if it's there. This would only happen # if all models are disabled in dbt_project if sql_node.config.enabled is False: sql_node.config.enabled = True self.manifest.disabled.pop(sql_node.unique_id) self.manifest.nodes[sql_node.unique_id] = sql_node # keep track of the node added to the manifest self._inline_node_id = sql_node.unique_id except CompilationError as exc: fire_event( ParseInlineNodeError( exc=str(exc.msg), node_info={ "node_path": "sql/inline_query", "node_name": "inline_query", "unique_id": "sqloperation.test.inline_query", "node_status": "failed", }, ) ) raise DbtException("Error parsing inline query") super()._runtime_initialize() def after_run(self, adapter, results) -> None: # remove inline node from manifest if self._inline_node_id: self.manifest.nodes.pop(self._inline_node_id) self._inline_node_id = None super().after_run(adapter, results) def _handle_result(self, result) -> None: super()._handle_result(result) if ( result.node.is_ephemeral_model and type(self) is CompileTask and (self.args.select or getattr(self.args, "inline", None)) ): self.node_results.append(result) ================================================ FILE: core/dbt/task/debug.py ================================================ # coding=utf-8 import importlib import os import platform import sys from collections import namedtuple from enum import Flag from pathlib import Path from typing import Any, Dict, List, Optional, Tuple import dbt.exceptions import dbt_common.clients.system import dbt_common.exceptions from dbt.adapters.factory import get_adapter, register_adapter from dbt.artifacts.schemas.results import RunStatus from dbt.cli.flags import Flags from dbt.clients.yaml_helper import load_yaml_text from dbt.config import PartialProject, Profile, Project from dbt.config.renderer import DbtProjectYamlRenderer, ProfileRenderer from dbt.events.types import DebugCmdOut, DebugCmdResult, OpenCommand from dbt.links import ProfileConfigDocs from dbt.mp_context import get_mp_context from dbt.task.base import BaseTask, get_nearest_project_dir from dbt.version import get_installed_version from dbt_common.events.format import pluralize from dbt_common.events.functions import fire_event from dbt_common.ui import green, red ONLY_PROFILE_MESSAGE = """ A `dbt_project.yml` file was not found in this directory. Using the only profile `{}`. """.lstrip() MULTIPLE_PROFILE_MESSAGE = """ A `dbt_project.yml` file was not found in this directory. dbt found the following profiles: {} To debug one of these profiles, run: dbt debug --profile [profile-name] """.lstrip() COULD_NOT_CONNECT_MESSAGE = """ dbt was unable to connect to the specified database. The database returned the following error: >{err} Check your database credentials and try again. For more information, visit: {url} """.lstrip() MISSING_PROFILE_MESSAGE = """ dbt looked for a profiles.yml file in {path}, but did not find one. For more information on configuring your profile, consult the documentation: {url} """.lstrip() FILE_NOT_FOUND = "file not found" SubtaskStatus = namedtuple( "SubtaskStatus", ["log_msg", "run_status", "details", "summary_message"] ) class DebugRunStatus(Flag): SUCCESS = True FAIL = False class DebugTask(BaseTask): def __init__(self, args: Flags) -> None: super().__init__(args) self.profiles_dir = args.PROFILES_DIR self.profile_path = os.path.join(self.profiles_dir, "profiles.yml") try: self.project_dir = get_nearest_project_dir(self.args.project_dir) except dbt_common.exceptions.DbtBaseException: # we probably couldn't find a project directory. Set project dir # to whatever was given, or default to the current directory. if args.project_dir: self.project_dir = args.project_dir else: self.project_dir = Path.cwd() self.project_path = os.path.join(self.project_dir, "dbt_project.yml") self.cli_vars: Dict[str, Any] = args.vars # set by _load_* self.profile: Optional[Profile] = None self.raw_profile_data: Optional[Dict[str, Any]] = None self.profile_name: Optional[str] = None def run(self) -> bool: # WARN: this is a legacy workflow that is not compatible with other runtime flags if getattr(self.args, "config_dir", None): fire_event( OpenCommand( open_cmd=dbt_common.clients.system.open_dir_cmd(), profiles_dir=str(self.profiles_dir), ) ) return DebugRunStatus.SUCCESS.value version: str = get_installed_version().to_version_string(skip_matcher=True) fire_event(DebugCmdOut(msg="dbt version: {}".format(version))) fire_event(DebugCmdOut(msg="python version: {}".format(sys.version.split()[0]))) fire_event(DebugCmdOut(msg="python path: {}".format(sys.executable))) fire_event(DebugCmdOut(msg="os info: {}".format(platform.platform()))) # Load profile if possible, then load adapter info (which requires the profile) load_profile_status: SubtaskStatus = self._load_profile() fire_event(DebugCmdOut(msg="Using profiles dir at {}".format(self.profiles_dir))) fire_event(DebugCmdOut(msg="Using profiles.yml file at {}".format(self.profile_path))) fire_event(DebugCmdOut(msg="Using dbt_project.yml file at {}".format(self.project_path))) if load_profile_status.run_status == RunStatus.Success: if self.profile is None: raise dbt_common.exceptions.DbtInternalError( "Profile should not be None if loading profile completed" ) else: adapter_type: str = self.profile.credentials.type adapter_version: str = self._read_adapter_version( f"dbt.adapters.{adapter_type}.__version__" ) fire_event(DebugCmdOut(msg="adapter type: {}".format(adapter_type))) fire_event(DebugCmdOut(msg="adapter version: {}".format(adapter_version))) # Get project loaded to do additional checks load_project_status: SubtaskStatus = self._load_project() dependencies_statuses: List[SubtaskStatus] = [] if getattr(self.args, "connection", False): fire_event(DebugCmdOut(msg="Skipping steps before connection verification")) else: # this job's status not logged since already accounted for in _load_* commands self.test_configuration(load_profile_status.log_msg, load_project_status.log_msg) dependencies_statuses = self.test_dependencies() # Test connection connection_status = self.test_connection() # Log messages from any fails all_statuses: List[SubtaskStatus] = [ load_profile_status, load_project_status, *dependencies_statuses, connection_status, ] all_failing_statuses: List[SubtaskStatus] = list( filter(lambda status: status.run_status == RunStatus.Error, all_statuses) ) failure_count: int = len(all_failing_statuses) if failure_count > 0: fire_event(DebugCmdResult(msg=red(f"{(pluralize(failure_count, 'check'))} failed:"))) for status in all_failing_statuses: fire_event(DebugCmdResult(msg=f"{status.summary_message}\n")) return DebugRunStatus.FAIL.value else: fire_event(DebugCmdResult(msg=green("All checks passed!"))) return DebugRunStatus.SUCCESS.value # ============================== # Override for elsewhere in core # ============================== def interpret_results(self, results): return results # =============== # Loading profile # =============== def _load_profile(self) -> SubtaskStatus: """ Side effects: load self.profile load self.target_name load self.raw_profile_data """ if not os.path.exists(self.profile_path): return SubtaskStatus( log_msg=red("ERROR not found"), run_status=RunStatus.Error, details=FILE_NOT_FOUND, summary_message=MISSING_PROFILE_MESSAGE.format( path=self.profile_path, url=ProfileConfigDocs ), ) raw_profile_data = load_yaml_text( dbt_common.clients.system.load_file_contents(self.profile_path) ) if isinstance(raw_profile_data, dict): self.raw_profile_data = raw_profile_data profile_errors = [] profile_names, summary_message = self._choose_profile_names() renderer = ProfileRenderer(self.cli_vars) for profile_name in profile_names: try: profile: Profile = Profile.render( renderer, profile_name, self.args.profile, self.args.target, # TODO: Generalize safe access to flags.THREADS: # https://github.com/dbt-labs/dbt-core/issues/6259 getattr(self.args, "threads", None), ) except dbt_common.exceptions.DbtConfigError as exc: profile_errors.append(str(exc)) else: if len(profile_names) == 1: # if a profile was specified, set it on the task self.target_name = self._choose_target_name(profile_name) self.profile = profile if profile_errors: details = "\n\n".join(profile_errors) return SubtaskStatus( log_msg=red("ERROR invalid"), run_status=RunStatus.Error, details=details, summary_message=( summary_message + f"Profile loading failed for the following reason:" f"\n{details}" f"\n" ), ) else: return SubtaskStatus( log_msg=green("OK found and valid"), run_status=RunStatus.Success, details="", summary_message="Profile is valid", ) def _choose_profile_names(self) -> Tuple[List[str], str]: project_profile: Optional[str] = None if os.path.exists(self.project_path): try: partial = PartialProject.from_project_root( os.path.dirname(self.project_path), verify_version=bool(self.args.VERSION_CHECK), ) renderer = DbtProjectYamlRenderer(None, self.cli_vars) project_profile = partial.render_profile_name(renderer) except dbt.exceptions.DbtProjectError: pass args_profile: Optional[str] = getattr(self.args, "profile", None) try: return [Profile.pick_profile_name(args_profile, project_profile)], "" except dbt_common.exceptions.DbtConfigError: pass # try to guess profiles = [] if self.raw_profile_data: profiles = [k for k in self.raw_profile_data if k != "config"] if project_profile is None: summary_message = "Could not load dbt_project.yml\n" elif len(profiles) == 0: summary_message = "The profiles.yml has no profiles\n" elif len(profiles) == 1: summary_message = ONLY_PROFILE_MESSAGE.format(profiles[0]) else: summary_message = MULTIPLE_PROFILE_MESSAGE.format( "\n".join(" - {}".format(o) for o in profiles) ) return profiles, summary_message def _read_adapter_version(self, module) -> str: """read the version out of a standard adapter file""" try: version = importlib.import_module(module).version except ModuleNotFoundError: version = red("ERROR not found") except Exception as exc: version = red("ERROR {}".format(exc)) raise dbt.exceptions.DbtInternalError( f"Error when reading adapter version from {module}: {exc}" ) return version def _choose_target_name(self, profile_name: str): has_raw_profile = ( self.raw_profile_data is not None and profile_name in self.raw_profile_data ) if not has_raw_profile: return None # mypy appeasement, we checked just above assert self.raw_profile_data is not None raw_profile = self.raw_profile_data[profile_name] renderer = ProfileRenderer(self.cli_vars) target_name, _ = Profile.render_profile( raw_profile=raw_profile, profile_name=profile_name, target_override=getattr(self.args, "target", None), renderer=renderer, ) return target_name # =============== # Loading project # =============== def _load_project(self) -> SubtaskStatus: """ Side effect: load self.project """ if not os.path.exists(self.project_path): return SubtaskStatus( log_msg=red("ERROR not found"), run_status=RunStatus.Error, details=FILE_NOT_FOUND, summary_message=( f"Project loading failed for the following reason:" f"\n project path <{self.project_path}> not found" ), ) renderer = DbtProjectYamlRenderer(self.profile, self.cli_vars) try: self.project = Project.from_project_root( str(self.project_dir), renderer, verify_version=self.args.VERSION_CHECK, ) except dbt_common.exceptions.DbtConfigError as exc: return SubtaskStatus( log_msg=red("ERROR invalid"), run_status=RunStatus.Error, details=str(exc), summary_message=( f"Project loading failed for the following reason:" f"\n{str(exc)}" f"\n" ), ) else: return SubtaskStatus( log_msg=green("OK found and valid"), run_status=RunStatus.Success, details="", summary_message="Project is valid", ) def _profile_found(self) -> str: if not self.raw_profile_data: return red("ERROR not found") assert self.raw_profile_data is not None if self.profile_name in self.raw_profile_data: return green("OK found") else: return red("ERROR not found") def _target_found(self) -> str: requirements = self.raw_profile_data and self.profile_name and self.target_name if not requirements: return red("ERROR not found") # mypy appeasement, we checked just above assert self.raw_profile_data is not None assert self.profile_name is not None assert self.target_name is not None if self.profile_name not in self.raw_profile_data: return red("ERROR not found") profiles = self.raw_profile_data[self.profile_name]["outputs"] if self.target_name not in profiles: return red("ERROR not found") else: return green("OK found") # ============ # Config tests # ============ def test_git(self) -> SubtaskStatus: try: dbt_common.clients.system.run_cmd(os.getcwd(), ["git", "--help"]) except dbt_common.exceptions.ExecutableError as exc: return SubtaskStatus( log_msg=red("ERROR"), run_status=RunStatus.Error, details="git error", summary_message="Error from git --help: {!s}".format(exc), ) else: return SubtaskStatus( log_msg=green("OK found"), run_status=RunStatus.Success, details="", summary_message="git is installed and on the path", ) def test_dependencies(self) -> List[SubtaskStatus]: fire_event(DebugCmdOut(msg="Required dependencies:")) git_test_status = self.test_git() fire_event(DebugCmdResult(msg=f" - git [{git_test_status.log_msg}]\n")) return [git_test_status] def test_configuration(self, profile_status_msg, project_status_msg): fire_event(DebugCmdOut(msg="Configuration:")) fire_event(DebugCmdOut(msg=f" profiles.yml file [{profile_status_msg}]")) fire_event(DebugCmdOut(msg=f" dbt_project.yml file [{project_status_msg}]")) # skip profile stuff if we can't find a profile name if self.profile_name is not None: fire_event( DebugCmdOut( msg=" profile: {} [{}]\n".format(self.profile_name, self._profile_found()) ) ) fire_event( DebugCmdOut( msg=" target: {} [{}]\n".format(self.target_name, self._target_found()) ) ) # =============== # Connection test # =============== @staticmethod def attempt_connection(profile) -> Optional[str]: """Return a string containing the error message, or None if there was no error.""" register_adapter(profile, get_mp_context()) adapter = get_adapter(profile) try: with adapter.connection_named("debug"): # is defined in adapter class adapter.debug_query() except Exception as exc: return COULD_NOT_CONNECT_MESSAGE.format( err=str(exc), url=ProfileConfigDocs, ) return None def test_connection(self) -> SubtaskStatus: if self.profile is None: fire_event(DebugCmdOut(msg="Connection test skipped since no profile was found")) return SubtaskStatus( log_msg=red("SKIPPED"), run_status=RunStatus.Skipped, details="No profile found", summary_message="Connection test skipped since no profile was found", ) fire_event(DebugCmdOut(msg="Connection:")) for k, v in self.profile.credentials.connection_info(): fire_event(DebugCmdOut(msg=f" {k}: {v}")) connection_result = self.attempt_connection(self.profile) if connection_result is None: status = SubtaskStatus( log_msg=green("OK connection ok"), run_status=RunStatus.Success, details="", summary_message="Connection test passed", ) else: status = SubtaskStatus( log_msg=red("ERROR"), run_status=RunStatus.Error, details="Failure in connecting to db", summary_message=connection_result, ) fire_event(DebugCmdOut(msg=f" Connection test: [{status.log_msg}]\n")) return status @classmethod def validate_connection(cls, target_dict) -> None: """Validate a connection dictionary. On error, raises a DbtConfigError.""" target_name = "test" # make a fake profile that we can parse profile_data = { "outputs": { target_name: target_dict, }, } # this will raise a DbtConfigError on failure profile = Profile.from_raw_profile_info( raw_profile=profile_data, profile_name="", target_override=target_name, renderer=ProfileRenderer({}), ) result = cls.attempt_connection(profile) if result is not None: raise dbt.exceptions.DbtProfileError(result, result_type="connection_failure") ================================================ FILE: core/dbt/task/deps.py ================================================ import json from hashlib import sha1 from pathlib import Path from typing import Any, Dict, List, Optional import yaml import dbt.deprecations import dbt.exceptions import dbt.utils from dbt.config import Project from dbt.config.project import load_yml_dict, package_config_from_data from dbt.config.renderer import PackageRenderer from dbt.constants import PACKAGE_LOCK_FILE_NAME, PACKAGE_LOCK_HASH_KEY from dbt.contracts.project import PackageSpec from dbt.deps.base import downloads_directory from dbt.deps.registry import RegistryPinnedPackage from dbt.deps.resolver import resolve_lock_packages, resolve_packages from dbt.events.types import ( DepsAddPackage, DepsFoundDuplicatePackage, DepsInstallInfo, DepsListSubdirectory, DepsLockUpdating, DepsNoPackagesFound, DepsNotifyUpdatesAvailable, DepsStartPackageInstall, DepsUpdateAvailable, DepsUpToDate, ) from dbt.task.base import BaseTask, move_to_nearest_project_dir from dbt_common.clients import system from dbt_common.events.functions import fire_event from dbt_common.events.types import Formatting class dbtPackageDumper(yaml.Dumper): def increase_indent(self, flow=False, indentless=False): return super(dbtPackageDumper, self).increase_indent(flow, False) def _create_sha1_hash(packages: List[PackageSpec]) -> str: """Create a SHA1 hash of the packages list, this is used to determine if the packages for current execution matches the previous lock. Args: list[Packages]: list of packages specified that are already rendered Returns: str: SHA1 hash of the packages list """ package_strs = [json.dumps(package.to_dict(), sort_keys=True) for package in packages] package_strs = sorted(package_strs) return sha1("\n".join(package_strs).encode("utf-8")).hexdigest() def _create_packages_yml_entry(package: str, version: Optional[str], source: str) -> dict: """Create a formatted entry to add to `packages.yml` or `package-lock.yml` file Args: package (str): Name of package to download version (str): Version of package to download source (str): Source of where to download package from Returns: dict: Formatted dict to write to `packages.yml` or `package-lock.yml` file """ package_key = source version_key = "version" if source == "hub": package_key = "package" packages_yml_entry = {package_key: package} if source == "git": version_key = "revision" if version: if "," in version: version = version.split(",") # type: ignore packages_yml_entry[version_key] = version return packages_yml_entry class DepsTask(BaseTask): def __init__(self, args: Any, project: Project) -> None: super().__init__(args=args) # N.B. This is a temporary fix for a bug when using relative paths via # --project-dir with deps. A larger overhaul of our path handling methods # is needed to fix this the "right" way. # See GH-7615 project.project_root = str(Path(project.project_root).resolve()) self.project = project self.cli_vars = args.vars def track_package_install( self, package_name: str, source_type: str, version: Optional[str] ) -> None: # Hub packages do not need to be hashed, as they are public if source_type == "local": package_name = dbt.utils.md5(package_name) version = "local" elif source_type == "tarball": package_name = dbt.utils.md5(package_name) version = "tarball" elif source_type != "hub": package_name = dbt.utils.md5(package_name) version = dbt.utils.md5(version) dbt.tracking.track_package_install( "deps", self.project.hashed_name(), {"name": package_name, "source": source_type, "version": version}, ) def check_for_duplicate_packages(self, packages_yml): """Loop through contents of `packages.yml` to remove entries that match the package being added. This method is called only during `dbt deps --add-package` to check if the package being added already exists in packages.yml. It uses substring matching to identify duplicates, which means it will match across different package sources. For example, adding a hub package "dbt-labs/dbt_utils" will remove an existing git package "https://github.com/dbt-labs/dbt-utils.git" since both contain "dbt_utils" or "dbt-utils". The matching is flexible to handle both underscore and hyphen variants of package names, as git repos often use hyphens (dbt-utils) while package names use underscores (dbt_utils). Word boundaries (/, .) are enforced to prevent false matches like "dbt-core" matching "dbt-core-utils". Args: packages_yml (dict): In-memory read of `packages.yml` contents Returns: dict: Updated packages_yml contents with matching packages removed """ # Extract the package name for matching package_name = self.args.add_package["name"] # Create variants for flexible matching (handle _ vs -) # Check multiple variants to handle naming inconsistencies between hub and git package_name_parts = [ package_name, # Original: "dbt-labs/dbt_utils" package_name.replace("_", "-"), # Hyphens: "dbt-labs/dbt-utils" package_name.replace("-", "_"), # Underscores: "dbt_labs/dbt_utils" ] # Extract just the package name without org (after last /) if "/" in package_name: short_name = package_name.split("/")[-1] package_name_parts.extend( [ short_name, # "dbt_utils" short_name.replace("_", "-"), # "dbt-utils" short_name.replace("-", "_"), # "dbt_utils" (deduplicated) ] ) # Remove duplicates from package_name_parts package_name_parts = list(set(package_name_parts)) # Iterate backwards to safely delete items without index shifting issues for i in range(len(packages_yml["packages"]) - 1, -1, -1): pkg_entry = packages_yml["packages"][i] # Get the package identifier key (package type determines which key exists) # This avoids iterating over non-string values like warn-unpinned: false package_identifier = ( pkg_entry.get("package") # hub/registry package or pkg_entry.get("git") # git package or pkg_entry.get("local") # local package or pkg_entry.get("tarball") # tarball package or pkg_entry.get("private") # private package ) # Check if any variant of the package name appears in the identifier # Use word boundaries to avoid false matches (e.g., "dbt-core" shouldn't match "dbt-core-utils") # Word boundaries are: start/end of string, /, or . # Note: - and _ are NOT boundaries since they're used within compound package names if package_identifier: is_duplicate = False for name_variant in package_name_parts: if name_variant in package_identifier: # Found a match, now verify it's not a substring of a larger word # Check characters before and after the match idx = package_identifier.find(name_variant) start_ok = idx == 0 or package_identifier[idx - 1] in "/." end_idx = idx + len(name_variant) end_ok = ( end_idx == len(package_identifier) or package_identifier[end_idx] in "/." ) if start_ok and end_ok: is_duplicate = True break if is_duplicate: del packages_yml["packages"][i] # Filter out non-string values (like warn-unpinned boolean) before logging # Note: Check for bool first since bool is a subclass of int in Python loggable_package = { k: v for k, v in pkg_entry.items() if not isinstance(v, bool) and isinstance(v, (str, int, float)) and k != "unrendered" } fire_event(DepsFoundDuplicatePackage(removed_package=loggable_package)) return packages_yml def add(self): packages_yml_filepath = ( f"{self.project.project_root}/{self.project.packages_specified_path}" ) if not system.path_exists(packages_yml_filepath): with open(packages_yml_filepath, "w") as package_yml: yaml.safe_dump({"packages": []}, package_yml) fire_event(Formatting("Created packages.yml")) new_package_entry = _create_packages_yml_entry( self.args.add_package["name"], self.args.add_package["version"], self.args.source ) with open(packages_yml_filepath, "r") as user_yml_obj: packages_yml = yaml.safe_load(user_yml_obj) packages_yml = self.check_for_duplicate_packages(packages_yml) packages_yml["packages"].append(new_package_entry) self.project.packages.packages = package_config_from_data(packages_yml).packages if packages_yml: with open(packages_yml_filepath, "w") as pkg_obj: pkg_obj.write( yaml.dump(packages_yml, Dumper=dbtPackageDumper, default_flow_style=False) ) fire_event( DepsAddPackage( package_name=self.args.add_package["name"], version=self.args.add_package["version"], packages_filepath=packages_yml_filepath, ) ) def lock(self) -> None: lock_filepath = f"{self.project.project_root}/{PACKAGE_LOCK_FILE_NAME}" packages = self.project.packages.packages packages_installed: Dict[str, Any] = {"packages": []} if not packages: fire_event(DepsNoPackagesFound()) return with downloads_directory(): resolved_deps = resolve_packages(packages, self.project, self.cli_vars) # this loop is to create the package-lock.yml in the same format as original packages.yml # package-lock.yml includes both the stated packages in packages.yml along with dependent packages renderer = PackageRenderer(self.cli_vars) for package in resolved_deps: package_dict = package.to_dict() package_dict["name"] = package.get_project_name(self.project, renderer) packages_installed["packages"].append(package_dict) packages_installed[PACKAGE_LOCK_HASH_KEY] = _create_sha1_hash( self.project.packages.packages ) with open(lock_filepath, "w") as lock_obj: yaml.dump(packages_installed, lock_obj, Dumper=dbtPackageDumper) fire_event(DepsLockUpdating(lock_filepath=lock_filepath)) def run(self) -> None: move_to_nearest_project_dir(self.args.project_dir) if self.args.add_package: self.add() # Check lock file exist and generated by the same packages.yml # or dependencies.yml. lock_file_path = f"{self.project.project_root}/{PACKAGE_LOCK_FILE_NAME}" if not system.path_exists(lock_file_path): self.lock() elif self.args.upgrade: self.lock() else: # Check dependency definition is modified or not. current_hash = _create_sha1_hash(self.project.packages.packages) previous_hash = load_yml_dict(lock_file_path).get(PACKAGE_LOCK_HASH_KEY, None) if previous_hash != current_hash: self.lock() # Early return when 'dbt deps --lock' # Just resolve packages and write lock file, don't actually install packages if self.args.lock: return if system.path_exists(self.project.packages_install_path): system.rmtree(self.project.packages_install_path) system.make_directory(self.project.packages_install_path) packages_lock_dict = load_yml_dict(f"{self.project.project_root}/{PACKAGE_LOCK_FILE_NAME}") renderer = PackageRenderer(self.cli_vars) packages_lock_config = package_config_from_data( renderer.render_data(packages_lock_dict), packages_lock_dict ).packages if not packages_lock_config: fire_event(DepsNoPackagesFound()) return with downloads_directory(): lock_defined_deps = resolve_lock_packages(packages_lock_config) renderer = PackageRenderer(self.cli_vars) packages_to_upgrade = [] for package in lock_defined_deps: package_name = package.name source_type = package.source_type() version = package.get_version() fire_event(DepsStartPackageInstall(package_name=package_name)) package.install(self.project, renderer) fire_event(DepsInstallInfo(version_name=package.nice_version_name())) if isinstance(package, RegistryPinnedPackage): version_latest = package.get_version_latest() if version_latest != version: packages_to_upgrade.append(package_name) fire_event(DepsUpdateAvailable(version_latest=version_latest)) else: fire_event(DepsUpToDate()) if package.get_subdirectory(): fire_event(DepsListSubdirectory(subdirectory=package.get_subdirectory())) self.track_package_install( package_name=package_name, source_type=source_type, version=version ) if packages_to_upgrade: fire_event(Formatting("")) fire_event(DepsNotifyUpdatesAvailable(packages=packages_to_upgrade)) ================================================ FILE: core/dbt/task/docs/__init__.py ================================================ import os DOCS_INDEX_FILE_PATH = os.path.normpath(os.path.join(os.path.dirname(__file__), "index.html")) ================================================ FILE: core/dbt/task/docs/generate.py ================================================ import os import shutil from dataclasses import replace from datetime import datetime, timezone from itertools import chain from typing import Any, Dict, Iterable, List, Optional, Set, Tuple import agate import dbt.compilation import dbt.exceptions import dbt.utils import dbt_common.utils.formatting from dbt.adapters.events.types import ( BuildingCatalog, CannotGenerateDocs, CatalogWritten, WriteCatalogFailure, ) from dbt.adapters.factory import get_adapter from dbt.artifacts.schemas.catalog import ( CatalogArtifact, CatalogKey, CatalogResults, CatalogTable, ColumnMetadata, PrimitiveDict, StatsDict, StatsItem, TableMetadata, ) from dbt.artifacts.schemas.results import NodeStatus from dbt.constants import CATALOG_FILENAME, MANIFEST_FILE_NAME from dbt.contracts.graph.manifest import Manifest from dbt.contracts.graph.nodes import ResultNode from dbt.events.types import ArtifactWritten from dbt.exceptions import AmbiguousCatalogMatchError from dbt.graph import ResourceTypeSelector from dbt.graph.graph import UniqueId from dbt.node_types import EXECUTABLE_NODE_TYPES, NodeType from dbt.parser.manifest import write_manifest from dbt.task.compile import CompileTask from dbt.task.docs import DOCS_INDEX_FILE_PATH from dbt.utils.artifact_upload import add_artifact_produced from dbt_common.clients.system import load_file_contents from dbt_common.dataclass_schema import ValidationError from dbt_common.events.functions import fire_event from dbt_common.exceptions import DbtInternalError def get_stripped_prefix(source: Dict[str, Any], prefix: str) -> Dict[str, Any]: """Go through the source, extracting every key/value pair where the key starts with the given prefix. """ cut = len(prefix) return {k[cut:]: v for k, v in source.items() if k.startswith(prefix)} def build_catalog_table(data) -> CatalogTable: # build the new table's metadata + stats metadata = TableMetadata.from_dict(get_stripped_prefix(data, "table_")) stats = format_stats(get_stripped_prefix(data, "stats:")) return CatalogTable( metadata=metadata, stats=stats, columns={}, ) # keys are database name, schema name, table name class Catalog(Dict[CatalogKey, CatalogTable]): def __init__(self, columns: List[PrimitiveDict]) -> None: super().__init__() for col in columns: self.add_column(col) def get_table(self, data: PrimitiveDict) -> CatalogTable: database = data.get("table_database") if database is None: dkey: Optional[str] = None else: dkey = str(database) try: key = CatalogKey( dkey, str(data["table_schema"]), str(data["table_name"]), ) except KeyError as exc: raise dbt_common.exceptions.CompilationError( "Catalog information missing required key {} (got {})".format(exc, data) ) table: CatalogTable if key in self: table = self[key] else: table = build_catalog_table(data) self[key] = table return table def add_column(self, data: PrimitiveDict): table = self.get_table(data) column_data = get_stripped_prefix(data, "column_") # the index should really never be that big so it's ok to end up # serializing this to JSON (2^53 is the max safe value there) column_data["index"] = int(column_data["index"]) column = ColumnMetadata.from_dict(column_data) table.columns[column.name] = column def make_unique_id_map( self, manifest: Manifest, selected_node_ids: Optional[Set[UniqueId]] = None ) -> Tuple[Dict[str, CatalogTable], Dict[str, CatalogTable]]: """ Create mappings between CatalogKeys and CatalogTables for nodes and sources, filtered by selected_node_ids. By default, selected_node_ids is None and all nodes and sources defined in the manifest are included in the mappings. """ nodes: Dict[str, CatalogTable] = {} sources: Dict[str, CatalogTable] = {} node_map, source_map = get_unique_id_mapping(manifest) table: CatalogTable for table in self.values(): key = table.key() if key in node_map: unique_id = node_map[key] if selected_node_ids is None or unique_id in selected_node_ids: nodes[unique_id] = replace(table, unique_id=unique_id) unique_ids = source_map.get(table.key(), set()) for unique_id in unique_ids: if unique_id in sources: raise AmbiguousCatalogMatchError( unique_id, sources[unique_id].to_dict(omit_none=True), table.to_dict(omit_none=True), ) elif selected_node_ids is None or unique_id in selected_node_ids: sources[unique_id] = replace(table, unique_id=unique_id) return nodes, sources def format_stats(stats: PrimitiveDict) -> StatsDict: """Given a dictionary following this layout: { 'encoded:label': 'Encoded', 'encoded:value': 'Yes', 'encoded:description': 'Indicates if the column is encoded', 'encoded:include': True, 'size:label': 'Size', 'size:value': 128, 'size:description': 'Size of the table in MB', 'size:include': True, } format_stats will convert the dict into a StatsDict with keys of 'encoded' and 'size'. """ stats_collector: StatsDict = {} base_keys = {k.split(":")[0] for k in stats} for key in base_keys: dct: PrimitiveDict = {"id": key} for subkey in ("label", "value", "description", "include"): dct[subkey] = stats["{}:{}".format(key, subkey)] try: stats_item = StatsItem.from_dict(dct) except ValidationError: continue if stats_item.include: stats_collector[key] = stats_item # we always have a 'has_stats' field, it's never included has_stats = StatsItem( id="has_stats", label="Has Stats?", value=len(stats_collector) > 0, description="Indicates whether there are statistics for this table", include=False, ) stats_collector["has_stats"] = has_stats return stats_collector def mapping_key(node: ResultNode) -> CatalogKey: dkey = dbt_common.utils.formatting.lowercase(node.database) return CatalogKey(dkey, node.schema.lower(), node.identifier.lower()) def get_unique_id_mapping( manifest: Manifest, ) -> Tuple[Dict[CatalogKey, str], Dict[CatalogKey, Set[str]]]: # A single relation could have multiple unique IDs pointing to it if a # source were also a node. node_map: Dict[CatalogKey, str] = {} source_map: Dict[CatalogKey, Set[str]] = {} for unique_id, node in manifest.nodes.items(): key = mapping_key(node) node_map[key] = unique_id for unique_id, source in manifest.sources.items(): key = mapping_key(source) if key not in source_map: source_map[key] = set() source_map[key].add(unique_id) return node_map, source_map class GenerateTask(CompileTask): def run(self) -> CatalogArtifact: compile_results = None if self.args.compile: compile_results = CompileTask.run(self) if any(r.status == NodeStatus.Error for r in compile_results): fire_event(CannotGenerateDocs()) return CatalogArtifact.from_results( nodes={}, sources={}, generated_at=datetime.now(timezone.utc).replace(tzinfo=None), errors=None, compile_results=compile_results, ) shutil.copyfile( DOCS_INDEX_FILE_PATH, os.path.join(self.config.project_target_path, "index.html") ) for asset_path in self.config.asset_paths: to_asset_path = os.path.join(self.config.project_target_path, asset_path) if os.path.exists(to_asset_path): shutil.rmtree(to_asset_path) from_asset_path = os.path.join(self.config.project_root, asset_path) if os.path.exists(from_asset_path): shutil.copytree(from_asset_path, to_asset_path) if self.manifest is None: raise DbtInternalError("self.manifest was None in run!") selected_node_ids: Optional[Set[UniqueId]] = None if self.args.empty_catalog: catalog_table: agate.Table = agate.Table([]) exceptions: List[Exception] = [] selected_node_ids = set() else: adapter = get_adapter(self.config) with adapter.connection_named("generate_catalog"): fire_event(BuildingCatalog()) # Get a list of relations we need from the catalog relations = None if self.job_queue is not None: selected_node_ids = self.job_queue.get_selected_nodes() selected_nodes = self._get_nodes_from_ids(self.manifest, selected_node_ids) # Source selection is handled separately from main job_queue selection because # SourceDefinition nodes cannot be safely compiled / run by the CompileRunner / CompileTask, # but should still be included in the catalog based on the selection spec selected_source_ids = self._get_selected_source_ids() selected_source_nodes = self._get_nodes_from_ids( self.manifest, selected_source_ids ) selected_node_ids.update(selected_source_ids) selected_nodes.extend(selected_source_nodes) relations = { adapter.Relation.create_from(adapter.config, node) for node in selected_nodes } # This generates the catalog as an agate.Table catalogable_nodes = chain( [ node for node in self.manifest.nodes.values() if (node.is_relational and not node.is_ephemeral_model) ], self.manifest.sources.values(), ) used_schemas = self.manifest.get_used_schemas() catalog_table, exceptions = adapter.get_filtered_catalog( catalogable_nodes, used_schemas, relations ) catalog_data: List[PrimitiveDict] = [ dict(zip(catalog_table.column_names, map(dbt.utils._coerce_decimal, row))) for row in catalog_table ] catalog = Catalog(catalog_data) errors: Optional[List[str]] = None if exceptions: errors = [str(e) for e in exceptions] nodes, sources = catalog.make_unique_id_map(self.manifest, selected_node_ids) results = self.get_catalog_results( nodes=nodes, sources=sources, generated_at=datetime.now(timezone.utc).replace(tzinfo=None), compile_results=compile_results, errors=errors, ) catalog_path = os.path.join(self.config.project_target_path, CATALOG_FILENAME) results.write(catalog_path) add_artifact_produced(catalog_path) fire_event( ArtifactWritten(artifact_type=results.__class__.__name__, artifact_path=catalog_path) ) if self.args.compile: write_manifest(self.manifest, self.config.project_target_path) if self.args.static: # Read manifest.json and catalog.json read_manifest_data = load_file_contents( os.path.join(self.config.project_target_path, MANIFEST_FILE_NAME) ) read_catalog_data = load_file_contents(catalog_path) # Create new static index file contents index_data = load_file_contents(DOCS_INDEX_FILE_PATH) index_data = index_data.replace('"MANIFEST.JSON INLINE DATA"', read_manifest_data) index_data = index_data.replace('"CATALOG.JSON INLINE DATA"', read_catalog_data) # Write out the new index file static_index_path = os.path.join(self.config.project_target_path, "static_index.html") with open(static_index_path, "wb") as static_index_file: static_index_file.write(bytes(index_data, "utf8")) if exceptions: fire_event(WriteCatalogFailure(num_exceptions=len(exceptions))) fire_event(CatalogWritten(path=os.path.abspath(catalog_path))) return results def get_node_selector(self) -> ResourceTypeSelector: if self.manifest is None or self.graph is None: raise DbtInternalError("manifest and graph must be set to perform node selection") return ResourceTypeSelector( graph=self.graph, manifest=self.manifest, previous_state=self.previous_state, resource_types=EXECUTABLE_NODE_TYPES, include_empty_nodes=True, selectors=self.config.selectors, ) def get_catalog_results( self, nodes: Dict[str, CatalogTable], sources: Dict[str, CatalogTable], generated_at: datetime, compile_results: Optional[Any], errors: Optional[List[str]], ) -> CatalogArtifact: return CatalogArtifact.from_results( generated_at=generated_at, nodes=nodes, sources=sources, compile_results=compile_results, errors=errors, ) @classmethod def interpret_results(self, results: Optional[CatalogResults]) -> bool: if results is None: return False if results.errors: return False compile_results = results._compile_results if compile_results is None: return True return super().interpret_results(compile_results) @staticmethod def _get_nodes_from_ids(manifest: Manifest, node_ids: Iterable[str]) -> List[ResultNode]: selected: List[ResultNode] = [] for unique_id in node_ids: if unique_id in manifest.nodes: node = manifest.nodes[unique_id] if node.is_relational and not node.is_ephemeral_model: selected.append(node) elif unique_id in manifest.sources: source = manifest.sources[unique_id] selected.append(source) return selected def _get_selected_source_ids(self) -> Set[UniqueId]: if self.manifest is None or self.graph is None: raise DbtInternalError("manifest and graph must be set to perform node selection") source_selector = ResourceTypeSelector( graph=self.graph, manifest=self.manifest, previous_state=self.previous_state, resource_types=[NodeType.Source], selectors=self.config.selectors, ) return source_selector.get_graph_queue(self.get_selection_spec()).get_selected_nodes() ================================================ FILE: core/dbt/task/docs/serve.py ================================================ import os import shutil import socketserver import webbrowser from http.server import SimpleHTTPRequestHandler import click from dbt.task.base import ConfiguredTask from dbt.task.docs import DOCS_INDEX_FILE_PATH class ServeTask(ConfiguredTask): def run(self): os.chdir(self.config.project_target_path) shutil.copyfile(DOCS_INDEX_FILE_PATH, "index.html") port = self.args.port host = self.args.host if self.args.browser: webbrowser.open_new_tab(f"http://localhost:{port}") with socketserver.TCPServer((host, port), SimpleHTTPRequestHandler) as httpd: click.echo(f"Serving docs at {port}") click.echo(f"To access from your browser, navigate to: http://localhost:{port}") click.echo("\n\n") click.echo("Press Ctrl+C to exit.") httpd.serve_forever() ================================================ FILE: core/dbt/task/freshness.py ================================================ import os import threading import time from typing import AbstractSet, Dict, List, Optional, Type from dbt import deprecations from dbt.adapters.base import BaseAdapter from dbt.adapters.base.impl import FreshnessResponse from dbt.adapters.base.relation import BaseRelation from dbt.adapters.capability import Capability from dbt.adapters.contracts.connection import AdapterResponse from dbt.artifacts.schemas.freshness import ( FreshnessResult, FreshnessStatus, PartialSourceFreshnessResult, SourceFreshnessResult, ) from dbt.clients import jinja from dbt.constants import SOURCE_RESULT_FILE_NAME from dbt.context.providers import RuntimeProvider, SourceContext from dbt.contracts.graph.manifest import Manifest from dbt.contracts.graph.nodes import HookNode, SourceDefinition from dbt.contracts.results import RunStatus from dbt.events.types import FreshnessCheckComplete, LogFreshnessResult, LogStartLine from dbt.graph import ResourceTypeSelector from dbt.node_types import NodeType, RunHookType from dbt_common.events.base_types import EventLevel from dbt_common.events.functions import fire_event from dbt_common.events.types import Note from dbt_common.exceptions import DbtInternalError, DbtRuntimeError from .base import BaseRunner from .printer import print_run_result_error from .run import RunTask class FreshnessRunner(BaseRunner): def __init__(self, config, adapter, node, node_index, num_nodes) -> None: super().__init__(config, adapter, node, node_index, num_nodes) self._metadata_freshness_cache: Dict[BaseRelation, FreshnessResult] = {} def set_metadata_freshness_cache( self, metadata_freshness_cache: Dict[BaseRelation, FreshnessResult] ) -> None: self._metadata_freshness_cache = metadata_freshness_cache def on_skip(self): raise DbtRuntimeError("Freshness: nodes cannot be skipped!") def before_execute(self) -> None: description = "freshness of {0.source_name}.{0.name}".format(self.node) fire_event( LogStartLine( description=description, index=self.node_index, total=self.num_nodes, node_info=self.node.node_info, ) ) def after_execute(self, result) -> None: if hasattr(result, "node"): source_name = result.node.source_name table_name = result.node.name else: source_name = result.source_name table_name = result.table_name level = LogFreshnessResult.status_to_level(str(result.status)) fire_event( LogFreshnessResult( status=result.status, source_name=source_name, table_name=table_name, index=self.node_index, total=self.num_nodes, execution_time=result.execution_time, node_info=self.node.node_info, ), level=level, ) def error_result(self, node, message, start_time, timing_info): return self._build_run_result( node=node, start_time=start_time, status=FreshnessStatus.RuntimeErr, timing_info=timing_info, message=message, ) def _build_run_result(self, node, start_time, status, timing_info, message): execution_time = time.time() - start_time thread_id = threading.current_thread().name return PartialSourceFreshnessResult( status=status, thread_id=thread_id, execution_time=execution_time, timing=timing_info, message=message, node=node, adapter_response={}, failures=None, ) def from_run_result(self, result, start_time, timing_info): result.execution_time = time.time() - start_time result.timing.extend(timing_info) return result def execute(self, compiled_node, manifest): relation = self.adapter.Relation.create_from(self.config, compiled_node) # given a Source, calculate its freshness. with self.adapter.connection_named(compiled_node.unique_id, compiled_node): self.adapter.clear_transaction() adapter_response: Optional[AdapterResponse] = None freshness: Optional[FreshnessResponse] = None if compiled_node.loaded_at_query is not None: # within the context user can have access to `this`, `source_node`(`model` will point to the same thing), etc compiled_code = jinja.get_rendered( compiled_node.loaded_at_query, SourceContext( compiled_node, self.config, manifest, RuntimeProvider(), None ).to_dict(), compiled_node, ) adapter_response, freshness = self.adapter.calculate_freshness_from_custom_sql( relation, compiled_code, macro_resolver=manifest, ) status = compiled_node.freshness.status(freshness["age"]) elif compiled_node.loaded_at_field is not None: adapter_response, freshness = self.adapter.calculate_freshness( relation, compiled_node.loaded_at_field, compiled_node.freshness.filter, macro_resolver=manifest, ) status = compiled_node.freshness.status(freshness["age"]) elif self.adapter.supports(Capability.TableLastModifiedMetadata): if compiled_node.freshness.filter is not None: fire_event( Note( msg=f"A filter cannot be applied to a metadata freshness check on source '{compiled_node.name}'." ), EventLevel.WARN, ) metadata_source = self.adapter.Relation.create_from(self.config, compiled_node) if metadata_source in self._metadata_freshness_cache: freshness = self._metadata_freshness_cache[metadata_source] else: adapter_response, freshness = self.adapter.calculate_freshness_from_metadata( relation, macro_resolver=manifest, ) status = compiled_node.freshness.status(freshness["age"]) else: raise DbtRuntimeError( f"Could not compute freshness for source {compiled_node.name}: no 'loaded_at_field' provided and {self.adapter.type()} adapter does not support metadata-based freshness checks." ) # adapter_response was not returned in previous versions, so this will be None # we cannot call to_dict() on NoneType if adapter_response: adapter_response = adapter_response.to_dict(omit_none=True) return SourceFreshnessResult( node=compiled_node, status=status, thread_id=threading.current_thread().name, timing=[], execution_time=0, message=None, adapter_response=adapter_response or {}, failures=None, **freshness, ) def compile(self, manifest: Manifest): if self.node.resource_type != NodeType.Source: # should be unreachable... raise DbtRuntimeError("freshness runner: got a non-Source") # we don't do anything interesting when we compile a source node return self.node class FreshnessSelector(ResourceTypeSelector): def node_is_match(self, node): if not super().node_is_match(node): return False if not isinstance(node, SourceDefinition): return False return node.has_freshness class FreshnessTask(RunTask): def __init__(self, args, config, manifest) -> None: super().__init__(args, config, manifest) if self.args.output: deprecations.warn( "custom-output-path-in-source-freshness-deprecation", path=str(self.args.output) ) self._metadata_freshness_cache: Dict[BaseRelation, FreshnessResult] = {} def result_path(self) -> str: if self.args.output: return os.path.realpath(self.args.output) else: return os.path.join(self.config.project_target_path, SOURCE_RESULT_FILE_NAME) def raise_on_first_error(self) -> bool: return False def get_node_selector(self): if self.manifest is None or self.graph is None: raise DbtInternalError("manifest and graph must be set to get perform node selection") return FreshnessSelector( graph=self.graph, manifest=self.manifest, previous_state=self.previous_state, resource_types=[NodeType.Source], ) def before_run(self, adapter: BaseAdapter, selected_uids: AbstractSet[str]) -> RunStatus: populate_metadata_freshness_cache_status = RunStatus.Success before_run_status = super().before_run(adapter, selected_uids) if before_run_status == RunStatus.Success and adapter.supports( Capability.TableLastModifiedMetadataBatch ): populate_metadata_freshness_cache_status = self.populate_metadata_freshness_cache( adapter, selected_uids ) if ( before_run_status == RunStatus.Success and populate_metadata_freshness_cache_status == RunStatus.Success ): return RunStatus.Success else: return RunStatus.Error def get_runner(self, node) -> BaseRunner: freshness_runner = super().get_runner(node) assert isinstance(freshness_runner, FreshnessRunner) freshness_runner.set_metadata_freshness_cache(self._metadata_freshness_cache) return freshness_runner def get_runner_type(self, _) -> Optional[Type[BaseRunner]]: return FreshnessRunner def get_result(self, results, elapsed_time, generated_at): return FreshnessResult.from_node_results( elapsed_time=elapsed_time, generated_at=generated_at, results=results ) def task_end_messages(self, results) -> None: for result in results: if result.status in ( FreshnessStatus.Error, FreshnessStatus.RuntimeErr, RunStatus.Error, ): print_run_result_error(result) fire_event(FreshnessCheckComplete()) def get_hooks_by_type(self, hook_type: RunHookType) -> List[HookNode]: hooks = super().get_hooks_by_type(hook_type) if self.args.source_freshness_run_project_hooks: return hooks else: if hooks: deprecations.warn("source-freshness-project-hooks") return [] def populate_metadata_freshness_cache( self, adapter, selected_uids: AbstractSet[str] ) -> RunStatus: if self.manifest is None: raise DbtInternalError("Manifest must be set to populate metadata freshness cache") batch_metadata_sources: List[BaseRelation] = [] for selected_source_uid in list(selected_uids): source = self.manifest.sources.get(selected_source_uid) if source and source.loaded_at_field is None: metadata_source = adapter.Relation.create_from(self.config, source) batch_metadata_sources.append(metadata_source) fire_event( Note( msg=f"Pulling freshness from warehouse metadata tables for {len(batch_metadata_sources)} sources" ), EventLevel.INFO, ) try: _, metadata_freshness_results = adapter.calculate_freshness_from_metadata_batch( batch_metadata_sources ) self._metadata_freshness_cache.update(metadata_freshness_results) return RunStatus.Success except Exception as e: # This error handling is intentionally very coarse. # If anything goes wrong during batch metadata calculation, we can safely # leave _metadata_freshness_cache unpopulated. # Downstream, this will be gracefully handled as a cache miss and non-batch # metadata-based freshness will still be performed on a source-by-source basis. fire_event( Note(msg=f"Metadata freshness could not be computed in batch: {e}"), EventLevel.WARN, ) return RunStatus.Error def get_freshness_metadata_cache(self) -> Dict[BaseRelation, FreshnessResult]: return self._metadata_freshness_cache ================================================ FILE: core/dbt/task/function.py ================================================ import threading from typing import Any, Dict from dbt.adapters.exceptions import MissingMaterializationError from dbt.artifacts.schemas.results import NodeStatus, RunStatus from dbt.artifacts.schemas.run import RunResult from dbt.clients.jinja import MacroGenerator from dbt.context.providers import generate_runtime_function_context from dbt.contracts.graph.manifest import Manifest from dbt.contracts.graph.nodes import FunctionNode from dbt.events.types import LogFunctionResult, LogStartLine from dbt.task import group_lookup from dbt.task.compile import CompileRunner from dbt_common.clients.jinja import MacroProtocol from dbt_common.events.base_types import EventLevel from dbt_common.events.functions import fire_event from dbt_common.exceptions import DbtValidationError class FunctionRunner(CompileRunner): def __init__(self, config, adapter, node, node_index: int, num_nodes: int) -> None: super().__init__(config, adapter, node, node_index, num_nodes) # doing this gives us type hints for the node :D assert isinstance(node, FunctionNode) self.node = node def describe_node(self) -> str: return f"function {self.get_node_representation()}" def before_execute(self) -> None: fire_event( LogStartLine( description=self.describe_node(), index=self.node_index, total=self.num_nodes, node_info=self.node.node_info, ) ) def _get_materialization_macro( self, compiled_node: FunctionNode, manifest: Manifest ) -> MacroProtocol: materialization_macro = manifest.find_materialization_macro_by_name( self.config.project_name, compiled_node.get_materialization(), self.adapter.type() ) if materialization_macro is None: raise MissingMaterializationError( materialization=compiled_node.get_materialization(), adapter_type=self.adapter.type(), ) return materialization_macro def _check_lang_supported( self, compiled_node: FunctionNode, materialization_macro: MacroProtocol ) -> None: # TODO: This function and its typing is a bit wonky, we should fix it # Specifically, a MacroProtocol doesn't have a supported_languags attribute, but a macro does. We're acting # like the materialization_macro might not have a supported_languages attribute, but we access it in an unguarded manner. # So are we guaranteed to always have a Macro here? (because a Macro always has a supported_languages attribute) # This logic is a copy of of the logic in the run.py file, so the same logical conundrum applies there. Also perhaps # we can refactor to having one definition, and maybe a logically consistent one... mat_has_supported_langs = hasattr(materialization_macro, "supported_languages") function_lang_supported = compiled_node.language in materialization_macro.supported_languages # type: ignore if mat_has_supported_langs and not function_lang_supported: str_langs = [str(lang) for lang in materialization_macro.supported_languages] # type: ignore raise DbtValidationError( f'Materialization "{materialization_macro.name}" only supports languages {str_langs}; ' f'got "{compiled_node.language}"' ) def build_result(self, compiled_node: FunctionNode, context: Dict[str, Any]) -> RunResult: loaded_result = context["load_result"]("main") return RunResult( node=compiled_node, status=RunStatus.Success, timing=[], thread_id=threading.current_thread().name, # This gets set later in `from_run_result` called by `BaseRunner.safe_run` execution_time=0.0, message=str(loaded_result.response), adapter_response=loaded_result.response.to_dict(omit_none=True), failures=loaded_result.get("failures"), batch_results=None, ) def execute(self, compiled_node: FunctionNode, manifest: Manifest) -> RunResult: materialization_macro = self._get_materialization_macro(compiled_node, manifest) self._check_lang_supported(compiled_node, materialization_macro) context = generate_runtime_function_context(compiled_node, self.config, manifest) MacroGenerator(materialization_macro, context=context)() return self.build_result(compiled_node, context) def after_execute(self, result: RunResult) -> None: self.print_result_line(result) # def compile() defined on CompileRunner def print_result_line(self, result: RunResult) -> None: node = result.node assert isinstance(node, FunctionNode) group = group_lookup.get(node.unique_id) level = EventLevel.ERROR if result.status == NodeStatus.Error else EventLevel.INFO fire_event( LogFunctionResult( description=self.describe_node(), status=result.status, index=self.node_index, total=self.num_nodes, execution_time=result.execution_time, node_info=self.node.node_info, group=group, ), level=level, ) ================================================ FILE: core/dbt/task/group_lookup.py ================================================ from typing import AbstractSet, Dict, Optional, Union from dbt.contracts.graph.manifest import Manifest from dbt.contracts.graph.nodes import Group _node_id_to_group_name_map: Dict[str, str] = {} _group_name_to_group_map: Dict[str, Group] = {} def init(manifest: Optional[Manifest], selected_ids: AbstractSet[str]) -> None: if not manifest: return if not manifest.groups: return if not hasattr(manifest, "group_map"): manifest.build_group_map() _every_group_name_to_group_map = {v.name: v for v in manifest.groups.values()} for group_name, node_ids in manifest.group_map.items(): for node_id in node_ids: # only add node to lookup if it's selected if node_id in selected_ids: _node_id_to_group_name_map[node_id] = group_name # only add group to lookup if it's not already there and if node is selected if group_name not in _group_name_to_group_map: _group_name_to_group_map[group_name] = _every_group_name_to_group_map[ group_name ] def get(node_id: str) -> Optional[Dict[str, Union[str, Dict[str, str]]]]: group_name = _node_id_to_group_name_map.get(node_id) if group_name is None: return None group = _group_name_to_group_map.get(group_name) if group is None: return None return group.to_logging_dict() ================================================ FILE: core/dbt/task/init.py ================================================ import copy import os import re import shutil from pathlib import Path from typing import Optional import click import yaml import dbt.config import dbt_common.clients.system from dbt.adapters.factory import get_include_paths, load_plugin from dbt.config.profile import read_profile from dbt.contracts.util import Identifier as ProjectName from dbt.events.types import ( ConfigFolderDirectory, InvalidProfileTemplateYAML, NoSampleProfileFound, ProfileWrittenWithProjectTemplateYAML, ProfileWrittenWithSample, ProfileWrittenWithTargetTemplateYAML, ProjectCreated, ProjectNameAlreadyExists, SettingUpProfile, StarterProjectPath, ) from dbt.flags import get_flags from dbt.task.base import BaseTask, move_to_nearest_project_dir from dbt.version import _get_adapter_plugin_names from dbt_common.events.functions import fire_event from dbt_common.events.types import Note from dbt_common.exceptions import DbtRuntimeError from dbt_common.ui import red DOCS_URL = "https://docs.getdbt.com/docs/configure-your-profile" SLACK_URL = "https://community.getdbt.com/" # This file is not needed for the starter project but exists for finding the resource path IGNORE_FILES = ["__init__.py", "__pycache__"] # https://click.palletsprojects.com/en/8.0.x/api/#types # click v7.0 has UNPROCESSED, STRING, INT, FLOAT, BOOL, and UUID available. click_type_mapping = { "string": click.STRING, "int": click.INT, "float": click.FLOAT, "bool": click.BOOL, None: None, } class InitTask(BaseTask): def copy_starter_repo(self, project_name: str) -> None: # Lazy import to avoid ModuleNotFoundError from dbt.include.starter_project import ( PACKAGE_PATH as starter_project_directory, ) fire_event(StarterProjectPath(dir=starter_project_directory)) shutil.copytree( starter_project_directory, project_name, ignore=shutil.ignore_patterns(*IGNORE_FILES) ) def create_profiles_dir(self, profiles_dir: str) -> bool: """Create the user's profiles directory if it doesn't already exist.""" profiles_path = Path(profiles_dir) if not profiles_path.exists(): fire_event(ConfigFolderDirectory(dir=str(profiles_dir))) dbt_common.clients.system.make_directory(profiles_dir) return True return False def create_profile_from_sample(self, adapter: str, profile_name: str): """Create a profile entry using the adapter's sample_profiles.yml Renames the profile in sample_profiles.yml to match that of the project.""" # Line below raises an exception if the specified adapter is not found load_plugin(adapter) adapter_path = get_include_paths(adapter)[0] sample_profiles_path = adapter_path / "sample_profiles.yml" if not sample_profiles_path.exists(): fire_event(NoSampleProfileFound(adapter=adapter)) else: with open(sample_profiles_path, "r") as f: sample_profile = f.read() sample_profile_name = list(yaml.safe_load(sample_profile).keys())[0] # Use a regex to replace the name of the sample_profile with # that of the project without losing any comments from the sample sample_profile = re.sub(f"^{sample_profile_name}:", f"{profile_name}:", sample_profile) profiles_filepath = Path(get_flags().PROFILES_DIR) / Path("profiles.yml") if profiles_filepath.exists(): with open(profiles_filepath, "a") as f: f.write("\n" + sample_profile) else: with open(profiles_filepath, "w") as f: f.write(sample_profile) fire_event( ProfileWrittenWithSample(name=profile_name, path=str(profiles_filepath)) ) def generate_target_from_input(self, profile_template: dict, target: dict = {}) -> dict: """Generate a target configuration from profile_template and user input.""" profile_template_local = copy.deepcopy(profile_template) for key, value in profile_template_local.items(): if key.startswith("_choose"): choice_type = key[8:].replace("_", " ") option_list = list(value.keys()) prompt_msg = ( "\n".join([f"[{n+1}] {v}" for n, v in enumerate(option_list)]) + f"\nDesired {choice_type} option (enter a number)" ) numeric_choice = click.prompt(prompt_msg, type=click.INT) choice = option_list[numeric_choice - 1] # Complete the chosen option's values in a recursive call target = self.generate_target_from_input( profile_template_local[key][choice], target ) else: if key.startswith("_fixed"): # _fixed prefixed keys are not presented to the user target[key[7:]] = value else: hide_input = value.get("hide_input", False) default = value.get("default", None) hint = value.get("hint", None) type = click_type_mapping[value.get("type", None)] text = key + (f" ({hint})" if hint else "") target[key] = click.prompt( text, default=default, hide_input=hide_input, type=type ) return target def get_profile_name_from_current_project(self) -> str: """Reads dbt_project.yml in the current directory to retrieve the profile name. """ with open("dbt_project.yml") as f: dbt_project = yaml.safe_load(f) return dbt_project["profile"] def write_profile(self, profile: dict, profile_name: str): """Given a profile, write it to the current project's profiles.yml. This will overwrite any profile with a matching name.""" # Create the profile directory if it doesn't exist profiles_filepath = Path(get_flags().PROFILES_DIR) / Path("profiles.yml") profiles = {profile_name: profile} if profiles_filepath.exists(): with open(profiles_filepath, "r") as f: profiles = yaml.safe_load(f) or {} profiles[profile_name] = profile # Write the profiles dictionary to a brand-new or pre-existing file with open(profiles_filepath, "w") as f: yaml.dump(profiles, f) def create_profile_from_profile_template(self, profile_template: dict, profile_name: str): """Create and write a profile using the supplied profile_template.""" initial_target = profile_template.get("fixed", {}) prompts = profile_template.get("prompts", {}) target = self.generate_target_from_input(prompts, initial_target) target_name = target.pop("target", "dev") profile = {"outputs": {target_name: target}, "target": target_name} self.write_profile(profile, profile_name) def create_profile_from_target(self, adapter: str, profile_name: str): """Create a profile without defaults using target's profile_template.yml if available, or sample_profiles.yml as a fallback.""" # Line below raises an exception if the specified adapter is not found load_plugin(adapter) adapter_path = get_include_paths(adapter)[0] profile_template_path = adapter_path / "profile_template.yml" if profile_template_path.exists(): with open(profile_template_path) as f: profile_template = yaml.safe_load(f) self.create_profile_from_profile_template(profile_template, profile_name) profiles_filepath = Path(get_flags().PROFILES_DIR) / Path("profiles.yml") fire_event( ProfileWrittenWithTargetTemplateYAML( name=profile_name, path=str(profiles_filepath) ) ) else: # For adapters without a profile_template.yml defined, fallback on # sample_profiles.yml self.create_profile_from_sample(adapter, profile_name) def check_if_profile_exists(self, profile_name: str) -> bool: """ Validate that the specified profile exists. Can't use the regular profile validation routine because it assumes the project file exists """ profiles_dir = get_flags().PROFILES_DIR raw_profiles = read_profile(profiles_dir) return profile_name in raw_profiles def check_if_can_write_profile(self, profile_name: Optional[str] = None) -> bool: """Using either a provided profile name or that specified in dbt_project.yml, check if the profile already exists in profiles.yml, and if so ask the user whether to proceed and overwrite it.""" profiles_file = Path(get_flags().PROFILES_DIR) / Path("profiles.yml") if not profiles_file.exists(): return True profile_name = profile_name or self.get_profile_name_from_current_project() with open(profiles_file, "r") as f: profiles = yaml.safe_load(f) or {} if profile_name in profiles.keys(): response = click.confirm( f"The profile {profile_name} already exists in " f"{profiles_file}. Continue and overwrite it?" ) return response else: return True def create_profile_using_project_profile_template(self, profile_name): """Create a profile using the project's profile_template.yml""" with open("profile_template.yml") as f: profile_template = yaml.safe_load(f) self.create_profile_from_profile_template(profile_template, profile_name) profiles_filepath = Path(get_flags().PROFILES_DIR) / Path("profiles.yml") fire_event( ProfileWrittenWithProjectTemplateYAML(name=profile_name, path=str(profiles_filepath)) ) def ask_for_adapter_choice(self) -> str: """Ask the user which adapter (database) they'd like to use.""" available_adapters = list(_get_adapter_plugin_names()) if not available_adapters: raise dbt.exceptions.NoAdaptersAvailableError() prompt_msg = ( "Which database would you like to use?\n" + "\n".join([f"[{n+1}] {v}" for n, v in enumerate(available_adapters)]) + "\n\n(Don't see the one you want? https://docs.getdbt.com/docs/available-adapters)" + "\n\nEnter a number" ) numeric_choice = click.prompt(prompt_msg, type=click.INT) return available_adapters[numeric_choice - 1] def setup_profile(self, profile_name: str) -> None: """Set up a new profile for a project""" fire_event(SettingUpProfile()) if not self.check_if_can_write_profile(profile_name=profile_name): return # If a profile_template.yml exists in the project root, that effectively # overrides the profile_template.yml for the given target. profile_template_path = Path("profile_template.yml") if profile_template_path.exists(): try: # This relies on a valid profile_template.yml from the user, # so use a try: except to fall back to the default on failure self.create_profile_using_project_profile_template(profile_name) return except Exception: fire_event(InvalidProfileTemplateYAML()) adapter = self.ask_for_adapter_choice() self.create_profile_from_target(adapter, profile_name=profile_name) def get_valid_project_name(self) -> str: """Returns a valid project name, either from CLI arg or user prompt.""" # Lazy import to avoid ModuleNotFoundError from dbt.include.global_project import PROJECT_NAME as GLOBAL_PROJECT_NAME name = self.args.project_name internal_package_names = {GLOBAL_PROJECT_NAME} available_adapters = list(_get_adapter_plugin_names()) for adapter_name in available_adapters: internal_package_names.update(f"dbt_{adapter_name}") while not ProjectName.is_valid(name) or name in internal_package_names: if name: click.echo(name + " is not a valid project name.") name = click.prompt("Enter a name for your project (letters, digits, underscore)") return name def _run_debug(self) -> Optional[bool]: if self.args.skip_debug: return None from dbt.task.debug import DebugTask fire_event(Note(msg="Running dbt debug to validate the project...")) try: debug_task = DebugTask(self.args) debug_task.project_dir = Path.cwd() debug_task.project_path = os.path.join(Path.cwd(), "dbt_project.yml") return debug_task.run() except Exception: fire_event(Note(msg="Debug validation encountered an error")) return False def create_new_project(self, project_name: str, profile_name: str): self.copy_starter_repo(project_name) os.chdir(project_name) with open("dbt_project.yml", "r") as f: content = f"{f.read()}".format(project_name=project_name, profile_name=profile_name) with open("dbt_project.yml", "w") as f: f.write(content) def run(self): """Entry point for the init task.""" profiles_dir = get_flags().PROFILES_DIR self.create_profiles_dir(profiles_dir) try: move_to_nearest_project_dir(self.args.project_dir) in_project = True except dbt_common.exceptions.DbtRuntimeError: in_project = False if in_project: # If --profile was specified, it means use an existing profile, which is not # applicable to this case if self.args.profile: raise DbtRuntimeError( msg="Can not init existing project with specified profile, edit dbt_project.yml instead" ) # When dbt init is run inside an existing project, # just setup the user's profile. if not self.args.skip_profile_setup: profile_name = self.get_profile_name_from_current_project() self.setup_profile(profile_name) self._run_debug() else: # When dbt init is run outside of an existing project, # create a new project and set up the user's profile. project_name = self.get_valid_project_name() project_path = Path(project_name) if project_path.exists(): fire_event(ProjectNameAlreadyExists(name=project_name)) return # If the user specified an existing profile to use, use it instead of generating a new one user_profile_name = self.args.profile if user_profile_name: if not self.check_if_profile_exists(user_profile_name): raise DbtRuntimeError( msg="Could not find profile named '{}'".format(user_profile_name) ) self.create_new_project(project_name, user_profile_name) debug_success = self._run_debug() else: profile_name = project_name # Create the profile after creating the project to avoid leaving a random profile # if the former fails. self.create_new_project(project_name, profile_name) # Ask for adapter only if skip_profile_setup flag is not provided if not self.args.skip_profile_setup: self.setup_profile(profile_name) debug_success = self._run_debug() else: debug_success = None fire_event( ProjectCreated( project_name=project_name, docs_url=DOCS_URL, slack_url=SLACK_URL, ) ) if debug_success is False: fire_event( Note( msg=red( "Your profile may need adjustments. You can find the logs of `dbt debug` above" ) ) ) ================================================ FILE: core/dbt/task/list.py ================================================ import json from typing import Iterator, List from dbt.cli.flags import Flags from dbt.config.runtime import RuntimeConfig from dbt.contracts.graph.manifest import Manifest from dbt.contracts.graph.nodes import ( Exposure, Metric, SavedQuery, SemanticModel, SourceDefinition, UnitTestDefinition, ) from dbt.events.types import NoNodesSelected from dbt.graph import ResourceTypeSelector from dbt.node_types import NodeType from dbt.task.base import resource_types_from_args from dbt.task.runnable import GraphRunnableTask from dbt.utils import JSONEncoder from dbt_common.events.contextvars import task_contextvars from dbt_common.events.functions import fire_event, warn_or_error from dbt_common.events.types import PrintEvent from dbt_common.exceptions import DbtInternalError, DbtRuntimeError class ListTask(GraphRunnableTask): DEFAULT_RESOURCE_VALUES = frozenset( ( NodeType.Model, NodeType.Snapshot, NodeType.Seed, NodeType.Test, NodeType.Source, NodeType.Exposure, NodeType.Metric, NodeType.SavedQuery, NodeType.SemanticModel, NodeType.Unit, NodeType.Function, ) ) ALL_RESOURCE_VALUES = DEFAULT_RESOURCE_VALUES | frozenset((NodeType.Analysis,)) ALLOWED_KEYS = frozenset( ( "alias", "name", "package_name", "depends_on", "tags", "config", "resource_type", "source_name", "original_file_path", "unique_id", ) ) def __init__(self, args: Flags, config: RuntimeConfig, manifest: Manifest) -> None: super().__init__(args, config, manifest) if self.args.models: if self.args.select: raise DbtRuntimeError('"models" and "select" are mutually exclusive arguments') if self.args.resource_types: raise DbtRuntimeError( '"models" and "resource_type" are mutually exclusive ' "arguments" ) def _iterate_selected_nodes(self): selector = self.get_node_selector() spec = self.get_selection_spec() unique_ids = sorted(selector.get_selected(spec)) if not unique_ids: warn_or_error(NoNodesSelected()) return if self.manifest is None: raise DbtInternalError("manifest is None in _iterate_selected_nodes") for unique_id in unique_ids: if unique_id in self.manifest.nodes: yield self.manifest.nodes[unique_id] elif unique_id in self.manifest.sources: yield self.manifest.sources[unique_id] elif unique_id in self.manifest.exposures: yield self.manifest.exposures[unique_id] elif unique_id in self.manifest.metrics: yield self.manifest.metrics[unique_id] elif unique_id in self.manifest.semantic_models: yield self.manifest.semantic_models[unique_id] elif unique_id in self.manifest.unit_tests: yield self.manifest.unit_tests[unique_id] elif unique_id in self.manifest.saved_queries: yield self.manifest.saved_queries[unique_id] elif unique_id in self.manifest.functions: yield self.manifest.functions[unique_id] else: raise DbtRuntimeError( f'Got an unexpected result from node selection: "{unique_id}"' f"Listing this node type is not yet supported!" ) def generate_selectors(self): for node in self._iterate_selected_nodes(): if node.resource_type == NodeType.Source: assert isinstance(node, SourceDefinition) # sources are searched for by pkg.source_name.table_name source_selector = ".".join([node.package_name, node.source_name, node.name]) yield f"source:{source_selector}" elif node.resource_type == NodeType.Exposure: assert isinstance(node, Exposure) # exposures are searched for by pkg.exposure_name exposure_selector = ".".join([node.package_name, node.name]) yield f"exposure:{exposure_selector}" elif node.resource_type == NodeType.Metric: assert isinstance(node, Metric) # metrics are searched for by pkg.metric_name metric_selector = ".".join([node.package_name, node.name]) yield f"metric:{metric_selector}" elif node.resource_type == NodeType.SavedQuery: assert isinstance(node, SavedQuery) saved_query_selector = ".".join([node.package_name, node.name]) yield f"saved_query:{saved_query_selector}" elif node.resource_type == NodeType.SemanticModel: assert isinstance(node, SemanticModel) semantic_model_selector = ".".join([node.package_name, node.name]) yield f"semantic_model:{semantic_model_selector}" elif node.resource_type == NodeType.Unit: assert isinstance(node, UnitTestDefinition) unit_test_selector = ".".join([node.package_name, node.versioned_name]) yield f"unit_test:{unit_test_selector}" else: # everything else is from `fqn` yield ".".join(node.fqn) def generate_names(self): for node in self._iterate_selected_nodes(): yield node.search_name def _get_nested_value(self, data, key_path): """Get nested value using dot notation (e.g., 'config.materialized')""" keys = key_path.split(".") current = data for key in keys: if isinstance(current, dict) and key in current: current = current[key] else: return None return current def generate_json(self): for node in self._iterate_selected_nodes(): node_dict = node.to_dict(omit_none=False) if self.args.output_keys: # Handle both nested and regular keys result = {} for key in self.args.output_keys: if "." in key: # Handle nested key (e.g., 'config.materialized') value = self._get_nested_value(node_dict, key) if value is not None: result[key] = value else: # Handle regular key if key in node_dict: result[key] = node_dict[key] else: # Use default allowed keys result = {k: v for k, v in node_dict.items() if k in self.ALLOWED_KEYS} yield json.dumps(result, cls=JSONEncoder) def generate_paths(self) -> Iterator[str]: for node in self._iterate_selected_nodes(): yield node.original_file_path def run(self): # We set up a context manager here with "task_contextvars" because we # we need the project_root in compile_manifest. with task_contextvars(project_root=self.config.project_root): self.compile_manifest() output = self.args.output if output == "selector": generator = self.generate_selectors elif output == "name": generator = self.generate_names elif output == "json": generator = self.generate_json elif output == "path": generator = self.generate_paths else: raise DbtInternalError("Invalid output {}".format(output)) return self.output_results(generator()) def output_results(self, results): """Log, or output a plain, newline-delimited, and ready-to-pipe list of nodes found.""" for result in results: self.node_results.append(result) # No formatting, still get to stdout when --quiet is used fire_event(PrintEvent(msg=result)) return self.node_results @property def resource_types(self) -> List[NodeType]: if self.args.models: return [NodeType.Model] resource_types = resource_types_from_args( self.args, set(self.ALL_RESOURCE_VALUES), set(self.DEFAULT_RESOURCE_VALUES) ) return list(resource_types) @property def selection_arg(self): # for backwards compatibility, list accepts both --models and --select, # with slightly different behavior: --models implies --resource-type model if self.args.models: return self.args.models else: return self.args.select def get_node_selector(self) -> ResourceTypeSelector: if self.manifest is None or self.graph is None: raise DbtInternalError("manifest and graph must be set to get perform node selection") return ResourceTypeSelector( graph=self.graph, manifest=self.manifest, previous_state=self.previous_state, resource_types=self.resource_types, include_empty_nodes=True, selectors=self.config.selectors, ) def interpret_results(self, results): # list command should always return 0 as exit code return True ================================================ FILE: core/dbt/task/printer.py ================================================ from typing import Dict, Optional, Union from dbt.artifacts.schemas.results import NodeStatus from dbt.contracts.graph.nodes import Exposure from dbt.events.types import ( CheckNodeTestFailure, EndOfRunSummary, RunResultError, RunResultErrorNoMessage, RunResultFailure, RunResultWarning, RunResultWarningMessage, SQLCompiledPath, StatsLine, ) from dbt.node_types import NodeType from dbt.task import group_lookup from dbt_common.events.base_types import EventLevel from dbt_common.events.format import pluralize from dbt_common.events.functions import fire_event from dbt_common.events.types import Formatting def get_counts(flat_nodes) -> str: counts: Dict[str, int] = {} for node in flat_nodes: t = node.resource_type if node.resource_type == NodeType.Model: t = "{} {}".format(node.get_materialization(), t) elif node.resource_type == NodeType.Operation: t = "project hook" counts[t] = counts.get(t, 0) + 1 sorted_items = sorted(counts.items(), key=lambda x: x[0]) stat_line = ", ".join([pluralize(v, k).replace("_", " ") for k, v in sorted_items]) return stat_line def interpret_run_result(result) -> str: if result.status in (NodeStatus.Error, NodeStatus.Fail, NodeStatus.PartialSuccess): return "error" elif result.status == NodeStatus.Skipped: return "skip" elif result.status == NodeStatus.Warn: return "warn" elif result.status in (NodeStatus.Pass, NodeStatus.Success): return "pass" elif result.status == NodeStatus.NoOp: return "noop" else: raise RuntimeError(f"unhandled result {result}") def print_run_status_line(results) -> None: stats = { "error": 0, "skip": 0, "pass": 0, "warn": 0, "noop": 0, "total": 0, } for r in results: result_type = interpret_run_result(r) stats[result_type] += 1 stats["total"] += 1 fire_event(Formatting("")) fire_event(StatsLine(stats=stats)) def print_run_result_error( result, newline: bool = True, is_warning: bool = False, group: Optional[Dict[str, Union[str, Dict[str, str]]]] = None, ) -> None: # set node_info for logging events node_info = None if hasattr(result, "node") and result.node: node_info = result.node.node_info if result.status in (NodeStatus.Fail, NodeStatus.Error) or ( is_warning and result.status == NodeStatus.Warn ): if newline: fire_event(Formatting("")) if is_warning: fire_event( RunResultWarning( resource_type=result.node.resource_type, node_name=result.node.name, path=result.node.original_file_path, node_info=node_info, group=group, ) ) else: fire_event( RunResultFailure( resource_type=result.node.resource_type, node_name=result.node.name, path=result.node.original_file_path, node_info=node_info, group=group, ) ) if result.message: if is_warning: fire_event(RunResultWarningMessage(msg=result.message, node_info=node_info)) else: fire_event(RunResultError(msg=result.message, node_info=node_info, group=group)) else: fire_event(RunResultErrorNoMessage(status=result.status, node_info=node_info)) if getattr(result.node, "compiled_path", None): fire_event(Formatting("")) fire_event(SQLCompiledPath(path=result.node.compiled_path, node_info=node_info)) if getattr(result.node, "should_store_failures", None): fire_event(Formatting("")) fire_event( CheckNodeTestFailure(relation_name=result.node.relation_name, node_info=node_info) ) elif result.status == NodeStatus.Skipped and result.message is not None: if newline: fire_event(Formatting(""), level=EventLevel.DEBUG) fire_event(RunResultError(msg=result.message), level=EventLevel.DEBUG) elif result.message is not None: if newline: fire_event(Formatting("")) fire_event(RunResultError(msg=result.message, node_info=node_info, group=group)) def print_run_end_messages(results, keyboard_interrupt: bool = False) -> None: errors, warnings, partial_successes = [], [], [] for r in results: if r.status in (NodeStatus.RuntimeErr, NodeStatus.Error, NodeStatus.Fail): errors.append(r) elif r.status == NodeStatus.Skipped and r.message: if isinstance(r.node, Exposure): # Don't include exposure skips in errors list continue else: # This means we skipped a node because of an issue upstream, so include it as an error errors.append(r) elif r.status == NodeStatus.Warn: warnings.append(r) elif r.status == NodeStatus.PartialSuccess: partial_successes.append(r) fire_event(Formatting("")) fire_event( EndOfRunSummary( num_errors=len(errors), num_warnings=len(warnings), num_partial_success=len(partial_successes), keyboard_interrupt=keyboard_interrupt, ) ) for error in errors: group = group_lookup.get(error.node.unique_id) if hasattr(error, "node") else None print_run_result_error(error, is_warning=False, group=group) for warning in warnings: group = group_lookup.get(warning.node.unique_id) if hasattr(warning, "node") else None print_run_result_error(warning, is_warning=True, group=group) print_run_status_line(results) ================================================ FILE: core/dbt/task/retry.py ================================================ from pathlib import Path from click import get_current_context from click.core import ParameterSource from dbt.artifacts.schemas.results import NodeStatus from dbt.cli.flags import Flags from dbt.cli.types import Command as CliCommand from dbt.config import RuntimeConfig from dbt.constants import RUN_RESULTS_FILE_NAME from dbt.contracts.state import load_result_state from dbt.flags import get_flags, set_flags from dbt.graph import GraphQueue from dbt.parser.manifest import parse_manifest from dbt.task.base import ConfiguredTask from dbt.task.build import BuildTask from dbt.task.clone import CloneTask from dbt.task.compile import CompileTask from dbt.task.docs.generate import GenerateTask from dbt.task.run import RunTask from dbt.task.run_operation import RunOperationTask from dbt.task.seed import SeedTask from dbt.task.snapshot import SnapshotTask from dbt.task.test import TestTask from dbt_common.exceptions import DbtRuntimeError RETRYABLE_STATUSES = { NodeStatus.Error, NodeStatus.Fail, NodeStatus.Skipped, NodeStatus.RuntimeErr, NodeStatus.PartialSuccess, } IGNORE_PARENT_FLAGS = { "log_path", "output_path", "profiles_dir", "profiles_dir_exists_false", "project_dir", "defer_state", "deprecated_state", "target_path", "warn_error", } ALLOW_CLI_OVERRIDE_FLAGS = {"vars", "threads"} TASK_DICT = { "build": BuildTask, "compile": CompileTask, "clone": CloneTask, "generate": GenerateTask, "seed": SeedTask, "snapshot": SnapshotTask, "test": TestTask, "run": RunTask, "run-operation": RunOperationTask, } CMD_DICT = { "build": CliCommand.BUILD, "compile": CliCommand.COMPILE, "clone": CliCommand.CLONE, "generate": CliCommand.DOCS_GENERATE, "seed": CliCommand.SEED, "snapshot": CliCommand.SNAPSHOT, "test": CliCommand.TEST, "run": CliCommand.RUN, "run-operation": CliCommand.RUN_OPERATION, } class RetryTask(ConfiguredTask): def __init__(self, args: Flags, config: RuntimeConfig) -> None: # load previous run results state_path = args.state or config.target_path self.previous_results = load_result_state( Path(config.project_root) / Path(state_path) / RUN_RESULTS_FILE_NAME ) if not self.previous_results: raise DbtRuntimeError( f"Could not find previous run in '{state_path}' target directory" ) self.previous_args = self.previous_results.args self.previous_command_name = self.previous_args.get("which") # Reslove flags and config if args.warn_error: RETRYABLE_STATUSES.add(NodeStatus.Warn) cli_command = CMD_DICT.get(self.previous_command_name) # type: ignore # Remove these args when their default values are present, otherwise they'll raise an exception args_to_remove = { "show": lambda x: True, "resource_types": lambda x: x == [], "warn_error_options": lambda x: x == {"warn": [], "error": [], "silence": []}, } for k, v in args_to_remove.items(): if k in self.previous_args and v(self.previous_args[k]): del self.previous_args[k] previous_args = { k: v for k, v in self.previous_args.items() if k not in IGNORE_PARENT_FLAGS } click_context = get_current_context() current_args = { k: v for k, v in args.__dict__.items() if k in IGNORE_PARENT_FLAGS or ( click_context.get_parameter_source(k) == ParameterSource.COMMANDLINE and k in ALLOW_CLI_OVERRIDE_FLAGS ) } combined_args = {**previous_args, **current_args} retry_flags = Flags.from_dict(cli_command, combined_args) # type: ignore set_flags(retry_flags) retry_config = RuntimeConfig.from_args(args=retry_flags) # Parse manifest using resolved config/flags manifest = parse_manifest(retry_config, False, True, retry_flags.write_json, []) # type: ignore super().__init__(args, retry_config, manifest) self.task_class = TASK_DICT.get(self.previous_command_name) # type: ignore def run(self): unique_ids = { result.unique_id for result in self.previous_results.results if result.status in RETRYABLE_STATUSES # Avoid retrying operation nodes unless we are retrying the run-operation command and not ( self.previous_command_name != "run-operation" and result.unique_id.startswith("operation.") ) } # We need this so that re-running of a microbatch model will only rerun # batches that previously failed. Note _explicitly_ do no pass the # batch info if there were _no_ successful batches previously. This is # because passing the batch info _forces_ the microbatch process into # _incremental_ model, and it may be that we need to be in full refresh # mode which is only handled if previous_batch_results _isn't_ passed for a node batch_map = { result.unique_id: result.batch_results for result in self.previous_results.results if result.batch_results is not None and len(result.batch_results.successful) != 0 and len(result.batch_results.failed) > 0 and not ( self.previous_command_name != "run-operation" and result.unique_id.startswith("operation.") ) } # Tasks without get_graph_queue (e.g. run-operation) and no failed nodes to retry. if not unique_ids and not hasattr(self.task_class, "get_graph_queue"): # Return early with the previous results as the past invocation was successful return self.previous_results class TaskWrapper(self.task_class): def get_graph_queue(self): new_graph = self.graph.get_subset_graph(unique_ids) return GraphQueue( new_graph.graph, self.manifest, unique_ids, ) task = TaskWrapper( get_flags(), self.config, self.manifest, ) if self.task_class == RunTask or self.task_class == BuildTask: task.batch_map = batch_map task.original_invocation_started_at = ( self.previous_results.metadata.invocation_started_at or self.previous_results.metadata.generated_at ) return_value = task.run() return return_value def interpret_results(self, *args, **kwargs): return self.task_class.interpret_results(*args, **kwargs) ================================================ FILE: core/dbt/task/run.py ================================================ from __future__ import annotations import functools import threading import time from copy import deepcopy from dataclasses import asdict from datetime import datetime, timezone from typing import ( AbstractSet, Any, Callable, Dict, Iterable, List, Optional, Set, Tuple, Type, ) from dbt import tracking, utils from dbt.adapters.base import BaseAdapter, BaseRelation from dbt.adapters.capability import Capability from dbt.adapters.events.types import FinishedRunningStats from dbt.adapters.exceptions import MissingMaterializationError from dbt.artifacts.resources import Hook from dbt.artifacts.schemas.batch_results import BatchResults, BatchType from dbt.artifacts.schemas.results import ( NodeStatus, RunningStatus, RunStatus, TimingInfo, collect_timing_info, ) from dbt.artifacts.schemas.run import RunResult from dbt.cli.flags import Flags from dbt.clients.jinja import MacroGenerator from dbt.config import RuntimeConfig from dbt.context.providers import generate_runtime_model_context from dbt.contracts.graph.manifest import Manifest from dbt.contracts.graph.nodes import BatchContext, HookNode, ModelNode, ResultNode from dbt.events.types import ( GenericExceptionOnRun, LogBatchResult, LogHookEndLine, LogHookStartLine, LogModelResult, LogStartBatch, LogStartLine, MicrobatchExecutionDebug, ) from dbt.exceptions import CompilationError, DbtInternalError, DbtRuntimeError from dbt.graph import ResourceTypeSelector from dbt.graph.thread_pool import DbtThreadPool from dbt.hooks import get_hook_dict from dbt.materializations.incremental.microbatch import MicrobatchBuilder from dbt.node_types import NodeType, RunHookType from dbt.task import group_lookup from dbt.task.base import BaseRunner from dbt.task.compile import CompileRunner, CompileTask from dbt.task.printer import get_counts, print_run_end_messages from dbt.utils.artifact_upload import add_artifact_produced from dbt_common.clients.jinja import MacroProtocol from dbt_common.dataclass_schema import dbtClassMixin from dbt_common.events.base_types import EventLevel from dbt_common.events.contextvars import log_contextvars from dbt_common.events.functions import fire_event, get_invocation_id from dbt_common.events.types import Formatting from dbt_common.exceptions import DbtValidationError from dbt_common.invocation import get_invocation_started_at @functools.total_ordering class BiggestName(str): def __lt__(self, other): return True def __eq__(self, other): return isinstance(other, self.__class__) def _hook_list() -> List[HookNode]: return [] def get_hooks_by_tags( nodes: Iterable[ResultNode], match_tags: Set[str], ) -> List[HookNode]: matched_nodes = [] for node in nodes: if not isinstance(node, HookNode): continue node_tags = node.tags if len(set(node_tags) & match_tags): matched_nodes.append(node) return matched_nodes def get_hook(source, index): hook_dict = get_hook_dict(source) hook_dict.setdefault("index", index) Hook.validate(hook_dict) return Hook.from_dict(hook_dict) def get_execution_status(sql: str, adapter: BaseAdapter) -> Tuple[RunStatus, str]: if not sql.strip(): return RunStatus.Success, "OK" try: response, _ = adapter.execute(sql, auto_begin=False, fetch=False) status = RunStatus.Success message = response._message except (KeyboardInterrupt, SystemExit): raise except DbtRuntimeError as exc: status = RunStatus.Error message = exc.msg except Exception as exc: status = RunStatus.Error message = str(exc) return (status, message) def _get_adapter_info(adapter, run_model_result) -> Dict[str, Any]: """Each adapter returns a dataclass with a flexible dictionary for adapter-specific fields. Only the non-'model_adapter_details' fields are guaranteed cross adapter.""" return asdict(adapter.get_adapter_run_info(run_model_result.node.config)) if adapter else {} def track_model_run(index, num_nodes, run_model_result, adapter=None): if tracking.active_user is None: raise DbtInternalError("cannot track model run with no active user") invocation_id = get_invocation_id() node = run_model_result.node has_group = True if hasattr(node, "group") and node.group else False if node.resource_type == NodeType.Model: access = node.access.value if node.access is not None else None contract_enforced = node.contract.enforced versioned = True if node.version else False incremental_strategy = node.config.incremental_strategy else: access = None contract_enforced = False versioned = False incremental_strategy = None tracking.track_model_run( { "invocation_id": invocation_id, "index": index, "total": num_nodes, "execution_time": run_model_result.execution_time, "run_status": str(run_model_result.status).upper(), "run_skipped": run_model_result.status == NodeStatus.Skipped, "run_error": run_model_result.status == NodeStatus.Error, "model_materialization": node.get_materialization(), "model_incremental_strategy": incremental_strategy, "model_id": utils.get_hash(node), "hashed_contents": utils.get_hashed_contents(node), "timing": [t.to_dict(omit_none=True) for t in run_model_result.timing], "language": str(node.language), "has_group": has_group, "contract_enforced": contract_enforced, "access": access, "versioned": versioned, "adapter_info": _get_adapter_info(adapter, run_model_result), } ) # make sure that we got an ok result back from a materialization def _validate_materialization_relations_dict(inp: Dict[Any, Any], model) -> List[BaseRelation]: try: relations_value = inp["relations"] except KeyError: msg = ( 'Invalid return value from materialization, "relations" ' "not found, got keys: {}".format(list(inp)) ) raise CompilationError(msg, node=model) from None if not isinstance(relations_value, list): msg = ( 'Invalid return value from materialization, "relations" ' "not a list, got: {}".format(relations_value) ) raise CompilationError(msg, node=model) from None relations: List[BaseRelation] = [] for relation in relations_value: if not isinstance(relation, BaseRelation): msg = ( "Invalid return value from materialization, " '"relations" contains non-Relation: {}'.format(relation) ) raise CompilationError(msg, node=model) assert isinstance(relation, BaseRelation) relations.append(relation) return relations class ModelRunner(CompileRunner): def describe_node(self) -> str: # TODO CL 'language' will be moved to node level when we change representation return f"{self.node.language} {self.node.get_materialization()} model {self.get_node_representation()}" def print_start_line(self): fire_event( LogStartLine( description=self.describe_node(), index=self.node_index, total=self.num_nodes, node_info=self.node.node_info, ) ) def print_result_line(self, result): description = self.describe_node() group = group_lookup.get(self.node.unique_id) if result.status == NodeStatus.Error: status = result.status level = EventLevel.ERROR else: status = result.message level = EventLevel.INFO fire_event( LogModelResult( description=description, status=status, index=self.node_index, total=self.num_nodes, execution_time=result.execution_time, node_info=self.node.node_info, group=group, ), level=level, ) def before_execute(self) -> None: self.print_start_line() def after_execute(self, result) -> None: track_model_run(self.node_index, self.num_nodes, result, adapter=self.adapter) self.print_result_line(result) def _build_run_model_result(self, model, context, elapsed_time: float = 0.0): result = context["load_result"]("main") if not result: raise DbtRuntimeError("main is not being called during running model") adapter_response = {} if isinstance(result.response, dbtClassMixin): adapter_response = result.response.to_dict(omit_none=True) return RunResult( node=model, status=RunStatus.Success, timing=[], thread_id=threading.current_thread().name, execution_time=elapsed_time, message=str(result.response), adapter_response=adapter_response, failures=result.get("failures"), batch_results=None, ) def _materialization_relations(self, result: Any, model) -> List[BaseRelation]: if isinstance(result, str): msg = ( 'The materialization ("{}") did not explicitly return a ' "list of relations to add to the cache.".format(str(model.get_materialization())) ) raise CompilationError(msg, node=model) if isinstance(result, dict): return _validate_materialization_relations_dict(result, model) msg = ( "Invalid return value from materialization, expected a dict " 'with key "relations", got: {}'.format(str(result)) ) raise CompilationError(msg, node=model) def _execute_model( self, hook_ctx: Any, context_config: Any, model: ModelNode, context: Dict[str, Any], materialization_macro: MacroProtocol, ) -> RunResult: try: result = MacroGenerator( materialization_macro, context, stack=context["context_macro_stack"] )() finally: self.adapter.post_model_hook(context_config, hook_ctx) for relation in self._materialization_relations(result, model): self.adapter.cache_added(relation.incorporate(dbt_created=True)) return self._build_run_model_result(model, context) def execute(self, model, manifest): context = generate_runtime_model_context(model, self.config, manifest) materialization_macro = manifest.find_materialization_macro_by_name( self.config.project_name, model.get_materialization(), self.adapter.type() ) if materialization_macro is None: raise MissingMaterializationError( materialization=model.get_materialization(), adapter_type=self.adapter.type() ) if "config" not in context: raise DbtInternalError( "Invalid materialization context generated, missing config: {}".format(context) ) context_config = context["config"] mat_has_supported_langs = hasattr(materialization_macro, "supported_languages") model_lang_supported = model.language in materialization_macro.supported_languages if mat_has_supported_langs and not model_lang_supported: str_langs = [str(lang) for lang in materialization_macro.supported_languages] raise DbtValidationError( f'Materialization "{materialization_macro.name}" only supports languages {str_langs}; ' f'got "{model.language}"' ) hook_ctx = self.adapter.pre_model_hook(context_config) return self._execute_model(hook_ctx, context_config, model, context, materialization_macro) class MicrobatchBatchRunner(ModelRunner): """Handles the running of individual batches""" def __init__( self, config, adapter, node, node_index: int, num_nodes: int, batch_idx: int, batches: Dict[int, BatchType], relation_exists: bool, incremental_batch: bool, ): super().__init__(config, adapter, node, node_index, num_nodes) self.batch_idx = batch_idx self.batches = batches self.relation_exists = relation_exists self.incremental_batch = incremental_batch def describe_batch(self) -> str: batch_start = self.batches[self.batch_idx][0] formatted_batch_start = MicrobatchBuilder.format_batch_start( batch_start, self.node.config.batch_size ) return f"batch {formatted_batch_start} of {self.get_node_representation()}" def print_result_line(self, result: RunResult): if result.status == NodeStatus.Error: status = result.status level = EventLevel.ERROR elif result.status == NodeStatus.Skipped: status = result.status level = EventLevel.INFO else: status = result.message level = EventLevel.INFO fire_event( LogBatchResult( description=self.describe_batch(), status=status, batch_index=self.batch_idx + 1, total_batches=len(self.batches), execution_time=result.execution_time, node_info=self.node.node_info, group=group_lookup.get(self.node.unique_id), ), level=level, ) def print_start_line(self) -> None: fire_event( LogStartBatch( description=self.describe_batch(), batch_index=self.batch_idx + 1, total_batches=len(self.batches), node_info=self.node.node_info, ) ) def should_run_in_parallel(self) -> bool: if not self.adapter.supports(Capability.MicrobatchConcurrency): run_in_parallel = False elif not self.relation_exists: # If the relation doesn't exist, we can't run in parallel run_in_parallel = False elif self.node.config.concurrent_batches is not None: # If the relation exists and the `concurrent_batches` config isn't None, use the config value run_in_parallel = self.node.config.concurrent_batches else: # If the relation exists, the `concurrent_batches` config is None, check if the model self references `this`. # If the model self references `this` then we assume the model batches _can't_ be run in parallel run_in_parallel = not self.node.has_this return run_in_parallel def on_skip(self): result = RunResult( node=self.node, status=RunStatus.Skipped, timing=[], thread_id=threading.current_thread().name, execution_time=0.0, message="SKIPPED", adapter_response={}, failures=1, batch_results=BatchResults(failed=[self.batches[self.batch_idx]]), ) self.print_result_line(result=result) return result def error_result(self, node, message, start_time, timing_info): """Necessary to return a result with a batch result Called by `BaseRunner.safe_run` when an error occurs """ return self._build_run_result( node=node, start_time=start_time, status=RunStatus.Error, timing_info=timing_info, message=message, batch_results=BatchResults(failed=[self.batches[self.batch_idx]]), ) def compile(self, manifest: Manifest): batch = self.batches[self.batch_idx] # LEGACY: Set start/end in context prior to re-compiling (Will be removed for 1.10+) # TODO: REMOVE before 1.10 GA self.node.config["__dbt_internal_microbatch_event_time_start"] = batch[0] self.node.config["__dbt_internal_microbatch_event_time_end"] = batch[1] # Create batch context on model node prior to re-compiling self.node.batch = BatchContext( id=MicrobatchBuilder.batch_id(batch[0], self.node.config.batch_size), event_time_start=batch[0], event_time_end=batch[1], ) # Recompile node to re-resolve refs with event time filters rendered, update context self.compiler.compile_node( self.node, manifest, {}, split_suffix=MicrobatchBuilder.format_batch_start( batch[0], self.node.config.batch_size ), ) return self.node def _build_succesful_run_batch_result( self, model: ModelNode, context: Dict[str, Any], batch: BatchType, elapsed_time: float = 0.0, ) -> RunResult: run_result = self._build_run_model_result(model, context, elapsed_time) run_result.batch_results = BatchResults(successful=[batch]) return run_result def _build_failed_run_batch_result( self, model: ModelNode, batch: BatchType, elapsed_time: float = 0.0, ) -> RunResult: return RunResult( node=model, status=RunStatus.Error, timing=[], thread_id=threading.current_thread().name, execution_time=elapsed_time, message="ERROR", adapter_response={}, failures=1, batch_results=BatchResults(failed=[batch]), ) def _execute_microbatch_materialization( self, model: ModelNode, context: Dict[str, Any], materialization_macro: MacroProtocol, ) -> RunResult: batch = self.batches[self.batch_idx] # call materialization_macro to get a batch-level run result start_time = time.perf_counter() try: # Update jinja context with batch context members jinja_context = MicrobatchBuilder.build_jinja_context_for_batch( model=model, incremental_batch=self.incremental_batch, ) context.update(jinja_context) # Materialize batch and cache any materialized relations result = MacroGenerator( materialization_macro, context, stack=context["context_macro_stack"] )() for relation in self._materialization_relations(result, model): self.adapter.cache_added(relation.incorporate(dbt_created=True)) # Build result of executed batch batch_run_result = self._build_succesful_run_batch_result( model, context, batch, time.perf_counter() - start_time ) batch_result = batch_run_result # At least one batch has been inserted successfully! # Can proceed incrementally + in parallel self.relation_exists = True except (KeyboardInterrupt, SystemExit): # reraise it for GraphRunnableTask.execute_nodes to handle raise except Exception as e: fire_event( GenericExceptionOnRun( unique_id=self.node.unique_id, exc=f"Exception on worker thread. {str(e)}", node_info=self.node.node_info, ) ) batch_run_result = self._build_failed_run_batch_result( model, batch, time.perf_counter() - start_time ) batch_result = batch_run_result return batch_result def _execute_model( self, hook_ctx: Any, context_config: Any, model: ModelNode, context: Dict[str, Any], materialization_macro: MacroProtocol, ) -> RunResult: try: batch_result = self._execute_microbatch_materialization( model, context, materialization_macro ) finally: self.adapter.post_model_hook(context_config, hook_ctx) return batch_result class MicrobatchModelRunner(ModelRunner): """Handles the orchestration of batches to run for a given microbatch model""" def __init__(self, config, adapter, node, node_index: int, num_nodes: int): super().__init__(config, adapter, node, node_index, num_nodes) # The parent task is necessary because we need access to the `_submit_batch` and `submit` methods self._parent_task: Optional[RunTask] = None # The pool is necessary because we need to batches to be executed within the same thread pool self._pool: Optional[DbtThreadPool] = None def set_parent_task(self, parent_task: RunTask) -> None: self._parent_task = parent_task def set_pool(self, pool: DbtThreadPool) -> None: self._pool = pool @property def parent_task(self) -> RunTask: if self._parent_task is None: raise DbtInternalError( msg="Tried to access `parent_task` of `MicrobatchModelRunner` before it was set" ) return self._parent_task @property def pool(self) -> DbtThreadPool: if self._pool is None: raise DbtInternalError( msg="Tried to access `pool` of `MicrobatchModelRunner` before it was set" ) return self._pool def _has_relation(self, model: ModelNode) -> bool: """Check whether the relation for the model exists in the data warehouse""" relation_info = self.adapter.Relation.create_from(self.config, model) relation = self.adapter.get_relation( relation_info.database, relation_info.schema, relation_info.name ) return relation is not None def _is_incremental(self, model) -> bool: """Check whether the model should be run `incrementally` or as `full refresh`""" # TODO: Remove this whole function. This should be a temporary method. We're working with adapters on # a strategy to ensure we can access the `is_incremental` logic without drift relation_info = self.adapter.Relation.create_from(self.config, model) relation = self.adapter.get_relation( relation_info.database, relation_info.schema, relation_info.name ) if ( relation is not None and relation.type == "table" and model.config.materialized == "incremental" ): if model.config.full_refresh is not None: return not model.config.full_refresh else: return not getattr(self.config.args, "FULL_REFRESH", False) else: return False def _initial_run_microbatch_model_result(self, model: ModelNode) -> RunResult: return RunResult( node=model, status=RunStatus.Success, timing=[], thread_id=threading.current_thread().name, # The execution_time here doesn't get propagated to logs because # `safe_run_hooks` handles the elapsed time at the node level execution_time=0, message="", adapter_response={}, failures=0, batch_results=BatchResults(), ) def describe_node(self) -> str: return f"{self.node.language} microbatch model {self.get_node_representation()}" def merge_batch_results(self, result: RunResult, batch_results: List[RunResult]): """merge batch_results into result""" if result.batch_results is None: result.batch_results = BatchResults() for batch_result in batch_results: if batch_result.batch_results is not None: result.batch_results += batch_result.batch_results result.execution_time += batch_result.execution_time num_successes = len(result.batch_results.successful) num_failures = len(result.batch_results.failed) if num_failures == 0: status = RunStatus.Success msg = "SUCCESS" elif num_successes == 0: status = RunStatus.Error msg = "ERROR" else: status = RunStatus.PartialSuccess msg = f"PARTIAL SUCCESS ({num_successes}/{num_successes + num_failures})" result.status = status result.message = msg result.batch_results.successful = sorted(result.batch_results.successful) result.batch_results.failed = sorted(result.batch_results.failed) # # If retrying, propagate previously successful batches into final result, even thoguh they were not run in this invocation if self.node.previous_batch_results is not None: result.batch_results.successful += self.node.previous_batch_results.successful def _update_result_with_unfinished_batches( self, result: RunResult, batches: Dict[int, BatchType] ) -> None: """This method is really only to be used when the execution of a microbatch model is halted before all batches have had a chance to run""" batches_finished: Set[BatchType] = set() if result.batch_results: # build list of finished batches batches_finished = batches_finished.union(set(result.batch_results.successful)) batches_finished = batches_finished.union(set(result.batch_results.failed)) else: # instantiate `batch_results` if it was `None` result.batch_results = BatchResults() # skipped batches are any batch that was expected but didn't finish batches_expected = {batch for _, batch in batches.items()} skipped_batches = batches_expected.difference(batches_finished) result.batch_results.failed.extend(list(skipped_batches)) # We call this method, even though we are merging no new results, as it updates # the result witht he appropriate status (Success/Partial/Failed) self.merge_batch_results(result, []) def get_microbatch_builder(self, model: ModelNode) -> MicrobatchBuilder: # Intially set the start/end to values from args event_time_start = getattr(self.config.args, "EVENT_TIME_START", None) event_time_end = getattr(self.config.args, "EVENT_TIME_END", None) # If we're in sample mode, alter start/end to sample values if getattr(self.config.args, "SAMPLE", None) is not None: event_time_start = self.config.args.sample.start event_time_end = self.config.args.sample.end # During retry, use the original invocation time so that the same # batches are recomputed rather than batches relative to "now". default_end_time = ( self.parent_task.original_invocation_started_at or get_invocation_started_at() ) return MicrobatchBuilder( model=model, is_incremental=self._is_incremental(model), event_time_start=event_time_start, event_time_end=event_time_end, default_end_time=default_end_time, ) def get_batches(self, model: ModelNode) -> Dict[int, BatchType]: """Get the batches that should be run for the model""" # Note currently (02/23/2025) model.previous_batch_results is only ever _not_ `None` # IFF `dbt retry` is being run and the microbatch model had batches which # failed on the run of the model (which is being retried) if model.previous_batch_results is None: microbatch_builder = self.get_microbatch_builder(model) end = microbatch_builder.build_end_time() start = microbatch_builder.build_start_time(end) batches = microbatch_builder.build_batches(start, end) else: batches = model.previous_batch_results.failed return {batch_idx: batches[batch_idx] for batch_idx in range(len(batches))} def compile(self, manifest: Manifest): """Don't do anything here because this runner doesn't need to compile anything""" return self.node def execute(self, model: ModelNode, manifest: Manifest) -> RunResult: # Execution really means orchestration in this case batches = self.get_batches(model=model) relation_exists = self._has_relation(model=model) result = self._initial_run_microbatch_model_result(model=model) # No batches to run, so return initial result if len(batches) == 0: return result batch_results: List[RunResult] = [] batch_idx = 0 # Run first batch not in parallel relation_exists = self.parent_task._submit_batch( node=model, adapter=self.adapter, relation_exists=relation_exists, batches=batches, batch_idx=batch_idx, batch_results=batch_results, pool=self.pool, force_sequential_run=True, incremental_batch=self._is_incremental(model=model), ) batch_idx += 1 skip_batches = batch_results[0].status != RunStatus.Success # Run all batches except first and last batch, in parallel if possible while batch_idx < len(batches) - 1: relation_exists = self.parent_task._submit_batch( node=model, adapter=self.adapter, relation_exists=relation_exists, batches=batches, batch_idx=batch_idx, batch_results=batch_results, pool=self.pool, skip=skip_batches, ) batch_idx += 1 # Wait until all submitted batches have completed while len(batch_results) != batch_idx: # Check if the pool was closed, because if it was, then the main thread is trying to exit. # If the main thread is trying to exit, we need to shutdown. If we _don't_ shutdown, then # batches will continue to execute and we'll delay the run from stopping if self.pool.is_closed(): # It's technically possible for more results to come in while we clean up # instead we're going to say the didn't finish, regardless of if they finished # or not. Thus, lets get a copy of the results as they exist right "now". frozen_batch_results = deepcopy(batch_results) self.merge_batch_results(result, frozen_batch_results) self._update_result_with_unfinished_batches(result, batches) return result # breifly sleep so that this thread doesn't go brrrrr while waiting time.sleep(0.1) # Only run "last" batch if there is more than one batch if len(batches) != 1: # Final batch runs once all others complete to ensure post_hook runs at the end self.parent_task._submit_batch( node=model, adapter=self.adapter, relation_exists=relation_exists, batches=batches, batch_idx=batch_idx, batch_results=batch_results, pool=self.pool, force_sequential_run=True, skip=skip_batches, ) # Finalize run: merge results, track model run, and print final result line self.merge_batch_results(result, batch_results) return result class RunTask(CompileTask): def __init__( self, args: Flags, config: RuntimeConfig, manifest: Manifest, batch_map: Optional[Dict[str, BatchResults]] = None, ) -> None: super().__init__(args, config, manifest) self.batch_map = batch_map self.original_invocation_started_at: Optional[datetime] = None def raise_on_first_error(self) -> bool: return False def get_hook_sql(self, adapter, hook, idx, num_hooks, extra_context) -> str: if self.manifest is None: raise DbtInternalError("compile_node called before manifest was loaded") compiled = self.compiler.compile_node(hook, self.manifest, extra_context) statement = compiled.compiled_code hook_index = hook.index or num_hooks hook_obj = get_hook(statement, index=hook_index) return hook_obj.sql or "" def handle_job_queue(self, pool: DbtThreadPool, callback: Callable) -> None: assert self.job_queue is not None node = self.job_queue.get() self._raise_set_error() runner = self.get_runner(node) # we finally know what we're running! Make sure we haven't decided # to skip it due to upstream failures if runner.node.unique_id in self._skipped_children: cause = self._skipped_children.pop(runner.node.unique_id) runner.do_skip(cause=cause) if isinstance(runner, MicrobatchModelRunner): runner.set_parent_task(self) runner.set_pool(pool) args = [runner] # Ensure we don't submit more MicrobatchModelRunners than the pool allows self._maybe_wait_for_microbatch(pool, runner) self._submit(pool, args, callback) def _maybe_wait_for_microbatch(self, pool: DbtThreadPool, runner: BaseRunner) -> None: """ Checks if the runner is a MicrobatchModelRunner, and waits until the number of microbatch models in progress is less than the max number of microbatch models allowed by the pool. """ if isinstance(runner, MicrobatchModelRunner) and self.job_queue is not None: fire_event( MicrobatchExecutionDebug( msg=f"Waiting for microbatch model to be run: {runner.node.name}.\n\tpool.max_microbatch_models: {pool.max_microbatch_models}\n\tlen(self.job_queue.in_progress_microbatch): {len(self.job_queue.in_progress_microbatch)}" ) ) while pool.max_microbatch_models < len(self.job_queue.in_progress_microbatch): time.sleep(0.1) return def _submit_batch( self, node: ModelNode, adapter: BaseAdapter, relation_exists: bool, batches: Dict[int, BatchType], batch_idx: int, batch_results: List[RunResult], pool: DbtThreadPool, force_sequential_run: bool = False, skip: bool = False, incremental_batch: bool = True, ): node_copy = deepcopy(node) # Only run pre_hook(s) for first batch if batch_idx != 0: node_copy.config.pre_hook = [] # Only run post_hook(s) for last batch if batch_idx != len(batches) - 1: node_copy.config.post_hook = [] # TODO: We should be doing self.get_runner, however doing so # currently causes the tracking of how many nodes there are to # increment when we don't want it to batch_runner = MicrobatchBatchRunner( self.config, adapter, node_copy, self.run_count, self.num_nodes, batch_idx, batches, relation_exists, incremental_batch, ) if skip: batch_runner.do_skip() if not pool.is_closed(): # Only run the batch in parallel IFF: # 1. The batch runner is not forced to run sequentially # 2. The batch runner should be run in parallel # 3. There are available threads in the pool # a. This prevents deadlocks from occurring --threads=1 if ( not force_sequential_run and batch_runner.should_run_in_parallel() and self.job_queue is not None and len(self.job_queue.in_progress) < pool.max_threads ): fire_event( MicrobatchExecutionDebug( msg=f"{batch_runner.describe_batch()} is being run concurrently" ) ) self._submit(pool, [batch_runner], batch_results.append) else: fire_event( MicrobatchExecutionDebug( msg=f"{batch_runner.describe_batch()} is being run sequentially" ) ) batch_results.append(self.call_runner(batch_runner)) relation_exists = batch_runner.relation_exists else: batch_results.append( batch_runner._build_failed_run_batch_result(node_copy, batches[batch_idx]) ) return relation_exists def _hook_keyfunc(self, hook: HookNode) -> Tuple[str, Optional[int]]: package_name = hook.package_name if package_name == self.config.project_name: package_name = BiggestName("") return package_name, hook.index def get_hooks_by_type(self, hook_type: RunHookType) -> List[HookNode]: if self.manifest is None: raise DbtInternalError("self.manifest was None in get_hooks_by_type") nodes = self.manifest.nodes.values() # find all hooks defined in the manifest (could be multiple projects) hooks: List[HookNode] = get_hooks_by_tags(nodes, {hook_type}) hooks.sort(key=self._hook_keyfunc) return hooks def safe_run_hooks( self, adapter: BaseAdapter, hook_type: RunHookType, extra_context: Dict[str, Any] ) -> RunStatus: ordered_hooks = self.get_hooks_by_type(hook_type) if hook_type == RunHookType.End and ordered_hooks: fire_event(Formatting("")) # on-run-* hooks should run outside a transaction. This happens because psycopg2 automatically begins a transaction when a connection is created. adapter.clear_transaction() if not ordered_hooks: return RunStatus.Success status = RunStatus.Success failed = False num_hooks = len(ordered_hooks) for idx, hook in enumerate(ordered_hooks, 1): with log_contextvars(node_info=hook.node_info): hook.index = idx hook_name = f"{hook.package_name}.{hook_type}.{hook.index - 1}" execution_time = 0.0 timing: List[TimingInfo] = [] failures = 1 if not failed: with collect_timing_info("compile", timing.append): sql = self.get_hook_sql( adapter, hook, hook.index, num_hooks, extra_context ) started_at = timing[0].started_at or datetime.now(timezone.utc).replace( tzinfo=None ) hook.update_event_status( started_at=started_at.isoformat(), node_status=RunningStatus.Started ) fire_event( LogHookStartLine( statement=hook_name, index=hook.index, total=num_hooks, node_info=hook.node_info, ) ) with collect_timing_info("execute", timing.append): status, message = get_execution_status(sql, adapter) finished_at = timing[1].completed_at or datetime.now(timezone.utc).replace( tzinfo=None ) hook.update_event_status(finished_at=finished_at.isoformat()) execution_time = (finished_at - started_at).total_seconds() failures = 0 if status == RunStatus.Success else 1 if status == RunStatus.Success: message = f"{hook_name} passed" else: message = f"{hook_name} failed, error:\n {message}" failed = True else: status = RunStatus.Skipped message = f"{hook_name} skipped" hook.update_event_status(node_status=status) self.node_results.append( RunResult( status=status, thread_id="main", timing=timing, message=message, adapter_response={}, execution_time=execution_time, failures=failures, node=hook, ) ) fire_event( LogHookEndLine( statement=hook_name, status=status, index=hook.index, total=num_hooks, execution_time=execution_time, node_info=hook.node_info, ) ) if hook_type == RunHookType.Start and ordered_hooks: fire_event(Formatting("")) return status def print_results_line(self, results, execution_time) -> None: nodes = [r.node for r in results if hasattr(r, "node")] stat_line = get_counts(nodes) execution = "" if execution_time is not None: execution = utils.humanize_execution_time(execution_time=execution_time) fire_event(Formatting("")) fire_event( FinishedRunningStats( stat_line=stat_line, execution=execution, execution_time=execution_time ) ) def populate_microbatch_batches(self, selected_uids: AbstractSet[str]): if self.batch_map is not None and self.manifest is not None: for uid in selected_uids: if uid in self.batch_map: node = self.manifest.ref_lookup.perform_lookup(uid, self.manifest) if isinstance(node, ModelNode): node.previous_batch_results = self.batch_map[uid] def before_run(self, adapter: BaseAdapter, selected_uids: AbstractSet[str]) -> RunStatus: with adapter.connection_named("master"): self.defer_to_manifest() required_schemas = self.get_model_schemas(adapter, selected_uids) self.create_schemas(adapter, required_schemas) self.populate_adapter_cache(adapter, required_schemas) self.populate_microbatch_batches(selected_uids) group_lookup.init(self.manifest, selected_uids) run_hooks_status = self.safe_run_hooks(adapter, RunHookType.Start, {}) return run_hooks_status def after_run(self, adapter, results) -> None: # in on-run-end hooks, provide the value 'database_schemas', which is a # list of unique (database, schema) pairs that successfully executed # models were in. For backwards compatibility, include the old # 'schemas', which did not include database information. database_schema_set: Set[Tuple[Optional[str], str]] = { (r.node.database, r.node.schema) for r in results if (hasattr(r, "node") and r.node.is_relational) and r.status not in (NodeStatus.Error, NodeStatus.Fail, NodeStatus.Skipped) } extras = { "schemas": list({s for _, s in database_schema_set}), "results": [ r for r in results if r.thread_id != "main" or r.status == RunStatus.Error ], # exclude that didn't fail to preserve backwards compatibility "database_schemas": list(database_schema_set), } try: with adapter.connection_named("master"): self.safe_run_hooks(adapter, RunHookType.End, extras) except (KeyboardInterrupt, SystemExit, DbtRuntimeError): run_result = self.get_result( results=self.node_results, elapsed_time=time.time() - self.started_at, generated_at=datetime.now(timezone.utc).replace(tzinfo=None), ) if self.args.write_json and hasattr(run_result, "write"): run_result.write(self.result_path()) add_artifact_produced(self.result_path()) print_run_end_messages(self.node_results, keyboard_interrupt=True) raise def get_node_selector(self) -> ResourceTypeSelector: if self.manifest is None or self.graph is None: raise DbtInternalError("manifest and graph must be set to get perform node selection") return ResourceTypeSelector( graph=self.graph, manifest=self.manifest, previous_state=self.previous_state, resource_types=[NodeType.Model], selectors=self.config.selectors, ) def get_runner_type(self, node) -> Optional[Type[BaseRunner]]: if self.manifest is None: raise DbtInternalError("manifest must be set prior to calling get_runner_type") if ( node.config.materialized == "incremental" and node.config.incremental_strategy == "microbatch" and self.manifest.use_microbatch_batches(project_name=self.config.project_name) ): return MicrobatchModelRunner else: return ModelRunner def task_end_messages(self, results) -> None: if results: print_run_end_messages(results) ================================================ FILE: core/dbt/task/run_operation.py ================================================ import os import threading import traceback from datetime import datetime, timezone from typing import TYPE_CHECKING, List import dbt_common.exceptions from dbt.adapters.factory import get_adapter from dbt.artifacts.schemas.results import RunStatus, TimingInfo, collect_timing_info from dbt.artifacts.schemas.run import RunResult, RunResultsArtifact from dbt.constants import RUN_RESULTS_FILE_NAME from dbt.contracts.files import FileHash from dbt.contracts.graph.nodes import HookNode from dbt.events.types import ( ArtifactWritten, LogDebugStackTrace, RunningOperationCaughtError, RunningOperationUncaughtError, ) from dbt.node_types import NodeType from dbt.task.base import ConfiguredTask from dbt_common.events.functions import fire_event if TYPE_CHECKING: import agate class RunOperationTask(ConfiguredTask): def _get_macro_parts(self): macro_name = self.args.macro if "." in macro_name: package_name, macro_name = macro_name.split(".", 1) else: package_name = None return package_name, macro_name def _run_unsafe(self, package_name, macro_name) -> "agate.Table": adapter = get_adapter(self.config) macro_kwargs = self.args.args with adapter.connection_named("macro_{}".format(macro_name)): adapter.clear_transaction() res = adapter.execute_macro( macro_name, project=package_name, kwargs=macro_kwargs, macro_resolver=self.manifest ) return res def run(self) -> RunResultsArtifact: timing: List[TimingInfo] = [] with collect_timing_info("compile", timing.append): self.compile_manifest() start = timing[0].started_at success = True package_name, macro_name = self._get_macro_parts() with collect_timing_info("execute", timing.append): try: self._run_unsafe(package_name, macro_name) except dbt_common.exceptions.DbtBaseException as exc: fire_event(RunningOperationCaughtError(exc=str(exc))) fire_event(LogDebugStackTrace(exc_info=traceback.format_exc())) success = False except Exception as exc: fire_event(RunningOperationUncaughtError(exc=str(exc))) fire_event(LogDebugStackTrace(exc_info=traceback.format_exc())) success = False end = timing[1].completed_at macro = ( self.manifest.find_macro_by_name(macro_name, self.config.project_name, package_name) if self.manifest else None ) if macro: unique_id = macro.unique_id fqn = unique_id.split(".") else: raise dbt_common.exceptions.UndefinedMacroError( f"dbt could not find a macro with the name '{macro_name}' in any package" ) execution_time = (end - start).total_seconds() if start and end else 0.0 run_result = RunResult( adapter_response={}, status=RunStatus.Success if success else RunStatus.Error, execution_time=execution_time, failures=0 if success else 1, message=None, node=HookNode( alias=macro_name, checksum=FileHash.from_contents(unique_id), database=self.config.credentials.database, schema=self.config.credentials.schema, resource_type=NodeType.Operation, fqn=fqn, name=macro_name, unique_id=unique_id, package_name=package_name, path="", original_file_path="", ), thread_id=threading.current_thread().name, timing=timing, batch_results=None, ) results = RunResultsArtifact.from_execution_results( generated_at=end or datetime.now(timezone.utc).replace(tzinfo=None), elapsed_time=execution_time, args={ k: v for k, v in self.args.__dict__.items() if k.islower() and type(v) in (str, int, float, bool, list, dict) }, results=[run_result], ) result_path = os.path.join(self.config.project_target_path, RUN_RESULTS_FILE_NAME) if self.args.write_json: results.write(result_path) fire_event( ArtifactWritten( artifact_type=results.__class__.__name__, artifact_path=result_path ) ) return results @classmethod def interpret_results(cls, results): return results.results[0].status == RunStatus.Success ================================================ FILE: core/dbt/task/runnable.py ================================================ import os import time from abc import abstractmethod from concurrent.futures import as_completed from datetime import datetime, timezone from pathlib import Path from typing import ( AbstractSet, Any, Callable, Dict, Iterable, List, Optional, Set, Tuple, Type, Union, ) import dbt.exceptions import dbt.tracking import dbt.utils import dbt_common.utils.formatting from dbt.adapters.base import BaseAdapter, BaseRelation from dbt.adapters.factory import get_adapter from dbt.artifacts.schemas.results import ( BaseResult, NodeStatus, RunningStatus, RunStatus, ) from dbt.artifacts.schemas.run import RunExecutionResult, RunResult from dbt.cli.flags import Flags from dbt.config.runtime import RuntimeConfig from dbt.constants import RUN_RESULTS_FILE_NAME from dbt.contracts.graph.manifest import Manifest from dbt.contracts.graph.nodes import Exposure, ResultNode from dbt.contracts.state import PreviousState from dbt.events.types import ( ArtifactWritten, ConcurrencyLine, DefaultSelector, EndRunResult, GenericExceptionOnRun, LogCancelLine, MarkSkippedChildren, NodeFinished, NodeStart, NothingToDo, QueryCancelationUnsupported, SkippingDetails, ) from dbt.exceptions import DbtInternalError, DbtRuntimeError, FailFastError from dbt.flags import get_flags from dbt.graph import ( GraphQueue, NodeSelector, SelectionSpec, UniqueId, parse_difference, ) from dbt.graph.thread_pool import DbtThreadPool from dbt.parser.manifest import write_manifest from dbt.task import group_lookup from dbt.task.base import BaseRunner, ConfiguredTask from dbt.task.printer import print_run_end_messages, print_run_result_error from dbt.utils.artifact_upload import add_artifact_produced from dbt_common.context import _INVOCATION_CONTEXT_VAR, get_invocation_context from dbt_common.dataclass_schema import StrEnum from dbt_common.events.contextvars import log_contextvars, task_contextvars from dbt_common.events.functions import fire_event, warn_or_error from dbt_common.events.types import Formatting from dbt_common.exceptions import NotImplementedError class GraphRunnableMode(StrEnum): Topological = "topological" Independent = "independent" def mark_node_as_skipped( node: ResultNode, executed_node_ids: Set[str], message: Optional[str] ) -> Optional[RunResult]: if node.unique_id not in executed_node_ids: return RunResult.from_node(node, RunStatus.Skipped, message) return None class GraphRunnableTask(ConfiguredTask): MARK_DEPENDENT_ERRORS_STATUSES = [NodeStatus.Error, NodeStatus.PartialSuccess] def __init__(self, args: Flags, config: RuntimeConfig, manifest: Manifest) -> None: super().__init__(args, config, manifest) self.config = config self._flattened_nodes: Optional[List[ResultNode]] = None self._raise_next_tick: Optional[DbtRuntimeError] = None self._skipped_children: Dict[str, Optional[RunResult]] = {} self.job_queue: Optional[GraphQueue] = None self.node_results: List[BaseResult] = [] self.num_nodes: int = 0 self.previous_state: Optional[PreviousState] = None self.previous_defer_state: Optional[PreviousState] = None self.run_count: int = 0 self.started_at: float = 0 if self.args.state: self.previous_state = PreviousState( state_path=self.args.state, target_path=Path(self.config.target_path), project_root=Path(self.config.project_root), ) if self.args.defer_state: self.previous_defer_state = PreviousState( state_path=self.args.defer_state, target_path=Path(self.config.target_path), project_root=Path(self.config.project_root), ) def index_offset(self, value: int) -> int: return value @property def selection_arg(self): return self.args.select @property def exclusion_arg(self): return self.args.exclude def get_selection_spec(self) -> SelectionSpec: default_selector_name = self.config.get_default_selector_name() spec: Union[SelectionSpec, bool] if hasattr(self.args, "inline") and self.args.inline: # We want an empty selection spec. spec = parse_difference(None, None) elif self.args.selector: # use pre-defined selector (--selector) spec = self.config.get_selector(self.args.selector) elif not (self.selection_arg or self.exclusion_arg) and default_selector_name: # use pre-defined selector (--selector) with default: true fire_event(DefaultSelector(name=default_selector_name)) spec = self.config.get_selector(default_selector_name) else: # This is what's used with no default selector and no selection # use --select and --exclude args spec = parse_difference(self.selection_arg, self.exclusion_arg) # mypy complains because the return values of get_selector and parse_difference # are different return spec # type: ignore @abstractmethod def get_node_selector(self) -> NodeSelector: raise NotImplementedError(f"get_node_selector not implemented for task {type(self)}") def defer_to_manifest(self): deferred_manifest = self._get_deferred_manifest() if deferred_manifest is None: return if self.manifest is None: raise DbtInternalError( "Expected to defer to manifest, but there is no runtime manifest to defer from!" ) self.manifest.merge_from_artifact(other=deferred_manifest) def get_graph_queue(self) -> GraphQueue: selector = self.get_node_selector() # Following uses self.selection_arg and self.exclusion_arg spec = self.get_selection_spec() preserve_edges = True if self.get_run_mode() == GraphRunnableMode.Independent: preserve_edges = False return selector.get_graph_queue(spec, preserve_edges) def get_run_mode(self) -> GraphRunnableMode: return GraphRunnableMode.Topological def _runtime_initialize(self): self.compile_manifest() if self.manifest is None or self.graph is None: raise DbtInternalError("_runtime_initialize never loaded the graph!") self.job_queue = self.get_graph_queue() # Set selected node IDs on the compiler so FK constraint compilation # can determine whether to use deferred relations or current relations. # FK targets that ARE selected should use current relations (being built now). # FK targets that are NOT selected should use deferred relations (from state). self.compiler.selected_node_ids = set(self.job_queue.get_selected_nodes()) # we use this a couple of times. order does not matter. self._flattened_nodes = [] for uid in self.job_queue.get_selected_nodes(): if uid in self.manifest.nodes: self._flattened_nodes.append(self.manifest.nodes[uid]) elif uid in self.manifest.sources: self._flattened_nodes.append(self.manifest.sources[uid]) elif uid in self.manifest.saved_queries: self._flattened_nodes.append(self.manifest.saved_queries[uid]) elif uid in self.manifest.unit_tests: self._flattened_nodes.append(self.manifest.unit_tests[uid]) elif uid in self.manifest.exposures: self._flattened_nodes.append(self.manifest.exposures[uid]) elif uid in self.manifest.functions: self._flattened_nodes.append(self.manifest.functions[uid]) else: raise DbtInternalError( f"Node selection returned {uid}, expected an exposure, a function, a node, a saved query, a source, or a unit test" ) self.num_nodes = len([n for n in self._flattened_nodes if not n.is_ephemeral_model]) def raise_on_first_error(self) -> bool: return False def get_runner_type(self, node) -> Optional[Type[BaseRunner]]: raise NotImplementedError("Not Implemented") def result_path(self) -> str: return os.path.join(self.config.project_target_path, RUN_RESULTS_FILE_NAME) def get_runner(self, node) -> BaseRunner: adapter = get_adapter(self.config) run_count: int = 0 num_nodes: int = 0 if node.is_ephemeral_model: run_count = 0 num_nodes = 0 else: self.run_count += 1 run_count = self.run_count num_nodes = self.num_nodes cls = self.get_runner_type(node) if cls is None: raise DbtInternalError("Could not find runner type for node.") runner = cls(self.config, adapter, node, run_count, num_nodes) # Propagate selected node IDs to the runner's compiler for FK constraint resolution runner.compiler.selected_node_ids = self.compiler.selected_node_ids return runner def call_runner(self, runner: BaseRunner) -> RunResult: with log_contextvars(node_info=runner.node.node_info): runner.node.update_event_status( started_at=datetime.now(timezone.utc).replace(tzinfo=None).isoformat(), node_status=RunningStatus.Started, ) fire_event( NodeStart( node_info=runner.node.node_info, ) ) result = None thread_exception: Optional[Union[KeyboardInterrupt, SystemExit, Exception]] = None try: result = runner.run_with_hooks(self.manifest) except (KeyboardInterrupt, SystemExit) as exe: result = None thread_exception = exe raise except Exception as e: result = None thread_exception = e finally: if result is not None: try: fire_event( NodeFinished( node_info=runner.node.node_info, run_result=result.to_msg_dict(), ) ) except Exception as e: result = self._handle_thread_exception(runner, e) else: result = self._handle_thread_exception(runner, thread_exception) # `_event_status` dict is only used for logging. Make sure # it gets deleted when we're done with it runner.node.clear_event_status() fail_fast = get_flags().FAIL_FAST if ( result.status in (NodeStatus.Error, NodeStatus.Fail, NodeStatus.PartialSuccess) and fail_fast ): self._raise_next_tick = FailFastError( msg="Failing early due to test failure or runtime error", result=result, node=getattr(result, "node", None), ) elif result.status == NodeStatus.Error and self.raise_on_first_error(): # if we raise inside a thread, it'll just get silently swallowed. # stash the error message we want here, and it will check the # next 'tick' - should be soon since our thread is about to finish! self._raise_next_tick = DbtRuntimeError(result.message) return result def _submit(self, pool: DbtThreadPool, args: List[Any], callback: Callable) -> None: """If the caller has passed the magic 'single-threaded' flag, call the function directly instead of pool.apply_async. The single-threaded flag is intended for gathering more useful performance information about what happens beneath `call_runner`, since python's default profiling tools ignore child threads. This does still go through the callback path for result collection. """ if self.config.args.single_threaded: callback(self.call_runner(*args)) else: pool.apply_async(self.call_runner, args=args, callback=callback) def _raise_set_error(self): if self._raise_next_tick is not None: raise self._raise_next_tick def run_queue(self, pool: DbtThreadPool) -> None: """Given a pool, submit jobs from the queue to the pool.""" if self.job_queue is None: raise DbtInternalError("Got to run_queue with no job queue set") def callback(result): """Note: mark_done, at a minimum, must happen here or dbt will deadlock during ephemeral result error handling! """ self._handle_result(result) if self.job_queue is None: raise DbtInternalError("Got to run_queue callback with no job queue set") self.job_queue.mark_done(result.node.unique_id) while not self.job_queue.empty(): self.handle_job_queue(pool, callback) # block on completion if get_flags().FAIL_FAST: # checkout for an errors after task completion in case of # fast failure while self.job_queue.wait_until_something_was_done(): self._raise_set_error() else: # wait until every task will be complete self.job_queue.join() # if an error got set during join(), raise it. self._raise_set_error() return # The build command overrides this def handle_job_queue(self, pool: DbtThreadPool, callback: Callable) -> None: assert self.job_queue is not None node = self.job_queue.get() self._raise_set_error() runner = self.get_runner(node) # we finally know what we're running! Make sure we haven't decided # to skip it due to upstream failures if runner.node.unique_id in self._skipped_children: cause = self._skipped_children.pop(runner.node.unique_id) runner.do_skip(cause=cause) args = [runner] self._submit(pool, args, callback) def _handle_thread_exception( self, runner: BaseRunner, thread_exception: Optional[Union[KeyboardInterrupt, SystemExit, Exception]], ) -> RunResult: msg = f"Exception on worker thread. {thread_exception}" fire_event( GenericExceptionOnRun( unique_id=runner.node.unique_id, exc=str(thread_exception), node_info=runner.node.node_info, ) ) return RunResult( status=RunStatus.Error, # type: ignore timing=[], thread_id="", execution_time=0.0, adapter_response={}, message=msg, failures=None, batch_results=None, node=runner.node, ) def _handle_result(self, result: RunResult) -> None: """Mark the result as completed, insert the `CompileResultNode` into the manifest, and mark any descendants (potentially with a 'cause' if the result was an ephemeral model) as skipped. """ is_ephemeral = result.node.is_ephemeral_model if not is_ephemeral: self.node_results.append(result) node = result.node if self.manifest is None: raise DbtInternalError("manifest was None in _handle_result") # If result.status == NodeStatus.Error, plus Fail for build command if result.status in self.MARK_DEPENDENT_ERRORS_STATUSES: if is_ephemeral: cause = result else: cause = None self._mark_dependent_errors(node.unique_id, result, cause) def _cancel_connections(self, pool): """Given a pool, cancel all adapter connections and wait until all runners gentle terminates. """ pool.close() pool.terminate() adapter = get_adapter(self.config) if not adapter.is_cancelable(): fire_event(QueryCancelationUnsupported(type=adapter.type())) else: with adapter.connection_named("master"): for conn_name in adapter.cancel_open_connections(): if self.manifest is not None: node = self.manifest.nodes.get(conn_name) if node is not None and node.is_ephemeral_model: continue # if we don't have a manifest/don't have a node, print # anyway. fire_event(LogCancelLine(conn_name=conn_name)) pool.join() def execute_nodes(self): num_threads = self.config.threads pool = DbtThreadPool( num_threads, self._pool_thread_initializer, [get_invocation_context()] ) try: self.run_queue(pool) except FailFastError as failure: self._cancel_connections(pool) executed_node_ids = {r.node.unique_id for r in self.node_results} message = "Skipping due to fail_fast" for node in self._flattened_nodes: if node.unique_id not in executed_node_ids: self.node_results.append( mark_node_as_skipped(node, executed_node_ids, message) ) print_run_result_error(failure.result) # ensure information about all nodes is propagated to run results when failing fast return self.node_results except (KeyboardInterrupt, SystemExit): run_result = self.get_result( results=self.node_results, elapsed_time=time.time() - self.started_at, generated_at=datetime.now(timezone.utc).replace(tzinfo=None), ) if self.args.write_json and hasattr(run_result, "write"): run_result.write(self.result_path()) add_artifact_produced(self.result_path()) fire_event( ArtifactWritten( artifact_type=run_result.__class__.__name__, artifact_path=self.result_path(), ) ) self._cancel_connections(pool) print_run_end_messages(self.node_results, keyboard_interrupt=True) raise pool.close() pool.join() return self.node_results @staticmethod def _pool_thread_initializer(invocation_context): _INVOCATION_CONTEXT_VAR.set(invocation_context) def _mark_dependent_errors( self, node_id: str, result: RunResult, cause: Optional[RunResult] ) -> None: if self.graph is None: raise DbtInternalError("graph is None in _mark_dependent_errors") fire_event( MarkSkippedChildren( unique_id=node_id, status=result.status, run_result=result.to_msg_dict(), ) ) for dep_node_id in self.graph.get_dependent_nodes(UniqueId(node_id)): self._skipped_children[dep_node_id] = cause def populate_adapter_cache( self, adapter, required_schemas: Optional[Set[BaseRelation]] = None ): if not self.args.populate_cache: return if self.manifest is None: raise DbtInternalError("manifest was None in populate_adapter_cache") start_populate_cache = time.perf_counter() # the cache only cares about executable nodes cachable_nodes = [ node for node in self.manifest.nodes.values() if (node.is_relational and not node.is_ephemeral_model and not node.is_external_node) ] if get_flags().CACHE_SELECTED_ONLY is True: adapter.set_relations_cache(cachable_nodes, required_schemas=required_schemas) else: adapter.set_relations_cache(cachable_nodes) cache_populate_time = time.perf_counter() - start_populate_cache if dbt.tracking.active_user is not None: dbt.tracking.track_runnable_timing( {"adapter_cache_construction_elapsed": cache_populate_time} ) def before_run(self, adapter: BaseAdapter, selected_uids: AbstractSet[str]) -> RunStatus: with adapter.connection_named("master"): self.defer_to_manifest() self.populate_adapter_cache(adapter) return RunStatus.Success def after_run(self, adapter, results) -> None: pass def print_results_line(self, node_results, elapsed): pass def execute_with_hooks(self, selected_uids: AbstractSet[str]): adapter = get_adapter(self.config) fire_event(Formatting("")) fire_event( ConcurrencyLine( num_threads=self.config.threads, target_name=self.config.target_name, node_count=self.num_nodes, ) ) fire_event(Formatting("")) self.started_at = time.time() try: before_run_status = self.before_run(adapter, selected_uids) if before_run_status == RunStatus.Success or ( not get_flags().skip_nodes_if_on_run_start_fails ): res = self.execute_nodes() else: executed_node_ids = { r.node.unique_id for r in self.node_results if hasattr(r, "node") } res = [] for index, node in enumerate(self._flattened_nodes or []): group = group_lookup.get(node.unique_id) if node.unique_id not in executed_node_ids: fire_event( SkippingDetails( resource_type=node.resource_type, schema=node.schema, node_name=node.name, index=index + 1, total=self.num_nodes, node_info=node.node_info, group=group, ) ) skipped_node_result = mark_node_as_skipped(node, executed_node_ids, None) if skipped_node_result: self.node_results.append(skipped_node_result) self.after_run(adapter, res) finally: adapter.cleanup_connections() elapsed = time.time() - self.started_at self.print_results_line(self.node_results, elapsed) result = self.get_result( results=self.node_results, elapsed_time=elapsed, generated_at=datetime.now(timezone.utc).replace(tzinfo=None), ) return result def run(self): """ Run dbt for the query, based on the graph. """ # We set up a context manager here with "task_contextvars" because we # need the project_root in runtime_initialize. with task_contextvars(project_root=self.config.project_root): self._runtime_initialize() if self._flattened_nodes is None: raise DbtInternalError( "after _runtime_initialize, _flattened_nodes was still None" ) if len(self._flattened_nodes) == 0: warn_or_error(NothingToDo()) result = self.get_result( results=[], generated_at=datetime.now(timezone.utc).replace(tzinfo=None), elapsed_time=0.0, ) else: selected_uids = frozenset(n.unique_id for n in self._flattened_nodes) result = self.execute_with_hooks(selected_uids) # We have other result types here too, including FreshnessResult if isinstance(result, RunExecutionResult): result_msgs = [result.to_msg_dict() for result in result.results] fire_event( EndRunResult( results=result_msgs, generated_at=result.generated_at.strftime("%Y-%m-%dT%H:%M:%SZ"), elapsed_time=result.elapsed_time, success=GraphRunnableTask.interpret_results(result.results), ) ) if self.args.write_json: write_manifest(self.manifest, self.config.project_target_path) if hasattr(result, "write"): result.write(self.result_path()) add_artifact_produced(self.result_path()) fire_event( ArtifactWritten( artifact_type=result.__class__.__name__, artifact_path=self.result_path() ) ) self.task_end_messages(result.results) return result @classmethod def interpret_results(cls, results): if results is None: return False num_runtime_errors = len([r for r in results if r.status == NodeStatus.RuntimeErr]) num_errors = len([r for r in results if r.status == NodeStatus.Error]) num_fails = len([r for r in results if r.status == NodeStatus.Fail]) num_skipped = len( [ r for r in results if r.status == NodeStatus.Skipped and not isinstance(r.node, Exposure) ] ) num_partial_success = len([r for r in results if r.status == NodeStatus.PartialSuccess]) num_total = num_runtime_errors + num_errors + num_fails + num_skipped + num_partial_success return num_total == 0 def get_model_schemas(self, adapter, selected_uids: Iterable[str]) -> Set[BaseRelation]: if self.manifest is None: raise DbtInternalError("manifest was None in get_model_schemas") result: Set[BaseRelation] = set() for node in self.manifest.nodes.values(): if node.unique_id not in selected_uids: continue if node.is_relational and not node.is_ephemeral: relation = adapter.Relation.create_from(self.config, node) result.add(relation.without_identifier()) return result def create_schemas(self, adapter, required_schemas: Set[BaseRelation]): # we want the string form of the information schema database required_databases: Set[BaseRelation] = set() for required in required_schemas: db_only = required.include(database=True, schema=False, identifier=False) required_databases.add(db_only) existing_schemas_lowered: Set[Tuple[Optional[str], Optional[str]]] existing_schemas_lowered = set() def list_schemas(db_only: BaseRelation) -> List[Tuple[Optional[str], str]]: # the database can be None on some warehouses that don't support it database_quoted: Optional[str] db_lowercase = dbt_common.utils.formatting.lowercase(db_only.database) if db_only.database is None: database_quoted = None else: database_quoted = str(db_only) # we should never create a null schema, so just filter them out return [ (db_lowercase, s.lower()) for s in adapter.list_schemas(database_quoted) if s is not None ] def create_schema(relation: BaseRelation) -> None: db = relation.database or "" schema = relation.schema with adapter.connection_named(f"create_{db}_{schema}"): adapter.create_schema(relation) list_futures = [] create_futures = [] # TODO: following has a mypy issue because profile and project config # defines threads as int and HasThreadingConfig defines it as Optional[int] with dbt_common.utils.executor(self.config) as tpe: # type: ignore for req in required_databases: if req.database is None: name = "list_schemas" else: name = f"list_{req.database}" fut = tpe.submit_connected(adapter, name, list_schemas, req) list_futures.append(fut) for ls_future in as_completed(list_futures): existing_schemas_lowered.update(ls_future.result()) for info in required_schemas: if info.schema is None: # we are not in the business of creating null schemas, so # skip this continue db: Optional[str] = info.database db_lower: Optional[str] = dbt_common.utils.formatting.lowercase(db) schema: str = info.schema db_schema = (db_lower, schema.lower()) if db_schema not in existing_schemas_lowered: existing_schemas_lowered.add(db_schema) fut = tpe.submit_connected( adapter, f'create_{info.database or ""}_{info.schema}', create_schema, info ) create_futures.append(fut) for create_future in as_completed(create_futures): # trigger/re-raise any exceptions while creating schemas create_future.result() def get_result(self, results, elapsed_time, generated_at): return RunExecutionResult( results=results, elapsed_time=elapsed_time, generated_at=generated_at, args=dbt.utils.args_to_dict(self.args), ) def task_end_messages(self, results) -> None: print_run_end_messages(results) def _get_previous_state(self) -> Optional[Manifest]: state = self.previous_defer_state or self.previous_state if not state: raise DbtRuntimeError( "--state or --defer-state are required for deferral, but neither was provided" ) if not state.manifest: raise DbtRuntimeError(f'Could not find manifest in --state path: "{state.state_path}"') return state.manifest def _get_deferred_manifest(self) -> Optional[Manifest]: return self._get_previous_state() if self.args.defer else None ================================================ FILE: core/dbt/task/seed.py ================================================ import random from typing import Optional, Type from dbt.artifacts.schemas.results import NodeStatus, RunStatus from dbt.contracts.graph.manifest import Manifest from dbt.events.types import LogSeedResult, LogStartLine, SeedHeader from dbt.graph import ResourceTypeSelector from dbt.node_types import NodeType from dbt.task import group_lookup from dbt.task.base import BaseRunner from dbt.task.printer import print_run_end_messages from dbt.task.run import ModelRunner, RunTask from dbt_common.events.base_types import EventLevel from dbt_common.events.functions import fire_event from dbt_common.events.types import Formatting from dbt_common.exceptions import DbtInternalError class SeedRunner(ModelRunner): def describe_node(self) -> str: return "seed file {}".format(self.get_node_representation()) def before_execute(self) -> None: fire_event( LogStartLine( description=self.describe_node(), index=self.node_index, total=self.num_nodes, node_info=self.node.node_info, ) ) def _build_run_model_result(self, model, context): result = super()._build_run_model_result(model, context) agate_result = context["load_result"]("agate_table") result.agate_table = agate_result.table return result def compile(self, manifest: Manifest): return self.node def print_result_line(self, result): model = result.node group = group_lookup.get(model.unique_id) level = EventLevel.ERROR if result.status == NodeStatus.Error else EventLevel.INFO fire_event( LogSeedResult( status=result.status, result_message=result.message, index=self.node_index, total=self.num_nodes, execution_time=result.execution_time, schema=self.node.schema, relation=model.alias, node_info=model.node_info, group=group, ), level=level, ) class SeedTask(RunTask): def raise_on_first_error(self) -> bool: return False def get_node_selector(self): if self.manifest is None or self.graph is None: raise DbtInternalError("manifest and graph must be set to get perform node selection") return ResourceTypeSelector( graph=self.graph, manifest=self.manifest, previous_state=self.previous_state, resource_types=[NodeType.Seed], selectors=self.config.selectors, ) def get_runner_type(self, _) -> Optional[Type[BaseRunner]]: return SeedRunner def task_end_messages(self, results) -> None: if self.args.show: self.show_tables(results) print_run_end_messages(results) def show_table(self, result): table = result.agate_table rand_table = table.order_by(lambda x: random.random()) schema = result.node.schema alias = result.node.alias header = "Random sample of table: {}.{}".format(schema, alias) fire_event(Formatting("")) fire_event(SeedHeader(header=header)) fire_event(Formatting("-" * len(header))) rand_table.print_table(max_rows=10, max_columns=None) fire_event(Formatting("")) def show_tables(self, results): for result in results: if result.status != RunStatus.Error: self.show_table(result) ================================================ FILE: core/dbt/task/show.py ================================================ import io import threading import time from dbt.adapters.factory import get_adapter from dbt.artifacts.schemas.run import RunResult, RunStatus from dbt.context.providers import generate_runtime_model_context from dbt.contracts.graph.nodes import SeedNode from dbt.events.types import ShowNode from dbt.flags import get_flags from dbt.task.base import ConfiguredTask from dbt.task.compile import CompileRunner, CompileTask from dbt.task.seed import SeedRunner from dbt_common.events.base_types import EventLevel from dbt_common.events.functions import fire_event from dbt_common.events.types import Note from dbt_common.exceptions import DbtRuntimeError class ShowRunner(CompileRunner): def __init__(self, config, adapter, node, node_index, num_nodes) -> None: super().__init__(config, adapter, node, node_index, num_nodes) self.run_ephemeral_models = True def execute(self, compiled_node, manifest): start_time = time.time() # Allow passing in -1 (or any negative number) to get all rows limit = None if self.config.args.limit < 0 else self.config.args.limit model_context = generate_runtime_model_context(compiled_node, self.config, manifest) compiled_node.compiled_code = self.adapter.execute_macro( macro_name="get_show_sql", macro_resolver=manifest, context_override=model_context, kwargs={ "compiled_code": model_context["compiled_code"], "sql_header": model_context["config"].get("sql_header"), "limit": limit, }, ) adapter_response, execute_result = self.adapter.execute( compiled_node.compiled_code, fetch=True ) end_time = time.time() return RunResult( node=compiled_node, status=RunStatus.Success, timing=[], thread_id=threading.current_thread().name, execution_time=end_time - start_time, message=None, adapter_response=adapter_response.to_dict(), agate_table=execute_result, failures=None, batch_results=None, ) class ShowTask(CompileTask): def _runtime_initialize(self): if not (self.args.select or getattr(self.args, "inline", None)): raise DbtRuntimeError("Either --select or --inline must be passed to show") super()._runtime_initialize() def get_runner_type(self, node): if isinstance(node, SeedNode): return SeedRunner else: return ShowRunner def task_end_messages(self, results) -> None: is_inline = bool(getattr(self.args, "inline", None)) if is_inline: matched_results = [result for result in results if result.node.name == "inline_query"] else: matched_results = [] for result in results: if result.node.name in self.selection_arg[0]: matched_results.append(result) else: fire_event( Note(msg=f"Excluded node '{result.node.name}' from results"), EventLevel.DEBUG, ) for result in matched_results: table = result.agate_table # Hack to get Agate table output as string output = io.StringIO() if self.args.output == "json": table.to_json(path=output) else: table.print_table(output=output, max_rows=None) node_name = result.node.name if hasattr(result.node, "version") and result.node.version: node_name += f".v{result.node.version}" fire_event( ShowNode( node_name=node_name, preview=output.getvalue(), is_inline=is_inline, output_format=self.args.output, unique_id=result.node.unique_id, quiet=get_flags().QUIET, ) ) def _handle_result(self, result) -> None: super()._handle_result(result) if ( result.node.is_ephemeral_model and type(self) is ShowTask and (self.args.select or getattr(self.args, "inline", None)) ): self.node_results.append(result) class ShowTaskDirect(ConfiguredTask): def run(self): adapter = get_adapter(self.config) with adapter.connection_named("show", should_release_connection=False): limit = None if self.args.limit < 0 else self.args.limit response, table = adapter.execute(self.args.inline_direct, fetch=True, limit=limit) output = io.StringIO() if self.args.output == "json": table.to_json(path=output) else: table.print_table(output=output, max_rows=None) fire_event( ShowNode( node_name="direct-query", preview=output.getvalue(), is_inline=True, output_format=self.args.output, unique_id="direct-query", quiet=get_flags().QUIET, ) ) ================================================ FILE: core/dbt/task/snapshot.py ================================================ from typing import Optional, Type from dbt.artifacts.schemas.results import NodeStatus from dbt.events.types import LogSnapshotResult from dbt.graph import ResourceTypeSelector from dbt.node_types import NodeType from dbt.task import group_lookup from dbt.task.base import BaseRunner from dbt.task.run import ModelRunner, RunTask from dbt_common.events.base_types import EventLevel from dbt_common.events.functions import fire_event from dbt_common.exceptions import DbtInternalError from dbt_common.utils import cast_dict_to_dict_of_strings class SnapshotRunner(ModelRunner): def describe_node(self) -> str: return "snapshot {}".format(self.get_node_representation()) def print_result_line(self, result): model = result.node group = group_lookup.get(model.unique_id) cfg = model.config.to_dict(omit_none=True) level = EventLevel.ERROR if result.status == NodeStatus.Error else EventLevel.INFO fire_event( LogSnapshotResult( status=result.status, description=self.get_node_representation(), cfg=cast_dict_to_dict_of_strings(cfg), index=self.node_index, total=self.num_nodes, execution_time=result.execution_time, node_info=model.node_info, result_message=result.message, group=group, ), level=level, ) class SnapshotTask(RunTask): def raise_on_first_error(self) -> bool: return False def get_node_selector(self): if self.manifest is None or self.graph is None: raise DbtInternalError("manifest and graph must be set to get perform node selection") return ResourceTypeSelector( graph=self.graph, manifest=self.manifest, previous_state=self.previous_state, resource_types=[NodeType.Snapshot], selectors=self.config.selectors, ) def get_runner_type(self, _) -> Optional[Type[BaseRunner]]: return SnapshotRunner ================================================ FILE: core/dbt/task/sql.py ================================================ import traceback from abc import abstractmethod from datetime import datetime, timezone from typing import Generic, TypeVar import dbt.exceptions import dbt_common.exceptions.base from dbt.contracts.graph.manifest import Manifest from dbt.contracts.sql import ( RemoteCompileResult, RemoteCompileResultMixin, RemoteRunResult, ResultTable, ) from dbt.events.types import SQLRunnerException from dbt.task.compile import CompileRunner from dbt_common.events.functions import fire_event SQLResult = TypeVar("SQLResult", bound=RemoteCompileResultMixin) class GenericSqlRunner(CompileRunner, Generic[SQLResult]): def __init__(self, config, adapter, node, node_index, num_nodes) -> None: CompileRunner.__init__(self, config, adapter, node, node_index, num_nodes) def handle_exception(self, e, ctx): fire_event( SQLRunnerException( exc=str(e), exc_info=traceback.format_exc(), node_info=self.node.node_info ) ) # REVIEW: This code is invalid and will always throw. if isinstance(e, dbt.exceptions.Exception): if isinstance(e, dbt_common.exceptions.DbtRuntimeError): e.add_node(ctx.node) return e def before_execute(self) -> None: pass def after_execute(self, result) -> None: pass def compile(self, manifest: Manifest): return self.compiler.compile_node(self.node, manifest, {}, write=False) @abstractmethod def execute(self, compiled_node, manifest) -> SQLResult: pass @abstractmethod def from_run_result(self, result, start_time, timing_info) -> SQLResult: pass def error_result(self, node, error, start_time, timing_info): raise error def ephemeral_result(self, node, start_time, timing_info): raise dbt_common.exceptions.base.NotImplementedError( "cannot execute ephemeral nodes remotely!" ) class SqlCompileRunner(GenericSqlRunner[RemoteCompileResult]): def execute(self, compiled_node, manifest) -> RemoteCompileResult: return RemoteCompileResult( raw_code=compiled_node.raw_code, compiled_code=compiled_node.compiled_code, node=compiled_node, timing=[], # this will get added later generated_at=datetime.now(timezone.utc).replace(tzinfo=None), ) def from_run_result(self, result, start_time, timing_info) -> RemoteCompileResult: return RemoteCompileResult( raw_code=result.raw_code, compiled_code=result.compiled_code, node=result.node, timing=timing_info, generated_at=datetime.now(timezone.utc).replace(tzinfo=None), ) class SqlExecuteRunner(GenericSqlRunner[RemoteRunResult]): def execute(self, compiled_node, manifest) -> RemoteRunResult: _, execute_result = self.adapter.execute(compiled_node.compiled_code, fetch=True) table = ResultTable( column_names=list(execute_result.column_names), rows=[list(row) for row in execute_result], ) return RemoteRunResult( raw_code=compiled_node.raw_code, compiled_code=compiled_node.compiled_code, node=compiled_node, table=table, timing=[], generated_at=datetime.now(timezone.utc).replace(tzinfo=None), ) def from_run_result(self, result, start_time, timing_info) -> RemoteRunResult: return RemoteRunResult( raw_code=result.raw_code, compiled_code=result.compiled_code, node=result.node, table=result.table, timing=timing_info, generated_at=datetime.now(timezone.utc).replace(tzinfo=None), ) ================================================ FILE: core/dbt/task/test.py ================================================ import io import json import re import threading from dataclasses import dataclass from typing import ( TYPE_CHECKING, Any, Collection, Dict, List, Optional, Tuple, Type, Union, ) import daff from dbt.adapters.exceptions import MissingMaterializationError from dbt.artifacts.schemas.catalog import PrimitiveDict from dbt.artifacts.schemas.results import TestStatus from dbt.artifacts.schemas.run import RunResult from dbt.clients.jinja import MacroGenerator from dbt.context.providers import generate_runtime_model_context from dbt.contracts.graph.manifest import Manifest from dbt.contracts.graph.nodes import ( GenericTestNode, SingularTestNode, TestNode, UnitTestDefinition, UnitTestNode, ) from dbt.events.types import LogStartLine, LogTestResult from dbt.exceptions import BooleanError, DbtInternalError from dbt.flags import get_flags from dbt.graph import ResourceTypeSelector from dbt.node_types import TEST_NODE_TYPES, NodeType from dbt.parser.unit_tests import UnitTestManifestLoader from dbt.task import group_lookup from dbt.task.base import BaseRunner, resource_types_from_args from dbt.task.compile import CompileRunner from dbt.task.run import RunTask from dbt.utils import _coerce_decimal, strtobool from dbt_common.dataclass_schema import dbtClassMixin from dbt_common.events.format import pluralize from dbt_common.events.functions import fire_event from dbt_common.exceptions import DbtBaseException, DbtRuntimeError from dbt_common.ui import green, red if TYPE_CHECKING: import agate @dataclass class UnitTestDiff(dbtClassMixin): actual: List[Dict[str, Any]] expected: List[Dict[str, Any]] rendered: str @dataclass class TestResultData(dbtClassMixin): failures: int should_warn: bool should_error: bool adapter_response: Dict[str, Any] @classmethod def validate(cls, data): data["should_warn"] = cls.convert_bool_type(data["should_warn"]) data["should_error"] = cls.convert_bool_type(data["should_error"]) super().validate(data) def convert_bool_type(field) -> bool: # if it's type string let python decide if it's a valid value to convert to bool if isinstance(field, str): try: return bool(strtobool(field)) # type: ignore except ValueError: raise BooleanError(field, "get_test_sql") # need this so we catch both true bools and 0/1 return bool(field) @dataclass class UnitTestResultData(dbtClassMixin): should_error: bool adapter_response: Dict[str, Any] diff: Optional[UnitTestDiff] = None class TestRunner(CompileRunner): _ANSI_ESCAPE = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") def describe_node_name(self) -> str: if self.node.resource_type == NodeType.Unit: name = f"{self.node.model}::{self.node.versioned_name}" return name else: return self.node.name def describe_node(self) -> str: return f"{self.node.resource_type} {self.describe_node_name()}" def print_result_line(self, result): model = result.node group = group_lookup.get(model.unique_id) attached_node = ( result.node.attached_node if isinstance(result.node, GenericTestNode) else None ) fire_event( LogTestResult( name=self.describe_node_name(), status=str(result.status), index=self.node_index, num_models=self.num_nodes, execution_time=result.execution_time, node_info=model.node_info, num_failures=result.failures, group=group, attached_node=attached_node, ), level=LogTestResult.status_to_level(str(result.status)), ) def print_start_line(self): fire_event( LogStartLine( description=self.describe_node(), index=self.node_index, total=self.num_nodes, node_info=self.node.node_info, ) ) def before_execute(self) -> None: self.print_start_line() def execute_data_test(self, data_test: TestNode, manifest: Manifest) -> TestResultData: context = generate_runtime_model_context(data_test, self.config, manifest) hook_ctx = self.adapter.pre_model_hook(context["config"]) materialization_macro = manifest.find_materialization_macro_by_name( self.config.project_name, data_test.get_materialization(), self.adapter.type() ) if materialization_macro is None: raise MissingMaterializationError( materialization=data_test.get_materialization(), adapter_type=self.adapter.type() ) if "config" not in context: raise DbtInternalError( "Invalid materialization context generated, missing config: {}".format(context) ) # generate materialization macro macro_func = MacroGenerator(materialization_macro, context) try: # execute materialization macro macro_func() finally: self.adapter.post_model_hook(context, hook_ctx) # load results from context # could eventually be returned directly by materialization result = context["load_result"]("main") table = result["table"] num_rows = len(table.rows) if num_rows != 1: raise DbtInternalError( f"dbt internally failed to execute {data_test.unique_id}: " f"Returned {num_rows} rows, but expected " f"1 row" ) num_cols = len(table.columns) if num_cols != 3: raise DbtInternalError( f"dbt internally failed to execute {data_test.unique_id}: " f"Returned {num_cols} columns, but expected " f"3 columns" ) test_result_dct: PrimitiveDict = dict( zip( [column_name.lower() for column_name in table.column_names], map(_coerce_decimal, table.rows[0]), ) ) test_result_dct["adapter_response"] = result["response"].to_dict(omit_none=True) TestResultData.validate(test_result_dct) return TestResultData.from_dict(test_result_dct) def build_unit_test_manifest_from_test( self, unit_test_def: UnitTestDefinition, manifest: Manifest ) -> Manifest: # build a unit test manifest with only the test from this UnitTestDefinition loader = UnitTestManifestLoader(manifest, self.config, {unit_test_def.unique_id}) return loader.load() def execute_unit_test( self, unit_test_def: UnitTestDefinition, manifest: Manifest ) -> Tuple[UnitTestNode, UnitTestResultData]: unit_test_manifest = self.build_unit_test_manifest_from_test(unit_test_def, manifest) # The unit test node and definition have the same unique_id unit_test_node = unit_test_manifest.nodes[unit_test_def.unique_id] assert isinstance(unit_test_node, UnitTestNode) # Compile the node unit_test_node = self.compiler.compile_node(unit_test_node, unit_test_manifest, {}) assert isinstance(unit_test_node, UnitTestNode) # generate_runtime_unit_test_context not strictly needed - this is to run the 'unit' # materialization, not compile the node.compiled_code context = generate_runtime_model_context(unit_test_node, self.config, unit_test_manifest) hook_ctx = self.adapter.pre_model_hook(context["config"]) materialization_macro = unit_test_manifest.find_materialization_macro_by_name( self.config.project_name, unit_test_node.get_materialization(), self.adapter.type() ) if materialization_macro is None: raise MissingMaterializationError( materialization=unit_test_node.get_materialization(), adapter_type=self.adapter.type(), ) if "config" not in context: raise DbtInternalError( "Invalid materialization context generated, missing config: {}".format(context) ) # generate materialization macro macro_func = MacroGenerator(materialization_macro, context) try: # execute materialization macro macro_func() except DbtBaseException as e: raise DbtRuntimeError( f"An error occurred during execution of unit test '{unit_test_def.name}'. " f"There may be an error in the unit test definition: check the data types.\n {e}" ) finally: self.adapter.post_model_hook(context, hook_ctx) # load results from context # could eventually be returned directly by materialization result = context["load_result"]("main") adapter_response = result["response"].to_dict(omit_none=True) table = result["table"] actual = self._get_unit_test_agate_table(table, "actual") expected = self._get_unit_test_agate_table(table, "expected") # generate diff, if exists should_error, diff = False, None daff_diff = self._get_daff_diff(expected, actual) if daff_diff.hasDifference(): should_error = True rendered = self._render_daff_diff(daff_diff) rendered = f"\n\n{green('actual')} differs from {red('expected')}:\n\n{rendered}\n" diff = UnitTestDiff( actual=json_rows_from_table(actual), expected=json_rows_from_table(expected), rendered=rendered, ) unit_test_result_data = UnitTestResultData( diff=diff, should_error=should_error, adapter_response=adapter_response, ) return unit_test_node, unit_test_result_data def execute(self, test: Union[TestNode, UnitTestNode], manifest: Manifest): if isinstance(test, UnitTestDefinition): unit_test_node, unit_test_result = self.execute_unit_test(test, manifest) return self.build_unit_test_run_result(unit_test_node, unit_test_result) else: # Note: manifest here is a normal manifest assert isinstance(test, (SingularTestNode, GenericTestNode)) test_result = self.execute_data_test(test, manifest) return self.build_test_run_result(test, test_result) def build_test_run_result(self, test: TestNode, result: TestResultData) -> RunResult: severity = test.config.severity.upper() thread_id = threading.current_thread().name num_errors = pluralize(result.failures, "result") status = None message = None failures = 0 if severity == "ERROR" and result.should_error: status = TestStatus.Fail message = f"Got {num_errors}, configured to fail if {test.config.error_if}" failures = result.failures elif result.should_warn: if get_flags().WARN_ERROR or get_flags().WARN_ERROR_OPTIONS.includes( LogTestResult.__name__ ): status = TestStatus.Fail message = f"Got {num_errors}, configured to fail if {test.config.warn_if}" else: status = TestStatus.Warn message = f"Got {num_errors}, configured to warn if {test.config.warn_if}" failures = result.failures else: status = TestStatus.Pass run_result = RunResult( node=test, status=status, timing=[], thread_id=thread_id, execution_time=0, message=message, adapter_response=result.adapter_response, failures=failures, batch_results=None, ) return run_result def build_unit_test_run_result( self, test: UnitTestNode, result: UnitTestResultData ) -> RunResult: thread_id = threading.current_thread().name status = TestStatus.Pass message = None failures = 0 if result.should_error: status = TestStatus.Fail message = result.diff.rendered if result.diff else None failures = 1 return RunResult( node=test, status=status, timing=[], thread_id=thread_id, execution_time=0, message=message, adapter_response=result.adapter_response, failures=failures, batch_results=None, ) def after_execute(self, result) -> None: self.print_result_line(result) def _get_unit_test_agate_table(self, result_table, actual_or_expected: str): # lower case the column names as platforms like snowflake can sometimes return columns in uppercase result_table = result_table.rename([col.lower() for col in result_table.column_names]) unit_test_table = result_table.where( lambda row: row["actual_or_expected"] == actual_or_expected ) columns = list(unit_test_table.columns.keys()) columns.remove("actual_or_expected") return unit_test_table.select(columns) def _get_daff_diff( self, expected: "agate.Table", actual: "agate.Table", ordered: bool = False ) -> daff.TableDiff: # Sort expected and actual inputs prior to creating daff diff to ensure order insensitivity # https://github.com/paulfitz/daff/issues/200 expected_daff_table = daff.PythonTableView(list_rows_from_table(expected, sort=True)) actual_daff_table = daff.PythonTableView(list_rows_from_table(actual, sort=True)) flags = daff.CompareFlags() flags.ordered = ordered alignment = daff.Coopy.compareTables(expected_daff_table, actual_daff_table, flags).align() result = daff.PythonTableView([]) diff = daff.TableDiff(alignment, flags) diff.hilite(result) return diff def _render_daff_diff(self, daff_diff: daff.TableDiff) -> str: result = daff.PythonTableView([]) daff_diff.hilite(result) rendered = daff.TerminalDiffRender().render(result) # strip colors if necessary if not self.config.args.use_colors: rendered = self._ANSI_ESCAPE.sub("", rendered) return rendered class TestTask(RunTask): """ Testing: Read schema files + custom data tests and validate that constraints are satisfied. """ __test__ = False def raise_on_first_error(self) -> bool: return False @property def resource_types(self) -> List[NodeType]: resource_types: Collection[NodeType] = resource_types_from_args( self.args, set(TEST_NODE_TYPES), set(TEST_NODE_TYPES) ) # filter out any non-test node types resource_types = [rt for rt in resource_types if rt in TEST_NODE_TYPES] return list(resource_types) def get_node_selector(self) -> ResourceTypeSelector: if self.manifest is None or self.graph is None: raise DbtInternalError("manifest and graph must be set to get perform node selection") return ResourceTypeSelector( graph=self.graph, manifest=self.manifest, previous_state=self.previous_state, resource_types=self.resource_types, selectors=self.config.selectors, ) def get_runner_type(self, _) -> Optional[Type[BaseRunner]]: return TestRunner # This was originally in agate_helper, but that was moved out into dbt_common def json_rows_from_table(table: "agate.Table") -> List[Dict[str, Any]]: "Convert a table to a list of row dict objects" output = io.StringIO() table.to_json(path=output) # type: ignore return json.loads(output.getvalue()) # This was originally in agate_helper, but that was moved out into dbt_common def list_rows_from_table(table: "agate.Table", sort: bool = False) -> List[Any]: """ Convert given table to a list of lists, where the first element represents the header By default, sort is False and no sort order is applied to the non-header rows of the given table. If sort is True, sort the non-header rows hierarchically, treating None values as lower in order. Examples: * [['a','b','c'],[4,5,6],[1,2,3]] -> [['a','b','c'],[1,2,3],[4,5,6]] * [['a','b','c'],[4,5,6],[1,null,3]] -> [['a','b','c'],[1,null,3],[4,5,6]] * [['a','b','c'],[4,5,6],[null,2,3]] -> [['a','b','c'],[4,5,6],[null,2,3]] """ header = [col.name for col in table.columns] rows = [] for row in table.rows: rows.append(list(row.values())) if sort: rows = sorted(rows, key=lambda x: [(elem is None, elem) for elem in x]) return [header] + rows ================================================ FILE: core/dbt/tests/fixtures/__init__.py ================================================ # dbt.tests.fixtures directory ================================================ FILE: core/dbt/tests/fixtures/project.py ================================================ import os import random from argparse import Namespace from datetime import datetime, timezone from pathlib import Path from typing import Dict, Mapping import pytest # type: ignore import yaml import dbt.flags as flags from dbt.adapters.factory import ( get_adapter, get_adapter_by_type, register_adapter, reset_adapters, ) from dbt.config.runtime import RuntimeConfig from dbt.context.providers import generate_runtime_macro_context from dbt.deprecations import reset_deprecations from dbt.events.logging import setup_event_logger from dbt.mp_context import get_mp_context from dbt.parser.manifest import ManifestLoader from dbt.tests.util import ( TestProcessingException, get_connection, run_sql_with_adapter, write_file, ) from dbt_common.context import set_invocation_context from dbt_common.events.event_manager_client import cleanup_event_logger from dbt_common.exceptions import CompilationError, DbtDatabaseError from dbt_common.tests import enable_test_caching # These are the fixtures that are used in dbt core functional tests # # The main functional test fixture is the 'project' fixture, which combines # other fixtures, writes out a dbt project in a temporary directory, creates a temp # schema in the testing database, and returns a `TestProjInfo` object that # contains information from the other fixtures for convenience. # # The models, macros, seeds, snapshots, tests, and analyses fixtures all # represent directories in a dbt project, and are all dictionaries with # file name keys and file contents values. # # The other commonly used fixture is 'project_config_update'. Other # occasionally used fixtures are 'profiles_config_update', 'packages', # and 'selectors'. # # Most test cases have fairly small files which are best included in # the test case file itself as string variables, to make it easy to # understand what is happening in the test. Files which are used # in multiple test case files can be included in a common file, such as # files.py or fixtures.py. Large files, such as seed files, which would # just clutter the test file can be pulled in from 'data' subdirectories # in the test directory. # # Test logs are written in the 'logs' directory in the root of the repo. # Every test case writes to a log directory with the same 'prefix' as the # test's unique schema. # # These fixture have "class" scope. Class scope fixtures can be used both # in classes and in single test functions (which act as classes for this # purpose). Pytest will collect all classes starting with 'Test', so if # you have a class that you want to be subclassed, it's generally best to # not start the class name with 'Test'. All standalone functions starting with # 'test_' and methods in classes starting with 'test_' (in classes starting # with 'Test') will be collected. # # Please see the pytest docs for further information: # https://docs.pytest.org # Used in constructing the unique_schema and logs_dir @pytest.fixture(scope="class") def prefix(): # create a directory name that will be unique per test session _randint = random.randint(0, 9999) _runtime_timedelta = datetime.now(timezone.utc).replace(tzinfo=None) - datetime( 1970, 1, 1, 0, 0, 0 ) _runtime = (int(_runtime_timedelta.total_seconds() * 1e6)) + _runtime_timedelta.microseconds prefix = f"test{_runtime}{_randint:04}" return prefix # Every test has a unique schema @pytest.fixture(scope="class") def unique_schema(request, prefix) -> str: test_file = request.module.__name__ # We only want the last part of the name test_file = test_file.split(".")[-1] unique_schema = f"{prefix}_{test_file}" return unique_schema # Create a directory for the profile using tmpdir fixture @pytest.fixture(scope="class") def profiles_root(tmpdir_factory): return tmpdir_factory.mktemp("profile") # Create a directory for the project using tmpdir fixture @pytest.fixture(scope="class") def project_root(tmpdir_factory): # tmpdir docs - https://docs.pytest.org/en/6.2.x/tmpdir.html project_root = tmpdir_factory.mktemp("project") print(f"\n=== Test project_root: {project_root}") return project_root # This is for data used by multiple tests, in the 'tests/data' directory @pytest.fixture(scope="session") def shared_data_dir(request): return os.path.join(request.config.rootdir, "tests", "data") # This is for data for a specific test directory, i.e. tests/basic/data @pytest.fixture(scope="module") def test_data_dir(request): return os.path.join(request.fspath.dirname, "data") # This contains the profile target information, for simplicity in setting # up different profiles, particularly in the adapter repos. # Note: because we load the profile to create the adapter, this # fixture can't be used to test vars and env_vars or errors. The # profile must be written out after the test starts. @pytest.fixture(scope="class") def dbt_profile_target(): return { "type": "postgres", "threads": 4, "host": "localhost", "port": int(os.getenv("POSTGRES_TEST_PORT", 5432)), "user": os.getenv("POSTGRES_TEST_USER", "root"), "pass": os.getenv("POSTGRES_TEST_PASS", "password"), "dbname": os.getenv("POSTGRES_TEST_DATABASE", "dbt"), } @pytest.fixture(scope="class") def profile_user(dbt_profile_target): return dbt_profile_target["user"] # This fixture can be overridden in a project. The data provided in this # fixture will be merged into the default project dictionary via a python 'update'. @pytest.fixture(scope="class") def profiles_config_update(): return {} # The profile dictionary, used to write out profiles.yml. It will pull in updates # from two separate sources, the 'profile_target' and 'profiles_config_update'. # The second one is useful when using alternative targets, etc. @pytest.fixture(scope="class") def dbt_profile_data(unique_schema, dbt_profile_target, profiles_config_update): profile = { "test": { "outputs": { "default": {}, }, "target": "default", }, } target = dbt_profile_target target["schema"] = unique_schema profile["test"]["outputs"]["default"] = target if profiles_config_update: profile.update(profiles_config_update) return profile # Write out the profile data as a yaml file @pytest.fixture(scope="class") def profiles_yml(profiles_root, dbt_profile_data): os.environ["DBT_PROFILES_DIR"] = str(profiles_root) write_file(yaml.safe_dump(dbt_profile_data), profiles_root, "profiles.yml") yield dbt_profile_data del os.environ["DBT_PROFILES_DIR"] # Data used to update the dbt_project config data. @pytest.fixture(scope="class") def project_config_update(): return {} # Combines the project_config_update dictionary with project_config defaults to # produce a project_yml config and write it out as dbt_project.yml @pytest.fixture(scope="class") def dbt_project_yml(project_root, project_config_update, vars_yml): project_config = { "name": "test", "profile": "test", "flags": {"send_anonymous_usage_stats": False}, } if project_config_update: if isinstance(project_config_update, dict): project_config.update(project_config_update) elif isinstance(project_config_update, str): updates = yaml.safe_load(project_config_update) project_config.update(updates) write_file(yaml.safe_dump(project_config), project_root, "dbt_project.yml") return project_config # Fixture to provide dependencies @pytest.fixture(scope="class") def dependencies(): return {} # Write out the dependencies.yml file # Write out the packages.yml file @pytest.fixture(scope="class") def dependencies_yml(project_root, dependencies): if dependencies: if isinstance(dependencies, str): data = dependencies else: data = yaml.safe_dump(dependencies) write_file(data, project_root, "dependencies.yml") # Fixture to provide packages as either yaml or dictionary @pytest.fixture(scope="class") def packages(): return {} # Write out the packages.yml file @pytest.fixture(scope="class") def packages_yml(project_root, packages): if packages: if isinstance(packages, str): data = packages else: data = yaml.safe_dump(packages) write_file(data, project_root, "packages.yml") # Fixture to provide selectors as either yaml or dictionary @pytest.fixture(scope="class") def selectors(): return {} # Write out the selectors.yml file @pytest.fixture(scope="class") def selectors_yml(project_root, selectors): if selectors: if isinstance(selectors, str): data = selectors else: data = yaml.safe_dump(selectors) write_file(data, project_root, "selectors.yml") # Fixture to provide vars as either yaml or dictionary @pytest.fixture(scope="class") def vars_yml_update(): return {} # Write out the vars.yml file @pytest.fixture(scope="class") def vars_yml(project_root, vars_yml_update): if vars_yml_update: if isinstance(vars_yml_update, str): data = vars_yml_update else: data = yaml.safe_dump(vars_yml_update) write_file(data, project_root, "vars.yml") # This fixture ensures that the logging infrastructure does not accidentally # reuse streams configured on previous test runs, which might now be closed. # It should be run before (and so included as a parameter by) any other fixture # which runs dbt-core functions that might fire events. @pytest.fixture(scope="class") def clean_up_logging(): cleanup_event_logger() # This creates an adapter that is used for running test setup, such as creating # the test schema, and sql commands that are run in tests prior to the first # dbt command. After a dbt command is run, the project.adapter property will # return the current adapter (for this adapter type) from the adapter factory. # The adapter produced by this fixture will contain the "base" macros (not including # macros from dependencies). # # Anything used here must be actually working (dbt_project, profile, project and internal macros), # otherwise this will fail. So to test errors in those areas, you need to copy the files # into the project in the tests instead of putting them in the fixtures. @pytest.fixture(scope="class") def adapter( logs_dir, unique_schema, project_root, profiles_root, profiles_yml, clean_up_logging, dbt_project_yml, ): # The profiles.yml and dbt_project.yml should already be written out args = Namespace( profiles_dir=str(profiles_root), project_dir=str(project_root), target=None, profile=None, threads=None, ) flags.set_from_args(args, {}) runtime_config = RuntimeConfig.from_args(args) register_adapter(runtime_config, get_mp_context()) adapter = get_adapter(runtime_config) # We only need the base macros, not macros from dependencies, and don't want # to run 'dbt deps' here. manifest = ManifestLoader.load_macros( runtime_config, adapter.connections.set_query_header, base_macros_only=True, ) adapter.set_macro_resolver(manifest) adapter.set_macro_context_generator(generate_runtime_macro_context) yield adapter adapter.cleanup_connections() reset_adapters() # Start at directory level. def write_project_files(project_root, dir_name, file_dict): path = project_root.mkdir(dir_name) if file_dict: write_project_files_recursively(path, file_dict) # Write files out from file_dict. Can be nested directories... def write_project_files_recursively(path, file_dict): if type(file_dict) is not dict: raise TestProcessingException(f"File dict is not a dict: '{file_dict}' for path '{path}'") suffix_list = [".sql", ".csv", ".md", ".txt", ".py"] for name, value in file_dict.items(): if name.endswith(".yml") or name.endswith(".yaml"): if isinstance(value, str): data = value else: data = yaml.safe_dump(value) write_file(data, path, name) elif name.endswith(tuple(suffix_list)): write_file(value, path, name) else: write_project_files_recursively(path.mkdir(name), value) # models, macros, seeds, snapshots, tests, analyses # Provide a dictionary of file names to contents. Nested directories # are handle by nested dictionaries. # models directory @pytest.fixture(scope="class") def models(): return {} # macros directory @pytest.fixture(scope="class") def macros(): return {} # properties directory @pytest.fixture(scope="class") def properties(): return {} # seeds directory @pytest.fixture(scope="class") def seeds(): return {} # snapshots directory @pytest.fixture(scope="class") def snapshots(): return {} # tests directory @pytest.fixture(scope="class") def tests(): return {} # analyses directory @pytest.fixture(scope="class") def analyses(): return {} @pytest.fixture(scope="class") def functions() -> Dict[str, str]: return {} # Write out the files provided by models, macros, properties, snapshots, seeds, tests, analyses @pytest.fixture(scope="class") def project_files( project_root, models, macros, snapshots, properties, seeds, tests, analyses, functions, selectors_yml, dependencies_yml, packages_yml, vars_yml, dbt_project_yml, ): write_project_files(project_root, "models", {**models, **properties}) write_project_files(project_root, "macros", macros) write_project_files(project_root, "snapshots", snapshots) write_project_files(project_root, "seeds", seeds) write_project_files(project_root, "tests", tests) write_project_files(project_root, "analyses", analyses) write_project_files(project_root, "functions", functions) # We have a separate logs dir for every test @pytest.fixture(scope="class") def logs_dir(request, prefix): dbt_log_dir = os.path.join(request.config.rootdir, "logs", prefix) os.environ["DBT_LOG_PATH"] = str(dbt_log_dir) yield str(Path(dbt_log_dir)) del os.environ["DBT_LOG_PATH"] # This fixture is for customizing tests that need overrides in adapter # repos. Example in tests.functional.adapter.basic.test_base. @pytest.fixture(scope="class") def test_config(): return {} # This class is returned from the 'project' fixture, and contains information # from the pytest fixtures that may be needed in the test functions, including # a 'run_sql' method. class TestProjInfo: __test__ = False def __init__( self, project_root, profiles_dir, adapter_type, test_dir, shared_data_dir, test_data_dir, test_schema, database, test_config, ): self.project_root = project_root self.profiles_dir = profiles_dir self.adapter_type = adapter_type self.test_dir = test_dir self.shared_data_dir = shared_data_dir self.test_data_dir = test_data_dir self.test_schema = test_schema self.database = database self.test_config = test_config self.created_schemas = [] @property def adapter(self): # This returns the last created "adapter" from the adapter factory. Each # dbt command will create a new one. This allows us to avoid patching the # providers 'get_adapter' function. return get_adapter_by_type(self.adapter_type) # Run sql from a path def run_sql_file(self, sql_path, fetch=None): with open(sql_path, "r") as f: statements = f.read().split(";") for statement in statements: self.run_sql(statement, fetch) # Run sql from a string, using adapter saved at test startup def run_sql(self, sql, fetch=None): return run_sql_with_adapter(self.adapter, sql, fetch=fetch) # Create the unique test schema. Used in test setup, so that we're # ready for initial sql prior to a run_dbt command. def create_test_schema(self, schema_name=None): if schema_name is None: schema_name = self.test_schema with get_connection(self.adapter): relation = self.adapter.Relation.create(database=self.database, schema=schema_name) self.adapter.create_schema(relation) self.created_schemas.append(schema_name) # Drop the unique test schema, usually called in test cleanup def drop_test_schema(self): if self.adapter.get_macro_resolver() is None: manifest = ManifestLoader.load_macros( self.adapter.config, self.adapter.connections.set_query_header, base_macros_only=True, ) self.adapter.set_macro_resolver(manifest) with get_connection(self.adapter): for schema_name in self.created_schemas: relation = self.adapter.Relation.create(database=self.database, schema=schema_name) self.adapter.drop_schema(relation) self.created_schemas = [] # This return a dictionary of table names to 'view' or 'table' values. def get_tables_in_schema(self): sql = """ select table_name, case when table_type = 'BASE TABLE' then 'table' when table_type = 'VIEW' then 'view' else table_type end as materialization from information_schema.tables where {} order by table_name """ sql = sql.format("{} ilike '{}'".format("table_schema", self.test_schema)) result = self.run_sql(sql, fetch="all") return {model_name: materialization for (model_name, materialization) in result} @pytest.fixture(scope="class") def environment() -> Mapping[str, str]: # By default, fixture initialization is done with the following environment # from the os, but this fixture provides a way to customize the environment. return os.environ # Housekeeping that needs to be done before we start setting up any test fixtures. @pytest.fixture(scope="class") def initialization(environment) -> None: # Create an "invocation context," which dbt application code relies on. set_invocation_context(environment) # Enable caches used between test runs, for better testing performance. enable_test_caching() @pytest.fixture(scope="class") def project_setup( initialization, clean_up_logging, project_root, profiles_root, request, unique_schema, profiles_yml, adapter, shared_data_dir, test_data_dir, logs_dir, test_config, ): log_flags = Namespace( LOG_PATH=logs_dir, LOG_FORMAT="json", LOG_FORMAT_FILE="json", USE_COLORS=False, USE_COLORS_FILE=False, LOG_LEVEL="info", LOG_LEVEL_FILE="debug", DEBUG=False, LOG_CACHE_EVENTS=False, QUIET=False, LOG_FILE_MAX_BYTES=1000000, ) setup_event_logger(log_flags) orig_cwd = os.getcwd() os.chdir(project_root) # Return whatever is needed later in tests but can only come from fixtures, so we can keep # the signatures in the test signature to a minimum. project = TestProjInfo( project_root=project_root, profiles_dir=profiles_root, adapter_type=adapter.type(), test_dir=request.fspath.dirname, shared_data_dir=shared_data_dir, test_data_dir=test_data_dir, test_schema=unique_schema, database=adapter.config.credentials.database, test_config=test_config, ) project.drop_test_schema() project.create_test_schema() yield project # deps, debug and clean commands will not have an installed adapter when running and will raise # a KeyError here. Just pass for now. # See https://github.com/dbt-labs/dbt-core/issues/5041 # The debug command also results in an AttributeError since `Profile` doesn't have # a `load_dependencies` method. # Macros gets executed as part of drop_scheme in core/dbt/adapters/sql/impl.py. When # the macros have errors (which is what we're actually testing for...) they end up # throwing CompilationErrorss or DatabaseErrors try: project.drop_test_schema() except (KeyError, AttributeError, CompilationError, DbtDatabaseError): pass os.chdir(orig_cwd) cleanup_event_logger() reset_deprecations() # This is the main fixture that is used in all functional tests. It pulls in the other # fixtures that are necessary to set up a dbt project, and saves some of the information # in a TestProjInfo class, which it returns, so that individual test cases do not have # to pull in the other fixtures individually to access their information. # The order of arguments here determine which steps runs first. @pytest.fixture(scope="class") def project( project_setup: TestProjInfo, project_files, ): return project_setup ================================================ FILE: core/dbt/tests/util.py ================================================ import json import os import shutil from contextlib import contextmanager from contextvars import ContextVar, copy_context from datetime import datetime, timezone from io import StringIO from typing import Any, Callable, Dict, List, Optional from unittest import mock import pytz import yaml from dbt.adapters.base.relation import BaseRelation from dbt.adapters.factory import Adapter from dbt.cli.main import dbtRunner from dbt.contracts.graph.manifest import Manifest from dbt.materializations.incremental.microbatch import MicrobatchBuilder from dbt_common.context import _INVOCATION_CONTEXT_VAR, InvocationContext from dbt_common.events.base_types import EventLevel, EventMsg from dbt_common.events.functions import ( capture_stdout_logs, fire_event, reset_metadata_vars, stop_capture_stdout_logs, ) from dbt_common.events.types import Note # ============================================================================= # Test utilities # run_dbt # run_dbt_and_capture # get_manifest # copy_file # rm_file # write_file # read_file # mkdir # rm_dir # get_artifact # update_config_file # write_config_file # get_unique_ids_in_results # check_result_nodes_by_name # check_result_nodes_by_unique_id # SQL related utilities that use the adapter # run_sql_with_adapter # relation_from_name # check_relation_types (table/view) # check_relations_equal # check_relation_has_expected_schema # check_relations_equal_with_relations # check_table_does_exist # check_table_does_not_exist # get_relation_columns # update_rows # generate_update_clause # # Classes for comparing fields in dictionaries # AnyFloat # AnyInteger # AnyString # AnyStringWith # ============================================================================= # 'run_dbt' is used in pytest tests to run dbt commands. It will return # different objects depending on the command that is executed. # For a run command (and most other commands) it will return a list # of results. For the 'docs generate' command it returns a CatalogArtifact. # The first parameter is a list of dbt command line arguments, such as # run_dbt(["run", "--vars", "seed_name: base"]) # If the command is expected to fail, pass in "expect_pass=False"): # run_dbt(["test"], expect_pass=False) def run_dbt( args: Optional[List[str]] = None, expect_pass: bool = True, callbacks: Optional[List[Callable[[EventMsg], None]]] = None, ): # reset global vars reset_metadata_vars() if args is None: args = ["run"] print("\n\nInvoking dbt with {}".format(args)) from dbt.flags import get_flags flags = get_flags() project_dir = getattr(flags, "PROJECT_DIR", None) profiles_dir = getattr(flags, "PROFILES_DIR", None) if project_dir and "--project-dir" not in args: args.extend(["--project-dir", project_dir]) if profiles_dir and "--profiles-dir" not in args: args.extend(["--profiles-dir", profiles_dir]) dbt = dbtRunner(callbacks=callbacks) res = dbt.invoke(args) # the exception is immediately raised to be caught in tests # using a pattern like `with pytest.raises(SomeException):` if res.exception is not None: raise res.exception if expect_pass is not None: assert res.success == expect_pass, "dbt exit state did not match expected" return res.result # Use this if you need to capture the command logs in a test. # If you want the logs that are normally written to a file, you must # start with the "--debug" flag. The structured schema log CI test # will turn the logs into json, so you have to be prepared for that. def run_dbt_and_capture( args: Optional[List[str]] = None, expect_pass: bool = True, ): try: stringbuf = StringIO() capture_stdout_logs(stringbuf) res = run_dbt(args, expect_pass=expect_pass) stdout = stringbuf.getvalue() finally: stop_capture_stdout_logs() return res, stdout def get_logging_events(log_output, event_name): logging_events = [] for log_line in log_output.split("\n"): # skip empty lines if len(log_line) == 0: continue # The adapter logging also shows up, so skip non-json lines if not log_line.startswith("{"): continue if event_name in log_line: log_dct = json.loads(log_line) if log_dct["info"]["name"] == event_name: logging_events.append(log_dct) return logging_events # Used in test cases to get the manifest from the partial parsing file # Note: this uses an internal version of the manifest, and in the future # parts of it will not be supported for external use. def get_manifest(project_root) -> Optional[Manifest]: path = os.path.join(project_root, "target", "partial_parse.msgpack") if os.path.exists(path): with open(path, "rb") as fp: manifest_mp = fp.read() manifest: Manifest = Manifest.from_msgpack(manifest_mp) # type: ignore[attr-defined] return manifest else: return None # Used in test cases to get the run_results.json file. def get_run_results(project_root) -> Any: path = os.path.join(project_root, "target", "run_results.json") if os.path.exists(path): with open(path) as run_result_text: return json.load(run_result_text) else: return None # Used in tests to copy a file, usually from a data directory to the project directory def copy_file(src_path, src, dest_path, dest) -> None: # dest is a list, so that we can provide nested directories, like 'models' etc. # copy files from the data_dir to appropriate project directory shutil.copyfile( os.path.join(src_path, src), os.path.join(dest_path, *dest), ) # Used in tests when you want to remove a file from the project directory def rm_file(*paths) -> None: # remove files from proj_path os.remove(os.path.join(*paths)) # Used in tests to write out the string contents of a file to a # file in the project directory. # We need to explicitly use encoding="utf-8" because otherwise on # Windows we'll get codepage 1252 and things might break def write_file(contents, *paths): with open(os.path.join(*paths), "w", encoding="utf-8") as fp: fp.write(contents) def file_exists(*paths): """Check if file exists at path""" return os.path.exists(os.path.join(*paths)) # Used in test utilities def read_file(*paths): contents = "" with open(os.path.join(*paths), "r") as fp: contents = fp.read() return contents # To create a directory def mkdir(directory_path): try: os.makedirs(directory_path) except FileExistsError: raise FileExistsError(f"{directory_path} already exists.") # To remove a directory def rm_dir(directory_path): try: shutil.rmtree(directory_path) except FileNotFoundError: raise FileNotFoundError(f"{directory_path} does not exist.") def rename_dir(src_directory_path, dest_directory_path): os.rename(src_directory_path, dest_directory_path) # Get an artifact (usually from the target directory) such as # manifest.json or catalog.json to use in a test def get_artifact(*paths): contents = read_file(*paths) dct = json.loads(contents) return dct def write_artifact(dct, *paths): json_output = json.dumps(dct) write_file(json_output, *paths) # For updating yaml config files def update_config_file(updates, *paths): current_yaml = read_file(*paths) config = yaml.safe_load(current_yaml) config.update(updates) new_yaml = yaml.safe_dump(config) write_file(new_yaml, *paths) # Write new config file def write_config_file(data, *paths): if type(data) is dict: data = yaml.safe_dump(data) write_file(data, *paths) # Get the unique_ids in dbt command results def get_unique_ids_in_results(results): unique_ids = [] for result in results: unique_ids.append(result.node.unique_id) return unique_ids # Check the nodes in the results returned by a dbt run command def check_result_nodes_by_name(results, names): result_names = [] for result in results: result_names.append(result.node.name) assert set(names) == set(result_names) # Check the nodes in the results returned by a dbt run command def check_result_nodes_by_unique_id(results, unique_ids): result_unique_ids = [] for result in results: result_unique_ids.append(result.node.unique_id) assert set(unique_ids) == set(result_unique_ids) # Check datetime is between start and end/now def check_datetime_between(timestr, start, end=None): datefmt = "%Y-%m-%dT%H:%M:%S.%fZ" if end is None: end = datetime.now(timezone.utc).replace(tzinfo=None) parsed = datetime.strptime(timestr, datefmt) assert start <= parsed assert end >= parsed class TestProcessingException(Exception): pass # Testing utilities that use adapter code # Uses: # adapter.config.credentials # adapter.quote # adapter.run_sql_for_tests def run_sql_with_adapter(adapter, sql, fetch=None): if sql.strip() == "": return # substitute schema and database in sql kwargs = { "schema": adapter.config.credentials.schema, "database": adapter.quote(adapter.config.credentials.database), } sql = sql.format(**kwargs) msg = f'test connection "__test" executing: {sql}' fire_event(Note(msg=msg), level=EventLevel.DEBUG) with get_connection(adapter) as conn: return adapter.run_sql_for_tests(sql, fetch, conn) # Get a Relation object from the identifier (name of table/view). # Uses the default database and schema. If you need a relation # with a different schema, it should be constructed in the test. # Uses: # adapter.Relation # adapter.config.credentials # Relation.get_default_quote_policy # Relation.get_default_include_policy def relation_from_name(adapter, name: str): """reverse-engineer a relation from a given name and the adapter. The relation name is split by the '.' character. """ # Different adapters have different Relation classes cls = adapter.Relation credentials = adapter.config.credentials quote_policy = cls.get_default_quote_policy().to_dict() include_policy = cls.get_default_include_policy().to_dict() # Make sure we have database/schema/identifier parts, even if # only identifier was supplied. relation_parts = name.split(".") if len(relation_parts) == 1: relation_parts.insert(0, credentials.schema) if len(relation_parts) == 2: relation_parts.insert(0, credentials.database) kwargs = { "database": relation_parts[0], "schema": relation_parts[1], "identifier": relation_parts[2], } relation = cls.create( include_policy=include_policy, quote_policy=quote_policy, **kwargs, ) return relation # Ensure that models with different materialiations have the # current table/view. # Uses: # adapter.list_relations_without_caching def check_relation_types(adapter, relation_to_type): """ Relation name to table/view { "base": "table", "other": "view", } """ expected_relation_values = {} found_relations = [] schemas = set() for key, value in relation_to_type.items(): relation = relation_from_name(adapter, key) expected_relation_values[relation] = value schemas.add(relation.without_identifier()) with get_connection(adapter): for schema in schemas: found_relations.extend(adapter.list_relations_without_caching(schema)) for key, value in relation_to_type.items(): for relation in found_relations: # this might be too broad if relation.identifier == key: assert relation.type == value, ( f"Got an unexpected relation type of {relation.type} " f"for relation {key}, expected {value}" ) # Replaces assertTablesEqual. assertManyTablesEqual can be replaced # by doing a separate call for each set of tables/relations. # Wraps check_relations_equal_with_relations by creating relations # from the list of names passed in. def check_relations_equal(adapter, relation_names: List, compare_snapshot_cols=False): if len(relation_names) < 2: raise TestProcessingException( "Not enough relations to compare", ) relations = [relation_from_name(adapter, name) for name in relation_names] return check_relations_equal_with_relations( adapter, relations, compare_snapshot_cols=compare_snapshot_cols ) # Used to check that a particular relation has an expected schema # expected_schema should look like {"column_name": "expected datatype"} def check_relation_has_expected_schema(adapter, relation_name, expected_schema: Dict): relation = relation_from_name(adapter, relation_name) with get_connection(adapter): actual_columns = {c.name: c.data_type for c in adapter.get_columns_in_relation(relation)} assert ( actual_columns == expected_schema ), f"Actual schema did not match expected, actual: {json.dumps(actual_columns)}" # This can be used when checking relations in different schemas, by supplying # a list of relations. Called by 'check_relations_equal'. # Uses: # adapter.get_columns_in_relation # adapter.get_rows_different_sql # adapter.execute def check_relations_equal_with_relations( adapter: Adapter, relations: List, compare_snapshot_cols=False ): with get_connection(adapter): basis, compares = relations[0], relations[1:] # Skip columns starting with "dbt_" because we don't want to # compare those, since they are time sensitive # (unless comparing "dbt_" snapshot columns is explicitly enabled) column_names = [ c.name for c in adapter.get_columns_in_relation(basis) # type: ignore if not c.name.lower().startswith("dbt_") or compare_snapshot_cols ] for relation in compares: sql = adapter.get_rows_different_sql(basis, relation, column_names=column_names) # type: ignore _, tbl = adapter.execute(sql, fetch=True) num_rows = len(tbl) assert ( num_rows == 1 ), f"Invalid sql query from get_rows_different_sql: incorrect number of rows ({num_rows})" num_cols = len(tbl[0]) assert ( num_cols == 2 ), f"Invalid sql query from get_rows_different_sql: incorrect number of cols ({num_cols})" row_count_difference = tbl[0][0] assert ( row_count_difference == 0 ), f"Got {row_count_difference} difference in row count betwen {basis} and {relation}" rows_mismatched = tbl[0][1] assert ( rows_mismatched == 0 ), f"Got {rows_mismatched} different rows between {basis} and {relation}" # Uses: # adapter.update_column_sql # adapter.execute # adapter.commit_if_has_connection def update_rows(adapter, update_rows_config): """ { "name": "base", "dst_col": "some_date" "clause": { "type": "add_timestamp", "src_col": "some_date", "where" "id > 10" } """ for key in ["name", "dst_col", "clause"]: if key not in update_rows_config: raise TestProcessingException(f"Invalid update_rows: no {key}") clause = update_rows_config["clause"] clause = generate_update_clause(adapter, clause) where = None if "where" in update_rows_config: where = update_rows_config["where"] name = update_rows_config["name"] dst_col = update_rows_config["dst_col"] relation = relation_from_name(adapter, name) with get_connection(adapter): sql = adapter.update_column_sql( dst_name=str(relation), dst_column=dst_col, clause=clause, where_clause=where, ) adapter.execute(sql, auto_begin=True) adapter.commit_if_has_connection() # This is called by the 'update_rows' function. # Uses: # adapter.timestamp_add_sql # adapter.string_add_sql def generate_update_clause(adapter, clause) -> str: """ Called by update_rows function. Expects the "clause" dictionary documented in 'update_rows. """ if "type" not in clause or clause["type"] not in ["add_timestamp", "add_string"]: raise TestProcessingException("invalid update_rows clause: type missing or incorrect") clause_type = clause["type"] if clause_type == "add_timestamp": if "src_col" not in clause: raise TestProcessingException("Invalid update_rows clause: no src_col") add_to = clause["src_col"] kwargs = {k: v for k, v in clause.items() if k in ("interval", "number")} with get_connection(adapter): return adapter.timestamp_add_sql(add_to=add_to, **kwargs) elif clause_type == "add_string": for key in ["src_col", "value"]: if key not in clause: raise TestProcessingException(f"Invalid update_rows clause: no {key}") src_col = clause["src_col"] value = clause["value"] location = clause.get("location", "append") with get_connection(adapter): return adapter.string_add_sql(src_col, value, location) return "" @contextmanager def get_connection(adapter, name="_test"): with adapter.connection_named(name): conn = adapter.connections.get_thread_connection() yield conn # Uses: # adapter.get_columns_in_relation def get_relation_columns(adapter, name): relation = relation_from_name(adapter, name) with get_connection(adapter): columns = adapter.get_columns_in_relation(relation) return sorted(((c.name, c.dtype, c.char_size) for c in columns), key=lambda x: x[0]) def check_table_does_not_exist(adapter, name): columns = get_relation_columns(adapter, name) assert len(columns) == 0 def check_table_does_exist(adapter, name): columns = get_relation_columns(adapter, name) assert len(columns) > 0 # Utility classes for enabling comparison of dictionaries class AnyFloat: """Any float. Use this in assert calls""" def __eq__(self, other): return isinstance(other, float) class AnyInteger: """Any Integer. Use this in assert calls""" def __eq__(self, other): return isinstance(other, int) class AnyString: """Any string. Use this in assert calls""" def __eq__(self, other): return isinstance(other, str) class AnyStringWith: """AnyStringWith("AUTO")""" def __init__(self, contains=None): self.contains = contains def __eq__(self, other): if not isinstance(other, str): return False if self.contains is None: return True return self.contains in other def __repr__(self): return "AnyStringWith<{!r}>".format(self.contains) def assert_message_in_logs(message: str, logs: str, expected_pass: bool = True): # if the logs are json strings, then 'jsonify' the message because of things like escape quotes if os.environ.get("DBT_LOG_FORMAT", "") == "json": message = message.replace(r'"', r"\"") if expected_pass: assert message in logs else: assert message not in logs def get_project_config(project): file_yaml = read_file(project.project_root, "dbt_project.yml") return yaml.safe_load(file_yaml) def set_project_config(project, config): config_yaml = yaml.safe_dump(config) write_file(config_yaml, project.project_root, "dbt_project.yml") def get_model_file(project, relation: BaseRelation) -> str: return read_file(project.project_root, "models", f"{relation.name}.sql") def set_model_file(project, relation: BaseRelation, model_sql: str): write_file(model_sql, project.project_root, "models", f"{relation.name}.sql") def safe_set_invocation_context(): """In order to deal with a problem with the way the pytest runner interacts with ContextVars, this function provides a mechanism for setting the invocation context reliably, using its name rather than the reference variable, which may have been loaded in a separate context.""" invocation_var: Optional[ContextVar] = next( iter([cv for cv in copy_context() if cv.name == _INVOCATION_CONTEXT_VAR.name]), None ) if invocation_var is None: invocation_var = _INVOCATION_CONTEXT_VAR invocation_var.set(InvocationContext(os.environ)) def patch_microbatch_end_time(dt_str: str): dt = datetime.strptime(dt_str, "%Y-%m-%d %H:%M:%S").replace(tzinfo=pytz.UTC) return mock.patch.object(MicrobatchBuilder, "build_end_time", return_value=dt) ================================================ FILE: core/dbt/tracking.py ================================================ import os import platform import traceback import uuid from contextlib import contextmanager from datetime import datetime from typing import Optional import pytz import requests from packaging.version import Version from snowplow_tracker import Emitter, SelfDescribingJson, Subject, Tracker from snowplow_tracker import __version__ as snowplow_version # type: ignore from snowplow_tracker import logger as sp_logger from snowplow_tracker.events import StructuredEvent from dbt import version as dbt_version from dbt.adapters.exceptions import FailedToConnectError from dbt.clients.yaml_helper import safe_load, yaml # noqa:F401 from dbt.events.types import ( DisableTracking, FlushEvents, FlushEventsFailure, MainEncounteredError, SendEventFailure, SendingEvent, TrackingInitializeFailure, ) from dbt_common.events.base_types import EventMsg from dbt_common.events.functions import fire_event, get_invocation_id, msg_to_dict from dbt_common.exceptions import NotImplementedError sp_logger.setLevel(100) COLLECTOR_URL = "fishtownanalytics.sinter-collect.com" COLLECTOR_PROTOCOL = "https" DBT_INVOCATION_ENV = "DBT_INVOCATION_ENV" ADAPTER_INFO_SPEC = "iglu:com.dbt/adapter_info/jsonschema/1-0-1" DEPRECATION_WARN_SPEC = "iglu:com.dbt/deprecation_warn/jsonschema/1-0-0" BEHAVIOR_CHANGE_WARN_SPEC = "iglu:com.dbt/behavior_change_warn/jsonschema/1-0-0" EXPERIMENTAL_PARSER = "iglu:com.dbt/experimental_parser/jsonschema/1-0-0" INVOCATION_ENV_SPEC = "iglu:com.dbt/invocation_env/jsonschema/1-0-0" INVOCATION_SPEC = "iglu:com.dbt/invocation/jsonschema/1-0-2" LOAD_ALL_TIMING_SPEC = "iglu:com.dbt/load_all_timing/jsonschema/1-0-3" PACKAGE_INSTALL_SPEC = "iglu:com.dbt/package_install/jsonschema/1-0-0" PARTIAL_PARSER = "iglu:com.dbt/partial_parser/jsonschema/1-0-1" PLATFORM_SPEC = "iglu:com.dbt/platform/jsonschema/1-0-0" PROJECT_ID_SPEC = "iglu:com.dbt/project_id/jsonschema/1-0-1" RESOURCE_COUNTS = "iglu:com.dbt/resource_counts/jsonschema/1-0-1" RPC_REQUEST_SPEC = "iglu:com.dbt/rpc_request/jsonschema/1-0-1" RUNNABLE_TIMING = "iglu:com.dbt/runnable/jsonschema/1-0-0" RUN_MODEL_SPEC = "iglu:com.dbt/run_model/jsonschema/1-1-0" PLUGIN_GET_NODES = "iglu:com.dbt/plugin_get_nodes/jsonschema/1-0-0" ARTIFACT_UPLOAD = "iglu:com.dbt/artifact_upload/jsonschema/1-0-0" SNOWPLOW_TRACKER_VERSION = Version(snowplow_version) # workaround in case real snowplow tracker is in the env # the argument was renamed in https://github.com/snowplow/snowplow-python-tracker/commit/39fd50a3aff98a5efdd5c5c7fb5518fe4761305b INIT_KW_ARGS = ( {"buffer_size": 30} if SNOWPLOW_TRACKER_VERSION < Version("0.13.0") else {"batch_size": 30} ) class TimeoutEmitter(Emitter): def __init__(self) -> None: super().__init__( COLLECTOR_URL, protocol=COLLECTOR_PROTOCOL, on_failure=self.handle_failure, method="post", # don't set this. byte_limit=None, **INIT_KW_ARGS, ) @staticmethod def handle_failure(num_ok, unsent): # num_ok will always be 0, unsent will always be 1 entry long, because # the buffer is length 1, so not much to talk about fire_event(DisableTracking()) disable_tracking() def _log_request(self, request, payload): sp_logger.info(f"Sending {request} request to {self.endpoint}...") sp_logger.debug(f"Payload: {payload}") def _log_result(self, request, status_code): msg = f"{request} request finished with status code: {status_code}" if self.is_good_status_code(status_code): sp_logger.info(msg) else: sp_logger.warning(msg) def http_post(self, payload): self._log_request("POST", payload) r = requests.post( self.endpoint, data=payload, headers={"content-type": "application/json; charset=utf-8"}, timeout=5.0, ) self._log_result("GET", r.status_code) return r def http_get(self, payload): self._log_request("GET", payload) r = requests.get(self.endpoint, params=payload, timeout=5.0) self._log_result("GET", r.status_code) return r emitter = TimeoutEmitter() tracker = Tracker( emitters=emitter, namespace="cf", app_id="dbt", ) class User: def __init__(self, cookie_dir) -> None: self.do_not_track = True self.cookie_dir = cookie_dir self.id = None self.invocation_id = get_invocation_id() self.run_started_at = datetime.now(tz=pytz.utc) def state(self): return "do not track" if self.do_not_track else "tracking" @property def cookie_path(self): return os.path.join(self.cookie_dir, ".user.yml") def initialize(self): self.do_not_track = False cookie = self.get_cookie() self.id = cookie.get("id") subject = Subject() subject.set_user_id(self.id) tracker.set_subject(subject) def disable_tracking(self): self.do_not_track = True self.id = None self.cookie_dir = None tracker.set_subject(None) def set_cookie(self): # If the user points dbt to a profile directory which exists AND # contains a profiles.yml file, then we can set a cookie. If the # specified folder does not exist, or if there is not a profiles.yml # file in this folder, then an inconsistent cookie can be used. This # will change in every dbt invocation until the user points to a # profile dir file which contains a valid profiles.yml file. # # See: https://github.com/dbt-labs/dbt-core/issues/1645 user = {"id": str(uuid.uuid4())} cookie_path = os.path.abspath(self.cookie_dir) profiles_file = os.path.join(cookie_path, "profiles.yml") if os.path.exists(cookie_path) and os.path.exists(profiles_file): with open(self.cookie_path, "w") as fh: yaml.dump(user, fh) return user def get_cookie(self): if not os.path.isfile(self.cookie_path): user = self.set_cookie() else: with open(self.cookie_path, "r") as fh: try: user = safe_load(fh) if user is None: user = self.set_cookie() except yaml.reader.ReaderError: user = self.set_cookie() return user active_user: Optional[User] = None def get_platform_context(): data = { "platform": platform.platform(), "python": platform.python_version(), "python_version": platform.python_implementation(), } return SelfDescribingJson(PLATFORM_SPEC, data) def get_dbt_env_context(): default = "manual" dbt_invocation_env = os.getenv(DBT_INVOCATION_ENV, default) if dbt_invocation_env == "": dbt_invocation_env = default data = { "environment": dbt_invocation_env, } return SelfDescribingJson(INVOCATION_ENV_SPEC, data) def track(user, *args, **kwargs): if user.do_not_track: return fire_event(SendingEvent(kwargs=str(kwargs))) try: tracker.track(StructuredEvent(*args, **kwargs)) except Exception: fire_event(SendEventFailure()) def track_project_id(options): assert active_user is not None, "Cannot track project_id when active user is None" context = [SelfDescribingJson(PROJECT_ID_SPEC, options)] track( active_user, category="dbt", action="project_id", label=get_invocation_id(), context=context, ) def track_adapter_info(options): assert active_user is not None, "Cannot track adapter_info when active user is None" context = [SelfDescribingJson(ADAPTER_INFO_SPEC, options)] track( active_user, category="dbt", action="adapter_info", label=get_invocation_id(), context=context, ) def track_invocation_start(invocation_context): data = {"progress": "start", "result_type": None, "result": None} data.update(invocation_context) context = [ SelfDescribingJson(INVOCATION_SPEC, data), get_platform_context(), get_dbt_env_context(), ] track(active_user, category="dbt", action="invocation", label="start", context=context) def track_project_load(options): context = [SelfDescribingJson(LOAD_ALL_TIMING_SPEC, options)] assert active_user is not None, "Cannot track project loading time when active user is None" track( active_user, category="dbt", action="load_project", label=get_invocation_id(), context=context, ) def track_resource_counts(resource_counts): context = [SelfDescribingJson(RESOURCE_COUNTS, resource_counts)] assert active_user is not None, "Cannot track resource counts when active user is None" track( active_user, category="dbt", action="resource_counts", label=get_invocation_id(), context=context, ) def track_model_run(options): context = [SelfDescribingJson(RUN_MODEL_SPEC, options)] assert active_user is not None, "Cannot track model runs when active user is None" track( active_user, category="dbt", action="run_model", label=get_invocation_id(), context=context ) def track_rpc_request(options): context = [SelfDescribingJson(RPC_REQUEST_SPEC, options)] assert active_user is not None, "Cannot track rpc requests when active user is None" track( active_user, category="dbt", action="rpc_request", label=get_invocation_id(), context=context, ) def get_base_invocation_context(): assert ( active_user is not None ), "initialize active user before calling get_base_invocation_context" return { "project_id": None, "user_id": active_user.id, "invocation_id": active_user.invocation_id, "command": None, "options": None, "version": str(dbt_version.installed), "run_type": "regular", "adapter_type": None, "adapter_unique_id": None, } def track_package_install(command_name: str, project_hashed_name: Optional[str], options): assert active_user is not None, "Cannot track package installs when active user is None" invocation_data = get_base_invocation_context() invocation_data.update({"project_id": project_hashed_name, "command": command_name}) context = [ SelfDescribingJson(INVOCATION_SPEC, invocation_data), SelfDescribingJson(PACKAGE_INSTALL_SPEC, options), ] track( active_user, category="dbt", action="package", label=get_invocation_id(), property_="install", context=context, ) def track_deprecation_warn(options): assert active_user is not None, "Cannot track deprecation warnings when active user is None" context = [SelfDescribingJson(DEPRECATION_WARN_SPEC, options)] track( active_user, category="dbt", action="deprecation", label=get_invocation_id(), property_="warn", context=context, ) def track_behavior_change_warn(msg: EventMsg) -> None: if msg.info.name != "BehaviorChangeEvent" or active_user is None: return context = [SelfDescribingJson(BEHAVIOR_CHANGE_WARN_SPEC, msg_to_dict(msg))] track( active_user, category="dbt", action=msg.info.name, label=get_invocation_id(), context=context, ) def track_invocation_end(invocation_context, result_type=None): data = {"progress": "end", "result_type": result_type, "result": None} data.update(invocation_context) context = [ SelfDescribingJson(INVOCATION_SPEC, data), get_platform_context(), get_dbt_env_context(), ] assert active_user is not None, "Cannot track invocation end when active user is None" track(active_user, category="dbt", action="invocation", label="end", context=context) def track_invalid_invocation(args=None, result_type=None): assert active_user is not None, "Cannot track invalid invocations when active user is None" invocation_context = get_base_invocation_context() invocation_context.update({"command": args.which}) data = {"progress": "invalid", "result_type": result_type, "result": None} data.update(invocation_context) context = [ SelfDescribingJson(INVOCATION_SPEC, data), get_platform_context(), get_dbt_env_context(), ] track(active_user, category="dbt", action="invocation", label="invalid", context=context) def track_experimental_parser_sample(options): context = [SelfDescribingJson(EXPERIMENTAL_PARSER, options)] assert ( active_user is not None ), "Cannot track experimental parser info when active user is None" track( active_user, category="dbt", action="experimental_parser", label=get_invocation_id(), context=context, ) def track_partial_parser(options): context = [SelfDescribingJson(PARTIAL_PARSER, options)] assert active_user is not None, "Cannot track partial parser info when active user is None" track( active_user, category="dbt", action="partial_parser", label=get_invocation_id(), context=context, ) def track_plugin_get_nodes(options): context = [SelfDescribingJson(PLUGIN_GET_NODES, options)] assert active_user is not None, "Cannot track plugin node info when active user is None" track( active_user, category="dbt", action="plugin_get_nodes", label=get_invocation_id(), context=context, ) def track_runnable_timing(options): context = [SelfDescribingJson(RUNNABLE_TIMING, options)] assert active_user is not None, "Cannot track runnable info when active user is None" track( active_user, category="dbt", action="runnable_timing", label=get_invocation_id(), context=context, ) def track_artifact_upload(options): context = [SelfDescribingJson(ARTIFACT_UPLOAD, options)] assert active_user is not None, "Cannot track artifact upload when active user is None" track( active_user, category="dbt", action="artifact_upload", label=get_invocation_id(), context=context, ) def flush(): fire_event(FlushEvents()) try: tracker.flush() except Exception: fire_event(FlushEventsFailure()) def disable_tracking(): global active_user if active_user is not None: active_user.disable_tracking() else: active_user = User(None) def do_not_track(): global active_user active_user = User(None) def initialize_from_flags(send_anonymous_usage_stats, profiles_dir): global active_user if send_anonymous_usage_stats: active_user = User(profiles_dir) try: active_user.initialize() except Exception: fire_event(TrackingInitializeFailure(exc_info=traceback.format_exc())) active_user = User(None) else: active_user = User(None) @contextmanager def track_run(run_command=None): invocation_context = get_base_invocation_context() invocation_context["command"] = run_command track_invocation_start(invocation_context) try: yield track_invocation_end(invocation_context, result_type="ok") except (NotImplementedError, FailedToConnectError) as e: fire_event(MainEncounteredError(exc=str(e))) track_invocation_end(invocation_context, result_type="error") except Exception: track_invocation_end(invocation_context, result_type="error") raise finally: flush() ================================================ FILE: core/dbt/utils/__init__.py ================================================ # re-export utils from .utils import * # noqa: F403 from .utils import _coerce_decimal # noqa: F401 somewhere in the codebase we use this ================================================ FILE: core/dbt/utils/artifact_upload.py ================================================ import time import uuid import zipfile import requests import dbt.tracking from dbt._pydantic_shim import BaseSettings # type: ignore from dbt.config.runtime import UnsetProfile, load_project from dbt.constants import MANIFEST_FILE_NAME, RUN_RESULTS_FILE_NAME from dbt.events.types import ArtifactUploadSkipped, ArtifactUploadSuccess from dbt.exceptions import DbtProjectError from dbt_common.events.functions import fire_event from dbt_common.exceptions import DbtBaseException as DbtException MAX_RETRIES = 3 EXECUTION_ARTIFACTS = [MANIFEST_FILE_NAME, RUN_RESULTS_FILE_NAME] PRODUCED_ARTIFACTS_PATHS: set[str] = set() # artifact paths calling this will be uploaded to dbt Cloud def add_artifact_produced(artifact_path: str): PRODUCED_ARTIFACTS_PATHS.add(artifact_path) class ArtifactUploadConfig(BaseSettings): tenant_hostname: str DBT_CLOUD_TOKEN: str DBT_CLOUD_ACCOUNT_ID: str DBT_CLOUD_ENVIRONMENT_ID: str def get_ingest_url(self): return f"https://{self.tenant_hostname}/api/private/accounts/{self.DBT_CLOUD_ACCOUNT_ID}/environments/{self.DBT_CLOUD_ENVIRONMENT_ID}/ingests/" def get_complete_url(self, ingest_id): return f"{self.get_ingest_url()}{ingest_id}/" def get_headers(self, invocation_id=None): if invocation_id is None: invocation_id = str(uuid.uuid4()) return { "Accept": "application/json", "X-Invocation-Id": invocation_id, "Authorization": f"Token {self.DBT_CLOUD_TOKEN}", } def _retry_with_backoff(operation_name, func, max_retries=MAX_RETRIES, retry_codes=None): """Execute a function with exponential backoff retry logic. Args: operation_name: Name of the operation for error messages func: Function to execute that returns (success, result) max_retries: Maximum number of retry attempts Returns: The result from the function if successful Raises: DbtException: If all retry attempts fail """ if retry_codes is None: retry_codes = [500, 502, 503, 504] retry_delay = 1 for attempt in range(max_retries + 1): try: success, result = func() if success: return result if result.status_code not in retry_codes: raise DbtException(f"Error {operation_name}: {result}") if attempt == max_retries: # Last attempt raise DbtException(f"Error {operation_name}: {result}") except requests.RequestException as e: if attempt == max_retries: # Last attempt raise DbtException(f"Error {operation_name}: {str(e)}") time.sleep(retry_delay) retry_delay *= 2 # exponential backoff def upload_artifacts(project_dir, target_path, command): # Check if there are artifacts to upload for this command if not PRODUCED_ARTIFACTS_PATHS: fire_event(ArtifactUploadSkipped(msg="No artifacts to upload for current command")) return # read configurations try: project = load_project( project_dir, version_check=False, profile=UnsetProfile(), cli_vars=None ) if not project.dbt_cloud or "tenant_hostname" not in project.dbt_cloud: raise DbtProjectError("dbt_cloud.tenant_hostname not found in dbt_project.yml") tenant_hostname = project.dbt_cloud["tenant_hostname"] if not tenant_hostname: raise DbtProjectError("dbt_cloud.tenant_hostname is empty in dbt_project.yml") except Exception as e: raise DbtProjectError( f"Error reading dbt_cloud.tenant_hostname from dbt_project.yml: {str(e)}" ) config = ArtifactUploadConfig(tenant_hostname=tenant_hostname) if not target_path: target_path = "target" # Create zip file with artifacts zip_file_name = "target.zip" with zipfile.ZipFile(zip_file_name, "w") as z: for artifact_path in PRODUCED_ARTIFACTS_PATHS: z.write(artifact_path, artifact_path.split("/")[-1]) # Step 1: Create ingest request with retry def create_ingest(): response = requests.post(url=config.get_ingest_url(), headers=config.get_headers()) return response.status_code == 200, response response = _retry_with_backoff("creating ingest request", create_ingest) response_data = response.json() ingest_id = response_data["data"]["id"] upload_url = response_data["data"]["upload_url"] # Step 2: Upload the zip file to the provided URL with retry with open(zip_file_name, "rb") as f: file_data = f.read() def upload_file(): upload_response = requests.put(url=upload_url, data=file_data) return upload_response.status_code in (200, 204), upload_response _retry_with_backoff("uploading artifacts", upload_file) # Step 3: Mark the ingest as successful with retry def complete_ingest(): complete_response = requests.patch( url=config.get_complete_url(ingest_id), headers=config.get_headers(), json={"upload_status": "SUCCESS"}, ) return complete_response.status_code == 204, complete_response _retry_with_backoff("completing ingest", complete_ingest) fire_event(ArtifactUploadSuccess(msg=f"command {command} completed successfully")) if dbt.tracking.active_user is not None: dbt.tracking.track_artifact_upload({"command": command}) PRODUCED_ARTIFACTS_PATHS.clear() ================================================ FILE: core/dbt/utils/utils.py ================================================ import collections import decimal import functools import itertools import json import os import sys from datetime import date, datetime, time, timezone from enum import Enum from pathlib import PosixPath, WindowsPath from typing import ( AbstractSet, Any, Dict, Iterable, Iterator, List, Mapping, Optional, Sequence, Set, Tuple, Type, ) import jinja2 from dbt import flags from dbt.exceptions import DuplicateAliasError from dbt_common.exceptions import RecursionError from dbt_common.helper_types import WarnErrorOptionsV2 from dbt_common.utils import md5 DECIMALS: Tuple[Type[Any], ...] try: import cdecimal # typing: ignore except ImportError: DECIMALS = (decimal.Decimal,) else: DECIMALS = (decimal.Decimal, cdecimal.Decimal) class ExitCodes(int, Enum): Success = 0 ModelError = 1 UnhandledError = 2 def coalesce(*args): for arg in args: if arg is not None: return arg return None def get_profile_from_project(project): target_name = project.get("target", {}) profile = project.get("outputs", {}).get(target_name, {}) return profile def get_model_name_or_none(model): if model is None: name = "" elif isinstance(model, str): name = model elif isinstance(model, dict): name = model.get("alias", model.get("name")) elif hasattr(model, "alias"): name = model.alias elif hasattr(model, "name"): name = model.name else: name = str(model) return name def split_path(path): return path.split(os.sep) def get_pseudo_test_path(node_name, source_path): "schema tests all come from schema.yml files. fake a source sql file" source_path_parts = split_path(source_path) source_path_parts.pop() # ignore filename suffix = ["{}.sql".format(node_name)] pseudo_path_parts = source_path_parts + suffix return os.path.join(*pseudo_path_parts) def get_pseudo_hook_path(hook_name): path_parts = ["hooks", "{}.sql".format(hook_name)] return os.path.join(*path_parts) def get_hash(model): return md5(model.unique_id) def get_hashed_contents(model): return md5(model.raw_code) def flatten_nodes(dep_list): return list(itertools.chain.from_iterable(dep_list)) class memoized: """Decorator. Caches a function's return value each time it is called. If called later with the same arguments, the cached value is returned (not reevaluated). Taken from https://wiki.python.org/moin/PythonDecoratorLibrary#Memoize""" def __init__(self, func) -> None: self.func = func self.cache: Dict[Any, Any] = {} def __call__(self, *args): if not isinstance(args, collections.abc.Hashable): # uncacheable. a list, for instance. # better to not cache than blow up. return self.func(*args) if args in self.cache: return self.cache[args] value = self.func(*args) self.cache[args] = value return value def __repr__(self): """Return the function's docstring.""" return self.func.__doc__ def __get__(self, obj, objtype): """Support instance methods.""" return functools.partial(self.__call__, obj) def add_ephemeral_model_prefix(s: str) -> str: return "__dbt__cte__{}".format(s) def timestring() -> str: """Get the current datetime as an RFC 3339-compliant string""" # isoformat doesn't include the mandatory trailing 'Z' for UTC. return datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + "Z" def humanize_execution_time(execution_time: int) -> str: minutes, seconds = divmod(execution_time, 60) hours, minutes = divmod(minutes, 60) return f" in {int(hours)} hours {int(minutes)} minutes and {seconds:0.2f} seconds" class JSONEncoder(json.JSONEncoder): """A 'custom' json encoder that does normal json encoder things, but also handles `Decimal`s and `Undefined`s. Decimals can lose precision because they get converted to floats. Undefined's are serialized to an empty string """ def default(self, obj): if isinstance(obj, DECIMALS): return float(obj) elif isinstance(obj, (datetime, date, time)): return obj.isoformat() elif isinstance(obj, jinja2.Undefined): return "" elif isinstance(obj, Exception): return repr(obj) elif hasattr(obj, "to_dict"): # if we have a to_dict we should try to serialize the result of # that! return obj.to_dict(omit_none=True) else: return super().default(obj) class Translator: def __init__(self, aliases: Mapping[str, str], recursive: bool = False) -> None: self.aliases = aliases self.recursive = recursive def translate_mapping(self, kwargs: Mapping[str, Any]) -> Dict[str, Any]: result: Dict[str, Any] = {} for key, value in kwargs.items(): canonical_key = self.aliases.get(key, key) if canonical_key in result: raise DuplicateAliasError(kwargs, self.aliases, canonical_key) result[canonical_key] = self.translate_value(value) return result def translate_sequence(self, value: Sequence[Any]) -> List[Any]: return [self.translate_value(v) for v in value] def translate_value(self, value: Any) -> Any: if self.recursive: if isinstance(value, Mapping): return self.translate_mapping(value) elif isinstance(value, (list, tuple)): return self.translate_sequence(value) return value def translate(self, value: Mapping[str, Any]) -> Dict[str, Any]: try: return self.translate_mapping(value) except RuntimeError as exc: if "maximum recursion depth exceeded" in str(exc): raise RecursionError("Cycle detected in a value passed to translate!") raise def translate_aliases( kwargs: Dict[str, Any], aliases: Dict[str, str], recurse: bool = False, ) -> Dict[str, Any]: """Given a dict of keyword arguments and a dict mapping aliases to their canonical values, canonicalize the keys in the kwargs dict. If recurse is True, perform this operation recursively. :returns: A dict containing all the values in kwargs referenced by their canonical key. :raises: `AliasError`, if a canonical key is defined more than once. """ translator = Translator(aliases, recurse) return translator.translate(kwargs) # Note that this only affects hologram json validation. # It has no effect on mashumaro serialization. # Q: Can this be removed? def restrict_to(*restrictions): """Create the metadata for a restricted dataclass field""" return {"restrict": list(restrictions)} def coerce_dict_str(value: Any) -> Optional[Dict[str, Any]]: """For annoying mypy reasons, this helper makes dealing with nested dicts easier. You get either `None` if it's not a Dict[str, Any], or the Dict[str, Any] you expected (to pass it to dbtClassMixin.from_dict(...)). """ if isinstance(value, dict) and all(isinstance(k, str) for k in value): return value else: return None def _coerce_decimal(value): if isinstance(value, DECIMALS): return float(value) return value def fqn_search(root: Dict[str, Any], fqn: List[str]) -> Iterator[Dict[str, Any]]: """Iterate into a nested dictionary, looking for keys in the fqn as levels. Yield the level config. """ yield root for level in fqn: level_config = root.get(level, None) if not isinstance(level_config, dict): break # This used to do a 'deepcopy', # but it didn't seem to be necessary yield level_config root = level_config StringMap = Mapping[str, Any] StringMapList = List[StringMap] StringMapIter = Iterable[StringMap] class MultiDict(Mapping[str, Any]): """Implement the mapping protocol using a list of mappings. The most recently added mapping "wins". """ def __init__(self, sources: Optional[StringMapList] = None) -> None: super().__init__() self.sources: StringMapList if sources is None: self.sources = [] else: self.sources = sources def add_from(self, sources: StringMapIter): self.sources.extend(sources) def add(self, source: StringMap): self.sources.append(source) def _keyset(self) -> AbstractSet[str]: # return the set of keys keys: Set[str] = set() for entry in self._itersource(): keys.update(entry) return keys def _itersource(self) -> StringMapIter: return reversed(self.sources) def __iter__(self) -> Iterator[str]: # we need to avoid duplicate keys return iter(self._keyset()) def __len__(self): return len(self._keyset()) def __getitem__(self, name: str) -> Any: for entry in self._itersource(): if name in entry: return entry[name] raise KeyError(name) def __contains__(self, name) -> bool: return any((name in entry for entry in self._itersource())) # This is used to serialize the args in the run_results and in the logs. # We do this separately because there are a few fields that don't serialize, # i.e. PosixPath, WindowsPath, and types. It also includes args from both # cli args and flags, which is more complete than just the cli args. # If new args are added that are false by default (particularly in the # global options) they should be added to the 'default_false_keys' list. def args_to_dict(args): var_args = vars(args).copy() # update the args with the flags, which could also come from environment # variables or project_flags flag_dict = flags.get_flag_dict() var_args.update(flag_dict) dict_args = {} # remove args keys that clutter up the dictionary for key in var_args: if key.lower() in var_args and key == key.upper(): # skip all capped keys being introduced by Flags in dbt.cli.flags continue if key in ["cls", "mp_context"]: continue if var_args[key] is None: continue # TODO: add more default_false_keys default_false_keys = ( "debug", "full_refresh", "fail_fast", "warn_error", "single_threaded", "log_cache_events", "store_failures", "use_experimental_parser", ) default_empty_yaml_dict_keys = ("vars", "warn_error_options") if key in default_false_keys and var_args[key] is False: continue if key in default_empty_yaml_dict_keys and var_args[key] == "{}": continue # this was required for a test case if isinstance(var_args[key], PosixPath) or isinstance(var_args[key], WindowsPath): var_args[key] = str(var_args[key]) if isinstance(var_args[key], WarnErrorOptionsV2): var_args[key] = var_args[key].to_dict() dict_args[key] = var_args[key] return dict_args # Taken from https://github.com/python/cpython/blob/3.11/Lib/distutils/util.py # This is a copy of the function from distutils.util, which was removed in Python 3.12. def strtobool(val: str) -> bool: """Convert a string representation of truth to True or False. True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values are 'n', 'no', 'f', 'false', 'off', and '0'. Raises ValueError if 'val' is anything else. """ val = val.lower() if val in ("y", "yes", "t", "true", "on", "1"): return True elif val in ("n", "no", "f", "false", "off", "0"): return False else: raise ValueError("invalid truth value %r" % (val,)) def try_get_max_rss_kb() -> Optional[int]: """Attempts to get the high water mark for this process's memory use via the most reliable and accurate mechanism available through the host OS. Currently only implemented for Linux.""" if sys.platform == "linux" and os.path.isfile("/proc/self/status"): try: # On Linux, the most reliable documented mechanism for getting the RSS # high-water-mark comes from the line confusingly labeled VmHWM in the # /proc/self/status virtual file. with open("/proc/self/status") as f: for line in f: if line.startswith("VmHWM:"): return int(str.split(line)[1]) except Exception: pass return None ================================================ FILE: core/dbt/version.py ================================================ import glob import importlib import importlib.util import json import os from importlib import metadata as importlib_metadata from typing import Iterator, List, Optional, Tuple import requests import dbt_common.semver as semver from dbt.__version__ import version as __version_string from dbt_common.ui import green, yellow PYPI_VERSION_URL = "https://pypi.org/pypi/dbt-core/json" def get_version_information() -> str: installed = get_installed_version() latest = get_latest_version() core_msg_lines, core_info_msg = _get_core_msg_lines(installed, latest) core_msg = _format_core_msg(core_msg_lines) plugin_version_msg = _get_plugins_msg() msg_lines = [core_msg] if core_info_msg != "": msg_lines.append(core_info_msg) msg_lines.append(plugin_version_msg) msg_lines.append("") return "\n\n".join(msg_lines) def get_installed_version() -> semver.VersionSpecifier: return semver.VersionSpecifier.from_version_string(__version__) def get_latest_version( version_url: str = PYPI_VERSION_URL, ) -> Optional[semver.VersionSpecifier]: try: resp = requests.get(version_url, timeout=1) data = resp.json() version_string = data["info"]["version"] except (json.JSONDecodeError, KeyError, requests.RequestException): return None return semver.VersionSpecifier.from_version_string(version_string) def _get_core_msg_lines( installed: semver.VersionSpecifier, latest: Optional[semver.VersionSpecifier], ) -> Tuple[List[List[str]], str]: installed_s = installed.to_version_string(skip_matcher=True) installed_line = ["installed", installed_s, ""] update_info = "" if latest is None: update_info = ( " The latest version of dbt-core could not be determined!\n" " Make sure that the following URL is accessible:\n" f" {PYPI_VERSION_URL}" ) return [installed_line], update_info latest_s = latest.to_version_string(skip_matcher=True) latest_line = ["latest", latest_s, green("Up to date!")] if installed > latest: latest_line[2] = yellow("Ahead of latest version!") elif installed < latest: latest_line[2] = yellow("Update available!") update_info = ( " Your version of dbt-core is out of date!\n" " You can find instructions for upgrading here:\n" " https://docs.getdbt.com/docs/installation" ) return [ installed_line, latest_line, ], update_info def _format_core_msg(lines: List[List[str]]) -> str: msg = "Core:\n" msg_lines = [] for name, version, update_msg in _pad_lines(lines, seperator=":"): line_msg = f" - {name} {version}" if update_msg != "": line_msg += f" - {update_msg}" msg_lines.append(line_msg) return msg + "\n".join(msg_lines) def _get_plugins_msg() -> str: msg_lines = ["Plugins:"] plugins = [] display_update_msg = False for name, version_s in _get_dbt_plugins_info(): compatability_msg, needs_update = _get_plugin_msg_info(name, version_s, installed) if needs_update: display_update_msg = True plugins.append([name, version_s, compatability_msg]) for plugin in _pad_lines(plugins, seperator=":"): msg_lines.append(_format_single_plugin(plugin, "")) if display_update_msg: update_msg = ( " At least one plugin is out of date with dbt-core.\n" " You can find instructions for upgrading here:\n" " https://docs.getdbt.com/docs/installation" ) msg_lines += ["", update_msg] return "\n".join(msg_lines) def _get_plugin_msg_info( name: str, version_s: str, core: semver.VersionSpecifier ) -> Tuple[str, bool]: plugin = semver.VersionSpecifier.from_version_string(version_s) latest_plugin = get_latest_version(version_url=get_package_pypi_url(name)) needs_update = False if not latest_plugin: compatibility_msg = yellow("Could not determine latest version") return (compatibility_msg, needs_update) if plugin < latest_plugin: compatibility_msg = yellow("Update available!") needs_update = True elif plugin > latest_plugin: compatibility_msg = yellow("Ahead of latest version!") else: compatibility_msg = green("Up to date!") return (compatibility_msg, needs_update) def _format_single_plugin(plugin: List[str], update_msg: str) -> str: name, version_s, compatability_msg = plugin msg = f" - {name} {version_s} - {compatability_msg}" if update_msg != "": msg += f"\n{update_msg}\n" return msg def _pad_lines(lines: List[List[str]], seperator: str = "") -> List[List[str]]: if len(lines) == 0: return [] # count the max line length for each column in the line counter = [0] * len(lines[0]) for line in lines: for i, item in enumerate(line): counter[i] = max(counter[i], len(item)) result: List[List[str]] = [] for i, line in enumerate(lines): # add another list to hold padded strings if len(result) == i: result.append([""] * len(line)) # iterate over columns in the line for j, item in enumerate(line): # the last column does not need padding if j == len(line) - 1: result[i][j] = item continue # if the following column has no length # the string does not need padding if counter[j + 1] == 0: result[i][j] = item continue # only add the seperator to the first column offset = 0 if j == 0 and seperator != "": item += seperator offset = len(seperator) result[i][j] = item.ljust(counter[j] + offset) return result def get_package_pypi_url(package_name: str) -> str: return f"https://pypi.org/pypi/dbt-{package_name}/json" def _get_dbt_plugins_info() -> Iterator[Tuple[str, str]]: for plugin_name in _get_adapter_plugin_names(): if plugin_name == "core": continue try: mod = importlib.import_module(f"dbt.adapters.{plugin_name}.__version__") except ImportError: # not an adapter continue yield plugin_name, mod.version def _get_adapter_plugin_names() -> Iterator[str]: spec = importlib.util.find_spec("dbt.adapters") # If None, then nothing provides an importable 'dbt.adapters', so we will # not be reporting plugin versions today if spec is None or spec.submodule_search_locations is None: return for adapters_path in spec.submodule_search_locations: version_glob = os.path.join(adapters_path, "*", "__version__.py") for version_path in glob.glob(version_glob): # the path is like .../dbt/adapters/{plugin_name}/__version__.py # except it could be \\ on windows! plugin_root, _ = os.path.split(version_path) _, plugin_name = os.path.split(plugin_root) yield plugin_name def _resolve_version() -> str: try: return importlib_metadata.version("dbt-core") except importlib_metadata.PackageNotFoundError: # When running from source (not installed), use version from __version__.py return __version_string __version__ = _resolve_version() installed = get_installed_version() ================================================ FILE: core/hatch.toml ================================================ [version] path = "dbt/__version__.py" [build.targets.wheel] packages = ["dbt"] only-packages = true exclude = [ "**/*.md", ] artifacts = [ "dbt/include/**/*.py", "dbt/include/**/*.sql", "dbt/include/**/*.yml", "dbt/include/**/*.html", "dbt/include/**/*.md", "dbt/include/**/.gitkeep", "dbt/include/**/.gitignore", "dbt/task/docs/**/*.html", "dbt/jsonschemas/**/*.json", "dbt/py.typed", # Directories without __init__.py (namespace packages) "dbt/artifacts/resources/v1/**/*.py", "dbt/artifacts/utils/**/*.py", "dbt/event_time/**/*.py", "dbt/docs/source/**/*.py", "dbt/tests/util.py", ] [build.targets.sdist] include = [ "/dbt", "/README.md", ] [build.targets.sdist.force-include] "dbt/task/docs/index.html" = "dbt/task/docs/index.html" [envs.default] # Python 3.10-3.11 required locally due to flake8==4.0.1 compatibility # CI uses [envs.ci] which doesn't set python, allowing matrix testing python = "3.11" dependencies = [ # Git dependencies for development against main branches "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@main#subdirectory=dbt-adapters", "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git@main#subdirectory=dbt-tests-adapter", "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git@main", "dbt-postgres @ git+https://github.com/dbt-labs/dbt-adapters.git@main#subdirectory=dbt-postgres", # Code quality "pre-commit~=3.7.0", "black>=24.3,<25.0", "flake8==4.0.1", # requires python <3.12 "mypy==1.4.1", # update requires code fixes "isort==5.13.2", # Testing "pytest>=7.0,<8.0", "pytest-xdist~=3.6", "pytest-csv~=3.0", "pytest-cov", "pytest-dotenv", "pytest-ignore-test-results", "pytest-mock", "pytest-split", "pytest-logbook~=1.2", "logbook<1.9", "flaky", "freezegun>=1.5.1", "hypothesis", "mocker", # Debugging "ipdb", "ddtrace==2.21.3", # Documentation "docutils", "sphinx", # Type stubs "types-docutils", "types-PyYAML", "types-Jinja2", "types-jsonschema", "types-mock", "types-protobuf>=5.0,<6.0", "types-python-dateutil", "types-pytz", "types-requests", "types-setuptools", # Other "pip-tools", "protobuf>=6.0,<7.0", ] pre-install-commands = [ "pip install -e .", ] [envs.default.scripts] # Setup commands setup = [ "pre-commit install", ] dev-req = [ "pip install -e .", ] # Code quality commands code-quality = "pre-commit run --all-files" lint = [ "pre-commit run flake8-check --hook-stage manual --all-files", "pre-commit run mypy-check --hook-stage manual --all-files", ] flake8 = "pre-commit run flake8-check --hook-stage manual --all-files" mypy = "pre-commit run mypy-check --hook-stage manual --all-files" black = "pre-commit run black-check --hook-stage manual --all-files" # Testing commands unit-tests = "python -m pytest {args} ../tests/unit" integration-tests = "python -m pytest -nauto {args} ../tests/functional" integration-tests-fail-fast = "python -m pytest -x -nauto {args} ../tests/functional" test = [ "python -m pytest ../tests/unit", "pre-commit run black-check --hook-stage manual --all-files", "pre-commit run flake8-check --hook-stage manual --all-files", "pre-commit run mypy-check --hook-stage manual --all-files", ] # Database setup setup-db = [ "docker compose up -d database", "bash ../scripts/setup_db.sh", ] # Utility commands clean = [ "rm -f .coverage", "rm -f .coverage.*", "rm -rf .eggs/", "rm -rf build/", "rm -rf dbt.egg-info/", "rm -f dbt_project.yml", "rm -rf dist/", "find . -type f -name '*.pyc' -delete", "find . -type d -name __pycache__ -exec rm -rf {} +", ] json-schema = "python ../scripts/collect-artifact-schema.py --path ../schemas" [envs.build] python = "3.11" detached = true dependencies = [ "wheel", "twine", "check-wheel-contents", ] [envs.build.scripts] check-all = [ # Run check-wheel-contents first, before any installation overwrites dependencies "check-wheel-contents ./dist/*.whl --ignore W007,W008", "check-wheel", "check-sdist", ] check-wheel = [ "twine check ./dist/*", "find ./dist/dbt_core-*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=./dist/", "pip freeze | grep dbt-core", "dbt --version", ] check-sdist = [ "find ./dist/dbt_core-*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=./dist/", "pip freeze | grep dbt-core", "dbt --version", ] # CI environment - isolated environment with test dependencies [envs.ci] dependencies = [ # Git dependencies for development against main branches "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@main#subdirectory=dbt-adapters", "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git@main#subdirectory=dbt-tests-adapter", "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git@main", "dbt-postgres @ git+https://github.com/dbt-labs/dbt-adapters.git@main#subdirectory=dbt-postgres", # Testing "pre-commit~=3.7.0", "pytest>=7.0,<8.0", "pytest-cov", "pytest-xdist~=3.6", "pytest-csv~=3.0", "pytest-dotenv", "pytest-ignore-test-results", "pytest-mock", "pytest-split", "ddtrace==2.21.3", "flaky", "freezegun>=1.5.1", "hypothesis", ] pre-install-commands = [ "pip install -e .", ] [envs.ci.env-vars] DBT_TEST_USER_1 = "dbt_test_user_1" DBT_TEST_USER_2 = "dbt_test_user_2" DBT_TEST_USER_3 = "dbt_test_user_3" # Use new ddtrace pytest plugin to avoid deprecation warnings (old plugin removed in 3.0.0) DD_PYTEST_USE_NEW_PLUGIN_BETA = "true" [envs.ci.scripts] unit-tests = "python -m pytest --cov=dbt --cov-report=xml {args} ../tests/unit" code-quality = "pre-commit run --all-files" # pytest-split algorithm: we use 'least_duration' instead of the default 'duration_based_chunks' # because duration_based_chunks maintains alphabetical order and can create very uneven test # counts per group (e.g., 294 tests in group 1 vs 90 in others). Even with equal total duration, # more tests means more per-test overhead (fixture setup/teardown, DB connections, temp files) # which causes timeouts. least_duration distributes tests evenly across groups by count while # still balancing duration, avoiding the overhead problem. integration-tests = "python -m pytest --cov=dbt --cov-report=xml -nauto --durations-path .test_durations --splitting-algorithm least_duration {args} ../tests/functional" # Used by update-test-durations.yml to generate duration data (runs in parallel split groups) integration-tests-generate-durations = "python -m pytest -nauto --durations-path .test_durations --splitting-algorithm least_duration --store-durations {args} ../tests/functional" # Note: Python version matrix is handled by GitHub Actions CI, not hatch. # This avoids running tests 4x per job. The CI sets up the Python version # and hatch uses whatever Python is active. ================================================ FILE: core/pyproject.toml ================================================ [project] name = "dbt-core" dynamic = ["version"] description = "With dbt, data analysts and engineers can build analytics the way engineers build applications." readme = "README.md" requires-python = ">=3.10" license = "Apache-2.0" license-files = { globs = ["LICENSE"] } keywords = [] authors = [ { name = "dbt Labs", email = "info@dbtlabs.com" }, ] maintainers = [ { name = "dbt Labs", email = "info@dbtlabs.com" }, ] classifiers = [ "Development Status :: 5 - Production/Stable", "Operating System :: Microsoft :: Windows", "Operating System :: MacOS :: MacOS X", "Operating System :: POSIX :: Linux", "Programming Language :: Python", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", ] dependencies = [ # ---- # dbt-core uses these packages deeply, throughout the codebase, and there have been breaking changes in past patch releases (even though these are major-version-one). # Pin to the patch or minor version, and bump in each new minor version of dbt-core. "agate>=1.7.0,<1.10", "Jinja2>=3.1.3,<4", "mashumaro[msgpack]>=3.9,<3.15", # ---- # dbt-core uses these packages in standard ways. Pin to the major version, and check compatibility # with major versions in each new minor version of dbt-core. "click>=8.3.0,<9.0", "jsonschema>=4.19.1,<5.0", "networkx>=2.3,<4.0", "protobuf>=6.0,<7.0", "requests<3.0.0", # should match dbt-common "snowplow-tracker>=1.0.2,<2.0", # ---- # These packages are major-version-0. Keep upper bounds on upcoming minor versions (which could have breaking changes) # and check compatibility / bump in each new minor version of dbt-core. "pathspec>=0.9,<0.13", "sqlparse>=0.5.5,<0.6.0", # ---- # These are major-version-0 packages also maintained by dbt-labs. # Accept patches but avoid automatically updating past a set minor version range. "dbt-extractor>=0.5.0,<=0.6", "dbt-semantic-interfaces>=0.10.2,<0.11", # Minor versions for these are expected to be backwards-compatible "dbt-common>=1.37.3,<2.0", "dbt-adapters>=1.22.8,<2.0", "dbt-protos>=1.0.419,<2.0", "pydantic<3", # ---- # Expect compatibility with all new versions of these packages, so lower bounds only. "packaging>20.9", "pytz>=2015.7", "pyyaml>=6.0", "daff>=1.3.46", "typing-extensions>=4.4", ] [project.urls] Homepage = "https://github.com/dbt-labs/dbt-core" Repository = "https://github.com/dbt-labs/dbt-core.git" Issues = "https://github.com/dbt-labs/dbt-core/issues" Changelog = "https://github.com/dbt-labs/dbt-core/blob/main/CHANGELOG.md" [project.scripts] dbt = "dbt.cli.main:cli" [tool.hatch.version] path = "dbt/__version__.py" [build-system] requires = ["hatchling"] build-backend = "hatchling.build" ================================================ FILE: docker/Dockerfile ================================================ ARG py_version=3.11.2 FROM python:$py_version-slim-bullseye AS base RUN apt-get update \ # Install version-pinned packages for reproducible builds && apt-get install -y --no-install-recommends \ build-essential=12.9 \ ca-certificates=20210119 \ libpq-dev=13.23-0+deb11u1 \ make=4.3-4.1 \ openssh-client=1:8.4p1-5+deb11u3 \ software-properties-common=0.96.20.2-2.1 \ # Prevent dist-upgrade from changing pinned package versions && apt-mark hold \ build-essential \ ca-certificates \ libpq-dev \ make \ openssh-client \ software-properties-common \ # Apply security updates to non-pinned base image packages && apt-get dist-upgrade -y \ # Install temporary dependencies for building git from source (removed later) && apt-get install -y --no-install-recommends \ wget \ libcurl4-gnutls-dev \ libexpat1-dev \ libssl-dev \ zlib1g-dev \ # Build and install git from source to get 2.50.1 since debian only ships with 2.47 for now && wget https://github.com/git/git/archive/refs/tags/v2.50.1.tar.gz \ && tar -xf v2.50.1.tar.gz \ && cd git-2.50.1 \ && make prefix=/usr/local NO_TCLTK=1 NO_GETTEXT=1 all \ && make prefix=/usr/local NO_TCLTK=1 NO_GETTEXT=1 install \ && cd .. \ && rm -rf git-2.50.1 v2.50.1.tar.gz \ # Remove build dependencies to keep image slim && apt-get remove -y \ libcurl4-gnutls-dev \ libexpat1-dev \ libssl-dev \ wget \ zlib1g-dev \ && apt-get autoremove -y \ && apt-get clean \ && rm -rf \ /var/lib/apt/lists/* \ /tmp/* \ /var/tmp/* ENV PYTHONIOENCODING=utf-8 ENV LANG=C.UTF-8 RUN python -m pip install --upgrade "pip==24.0" "setuptools==69.2.0" "wheel==0.43.0" --no-cache-dir FROM base AS dbt-core ARG commit_ref=main HEALTHCHECK CMD dbt --version || exit 1 WORKDIR /usr/app/dbt/ ENTRYPOINT ["dbt"] RUN python -m pip install --no-cache-dir "dbt-core @ git+https://github.com/dbt-labs/dbt-core@${commit_ref}#subdirectory=core" FROM base AS dbt-postgres ARG commit_ref=main HEALTHCHECK CMD dbt --version || exit 1 WORKDIR /usr/app/dbt/ ENTRYPOINT ["dbt"] RUN python -m pip install --no-cache-dir "dbt-postgres @ git+https://github.com/dbt-labs/dbt-core@${commit_ref}#subdirectory=plugins/postgres" FROM dbt-core AS dbt-third-party ARG dbt_third_party RUN if [ "$dbt_third_party" ]; then \ python -m pip install --no-cache-dir "${dbt_third_party}"; \ else \ echo "No third party adapter provided"; \ fi \ ================================================ FILE: docker/README.md ================================================ # Docker for dbt This docker file is suitable for building dbt Docker images locally or using with CI/CD to automate populating a container registry. ## Building an image: This Dockerfile can create images for the following targets, each named after the database they support: * `dbt-core` _(no db-adapter support)_ * `dbt-third-party` _(requires additional build-arg)_ For platform-specific images, please refer to that platform's repository (eg. [dbt-postgres](https://github.com/dbt-labs/dbt-adapters/blob/main/dbt-postgres/docker/README.md)) In order to build a new image, run the following docker command. ``` docker build --tag --target ``` --- > **Note:** Docker must be configured to use [BuildKit](https://docs.docker.com/develop/develop-images/build_enhancements/) in order for images to build properly! --- By default the images will be populated with `dbt-core` on `main`. If you need to use a different version you can specify it by git ref (tag, branch, sha) using the `--build-arg` flag: ``` docker build --tag \ --target \ --build-arg commit_ref= \ ``` If you wish to build an image with a third-party adapter you can use the `dbt-third-party` target. This target requires you provide a path to the adapter that can be processed by `pip` by using the `dbt_third_party` build arg: ``` docker build --tag \ --target dbt-third-party \ --build-arg dbt_third_party= \ ``` This can also be combined with the `commit_ref` build arg to specify a version of `dbt-core`. ### Examples: To build an image named "my-third-party-dbt" that uses the latest release of [Materialize third party adapter](https://github.com/MaterializeInc/materialize/tree/main/misc/dbt-materialize) and the latest dev version of `dbt-core`: ``` cd dbt-core/docker docker build --tag my-third-party-dbt \ --target dbt-third-party \ --build-arg dbt_third_party=dbt-materialize \ . ``` ## Running an image in a container: The `ENTRYPOINT` for this Dockerfile is the command `dbt` so you can bind-mount your project to `/usr/app` and use dbt as normal: ``` docker run \ --network=host \ --mount type=bind,source=path/to/project,target=/usr/app \ --mount type=bind,source=path/to/profiles.yml,target=/root/.dbt/profiles.yml \ my-dbt \ ls ``` --- **Notes:** * Bind-mount sources _must_ be an absolute path * You may need to make adjustments to the docker networking setting depending on the specifics of your data warehouse/database host. --- ================================================ FILE: docker/test.sh ================================================ # - VERY rudimentary test script to run latest + specific branch image builds and test them all by running `--version` # TODO: create a real test suite clear \ && echo "\n\n"\ "###################################\n"\ "##### Testing dbt-core latest #####\n"\ "###################################\n"\ && docker build --tag dbt-core \ --target dbt-core \ docker \ && docker run dbt-core --version \ \ && echo "\n\n"\ "####################################\n"\ "##### Testing dbt-core-1.0.0b1 #####\n"\ "####################################\n"\ && docker build --tag dbt-core-1.0.0b1 \ --target dbt-core \ --build-arg dbt_core_ref=dbt-core@v1.0.0b1 \ docker \ && docker run dbt-core-1.0.0b1 --version \ \ && echo "\n\n"\ "#######################################\n"\ "##### Testing dbt-postgres latest #####\n"\ "#######################################\n"\ && docker build --tag dbt-postgres \ --target dbt-postgres \ docker \ && docker run dbt-postgres --version \ \ && echo "\n\n"\ "########################################\n"\ "##### Testing dbt-postgres-1.0.0b1 #####\n"\ "########################################\n"\ && docker build --tag dbt-postgres-1.0.0b1 \ --target dbt-postgres \ --build-arg dbt_postgres_ref=dbt-core@v1.0.0b1 \ docker \ && docker run dbt-postgres-1.0.0b1 --version \ \ && echo "\n\n"\ "#######################################\n"\ "##### Testing dbt-redshift latest #####\n"\ "#######################################\n"\ && docker build --tag dbt-redshift \ --target dbt-redshift \ docker \ && docker run dbt-redshift --version \ \ && echo "\n\n"\ "########################################\n"\ "##### Testing dbt-redshift-1.0.0b1 #####\n"\ "########################################\n"\ && docker build --tag dbt-redshift-1.0.0b1 \ --target dbt-redshift \ --build-arg dbt_redshift_ref=dbt-redshift@v1.0.0b1 \ docker \ && docker run dbt-redshift-1.0.0b1 --version \ \ && echo "\n\n"\ "#######################################\n"\ "##### Testing dbt-bigquery latest #####\n"\ "#######################################\n"\ && docker build --tag dbt-bigquery \ --target dbt-bigquery \ docker \ && docker run dbt-bigquery --version \ \ && echo "\n\n"\ "########################################\n"\ "##### Testing dbt-bigquery-1.0.0b1 #####\n"\ "########################################\n"\ && docker build --tag dbt-bigquery-1.0.0b1 \ --target dbt-bigquery \ --build-arg dbt_bigquery_ref=dbt-bigquery@v1.0.0b1 \ docker \ && docker run dbt-bigquery-1.0.0b1 --version \ \ && echo "\n\n"\ "########################################\n"\ "##### Testing dbt-snowflake latest #####\n"\ "########################################\n"\ && docker build --tag dbt-snowflake \ --target dbt-snowflake \ docker \ && docker run dbt-snowflake --version \ \ && echo "\n\n"\ "#########################################\n"\ "##### Testing dbt-snowflake-1.0.0b1 #####\n"\ "#########################################\n"\ && docker build --tag dbt-snowflake-1.0.0b1 \ --target dbt-snowflake\ --build-arg dbt_snowflake_ref=dbt-snowflake@v1.0.0b1 \ docker \ && docker run dbt-snowflake-1.0.0b1 --version \ \ && echo "\n\n"\ "####################################\n"\ "##### Testing dbt-spark latest #####\n"\ "####################################\n"\ && docker build --tag dbt-spark \ --target dbt-spark \ docker \ && docker run dbt-spark --version \ \ && echo "\n\n"\ "#####################################\n"\ "##### Testing dbt-spark-1.0.0rc2 ####\n"\ "#####################################\n"\ && docker build --tag dbt-spark-1.0.0rc2 \ --target dbt-spark \ --build-arg dbt_spark_ref=dbt-spark@v1.0.0rc2 \ docker \ && docker run dbt-spark-1.0.0rc2 --version \ \ && echo "\n\n"\ "###########################\n"\ "##### Testing dbt-all #####\n"\ "###########################\n"\ && docker build --tag dbt-all \ --target dbt-all \ docker \ && docker run dbt-all --version \ \ && echo "\n\n"\ "##########################################\n"\ "##### Testing third party db adapter #####\n"\ "##########################################\n"\ && docker build --tag dbt-materialize \ --target dbt-third-party \ --build-arg dbt_third_party="dbt-materialize" \ docker \ && docker run dbt-materialize --version ================================================ FILE: docker-compose.yml ================================================ ## # This compose file is used for local development and adapter testing only. # See `/docker` for a generic and production-ready docker file ## version: "3.5" services: database: image: postgres shm_size: 1gb environment: POSTGRES_USER: "root" POSTGRES_PASSWORD: "password" POSTGRES_DB: "dbt" ports: - "5432:5432" test: build: context: . dockerfile: Dockerfile.test args: # Run `make .env` to set $USER_ID and $GROUP_ID USER_ID: ${USER_ID:-} GROUP_ID: ${GROUP_ID:-} command: "bash -c 'cd core && hatch run ci:unit-tests'" environment: POSTGRES_TEST_HOST: "database" volumes: - .:/usr/app working_dir: /usr/app depends_on: - database ================================================ FILE: docs/arch/1_Overview.md ================================================ ================================================ FILE: docs/arch/2_CLI.md ================================================ # CLI Architecture ## Overview The CLI layer serves as the primary entry point for all dbt commands. It handles command-line parsing, configuration resolution, and orchestration of the execution lifecycle. The implementation is built on the [Click](https://click.palletsprojects.com/) framework, with custom extensions for dbt-specific needs like multi-source configuration and decorator-based dependency injection. dbt can be invoked in two ways: via the command line or programmatically. Command-line invocation flows through the `cli` click group defined in `main.py`. For programmatic use, the `dbtRunner` class provides a Python API that wraps click invocation and returns structured `dbtRunnerResult` objects containing success status, results, and any exceptions. ### Command Structure (`main.py`) Commands are defined as click-decorated functions under the root `cli` group. Each command follows a consistent pattern: it applies global flags via `@global_flags` (e.g., `--target`, `--debug`, `--fail-fast`, etc), command-specific options from the `params` module, and `@requires` decorators that build up execution context. The command body then instantiates the appropriate `Task` class for the command and calls `task.run()`. Nested command groups like `docs` and `source` contain subcommands (e.g., `docs generate`, `source freshness`). ### Parameters (`params.py`) All CLI options are defined in `params.py` as click option decorators. Each option specifies its flags, environment variable mapping, help text, type, and default value. Custom click types in `option_types.py` handle complex inputs like YAML strings (`--vars`, `--args`) and warn-error configurations. The `MultiOption` class in `options.py` allows options like `--select` to accept multiple space-separated values. ### Flags and Configuration (`flags.py`) The `Flags` dataclass is the primary configuration handler for running dbt. It consolidates configuration from multiple sources in priority order: CLI options take precedence over environment variables, which take precedence over project flags from `dbt_project.yml`, which take precedence over defaults. The `__init__` method walks the click context hierarchy to collect all parameter values, handling deprecated env vars and mutually exclusive options along the way. ### `@requires` Decorators (`requires.py`) The `requires` module provides decorators that progressively build up the execution context stored in Click's `ctx.obj`. The `preflight` decorator handles initialization: creating `Flags`, setting up logging, and initializing tracking. Resource decorators—`profile`, `project`, `runtime_config`, `manifest`, and `catalogs`—each load their respective configuration or artifact and add it to context. The `postflight` decorator wraps command execution with exception handling and fires completion events. ## Execution Flow When a command is invoked, execution flows through the decorator chain: `preflight` initializes flags and logging, then resource decorators (`profile` → `project` → `runtime_config` → `manifest`) progressively load configuration and add it to `ctx.obj`. The command body receives this fully-populated context, instantiates a `Task` with the flags, config, and manifest, then runs it. Finally, `postflight` handles any exceptions and emits completion events. ### Exception Handling In the `postflight` decorator, the click command is invoked (i.e. `func(*args, **kwargs)`) and wrapped in a `try/except` block to handle any exceptions thrown. Any exceptions thrown from `postflight` are wrapped by custom exceptions from the `dbt.cli.exceptions` module (i.e. `ResultExit`, `ExceptionExit`) to instruct click to complete execution with a particular exit code. Some `dbt-core` handled exceptions have an attribute named `results` which contains results from running nodes (e.g. `FailFastError`). These are wrapped in the `ResultExit` exception to represent runs that have failed in a way that `dbt-core` expects. If the invocation of the command does not throw any exceptions but does not succeed, `postflight` will still raise the `ResultExit` exception to make use of the exit code. These exceptions produce an exit code of `1`. Exceptions wrapped with `ExceptionExit` may be thrown by `dbt-core` intentionally (i.e. an exception that inherits from `dbt.exceptions.Exception`) or unintentionally (i.e. exceptions thrown by the python runtime). In either case these are considered errors that `dbt-core` did not expect and are treated as genuine exceptions. These exceptions produce an exit code of `2`. If no exceptions are thrown from invoking the command and the command succeeds, `postflight` will not raise any exceptions. When no exceptions are raised an exit code of `0` is produced. ### `dbtRunner` `dbtRunner` provides a programmatic interface for our click CLI and wraps the invocation of the click commands to handle any exceptions thrown. It is a feature available for users of dbt Core, but in some instances we also use it internally for testing. `dbtRunner.invoke` should ideally only ever return an instantiated `dbtRunnerResult` which contains the following fields: - `success`: A boolean representing whether the command invocation was successful[ - `result`: The optional result of the command invoked. This attribute can have many types, please see [the definition of `dbtRunnerResult` for more information](https://github.com/dbt-labs/dbt-core/blob/7634345985a86b113f51b74b9b776e346b59bdbf/core/dbt/cli/main.py#L23-L37) - `exception`: If an exception was thrown during command invocation it will be saved here, otherwise it will be `None`. Please note that the exceptions held in this attribute are not the exceptions thrown by `postflight` but instead the exceptions that `ResultExit` and `ExceptionExit` wrap Programmatic exception handling might look like the following: ```python from dbt.cli.main import dbtRunner, dbtRunnerResult # initialize dbt = dbtRunner() # create CLI args as a list of strings cli_args = ["run", "--select", "tag:my_tag"] # run the command res: dbtRunnerResult = dbt.invoke(cli_args) # inspect the results for r in res.result: print(f"{r.node.name}: {r.status}") ``` Reference: https://docs.getdbt.com/reference/programmatic-invocations ## Adding a New Command To add a new command: (1) define the command function in `main.py` with appropriate decorators, (2) add an entry to the `Command` enum in `types.py`, and (3) add the command to the `CMD_DICT` in `flags.py`'s `command_args` function. Every command needs at minimum the `@cli.command()` decorator, `@requires.postflight`, and `@requires.preflight`. ```python @cli.command("my-new-command") @requires.postflight @requires.preflight def my_new_command(ctx, **kwargs): ... ``` ================================================ FILE: docs/arch/3.1_Partial_Parsing.md ================================================ # Partial Parsing ## Overview Partial parsing improves parse performance by reusing the previous manifest and only re-parsing files that have changed. Instead of reading and processing every file on each invocation, dbt compares file checksums against the saved manifest and selectively updates only what's necessary. The saved manifest is stored in `target/partial_parse.msgpack`. On subsequent runs, if the state check passes and files can be diffed, dbt performs an incremental update rather than a full re-parse. ## State Check Before using a saved manifest, `ManifestLoader.is_partial_parsable()` validates that the environment hasn't changed in ways that would invalidate the cached parse results: - **Version match** — The saved manifest's dbt version must match the current version - **Vars hash** — CLI `--vars`, profile name, and target name are hashed and compared - **Profile hash** — Connection credentials info is hashed (changes could affect parsing) - **Project env vars** — Environment variables used in `dbt_project.yml` are tracked - **Project hashes** — Each project's `dbt_project.yml` content is hashed If any check fails, a full re-parse is triggered and the reason is logged. ## File Diff Detection The `PartialParsing` class compares the saved manifest's files against newly-read files: ``` prior run files ∩ current run files → check checksums → changed / unchanged prior run files - current run files → deleted current run files - prior run files → added ``` Schema files (YAML) are tracked separately because they require element-level diffing: a change to one model's config in a YAML file shouldn't require re-parsing unrelated models in the same file. ### When Partial Parsing Is Bypassed Partial parsing is skipped or abandoned in these cases: - `--no-partial-parse` flag is set - `partial_parse.msgpack` doesn't exist or can't be loaded - dbt version mismatch - Vars, profile, or project config changed - An error occurs during partial parse processing (falls back to full re-parse) - Special override macros were changed When bypassed, a full re-parse occurs and a new `partial_parse.msgpack` is written for the next run. ## Processing Changes `PartialParsing.get_parsing_files()` processes the file diff and returns a dictionary of files that need parsing: ### Added Files - Add the file to `saved_manifest.files` - Schedule for parsing ### Deleted Files - Remove associated nodes from the manifest - Schedule dependent nodes (children in the DAG) for re-parsing ### Changed Files - Remove old nodes from the manifest - Copy the new file content to saved files - Schedule for parsing - If the node had a schema patch, re-apply it ### Changed Schema Files For YAML files, `change_schema_file()` performs element-level diffing: - Compare saved vs. new YAML dictionaries key by key (models, sources, etc.) - For each key, identify added/deleted/changed elements by name - Only re-process affected elements, preserving unaffected nodes ## Special Macro Handling Certain macros have global effects that make incremental updates unsafe. Changes to these "special override macros" trigger a full re-parse: - `ref` - `source` - `config` - `generate_schema_name` - `generate_database_name` - `generate_alias_name` If a macro file containing any of these is changed or deleted, dbt abandons partial parsing and does a full re-parse. ## The `PartialParsing` Class Located in `partial.py`, this class encapsulates the partial parsing logic: **Constructor inputs:** - `saved_manifest` — The previously-saved manifest - `current run files` — Files read from the current project **Key methods:** - `skip_parsing()` — Returns `True` if no files changed (nothing to do) - `build_file_diff()` — Compares saved vs. new files, populates `file_diff` dict - `get_parsing_files()` — Processes the diff and returns files to parse - `delete_from_saved()` / `update_in_saved()` — Handle individual file changes - `change_schema_file()` — Handle YAML file changes with element-level granularity **Key attributes:** - `file_diff` — Dict with keys: `added`, `deleted`, `changed`, `unchanged`, `changed_schema_files`, `deleted_schema_files` - `project_parser_files` — Output dict mapping `project → parser → [file_ids]` ================================================ FILE: docs/arch/3.2_Deferral.md ================================================ # Deferral Deferral allows dbt to resolve `ref()` calls to objects that exist in a different environment (typically production) when the referenced model isn't being built in the current run. This is essential for CI workflows where you want to build and test only modified models without rebuilding the entire DAG. ## Overview When you run `dbt run --select my_model --defer --state prod-artifacts/`, dbt will: 1. Build `my_model` in your target schema 2. For any `ref()` to an unselected model, resolve to the production location instead of the target schema This enables "slim CI" workflows where only changed models are built, while still being able to reference their unchanged upstream dependencies in production. ## Key Flags | Flag | Description | |------|-------------| | `--defer` | Enable deferral. Resolve unselected refs to the state manifest. | | `--state PATH` | Path to directory containing production `manifest.json` | | `--defer-state PATH` | Override state path for deferral only (separate from `--state` used for `state:modified`) | | `--favor-state` | Always prefer deferred relations for unselected nodes, even if they exist locally | ## How Deferral is Set Up Deferral is configured during the execution phase, specifically in `GraphRunnableTask.before_run()`: ### 1. Load Previous State When `--state` or `--defer-state` is provided, dbt creates a `PreviousState` object that loads the production `manifest.json` from the specified path. This manifest is deserialized and version-checked to ensure compatibility with the current dbt version. ### 2. Merge Defer Relations Before execution begins, `GraphRunnableTask.before_run()` calls `defer_to_manifest()`. This method retrieves the deferred manifest (loaded from the state path) and merges it into the current manifest. ### 3. Attaching Defer Relations to Nodes The merge process iterates through every node in the production manifest. For each node that also exists in the current manifest and is a refable resource type (not ephemeral), dbt creates a `DeferRelation` object containing the production location metadata: database, schema, alias, and fully-qualified relation name. This `DeferRelation` is then attached to the corresponding node in the current manifest. After this merge, each eligible node carries information about where it exists in production, which can be used later during ref resolution if that node is not selected for execution. ## The `DeferRelation` Object `DeferRelation` (`core/dbt/artifacts/resources/v1/components.py`) is a lightweight object that captures the production location of a node: - `database`: Production database name - `schema`: Production schema name - `alias`: Production table/view name - `relation_name`: Fully-qualified relation name - `compiled_code`: The production compiled SQL (for reference) - `meta`, `tags`, `config`: Metadata from production ## How Refs are Resolved with Deferral See [Node Compilation — Deferred Ref Resolution](4.4_Node_Compilation.md#deferred-ref-resolution) for how `ref()` calls use the `defer_relation` during compilation. ## `--favor-state` Behavior By default, deferral only applies when the referenced relation doesn't exist in the target environment (checked via adapter cache lookup). With `--favor-state`: - Unselected nodes **always** resolve to their `defer_relation` - Even if the local relation exists, production is preferred - Useful when you want strict isolation from local state ## Use Cases ### Slim CI ```bash dbt run --select state:modified+ --defer --state prod-run-artifacts/ ``` Only build modified models and their descendants, referencing production for everything else. ### Development with Production Data ```bash dbt run --select my_new_model --defer --state prod-run-artifacts/ --favor-state ``` Develop a new model that references production tables, without rebuilding upstream models locally. ### Clone with Deferral ```bash dbt clone --select my_model --defer --state prod-run-artifacts/ ``` Clone specific models from production to a development schema. ================================================ FILE: docs/arch/3_Parsing.md ================================================ # Parsing ## Overview Parsing reads all files in the project and constructs an internal representation called the **Manifest**. The manifest contains all project resources models, tests, seeds, snapshots, sources, macros, docs, etc, and their relationships. Parsing captures dependencies (`ref()`, `source()`) and configuration, but does not compile SQL or execute anything against the database—those happen later during execution. The manifest is written to `target/manifest.json` for external tooling and for stateful dbt behavior such as `state:modified` selection and deferral. It is also serialized to msgpack to `target/partial_parse.msgpack` for reuse on subsequent runs in partial parsing. Note that the manifest produced by parsing is not complete: fields like `compiled_code` are populated later during compilation, and graph validation (e.g., cycle detection) happens after parsing. ## Entry Point The main entry point is `ManifestLoader.get_full_manifest()`, called from the `@requires.manifest` decorator. This method: 1. Loads project dependencies 2. Creates a `ManifestLoader` instance 3. Calls `loader.load()` to perform the actual parsing 4. Runs post-parse validation (`_check_manifest`—checks resource uniqueness, warns for unused configs) ## Parsing Phases The `ManifestLoader.load()` method orchestrates parsing in several phases: ### 1. Read Files Scan project directories and build `manifest.files`—a dictionary of file IDs to `SourceFile` objects containing paths, checksums, and metadata. This phase also checks for partial parsing opportunities (see [Partial Parsing](3.1_Partial_Parsing.md)). ### 2. Load Macros Macros must be parsed first because they're needed for Jinja rendering during subsequent parsing. Additonally Generic Tests also get parsed at this time due to their similarity to macros. `MacroParser` processes `.sql` files in macro directories, and `GenericTestParser` handles generic test definitions. After loading, `build_macro_resolver()` creates the `MacroResolver` for looking up macros by name. ### 3. Parse Project Files For each project (root + dependencies), run the appropriate parsers based on file type: - `ModelParser` — SQL/Python models - `SnapshotParser` — Snapshot definitions - `AnalysisParser` — Analysis files - `SingularTestParser` — Singular test SQL files - `SeedParser` — CSV seed files - `DocumentationParser` — Markdown docs - `HookParser` — on-run-start/end hooks from `dbt_project.yml` - `FixtureParser` — Unit test fixtures - `FunctionParser` — Function definitions ### 4. Parse Schema Files `SchemaParser` processes YAML files, extracting: - Model/seed/snapshot/analysis/function patches (descriptions, configs, columns) - Source definitions - Exposures, metrics, semantic models, saved queries - Generic tests attached to models/sources - Groups and unit tests ### 5. Patch Sources Note: The term 'patch' refers to the schema.yml contents corresponding to a particular resource in a dbt project. `SourcePatcher.construct_sources()` converts unparsed source definitions into `SourceDefinition` nodes, applying any source patches (overrides). ### 6. Process References Resolve symbolic references captured during parsing: - `process_refs()` — Look up `ref()` targets and populate `depends_on.nodes` - `process_sources()` — Look up `source()` targets - `process_docs()` — Render `{{ doc() }}` blocks in descriptions - `process_metrics()` — Resolve metric dependencies ### 7. Validation - Check resource uniqueness (no duplicate names/aliases) - Validate group and access configurations - Validate snapshot and microbatch configs ## Key Classes ### `ManifestLoader` (`manifest.py`) Orchestrates the entire parsing process. Key attributes: - `root_project` / `all_projects` — Project configurations - `manifest` — The manifest being built - `saved_manifest` — Previous manifest for partial parsing - `partial_parser` — `PartialParsing` instance if doing incremental parse ### Parser Hierarchy (`base.py`) - `BaseParser` — Abstract base with `parse_file()` method and `resource_type` property - `Parser` — Adds `root_project` reference for cross-project parsing - `ConfiguredParser` — Handles config resolution, FQN generation, and relation name updates (database/schema/alias) - `SimpleSQLParser` — Convenience class for straightforward SQL file parsing Each parser reads `FileBlock` objects and adds parsed nodes to the manifest via `manifest.add_node()`. ### `SchemaParser` (`schemas.py`) The most complex parser: handles YAML property files with multiple sub-parsers for different top-level keys (models, sources, exposures, etc.). Produces both nodes and "patches" that are applied to nodes parsed from SQL files. ## Parsing vs. Compilation vs. Runtime See `docs/guides/parsing-vs-compilation-vs-runtime.md` for a detailed explanation of these distinctions. - **Parsing** — Read files, construct manifest, capture `ref()`/`source()`/`config()` calls. No database connection required. - **Compilation** — Render Jinja with `execute=True`, run introspective queries (requiring an adapter / warehouse connection), populate `compiled_code`. Happens at runtime, in DAG order. - **Runtime** — Execute materializations, run tests, apply DDL/DML to the database. ## Partial Parsing To improve performance on subsequent invocations, dbt can reuse the previous manifest and only re-parse files that have changed. This is controlled by the `--partial-parse` flag (enabled by default). See [Partial Parsing](3.1_Partial_Parsing.md) ================================================ FILE: docs/arch/4.1_Task_Framework.md ================================================ # Task Framework The `Task` framework provides the execution layer that coordinates running dbt commands. It establishes a layered architecture where `Task`s handle command-level orchestration, `Runner`s handle per-node execution, and `Selector`s manage graph-based node selection and queuing. ## Task Hierarchy Tasks are organized in a class hierarchy that adds capabilities at each level. The base classes live in [`core/dbt/task/base.py`](https://github.com/dbt-labs/dbt-core/blob/main/core/dbt/task/base.py) and [`core/dbt/task/runnable.py`](https://github.com/dbt-labs/dbt-core/blob/main/core/dbt/task/runnable.py). ``` BaseTask ├── CleanTask ├── DebugTask ├── DepsTask ├── InitTask └── ConfiguredTask ├── RetryTask ├── RunOperationTask ├── ServeTask └── GraphRunnableTask ├── CloneTask ├── ListTask └── CompileTask ├── GenerateTask ├── ShowTask └── RunTask ├── BuildTask ├── FreshnessTask ├── SeedTask ├── SnapshotTask └── TestTask ``` ### BaseTask The abstract base class for all tasks. It holds `args: Flags` and defines the `run()` abstract method that each task must implement. Simple commands like `dbt clean`, `dbt debug`, `dbt deps`, and `dbt init` inherit directly from `BaseTask` since they don't require project configuration or manifest access. ### ConfiguredTask Extends `BaseTask` with project awareness. Adds `config: RuntimeConfig`, `manifest: Manifest`, and a `Compiler` instance. The `compile_manifest()` method invokes graph compilation (see [Graph Compilation](4.2_Graph_Compilation.md)), producing a `Graph` object. Commands that need project context but don't execute nodes against the graph (like `dbt run-operation`) inherit from this level. ### GraphRunnableTask The main workhorse for DAG-based execution. Manages the full execution lifecycle: node selection, thread pool management, and result collection. Key methods include: - `_runtime_initialize()`: Compiles the manifest and builds the `GraphQueue` via `get_graph_queue()` - `get_node_selector()`: Returns a `NodeSelector` (or subclass) for resolving `--select`/`--exclude` arguments - `execute_nodes()`: Creates a thread pool and drives `run_queue()` - `run_queue()`: Pulls nodes from the queue and dispatches them to `call_runner()` - `call_runner()`: Invokes the Runner for a single node, handles results and error propagation The `run()` method orchestrates the full flow: `_runtime_initialize()` → `execute_with_hooks()` → result writing. ### CompileTask Specializes `GraphRunnableTask` for compilation-focused commands. Sets `raise_on_first_error()` to `True` and uses `CompileRunner` for all nodes. The `get_node_selector()` returns a `ResourceTypeSelector` filtering to executable node types. `CompileTask` also supports inline SQL compilation via the `--inline` flag (e.g., `dbt compile --inline "select * from {{ ref('my_model') }}"`). This allows users to compile arbitrary SQL strings that use dbt Jinja functions like `ref()` and `source()` without needing a corresponding model file. The inline SQL is parsed into a temporary `SqlOperation` node, added to the manifest, compiled, and then removed after execution. ### RunTask and BuildTask `RunTask` extends `CompileTask` for model execution, adding hook execution (`on-run-start`, `on-run-end`), schema creation, and model-specific result tracking. It uses `ModelRunner` for models and handles microbatch incremental models specially. `BuildTask` extends `RunTask` to process multiple resource types in a single invocation. It maintains a `RUNNER_MAP` dispatching each `NodeType` to the appropriate Runner, and overrides `compile_manifest()` to add test edges to the graph (see [Graph Compilation](4.2_Graph_Compilation.md)). ## Runner Hierarchy Runners handle per-node execution. The base class lives in `core/dbt/task/base.py`, with specialized runners in their respective task modules. ``` BaseRunner ├── CloneRunner ├── FreshnessRunner ├── SavedQueryRunner └── CompileRunner ├── ShowRunner ├── TestRunner ├── GenericSqlRunner │ ├── SqlCompileRunner │ └── SqlExecuteRunner └── ModelRunner ├── SeedRunner └── SnapshotRunner ``` ### BaseRunner Abstract base for per-node execution. Holds references to `config`, `adapter`, `node`, and a `Compiler` instance. Key methods: - `run_with_hooks()`: Entry point called by the Task. Invokes `before_execute()`, `safe_run()`, `after_execute()` - `safe_run()`: Wraps `compile_and_execute()` with error handling and connection management - `compile_and_execute()`: The core two-phase execution—compiles the node, then executes it (for non-ephemeral nodes) - `compile()`: Abstract method implemented by subclasses to compile the node - `execute()`: Abstract method implemented by subclasses to execute the compiled node ### CompileRunner Basic compilation runner. The `compile()` method calls `compiler.compile_node()` (see [Node Compilation](4.4_Node_Compilation.md)). The `execute()` method simply returns a success result without running SQL and is used for `dbt compile`. ### ModelRunner Executes models by invoking materialization macros. The `execute()` method generates a runtime context, looks up the appropriate materialization macro, and invokes it via `MacroGenerator`. The materialization macro handles all adapter interactions (see [Node Materialization](4.5_Node_Materialization.md)). ### TestRunner Executes tests (data tests and unit tests). For data tests, it runs the test SQL and interprets row counts as pass/fail. For unit tests, it sets up fixtures, runs the model, and compares actual vs expected results. ## Node Selection Node selection determines which nodes from the manifest will be executed. The selector framework lives in `core/dbt/graph/`. ### NodeSelector The main class for resolving selection specs against the compiled graph. Given a `SelectionSpec` (parsed from `--select`/`--exclude` arguments), it traverses the graph to find matching nodes, applies graph operators (`+`, `@`), and handles indirect selection for tests. The `get_selected()` method returns the final set of unique IDs. See [Node Selection](4.3_Node_Selection.md) for details on selection syntax and methods. ### ResourceTypeSelector Extends `NodeSelector` to filter by `NodeType`. Used by most commands to restrict selection to relevant resource types (e.g., `RunTask` selects only models, `TestTask` selects only tests). ### GraphQueue A thread-safe priority queue backed by the dependency graph. Nodes are scored by topological depth (lower depth = higher priority). The queue tracks in-progress nodes and releases downstream nodes only when their dependencies complete via `mark_done()`. This ensures correct execution order while maximizing parallelism. ## Execution Flow The typical execution flow for a graph-based command: 1. **Task.run()**: Entry point, sets up context 2. **_runtime_initialize()**: Calls `compile_manifest()` to build the `Graph`, then `get_graph_queue()` for node selection 3. **execute_with_hooks()**: Gets adapter, calls `before_run()`, `execute_nodes()`, `after_run()` 4. **execute_nodes()**: Creates `DbtThreadPool`, calls `run_queue()` 5. **run_queue()**: Loop pulling nodes from `GraphQueue`, dispatching to thread pool via `call_runner()` 6. **call_runner()**: Creates Runner via `get_runner()`, invokes `runner.run_with_hooks()` 7. **Runner.run_with_hooks()** → **safe_run()** → **compile_and_execute()**: Compiles node, executes it, returns `RunResult` 8. **_handle_result()**: Records result, marks dependent nodes as skipped if needed 9. **GraphQueue.mark_done()**: Releases downstream nodes Results are collected in `node_results` and written to `run_results.json` at the end of execution. ================================================ FILE: docs/arch/4.2_Graph_Compilation.md ================================================ # Graph Compilation ## Overview Graph compilation transforms the parsed Manifest into an executable dependency graph. This happens in `Compiler.compile()` ([`core/dbt/compilation.py`](https://github.com/dbt-labs/dbt-core/blob/main/core/dbt/compilation.py)) and produces a `Graph` object wrapping a NetworkX DiGraph. ## Key Classes ### Compiler - `compile()`: Main entry point. Creates a `Linker`, calls `link_graph()`, optionally adds test edges, writes `graph.gpickle` - `compile_node()`: Per-node compilation (covered in [Node Compilation](4.4_Node_Compilation.md)) ### Linker Wrapper around `nx.DiGraph` that builds the dependency graph: - `link_graph()`: Iterates all manifest nodes, calls `link_node()` for each - `link_node()`: Adds edges based on `node.depends_on_nodes` - `find_cycles()`: Detects cycles in the graph, raises error if found - `add_test_edges()`: For `dbt build`, adds edges from tests to downstream nodes ## Graph Structure - Nodes: unique_ids of all manifest resources (models, tests, seeds, sources, etc.) - Edges: dependency relationships from `depends_on_nodes` - Direction: edges point from dependency to dependent (A → B means B depends on A) ## Test Edge Addition For `dbt build`, tests should block downstream models from executing. The `add_test_edges()` method adds edges from upstream tests to downstream non-test nodes, ensuring tests run before models that depend on the tested resources. Two implementations exist: - `add_test_edges_1()`: Original algorithm, comprehensive but adds many redundant edges - `add_test_edges_2()`: Optimized algorithm (behind `USE_FAST_TEST_EDGES` flag), adds minimal edges ## Cycle Detection After linking, `find_cycles()` uses NetworkX to detect cycles. If found, raises a `RuntimeError` with the cycle path. This catches circular `ref()` dependencies that would cause infinite loops. ================================================ FILE: docs/arch/4.3_Node_Selection.md ================================================ # Node Selection ## Overview Node selection determines which nodes from the manifest will be executed based on `--select` and `--exclude` arguments. The selection framework lives in `core/dbt/graph/`. ## Selection Syntax ### Basic Selectors - `model_name`: Select by name - `tag:value`: Select by tag - `path:models/staging`: Select by file path - `package:my_package`: Select by package - `config.materialized:view`: Select by config value - `state:modified`: Select modified nodes (requires `--state`, see below) ### Graph Operators - `+model`: Select model and all ancestors - `model+`: Select model and all descendants - `+model+`: Select model, ancestors, and descendants - `@model`: Select model, ancestors, descendants, and tests of ancestors ### Set Operations - `model_a model_b`: Union (space-separated) - `model_a,model_b`: Union (comma-separated) - `--exclude model_c`: Exclusion ## Key Classes ### SelectionSpec / SelectionCriteria Parsed representation of selection arguments. `parse_difference()` in [`core/dbt/graph/cli.py`](https://github.com/dbt-labs/dbt-core/blob/main/core/dbt/graph/cli.py) parses CLI arguments into a `SelectionSpec`. ### SelectorMethod Base class for selector implementations. Each selector type (tag, path, config, etc.) has a corresponding method class in [`core/dbt/graph/selector_methods.py`](https://github.com/dbt-labs/dbt-core/blob/main/core/dbt/graph/selector_methods.py). ### NodeSelector Main class for resolving selection specs against the graph: - `select_nodes()`: Recursively resolves selection spec, applies set operations - `get_nodes_from_criteria()`: Resolves a single criterion, applies graph operators - `expand_selection()`: Handles indirect test selection - `filter_selection()`: Final filtering (e.g., by resource type) ### ResourceTypeSelector Extends `NodeSelector` to filter by `NodeType`. The `node_is_match()` method checks if a node's resource type is in the allowed set. ## State-Based Selection The `state:modified` selector is designed for CI/CD workflows where you want to run only the resources that have changed between two project states, allowing you to detect potential regressions without running the entire project. This requires the `--state` flag pointing to a directory containing a previous manifest (typically from production). ### How Modification Detection Works The `StateSelectorMethod` in [`core/dbt/graph/selector_methods.py`](https://github.com/dbt-labs/dbt-core/blob/main/core/dbt/graph/selector_methods.py) compares each node in the current manifest against its counterpart in the previous state's manifest. Each resource type implements a `same_contents()` method that encodes the semantic comparison logic for that resource type. This method returns `True` if the node is considered unchanged, and `False` if it should be marked as modified. The `same_contents()` comparison typically checks: - `raw_code` / `raw_sql`: The source SQL or Python code - `config`: Materialization, schema, tags, and other configuration - `depends_on`: References to other nodes (`ref()`, `source()`) - Resource-specific fields: e.g., `columns` for sources, `test_metadata` for tests ### State Selector Variants - `state:modified`: Nodes where `same_contents()` returns `False` - `state:new`: Nodes that exist in current manifest but not in previous state - `state:modified.body`: Only code changes (ignores config changes) - `state:modified.configs`: Only config changes ## Indirect Selection Tests can be selected indirectly when their parent models are selected. Modes: - **eager** (default): Select test if ANY parent is selected - **cautious**: Select test only if ALL parents are selected - **buildable**: Select test if all parents are selected or are ancestors of selected nodes - **empty**: Don't expand to tests These modes are set by the user via the `--indirect-selection` flag (e.g. `--indirect-selection=cautious`). ## GraphQueue Construction `NodeSelector.get_graph_queue()` creates the execution queue: 1. `get_selected()`: Get final set of selected node IDs 2. `full_graph.get_subset_graph()`: Create subgraph with only selected nodes 3. `GraphQueue(subgraph, ...)`: Wrap in thread-safe queue with topological ordering ================================================ FILE: docs/arch/4.4_Node_Compilation.md ================================================ # Node Compilation ## Overview Node compilation renders Jinja templates into executable SQL. This happens in `Compiler.compile_node()` ([`core/dbt/compilation.py`](https://github.com/dbt-labs/dbt-core/blob/main/core/dbt/compilation.py)) and is invoked by Runners during execution. ### Example: Before and After Compilation **Before (raw SQL with Jinja)** — `models/marts/orders_with_payments.sql`: ```sql {{ config( materialized='table', schema='marts' ) }} with orders as ( select * from {{ ref('stg_orders') }} ), payments as ( select * from {{ source('stripe', 'payments') }} ) select orders.order_id, orders.customer_id, orders.order_date, payments.amount from orders left join payments on orders.payment_id = payments.id ``` **After (compiled SQL)** — `target/compiled/my_project/models/marts/orders_with_payments.sql`: ```sql with orders as ( select * from "database"."staging"."stg_orders" ), payments as ( select * from "database"."stripe"."payments" ) select orders.order_id, orders.customer_id, orders.order_date, payments.amount from orders left join payments on orders.payment_id = payments.id ``` The `config()` block is processed during parsing (setting node properties like `materialized='table'`) and stripped from the compiled output. The `ref()` and `source()` calls are resolved to fully-qualified database relations based on the target environment's configuration. ## Key Methods ### Compiler.compile_node() Main entry point for per-node compilation: 1. Calls `_compile_code()` to render Jinja → `compiled_code` 2. Calls `_recursively_prepend_ctes()` for ephemeral model CTE injection 3. Writes compiled SQL to `target/compiled/` directory ### _compile_code() Renders the node's `raw_code` using Jinja: 1. Creates node context via `_create_node_context()` 2. Calls `jinja.render()` with the context 3. Sets `node.compiled_code` and `node.compiled = True` ### _create_node_context() Builds the Jinja rendering context: - For regular nodes: `generate_runtime_model_context()` - For unit tests: `generate_runtime_unit_test_context()` - Adds test kwargs for generic tests ## Ephemeral Model Handling Ephemeral models don't create database objects—they're injected as CTEs into referencing models. ### _recursively_prepend_ctes() When a model references an ephemeral model via `ref()`: 1. The ephemeral model is added to `node.extra_ctes` during parsing 2. During compilation, `_recursively_prepend_ctes()` compiles each ephemeral dependency 3. Recursively processes nested ephemeral references 4. Injects compiled SQL as CTEs at the beginning of the referencing model's SQL ### CTE Injection The `_inject_ctes_into_sql()` method prepends CTEs to the model's SQL: ```sql WITH __dbt__cte__ephemeral_model AS ( -- compiled SQL from ephemeral model ), -- any additional CTEs from the model itself SELECT * FROM __dbt__cte__ephemeral_model ``` ## Deferred Ref Resolution When deferral is enabled (`--defer`), `ref()` calls may resolve to production relations instead of the target schema. This happens in `RuntimeRefResolver.create_relation()` ([`core/dbt/context/providers.py`](https://github.com/dbt-labs/dbt-core/blob/main/core/dbt/context/providers.py)). The resolution logic checks three conditions: 1. **Node has a `defer_relation`**: Set during `merge_from_artifact()` (see [Deferral](3.2_Deferral.md)) 2. **`--defer` flag is enabled** 3. **One of**: - `--favor-state` is set AND the node is not in the current selection - OR the relation doesn't exist in the target environment (adapter cache lookup) ```python if ( target_model.defer_relation and self.config.args.defer and ( (self.config.args.favor_state and target_model.unique_id not in SELECTED_RESOURCES) or not get_adapter(self.config).get_relation(database, schema, identifier) ) ): # Use defer_relation (production) instead of target_model (local) return self.Relation.create_from(self.config, target_model.defer_relation, ...) ``` ### Example With `--defer --state prod-artifacts/`: ```sql -- Source: models/marts/report.sql select * from {{ ref('dim_customers') }} -- If dim_customers is NOT selected and exists in prod: -- Compiles to: select * from "prod_db"."prod_schema"."dim_customers" -- If dim_customers IS selected (being built in this run): -- Compiles to: select * from "dev_db"."dev_schema"."dim_customers" ``` ## Compilation vs Parsing Key distinction (see [Parsing vs Compilation vs Runtime](../guides/parsing-vs-compilation-vs-runtime.md)): - **Parsing**: Extract structure, resolve refs, no Jinja rendering - **Compilation**: Render Jinja, produce `compiled_code`, inject CTEs - **Runtime**: Execute against database Compilation requires adapter connection for `run_query()` in Jinja. It happens during Task execution, not during manifest loading. ================================================ FILE: docs/arch/4.5_Node_Materialization.md ================================================ # Node Materialization ## Overview Node materialization is the final execution step where compiled SQL is run against the data warehouse. For models, this involves invoking materialization macros that handle the specifics of creating tables, views, or incremental updates. ### Example: Table Materialization (Atomic Swap Pattern) When a model is configured with `materialized='table'`, the materialization macro generates SQL that follows an atomic swap pattern to avoid downtime: ```sql -- 1. Create a temporary table with the model's SQL CREATE TABLE "database"."schema"."my_model__dbt_tmp" AS ( SELECT * FROM ... -- compiled model SQL ); -- 2. Drop the existing table (if it exists) DROP TABLE IF EXISTS "database"."schema"."my_model"; -- 3. Rename the temp table to the final name ALTER TABLE "database"."schema"."my_model__dbt_tmp" RENAME TO "my_model"; ``` This pattern ensures that the model is never in a partially-built state—readers see either the old version or the new version, never an incomplete table. Note that this is a general pattern and assumes DWH support for an atomic `RENAME` operation. Different strategies are implemented for different warehouses based on the capabilities they support with the goals of the materialization in mind. ## ModelRunner.execute() The main execution method for models (`core/dbt/task/run.py`): 1. Generates runtime context via `generate_runtime_model_context()` 2. Looks up materialization macro via `manifest.find_materialization_macro_by_name()` 3. Invokes the macro via `MacroGenerator` 4. Caches created relations via `adapter.cache_added()` 5. Returns `RunResult` ## Materialization Macro Lookup When looking up a materialization macro, dbt searches in a specific order (`core/dbt/contracts/graph/manifest.py`): 1. **Adapter-specific, project-local**: `materialization_table_snowflake` in your project 2. **Adapter-specific, imported package**: `materialization_table_snowflake` from a dependency 3. **Adapter-specific, adapter package**: Built into `dbt-snowflake` 4. **Default, project-local**: `materialization_table_default` in your project 5. **Default, core**: `materialization_table_default` from `dbt-core` The adapter type hierarchy is also considered. For example, Snowflake inherits from the base SQL adapter, so if no Snowflake-specific materialization exists, the default SQL one is used. ## Materialization Macros Materializations are Jinja macros that define how to create database objects. Built-in materializations live in [`core/dbt/include/global_project/macros/materializations/`](https://github.com/dbt-labs/dbt-adapters/tree/main/dbt-adapters/src/dbt/include/global_project) in the `dbt-adapters` repository. ### Common Materializations - **view**: `CREATE VIEW AS SELECT ...` - **table**: `CREATE TABLE AS SELECT ...` (with atomic swap) - **incremental**: Merge/insert new rows into existing table - **ephemeral**: No database object (CTE injection only, handled in compilation) ### Materialization Interface Macros receive a context including: - `model`: The node being materialized - `config`: Node configuration (via `config.get()`, `config.require()`) - `adapter`: Adapter methods for database operations - `statement()`: Execute SQL and capture results - `run_query()`: Execute SQL and return results Must return `{'relations': [relation]}` for cache management. ## The `statement()` Block The `statement()` block is the core primitive that materializations use to execute SQL. It: 1. Executes the provided SQL against the database via the adapter 2. Captures the result (rows affected, timing, etc.) 3. Stores the result in `sql_results` for later retrieval via `load_result()` Example usage in a materialization: ```jinja {% call statement('main') %} {{ sql }} {% endcall %} {# Later, retrieve the result #} {% set result = load_result('main') %} ``` The `'main'` statement is special: it's used by `ModelRunner` to extract the adapter response for the `RunResult`. ## Adapter Interactions Materializations use adapter methods within their jinja definitions for database operations: - `adapter.execute()`: Run SQL and return response - `adapter.create_schema()`: Create schema if needed - `adapter.drop_relation()`: Drop existing object - `adapter.rename_relation()`: Atomic swap for table replacement - `adapter.Relation.create()`: Create relation references - `adapter.get_columns_in_relation()`: Introspect table schema ## Python Models Python models follow a different execution path than SQL models. Instead of rendering Jinja and executing SQL: 1. The Python code is not Jinja-rendered (no `ref()` in Python syntax) 2. A postfix is appended that calls the `model()` function 3. The materialization macro calls `submit_python_job()` in the context 4. `submit_python_job()` delegates to `adapter.submit_python_job()` 5. The adapter submits the Python code to the warehouse's execution environment (e.g., Snowpark, Databricks notebooks) Python models are supported only on adapters that implement `submit_python_job()`. ## Seeds and Snapshots ### Seeds Seeds (`SeedRunner`) load CSV files into database tables: 1. `compile()` is a no-op (returns the node unchanged) 2. `execute()` invokes the seed materialization macro 3. The macro uses `adapter.load_dataframe()` to bulk-insert CSV data 4. Results include the agate table for optional display (`--show`) ### Snapshots Snapshots (`SnapshotRunner`) implement SCD Type 2 (slowly changing dimensions): 1. Uses the `snapshot` materialization macro 2. Tracks record changes with `dbt_valid_from` and `dbt_valid_to` columns 3. Supports `strategy='timestamp'` (track by updated_at column) or `strategy='check'` (track by column hash) 4. On each run, invalidates changed records and inserts new versions ## Test Execution Tests are "materialized" by running their SQL and interpreting results: ### Data Tests (Generic and Singular) 1. `TestRunner.execute_data_test()` invokes the `test` materialization 2. The materialization runs the test SQL 3. Results are a single row with columns: `failures`, `should_warn`, `should_error` 4. Row count interpretation: 0 failures = pass, >0 = fail (or warn based on config) ### Unit Tests 1. `TestRunner.execute_unit_test()` builds a special unit test manifest 2. Compiles and runs the `unit` materialization 3. Compares `actual` vs `expected` result sets 4. Reports diff if mismatch occurs ## Incremental Strategies Incremental models support multiple strategies: - **append**: Insert all new rows - **delete+insert**: Delete matching rows, then insert new - **merge**: Upsert based on unique key (requires adapter support) - **microbatch**: Process in time-based batches (see `MicrobatchModelRunner`) Strategy selection based on `config.incremental_strategy` and adapter capabilities. ## Hook Execution RunTask executes project hooks around model execution: - `on-run-start`: Before any models run - `on-run-end`: After all models complete - Pre/post hooks on individual models via `config.pre_hook` / `config.post_hook` ## Relation Cache Management dbt maintains an in-memory cache of database relations to avoid repeated introspection queries. Materializations must keep this cache in sync: - `adapter.cache_added(relation)`: Register a newly created relation - `adapter.cache_dropped(relation)`: Remove a dropped relation - `adapter.cache_renamed(from_relation, to_relation)`: Update after rename The `_materialization_relations()` return value (`{'relations': [...]}`) is used by `ModelRunner` to automatically call `cache_added()` for each relation. ## Result Handling `RunResult` captures execution outcome: - `status`: Success, Error, Skipped, etc. - `timing`: Compile and execute timing info - `adapter_response`: Database-specific response (rows affected, etc.) - `message`: Human-readable status message ================================================ FILE: docs/arch/4_Execution.md ================================================ # Execution After parsing produces the Manifest, dbt enters the execution phase where selected nodes are compiled and run against the data warehouse. This phase is orchestrated by the `Task` framework, which provides a layered abstraction for command execution, node selection, and per-node processing. The execution flow begins when a command's Task (e.g., `RunTask`, `BuildTask`) calls `run()`. For graph-based commands, this triggers manifest compilation into a dependency graph—including cycle detection to catch circular `ref()` dependencies—followed by node selection based on CLI arguments (`--select`, `--exclude`), and construction of a `GraphQueue` that respects topological ordering. The Task then spawns a thread pool and processes nodes concurrently, with the queue releasing downstream nodes only after their dependencies complete. Each node is processed by a dedicated `Runner` instance, which handles the compile-then-execute lifecycle. During compilation, Jinja templates are rendered to produce `compiled_code`, with ephemeral model references resolved into CTEs. During execution, the `Runner` interacts with the adapter to run the compiled SQL or invoke materialization macros. Results are collected and used to determine whether dependent nodes should proceed or be skipped due to upstream failures. The framework supports different execution modes: topological (default, respects dependencies) and independent (parallel execution ignoring edges, used by `dbt list`). Special handling exists for `dbt build`, which adds test edges to the graph so that tests on upstream models block downstream model execution, and coordinates unit tests to run before their associated models. ## Subsections - [Task Framework](4.1_Task_Framework.md) - Task and Runner class hierarchies, execution orchestration - [Graph Compilation](4.2_Graph_Compilation.md) - Building the dependency graph from the manifest - [Node Selection](4.3_Node_Selection.md) - Selection syntax and the selector framework - [Node Compilation](4.4_Node_Compilation.md) - Jinja rendering and CTE injection - [Node Materialization](4.5_Node_Materialization.md) - Materialization macros and adapter interactions ================================================ FILE: docs/arch/5_Adapter.md ================================================ # Adapter Framework The adapter framework provides the abstraction layer between dbt-core and data warehouses. Each adapter (e.g., `dbt-snowflake`, `dbt-postgres`, `dbt-bigquery`) implements a common interface for database operations, allowing dbt-core to remain warehouse-agnostic while adapters handle the specifics of SQL dialect, connection management, and platform capabilities. ## Architecture Overview The adapter framework consists of several key components: - **Credentials**: Warehouse connection parameters parsed from `profiles.yml` - **Adapter Plugin**: The adapter implementation loaded dynamically based on credential type - **Connection Manager**: Thread-safe connection pooling and lifecycle management - **Relation**: Database object abstraction (database, schema, identifier) - **Macro Resolver**: Resolution of adapter-specific macro implementations via `adapter.dispatch()` The base adapter classes live in the `dbt-adapters` package, with warehouse-specific implementations in their respective packages (e.g., `dbt-snowflake`). ## Initialization Flow Adapter initialization happens during the CLI decorator chain in `core/dbt/cli/requires.py`: ### 1. Profile Loading (`@profile` decorator) ``` profiles.yml → load_profile() → Profile object with Credentials ``` The `load_profile()` function: 1. Reads `profiles.yml` from the profiles directory 2. Extracts the target configuration (e.g., `dev`, `prod`) 3. Parses the `type` field to determine which adapter to use 4. Calls `load_plugin(adapter_type)` to dynamically load the adapter package 5. Instantiates the adapter's `Credentials` class with connection parameters 6. Returns a `Profile` object containing the credentials ### 2. Runtime Config Creation (`@runtime_config` decorator) ``` Profile + Project → RuntimeConfig ``` The `RuntimeConfig` combines: - Profile (credentials, target, threads) - Project (dbt_project.yml settings) - CLI flags This config object is passed to the adapter and provides all context needed for database operations. ### 3. Adapter Registration (`@manifest` decorator) ``` RuntimeConfig → register_adapter() → get_adapter() ``` During manifest loading in `parse_manifest()`: 1. `register_adapter(runtime_config, mp_context)` registers the adapter in a global factory 2. `get_adapter(runtime_config)` retrieves the singleton adapter instance 3. `adapter.set_macro_context_generator()` configures how macro contexts are created 4. `adapter.set_macro_resolver(manifest)` provides the manifest for macro resolution 5. `adapter.connections.set_query_header()` sets up query comment headers The adapter is now fully initialized and ready for use. ## Adapter Configuration with Manifest After parsing completes, the adapter receives the manifest to enable macro resolution: ```python adapter = get_adapter(runtime_config) adapter.set_macro_resolver(manifest) adapter.set_macro_context_generator(generate_runtime_macro_context) ``` This connection is critical because: - Materializations and other macros use `adapter.dispatch()` to find implementations - The adapter needs to resolve macro calls like `adapter.dispatch('create_table_as')` - Query headers include project metadata from the manifest ## Connection Management The adapter manages database connections through its `ConnectionManager`: ### Connection Lifecycle ```python # Named connection context with adapter.connection_named("master"): adapter.execute(sql) # Connection automatically released on exit ``` ### Thread-Safe Pooling - Each thread gets its own connection via `get_thread_connection()` - Connections are named (e.g., node unique_id) for debugging and cancellation - `cleanup_connections()` releases all connections at task end ### Connection Naming by Node During execution, each node runs with a connection named after its unique_id: ```python with self.adapter.connection_named(self.node.unique_id, self.node): # compile and execute the node ``` This enables: - Query cancellation by node name on `KeyboardInterrupt` - Connection tracking and debugging - Proper isolation between concurrent node executions ## Critical Usage Points During Execution ### Before Execution (`RunTask.before_run()`) ```python with adapter.connection_named("master"): self.create_schemas(adapter, required_schemas) self.populate_adapter_cache(adapter) self.safe_run_hooks(adapter, RunHookType.Start, {}) ``` - Creates any missing schemas - Populates the relation cache for fast lookups - Runs `on-run-start` hooks ### During Node Execution Runners interact with the adapter through the Jinja context: ```python context = generate_runtime_model_context(model, self.config, manifest) # context['adapter'] is a DatabaseWrapper around the real adapter ``` Key adapter methods used in materializations: - `adapter.execute(sql)`: Run SQL, return response - `adapter.get_columns_in_relation(relation)`: Introspect schema - `adapter.create_schema(relation)`: Create schema if not exists - `adapter.drop_relation(relation)`: Drop table/view - `adapter.rename_relation(from, to)`: Atomic rename - `adapter.dispatch(macro_name)`: Find adapter-specific macro ### Relation Cache The adapter maintains an in-memory cache of database relations: ```python adapter.cache_added(relation) # Register new relation adapter.cache_dropped(relation) # Remove dropped relation adapter.cache_renamed(from, to) # Update after rename ``` Cache population happens in `populate_adapter_cache()` before execution, querying `information_schema` or equivalent to discover existing objects. ### Query Cancellation On `KeyboardInterrupt`, the task cancels in-flight queries: ```python adapter = get_adapter(self.config) if adapter.is_cancelable(): with adapter.connection_named("master"): for conn_name in adapter.cancel_open_connections(): # Log cancelled connections ``` ### After Execution (`RunTask.after_run()`) ```python self.safe_run_hooks(adapter, RunHookType.End, extra_context) adapter.cleanup_connections() ``` - Runs `on-run-end` hooks - Cleans up all open connections ## The `adapter.dispatch()` Mechanism `adapter.dispatch()` enables polymorphic macro invocation—the same macro call resolves to different implementations based on the active adapter. Resolution order: 1. Current adapter type (e.g., `snowflake__create_table_as`) 2. Parent adapter types in inheritance chain 3. Default implementation (`default__create_table_as`) This allows adapters to override specific behaviors while falling back to common implementations. ## Key Adapter Methods | Method | Purpose | |--------|---------| | `execute(sql, auto_begin, fetch)` | Execute SQL statement | | `get_columns_in_relation(relation)` | Get column metadata | | `create_schema(relation)` | Create schema | | `drop_relation(relation)` | Drop relation | | `rename_relation(from, to)` | Rename relation | | `list_schemas(database)` | List schemas in database | | `list_relations_without_caching(schema)` | Query relations directly | | `expand_column_types(...)` | Handle type expansion for incremental | | `submit_python_job(model, code)` | Execute Python model (if supported) | ================================================ FILE: docs/arch/6.10_dbt_compile.md ================================================ # dbt compile ## Overview `dbt compile` generates executable SQL from Jinja templates without executing anything against the database. Compiled SQL files are written to the `target/compiled/` directory. **Reference**: https://docs.getdbt.com/reference/commands/compile ## Task Class: `CompileTask` `CompileTask` extends `GraphRunnableTask` and is the base class for many execution commands (`RunTask`, `TestTask`, `ShowTask`, etc.). ### CompileRunner `CompileRunner` extends `BaseRunner` and handles the compilation step without execution: ```python class CompileRunner(BaseRunner): def compile(self, manifest): return self.compiler.compile_node(self.node, manifest, {}) def execute(self, compiled_node, manifest): # For pure compilation, just return success return RunResult(node=compiled_node, status=RunStatus.Success, ...) ``` ## Implementation Quirks ### `--inline` Support `CompileTask` supports inline SQL compilation via `--inline`. After compilation, the inline node is removed from the manifest. ### Compiled SQL Location Compiled files are written to `target/compiled/{project_name}/{path}`: ``` target/ compiled/ my_project/ models/ marts/ orders.sql # Compiled SQL (refs resolved) tests/ unique_orders_id.sql ``` ### `--introspect` Flag Controls whether to run introspective queries (e.g., fetching column types). It is enabled by default. ```python if self.args.introspect: # Allow adapter queries during compilation ``` ### Ephemeral CTE Injection When `--inject-ephemeral-ctes` is enabled, ephemeral model CTEs are inlined into the compiled SQL. This is enabled by default. ### Output Formats The `--output` flag controls how compiled SQL is displayed: | Value | Description | |-------|-------------| | `text` | Plain SQL text | | `json` | JSON with node metadata | ## Flags | Flag | Description | |------|-------------| | `--select` | Compile selected nodes | | `--inline` | Compile inline SQL string | | `--output` | Output format: `text` or `json` | | `--introspect` / `--no-introspect` | Enable/disable adapter queries | | `--inject-ephemeral-ctes` | Inline ephemeral model CTEs | ## Examples ```bash # Compile all models dbt compile # Compile specific model dbt compile --select my_model # Compile inline SQL dbt compile --inline "select * from {{ ref('orders') }}" # Output as JSON dbt compile --select my_model --output json ``` ## Use Cases 1. **Preview SQL**: See exactly what SQL will run 2. **Debug Jinja**: Verify template rendering 3. **IDE Integration**: Generate SQL for external tools 4. **CI Validation**: Ensure models compile without database access ================================================ FILE: docs/arch/6.11_dbt_source.md ================================================ # dbt source ## Overview `dbt source` is a command group for managing sources. Currently, it has one subcommand: - `dbt source freshness`: Check the freshness of source data **Reference**: https://docs.getdbt.com/reference/commands/source ## Subcommand: `dbt source freshness` ### Task Class: `FreshnessTask` `FreshnessTask` extends `RunTask` (via different inheritance) and queries sources to check data freshness. ### FreshnessRunner `FreshnessRunner` extends `BaseRunner` and queries source tables for freshness. ## Implementation Quirks ### Metadata Freshness Cache For adapters supporting metadata-based freshness (table modification time as opposed to based on the max value of a dedicated event timestamp column in the data), results are cached: ```python class FreshnessRunner(BaseRunner): def __init__(self, ...): self._metadata_freshness_cache: Dict[BaseRelation, FreshnessResult] = {} ``` This avoids re-querying metadata for tables that share the same physical relation. ### Two Freshness Strategies 1. **Query-based**: `SELECT MAX(loaded_at_field) FROM source` 2. **Metadata-based**: Use adapter's `TableLastModifiedMetadata` capability ```python if adapter.supports(Capability.TableLastModifiedMetadata): # Use metadata instead of query metadata = adapter.get_relation_last_modified(relation) ``` ### Output Artifact Writes `sources.json` (not `run_results.json`): ### Freshness Thresholds Source freshness config defines thresholds: ```yaml sources: - name: jaffle_shop freshness: warn_after: {count: 12, period: hour} error_after: {count: 24, period: hour} loaded_at_field: _etl_loaded_at tables: - name: orders ``` Status is determined by comparing the data age (time since `max_loaded_at`) against configured thresholds: - If age exceeds `error_after` → **Error** - If age exceeds `warn_after` → **Warn** - Otherwise → **Pass** ## Flags | Flag | Description | |------|-------------| | `--select` | Select sources to check | | `--exclude` | Exclude sources | | `--output` | Path for `sources.json` output | ## Output Format `sources.json` structure: ```json { "metadata": {...}, "results": [ { "unique_id": "source.my_project.jaffle_shop.orders", "status": "pass", "max_loaded_at": "2024-01-15T10:30:00Z", "snapshotted_at": "2024-01-15T11:00:00Z", "age": 1800.0, "criteria": { "warn_after": {"count": 12, "period": "hour"}, "error_after": {"count": 24, "period": "hour"} } } ] } ``` ## Use Cases 1. **Data Quality Monitoring**: Alert when source data is stale 2. **CI/CD Gates**: Block deployments if sources are stale 3. **Dashboards**: Power freshness status dashboards 4. **Incident Detection**: Identify broken upstream pipelines ## Example ```bash # Check all sources dbt source freshness # Check specific sources dbt source freshness --select source:jaffle_shop # Output to specific path dbt source freshness --output freshness_results/sources.json ``` ================================================ FILE: docs/arch/6.12_dbt_run-operation.md ================================================ # dbt run-operation ## Overview `dbt run-operation` executes a macro by name, allowing you to run arbitrary operations defined in your project. It's useful for administrative tasks, data migrations, or any custom logic that doesn't fit the standard model/test workflow. **Reference**: https://docs.getdbt.com/reference/commands/run-operation ## Task Class: `RunOperationTask` `RunOperationTask` extends `ConfiguredTask` (not `GraphRunnableTask`) since it executes a single macro, not a graph of nodes. ## Implementation Quirks ### Macro Name Parsing Supports both local and package-qualified macro names as input. Examples: - `my_macro` → searches in project - `my_package.my_macro` → searches in specific package ### Macro Arguments Arguments are passed via `--args` as a YAML dictionary: ```bash dbt run-operation my_macro --args '{"my_arg": "value"}' ``` ### Direct Adapter Execution Unlike model execution, `run-operation` calls `adapter.execute_macro()` directly: ### Synthetic HookNode Creates a `HookNode` for the result artifact (since macros aren't nodes): ```python run_result = RunResult( node=HookNode( alias=macro_name, checksum=FileHash.from_contents(unique_id), resource_type=NodeType.Operation, fqn=fqn, name=macro_name, unique_id=unique_id, package_name=package_name, path="", original_file_path="", ), ... ) ``` ### Transaction Clearing Clears any existing transaction before execution: ### Return Value Not Used The macro's return value is captured but not displayed: ```python res = adapter.execute_macro(...) # res is not used further ``` To output results, the macro should use `log()` or fire events. ## Flags | Flag | Description | |------|-------------| | `MACRO` | Positional argument: macro name | | `--args` | YAML dictionary of arguments | ## Use Cases 1. **Grant Management**: `dbt run-operation grant_select --args '{"role": "analyst"}'` 2. **Data Cleanup**: `dbt run-operation drop_old_partitions` 3. **Metadata Updates**: `dbt run-operation update_table_comments` 4. **Custom Maintenance**: Any DBA-style operations ## Example Macro ```sql -- macros/grant_select.sql {% macro grant_select(role) %} {% set sql %} GRANT SELECT ON ALL TABLES IN SCHEMA {{ target.schema }} TO ROLE {{ role }} {% endset %} {% do run_query(sql) %} {{ log("Granted SELECT to " ~ role, info=True) }} {% endmacro %} ``` ```bash dbt run-operation grant_select --args '{"role": "analyst"}' ``` ## Retry Behavior `run-operation` results appear in `run_results.json` and are handled specially by `dbt retry`: - Only retried if the previous command was `run-operation` - Operation nodes are filtered out when retrying other commands ================================================ FILE: docs/arch/6.13_dbt_init.md ================================================ # dbt init ## Overview `dbt init` scaffolds a new dbt project or initializes a profile for an existing project. It creates the standard dbt directory structure and optionally configures database credentials. **Reference**: https://docs.getdbt.com/reference/commands/init ## Task Class: `InitTask` `InitTask` extends `BaseTask` and handles interactive project/profile setup. ## Implementation Quirks ### Minimal Context `init` requires almost nothing—no profile, no project, no manifest: ```python @requires.postflight @requires.preflight def init(ctx, **kwargs): with InitTask(ctx.obj["flags"]) as task: ... ``` ### Positional Argument for Project Name For backwards compatibility, project name can be passed as a positional argument: ```python @click.argument("project_name", required=False) def init(ctx, project_name=None, **kwargs): ... ``` ### Starter Project Template Uses a built-in starter project template is defined in `core/dbt/include/starter_project`. ### Profile Creation Strategies Three methods to create a profile: 1. **From adapter sample**: Uses adapter's `sample_profiles.yml` 2. **From project template**: Uses project's profile template if provided 3. **Interactive prompts**: Asks user for connection details ### Adapter Discovery Discovers available adapters from installed packages: ```python adapter_names = _get_adapter_plugin_names() # Returns list like ['postgres', 'snowflake', 'bigquery', ...] ``` ## Flags | Flag | Description | |------|-------------| | `--skip-profile-setup` | Skip interactive profile configuration | ## Created Structure ``` my_project/ ├── dbt_project.yml ├── README.md ├── models/ │ └── example/ │ ├── my_first_dbt_model.sql │ ├── my_second_dbt_model.sql │ └── schema.yml ├── analyses/ ├── macros/ ├── seeds/ ├── snapshots/ └── tests/ ``` ## Interactive Flow 1. **Project Name**: Prompt for name (if not provided) 2. **Adapter Selection**: Choose from installed adapters 3. **Connection Details**: Adapter-specific prompts (host, database, credentials) 4. **Profile Location**: Written to `~/.dbt/profiles.yml` ================================================ FILE: docs/arch/6.14_dbt_list.md ================================================ # dbt list ## Overview `dbt list` (alias: `dbt ls`) lists resources in your dbt project matching selection criteria. It's useful for exploring the project, debugging selectors, and scripting. **Reference**: https://docs.getdbt.com/reference/commands/list ## Task Class: `ListTask` `ListTask` extends `GraphRunnableTask` but doesn't execute nodes. It only performs lookups against the manifest. ## Implementation Quirks ### Always Returns Success Unlike execution commands, `list` always returns exit code 0: ### Output Modes Four output formats via `--output`: | Mode | Example Output | |------|----------------| | `selector` | `my_model`, `source:jaffle_shop.orders` | | `name` | `my_model`, `orders` | | `json` | `{"name": "my_model", "resource_type": "model"}` | | `path` | `models/staging/my_model.sql` | ### `--models` vs `--select` For backwards compatibility, both flags work but are mutually exclusive. `--models` implies `--resource-type model`. ### JSON Output Keys Default keys in JSON output: ```python ALLOWED_KEYS = frozenset(( "alias", "name", "package_name", "depends_on", "tags", "config", "resource_type", "source_name", "original_file_path", "unique_id", )) ``` Override with `--output-keys`: ```bash dbt ls --output json --output-keys name config.materialized ``` ### Nested Key Support `--output-keys` supports dot notation for nested values. ### Hidden Alias `ls` is a hidden alias for `list`: ```python -- core/dbt/cli/main.py ls = copy(cli.commands["list"]) ls.hidden = True cli.add_command(ls, "ls") ``` ## Flags | Flag | Description | |------|-------------| | `--select` | Selection criteria | | `--exclude` | Exclude nodes | | `--models` | Shorthand for `--select --resource-type model` | | `--output` | Format: `selector`, `name`, `json`, `path` | | `--output-keys` | JSON keys to include | | `--resource-type` | Filter by type | ## Examples ```bash # List all models dbt ls --resource-type model # List modified models (requires --state) dbt ls --select state:modified --state prod-artifacts/ # Get JSON with specific fields dbt ls --output json --output-keys name config.materialized # Get file paths for shell scripting dbt ls --select tag:nightly --output path | xargs wc -l ``` ================================================ FILE: docs/arch/6.15_dbt_retry.md ================================================ # dbt retry ## Overview `dbt retry` re-executes failed nodes from the previous run. It reads the `run_results.json` artifact to determine which nodes failed and runs them using the same command and flags from the original invocation. **Reference**: https://docs.getdbt.com/reference/commands/retry ## Task Class: `RetryTask` `RetryTask` extends `ConfiguredTask` and dynamically creates the appropriate task for the original command based on `run_results.json`. ```python class RetryTask(ConfiguredTask): def __init__(self, args, config): # Load previous run results self.previous_results = load_result_state( Path(config.project_root) / Path(state_path) / "run_results.json" ) # Determine which command was run self.previous_command_name = self.previous_args.get("which") self.task_class = TASK_DICT.get(self.previous_command_name) ``` ## Implementation Quirks ### Supported Commands Only certain commands are retryable: ```python TASK_DICT = { "build": BuildTask, "compile": CompileTask, "clone": CloneTask, "generate": GenerateTask, "seed": SeedTask, "snapshot": SnapshotTask, "test": TestTask, "run": RunTask, "run-operation": RunOperationTask, } ``` ### Retryable Statuses Nodes with these statuses are retried: ```python RETRYABLE_STATUSES = { NodeStatus.Error, NodeStatus.Fail, NodeStatus.Skipped, NodeStatus.RuntimeErr, NodeStatus.PartialSuccess, # Microbatch partial failure } ``` ### Flag Merging Retry combines previous flags with current CLI arguments: Flags are merged as follows: 1. **From previous run**: Most flags are restored (e.g., `--select`, `--full-refresh`, `--defer`) 2. **Always from current invocation**: Path-related flags (`--profiles-dir`, `--project-dir`, `--target-path`, `--log-path`) and `--warn-error` 3. **CLI override allowed**: `--vars` and `--threads` can be overridden if explicitly passed on the command line ### Dynamic Task Wrapping Retry creates a wrapper class that overrides the original Task's `get_graph_queue()`: ```python class TaskWrapper(self.task_class): def get_graph_queue(self): new_graph = self.graph.get_subset_graph(unique_ids) return GraphQueue(new_graph.graph, self.manifest, unique_ids) ``` ### Microbatch Batch-Level Retry For microbatch models, retry only re-runs failed batches by passing a `batch_map` to the `RunTask`. ### Operation Node Filtering `run-operation` results aren't retried if not retrying `dbt run-operation` directly. ### `--warn-error` Interaction If `--warn-error` is set for retry, warnings from previous runs become failures: ```python if args.warn_error: RETRYABLE_STATUSES.add(NodeStatus.Warn) ``` ### Late Manifest Parsing Unlike most commands, retry parses the manifest inside the task after resolving flags: ```python # Parse manifest using resolved config/flags manifest = parse_manifest(retry_config, ...) ``` ## Flags | Flag | Description | |------|-------------| | `--state` | Override path to previous run results | | `--threads` | Override thread count | | `--vars` | Override variables | | `--full-refresh` | Force full refresh for incremental models | ## Use Cases 1. **CI Recovery**: Re-run failed nodes after fixing issues 2. **Transient Failures**: Retry after network/database hiccups 3. **Partial Deployments**: Complete interrupted runs 4. **Microbatch Resumption**: Continue from failed batch ## Example Flow ```bash # Original run fails on 3 models dbt run --select tag:nightly # Fix the issues, then retry just the failures dbt retry # Or retry with different parallelism dbt retry --threads 1 ``` ================================================ FILE: docs/arch/6.16_dbt_clone.md ================================================ # dbt clone ## Overview `dbt clone` creates zero-copy clones of database objects from a source environment (typically production) to a target environment. This enables fast environment setup without data copying, using database-native cloning features. **Reference**: https://docs.getdbt.com/reference/commands/clone ## Task Class: `CloneTask` `CloneTask` extends `GraphRunnableTask` and uses database cloning instead of materialization. ## Implementation Quirks ### Always Requires State Unlike other commands where `--defer` and `--state` are optional, clone always requires knowing where to clone from. ### Independent Execution Mode Clone uses `GraphRunnableMode.Independent`, which means nodes are run without topological ordering. This is because clones don't depend on each other—they all reference the source environment. ### No Compilation `CloneRunner.compile()` is a no-op since there's no SQL to compile! ### Clone Materialization Macro Cloning is handled by the `clone` materialization macro. The macro calls adapter-specific clone methods (e.g., `CREATE TABLE ... CLONE` on Snowflake). ### Schema Caching Clone caches both target and source (defer_relation) schemas. ### Refable Node Types Only Clone only works with "refable" nodes (models, seeds, snapshots). ## Flags | Flag | Description | |------|-------------| | `--select` | Select nodes to clone | | `--exclude` | Exclude nodes | | `--state` | Path to production manifest (required) | | `--resource-type` | Filter by type (model, seed, snapshot) | | `--full-refresh` | Re-clone even if target exists | ## Database Support Clone support varies by adapter: | Adapter | Clone Method | |---------|--------------| | Snowflake | `CREATE TABLE ... CLONE` | | BigQuery | Table copy/clone | | Databricks | Delta Lake shallow clone | | Others | Falls back to `CREATE TABLE AS SELECT` | ## Use Cases 1. **Dev Environment Setup**: Clone production to development instantly 2. **CI Environments**: Create isolated test environments 3. **Blue/Green Deployment**: Clone before schema migrations 4. **Data Sampling**: Clone structure without full data ## Example ```bash # Clone production models to development dbt clone --select tag:core --state prod-artifacts/ # Clone with full refresh dbt clone --select my_model --state prod-artifacts/ --full-refresh ``` ================================================ FILE: docs/arch/6.17_dbt_debug.md ================================================ # dbt debug ## Overview `dbt debug` validates the data warehouse connection by checking profile.yml configuration files, testing the database connection, and reporting dependency versions. It's the first troubleshooting command when dbt isn't working correctly. **Reference**: https://docs.getdbt.com/reference/commands/debug ## Task Class: `DebugTask` `DebugTask` extends `BaseTask` and performs a series of validation checks: `DebugTask` performs the following checks in order: 1. Print version info (dbt, Python, OS) 2. Load and validate `profiles.yml` 3. Load and validate `dbt_project.yml` 4. Test required dependencies (e.g., git) 5. Test database connection ## Implementation Quirks ### Minimal Context Requirements `dbt debug` uses minimal `@requires`. decorator as it doesn't require a loaded manifest, profile, or project to start. Only `@requires.preflight` and `@requires.postflight` are used. ### Graceful Failure Handling Unlike other commands, `debug` continues checking even when individual steps fail. Each check returns a `SubtaskStatus` with success/failure status and details, allowing the full diagnostic report to be generated. ### `--config-dir` Flag Opens the profiles directory in the system file manager using the OS-appropriate command. ### `--connection` Flag Skips configuration validation (profiles.yml, dbt_project.yml, dependencies) and jumps straight to connection testing. ### Multiple Profile Handling If no `dbt_project.yml` exists, debug lists available profiles from `profiles.yml` and prompts the user to specify one with `--profile`. ### Adapter Version Detection Debug dynamically imports adapter packages (e.g., `dbt.adapters.postgres.__version__`) to report installed versions. Returns "unknown" if the adapter module can't be found. ## Checks Performed | Check | Description | |-------|-------------| | dbt version | Installed dbt-core version | | python version | Python interpreter version | | python path | Path to Python executable | | os info | Operating system details | | profiles.yml | Profile file exists and is valid YAML | | dbt_project.yml | Project file exists and is valid | | adapter type | Which adapter is configured | | adapter version | Installed adapter package version | | Connection | Test database connectivity | ## Flags | Flag | Description | |------|-------------| | `--config-dir` | Open profiles directory in file manager | | `--connection` | Skip to connection test only | ## Output Example ``` dbt version: 1.7.0 python version: 3.10.12 python path: /usr/bin/python3 os info: Linux-5.15.0-x86_64 Using profiles dir at /home/user/.dbt Using profiles.yml file at /home/user/.dbt/profiles.yml Using dbt_project.yml file at /home/user/project/dbt_project.yml Configuration: profiles.yml file [OK found and valid] dbt_project.yml file [OK found and valid] Required dependencies: - git [OK found] Connection: adapter: postgres connection_info: host=localhost port=5432 user=analyst Connection test: [OK connection ok] ``` ================================================ FILE: docs/arch/6.18_dbt_clean.md ================================================ # dbt clean ## Overview `dbt clean` removes directories specified in the `clean-targets` config, typically `target/` (compiled artifacts) and `dbt_packages/` (installed packages). It's useful for resetting state before a fresh build. **Reference**: https://docs.getdbt.com/reference/commands/clean ## Task Class: `CleanTask` `CleanTask` extends `BaseTask` and requires minimal context—no manifest, profile, or database connection. ## Implementation Quirks ### No Profile Required `@requires.unset_profile` indicates clean doesn't need database credentials: ```python @requires.unset_profile @requires.project def clean(ctx, **kwargs): with CleanTask(ctx.obj["flags"], ctx.obj["project"]) as task: ... ``` ### `--clean-project-files-only` Controls scope of cleaning: - Without flag: Cleans both project-level and dbt-managed directories - With flag: Only cleans project-defined `clean-targets` ### Default Clean Targets If not specified in `dbt_project.yml`, defaults to: - `target/` - `dbt_packages/` ## Flags | Flag | Description | |------|-------------| | `--clean-project-files-only` | Only clean project-defined targets | ## Configuration ```yaml # dbt_project.yml clean-targets: - target - dbt_packages - logs - compile_output # Custom directory ``` ## Safety `CleanTask` only removes directories explicitly listed in `clean-targets`. It won't remove: - Source files (`models/`, `seeds/`, etc.) - Configuration files (`dbt_project.yml`, `profiles.yml`) - Version control (`.git/`) ================================================ FILE: docs/arch/6.1_dbt_parse.md ================================================ # dbt parse ## Overview `dbt parse` parses the project and writes the manifest without executing any nodes. It's useful for validating project structure, measuring parse performance, and generating the manifest for external tools. **Reference**: https://docs.getdbt.com/reference/commands/parse ## Task Implementation Unlike most commands, `parse` has no dedicated Task class—the work happens in the `@requires.manifest` decorator: ```python @requires.manifest(write_perf_info=True) def parse(ctx, **kwargs): # manifest generation and writing happens in @requires.manifest return ctx.obj["manifest"], True ``` ## Implementation Quirks ### Performance Info The `write_perf_info=True` flag causes parsing performance data to be written: ```python @requires.manifest(write_perf_info=True) ``` This writes timing information for each parsing phase to help diagnose slow parses. ### Return Value `parse` is unique in returning the `Manifest` object directly Other commands return execution results like `RunExecutionResult`. ### Partial Parsing Integration `parse` respects partial parsing settings. A full parse happens when: - `--no-partial-parse` is specified - `partial_parse.msgpack` is missing or invalid - Project configuration changed ### No Compilation `parse` only parses. It doesn't compile SQL, meaning: - Jinja is not rendered - `compiled_code` is not populated - Refs/sources are extracted but not fully resolved ### Manifest Output Writes `manifest.json` to the target directory (respects `--write-json` flag, which is enabled by default). ## Flags | Flag | Description | |------|-------------| | `--partial-parse` / `--no-partial-parse` | Enable/disable partial parsing | | `--write-json` / `--no-write-json` | Write manifest.json | ## Use Cases 1. **Validate Syntax**: Check YAML and SQL for errors 2. **Measure Performance**: Diagnose slow parsing with perf info 3. **Generate Manifest**: Create manifest for external tools (IDE plugins, CI systems) 4. **Pre-warm Cache**: Populate partial parse cache before execution 5. **CI Validation**: Ensure project parses without database access ## Performance Metrics With `write_perf_info=True`, the following are tracked: - File reading time - YAML parsing time - Per-parser timing (models, tests, sources, etc.) - Partial parsing hit/miss - Total parse duration ================================================ FILE: docs/arch/6.2_dbt_run.md ================================================ # dbt run ## Overview `dbt run` compiles and executes models against the target database. It's the core command for materializing SQL and Python models as tables, views, or incremental updates. **Reference**: https://docs.getdbt.com/reference/commands/run ## Task Class: `RunTask` `RunTask` extends `CompileTask` (which extends `GraphRunnableTask`) and is the base class for most execution commands. Key characteristics: - **Runner**: Uses `ModelRunner` for standard models, `MicrobatchModelRunner` for microbatch incremental models - **Resource Types**: Models only (`NodeType.Model`) - **Error Handling**: `raise_on_first_error()` returns `False`—continues processing other nodes after failures - **Hooks**: Executes `on-run-start` and `on-run-end` hooks via `safe_run_hooks()` ### Execution Flow 1. **before_run()**: Creates schemas, populates adapter cache, calls `defer_to_manifest()` if `--defer` is set 2. **Hook execution**: Runs `on-run-start` hooks 3. **Node execution**: For each selected model, `ModelRunner` compiles and materializes 4. **Hook execution**: Runs `on-run-end` hooks ### ModelRunner `ModelRunner` extends `CompileRunner` and handles the compile-then-execute lifecycle: ``` compile() → execute() → after_execute() ``` In `execute()`: 1. Generates runtime context via `generate_runtime_model_context()` 2. Looks up materialization macro via `manifest.find_materialization_macro_by_name()` 3. Invokes the macro via `MacroGenerator` 4. Caches created relations for the adapter cache 5. Returns `RunResult` ## Implementation Quirks ### Tracking `RunTask` includes model run tracking via `track_model_run()` which sends anonymized telemetry including execution time, materialization type, incremental strategy, contract enforcement status, and versioning. ### Empty Execution The `--empty` flag allows running with `LIMIT 0` to test compilation and downstream effects without processing data. This is handled by models' ref/source resolution injecting empty result sets. ## Microbatch Models Microbatch is a special incremental strategy for processing time-series data in batches. When a model has `incremental_strategy='microbatch'`, dbt uses `MicrobatchModelRunner` instead of `ModelRunner`. ### How Microbatch Works Rather than processing all data in a single transaction, microbatch: 1. Divides the time range into batches based on `batch_size` (hour, day, month, year) 2. Executes each batch as a separate transaction 3. Supports parallel batch execution after the first batch succeeds 4. Tracks batch-level success/failure for retry capability ### Key Classes **`MicrobatchBuilder`** (`core/dbt/materializations/incremental/microbatch.py`): - Constructs batch definitions from model config (`begin`, `batch_size`, `lookback`) - Builds start/end times for each batch - Creates Jinja context for batch execution **`MicrobatchModelRunner`**: - Orchestrates batch execution - Manages parallel execution via thread pool - Tracks `relation_exists` to know when parallel execution is safe **`BatchRunner`**: - Handles single batch execution - Called by `MicrobatchModelRunner._execute_microbatch_materialization()` ### Batch Execution Flow ``` MicrobatchModelRunner.execute() └── For each batch: └── BatchRunner._execute_microbatch_materialization() ├── Build jinja context with batch bounds ├── Execute materialization macro └── Return batch result ``` ### CLI Flags for Microbatch | Flag | Description | |------|-------------| | `--event-time-start` | Override start time for batch range | | `--event-time-end` | Override end time for batch range | ### Retry Behavior Microbatch models support granular retry—only failed batches are re-executed on `dbt retry`. This is tracked via `batch_results` in `RunResult`, which stores `BatchResults` containing `successful` and `failed` batch lists. When retrying: - If some batches succeeded previously, only failed batches run - If no batches succeeded, the model runs in full-refresh mode (to handle schema changes) ### Configuration ```yaml models: my_model: materialized: incremental incremental_strategy: microbatch batch_size: day # hour, day, month, year begin: '2020-01-01' lookback: 1 # number of batches to re-process for late-arriving data ``` ================================================ FILE: docs/arch/6.3_dbt_build.md ================================================ # dbt build ## Overview `dbt build` runs all seeds, models, snapshots, and tests in DAG order. It's the "do everything" command that respects dependencies across resource types. **Reference**: https://docs.getdbt.com/reference/commands/build ## Task Class: `BuildTask` `BuildTask` extends `RunTask` and adds multi-resource-type execution with test edges in the graph. ### Key Differences from `RunTask` 1. **Multiple Resource Types**: Handles seeds, models, snapshots, tests, unit tests, saved queries, exposures, and functions via `RUNNER_MAP` 2. **Test Edges**: Calls `compiler.compile(manifest, add_test_edges=True)` to add edges from tests to downstream nodes 3. **Unit Test Handling**: Runs unit tests *before* their associated model (see below) ### RUNNER_MAP ```python RUNNER_MAP = { NodeType.Model: ModelRunner, NodeType.Snapshot: SnapshotRunner, NodeType.Seed: SeedRunner, NodeType.Test: TestRunner, NodeType.Unit: TestRunner, NodeType.SavedQuery: SavedQueryRunner, NodeType.Exposure: ExposureRunner, NodeType.Function: FunctionRunner, } ``` ## Implementation Quirks ### Test Edge Injection The `add_test_edges=True` flag adds graph edges that ensure tests run at the right time: - Tests run *after* all their parent nodes complete - If a test fails, downstream nodes are skipped This differs from `dbt run` + `dbt test` where either models or tests are run, but not both resource types. ### Unit Test Ordering Unit tests run *before* their associated model, not after. This is handled specially: 1. `get_graph_queue()` builds `model_to_unit_test_map` mapping model IDs to their unit tests 2. When a model is dequeued, `handle_model_with_unit_tests_node()` runs unit tests first 3. If any unit test fails, the model itself is skipped, via `call_model_and_unit_tests_runner()` ### Node Count Adjustment Because unit tests aren't in the main graph queue but are run inline with their models, `BuildTask` adjusts `num_nodes` in `handle_job_queue()`: ```python if self.run_count == 0: self.num_nodes = self.num_nodes + len(self.selected_unit_tests) ``` ### Failure Propagation `BuildTask.MARK_DEPENDENT_ERRORS_STATUSES` determines which statuses cause downstream skips: ```python MARK_DEPENDENT_ERRORS_STATUSES = [ NodeStatus.Error, NodeStatus.Fail, NodeStatus.Skipped, NodeStatus.PartialSuccess, # microbatch partial failure ] ``` ### Two-Pass Node Selection To correctly count nodes and handle unit tests, `get_graph_queue()` runs selection twice: 1. With unit tests, in order to identify selected unit tests 2. Without unit tests—to build the actual execution queue The difference is stored in `self.selected_unit_tests`. ## Flags | Flag | Description | |------|-------------| | `--select` | Select nodes to build | | `--exclude` | Exclude nodes from build | | `--resource-type` | Filter by resource type (model, seed, snapshot, test) | | `--full-refresh` | Treat incremental models as full refreshes | | `--store-failures` | Store test failures in the database | ================================================ FILE: docs/arch/6.4_dbt_seed.md ================================================ # dbt seed ## Overview `dbt seed` loads CSV files from the `seeds/` (or configured `seeds-dir` in dbt_project.yml) directory into the data warehouse as tables. Seeds are useful for small lookup tables, test fixtures, or static reference data. **Reference**: https://docs.getdbt.com/reference/commands/seed ## Task Class: `SeedTask` `SeedTask` extends `RunTask` but filters to only `NodeType.Seed` resources: ```python class SeedTask(RunTask): def get_node_selector(self): return ResourceTypeSelector( resource_types=[NodeType.Seed], ... ) ``` ### SeedRunner `SeedRunner` extends `ModelRunner` but with key differences: 1. **No Compilation**: `compile()` returns the node unchanged—no Jinja rendering 2. **Agate Table Result**: Returns the loaded agate table in the result for `--show` output ```python def compile(self, manifest: Manifest): return self.node # No Jinja compilation for seeds def _build_run_model_result(self, model, context): result = super()._build_run_model_result(model, context) result.agate_table = context["load_result"]("agate_table").table return result ``` ## Implementation Quirks ### CSV Loading Seed CSV files are discovered during manifest loading, but the CSV content is **not** loaded during parsing. The `SeedParser` creates a `SeedNode` with file metadata (path, name, config) but explicitly skips content loading—the `render_with_context()` method is a no-op for seeds. The actual CSV loading happens at **runtime** via `load_agate_table()` ([`core/dbt/context/providers.py`](https://github.com/dbt-labs/dbt-core/blob/main/core/dbt/context/providers.py)), which is called from the `seed` materialization macro. The adapter then handles bulk insert to the database. The CSV is loaded into memory as an agate table via `load_agate_table()` in the Jinja context. ### Large CSV Handling Seeds over **1 MB** (`MAXIMUM_SEED_SIZE`) receive special handling for partial parsing: ```python # core/dbt/constants.py MAXIMUM_SEED_SIZE = 1 * 1024 * 1024 # 1 MB ``` For large seeds: - **No content hashing**: Instead of hashing file contents, dbt uses the file path as the checksum (`FileHash.path()`). This avoids reading the entire file into memory during parsing. - **Change detection limitations**: Since content isn't hashed, dbt cannot reliably detect changes. It uses heuristics based on file path. **Warning messages for large seeds:** - `SeedIncreased`: Seed grew past 1MB threshold - `SeedExceedsLimitSamePath`: Large seed at same path, assuming unchanged - `SeedExceedsLimitAndPathChanged`: Large seed moved, assuming changed ### `--show` Flag The `--show` flag displays a random sample of each loaded seed: ### Error Handling `raise_on_first_error()` returns `False`, so seed loading continues even if individual seed nodes fail. ## Flags | Flag | Description | |------|-------------| | `--select` | Select specific seeds to load | | `--show` | Display random sample of loaded data | ================================================ FILE: docs/arch/6.5_dbt_snapshot.md ================================================ # dbt snapshot ## Overview `dbt snapshot` executes snapshot definitions to capture slowly-changing dimension (SCD Type 2) history. Snapshots track changes to source data over time by maintaining historical records with validity timestamps. **Reference**: https://docs.getdbt.com/reference/commands/snapshot ## Task Class: `SnapshotTask` `SnapshotTask` extends `RunTask` but filters to only `NodeType.Snapshot` resources. ```python class SnapshotTask(RunTask): def get_node_selector(self): return ResourceTypeSelector( resource_types=[NodeType.Snapshot], ... ) ``` ### SnapshotRunner `SnapshotRunner` extends `ModelRunner` with snapshot-specific logging: ```python class SnapshotRunner(ModelRunner): def describe_node(self) -> str: return "snapshot {}".format(self.get_node_representation()) def print_result_line(self, result): # Includes snapshot config in output (strategy, unique_key, etc.) cfg = model.config.to_dict(omit_none=True) fire_event(LogSnapshotResult(..., cfg=cfg)) ``` ## Implementation Quirks ### Snapshot Strategy Detection Snapshots use either `timestamp` or `check` strategy, configured in the snapshot definition: - **timestamp**: Tracks changes via `updated_at` column - **check**: Tracks changes by comparing all `check_cols` The strategy is resolved during parsing and stored in `SnapshotNode.config.strategy`. ### Materialization Macro Under the hood, snapshots use the `snapshot` materialization macro (not `table` or `incremental`). This macro: 1. Creates the snapshot table if it doesn't exist (with SCD columns: `dbt_scd_id`, `dbt_updated_at`, `dbt_valid_from`, `dbt_valid_to`) 2. Identifies new and changed records 3. Invalidates changed records (sets `dbt_valid_to`) 4. Inserts new records with current validity ### No Incremental Semantics Unlike models with `incremental` materialization, snapshots don't support `--full-refresh` in the same way. The `--full-refresh` flag for snapshots rebuilds from scratch, losing historical data. ## Flags | Flag | Description | |------|-------------| | `--select` | Select specific snapshots to execute | | `--exclude` | Exclude snapshots | ================================================ FILE: docs/arch/6.6_dbt_test.md ================================================ # dbt test ## Overview `dbt test` runs data tests and unit tests defined in your project. Data tests validate assumptions about your data (e.g., uniqueness, not null), while unit tests validate model logic with fixed inputs. **Reference**: https://docs.getdbt.com/reference/commands/test ## Task Class: `TestTask` `TestTask` extends `RunTask` and handles both singular tests, generic tests, and unit tests. ```python class TestTask(RunTask): @property def resource_types(self) -> List[NodeType]: # TEST_NODE_TYPES = [NodeType.Test, NodeType.Unit] return [rt for rt in resource_types if rt in TEST_NODE_TYPES] ``` ### TestRunner `TestRunner` extends `CompileRunner` and handles three test types: 1. **Singular Tests**: Custom SQL test files in `tests/` directory 2. **Generic Tests**: Schema tests like `unique`, `not_null` from YAML 3. **Unit Tests**: Tests with mocked inputs defined in YAML ## Implementation Quirks ### Test Execution Path Split `TestRunner.execute()` branches based on test type: ```python def execute(self, test, manifest): if isinstance(test, UnitTestDefinition): unit_test_node, result = self.execute_unit_test(test, manifest) return self.build_unit_test_run_result(unit_test_node, result) else: test_result = self.execute_data_test(test, manifest) return self.build_test_run_result(test, test_result) ``` ### Unit Test Manifest Loading Unit tests require a separate mini-manifest containing mocked versions of referenced models: ```python def execute_unit_test(self, unit_test_def, manifest): # Build isolated manifest with just this unit test loader = UnitTestManifestLoader(manifest, self.config, {unit_test_def.unique_id}) unit_test_manifest = loader.load() # Compile and execute with the mocked manifest unit_test_node = self.compiler.compile_node(unit_test_node, unit_test_manifest, {}) ... ``` ### Unit Test Diff Rendering Unit tests compare actual vs expected results using the `daff` library for diff visualization: ```python def _get_daff_diff(self, expected, actual): expected_table = daff.PythonTableView(list_rows_from_table(expected, sort=True)) actual_table = daff.PythonTableView(list_rows_from_table(actual, sort=True)) diff = daff.TableDiff(alignment, flags) return diff ``` The diff is rendered with colors (`green` for actual, `red` for expected). ### Test Result Status Logic Data test status depends on the `severity` config (`error` or `warn`) and the failure count thresholds (`error_if`, `warn_if`): - If severity is `ERROR` and failures exceed `error_if` → **Fail** - If failures exceed `warn_if` → **Warn** (or **Fail** if `--warn-error` is set) - Otherwise → **Pass** ### `--store-failures` When enabled, test failures are stored in the database for later analysis. The `test` materialization creates a table with failing rows. ## Flags | Flag | Description | |------|-------------| | `--select` | Select tests to run | | `--exclude` | Exclude tests | | `--store-failures` | Persist test failures to the database | | `--resource-type` | Filter by `test` or `unit` | ## Test Configuration ```yaml models: - name: orders columns: - name: order_id tests: - unique - not_null - name: status tests: - accepted_values: values: ['placed', 'shipped', 'delivered'] unit_tests: - name: test_order_total model: orders given: - input: ref('order_items') rows: - {order_id: 1, amount: 10} - {order_id: 1, amount: 20} expect: rows: - {order_id: 1, total: 30} ``` ================================================ FILE: docs/arch/6.7_dbt_show.md ================================================ # dbt show ## Overview `dbt show` compiles and executes a model or inline query, returning a preview of the results without materializing anything to the warehouse. It's useful for quickly testing SQL or viewing model output. **Reference**: https://docs.getdbt.com/reference/commands/show ## Task Classes ### `ShowTask` `ShowTask` extends `CompileTask` and uses `ShowRunner` to execute queries. ### `ShowTaskDirect` For `--inline-direct`, a simpler task that bypasses manifest loading entirely is invoked, `ShowTaskDirect` ## Implementation Quirks ### Ephemeral Model Handling `ShowRunner` sets `run_ephemeral_models = True`, which causes ephemeral upstream models to be executed inline (as CTEs) rather than skipped: ```python class ShowRunner(CompileRunner): def __init__(self, ...): super().__init__(...) self.run_ephemeral_models = True ``` ### `get_show_sql` Macro The compiled SQL is wrapped by the `get_show_sql` macro which adds `LIMIT` and handles `sql_header`. ### Output Formatting Results can be output as table (default) or JSON: ```python if self.args.output == "json": table.to_json(path=output) else: table.print_table(output=output, max_rows=None) ``` ### `--limit` Behavior `--limit` controls the number of rows returned. Use `-1` for unlimited (returning all available rows). ### Direct Mode (`--inline-direct`) The `--inline-direct` flag executes raw SQL without any Jinja processing or manifest loading. This is the fastest path for quick queries. ## Examples ```bash # Preview a model dbt show --select my_model # Preview with more rows dbt show --select my_model --limit 100 # Execute inline query with refs dbt show --inline "select * from {{ ref('orders') }} limit 10" # Execute raw SQL directly (fastest) dbt show --inline-direct "select current_timestamp()" ``` ================================================ FILE: docs/arch/6.8_dbt_deps.md ================================================ # dbt deps ## Overview `dbt deps` downloads and installs dbt packages specified in `packages.yml` or `dependencies.yml`. It manages package resolution, version locking, and installation from the dbt Hub, git repositories, or local paths. **Reference**: https://docs.getdbt.com/reference/commands/deps ## Task Class: `DepsTask` `DepsTask` extends `BaseTask` (not `GraphRunnableTask`) because it doesn't require a manifest or database connection: ```python class DepsTask(BaseTask): def __init__(self, args, project): super().__init__(args=args) self.project = project # Only needs project config, not runtime config ``` ## Implementation Quirks ### Lock File Management `dbt deps` uses a two-file system: - `packages.yml`: User-specified dependencies (version ranges allowed) - `package-lock.yml`: Resolved, pinned versions Resolution flow: - If `packages.yml` changed or `--upgrade` is specified: resolve version ranges and update the lock file - Otherwise: use pinned versions from the lock file directly ### SHA1 Hash Validation The lock file includes a SHA1 hash of the serialized `packages.yml` contents. On each run, dbt computes the current hash and compares it to the stored hash. If they don't match, packages are re-resolved. ### `--add-package` Flag Allows adding packages from CLI without editing `packages.yml`. Version is required unless the source is `local`. ### Package Sources Three sources supported: 1. **Hub** (`package:`): Downloads from packages.getdbt.com 2. **Git** (`git:`): Clones from git repository 3. **Local** (`local:`): Uses local filesystem path ### No Profile Required `@requires.unset_profile` decorator indicates deps doesn't need database credentials: ```python @requires.unset_profile # Skip profile loading @requires.project # Only need project config def deps(ctx, **kwargs): ... ``` ## Flags | Flag | Description | |------|-------------| | `--source` | Package source: `hub`, `git`, or `local` | | `--lock` | Update only the lock file without installing | | `--upgrade` | Upgrade packages to latest matching versions | | `--add-package` | Add a package (format: `name@version`) | ## Package Resolution Packages are resolved in order of specificity: ```yaml # packages.yml packages: - package: dbt-labs/dbt_utils version: [">=1.0.0", "<2.0.0"] # Version range - git: https://github.com/org/repo revision: v1.0.0 # Git ref - local: ../my-local-package # Filesystem path ``` The lock file pins exact versions: ```yaml # package-lock.yml packages: - package: dbt-labs/dbt_utils version: 1.1.1 sha1_hash: abc123... ``` ================================================ FILE: docs/arch/6.9_dbt_docs.md ================================================ # dbt docs ## Overview `dbt docs` is a command group with two subcommands: - `dbt docs generate`: Creates documentation website artifacts - `dbt docs serve`: Starts a local web server to view documentation **Reference**: https://docs.getdbt.com/reference/commands/cmd-docs ## Subcommand: `dbt docs generate` ### Task Class: `GenerateTask` `GenerateTask` extends `CompileTask` and produces three artifacts: 1. `manifest.json`: Full project manifest 2. `catalog.json`: Database metadata (tables, columns, stats) 3. Static HTML/CSS/JS for the documentation site The task compiles the manifest, queries database metadata for the catalog (unless `--empty-catalog`), and writes both artifacts. ### Catalog Generation The catalog is populated by querying the database's information schema via `adapter.get_catalog()`. This retrieves table/column metadata for all relations in the selected schemas. ### `--static` Flag When set, copies the static documentation site files (HTML/CSS/JS) to the target directory. ### `--empty-catalog` Flag Skips catalog generation entirely, producing an empty catalog. Useful for testing or when database access is slow. ### `--compile` / `--no-compile` Controls whether to compile the manifest with full Jinja rendering, or use the manifest as-is from parsing. ### Flags | Flag | Description | |------|-------------| | `--select` | Generate docs for selected nodes only | | `--static` | Copy static site files to target | | `--empty-catalog` | Skip catalog generation | | `--no-compile` | Skip SQL compilation | --- ## Subcommand: `dbt docs serve` ### Task Class: `ServeTask` `ServeTask` extends `ConfiguredTask` and starts a local HTTP server in the target directory, then opens the browser. ### No Manifest Required Unlike most commands, `docs serve` doesn't load the manifest—it just serves static files. Only `@requires.runtime_config` is needed (for the target path). ### Browser Opening By default, opens the browser automatically. Disable with `--no-browser`. ### Flags | Flag | Description | |------|-------------| | `--host` | Server host (default: `127.0.0.1`) | | `--port` | Server port (default: `8080`) | | `--browser` / `--no-browser` | Open browser automatically | ## Artifacts ### `catalog.json` Contains database metadata: ```json { "nodes": { "model.my_project.orders": { "metadata": { "type": "table", "schema": "public", "name": "orders" }, "columns": { "id": {"type": "integer", "index": 1}, "status": {"type": "varchar", "index": 2} }, "stats": { "row_count": {"value": 1000, "label": "Row Count"} } } } } ``` ### Documentation Site The static site reads `manifest.json` and `catalog.json` at runtime to render: - Model lineage DAG - Column-level documentation - Test coverage - Source freshness ================================================ FILE: docs/arch/6_Commands.md ================================================ # Commands This section documents each dbt CLI command's implementation details, including task classes, runners, and implementation quirks. For detailed product documentation, see: https://docs.getdbt.com/category/list-of-commands ## Command Index | Command | Task Class | Description | |---------|------------|-------------| | [dbt parse](6.1_dbt_parse.md) | *(@requires.manifest decorator)* | Parse project and write manifest | | [dbt run](6.2_dbt_run.md) | `RunTask` | Execute models against the database | | [dbt build](6.3_dbt_build.md) | `BuildTask` | Run seeds, models, snapshots, and tests in DAG order | | [dbt seed](6.4_dbt_seed.md) | `SeedTask` | Load CSV files into the database | | [dbt snapshot](6.5_dbt_snapshot.md) | `SnapshotTask` | Execute SCD Type 2 snapshots | | [dbt test](6.6_dbt_test.md) | `TestTask` | Run data tests and unit tests | | [dbt show](6.7_dbt_show.md) | `ShowTask` | Preview query results without materializing | | [dbt deps](6.8_dbt_deps.md) | `DepsTask` | Install package dependencies | | [dbt docs](6.9_dbt_docs.md) | `GenerateTask` / `ServeTask` | Generate and serve documentation | | [dbt compile](6.10_dbt_compile.md) | `CompileTask` | Generate compiled SQL without executing | | [dbt source](6.11_dbt_source.md) | `FreshnessTask` | Check source freshness | | [dbt run-operation](6.12_dbt_run-operation.md) | `RunOperationTask` | Execute a macro | | [dbt init](6.13_dbt_init.md) | `InitTask` | Scaffold new project or profile | | [dbt list](6.14_dbt_list.md) | `ListTask` | List project resources | | [dbt retry](6.15_dbt_retry.md) | `RetryTask` | Re-execute failed nodes | | [dbt clone](6.16_dbt_clone.md) | `CloneTask` | Create zero-copy clones from production | | [dbt debug](6.17_dbt_debug.md) | `DebugTask` | Validate environment and connection | | [dbt clean](6.18_dbt_clean.md) | `CleanTask` | Remove target and packages directories | ## Command Categories ### Execution Commands Commands that execute work against the database: - `dbt run` — Models - `dbt build` — All resource types - `dbt seed` — CSV data loading - `dbt snapshot` — SCD snapshots - `dbt test` — Data/unit tests - `dbt clone` — Zero-copy clones - `dbt source freshness` — Source monitoring - `dbt retry` — Failure recovery - `dbt run-operation` — Ad-hoc macro execution ### Compilation Commands Commands that process SQL without executing: - `dbt compile` — Generate compiled SQL - `dbt show` — Preview results - `dbt parse` — Build manifest only ### Utility Commands Commands for project management: - `dbt deps` — Package management - `dbt clean` — Clear artifacts - `dbt init` — Project scaffolding - `dbt debug` — Warehouse connection validation - `dbt list` — Resource listing - `dbt docs` — Documentation generation/serving ## Common Patterns ### Context Requirements Commands use `@requires` decorators to build their execution context: | Decorator | Provides | |-----------|----------| | `@requires.preflight` | Logging, tracking setup | | `@requires.profile` | Database credentials | | `@requires.project` | Project configuration | | `@requires.runtime_config` | Merged profile + project | | `@requires.manifest` | Parsed manifest | ### Runner Pattern Execution commands use the Runner pattern: 1. Task creates `GraphQueue` of selected nodes 2. Thread pool processes nodes in parallel 3. Each node is handled by a `Runner` subclass 4. Runner calls `compile()` → `execute()` ### Result Artifacts Most commands write `run_results.json`. Exceptions: - `dbt source freshness` → `sources.json` - `dbt docs generate` → `catalog.json` - `dbt list` → stdout only ================================================ FILE: docs/arch/7_Artifacts.md ================================================ # Artifacts ## `dbt/artifacts` Directory This directory is meant to be a lightweight module that is independent (and upstream of) the rest of `dbt-core` internals. Its primary responsibility is to define simple data classes that represent the versioned artifact schemas that dbt writes as JSON files throughout execution. Eventually, this module may be released as a standalone package (e.g. `dbt-artifacts`) to support stable programmatic parsing of dbt artifacts. `dbt/artifacts` is organized into artifact 'schemas' and 'resources'. Schemas represent the final serialized artifact objects, while resources represent smaller components within those schemas. ### dbt/artifacts/schemas Each major version of a schema under `dbt/artifacts/schema` is defined in its corresponding `dbt/artifacts/schema//v` directory. Before `dbt/artifacts` artifact schemas were always modified in-place, which is why older artifacts are those missing class definitions. Currently, there are four artifact schemas defined in `dbt/artifact/schemas`: | Artifact name | File | Class | Latest definition | |---------------|------------------|----------------------------------|-----------------------------------| | manifest | manifest.json | WritableManifest | dbt/artifacts/schema/manifest/v12 | | catalog | catalog.json | CatalogArtifact | dbt/artifacts/schema/catalog/v1 | | run | run_results.json | RunResultsArtifact | dbt/artifacts/schema/run/v5 | | freshness | sources.json | FreshnessExecutionResultArtifact | dbt/artifacts/schema/freshness/v3 | ### dbt/artifacts/resources All existing resources are defined under `dbt/artifacts/resources/v1`. ## Making changes to dbt/artifacts ### All changes All changes to any fields will require a manual update to [dbt-jsonschema](https://github.com/dbt-labs/dbt-jsonschema) to ensure live checking continues to work. ### Non-breaking changes Freely make incremental, non-breaking changes in-place to the latest major version of any artifact (minor or patch bumps). The only changes that are fully forward and backward compatible are: * Adding a new field with a default * Deleting a field with a default. This is compatible in terms of serialization and deserialization, but still may be lead to suprising behaviour: * For artifact consumers relying on the fields existence (e.g. `manifest["deleted_field"]` will stop working unless the access was implemented safely) * Old code (e.g. in dbt-core) that relies on the value of the deleted field may have surprising behaviour given only the default value will be set when instantiated from the new schema These types of minor, non-breaking changes are tested by [tests/unit/artifacts/test_base_resource.py::TestMinorSchemaChange](https://github.com/dbt-labs/dbt-core/blob/main/tests/unit/artifacts/test_base_resource.py). #### Updating [schemas.getdbt.com](https://schemas.getdbt.com) Non-breaking changes to artifact schemas require an update to the corresponding jsonschemas published to [schemas.getdbt.com](https://schemas.getdbt.com), which are defined in https://github.com/dbt-labs/schemas.getdbt.com. To do so: Note this must be done AFTER the core pull request is merged, otherwise we may end up with unresolvable conflicts and schemas that are invalid prior to base pull request merge. You may create the schemas.getdbt.com pull request prior to merging the base pull request, but do not merge until afterward. 1. Create a PR in https://github.com/dbt-labs/schemas.getdbt.com which reflects the schema changes to the artifact. The schema can be updated in-place for non-breaking changes. Example PR: https://github.com/dbt-labs/schemas.getdbt.com/pull/39 2. Merge the https://github.com/dbt-labs/schemas.getdbt.com PR Note: Although `jsonschema` validation using the schemas in [schemas.getdbt.com](https://schemas.getdbt.com) is not encouraged or formally supported, `jsonschema` validation should still continue to work once the schemas are updated because they are forward-compatible and can therefore be used to validate previous minor versions of the schema. ### Breaking changes A breaking change is anything that: * Deletes a required field * Changes the name or type of an existing field * Removes the default value of an existing field These should be avoided however possible. When necessary, multiple breaking changes should be bundled together, to aim for minimal disruption across the ecosystem of tools that leverage dbt metadata. When it comes time to make breaking changes, a new versioned artifact should be created as follows: 1. Create a new version directory and file that defines the new artifact schema under `dbt/artifacts/schemas//v/.py` 2. If any resources are having breaking changes introduced, create a new resource class that defines the new resource schema under `dbt/artifacts/resources/v/.py` 3. Implement upgrade paths on the new versioned artifact class so it can be constructed given a dictionary representation of any previous version of the same artifact * TODO: link example once available 4. Implement downgrade paths on all previous versions of the artifact class so they can still be constructed given a dictionary representation of the new artifact schema * TODO: link example once available 5. Update the 'latest' aliases to point to the new version of the artifact and/or resource: * Artifact: `dbt/artifacts/schemas//__init__.py ` * Resource: `dbt/artifacts/resources/__init__.py ` Downstream consumers (e.g. `dbt-core`) importing from the latest alias are susceptible to breaking changes. Ideally, any incompatibilities should be caught my static type checking in those systems. However, it is always possible for consumers to pin imports to previous versions via `dbt.artifacts.schemas..v`. ================================================ FILE: docs/arch/8_Versioning_Branching_Strategy.md ================================================ # Release Versioning / Branching Strategy ## Context With `dbt` ever evolving and progressing, we continue to ship and release versions on a regular basis. As we release `dbt`, versioning and branching become key to identifying what is different for users and conveying the expectations of those changes. We currently push all our commits to `main` which is the latest version of our code. We don't always want to release the latest version of our code though. We want to be able to isolate commits to go into particular releases based on urgency (e.g. bug fixes) and stability (e.g. new features). We already have an established versioning strategy when it comes to major/minor/patch releases. The following is our approach as to what to expect from each release version: * Major version: This signifies that breaking changes which require user action are included in this release. * Minor version: This signifies that new features are included in this release. * Patch version: This signifies that bug and security fixes are included in this release. We need a versioning strategy that allows us to to convey to users the confidence and stability of code they are installing. We want a versioning strategy that allows users to try out new changes, and experiment with them to get early feedback. We want to make sure users can also get a preview of new features we are working on, so they can incorporate these changes into their work as soon as possible. We need a branching strategy to support the goals of the versioning strategy. We do not want to have to stop development on `main` as we await a release, nor do we want every change in `main` published in every release. Using branches, we can decide which changes go into which release. ## Common Terminology * Release version: refers to the numeric value of the release (eg. `1.0.1`, `0.21.0`) There are major, minor, and patch release versions that map to standard [semantic versioning practices](https://semver.org/#semantic-versioning-200). * Release phase: refers to the stability of the release (eg. beta, rc, final). ## Requirements The following are requirements that must be met for releasing dbt: 1. We must have major, minor, and patch releases. 1. We must have release phases that have the goal of garnering early feedback from the Community. 1. We must have release phases that aim to identify bugs and test for stability. 1. We must have release phases that are stable and of production quality. 1. We must have the ability to isolate changes from different versions and phases to release. 1. A version of `dbt-core` is considered released when it is available on GitHub Releases, PyPi, GitHub Container Registry, and Homebrew. This also applies to the individual `dbt` adapters. Otherwise, a version is considered partially released if only on a subset of those platforms. ## Branching Strategies Considered The 2 branching strategies we are considering using going forward are: 1. A new branch for each and every release 1. A new branch for each minor version which we release minor and patch versions from (current strategy) ### New branch for every release #### Pros: * Easy mapping between a release and a branch. Great for troubleshooting issues. * Less confusing and more straightforward for running a release on a branch. No question of what version is being released. #### Cons: * Breaks current adapters CI which needs to point to the exact `dbt-core` branch that it needs to test with. The branches would need to be updated in all adapter repos for each final and prerelease branch every time we have a new release branch for `dbt-core`. * Confusing for backporting. We will need to keep creating new labels and making sure we remove the old labels because we don't want to accidently backport to the wrong branch. ### Minor release branches (current strategy) #### Pros: * Adapter CI testing will continue to work with this strategy and makes it less complicated overall. Adapters and `dbt-core` need to be on the same minor version but can be on different patch versions. The current CI workflows verify that the changes slated for a new `1.1` patch are compatible across all repos by checking out the `1.1.latest` branch of each repo. The 1:1 mapping of adapter branches to `dbt-core` branches makes this easy. * Easy backporting. There is no need to figure out the exact branch that you need to backport to and accidently backporting to an older, stale branch. #### Cons: * Branches contain more than 1 release which is confusing. Without a 1:1 mapping between branch and release, you must rely on tags instead to know where in the branch the release was cut. * Releasing from the same branch multiple times can lead to confusion as to which version we are releasing. A branch named for the specific version could be extracted and used to run the bumpversion script instead of relying on human input at release time. ## Decisions ### Version phases
Header Explanations: - phases: defined in [Common Terminology](#common-terminology) - Released?: do we plan to release this phase to GitHub, PyPi, etc. - Branches: the branches where these release phases are present (e.g. betas should only exist on the `main` branch) - Applicable Release Versions: the release versions where the release phase is applicable (i.e. we will only have alphas for major and minor versions, we will not have alphas for patch versions) - Expectations: the stability of the code changes in the release
| Phase | Released? | Branches | Applicable Release Versions | Expectations | | ---- | --------- | -------- | ---------------- | ------------ | | Alpha | No | `main` | Major / Minor | Experimental | | Beta | Yes | `main` | Major / Minor | Experimental | | RC | Yes | Release branch | Major / Minor / Patch | Pre-production | | Final | Yes | Release branch | Major / Minor / Patch | Production | ### Branching Strategy Based off of the release version and expectations, the version phase's branching strategy can be determined. * Alpha and Beta versions are experimental, and therefore exist only on the `main` branch, where all our changes reside * Alpha versions are never released. They signify that development is underway, though the changes they contain have not been released in any form * A Release Candidate (RC) version denotes a more stable state. A release branch will be created for RC releases to limit the changes a release contains * Final versions are stable and tested. Only verified changes will go into these releases, which are inherited from the corresponding RC version * A release branch will exist for each unique major or minor version, to be named accordingly: `..latest` (e.g. `1.0.latest`) * This is keeping with our current stratgey that we use today. * The reasoning for this decision is largely based off of not wanting to break the current adapters CI workflows and needing to redesign these as well. With patch versions of adapters and `dbt-core` needing to be compatible, testing off of a single minor version branch is much simpler. * Making this current decision does not back us into a corner if we would like to revisit this again in the future. The release workflows created will be able to run on a branch that is version specific or not which makes this flexible as we test out our new processes. ![Branching Strategy](images/ReleasingBranchStrategy.png) ### Changes for Specific Release Versions There will be times that we will want to target changes to go into specific release versions. Below outlines 2 different scenarios in which this can occur and how we will approach them. 1. If a release branch already exists: This is the most common scenario where we want to add a change to an upcoming release and the release branch has already been cut. In this instance, developers will commit their changes to `main` first. Those changes (or some form of them) will then be applied to the release branch in question so that they can be included in the specific release version. \* Disclaimer: the how and when to apply changes to a release branch will be covered in another upcoming ADR. 1. If a release branch is not yet cut: This is the scenario where we start to develop a feature for a future version before we have cut a release branch for the upcoming version (eg. we want to work on a feature for `1.2.0` but we haven't cut the release branch for `1.1.0` yet). 1. Is this dead code? If so, then commiting the changes to `main` is acceptable. There must be 0 risk of users hitting this so if unsure, please use the alternative option. 1. Use a feature branch until the release branch is cut. Then when it makes sense, merge the feature branch into `main` and continue development. ** Sometimes feature flags are used by teams to also achieve this goal. In our case, feature flags aren't controlled by the team and instead dependent of a user to toggle them on/off. Core uses feature flags instead for offering experimental functionality for users to try and test out. This is different from hiding features from going into certain releases. ## Status Completed ## Consequences This doesn't drastically change our current versioning or branching strategies, but documents our existing flows. The only real change here is the addition of an Alpha release, the intent of which is to make developing across multiple repos easier. `dbt-core` and adapters should depend on minor versions. If we don't have a way to keep the `main` branches in sync with one another, integration tests will start to fail. It is also confusing when the `main` branch is marked as an RC or Final version when we never release from `main`. This change provides visibility around which versions live where. ## Outside Scope The following are topics that are outside the scope of this document and will be addressed in their own ADR: * Hotfixes and their branching strategies * How and why we bump release versions right before a release. * How and when do we backport changes to release branches ================================================ FILE: docs/guides/behavior-change-flags.md ================================================ # Playbook: Behavior Change Flags User documentation: https://docs.getdbt.com/reference/global-configs/legacy-behaviors ## Rules for introducing a new flag 1. **Naming.** All behavior change flags should be named so that their default value changes from **False → True**. This makes it significantly easier for us to document them and talk about them consistently, and it's more intuitive for end users. * (a) If the flag is prohibiting something that we previously allowed, use the verb "require." Examples: * `require_resource_names_without_spaces` * `require_explicit_package_overrides_for_builtin_materializations` * (b) All flags should be of boolean type, and False by default when introduced: `bool = False`. 2. **Documentation.** Start with the docs. What is the change? Who might be affected? What action will users need to take to mitigate this change? At this point, the dates for flag Introduction + Maturity are "TBD." 3. **Deprecation warnings**. As a general rule, **all** behavior changes should be accompanied by a deprecation warning. * (a) Always use our standard deprecations module: [https://github.com/dbt-labs/dbt-core/blob/main/core/dbt/deprecations.py](https://github.com/dbt-labs/dbt-core/blob/main/core/dbt/deprecations.py) * (b) This serves two purposes: Signalling the change to the user, and collecting telemetry so we can understand blast radius among users with telemtry enabled. * (c) These warning messages should link back to documentation: [https://docs.getdbt.com/reference/global-configs/legacy-behaviors](https://docs.getdbt.com/reference/global-configs/legacy-behaviors#deprecate_package_materialization_builtin_override) * (d) Even for additive behaviors that are not "breaking changes," there is still an opportunity to signal these changes for users, and to gather an estimate of the impact. E.g. `source_freshness_run_project_hooks` should still include a proactive message any time someone runs the `source freshness` command in a project that has `on-run-*` hooks defined. * (e) The call site for these deprecation warnings should be as close as possible to the place where we’re evaluating conditional logic based on the project flag. Essentially, any time we check the flag value and it returns `False`, we should raise a deprecation warning while preserving the legacy behavior. (In the future, we might be able to streamline more of this boilerplate code.) * (f) If users want to silence these deprecation warnings, they can do so via [`warn_error_options.silence`](https://docs.getdbt.com/reference/global-configs/warnings). Explicitly setting the flag to `False` in `dbt_project.yml` is not sufficient to silence the warning. 4. **Exceptions.** If the behavior change is to raise an exception that prohibits behavior which was previously permitted (e.g. spaces in model names), the exception message should also link to the docs on legacy behaviors. 5. **Backports.** Whenever possible, we should backport both the deprecation warning and the flag to the previous version of dbt Core. 6. **Open a GitHub issue** in the dbt-core repository that is the implementation ticket for switching the default from `false` to `true`. Add the `behavior_change_flag` issue label, and add it to the GitHub milestone for the next minor version. (This is true in most cases, see below for exceptional considerations.) During planning, we will bundle up the "introduced" behavior changes into an epic/tasklist that schedules their maturation. ## After introduction 1. **Maturing flag(s) by switching value from `False` → `True` in dbt-core `main`.** * (a) This should land in **the next minor (`1.X.0`) release of dbt-core**. * (b) If the behavior change is mitigating a security vulnerability, and the next minor release is still planned for several months away, we still backport the fix + flag (off by default) to supported OSS versions, and we strongly advise all users to opt into the flag sooner. 2. **Removing support for legacy behaviors.** * (a) As a general rule, we will not entirely remove support for any legacy behaviors until dbt v2.0. At the same time, we are not committing to supporting them forever (à la Rust editions). We need to strike the right balance between _too fast_ and _never_. * (b) On a case-by-case basis, if there is a strong compelling reason to remove a legacy behavior and we see minimal in-the-wild usage (<1% of relevant projects), we can remove it entirely. This needs to be communicated well in advance — at least 2 minor versions after introduction in dbt Core. * (d) These are project configurations, not temporary feature flags. They add complexity to our codebase; that complexity compounds the more we have, and the longer we have them. Such is the price of maintaining mature v1.* software. ================================================ FILE: docs/guides/parsing-vs-compilation-vs-runtime.md ================================================ # Parsing vs. Compilation vs. Runtime ## Context: Why this doc? There’s a lot of confusion about what dbt does at parse time vs. compile time / runtime. Even that separation is a relative simplification: parsing includes multiple steps, and while there are some distinctions between "compiling" and "running" a model, the two are **very** closely related. It's come up many times before, and we expect it will keep coming up! A decent number of bug reports in `dbt-core` are actually rooted in a misunderstanding of when configs are resolved, especially when folks are using pre/post hooks, or configs that alter materialization behavior (`partitions`, `merge_exclude_columns`, etc). So, here goes. ## What is "parsing"? **In a sentence:** dbt reads all the files in your project, and constructs an internal representation of the project ("manifest"). To keep it really simple, let’s say this happens in two steps: "Parsing" and "Resolving." ### Parsing As a user, you write models as SQL (or Python!) + YAML. For sake of simplicity, we'll mostly consider SQL models ("Jinja-SQL") with additional notes for Python models ("dbt-py") as-needed. dbt wants to understand and define each SQL model as an object in an internal data structure. It also wants to know its dependencies and configuration (= its place in the DAG). dbt reads your code **for that one model,** and attempts to construct that object, raising a **validation** error if it can’t.
(Toggle for many more details.) - (Because your SQL and YAML live in separate files, this is actually two steps. But for things like `sources`, `exposures`, `metrics`, `tests`, it’s a single pass.) - dbt needs to capture and store two vital pieces of information: **dependencies** and **configuration**. - We need to know the shape of the DAG. This includes which models are disabled. It also includes dependency relationships between models. - Plus, certain configurations have implications for **node selection**, which supports selecting models using the `tag:` and `config:` methods. - Parsing also resolves the configuration for that model, based on configs set in `dbt_project.yml`, and macros like `generate_schema_name`. (These are "special" macros, whose results are saved at parse time!) - The way dbt parses models depends on the language that model is written in. - dbt-py models are statically analyzed using the Python AST. - Simple Jinja-SQL models (using just `ref()`, `source()`, &/or `config()` with literal inputs) are also [statically analyzed](https://docs.getdbt.com/reference/parsing#static-parser), using [a thing we built](https://github.com/dbt-labs/dbt-extractor). This is **very** fast (~0.3 ms). - More complex Jinja-SQL models are parsed by actually rendering the Jinja, and "capturing" any instances of `ref()`, `source()`, &/or `config()`. This is kinda slow, but it’s more capable than our static parser. Those macros can receive `set` variables, or call other macros in turn, and we can still capture the right results because **we’re actually using real Jinja to render it.** - We capture any other macros called in `depends_on.macros`. This enables us to do clever things later on, such as select models downstream of changed macros (`state:modified.macros`). - **However:** If `ref()` is nested inside a conditional block that is false at parse time (e.g. `{% if execute %}`), we will miss capturing that macro call then. If the same conditional block resolves to true at runtime, we’re screwed! So [we have a runtime check](https://github.com/dbt-labs/dbt-core/blob/16f529e1d4e067bdbb6a659a622bead442f24b4e/core/dbt/context/providers.py#L495-L500) to validate that any `ref()` we see again at compile/runtime, is one we also previously captured at parse time. If we find a new `ref()` we weren’t expecting, there’s a risk that we’re running the DAG out of order!
### Resolving After we’ve parsed all the objects in a project, we need to resolve the links between them. This is when we look up all the `ref()`, `source()`, `metric()`, and `doc()` calls that we captured during parsing. This is the first step of (almost) every dbt command! When it's done, we have the **Manifest**.
(Toggle for many more details.) - If we find another node matching the lookup, we add it to the first node’s `depends_on.nodes`. - If we don’t find an enabled node matching the lookup, we raise an error. - (This is sometimes a failure mode for partial parsing, where we missed re-parsing a particular changed file/node, and it appears as though the node is missing when it clearly isn’t.) - Corollary: During the initial parse (previous step), we’re not actually ready to look up `ref()`, `source()`, etc. But during that first Jinja render, we still want them to return a `Relation` object, to avoid type errors if users are writing custom code that expects to operate on a `Relation`. (Otherwise, we’d see all sorts of errors like "NoneType has no attribute "identifier.") So, during parsing, we just have `ref()` and `source()` return a placeholder `Relation` pointing to the model currently being parsed. This can lead to some odd behavior, such as in [this recent issue](https://github.com/dbt-labs/dbt-core/issues/6382).
## What is "execution"? **In a sentence:** Now that dbt knows about all the stuff in your project, it can perform operations on top of it. Things it can do: - tell you about all the models that match certain criteria (`list`) - compile + run a set of models, in DAG order - interactively compile / preview some Jinja-SQL, that includes calls to macros or ref’s models defined in your project Depending on what’s involved, these operations may or may not require a live database connection. While executing, dbt produces metadata, which it returns as **log events** and **artifacts**. Put another way, dbt’s execution has required inputs, expected outputs, and the possibility for side effects: - **Inputs** (provided by user): project files, credentials, configuration → Manifest + runtime configuration - **Outputs** (returned to user): logs & artifacts - **Side effects** (not seen directly by user): changes in database state, depending on the operation being performed ### Compiling a model We use the word "compiling" in a way that’s confusing for most software engineers (and many other people). Most of what’s described above, parsing + validating + constructing a Manifest (internal representation), falls more squarely in the traditional role of a language compiler. By contrast, when we talk about "compiling SQL," we’re really talking about something that happens at **runtime**. Devils in the details; toggle away.
The mechanism of "compilation" varies by model language. - **Jinja-SQL** wants to compile down to "vanilla" SQL, appropriate for this database, where any calls to `ref('something')` have been replaced with `database.schema.something`. - dbt doesn’t directly modify or rewrite user-provided **dbt-py** code at all. Instead, "compilation" looks like code generation: appending more methods that allow calls to `dbt.ref()`, `dbt.source()`, and `dbt.config.get()` to return the correct results at runtime.
If your model’s code uses a dynamic query to template code, this requires a database connection. - At this point, [`execute`](https://docs.getdbt.com/reference/dbt-jinja-functions/execute) is set to `True`. - e.g. `dbt_utils.get_column_values`, `dbt_utils.star` - Jinja-SQL supports this sort of dynamic templating. dbt-py does not; there are other imperative ways to do this, using DataFrame methods / the Python interpreter at runtime.
Compilation is also when ephemeral model CTEs are interpolated into the models that `ref` them. - The code for this is *gnarly*. That’s all I’m going to say about it for now.
When compiling happens for a given node varies by command. - For example, if one model’s templated SQL depends on an introspective query that expects another model to have already been materialized, this can lead to errors. - In `dbt run`, models are operated on in DAG order, where operating on one model means compiling it and then running its materialization. This way, if a downstream model’s compiled SQL will depend on an introspective query against the materialized results of an upstream model, we wait to compile it until the upstream model has completely finishing running.

The outcome of compiling a model is updating its Manifest entry in two important ways: - `compiled` is set to `True` - `compiled_code` is populated with (what else) the compiled code for this model ### Running / materializing a model A model’s `compiled_code` is passed into the materialization macro, and the materialization macro is executed. That materialization macro will also call user-provided pre- and post-hooks, and other built-in macros that return the appropriate DDL + DML statements (`create`, `alter`, `merge`, etc.) (For legacy reasons, `compiled_code` is also available as a context variable named [`sql`](https://github.com/dbt-labs/dbt-core/blob/16f529e1d4e067bdbb6a659a622bead442f24b4e/core/dbt/context/providers.py#L1314-L1323). You'll see it referenced as `sql` in some materializations. Going forward, `model['compiled_code']` is a better way to access this.) ## Why does it matter? Keeping these pieces of logic separate is one of the most important & opinionated abstractions offered by dbt. - **The separation of "control plane" logic** (configurations & shape of the DAG) **from "data plane" logic** (how data should be manipulated & transformed remotely). - You must declare all dependencies & configurations ahead of time, rather than imperatively redefining them at runtime. You cannot dynamically redefine the DAG on the basis of a query result. - This is limiting for some advanced use cases, but it prevents you from solving hard problems in exactly the wrong ways. - **The separation of modeling code** ("logical" transformation written in SQL, or DataFrame manipulations) **from materialization code** ("physical" state changes via DDL/DML)**.** - Every model is "just" a `select` statement (for Jinja-SQL models), or a Python DataFrame (for dbt-py models). It can be developed, previewed, and tested as such, *without* mutating database state. Those mutations are defined declaratively, with reusable boilerplate ("view" vs. "table" vs. "incremental"), rather than imperatively each time. ## Appendix
Click to toggle notes on parsing ### Notes on parsing - **dbt has not yet connected to a database.** Every step performed thus far has required only project files, configuration, and `dbt-core`. You can perform parsing without an Internet connection. - There is a command called `parse`, which does **just** "parsing" + "resolving," as a way to measure parsing performance in large projects. That command is the fastest way to write `manifest.json` (since v1.5). - In large projects, the parsing step can also be quite slow: reading lots of files, doing lots of dataclass validation, creating lots of links between lots of nodes. (See below for details on two potential optimizations.) ### Two potential optimizations 1. [**"Partial parsing."**](https://docs.getdbt.com/reference/parsing#partial-parsing) dbt saves the mostly-done Manifest from last time, in a file called `target/partial_parse.msgpack`. dbt **just** reads the files that have changed (based on file system metadata), and makes partial updates to that mostly-done Manifest. Of course, if a user has updated configuration that could be relevant globally (e.g. `dbt_project.yml`, `--vars`), we have to opt for a full re-parse — better safe (slow & correct) than sorry (fast & incorrect). 2. [**"Reusing manifests."**](https://docs.getdbt.com/reference/programmatic-invocations#reusing-objects) Note that this is taking "full control," and there are failure modes (example: [dbt-core#7945](https://github.com/dbt-labs/dbt-core/issues/7945)).
================================================ FILE: docs/roadmap/2022-05-dbt-a-core-story.md ================================================ # Let's talk about dbt Any writer, any artist, who finds themselves doing work that invites [thoughtful, clear, forward-looking](https://thecreativeindependent.com/people/doreen-st-felix-on-entering-the-world-of-criticism) criticism should be counted lucky. As a person who builds dbt Core for a living, I feel lucky to have [Pedram](https://pedram.substack.com/p/we-need-to-talk-about-dbt). He's a person who understands exactly what dbt is, and who cares deeply about its future. It's possible that we're not building the right things. It's certain that we're not doing a good enough job of communicating what we're building. How can we expect to know the former without doing the latter? In the end, [transparency always wins](https://www.getdbt.com/dbt-labs/values/#:~:text=Transparency%20always%20wins.). ### What is this doc? This is my story of dbt Core in 2022: everything we're trying to build, and why. This Markdown table is your `tl;dr`. The rest is commentary. | Version | When | Stuff | Confidence* | |-----|----------|----------------------------------------------------------------------------------|------------| | 1.1 ✅ | April | Testing framework for dbt-core + adapters. Tools and processes for sustainable OSS maintenance tools. | 100% | | 1.2 | July | Refactoring core materializations (especially incremental). Built-in support for grants. "Cross-db" macros in dbt-core + plugins. Python-language models (beta). | 95% | | 1.3 | October | Python-language models. Built-in support for UDFs. Exposures / external nodes. | 80% | | 1.4 | Jan 2023 | Mechanisms for cross-project lineage (namespacing). Performance tune-up. Start incorporating SQL grammar (linting & lineage). | 50% | `updated_at: 2022-04-12` **\*Confidence?** I mean the certainty of which things we build for which release. dbt Core is relied on by many. The most important things stay the same, but some of the details vary. When looking 6+ months into the future, I expect us to do ~half of the things I've listed above and below, and shuffle ~half of them out in favor of the things that you will be telling us we need to build. ### Why this doc? A product philosophy of dbt, taking inspiration [from Perl](https://www.amazon.com/gp/feature.html?ie=UTF8&docId=7137#:~:text=%22Easy%20things%20should%20be%20easy%2C%20and%20hard%20things%20should%20be%20possible.%22): **"Make the easy things easy, and the hard things possible."** The analyst in you probably has some questions: - Which things? The entire analytics engineering workflow? - What ought to be easy? What deserves to be hard? - Who's making them, and when? I haven't been doing a good enough job of answering those questions, in a single easy-to-bookmark place. I don't do my long-form writing in a personal blog, I do it [in a GitHub repo](https://github.com/dbt-labs/dbt-core/issues?q=is%3Aissue+is%3Aopen+commenter%3Ajtcohen6+), and trawling through issues is a leisure activity enjoyed by precious few. dbt Core is a product, sort of; a CLI tool, for many; a software project, for sure; an open source project, definitively; but above all it's a guide, a set of opinionated practices about how you should do this work of data transformation, testing, documentation. We believe these are many (not all) of the essential components of the thing we call analytics engineering, which maybe you just call getting your job done. If you're reading this, even if you don't pay to use dbt, you've more than likely invested in it a meaningful quantity of your time, care, and attention—not to mention, your business logic. It matters a lot to me that you know how we're thinking about dbt's ongoing development. (That also means that the story I can tell in this document is necessarily incomplete: I can tell you what dbt Core can do for you, but just as important is [what you and other skilled practitioners can do with dbt Core](https://docs.getdbt.com/blog).) That is to say, _mea culpa_. The best time to have written this document was a few months ago; the second best time, I hope, is right now. ### Where are we now? dbt is a workflow tool. It was originally built by a team of full-time analytics consultants (us). We built it because there were jobs to be done, literally, and dbt made us unreasonably effective at doing them. As a fresh-faced Data Analyst, with a seat in the engine room (the only room there was), I was able to deliver high-quality work to clients ahead of schedule, [and go home early](https://www.getdbt.com/dbt-labs/values/#:~:text=We%20work%20hard%20and%20go%20home.) (or to the beach). The people who built dbt had the same names, faces, and hand gestures as the people who used it every day. That was our blessing and our curse. Each time we found a new idiosyncrasy in analytical databases, we coded it into dbt. Late-bound views on Redshift? Always. Quoted identifiers on Snowflake? Never. These turned into magic incantations, whose full meaning could only be revealed, slowly, through immersion in our shared ritual practice. To the uninitiated, the curly-braced characters of the incremental materialization may more resemble cuneiform than Python. As an end user, `dbt-core` felt like a magical experience, even as it grew to be an indecipherable mess of a codebase. Logging was random. Metadata was at once invaluable and totally undocumented. Our automated testing and release processes were "prayers up" undertakings. This makes sense for where our priorities were. It also meant that, at the start of 2021, two things were true: 1. `dbt parse` was really, really freaking slow for any project with more than 500 models in it. 2. dbt Core was major-version-zero software, a fundamentally unstable substratum, used in production by thousands of people. Those were our two top priorities in 2021. Speed up dbt-core parsing, especially in development. Work up to v1.0, and release it. We made steady progress on both of those, not without some public pains along the way. There were moments when I, like Phoebe, wondered if we'd make it through December. We did. [Read all about it, featuring a dorky picture of me, and my genuine gratitude to all the colleagues and community members who made it happen](https://www.getdbt.com/blog/dbt-core-v1-is-here/). Those were two big accomplishments. (We also grew the team of people working full-time on dbt Core from three to eight.) That doesn't mean we're done developing Core, though—not even close. I reprised the exercise. At the start of 2022, three things were true: 1. **There are easy things we need to make easy, and one hard thing we need to make possible.** Users of dbt are still totally constrained to what they can write in SQL (or Jinja-SQL). At the same time, there's still too many things that, while doable, require lots of insider knowledge, custom code, or magic incantations. dbt Core needs some **new constructs,** a mix of totally-new and just-plain-easier things. 2. **dbt-core is still missing the right modular interfaces for long-term development.** This is a codebase that grew organically over time. Its layout today is _much_ more sensible than it was a year ago, but we're not all the way there. The tight couplings, of tasks with configs with CLI initialization, tax our ability to build next-level capabilities with confidence, including all the stuff in (1). Not just that: it prevents community members and technology partners from building truly next-level integrations. dbt Core needs some **new interfaces.** 3. **And yet: dbt Core is major-version-one software.** There are commitments to backwards compatibility and ease of upgrade that we take incredibly seriously. This is a necessary constraint that's always top-of-mind for us when building the new (1) and refactoring the old (2). I believe those three concerns are and remain relevant for every user of dbt. There are going to be specific things we build along the way that might be less relevant for you as an individual—support for other databases, if you happily use Redshift/Snowflake/BigQuery; support for large projects, if you're at a happy medium; better programmatic interfaces, if you need only to invoke dbt from the CLI—but they all weave together into the larger story we're telling this year. There's inevitably some "inside dbt-ball" in this document. It can't just be a list of user-facing features we're shipping, or I wouldn't be doing the story justice. We're here to take big swings, the kinds of things I get excited about demo'ing at Staging and Coalesce—but we're also here to maintain critical infrastructure, to grow a roster of talented position players who can put in a solid nine innings day after day. So this will also discuss some behind-the-scenes work we're doing, and the people who rely on it. It's true that many of the people now building dbt aren't themselves users of it—they're talented software engineers who know the things we need to do to scale large, performant, maintainable codebases. It's my job, and the job of my long-time colleagues, to ensure that dbt continues to feel dbtonic along the way. # Program for the 2022 season Welcome to the official program for "dbt: A Core Story," the devised theatre piece where everyone has a part to play. First, a few housekeeping items. We'll be releasing a new minor version of dbt Core every 3 months: April, July, August, January (2023). We put out our new [adapter](https://docs.getdbt.com/docs/available-adapters) versions at the same time. Adapters maintained by other people might be available for upgrade a few weeks later. For each new minor version, we put up a ["migration guide"](https://docs.getdbt.com/docs/guides/migration-guide) in the docs. Everything we work on is public on GitHub. I attach issues to [milestones](https://github.com/dbt-labs/dbt-core/milestones) for upcoming releases. We're using [discussions](https://github.com/dbt-labs/dbt-core/discussions) now for bigger-picture conversations—stuff we're thinking about, even if we can't write the code for it right away. If the dbt Community Slack feels overwhelming, discussions still feel, for now, like a slower-paced forum for dreaming up ideas, talking through caveats and rough edges. I really welcome your voices there. As for this document: Each quarter, I'll swing back and update it, with the edits preserved in git. If you see something you don't like, you'll know whom to `blame`. ## Act I: Sharpening our tools (January–April) ### Scene 1: Rewrite the way we test dbt-core and adapters This probably isn't news to you: dbt-Jinja code is pretty hard to test. I'm not talking about tests for data quality, but tests of "application code," in the sense of making sure that the incremental materialization is doing the right thing in the right circumstances. We've been feeling the pain here, too—it's really hurt our ability to onboard new engineers to the team, and ship features quickly and confidently. It's also hurt our ability to help the people who, whether for their job or with the generosity of their free time, maintain the adapter plugins that let dbt talk to dozens of databases. As core maintainers, we owed them a lot more than we could give them. So, we wrote a new testing framework, based in `pytest`, that's much easier to extend and debug. Who benefits? Concretely, these automated tests offer us a way to stamp our approval on adapter plugins developed and maintained outside dbt Labs, while ensuring a consistent baseline experience, whichever adapter you use. I think everyone who wants healthy competition between database vendors, and to know that you can have the same magical experience of using dbt across them. Ask your neighborhood adapter maintainer if the new testing framework is working well for them. Is the new `pytest`-based framework actually the dbt feature called "unit testing" in disguise? Funny you asked—no, but I do think it's a step on the way there. It's possible to use this functional testing framework to quickly spin up "projects," with fixture inputs and outputs, that check the behavior of a macro. We can test that macro for consistency across many databases. (Check this out in that most complex project of them all, [`dbt_utils`](https://github.com/dbt-labs/dbt-utils/pull/588).) The dream is indeed to expose a dbt-code-only version of this workflow, along the lines of what people have been cooking up in https://github.com/dbt-labs/dbt-core/discussions/4455. (We also intentionally built this in a way that avoids locking us into Jinja-y code forever. Jinja is a tool, just like Python. It gets us a lot right now. There will come a time when it's no longer the right fit, and we'll need to be ready to move away from it.) ### Scene 2: Sustainable maintainership Since December, we've added three new folks to the team. There's 10 of us now! You may have seen some newer names, if you're used to hanging around our GitHub: @leahwicz @gshank @nathaniel-may @iknox-fa @emmyoop @McKnight-42 @VersusFacit @ChenyuLInx @stu-k That growth was in parallel to the growth of `dbt-core` active users, and the growth of issues in the `dbt-core` repo. We get 2 or 3 new open source issues every calendar day. When it was just me responding, I would take a week of vacation, and come back to a stack of a dozen or more. At the same time, I was learning a **ton** about dbt—what it is today, what it should be—by reading and responding to all your issues. That's invaluable stuff for the people building dbt, and it's stuff that was all living in my head only. So, for both reasons, I've been sharing the load with the engineering team building dbt Core. This requires more time, and process, but it also requires a level of rigor and organization that I didn't have when it was just me. Bugs don't get lost in the shuffle, and get resolved more quickly. Is this groundbreaking stuff? No, but it is major-version-one stuff. Is every one of these issues game-changing? No, but they matter a lot to the person who opens them—and I believe it matters a lot to you, a person who has invested meaningfully in standardizing on dbt Core. You should want to know that that core dbt functionality is well maintained, to know that when you run into a bug or have an idea there will be a human on the other end to receive and respond to it. To that end, I want to share a new doc we put together over the past few months: [Expectations for OSS Contributors](https://docs.getdbt.com/docs/contributing/oss-expectations). It took some doing, but we now live in a world where: - every issue gets a timely response - every community PR gets code review (with easier code checks! and conflict-free changlog entries!) - they're not all (or even mostly) from Drew or me I care about making contribution accessible, and making dbt a thing we can keep building together. I care a lot about what you have to say and think about feel about dbt. I still read every new issue, even if it's not me who responds. Maybe I'll stop, someday. But for now I read them all. ### Scene 3: v1.1 (Gloria Casarez) At the end of April, [we released v1.1](https://github.com/dbt-labs/dbt-core/releases/tag/v1.1.0), named for Gloria Casarez, the activist and advocate for the rights of LGBTQ+ people in Philadelphia. This release had [some good stuff in it](https://docs.getdbt.com/docs/guides/migration-guide/upgrading-to-v1.1), including a few boundary-pushing community contributions—but I'll be the first to admit that it was a lighter release than others in recent memory. The biggest work was sharpening our tools, and sharpening the team. We'd rather do this work upfront, so we can build and ship the bigger stuff, with alacrity and assurance. ## Act II: Ergonomic interfaces (May–July) This is where we are now. There are three big focus areas for the Core team in these three months. Two of them are user-facing, and one of them is software-facing. This is work we've just kicked off, and if you choose to look closely, you can see the full flurry of issues and PRs flying around. ### Scene 1: "Adapter ergonomics" For the full story, and to leave your thoughts, check out the discussion: https://github.com/dbt-labs/dbt-core/discussions/5091. Meanwhile, I'll give the quick & dirty version. **First, grants.** These have been, [in Tristan's words](https://roundup.getdbt.com/p/the-response-you-deserve), "super-budget literally forever." Database permissions matter; they're an easy thing, and yet dbt makes them hard. I've lost track of the number of issues (but linked to many of them in https://github.com/dbt-labs/dbt-core/issues/5090) where users have run into bad experiences with `pre-hook` and `post-hook`, which are [rendered a little bit differently](https://docs.getdbt.com/docs/building-a-dbt-project/dont-nest-your-curlies), in ways that can be very useful and very confusing. Honestly, we want to move hooks into the category of "advanced functionality"—use only if you must!—but we can't do that while `grants` remain hooked on hooks. There's another reason to do this work now: BigQuery added support for DCL statements [last summer](https://cloud.google.com/bigquery/docs/release-notes#June_28_2021). Databricks has built a [brand-new unity catalog](https://databricks.com/product/unity-catalog). Many warehouses are adding more-complex permissions—dynamic data masking, role-based access policies on rows and columns. If dbt can be a forcing function for baseline consistency among data warehouse vendors—make the easy stuff easy—it provides a foundation, and clears human brain space, for the trickier case-dependent capabilities. **Second, materializations:** specifically, the 'incremental' materialization, and the sorta-real thing that is "incremental strategy" on a handful of adapters. These are powerful, complex, misunderstood. We need more sensible code, more sensible docs. Lots of people (adapters, users) want to customize or contribute to materialization logic. It's really tricky to write today, in ways that go beyond the standard (fair) complaints about macros. Our functional testing framework helps here, by giving us the ability to define the superset of behaviors in `dbt-core`, and to let each adapter or user opt into and fine-tune the ones they wish to support. **Third, `dbt-utils`**, which is getting unsustainably big. As part of [the work of splitting it up](https://github.com/dbt-labs/dbt-utils/discussions/487), we decided that its lowest-level building blocks—the "cross-db" macros so useful to authors of other packages—really belong in `dbt-core` and plugins. This is an opportunity for us to continue expanding the "dbt-SQL" language, such as it exists today, for the benefit of the package maintainers who contribute so much to our ecosystem. **The v1.2 release** will include all of the above, with a beta prerelease in June, and a final releaes to arrive in mid-July. What else? v1.2 will also to include [support for ratio metrics](https://github.com/dbt-labs/dbt-core/issues/4884), to power dbt's rapidly iterating semantic layer. Last and never least, it will include the dozen or so capabilities for which community members have taken the initiative to contribute. (Fun fact: thanks to our new `changie`-powered changelog, you can always catch a glimpse of [upcoming changes](https://github.com/dbt-labs/dbt-core/tree/main/.changes/unreleased), merged but not yet released.) ### Scene 2: Modular programmatic interfaces If you've hung around the `dbt-core` GitHub repo lately, you may have seen that we've started tagging a lot of issues with `tech_debt`, as well as with `Team:Language`, `Team:Execution`, `Team:Adapters`. Not long ago, we had one team (one person), one codebase, one tightly coupled application that did all of the things, from beginning to end, without clear stopping points in the middle. Now, I'm truly lucky to work with a team of talented software engineers, as we try to make the experience of developing and interfacing with `dbt-core` as no-bs as it is to use. We believe the highest-leverage way to do this is by splitting up the team into specialties, and the codebase into modular interfaces. For too long now, the [docs for dbt's "Python API"](https://docs.getdbt.com/docs/running-a-dbt-project/dbt-api) have cautioned: > It _is_ possible to import and invoke dbt as a Python module. This API is still not contracted or documented, and it is liable to change in future versions of `dbt-core` without warning. Please use caution when upgrading across versions of dbt if you choose to run dbt in this manner! There are two big parts of this initiative that I'm hoping to tackle in July, after we've readied v1.2 for prerelease testing: 1. Decouple the CLI from the `dbt-core` "library." Support actual programmatic interfaces into initialization and tasks. Unbundle the massive blobs of config that get passed around during initialization today. 2. **Structured logging**, as the future of (real time) dbt metadata. In v1.0, we replaced all our "legacy" logging with a real event system and structured interface. We need to extend that system to handle errors/exceptions, too—a pain to debug when dbt spits back no or little information—and we need to add much more logging than we have today. There's so much more valuable information that dbt collects _while_ it runs, which we want to expose for you and everyone else. (Think: model table statistics _during_ `dbt run`, not just after `docs generate`.) Eventually, we see these logs eclipsing `manifest.json`—which doesn't go away, exactly, but is no longer the one (giant) file that answers every question about a dbt project. These things don't pay dividends as quickly as user-facing functionality, but we believe they more than pay for themselves in the long run. Here are the three ways I can think of: 1. Better interfaces makes us faster at onboarding new engineers to the Core team, and building new features in dbt Core. 2. External contributors (you!) can contribute code to `dbt-core` more easily: to find the right module, the right interfaces to extend, and fewer pitfalls that sap enthusiasm while doing it. 3. It's easier to build other tools on top of dbt Core. That includes dbt Cloud—which, candidly, has been hamstrung in its ability to develop differentiated experiences by the lack of reliable interfaces in Core. But it also includes way-cool tools, such as [`sqlfluff`](https://github.com/sqlfluff/sqlfluff) and [`fal`](https://github.com/fal-ai/fal), that already hook into `dbt-core`'s officially undocumented Python interface. Let's make that the practice, not the anti-pattern. ### Scene 3: Python-language dbt models (!!) Sorry to bury the lede for this one. This is shaping up to be the groundbreaking new entrant in the Core roadmap in 2022. If do our job well, we'll have made a hard thing possible; if we do it very well, we might just manage to make it feel easy. Over the past few weeks, I've been drafting thoughts for a forthcoming discussion, and Core engineers have been playing around with a little bit of experimental code. We're hoping to have something that's beta-ready by July. I'll have more to say about this next week, including my personal take on le grand débat of SQL ~versus~ and Python. For now, here's a taste: ```python def model(dbt): # look familiar? dbt.config( materialized='table', ) # this knows about the DAG upstream_model = dbt.ref("upstream_model") upstream_source = dbt.source("upstream", "source") # dataframe-style transformations sample = upstream_model.where(col("city")=="Philly") ... # this too import numpy as np ... # your final 'select' statement df = ... # return a dataset, just like SQL — dbt takes care of the rest return df ``` Once the beta is available, we'll want to hear everyone's thoughts. Our plan is to incorporate your thoughts, ideas, and feedback, as well as some complementary features we have in mind, as we build toward... ## Act III: Unified Lineage (August–October) v1.3 takes us to Coalesce, where I might just manage to meet (some of) you in person for the very first time. The theme of this release will be **unifying lineage.** We're pulling some new pieces of transformation logic into dbt's execution model, and offering more ways to push the rich metadata from your project out into the world. Things like: - Python-language dbt models, ready for production - Native support for UDFs (https://github.com/dbt-labs/dbt-core/issues/136: an oldie, a goodie, and newly relevant!) - Improvements to exposures, and external nodes, and maybe those are actually the same thing? (https://github.com/dbt-labs/dbt-core/discussions/5073) What else? 'Tis not so sweet now as it was before? I think there will be lots to discuss, some to debate, things to try out, a few to throw away, and maybe just maybe we'll find ourselves with an expanded standard, a thing we're proud to still call dbt Core. Thoughts? For my part, I think **unit testing** model code becomes an even more pressing question when: - dbt models can be written in Python, a language that _wants_ to be unit-tested even more than SQL - dbt nodes (models?) can return _functions_, with testable inputs and outputs - dbt can be interacting with external services that live outside the database I rest well knowing we'll be better equipped to solve that problem, using all that we learn about Python-language models, and the components of the `pytest` framework that we put together in the first few months of the year. Of course, all of that will be in addition to the ongoing work that we're doing around metrics and the semantic layer, first previewed at last year's Coalesce. It's not my place to spoil any of the surprises there. ## Act IV: Reflect and reinvest (November–January) v1.3 takes us through Coalesce, which is always a bit of an event horizon for the Product team at dbt Labs. Looking further than six months away, I'm only ~50% certain about which initiatives we'll be tackling into the new year. For now, I can share the ones that feel most important to me: **Better mechanisms for cross-project lineage** (https://github.com/dbt-labs/dbt-core/discussions/5244), to support large projects that should really be multiple projects. That would include support for namespaced models, at least in different packages. This is an initiative I care about a lot, and I'm trying to push up earlier if possible. **Building a SQL grammar**, or build compelling features on top of another open source one. Catch syntax errors at parse time, lint code, auto-format. Also, detect column-level lineage. This gets complicated, because of how some dbt models are templated dynamically, but I trust that we'll be able to find reasonable compromises when people are doing more-complex things to template model SQL. **Revisiting performance.** It will have been 2 years since we first began performance work in earnest, in November 2020. Then, we had the goal of speeding up dbt startup by 100x. Now, we should see where the bottlenecks are—I sense they're around database caching and cataloging. Then, we chose to write some low-lying parts of dbt-core in Rust, instead of Python. Now, with more clearly defined interfaces, it might make sense to rewrite some parts again. ## Epilogue: What's missing? It's not all accounted for above. There are many important things we won't be able to build. I can name a few; I'm sure you have more than these, and I want to hear them. **Meaningfully rebuilding, rewriting, or reinvesting in the `dbt-docs` site.** We're going to keep this up & running, and we'll review community PRs—but we haven't built a team of Angular (!) engineers. The fact that `dbt-core` ships with an auto-generated documentation site is essential; the next-level version of this thing needs to be built on a more powerful engine than a static site reading JSON files. **DRYer configuration.** YAML reuse. Macros in `.yml` files. Vars of vars, and docs blocks on docs blocks. Why not solve it sooner? I think column-level lineage has a part to play here; anything we build before then risks emphasizing the wrong details. But I appreciate the real frustrations around config duplication and code generation in the meantime. dbt can be verbose, not unlike yours truly. **Native plugins for VS Code, PyCharm, Vim, ...**—I know. I want these things _very_ badly. I think the role of the Core team, first and foremost, is to solidify and harden the programmatic interfaces that I mentioned above. There's no sense building a thing on a rocky (let alone undocumented) foundation. This is also the best way for us to expand our reach: by enabling our colleagues, yes, and also partners, and also every member of the community with a hack day, to build better and cooler stuff, and build it with confidence. It's the kindness of strangers, and then some—it's open source. **What about ``?** Tell me which one, and I'll give it another read. I'm `@jerco` in dbt Community Slack. ================================================ FILE: docs/roadmap/2022-08-back-for-more.md ================================================ # dbt Core: Back for more (August 2022) Me again! As promised, three months later. Since I wrote to you in May, a few things have happened: - We released a new final version of dbt Core (v1.2) - We put out a beta of the next version (v1.3) - Two new colleagues on the Product team, Cody and Florian, joined me in imagining the future of dbt Core This update is going to be a touch shorter, as we have many fewer surprises to share. That's a good thing. We've been talking more in public about what we're building: on [the blog](https://www.getdbt.com/blog/), on [the other (cooler) blog](https://docs.getdbt.com/blog), at [Staging](https://www.getdbt.com/blog/staging-highlights-the-latest-from-dbt-labs/), and in [GitHub discussions](https://github.com/dbt-labs/dbt-core/discussions). And, while the meta conceit was good once ("welcome to my mind, it's an improvised play in four acts, how's that for stream[-of-consciousness] transformation"), I don't want to risk overdoing it. Here's what you came for: | Version | When | Namesakea | Stuff | Confidenceb | | ------- | ------------- | -------------- | ----- | ------------ | | 1.1 ✅ | April | Gloria Casarez | Testing framework for dbt-core + adapters. Tools and processes for sustainable OSS maintenance. | 100% | | 1.2 ✅ | July | Henry George | Built-in support for grants. Migrate cross-db macros into dbt-core / adapters. Improvements to metrics. | 100% | | 1.3 🌀 | October | | Python models in dbt. More improvements to metrics. (Other things, too—but those are the main events.) | 95% | | 1.4 ⚒️ | Jan 2023 | | Behind-the-scenes improvements to technical interfaces. A real, documented Python API/library, with an improved CLI to wrap it. Further investments in structured logging. | 80% | | 1.5+ 💡 | Next yearc | | Multi-project deployments: split up the monolith. The same DAG, more active: external orchestration. Python in dbt: next steps. Start imagining dbt Core v2. | 50% | `updated_at: 2022-08-31` aAlways a [phamous Philadelphian](https://en.wikipedia.org/wiki/List_of_people_from_Philadelphia), true to our roots. If you have ideas or recommendations for future version namesakes, my DMs are open :) bdbt Core is, increasingly, a standard-bearer and direction-setter. We need to tell you about the things we're thinking about, long in advance of actually building them, because it has real impacts for the plans of data teams and the roadmaps of other tools in the ecosystem. We also know that we don't know now everything we will know a year from now. As new things come up, as you tell us which ones are important to you, we reserve the right to pivot. So we'll keep sharing our future plans, on an ongoing basis, wrapped in a confidence interval. cWe're sticking with one minor version release per quarter, for the foreseeable. I haven't split those out here because, 6+ months into the future, we care more about the _what_ and the _why_ than the _when_. As we get closer, we'll be able to detail the more-specific functionality that might land in specific releases. Note too that these ideas, though we're already devoting meaningful time and effort to thinking through them, are not definite commitments. # Introducing… Two new Product Managers dedicated to thinking about dbt Core full time! Cody, Florian, could you say a few words of introduction? What brought you to dbt Labs? What are some things you're excited about doing here? > Cody: Hello! My name is Cody, or [@lostmygithubaccount](https://github.com/lostmygithubaccount). The first programming language I learned was Python, my background is in electrical engineering and data science, and my previous experience as a product manager centered around operationalizing machine learning systems (MLOps). I'm excited about empowering you to build intelligent systems, the role dbt Labs has to play in converging the data & AI stacks while breaking down organizational silos, and to work on open-source software with an awesome community supporting! > Florian: Bonjour friends! You can find me around here under [@fleid](https://github.com/fleid). I started a long time ago as a SQL developer, writing monstrous queries to feed complex reports on top of operational databases. Version control was adding a version number in the filename of each report. No logic was shared across reports. Performance was horrendous. Overall data quality was… abysmal. Since then, by engaging in various communities I was able to learn a couple of neat tricks, like modern software engineering practices and dimensional modeling. So now I'm delighted to be able to contribute back to a community that supports a tool and a workflow that make these pitfalls obsolete. I only wish dbt existed all of these years ago! As for the future, I'm ramping up to help on the adapters, starting with a big backlog of issues and PRs that needs attention and love. I hope we can get to a better place, more reactive and ideally proactive, by end of the year. Please reach out if you've feelings or opinions to share about that space! (Jeremy again.) It's been a lot of fun to bring both of you into the fold over the past few months. The music does slow down when we add new chairs—there are processes that have become muscle memory for me, and turning those into collective endeavors takes time—but it's a thing worth doing. I can already say that our conversations have given me a huge boost of energy and excitement, in terms of what we can build together. We're grappling with some big questions—the biggest question being, what is dbt, *really?*—that I realize I've been taking for granted. I hadn't appreciated the risk that my old instinctive answers had calcified into constrained visions, or a GitHub issue comment from 2019 into binding precedent. Now that dbt Core is safely stable, it's the right time to ask those questions again. To start developing answers, always from the same principled viewpoint; to refine and critique them together; and to follow the logic where it leads. # Commentary Hopefully, you're already well aware of, and happily making use of, the capabilities that shipped in dbt Core v1.1 and v1.2 earlier this year. If you're not, the [upgrade guides](https://docs.getdbt.com/guides/migration/versions) are a good place to get up to speed. ## v1.3 (October): Coalesce I got to see a very small number of you a few months ago at a London dbt meetup, where I presented May's edition of roadmap. I'm looking forward—we're all (!) looking forward—to seeing and talking with many more of you in October for [Coalesce](https://coalesce.getdbt.com/). For those who are able and comfortable to make the trip to New Orleans, London, or Sydney—and for all the many more who are planning on a "classic Coalesce" experience, from the comfort of your laptop—we're very excited to have you. The big thing coming in dbt Core v1.3 is support for Python models. You already knew that. All the details are in [the beta docs](https://docs.getdbt.com/docs/building-a-dbt-project/building-models/python-models), so give 'em a read now if you haven't yet. There are a couple of FAQs (Frequently Associated Qualms) that I want to address, here and now, in case you're thinking one of them: **Will I need to know Python to start using dbt?** Not at all. At the same time, if you don't know it, and want to learn, that's great! The open source Python-for-data ecosystem is powerful, and often also overwhelming. We'll be creating resources to help light the way, plus highlighting guides, walkthroughs, and recommendations made by members of the community. **Is now the time to start experimenting with advanced statistical processing, predictive analytics, …?** Maybe! Or maybe not. Developing a solid, foundational set of data models should always come first. Deeply understand your data. Use it to power reliable analytical reporting. Whether you're ready to take the next step now or later, dbt is here to make it possible. **Does adding Python fundamentally change the nature of dbt Core?** Honestly, yes and no. - **No:** For all the SQL you're already writing and running successfully, we think you should keep it that way. SQL remains the most expedient and accessible way to write most forms of data transformation. We are *not* going to stop investing in dbt's support for SQL—and in Jinja as its templating engine, for the foreseeable future. (dbt-core v1.3 also includes a long-awaited upgrade to Jinja3, which should help folks installing alongside other Jinja-powered tools.) - **Yes:** Adding support for Python has clarified our thinking on multilingual dbt. It's helped us realize that the real value of dbt is not in Jinja-templated SQL. (Anyone can build that in a weekend; many have.) It's the framework, the environment-aware workflow, the DAG, the integrated testing and documentation—more things than I can name here. That's more language-agnostic than you might expect. At the same time, each language brings its strengths; there will be things you can do in Python that you cannot do in Jinja-SQL, and vice versa. Note that, while Python is the main event, it's not the only new thing coming in v1.3. Every release includes a bunch of exciting community contributions, and this one's got [something long awaited](https://docs.getdbt.com/reference/resource-configs/docs#custom-node-colors?version=1.3). We're also making improvements to metrics to support the launch of the **dbt Semantic Layer**. That's been a *huge* initiative, long in the works, and a long time coming. It's not my place to offer spoilers. I recommend you check out [Drew's blog](https://www.getdbt.com/blog/dbt-semantic-layer/), if you haven't already, and [join us for the show](https://coalesce.getdbt.com/). ## v1.4 (January): For us, for you, for Core After Coalesce, we'll be taking stock of all that we built this year, and all we're looking to build next year. We are dedicating the months of November through January to dbt Core's technical foundations. (Plus: taking some well-deserved vacation over the holidays.) This work is comprised of two big initiatives: 1. **API + CLI:** Improving and documenting dbt-core's internal Python APIs. Creating a new and better-constructed CLI to wrap around it. To be clear, this CLI will support all the same commands, flags, and arguments as it does today. 2. **Event + logging interface.** Supporting type-safe, language-agnostic ways to ingest structured logs produced by dbt-core. This will enable other tools (ours and yours) to provide reliable observability around dbt runs, as well as more-digestible and realer-time metadata. Over a longer term, providing more information in log events where it's missing today. This is work that largely happens behind the scenes. If we do it well, the average dbt user should not notice any immediate differences. So why are we doing it? **If you use dbt Core's CLI,** this will make it easier to manage the growing number and complexity of command line options. To make sure all the right flags and options are supported on all the right commands; to add and update help text; and to automatically coordinate updated documentation that's been, to date, hand crafted by resident artisans. **If you build tools that wrap around dbt-core,** the appeal of a stable and documented API to its internals should be obvious. This is a long initiative, and we won't get to all of it right away, but the right idea is there. (And apologies, in advance, for the undocumented internal methods we'll be breaking in the process.) **If you use dbt Cloud,** Core's ability to provide stable and sensible interfaces is a big part of what enables differentiated capabilities in dbt Cloud in the future. It's not the coolest stuff in its own right, but a necessary precondition for that cool stuff to exist. **If you use dbt at all,** you should care about this work, insofar as it will make it easier for us to build more features faster next year. We want more people to join us in building dbt Core, and a welcoming codebase to greet them. ## v1.5+ (Next year) If you've been following our GitHub discussions, or the Analytics Engineering roundup, none of these topics should come as too much of a surprise. They're neither definite commitments, nor the full set of things we expect to do next year. There's a lot of linear improvement to existing functionality that's always on our minds, and in our issues. But I want to start with the pair of ideas that we've been talking about nonstop, for which we're already dreaming up some code: 1. **Multi-project deployments.** `ref` a final model from someone else's project, wherever they've put it, without the need to run it first. Split up monolithic projects of 5000 models into 10 projects of 500, grouped by team and domain. This is more than just "namespacing": to really solve for this, we also need to solve for versioning and contracts, and support a variety of deployment mechanisms. The discussion for this has been in [#5244](https://github.com/dbt-labs/dbt-core/discussions/5244); I'll have more to share over the next few months. 2. **External orchestration.** The same dbt DAG, playing a more active role. We've been developing this idea internally, and have arrived at a few strong opinions. This would not be a new node type, but an upgrade to the ones we already have: sources, models, and exposures. Sources that can trigger their own ingest. Exposures that can trigger downstream data consumers (syncs, sinks, etc). Models that can define and run transformations in dedicated execution environments, reading from and writing back to centralized data storage. For each of those external integrations, a simple request where possible, and a dedicated plugin where justified. If you're someone who followed along the original "external nodes" discussion ([#5073](https://github.com/dbt-labs/dbt-core/discussions/5073))—especially if you've got a tool you'd be excited to integrate into dbt's DAG—let's talk. --- We also intend to keep pushing on existing capabilities in dbt Core. Again, a non-exhaustive list: **Python models, only just beginning.** What's the right DataFrame API to standardize on? Should dbt have a role in managing packages, model training, artifacts? Eventually, a full "MLOps" workflow? v1.3 in October will be our first foray, not the final story. Cody just opened some GitHub discussions, starting with [#5742](https://github.com/dbt-labs/dbt-core/discussions/5742). See what we're thinking, and weigh in. **Adapters, adapters, adapters.** We want to make it easier to build, test, and validate support for dbt on a new database, query engine, or runtime environment. We want to support more than one adapter for use in a single dbt-core invocation. We want to keep honing the performance of caching, cataloging, and incremental processing at scale, across data platforms. We want to offer more adapters in dbt Cloud. **Imagining dbt Core v2.** Last December, when announcing the v1.0 release, I predicted (wildly guessed) that dbt v2.0 would take 2-4 years to reach us (2023-2025). Then I put some things on a slide, asking everyone to imagine: - *dbt-SQL: The same capabilities. No Jinja.*1 - *The docs are always ready.*2 - *One dbt run across many databases and query engines.*3 - *Define your own tasks for the dbt DAG.*4 Most of that still feels about right. I don't see us ending next year with a v2.0 final release, but I do see us having a clear picture of what v2 will look like. In a sense, we've already started the work to get there, by combing our way through the rougher edges of v1. I'm excited for the next few months. I hope you are too. --- 1Now, I wonder if the answer is: Jinja-SQL and Python are just two of many supported languages for dbt Core. Some languages will make it dead-easy to unit test, to transpile across different databases, to infer column-level lineage. Others make it possible to run introspective queries that dynamically template transformation logic. It's an exciting future to consider. The challenge is to be clear and opinionated about what each one brings to the table, and when each one shines. 2Real-time metadata; see above. 3External orchestration; see above. 4This one, I'm not so sure! The task before us is the same as it ever was: build the DAG, as fast as possible, just what's needed, when it's needed. Still, I keep more advanced use cases that want to programmatically create, manipulate, and invoking the dbt DAG—and they may well be more plausible in a future where dbt-core has a documented, contracted set of internal APIs. That would be advanced-level stuff, guardrails not included. You probably don't need (or want) it, and if you do, you know it. ================================================ FILE: docs/roadmap/2023-02-back-to-basics.md ================================================ # dbt: Back to basics (February 2023) We're back, and there's a lot to say—so much that if we're not mindful, we risk writing a(nother) novella. We're going to try for concision this time; it's a year of focus. Of course, a lot more has already been written, so we'll also link to those issues & discussions, where we encourage you to weigh in with thoughts. Since last August, we've released two new versions of dbt Core: - v1.3 unleashed Python models onto the world. This is very new functionality, and we are still gathering feedback on where to go next. Read & comment on [the big ideas](https://github.com/dbt-labs/dbt-core/discussions/categories/ideas?discussions_q=label%3Apython_models+category%3AIdeas). - v1.4 reworked some internals, paid down some tech debt, and paved the way for better APIs going forward. | Version | When | Namesake | Stuff | | ------- | ------------- | -------------- | ----- | | 1.3 ✅ | Oct 2022 | Edgar Allen Poe | Python models in dbt. More improvements to metrics. | | 1.4 ✅ | Jan 2023 | Alain LeRoy Locke | Behind-the-scenes improvements to technical interfaces, especially structured logging. | This year, we're returning to our fundamentals. This means fewer big surprises—fewer new answers to the question, "What is dbt?"—and more of the things that dbt needs—the dbt that you know, the dbt that you & your teams have come to rely on for getting your job done. There are four big themes we want to tackle, reflected in the questions below. We will aim to provide compelling answers, sometimes with new functionality, sometimes with polish on top of existing capabilities. These aren't the only questions that interest us, but they are the ones we're prioritizing this year. 1. **Our APIs.** How can we enable the thousands of people who want to use dbt today, ? Can we enable community members to build more powerful extensions of dbt's framework, by exposing more & more of its functionality as a stable Python library? Can we provide an experience as delightful as dbt-core's CLI, via a reliable `dbt-server`, and RESTful APIs in dbt Cloud? 2. **Your models, as APIs.** Can dbt as a framework scale to complex deployments, across multiple teams, entering their third or fourth year of project maturity? Can it scale to some of the largest organizations who have adopted it as a standard pattern? What can we learn from the scaling challenges that software teams have encountered and surmounted over the last decade? 3. **Streaming.** How must dbt's essential building blocks—models, tests, sources, materializations—change (or not) to finally leverage data platforms' capabilities around streaming transformation? 4. **Semantic Layer.** How can we combine the existing power of dbt metrics, defined as an extension of your dbt DAG, with the depth of MetricFlow (!) as a framework for defining richer metrics and generating optimized queries? In a sentence: **The same dbt, for more people.** More community members who might build plugins and extensions, without having to read `dbt-core` source code and hack together undocumented internal methods. More embedded analysts who can confidently contribute the right change to the right model in the right project, without having to first navigate through thousands of preexisting models with unclear ownership. More use cases that can be solved in "the dbt way," for batch as well as streaming. More downstream queriers who can benefit from asking questions on top of a dbt Semantic Layer. We're sticking with one minor version release every three months. There won't be a version dedicated to _just_ API improvements, multi-project deployments, streaming, or semantic layer. Rather, we expect to make incremental progress as we go along. With that, here's our near-sighted lay of the land: | Version | When | Stuff | Confidence | | ------- | ------------- | -------------- | ---------- | | 1.5 ⚒️ | April | An initial Python API for programmatic invocations, and a cleaner CLI to match. The beginning of multi-project deployments ("Models as APIs"), and of support for streaming (materialized views). | 95% | | 1.6 🌀 | July | Next steps for multi-project deployments (cross-project `ref`, project-level namespacing, patterns for development & deployment). Continue the story around stream processing (materialized tests, managed sources). Integrating dbt metrics and MetricFlow. | 75% | | 1.7 | October | More on the same themes. The details will be based on velocity, feedback, and emergent discoveries. | 50% | | 1.8+ 💡 | 2024 | dbt-core as a library. A sketch of dbt v2. | 25% | `updated_at: 2023-02-28` As always, to keep track of what's happening between these roadmap updates: - [Milestones](https://github.com/dbt-labs/dbt-core/milestones) - [GitHub discussions](https://github.com/dbt-labs/dbt-core/discussions) - [Company blog](https://blog.getdbt.com/) & [dev blog](https://docs.getdbt.com/blog) Don't forget to ~~like and subscribe~~ [upgrade](https://docs.getdbt.com/guides/migration/versions)! # Commentary Let's keep it brief! ## A Python API for dbt-core dbt Core v1.5 will include: - A new CLI, based on `click`, with improved help text & documentation - Support for programmatic invocations, via a Python API, at parity with CLI functionality Is this it? I don't think so. We have a longer-term vision of dbt-core as a mature software library, with clear interfaces and plugin points. We aren't going to get all the way there by April. We will have a subset of capabilities that will enable a number of cool things for many. I believe we will be able to get there, over the next year, with carefully scoped initiatives tied to clear outcomes. We've been developing & sharing our visions as a team, and we'll have more to share over the coming months. _Read more: ["dbt-core as a library: first steps"](https://github.com/dbt-labs/dbt-core/issues/6356)_ ## Multi-project deployments (v1.5+) Here are three guiding principles: 1. Each team owns its data, and how that data is shared with other teams. 2. Organizations can maintain central governance, coordinating rules across teams. 3. All models are in one DAG. These are not necessarily the doctrine of "dbt mesh," but we are using them to describe the end state we're hoping to achieve, the core capabilities we need to unlock it, and the user flows (person-to-person, team-to-team, person-to-dbt) we want to facilitate along the way. Each person interacts with the subset(s) of the DAG relevant to them. Developing and deploying dbt should feel the same. The first step here is giving teams maintaining dbt projects the tools to start serving models as "APIs." The coup de grâce is being able to `ref` another team's stable, public, contracted model as the starting point for your own. _Read more: ["Multi-project deployments"](https://github.com/dbt-labs/dbt-core/discussions/6725) & linked discussions_ ## Support for streaming Back in 2020, one of Jeremy's very first projects, as newly designated Associate Product Manager, was investigating the implementation of Materialized Views across our most popular data platforms. The findings: while the dream of MVs was a happy one, every real MV was unhappy in its own way, motivated by different use cases and beset by subtle limitations. A few years later, the major data platform vendors are taking another swing at first-class support for streaming transformation. We're also lucky to have Florian, who talked & thought streaming databases for a living. Our vision is a dbt DAG that can combine batch & streaming, without distorting the core framework that's gotten dbt where it is. _Read more: ["Let's add Materialized View as a materialization, finally"](https://github.com/dbt-labs/dbt-core/issues/6911)_ ## dbt Semantic Layer We've officially welcomed many new colleagues from Transform. We're going to be spending time over the next several weeks talking about how to integrate dbt Core's existing metrics spec with MetricFlow's. We expect to be writing much more about this in public. Until then, you can read our previous thinking. The specifics are liable to change, but the foundational concept still holds: ["dbt should know more semantic information"](https://github.com/dbt-labs/dbt-core/discussions/6644) ## What's **not** here? In 2023, we need to be focused & disciplined. There's a lot we wish we could be making progress on, but we can only guarantee that progress in a precious few areas, by devoting our attention & energy to them. Each of the topics below has appeared in the lower-confidence portions of previous roadmaps, and they continue to interest us greatly. We have all been guilty of saying, "Let's just take a day—just one day!—and try to hack together a working demo." You might even see some proof-of-concept code appear, here or there. We won't turn down opportunities to take small steps, to make incremental forward progress. But none of these is something we expect us to be launching at Coalesce in October. - **External orchestration.** Can dbt trigger external APIs to ingest `sources`, and sync `exposures`? To run `models` that require tools outside the data platform? I'd like the answer here to be "yes," but it isn't a priority for this year. - **Next steps for Python models.** It's worth restating: This is very new functionality, and we're still very early. We have some ideas of what could be compelling and ergonomic, and there are some small usability improvements we'll try to make over the year—but we need to learn more about how you all are using, and want to be using, Python models, before we once again tackle these as a top priority. - **More modeling languages**, or "Bring Your Own SQL transpiler / Python framework / ???". This is one of the more boundary-pushing ideas we've had, for continuing to expand dbt's reach as a framework for (language-agnostic!) data transformation. It's not one of the foundational reinvestments that we must make sooner rather than later. We've also [discussed this pattern](https://github.com/dbt-labs/dbt-core/discussions/4458#discussioncomment-4176217) as one potential path toward unlocking **column-level lineage,** which—while always present in our hearts & among our [most popular discussions](https://github.com/dbt-labs/dbt-core/discussions?discussions_q=sort%3Atop+)—also doesn't appear on this year's list of top priorities. - **Unit testing.** ~Let's just take a day—just one day!—and try to hack together a working demo.~ (We might, though.) ## What we'll keep doing This covers the big rocks. The pebbles and the sand, we already have our ~~mouths~~ hands full. Most of the time, it's even fun. We'll keep releasing patches with fixes for bugs and any regressions that crop up in new versions of dbt-core. We'll keep a swimlane open for developer ergonomics. Not fundamental changes to the dbt framework, but quality-of-life improvements for those who use it every day. We've created [a new label to track these "paper cuts"](https://github.com/dbt-labs/dbt-core/issues?q=is%3Aissue+is%3Aopen+label%3Apaper_cut+sort%3Areactions-%2B1-desc)—often among the most upvoted issues!—and we're very interested in supporting community members who want to help us refine & contribute these improvements. We'll keep reading & responding to your issues, bug reports, feature requests, ideas. We can't respond to every comment everywhere, but we read them all. You come to dbt, and so we build it & keep building it—with ambition & with vision, with discipline & with focus. Yours truly - Jeremy, Florian, Doug, & the entire Core team ================================================ FILE: docs/roadmap/2023-11-dbt-tng.md ================================================ # dbt: The Next Generation (November 2023) To everyone we saw at [Coalesce](https://coalesce.getdbt.com/) last month: thank you for joining us! We got up on stage and shared the next chapters from this year’s featured stories: about [collaborating across multiple teams and projects at scale](https://www.youtube.com/watch?v=NIseH-Gd-U4); about [relaunching the dbt Semantic Layer](https://www.youtube.com/watch?v=2Qo5_CIsSH4); about [more flexibility in development](https://www.youtube.com/watch?v=UfraDWKsSvU); and about [more mature CI/CD](https://www.youtube.com/watch?v=3sp6tmYykVc). To anyone who missed us live, [catch us on the replays](https://www.youtube.com/@dbt-labs)! These are stories that span both dbt Core and dbt Cloud. Our aim is to push forward the open source standard for analytics engineering, and also the platform that makes it possible for more teams to adopt & deploy dbt at scale. In [his keynote presentation](https://youtu.be/lNZLcsHAdco?si=FdtTOOIokvm1pT8D&t=637), Tristan talked about these two priorities for dbt Labs. We remain committed to dbt Core, as a standard for the industry, and an open source project under an Apache 2 license. We are also committed to creating a business around dbt Cloud that is sustainable over the long term, to enable us to continue to invest in driving dbt forward. Those two goals are inseparable. To make them both happen, we need to strike an important balance. What has it looked like over the last six months, and what will it look like for the six months ahead? _[JC](https://github.com/jtcohen6) & [GG](https://github.com/graciegoheen)*_ > *Also, hi! I’m Grace Goheen, or [@graciegoheen](https://github.com/graciegoheen). Long time dbt user, new to the dbt Core product team. I joined the Professional Services team at dbt Labs back in 2021, where I’ve since had the opportunity to work hands-on in dozens of dbt projects - leading legacy migrations, consulting on architecture, optimizing project performance, and more. I lived through the joy (lineage! testing! documentation!) and pain (spaghetti DAGs! model bottlenecks! debugging code!) of being an analytics engineer, and realized I wanted to be a part of shaping the tool at the center of it all. So here I am, the newest Product Manager of dbt Core! I am so grateful to be building this industry-defining tool with all of you. > # The last six months: scale | Version | When | Namesake | Stuff | | --- | --- | --- | --- | | [v1.5](https://docs.getdbt.com/guides/migration/versions/upgrading-to-v1.5) | April | [Dawn Staley](https://github.com/dbt-labs/dbt-core/releases/tag/v1.5.0#:~:text=Dawn%20Staley%20(b.%201970)) | Revamped CLI. Programmatic invocations. Model governance features (contracts, access, groups, versions). | | [v1.6](https://docs.getdbt.com/guides/migration/versions/upgrading-to-v1.6) | July | [Quiara Alegría Hudes](https://github.com/dbt-labs/dbt-core/releases/tag/v1.6.0#:~:text=Quiara%20Alegr%C3%ADa%20Hudes%20(b.%201977)) | New Semantic Layer spec. More on model governance (deprecations). Saving time and $$ with retry + clone. Initial rollout of materialized views. | | [v1.7](https://docs.getdbt.com/guides/migration/versions/upgrading-to-v1.7) | November | Questlove | More flexible access to "applied state" in docs generate and source freshness. Improvements to model governance & semantic layer features (driven by user feedback). | We added a **lot** of stuff this year! Over the past three dbt Core minor versions, we’ve managed to knock out a litany of the most popular issues and discussions gathered over the past several years: - [CLI preview](https://github.com/dbt-labs/dbt-core/discussions/5418) (1.5) - [Invoking dbt as a Python module](https://github.com/dbt-labs/dbt-core/issues/2013) (1.5) - [Materialized views](https://github.com/dbt-labs/dbt-core/issues/1162) (1.6) - [Namespacing for dbt resources](https://github.com/dbt-labs/dbt-core/issues/1269) (1.6), in support of [multi-project collaboration](https://github.com/dbt-labs/dbt-core/discussions/6725) - [`docs generate --select` for slimmer catalog queries](https://github.com/dbt-labs/dbt-core/issues/6014) (1.7) - And… we’re finally taking aim at [unit testing for dbt-SQL models](https://github.com/dbt-labs/dbt-core/discussions/8275) (!), coming in 1.8, which you should read more about in the section below. Thank you for providing your upvotes, comments, and feedback. One of the best things about building dbt Core in the open is that we are all pushing forward the analytics engineering standard together. We’re able to prioritize these features and paper cuts because of your participation. We’ve got lots more to build - there are some highly upvoted issues and discussions that remain, and gaps in the analytics engineering workflow that we want to close. But before we keep building, we must ensure our foundation is a **stable** one. # The next six months: stability (& unit testing!) | Version | When | Stuff | Confidence | | --- | --- | --- | --- | | 1.8 | Spring 2024 | Stable interfaces for adapters & artifacts. Built-in support for unit testing dbt models. | 80% | Since the v1.0 release of dbt Core (almost two years ago), we’ve released a minor version of dbt Core every three months. The January release (post-Coalesce, post-holidays) tends to be an understated affair: tech debt, bug fixes, support for Python 3-dot-new. We’ve been measuring the rate of adoption for new versions, and we’ve seen that it takes more than 3 months for the wider user base to really adopt them. The plurality of dbt projects in the world are using a dbt Core version released between 6 and 12 months ago. We think this speaks to two things: It’s harder to upgrade than it should be.; and we can afford to take more time baking new releases. Between now and next April (2024), we plan to instead prepare **one** minor release that prioritizes **all-around interface stability**. We want to make it easier for _everyone_ to upgrade with confidence, regardless of their adapter or other integrated tooling. There is a _lot_ of value locked up in the features we’ve already released in 2023, and we want to lower the barrier for *tens of thousands* of existing projects who are still on older versions. That work is important, it takes time, and it has long-lasting implications. ### Adapters & artifacts With the v1.0 release, [we committed](https://www.getdbt.com/blog/getting-ready-for-v1-0) to minimizing breaking changes to project code, so that end users would be able to upgrade more easily. We haven’t perfected this, including earlier this year when we did a full relaunch of the metrics spec for the Semantic Layer. We are committed to getting better here. Even in v1.0, though, we intentionally carved out two less-stable interfaces, which would continue to evolve in minor releases: **adapter plugins** and **metadata artifacts**. At the time, these interfaces were newer and rapidly changing. Almost every minor version upgrade, from v1.0 through v1.7, has required some fast-follow compatibility changes for adapters and for tools that parse dbt manifests. This has been particularly difficult for adapter maintainers. As of this writing, while [the majority of third-party adapters support v1.4](https://github.com/dbt-labs/dbt-core/discussions/6624#discussioncomment-5663823) (released in January), [just over a third support v1.5](https://github.com/dbt-labs/dbt-core/discussions/7213#discussioncomment-5663790) (April), and [only a handful support v1.6](https://github.com/dbt-labs/dbt-core/discussions/7958#discussioncomment-6310276) (July). It isn’t fair of us to keep releasing in a way that *requires* this reactive compatibility work every 3 months. Instead, we will be defining a stable interface for adapters, in a separate codebase and versioned separately from dbt Core. Starting in v1.8, it will be forward-compatible for future versions. If you want to use `dbt-core` v1.X with `dbt-duckdb` v1.Y, you will be able to. For most people, we don’t want you to have to think about versions _at all_: just use latest & greatest dbt Core. For customers and users of dbt Cloud, this is the experience we want to provide: delivering dbt Core and dbt Cloud together, as one integrated and continuously delivered SaaS application — an experience where you don’t need to think about versions or upgrading, and where you get access to Cloud-enhanced & Cloud-only features as a matter of course. **An aside:** This was the first year in which we delivered [some functionality like that](https://github.com/dbt-labs/dbt-core/discussions/6725): built it in such a way that it _feels like Core_ while being actually powered by (and exclusive to) dbt Cloud. This has long been our pattern: Core defines the spec, and Cloud the scalable implementation, especially for Enterprise-geared functionality. I (Jeremy) wish I had communicated this delineation more clearly, and from the start. We are going to continue telling unified stories, across mature capabilities in Core and newer ones in Cloud, and we want all of you — open source community members, Cloud customers, longtime data practitioners and more-recent arrivals — to know that you are along for this journey with us. ### Summary: continuous & stable delivery Over the next 6-12 months, we will be spending less time on totally new constructs in dbt Core, and more time on the fundamentals that are already there: stabilizing, maintaining, iterating, improving. dbt Cloud customers will see enhancements and under-the-hood improvements delivered continuously, as we move towards this model of increased stability. Features that fit inside dbt Core’s traditional scope will also land in a subsequent minor version of dbt Core. This is an important part of our evolving story: a compelling commercial offering that makes it possible for us to keep developing, maintaining, and distributing dbt Core as Apache 2 software. ## Onwards dbt Core is as it has always been: an open source standard. It’s a framework, a coherent set of ideas, and a fully functional standalone tool that anyone can take for a spin — adopt, extend, integrate, imitate — without ever needing to ask us for permission. Adapters will keep moving at the pace of innovation for their respective data warehouse. dbt Docs remains a great "single-player" experience for getting hooked on data documentation. (The aesthetic isn’t dated, it’s *[retro](https://github.com/lightdash/dbt-docs-95).*) dbt Core remains the industry-defining way to author analytical models and ensure their quality in production. But wait! As many of you have voiced, there’s been no good way to ensure your SQL logic is correct without running expensive queries against your full production data. dbt does not have native unit testing functionality… yet. This gap in the standard is one we have been eager to work on, and we’re planning to land it in the next minor release of dbt Core. ### What is unit testing in dbt? For many years, dbt has supported "data" tests — testing your *data outputs* (dbt models, snapshots, seeds, etc.) based on that environment’s actual *inputs* (dbt sources in your warehouse), and ensure the resulting datasets match your defined expectations. Soon, we’re introducing "unit" tests — testing your modeling *logic,* using a small set of static inputs, to validate that your code is working as expected, faster and cheaper. ### What’s the plan? Thank you to everyone who has already provided feedback and thoughts on our [unit testing discussion](https://github.com/dbt-labs/dbt-core/discussions/8275) — or, we should say our _new_ unit testing discussion, since Michelle opened the [original one](https://github.com/dbt-labs/dbt-core/discussions/4455) back in 2020, before she joined dbt Labs :) We truly appreciate the amount of insights and energy y’all have already poured into helping us make sure we build the right thing. We are actively working on this feature and expect it to be ready for you all in our `1.8` release next year! If you have thoughts or opinions, please keep commenting in the discussion. We’re also planning a community feedback session for unit testing once we’ve released an initial beta of `1.8`, so keep an eye out. ### Bugs, regressions, paper cuts, ... We will continue to respond to your issues and review your PRs. We will continue to resolve regressions and high-priority bugs, fast as we’re able, and include those fixes in regular patch releases. Along with fixing bugs and regressions, we’d also like to keep tackling some of the highly requested "paper cuts”. Thank you to all those who have expressed their interest by upvoting and commenting. We’re unlikely to tackle all of these things in v1.8 — they’re lower-priority than the interface stability work, which we must do — they are all legitimate opportunities to solidify the existing, well-established Core functionality: - [Allow data tests to be documented](https://github.com/dbt-labs/dbt-core/issues/2578) - [Snapshot paper cuts](https://github.com/dbt-labs/dbt-core/discussions/7018) - [Making external tables native to dbt-core](https://github.com/dbt-labs/dbt-core/discussions/8617) - [Defining vars, folder-level configs outside `dbt_project.yml`](https://github.com/dbt-labs/dbt-core/issues/2955) - [Supporting additional formats for seeds](https://github.com/dbt-labs/dbt-core/issues/2365) (JSON) Let us know which ones speak to you — in that list, not in that list, the ideas in your head — on GitHub, on Slack, or wherever you may find us. ================================================ FILE: docs/roadmap/2024-12-play-on.md ================================================ # dbt: Play On (December 2024) Oh hi there :) We’ve had the opportunity to talk a lot about our investments in open source over the past few months: - [I [Grace] renewed my vows with the dbt Community in a Vegas wedding ceremony officiated by Elvis](https://youtu.be/DC9sbZBYzpI?si=uK9Aie6Jl-FIHggm) (aka [@jtcohen6](https://github.com/jtcohen6)). - [We spoke about dbt Labs’ continued commitment to Open Source in the Coalesce Keynote](https://youtu.be/I72yUtrmhbY?si=Iu6s9WXdHnFtyCvi). - [We wrote about the importance of extensibility for how we build features in dbt Core.](https://www.getdbt.com/blog/dbt-core-v1-9-is-ga#:~:text=Extensibility%20is%20what%20powers%20the%20community) All of these points and themes remain incredibly relevant to our team and are fueling our vision as we prepare for the new year: - we're committed to defending dbt Core as the open source standard for data transformation, which will remain licensed under Apache 2.0 - the dbt framework will continue to be shaped by a collaborative effort between you (the community) and us (the maintainers) - when we add something new to the standard, we are committing to the long term, which means we are intentional about *how* and *when* we expand its breadth - in the meantime, lean into dbt's extensibility (and show us how you're doing it!) Now that we’ve gotten into our new rhythm… Now that last year’s focus on stability has earned us the right to ship awesome new additions to the dbt framework… Now that we’re actually seeing a *much* higher % of projects running the latest & greatest dbt… …the next 6-12 months will feel similar to the last. dbt will keep getting better at doing what it does - being a mission-critical piece of your data stack, and a delightful part of your work. To that end, stability always comes first. And, we will be shipping some exciting new features to dbt Core. # Oh What A Year It’s Been! This year, we released two new minor versions of dbt Core. | **Version** | **When** | **Namesake** | **Stuff** | | --- | --- | --- | --- | | [v1.8](https://docs.getdbt.com/docs/dbt-versions/core-upgrade/upgrading-to-v1.8) | May | [Julian Abele](https://github.com/dbt-labs/dbt-core/releases/tag/v1.8.0) | Unit testing. Decoupling of dbt-core and adapters. Flags for managing changes to legacy behaviors. | | [v1.9](https://docs.getdbt.com/docs/dbt-versions/core-upgrade/upgrading-to-v1.9) | December | [Dr. Susan La Flesche Picotte](https://github.com/dbt-labs/dbt-core/releases/tag/v1.9.0) | Microbatch incremental strategy. New configurations and spec for snapshots. Standardizing support for Iceberg. | In the [last roadmap post](https://github.com/dbt-labs/dbt-core/blob/main/docs/roadmap/2023-11-dbt-tng.md), we committed to prioritizing all-around interface stability. This included decoupling dbt-core and adapters, introducing [behavior change flags](https://docs.getdbt.com/reference/global-configs/behavior-changes) to give you time to adjust to ~~breaking~~ changes, and improving the stability of metadata artifacts. You can read more about those efforts [here](https://docs.getdbt.com/blog/latest-dbt-stability). Stability means you can upgrade with confidence. Stability means less disruptions for our adapters and integrations in the dbt ecosystem. Stability means we’ve earned the right to **ship some big new features**. This past year, we were able to ship some long-awaited additions and enhancements to the dbt Core framework: - [**Unit tests**](https://github.com/dbt-labs/dbt-core/discussions/8275) allow you to validate your SQL modeling logic on a small set of static inputs before you materialize your full model in production. - [**Snapshots**](https://github.com/dbt-labs/dbt-core/discussions/7018) got the glow up they deserved - with new configurations and spec to make capturing your data changes easier to configure, run, and customize. - [**Microbatch**](https://github.com/dbt-labs/dbt-core/discussions/10672) incremental models enable you to optimize your largest datasets, by transforming your timeseries data in discrete periods with their own SQL queries, rather than all at once. - [**Iceberg**](https://docs.getdbt.com/blog/icebeg-is-an-implementation-detail) table format (an open standard for storing data and accessing metadata) is standardized across adapters, enabling you to store your data in a way that promises interoperability with multiple compute engines. We also were able to close out some highly-upvoted “paper cuts”: - `--empty` flag limits the `ref`s and `source`s to zero rows, which you can use for schema-only dry runs that validate your model SQL and run unit tests - dbt now issues a single (batch) query when calculating source freshness through metadata, instead of executing a query per source - improvements to `state:modified` help reduce the risk of false positives due to environment-aware logic - you can now document your data tests with `description`s A lot of these features are things that you (the community) have been discussing and experimenting with for years. To everyone who opened an issue, commented on a discussion, joined us for a feedback session, developed a package, or contributed to our code base… however you made your voice heard this year, **thank you** for continuing to care, for continuing to lean in, for continuing to help shape dbt. # New Year’s Resolutions To start off the new year, we’re focusing on three major areas of development to the dbt framework: - **typed macros** - configure Python type annotations for better macro validation - **catalogs** - first-class support for materializing dbt models into external catalogs, providing a warehouse-agnostic interface for managing data in object storage - **sample mode** - limit your data to smaller, time-based samples for faster development and CI testing ## Typed Macros In the simplest terms, [macros](https://docs.getdbt.com/docs/build/jinja-macros#macros) are are pieces of code, written in Jinja, that can be reused throughout your project – they are similar to "functions" in other programming languages. In practice, you can reference a macro in a model’s SQL (or config block, or hook) to: - make your SQL code more DRY by abstracting snippets of SQL into reusable “functions” - use the results of one query to generate a set of logic - change the way your project builds based on the current environment Or, if you really need to, run arbitrary SQL in your warehouse via `dbt run-operation`. Macros can depend on inputs, vars, `env_vars`, or even other macros — *and* there are “special” macros for defining custom generic tests and materializations. This immense flexibility is one of the great benefits of macros — you can use them to solve **a lot** of different problems. However, on the flip side, this flexibility — where macros can be whatever you want them to be — makes it challenging to validate that your macros are doing what you expect. Without a way to define expected types for the inputs and outputs of macros, our adapter maintainers struggle to validate that a built-in macro override will produce the correct output. Furthermore, *any* analytics engineer writing a macro in dbt should be able to validate the expected behavior. One of the things we aim to work on in the coming months is an interface for **typed macros.** Imagine being able to codify the expected types* for the inputs and outputs for your macros: ```sql {% macro cents_to_dollars(column_name: str, scale: int = 2 -> str) %} ({{ column_name }} / 100)::numeric(16, {{ scale }}) {% endmacro %} ``` **Note: These are a subset of Python types (internal to dbt), not the data types within the data warehouse.* Then, we could issue warnings at parse time when usage of a macro violates these types. By adding the ability to configure type expectations, user-created macros become more predicable, and [built-in macros become easier to override](https://github.com/dbt-labs/dbt-core/issues/9164). Should this type checking be on by default, or something you opt into? Should we also support dbt-specific types, such as `Relation`? Head over to the [github discussion](https://github.com/dbt-labs/dbt-core/discussions/11158) to participate in the conversation! ## Catalogs In `v1.9`, we shipped a set of standard configs to materialize dbt models in Iceberg table format: ```sql {{ config( materialized = "table", table_format = "iceberg", external_volume = "s3_iceberg_snow" ) }} ... ``` Supporting the Iceberg table format was our first step towards empowering users to adopt Iceberg as a standard storage format for their critical datasets. In the coming months, we want to add first-class support for “catalogs” in dbt. “Catalogs,” including Glue or Iceberg REST, operate at a level of abstraction above specific Iceberg tables — and they can provide a warehouse-agnostic interface for managing a large number of datasets in object storage. Imagine a new top-level `catalogs.yml` that tells dbt about the catalog integrations you want to write to: ```yaml catalogs: - catalog_name: my_first_catalog write_integrations: - integration_name: prod_glue_write_integration external_volume: my_prod_external_volume table_format: iceberg catalog_type: glue ``` Then, in your model’s configuration, simply specify the `catalog` field: ```sql {{ config( catalog = "my_first_catalog" ) }} ... ``` These new configurations will enable dbt to template the correct DDL statements for the platform (`CREATE GLUE ICEBERG TABLE` vs. `CREATE ICEBERG TABLE`, etc.). Now, when you run the above model, it will be materialized as an Iceberg table in s3 registered in the AWS Glue catalog. We believe the approach of writing datasets to a platform-agnostic storage layer, and registering those datasets with a similarly agnostic *catalog service,* will become important to the technical foundations of the dbt workflow — the [Analytics Development Lifecycle (ADLC)](https://www.getdbt.com/resources/guides/the-analytics-development-lifecycle) — moving forward. Support for materializing Iceberg tables in external catalogs is another step down this path. To read more about catalogs and participate in shaping this feature, head over to the [discussion on GitHub](https://github.com/dbt-labs/dbt-core/discussions/11171)! ## Sample Mode We began our [“event_time” journey](https://github.com/dbt-labs/dbt-core/discussions/10672) with microbatch incremental models. Next up is [sample mode](https://github.com/dbt-labs/dbt-core/discussions/10672#:~:text=%F0%9F%8C%80%5BNext%5D%20%E2%80%9CSample%E2%80%9D%20mode%20for%20dev%20%26%20CI%20runs) - we want to support a pattern for speeding up development and CI testing by filtering your dataset to a *time-limited sample.* Imagine your model contains a `ref` statements like so: ```sql select * from {{ ref('fct_orders') }} ``` During standard runs, this compiles to: ```sql select * from my_db.my_schema.fct_orders ``` But during a *sample* run, dbt could automatically filter down large tables to the last X days of data using the same `event_time` column used for microbatch. The exact syntax here is TBD, but imagine you run something like `dbt run --sample 3`, which then compiles your code to: ```sql select * from my_db.my_schema.fct_orders -- the event_time column in fct_orders is 'order_at' where order_at > dateadd(-3, day, current_date) ``` No more [overriding the `source` or `ref` macro](https://discourse.getdbt.com/t/limiting-dev-runs-with-a-dynamic-date-range/508) to hack together this functionality. Built-in support for “sample mode” — filtering to a consistent time-based “slice” across all models — means faster development and faster testing because you’re running on *less* data. This could be configured for a specific dbt invocation, a CI environment, or as a set-it-and-forget-it default for everyone who’s developing on your team’s project. [#11200 Sample Mode (for faster Development and CI 🚀)](https://github.com/dbt-labs/dbt-core/discussions/11200) is the place to think in public with us. ## and who could forget, paper cuts! Those are the big things we plan to tackle in the coming months. As always, we want to tackle some smaller `paper_cut`s as well. Your upvotes and comments help us prioritize these, so please make some noise if there’s something you care a whole lot about. Some that are top of mind for me already: - **DRY-er YML**, including: - [Defining vars configs outside dbt_project.yml](https://github.com/dbt-labs/dbt-core/issues/2955) - [Add ability to import/include YAML from other files](https://github.com/dbt-labs/dbt-core/issues/9695) - **Enhancements to model versions and contracts**, including: - [Automatically create view/clone of latest version](https://github.com/dbt-labs/dbt-core/issues/7442) - [Support constraints independently from enforcing a full model contract](https://github.com/dbt-labs/dbt-core/issues/10195) - and [warnings when configs are misspelled](https://github.com/dbt-labs/dbt-core/issues/8942) :) # [**Call me**, **beep me** if you wanna reach me](https://www.youtube.com/watch?v=GIgLqN_rAXU) If one of *your* New Year’s resolutions is to be more involved in the dbt community, here are some of the many ways you can contribute: - open, upvote, and comment on GitHub [issues](https://docs.getdbt.com/community/resources/oss-expectations#issues) - start a [discussion](https://docs.getdbt.com/community/resources/oss-expectations#discussions) or discourse post when you’ve got a Big Idea - engage in conversations in the [dbt Community Slack](https://www.getdbt.com/community/join-the-community) - [contribute code](https://docs.getdbt.com/community/resources/oss-expectations#pull-requests) back to one of our open source repos, for one of our issues tagged `help_wanted` or `good_first_issue`, and our engineering team will work with you to get it over the finish line - join us on zoom for feedback sessions (which we’ll post about in slack and in the relevant GitHub discussion) - share your creative solutions in a blog post, at a dbt meetup, or by talking at Coalesce Your feedback and thoughts are incredibly valuable to us, so make yourself heard this year. We’re excited to build some awesome new features together. [Your loving wife](https://youtu.be/DC9sbZBYzpI?si=xCGWoQDK-w13Fz6U&t=1594), Grace ================================================ FILE: docs/roadmap/2025-05-new-engine-same-language.md ================================================ # dbt: New Engine, Same Language (May 2025) Hello! Today was one of our biggest launches in dbt Labs history. We announced a bunch of new experiences for the dbt platform, which you can read about in [the roundup](https://www.getdbt.com/blog/dbt-launch-showcase-2025-recap). And we finally got to announce one big thing that we’ve been working on with [the team that joined us from SDF](https://www.getdbt.com/blog/dbt-labs-acquires-sdf-labs) — the **dbt Fusion engine**, written in Rust and optimized for speed and scale. Since [last December](https://github.com/dbt-labs/dbt-core/blob/main/docs/roadmap/2024-12-play-on.md), the Core team at dbt Labs has been busy developing new features for the [dbt Core v1.10 release](https://docs.getdbt.com/docs/dbt-versions/core-upgrade/upgrading-to-v1.10) (`rc1` available now!) … | **Version** | **When** | **Namesake** | **Stuff** | | --- | --- | --- | --- | | v1.10 | May (beta) | Florence Earle Coates | `--sample` mode. Catalogs. Macro argument validation. Calculate source freshness via a custom `loaded_at_query`. Deprecation warnings using new and improved jsonschemas. | … and **we’ve been building a [whole new dbt engine](https://github.com/dbt-labs/dbt-fusion) from the ground up**. First thing’s first: - We’re committed to maintaining dbt Core **indefinitely**, under the Apache 2.0 license. - Fusion will be available under ELv2. That means you can use Fusion internally in your own business, for free and without restriction, forever. A bunch of its components (dbt-jinja, adapters, grammars, specs) will also be available under Apache 2.0. - You can read all the fine print in the [Licensing FAQs](https://www.getdbt.com/licenses-faq). Let’s talk about what’s changing, and what’s staying the same. ## What’s the difference between dbt Core and the new dbt Fusion engine? The new dbt Fusion **engine** is brand-new code. It’s built for speed and SQL comprehension, meaning it already has some underlying capabilities that dbt Core doesn’t. The **language** is the same: it’s dbt. It’s the language you’ve learned to do your job, and learned to love because it’s how you get your job done. ``` .--------------. / dbt language \ | .--------. | | / \ | | | dbt engine | | | \ / | | '--------' | \ / '--------------' caption: the dbt framework ``` We’ve written about this difference between “language” and the “engine” in the FAQs (linked above), and we’re going to give some more specific examples below. The really important message is this one: Together, the **language** and the **engine** create **the dbt framework.** ### What is the dbt language? The “language” of dbt includes anything you can write in your dbt project. Some of us like to call this the “authoring layer” — what we mean is, *the code you can put in your project.* Every data test configuration, every unit test property, the `+` before configs in `dbt_project.yml`, everything you can put between `{% macro %}` and `{% endmacro %}` ([just one, though](https://github.com/dbt-labs/dbt-core/issues/11393)). The language also includes important abstractions, like the idea that one `.sql` file in the `models/` directory == one dbt model == one data warehouse object ([or none](https://docs.getdbt.com/docs/build/materializations#ephemeral)). This language is really important. You’ve invested years into learning it, and you’ve used it to define your organization’s critical business logic. That’s why we’re committed to supporting, improving, and expanding the language across both dbt Core and Fusion. Recently, that has included important work to *tighten up* the language with stricter validation of your project code, as discussed in [dbt-core#11493](https://github.com/dbt-labs/dbt-core/discussions/11493). We want dbt to give more-helpful feedback when you misspell a config or write bad code in your project — a longtime paper-cut we are thrilled to at last be solving (see: [#2606,](https://github.com/dbt-labs/dbt-core/issues/2606) [#4280, #5605](https://github.com/dbt-labs/dbt-core/issues/5605), [#8942](https://github.com/dbt-labs/dbt-core/issues/8942)). dbt Core v1.10 is firing deprecation warnings, and Fusion will raise errors — and both engines are using the *same strongly-typed schemas* do it. Those schemas are live in the [dbt-jsonschema](https://github.com/dbt-labs/dbt-jsonschema/tree/main/schemas/latest_fusion) repo (under an Apache 2.0 license), and they’ll keep getting better as we work through prerelease feedback for dbt Core v1.10 and Fusion. We also want to keep adding language features across both Core and Fusion. Because Fusion is newer technology and has additional capabilities, such as built-in SQL comprehension, some language features will be better or exclusively available on Fusion. Think: an _enhancement_ to `--sample` and `--empty` modes where the engine intelligently add filters and limits, [without the associated issues of subqueries](https://github.com/dbt-labs/dbt-adapters/issues/199). ### What is the dbt engine? The “engine” of dbt is the foundational technology for turning **the code in your dbt project into data platform reality*.* The engine: - takes your project code, validates it (against the dbt language spec), and turns it into a DAG - connects to remote data warehouse using **adapters** - executes that DAG in order — creating, updating, or queries data in the warehouse — based on the commands/flags/arguments you pass in - produces logs and metadata from that DAG execution These things may differ across the dbt Core and Fusion engines. To ease the upgrade path, Fusion supports most of the same CLI commands/flags/arguments as dbt Core, and Fusion adapters support the same authentication methods as Core adapters. But we’re not planning for exact conformance on logs, metadata, and every single runtime behavior. Two examples from [the upgrade guide](https://docs.getdbt.com/docs/dbt-versions/core-upgrade/upgrading-to-fusion): - Fusion can run unit tests first before building *any* models, because it can infer column schemas from SQL understanding. - There’s no `--partial-parse` flag for Fusion, because its project parsing is *just that much faster. (And it will manage the cache itself)* If there are things you think are great about the dbt Core engine, and you want Fusion to meet the same need — let us know! We’ve got some tips at the end about how to engage with the next few months of Fusion development. ## What’s next for dbt-the-language? We’ve been building the new dbt Fusion engine since January, and we’ve still got plenty more to go. Our biggest focus area for the next few months is stabilizing and enhancing Fusion to support more of dbt's existing features, and more existing dbt projects. ([Joel and Jeremy wrote about Fusion’s path to GA](https://docs.getdbt.com/blog/dbt-fusion-engine-path-to-ga).) At the same time, we know there’s appetite for new capabilities in the dbt language — including some ideas we’ve been talking about for a long time. Importantly, our goal isn’t just parity with dbt as it exists today. We intend to keep expanding the language, across both the Core and Fusion engines. In many cases, we will be able to deliver an even-better experience in Fusion, thanks to its speed and SQL awareness. The exact specs here are TBD, but here are some features we’ve been thinking about adding: - **Out-of-the-box support for UDFs** - **Sources from external catalogs** - **Model freshness checks** - **…along with Bugs, Polish Work, and Paper Cuts** To be clear, we’re not committing to building precisely these things on any specific timeline — our top priority is parity for the new engine, to support existing users and customers in upgrading to Fusion — but we’re including these ideas as an illustration of what the same-framework, multi-engine future could look like. We’d love your thoughts on these ideas — what would be most interesting/useful to you? ### **Out-of-the-box support for UDFs** The idea to manage user-defined functions (UDFs) with dbt is almost [as old as dbt](https://github.com/dbt-labs/dbt-core/issues/136) — and it’s one that has [come up](https://github.com/dbt-labs/dbt-core/discussions/5099) [every few](https://github.com/dbt-labs/dbt-core/discussions/5741) [years since](https://github.com/dbt-labs/dbt-core/discussions/10395). UDFs enable users to define and register custom functions within the warehouse. Like dbt macros, they enable DRY reuse of code; unlike macros, UDFs can be defined in languages other than SQL (Python, Java, Scala, …) and they can be used by queries outside of dbt. There are two direct benefits to dbt-managed UDFs: 1. dbt will manage (create, update, rename) UDFs as part of DAG execution. If you want to update a UDF *and* a model that calls it, you can test the changes together in a development environment; propose those changes in a single pull request that goes through CI; and deploy them together into your production environment. dbt will ensure that the UDF is created before attempting to build the model that references it. And we could even imagine supporting *unit tests* on UDFs — which are functions, after all! 2. Fusion’s SQL comprehension is dialect-aware, but it is not yet *UDF-aware.* Today, if you call UDFs within a dbt model, you need to turn off Fusion’s static analysis for that model’s SQL. By supporting UDFs in the dbt framework, Fusion can also support them in its static analysis. We’re taking inspiration from Fusion’s antecedent, SDF, which supported UDFs for exactly this reason: https://docs.sdf.com/guide/advanced/udf Imagine if the dbt framework knew about user-defined functions. The Core engine could manage the creation of UDFs as data warehouse objects. The Fusion engine could take it one step further, by *also* validating your UDFs’ SQL and statically analyzing the SQL of dbt models that call those UDFs. ### **Sources from external catalogs** Back in December, we discussed ([dbt-core#11171](https://github.com/dbt-labs/dbt-core/discussions/11171)) our plans for integrating catalogs: Glue, Iceberg, Snowflake Managed, Unity, … Our goal was to centralize the configuration for materializing Iceberg tables, try to abstract over the minute differences between catalog providers (where possible), and teach dbt about a new top-level construct (`catalogs`). You can check out [the docs on Iceberg catalogs](https://docs.getdbt.com/docs/mesh/iceberg/about-catalogs), new in dbt Core v1.10. The first supported use case for external catalogs is around materializing dbt models *as Iceberg tables* by *writing them to catalogs* — but we know that another popular use case is reading from source tables that have been ingested to Iceberg catalogs, a.k.a. bringing the functionality of [read-external-iceberg from the dbt-labs-experimental-features](https://github.com/dbt-labs/dbt-labs-experimental-features/tree/main/read-external-iceberg) repository [into the dbt framework](https://github.com/dbt-labs/dbt-core/discussions/11265): ```sql select * from {{ source('my_catalog', 'my_iceberg_table') }} ``` You can already hack this with the [experimental code](https://github.com/dbt-labs/dbt-labs-experimental-features/tree/main/read-external-iceberg) today — but we see an opportunity to build it into dbt out-of-the-box, and to standardize the configurations for everyone interacting with Iceberg tables, for both use cases (sources and models) in their dbt projects. ### **Model freshness checks** dbt Core has supported `source freshness` checks since the [v0.13 release in 2019](https://github.com/dbt-labs/dbt-core/releases/tag/v0.13.0) (!). The idea, then and now, is: for tables that are updated by processes outside of dbt, the least dbt can do is ask, “How fresh is in the data in this table?” By contrast, it should be simple for dbt to know the freshness in the models *that dbt is building*… right? In practice, this can be trickier than you’d think! Multiple dbt invocations, split across different jobs/tasks/DAGs/schedules/orchestrators/projects/…, can result in confusion about when a model actually last built. Users have closed the gap with generic tests in popular packages, such as [dbt_utils.recency](https://github.com/dbt-labs/dbt-utils?tab=readme-ov-file#recency-source) and [dbt_expectations.expect_row_values_to_have_recent_data](https://github.com/metaplane/dbt-expectations?tab=readme-ov-file#expect_row_values_to_have_recent_data) — but the idea that [*dbt should know about model freshness*](https://github.com/dbt-labs/dbt-core/discussions/5103) is one we’ve tossed around a few times. This is a case where we think we might be able to introduce a new language concept that powers a capability in both engines, and unlock something additional in Fusion. Imagine: - The dbt language introduces `freshness` as an optional config for models. - dbt Core, which is stateless, could execute `freshness` checks on models — using a configured column, or metadata from the data warehouse. Fusion could support this check, too. - And: When the dbt Fusion engine is connected to the dbt platform, [it can run with *state awareness*](https://docs.getdbt.com/docs/deploy/state-aware-about). Fusion checks the freshness of upstream data inputs, and users can configure `freshness.build_after` — to reduce overbuilding models when there’s no new data, or when users want to save costs by placing an upper limit on build frequency. ```yaml models: - name: stg_orders config: freshness: # Fusion-powered state-aware orchestration: build this model after 4 hours, as long as it has new data build_after: {count: 4, period: hour} # Future: check that model has successfully updated within expected SLA, warn/error owner otherwise warn_after: {count: 24, period: hour} error_after: {count: 48, period: hour} ``` ### **Bugs, Polish Work, and Paper Cuts** And then, there’s everything else. While our team has been focused on the v1.10 release of dbt Core, and building the new dbt Fusion engine, we’ve had less capacity to triage every new issue and externally contributed PR. Thank you for your patience over the past five months — and thank you to the community members who gave us the feedback that we need to ensure maintenance even while working on the exciting new stuff. Over the coming months, alongside our work ensuring Fusion framework parity, we will be tracking our backlog of: - externally-contributed PRs - [polishing on microbatch](https://github.com/dbt-labs/dbt-core/issues/11292) - polishing on jsonschemas (the long-tail of adapter-specific configs) - `paper_cut`s ## How can you, a community member, contribute? Mostly, in all the same ways as before: [https://docs.getdbt.com/community/resources/contributor-expectations](https://docs.getdbt.com/community/resources/contributor-expectations) In the next few weeks, we’d really like your help — trying out the new engine, mettle-testing the new jsonschemas against your project code, and opening up bug issues when you run into problems — as we get Fusion (and the stricter dbt language spec) ready for prime-time. You can contribute your ideas by opening GitHub discussions or feature requests. While we’re focused on getting dbt Fusion to GA, we're going to keep the `dbt-fusion` repo focused on engine bugs and framework parity, so we’ll keep net-new feature requests in the `dbt-core` repo for the foreseeable future. Be on the lookout for GitHub discussions about new framework features, once we’re ready to start building. You can contribute your code by opening pull requests against *any* of our repos — whether you’ve written Rust before, or you’re looking for an excuse to get started :) ## Let’s get to work Maintaining a common framework across two codebases, written in entirely different languages, will be challenging. We’ve got some ideas for how to make this easier: - Core and Fusion are already sharing a set of strongly-typed jsonschemas defining the spec of acceptable project code inputs - Core adapters and Fusion adapters could share the same “packages” of macros and materializations - *… and more to come …* No lie — it’s going to be tricky, any way we slice it. We’re going to keep figuring it out over the coming months, and we ask for your patience while we do. After all, [our commitment](https://www.youtube.com/watch?v=DC9sbZBYzpI) is to you, the community, not to any one codebase. We’re excited to welcome in some new codebases, and some new contributors, to this thing we’re all building together. (╭☞ ͡° ͜ʖ ͡°)╭☞ **J**erco **E**lias **G**race ================================================ FILE: docs/roadmap/2025-12-magic-to-do.md ================================================ # dbt: Magic to Do (December 2025) dbt Core will turn in 10 in March 🥳. That’s almost a decade of `select`-ing, data-testing, and docs-generating. ICYMI - [we threw a birthday bash for the dbt Community at Coalesce this year](https://www.youtube.com/watch?v=aMUAQjqTKtc), and we used the occasion to reflect on all the ways dbt has grown up: There are now 2 engines: dbt Core (which will remain Apache 2.0 forever) and dbt Fusion (which uses a [mix of licenses](https://www.getdbt.com/blog/new-code-new-license-understanding-the-new-license-for-the-dbt-fusion-engine))... ...And lots more Open Source code: [MetricFlow](https://www.getdbt.com/blog/open-source-metricflow-governed-metrics) (now Apache 2.0!), [Fusion adapters](https://github.com/dbt-labs/dbt-fusion), [dbt-jinja](https://github.com/dbt-labs/dbt-fusion/tree/main/crates/dbt-jinja), [jsonschemas](https://github.com/dbt-labs/dbt-jsonschema/tree/main/schemas/latest_fusion), and [dbt-autofix](https://github.com/dbt-labs/dbt-autofix). The language is codified: `dbt-jinja` for the Jinja you can use in dbt, `jsonschemas` for yaml... ...And will continue to grow, [with new features across Core and Fusion](https://www.youtube.com/watch?v=6lI9d-gPKW8). As always, there is more ~~work~~ magic to do. # 2025 Wrapped In 2025, we had: - **>90k** weekly active projects - **>10k** comments across over **>5k** issues from over **>1k** people across our open source and source available repos - **4** new product-led discussions: - [Sample Mode (for faster Development and CI 🚀)](https://github.com/dbt-labs/dbt-core/discussions/11200) - [Out-of-the-box support for UDFs](https://github.com/dbt-labs/dbt-core/discussions/11851) - [UX Feedback on Logging and stdout](https://github.com/dbt-labs/dbt-fusion/discussions/584) - [Source schemas should be first-class, versioned artifacts](https://github.com/dbt-labs/dbt-fusion/discussions/1042) - **10** [Community Awards](https://www.youtube.com/watch?v=I-DgySJ0Syg) - **1** [demo on roller skates](https://youtu.be/aMUAQjqTKtc?si=9ZCmz30wZ118HeHI&t=1116) picture of demo on roller skates - **2** minor dbt Core releases | **Version** | **When** | **Namesake** | **Stuff** | | --- | --- | --- | --- | | [v1.10](https://docs.getdbt.com/docs/dbt-versions/core-upgrade/upgrading-to-v1.10) | June | [Florence Earle Coates](https://github.com/dbt-labs/dbt-core/releases/tag/v1.10.0) | `--sample` mode. Catalogs. Macro argument validation. Calculate source freshness via a custom `loaded_at_query`. YAML `anchors:`. | | [v1.11](https://docs.getdbt.com/docs/dbt-versions/core-upgrade/upgrading-to-v1.11) | December | [Juan R. Torruella](https://github.com/dbt-labs/dbt-core/releases/tag/v1.11.0) | User-defined functions. `cluster_by` for dynamic tables. Deprecation warnings using new and improved jsonschemas, enabled by default. | - and **1** dbt community <3 ### New language features This past year we added three big new language features to the dbt framework: - **[sample mode](https://docs.getdbt.com/docs/build/sample-flag)** renders filtered `ref`s and `source`s using time-based sampling, allowing developers to validate outputs without building entire historical datasets. - **[catalogs](https://docs.getdbt.com/docs/mesh/iceberg/about-catalogs)** abstraction for materializing dbt models as Iceberg tables, with a consistent set of configs (more on this below). - **[user-defined functions](https://docs.getdbt.com/docs/build/udfs)** as a new dbt resource type - allowing you to define custom functions in your project, call them in models, and register them in your warehouse as part of building your DAG. All of these are supported now across BOTH engines - dbt Core and dbt Fusion. The language is also now **codified** - **[new and improved jsonschemas](https://github.com/dbt-labs/dbt-jsonschema/tree/main/schemas/latest_fusion)** power warnings in Core (and errors in Fusion) to help you proactively identify and update deprecated configurations (such as misspelled config keys, old properties, or incorrect data types). This stricter spec creates clearer separation between the built-in configurations of the dbt language (or flags/options of the dbt engine) and your custom code (nested under `meta`), reducing the risk of collisions as we continue to add new capabilities and configurations to the dbt framework in the future. Because we plan to keep building dbt for a long time to come, and for that we need… ### Stability [Two years ago](https://github.com/dbt-labs/dbt-core/blob/main/docs/roadmap/2023-11-dbt-tng.md), we saw that lots of projects were still running older versions of dbt Core, even as we pushed ahead with newer ones. Our top priority was making it easier to upgrade. To that end, we’ve made significant investments in interface stability, and updated the release cadence to 1 new minor version every 6 months. All of that work has paid off: **Today, the majority of active dbt Core projects run on the latest minor version.** On December 1, >50% of projects were running on dbt Core `v1.10`, which we released in June. We just released `v1.11` this week, and we expect that the majority of projects will upgrade in the next 6 months. This means that when we fix bugs in patch releases, or release exciting language features in new minor versions — such as UDFs in `v1.11` — more users can get *immediate* access to those fixes and features than ever before. It also means that more users can benefit from strongly-typed schemas for dbt’s properties and configurations. **And if those users want to try out the new Fusion engine while it’s in preview, they can.** In order for the Fusion engine to work on your project, your dbt code will need to be compatible with the very latest language spec — which means resolving any deprecation warnings that you would have started seeing in dbt Core `v1.10` and `v1.11`. We have a tool that can help: [**`dbt-autofix`**](https://github.com/dbt-labs/dbt-autofix) scans your dbt project for deprecations, and automatically updates your code to align with the latest spec. We closed out this year with "De*bug*-cember" - a month-long bug bash where we squashed 35 long-standing issues across parsing, execution, logging, error messages, and more in the lead up to the final `v1.11` release. (To see the full list, head over to #dbt-core-development in the community Slack.) screenshot of slack post # What’s in the Queue for 2026? Over the next six months, we’re adding a bunch of new team members (say hi to new faces as they pop up in slack and github). With their help, we’re eager to tackle the backlog of bugs and "paper cuts". You can upvote and comment on issues to help us prioritize! As for bigger features… In our roadmap post from May, we proposed three possibilities for "[What’s next for dbt-the-language?](https://github.com/dbt-labs/dbt-core/blob/main/docs/roadmap/2025-05-new-engine-same-language.md#whats-next-for-dbt-the-language)": 1. Out-of-the-box support for UDFs 2. Sources from external catalogs 3. Model freshness checks We tackled #1 in the `v1.11` release. We intend to work on #3 next (github discussion coming soon). As for #2… [Jeremy says "it’s complicated"](https://www.youtube.com/watch?v=bRJJkeJkUsE&t=1s): > We are always building dbt within the context of the larger data ecosystem. In summer 2024, Databricks and Snowflake acclaimed Iceberg as the most promising standard for interoperable data storage. Following the flurry of new Iceberg catalogs, we [proposed a spec](https://github.com/dbt-labs/dbt-core/discussions/11171), implemented in dbt Core `v1.10`, to configure those `catalogs` as a place to materialize dbt models. > > Over the past year, the major data warehouses have added support for writing tables to *external* Iceberg catalogs — and even for *synchronizing* metadata from those catalogs (e.g. with a Snowflake [catalog-linked database](https://docs.snowflake.com/en/user-guide/tables-iceberg-catalog-linked-database)) so that you can treat their contents like any other table. This makes it substantially easier to treat Iceberg as [just another implementation detail](https://docs.getdbt.com/blog/icebeg-is-an-implementation-detail), and as the interchange layer for [cross-platform workflows](https://www.getdbt.com/blog/introducing-cross-platform-dbt-mesh). > > It also means that other features of dbt "just work." Earlier this year, we added an [experimental feature](https://github.com/dbt-labs/dbt-labs-experimental-features/tree/main/read-external-iceberg) for registering and refreshing Iceberg tables (à la `dbt-external-sources`), and in our May roadmap we proposed "sources from external catalogs" as a potential new feature of the dbt language. But if the underlying data warehouse can synchronize tables with an external Iceberg catalog automatically, then `sources` pointing to tables in an external Iceberg catalog already work, without any changes to the dbt language. That’s not the full end of the story — Iceberg requires a lot of setup (could dbt help here?) and careful optimization (might it be faster/cheaper to batch-calculate `freshness` by integrating with an Iceberg REST catalog directly, rather than querying the data warehouse?) — it warrants more time and testing. The plan is to keep evolving dbt along with the Iceberg ecosystem in 2026. Last-but-not-least, we will continue to invest in improving the stability of dbt Core. There’s much more we can improve about: - the interfaces shared between Core and Fusion - the integration between dbt Core and MetricFlow, [now that the latter is Apache 2.0 too](https://www.getdbt.com/blog/open-source-metricflow-governed-metrics) - the ease of contributing to our open source and source-available codebases # [Join us, leave your fields to flower](https://www.youtube.com/watch?v=AqbYa-NXFOg) Looking for ways to get involved in the dbt community? - give or get help in the [community slack](https://www.getdbt.com/community/join-the-community) - open up bug reports, feature requests, or discussions in our repos: [dbt-core](https://github.com/dbt-labs/dbt-core), [dbt-adapters](https://github.com/dbt-labs/dbt-adapters) (our new Core adapters monorepo), and [dbt-fusion](https://github.com/dbt-labs/dbt-fusion) are great places to start - join zoom feedback sessions to help shape new features coming to dbt - follow along with Fusion progress by [reading our diaries](https://github.com/dbt-labs/dbt-fusion/discussions/categories/announcements) and [subscribing to our linkedin newsletter](https://www.linkedin.com/newsletters/fusion-diaries-7366935294090084360/) - find community in-person at our [dbt meetups](https://www.meetup.com/pro/dbt/) and [roadshows](https://www.getdbt.com/events), all around the world See you in the new year, your neighborhood theater kids (Jerco & Grace) picture of jerco and grace running away ================================================ FILE: hatch.toml ================================================ # Root-level hatch.toml - warns users to run from core/ directory # The actual hatch configuration is in core/hatch.toml [envs.default] skip-install = true dependencies = [] [envs.default.scripts] setup = "echo '⚠️ Error: hatch must be run from the core/ directory, not the repo root.\n\nRun:\n cd core && hatch run setup' && exit 1" dev-req = "echo '⚠️ Error: hatch must be run from the core/ directory, not the repo root.\n\nRun:\n cd core && hatch run dev-req' && exit 1" code-quality = "echo '⚠️ Error: hatch must be run from the core/ directory, not the repo root.\n\nRun:\n cd core && hatch run code-quality' && exit 1" lint = "echo '⚠️ Error: hatch must be run from the core/ directory, not the repo root.\n\nRun:\n cd core && hatch run lint' && exit 1" flake8 = "echo '⚠️ Error: hatch must be run from the core/ directory, not the repo root.\n\nRun:\n cd core && hatch run flake8' && exit 1" mypy = "echo '⚠️ Error: hatch must be run from the core/ directory, not the repo root.\n\nRun:\n cd core && hatch run mypy' && exit 1" black = "echo '⚠️ Error: hatch must be run from the core/ directory, not the repo root.\n\nRun:\n cd core && hatch run black' && exit 1" unit-tests = "echo '⚠️ Error: hatch must be run from the core/ directory, not the repo root.\n\nRun:\n cd core && hatch run unit-tests' && exit 1" integration-tests = "echo '⚠️ Error: hatch must be run from the core/ directory, not the repo root.\n\nRun:\n cd core && hatch run integration-tests' && exit 1" integration-tests-fail-fast = "echo '⚠️ Error: hatch must be run from the core/ directory, not the repo root.\n\nRun:\n cd core && hatch run integration-tests-fail-fast' && exit 1" test = "echo '⚠️ Error: hatch must be run from the core/ directory, not the repo root.\n\nRun:\n cd core && hatch run test' && exit 1" setup-db = "echo '⚠️ Error: hatch must be run from the core/ directory, not the repo root.\n\nRun:\n cd core && hatch run setup-db' && exit 1" clean = "echo '⚠️ Error: hatch must be run from the core/ directory, not the repo root.\n\nRun:\n cd core && hatch run clean' && exit 1" json-schema = "echo '⚠️ Error: hatch must be run from the core/ directory, not the repo root.\n\nRun:\n cd core && hatch run json-schema' && exit 1" [envs.build] skip-install = true dependencies = [] [envs.build.scripts] check-all = "echo '⚠️ Error: hatch must be run from the core/ directory, not the repo root.\n\nRun:\n cd core && hatch run build:check-all' && exit 1" check-wheel = "echo '⚠️ Error: hatch must be run from the core/ directory, not the repo root.\n\nRun:\n cd core && hatch run build:check-wheel' && exit 1" check-sdist = "echo '⚠️ Error: hatch must be run from the core/ directory, not the repo root.\n\nRun:\n cd core && hatch run build:check-sdist' && exit 1" [envs.ci] skip-install = true dependencies = [] [envs.ci.scripts] unit-tests = "echo '⚠️ Error: hatch must be run from the core/ directory, not the repo root.\n\nRun:\n cd core && hatch run ci:unit-tests' && exit 1" code-quality = "echo '⚠️ Error: hatch must be run from the core/ directory, not the repo root.\n\nRun:\n cd core && hatch run ci:code-quality' && exit 1" integration-tests = "echo '⚠️ Error: hatch must be run from the core/ directory, not the repo root.\n\nRun:\n cd core && hatch run ci:integration-tests' && exit 1" ================================================ FILE: pyproject.toml ================================================ # Root-level pyproject.toml for tool configurations # Packaging configuration is in core/pyproject.toml # This file exists so tools like mypy and black can find their config when run from root [tool.mypy] # TODO: widen range of files as we fix issues files = 'core/dbt' mypy_path = "third-party-stubs/" namespace_packages = true [tool.black] line-length = 99 target-version = ['py38'] # flake8 config is in .flake8 (flake8 4.0.1 has limited pyproject.toml support). Add here when we add support for flake8 >=5 [tool.isort] profile = "black" extend_skip_glob = [ ".github/*", "third-party-stubs/*", "scripts/*", ] known_first_party = [ "dbt", "dbt_adapters", "dbt_common", "dbt_extractor", "dbt_semantic_interfaces", ] ================================================ FILE: pytest.ini ================================================ [pytest] filterwarnings = ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning ignore:unclosed file .*:ResourceWarning env_files = test.env testpaths = tests/functional tests/unit pythonpath = core ================================================ FILE: requirements.txt ================================================ ./core ================================================ FILE: schemas/dbt/catalog/v1.json ================================================ { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "title": "CatalogArtifact", "properties": { "metadata": { "type": "object", "title": "CatalogMetadata", "properties": { "dbt_schema_version": { "type": "string" }, "dbt_version": { "type": "string", "default": "1.12.0a1" }, "generated_at": { "type": "string" }, "invocation_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "invocation_started_at": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "env": { "type": "object", "additionalProperties": { "type": "string" }, "propertyNames": { "type": "string" } } }, "additionalProperties": false }, "nodes": { "type": "object", "additionalProperties": { "type": "object", "title": "CatalogTable", "properties": { "metadata": { "type": "object", "title": "TableMetadata", "properties": { "type": { "type": "string" }, "schema": { "type": "string" }, "name": { "type": "string" }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "comment": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "owner": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "type", "schema", "name" ] }, "columns": { "type": "object", "additionalProperties": { "type": "object", "title": "ColumnMetadata", "properties": { "type": { "type": "string" }, "index": { "type": "integer" }, "name": { "type": "string" }, "comment": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "type", "index", "name" ] }, "propertyNames": { "type": "string" } }, "stats": { "type": "object", "additionalProperties": { "type": "object", "title": "StatsItem", "properties": { "id": { "type": "string" }, "label": { "type": "string" }, "value": { "anyOf": [ { "type": "boolean" }, { "type": "string" }, { "type": "number" }, { "type": "null" } ] }, "include": { "type": "boolean" }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "id", "label", "value", "include" ] }, "propertyNames": { "type": "string" } }, "unique_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "metadata", "columns", "stats" ] }, "propertyNames": { "type": "string" } }, "sources": { "type": "object", "additionalProperties": { "type": "object", "title": "CatalogTable", "properties": { "metadata": { "type": "object", "title": "TableMetadata", "properties": { "type": { "type": "string" }, "schema": { "type": "string" }, "name": { "type": "string" }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "comment": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "owner": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "type", "schema", "name" ] }, "columns": { "type": "object", "additionalProperties": { "type": "object", "title": "ColumnMetadata", "properties": { "type": { "type": "string" }, "index": { "type": "integer" }, "name": { "type": "string" }, "comment": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "type", "index", "name" ] }, "propertyNames": { "type": "string" } }, "stats": { "type": "object", "additionalProperties": { "type": "object", "title": "StatsItem", "properties": { "id": { "type": "string" }, "label": { "type": "string" }, "value": { "anyOf": [ { "type": "boolean" }, { "type": "string" }, { "type": "number" }, { "type": "null" } ] }, "include": { "type": "boolean" }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "id", "label", "value", "include" ] }, "propertyNames": { "type": "string" } }, "unique_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "metadata", "columns", "stats" ] }, "propertyNames": { "type": "string" } }, "errors": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "null" } ], "default": null }, "_compile_results": { "anyOf": [ {}, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "metadata", "nodes", "sources" ], "$id": "https://schemas.getdbt.com/dbt/catalog/v1.json" } ================================================ FILE: schemas/dbt/manifest/v10.json ================================================ { "type": "object", "required": [ "metadata", "nodes", "sources", "macros", "docs", "exposures", "metrics", "groups", "selectors", "semantic_models" ], "properties": { "metadata": { "$ref": "#/definitions/ManifestMetadata", "description": "Metadata about the manifest" }, "nodes": { "type": "object", "additionalProperties": { "oneOf": [ { "$ref": "#/definitions/AnalysisNode" }, { "$ref": "#/definitions/SingularTestNode" }, { "$ref": "#/definitions/HookNode" }, { "$ref": "#/definitions/ModelNode" }, { "$ref": "#/definitions/RPCNode" }, { "$ref": "#/definitions/SqlNode" }, { "$ref": "#/definitions/GenericTestNode" }, { "$ref": "#/definitions/SnapshotNode" }, { "$ref": "#/definitions/SeedNode" } ] }, "description": "The nodes defined in the dbt project and its dependencies" }, "sources": { "type": "object", "additionalProperties": { "$ref": "#/definitions/SourceDefinition" }, "description": "The sources defined in the dbt project and its dependencies" }, "macros": { "type": "object", "additionalProperties": { "$ref": "#/definitions/Macro" }, "description": "The macros defined in the dbt project and its dependencies" }, "docs": { "type": "object", "additionalProperties": { "$ref": "#/definitions/Documentation" }, "description": "The docs defined in the dbt project and its dependencies" }, "exposures": { "type": "object", "additionalProperties": { "$ref": "#/definitions/Exposure" }, "description": "The exposures defined in the dbt project and its dependencies" }, "metrics": { "type": "object", "additionalProperties": { "$ref": "#/definitions/Metric" }, "description": "The metrics defined in the dbt project and its dependencies" }, "groups": { "type": "object", "additionalProperties": { "$ref": "#/definitions/Group" }, "description": "The groups defined in the dbt project" }, "selectors": { "type": "object", "description": "The selectors defined in selectors.yml" }, "disabled": { "oneOf": [ { "type": "object", "additionalProperties": { "type": "array", "items": { "oneOf": [ { "$ref": "#/definitions/AnalysisNode" }, { "$ref": "#/definitions/SingularTestNode" }, { "$ref": "#/definitions/HookNode" }, { "$ref": "#/definitions/ModelNode" }, { "$ref": "#/definitions/RPCNode" }, { "$ref": "#/definitions/SqlNode" }, { "$ref": "#/definitions/GenericTestNode" }, { "$ref": "#/definitions/SnapshotNode" }, { "$ref": "#/definitions/SeedNode" }, { "$ref": "#/definitions/SourceDefinition" }, { "$ref": "#/definitions/Exposure" }, { "$ref": "#/definitions/Metric" }, { "$ref": "#/definitions/SemanticModel" } ] } } }, { "type": "null" } ], "description": "A mapping of the disabled nodes in the target" }, "parent_map": { "oneOf": [ { "type": "object", "additionalProperties": { "type": "array", "items": { "type": "string" } } }, { "type": "null" } ], "description": "A mapping from\u00a0child nodes to their dependencies" }, "child_map": { "oneOf": [ { "type": "object", "additionalProperties": { "type": "array", "items": { "type": "string" } } }, { "type": "null" } ], "description": "A mapping from parent nodes to their dependents" }, "group_map": { "oneOf": [ { "type": "object", "additionalProperties": { "type": "array", "items": { "type": "string" } } }, { "type": "null" } ], "description": "A mapping from group names to their nodes" }, "semantic_models": { "type": "object", "additionalProperties": { "$ref": "#/definitions/SemanticModel" }, "description": "The semantic models defined in the dbt project" } }, "additionalProperties": false, "description": "WritableManifest(metadata: dbt.contracts.graph.manifest.ManifestMetadata, nodes: Mapping[str, Union[dbt.contracts.graph.nodes.AnalysisNode, dbt.contracts.graph.nodes.SingularTestNode, dbt.contracts.graph.nodes.HookNode, dbt.contracts.graph.nodes.ModelNode, dbt.contracts.graph.nodes.RPCNode, dbt.contracts.graph.nodes.SqlNode, dbt.contracts.graph.nodes.GenericTestNode, dbt.contracts.graph.nodes.SnapshotNode, dbt.contracts.graph.nodes.SeedNode]], sources: Mapping[str, dbt.contracts.graph.nodes.SourceDefinition], macros: Mapping[str, dbt.contracts.graph.nodes.Macro], docs: Mapping[str, dbt.contracts.graph.nodes.Documentation], exposures: Mapping[str, dbt.contracts.graph.nodes.Exposure], metrics: Mapping[str, dbt.contracts.graph.nodes.Metric], groups: Mapping[str, dbt.contracts.graph.nodes.Group], selectors: Mapping[str, Any], disabled: Union[Mapping[str, List[Union[dbt.contracts.graph.nodes.AnalysisNode, dbt.contracts.graph.nodes.SingularTestNode, dbt.contracts.graph.nodes.HookNode, dbt.contracts.graph.nodes.ModelNode, dbt.contracts.graph.nodes.RPCNode, dbt.contracts.graph.nodes.SqlNode, dbt.contracts.graph.nodes.GenericTestNode, dbt.contracts.graph.nodes.SnapshotNode, dbt.contracts.graph.nodes.SeedNode, dbt.contracts.graph.nodes.SourceDefinition, dbt.contracts.graph.nodes.Exposure, dbt.contracts.graph.nodes.Metric, dbt.contracts.graph.nodes.SemanticModel]]], NoneType], parent_map: Union[Dict[str, List[str]], NoneType], child_map: Union[Dict[str, List[str]], NoneType], group_map: Union[Dict[str, List[str]], NoneType], semantic_models: Mapping[str, dbt.contracts.graph.nodes.SemanticModel])", "definitions": { "ManifestMetadata": { "type": "object", "required": [], "properties": { "dbt_schema_version": { "type": "string", "default": "https://schemas.getdbt.com/dbt/manifest/v10.json" }, "dbt_version": { "type": "string", "default": "1.6.5" }, "generated_at": { "type": "string", "format": "date-time", "default": "2023-10-05T00:33:14.410024Z" }, "invocation_id": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "default": "603e2fae-9c7d-4d17-8530-7d28c9875263" }, "env": { "type": "object", "additionalProperties": { "type": "string" }, "default": {} }, "project_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "description": "Name of the root project" }, "project_id": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "description": "A unique identifier for the project, hashed from the project name" }, "user_id": { "oneOf": [ { "type": "string", "pattern": "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" }, { "type": "null" } ], "description": "A unique identifier for the user" }, "send_anonymous_usage_stats": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ], "description": "Whether dbt is configured to send anonymous usage statistics" }, "adapter_type": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "description": "The type name of the adapter" } }, "additionalProperties": false, "description": "Metadata for the manifest." }, "AnalysisNode": { "type": "object", "required": [ "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ], "properties": { "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "analysis" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "contract": { "enforced": false }, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "group": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1696465994.411958 }, "config_call_dict": { "type": "object", "default": {} }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "raw_code": { "type": "string", "default": "" }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "$ref": "#/definitions/RefArgs" }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "contract": { "$ref": "#/definitions/Contract", "default": { "enforced": false, "checksum": null } } }, "additionalProperties": false, "description": "AnalysisNode(database: Union[str, NoneType], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , group: Union[str, NoneType] = None, docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Union[str, NoneType] = None, raw_code: str = '', language: str = 'sql', refs: List[dbt.contracts.graph.nodes.RefArgs] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , compiled_path: Union[str, NoneType] = None, compiled: bool = False, compiled_code: Union[str, NoneType] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = , _pre_injected_sql: Union[str, NoneType] = None, contract: dbt.contracts.graph.nodes.Contract = )" }, "FileHash": { "type": "object", "required": [ "name", "checksum" ], "properties": { "name": { "type": "string" }, "checksum": { "type": "string" } }, "additionalProperties": false, "description": "FileHash(name: str, checksum: str)" }, "NodeConfig": { "type": "object", "required": [], "properties": { "enabled": { "type": "boolean", "default": true }, "alias": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "tags": { "oneOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ], "default": [] }, "meta": { "type": "object", "default": {} }, "group": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "materialized": { "type": "string", "default": "view" }, "incremental_strategy": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "persist_docs": { "type": "object", "default": {} }, "post-hook": { "type": "array", "items": { "$ref": "#/definitions/Hook" }, "default": [] }, "pre-hook": { "type": "array", "items": { "$ref": "#/definitions/Hook" }, "default": [] }, "quoting": { "type": "object", "default": {} }, "column_types": { "type": "object", "default": {} }, "full_refresh": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "unique_key": { "oneOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ] }, "on_schema_change": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "on_configuration_change": { "type": "string", "enum": [ "apply", "continue", "fail" ], "default": "apply" }, "grants": { "type": "object", "default": {} }, "packages": { "type": "array", "items": { "type": "string" }, "default": [] }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "contract": { "$ref": "#/definitions/ContractConfig", "default": { "enforced": false } } }, "additionalProperties": true, "description": "NodeConfig(_extra: Dict[str, Any] = , enabled: bool = True, alias: Union[str, NoneType] = None, schema: Union[str, NoneType] = None, database: Union[str, NoneType] = None, tags: Union[List[str], str] = , meta: Dict[str, Any] = , group: Union[str, NoneType] = None, materialized: str = 'view', incremental_strategy: Union[str, NoneType] = None, persist_docs: Dict[str, Any] = , post_hook: List[dbt.contracts.graph.model_config.Hook] = , pre_hook: List[dbt.contracts.graph.model_config.Hook] = , quoting: Dict[str, Any] = , column_types: Dict[str, Any] = , full_refresh: Union[bool, NoneType] = None, unique_key: Union[str, List[str], NoneType] = None, on_schema_change: Union[str, NoneType] = 'ignore', on_configuration_change: dbt.contracts.graph.model_config.OnConfigurationChangeOption = , grants: Dict[str, Any] = , packages: List[str] = , docs: dbt.contracts.graph.unparsed.Docs = , contract: dbt.contracts.graph.model_config.ContractConfig = )" }, "Hook": { "type": "object", "required": [ "sql" ], "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "oneOf": [ { "type": "integer" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "Hook(sql: str, transaction: bool = True, index: Union[int, NoneType] = None)" }, "Docs": { "type": "object", "required": [], "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "Docs(show: bool = True, node_color: Union[str, NoneType] = None)" }, "ContractConfig": { "type": "object", "required": [], "properties": { "enforced": { "type": "boolean", "default": false } }, "additionalProperties": false, "description": "ContractConfig(enforced: bool = False)" }, "ColumnInfo": { "type": "object", "required": [ "name" ], "properties": { "name": { "type": "string" }, "description": { "type": "string", "default": "" }, "meta": { "type": "object", "default": {} }, "data_type": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "constraints": { "type": "array", "items": { "$ref": "#/definitions/ColumnLevelConstraint" }, "default": [] }, "quote": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] } }, "additionalProperties": true, "description": "Used in all ManifestNodes and SourceDefinition" }, "ColumnLevelConstraint": { "type": "object", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ "check", "not_null", "unique", "primary_key", "foreign_key", "custom" ] }, "name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "expression": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "warn_unenforced": { "type": "boolean", "default": true }, "warn_unsupported": { "type": "boolean", "default": true } }, "additionalProperties": false, "description": "ColumnLevelConstraint(type: dbt.contracts.graph.nodes.ConstraintType, name: Union[str, NoneType] = None, expression: Union[str, NoneType] = None, warn_unenforced: bool = True, warn_unsupported: bool = True)" }, "RefArgs": { "type": "object", "required": [ "name" ], "properties": { "name": { "type": "string" }, "package": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "version": { "oneOf": [ { "type": "string" }, { "type": "number" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "RefArgs(name: str, package: Union[str, NoneType] = None, version: Union[str, float, NoneType] = None)" }, "DependsOn": { "type": "object", "required": [], "properties": { "macros": { "type": "array", "items": { "type": "string" }, "default": [] }, "nodes": { "type": "array", "items": { "type": "string" }, "default": [] } }, "additionalProperties": false, "description": "DependsOn(macros: List[str] = , nodes: List[str] = )" }, "InjectedCTE": { "type": "object", "required": [ "id", "sql" ], "properties": { "id": { "type": "string" }, "sql": { "type": "string" } }, "additionalProperties": false, "description": "Used in CompiledNodes as part of ephemeral model processing" }, "Contract": { "type": "object", "required": [], "properties": { "enforced": { "type": "boolean", "default": false }, "checksum": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "Contract(enforced: bool = False, checksum: Union[str, NoneType] = None)" }, "SingularTestNode": { "type": "object", "required": [ "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ], "properties": { "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "test" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/TestConfig", "default": { "enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0" } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "group": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1696465994.413604 }, "config_call_dict": { "type": "object", "default": {} }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "raw_code": { "type": "string", "default": "" }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "$ref": "#/definitions/RefArgs" }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "contract": { "$ref": "#/definitions/Contract", "default": { "enforced": false, "checksum": null } } }, "additionalProperties": false, "description": "SingularTestNode(database: Union[str, NoneType], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.TestConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , group: Union[str, NoneType] = None, docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Union[str, NoneType] = None, raw_code: str = '', language: str = 'sql', refs: List[dbt.contracts.graph.nodes.RefArgs] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , compiled_path: Union[str, NoneType] = None, compiled: bool = False, compiled_code: Union[str, NoneType] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = , _pre_injected_sql: Union[str, NoneType] = None, contract: dbt.contracts.graph.nodes.Contract = )" }, "TestConfig": { "type": "object", "required": [], "properties": { "enabled": { "type": "boolean", "default": true }, "alias": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "default": "dbt_test__audit" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "tags": { "oneOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ], "default": [] }, "meta": { "type": "object", "default": {} }, "group": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "materialized": { "type": "string", "default": "test" }, "severity": { "type": "string", "pattern": "^([Ww][Aa][Rr][Nn]|[Ee][Rr][Rr][Oo][Rr])$", "default": "ERROR" }, "store_failures": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "where": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "limit": { "oneOf": [ { "type": "integer" }, { "type": "null" } ] }, "fail_calc": { "type": "string", "default": "count(*)" }, "warn_if": { "type": "string", "default": "!= 0" }, "error_if": { "type": "string", "default": "!= 0" } }, "additionalProperties": true, "description": "TestConfig(_extra: Dict[str, Any] = , enabled: bool = True, alias: Union[str, NoneType] = None, schema: Union[str, NoneType] = 'dbt_test__audit', database: Union[str, NoneType] = None, tags: Union[List[str], str] = , meta: Dict[str, Any] = , group: Union[str, NoneType] = None, materialized: str = 'test', severity: dbt.contracts.graph.model_config.Severity = 'ERROR', store_failures: Union[bool, NoneType] = None, where: Union[str, NoneType] = None, limit: Union[int, NoneType] = None, fail_calc: str = 'count(*)', warn_if: str = '!= 0', error_if: str = '!= 0')" }, "HookNode": { "type": "object", "required": [ "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ], "properties": { "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "operation" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "contract": { "enforced": false }, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "group": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1696465994.414359 }, "config_call_dict": { "type": "object", "default": {} }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "raw_code": { "type": "string", "default": "" }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "$ref": "#/definitions/RefArgs" }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "contract": { "$ref": "#/definitions/Contract", "default": { "enforced": false, "checksum": null } }, "index": { "oneOf": [ { "type": "integer" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "HookNode(database: Union[str, NoneType], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , group: Union[str, NoneType] = None, docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Union[str, NoneType] = None, raw_code: str = '', language: str = 'sql', refs: List[dbt.contracts.graph.nodes.RefArgs] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , compiled_path: Union[str, NoneType] = None, compiled: bool = False, compiled_code: Union[str, NoneType] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = , _pre_injected_sql: Union[str, NoneType] = None, contract: dbt.contracts.graph.nodes.Contract = , index: Union[int, NoneType] = None)" }, "ModelNode": { "type": "object", "required": [ "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ], "properties": { "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "model" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "contract": { "enforced": false }, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "group": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1696465994.4150689 }, "config_call_dict": { "type": "object", "default": {} }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "raw_code": { "type": "string", "default": "" }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "$ref": "#/definitions/RefArgs" }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "contract": { "$ref": "#/definitions/Contract", "default": { "enforced": false, "checksum": null } }, "access": { "type": "string", "enum": [ "private", "protected", "public" ], "default": "protected" }, "constraints": { "type": "array", "items": { "$ref": "#/definitions/ModelLevelConstraint" }, "default": [] }, "version": { "oneOf": [ { "type": "string" }, { "type": "number" }, { "type": "null" } ] }, "latest_version": { "oneOf": [ { "type": "string" }, { "type": "number" }, { "type": "null" } ] }, "deprecation_date": { "oneOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ] }, "defer_relation": { "oneOf": [ { "$ref": "#/definitions/DeferRelation" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "ModelNode(database: Union[str, NoneType], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , group: Union[str, NoneType] = None, docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Union[str, NoneType] = None, raw_code: str = '', language: str = 'sql', refs: List[dbt.contracts.graph.nodes.RefArgs] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , compiled_path: Union[str, NoneType] = None, compiled: bool = False, compiled_code: Union[str, NoneType] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = , _pre_injected_sql: Union[str, NoneType] = None, contract: dbt.contracts.graph.nodes.Contract = , access: dbt.node_types.AccessType = , constraints: List[dbt.contracts.graph.nodes.ModelLevelConstraint] = , version: Union[str, float, NoneType] = None, latest_version: Union[str, float, NoneType] = None, deprecation_date: Union[datetime.datetime, NoneType] = None, defer_relation: Union[dbt.contracts.graph.nodes.DeferRelation, NoneType] = None)" }, "ModelLevelConstraint": { "type": "object", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ "check", "not_null", "unique", "primary_key", "foreign_key", "custom" ] }, "name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "expression": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "warn_unenforced": { "type": "boolean", "default": true }, "warn_unsupported": { "type": "boolean", "default": true }, "columns": { "type": "array", "items": { "type": "string" }, "default": [] } }, "additionalProperties": false, "description": "ModelLevelConstraint(type: dbt.contracts.graph.nodes.ConstraintType, name: Union[str, NoneType] = None, expression: Union[str, NoneType] = None, warn_unenforced: bool = True, warn_unsupported: bool = True, columns: List[str] = )" }, "DeferRelation": { "type": "object", "required": [ "schema", "alias" ], "properties": { "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "alias": { "type": "string" }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "DeferRelation(database: Union[str, NoneType], schema: str, alias: str, relation_name: Union[str, NoneType])" }, "RPCNode": { "type": "object", "required": [ "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ], "properties": { "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "rpc" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "contract": { "enforced": false }, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "group": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1696465994.416128 }, "config_call_dict": { "type": "object", "default": {} }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "raw_code": { "type": "string", "default": "" }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "$ref": "#/definitions/RefArgs" }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "contract": { "$ref": "#/definitions/Contract", "default": { "enforced": false, "checksum": null } } }, "additionalProperties": false, "description": "RPCNode(database: Union[str, NoneType], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , group: Union[str, NoneType] = None, docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Union[str, NoneType] = None, raw_code: str = '', language: str = 'sql', refs: List[dbt.contracts.graph.nodes.RefArgs] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , compiled_path: Union[str, NoneType] = None, compiled: bool = False, compiled_code: Union[str, NoneType] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = , _pre_injected_sql: Union[str, NoneType] = None, contract: dbt.contracts.graph.nodes.Contract = )" }, "SqlNode": { "type": "object", "required": [ "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ], "properties": { "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "sql_operation" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "contract": { "enforced": false }, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "group": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1696465994.41679 }, "config_call_dict": { "type": "object", "default": {} }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "raw_code": { "type": "string", "default": "" }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "$ref": "#/definitions/RefArgs" }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "contract": { "$ref": "#/definitions/Contract", "default": { "enforced": false, "checksum": null } } }, "additionalProperties": false, "description": "SqlNode(database: Union[str, NoneType], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , group: Union[str, NoneType] = None, docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Union[str, NoneType] = None, raw_code: str = '', language: str = 'sql', refs: List[dbt.contracts.graph.nodes.RefArgs] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , compiled_path: Union[str, NoneType] = None, compiled: bool = False, compiled_code: Union[str, NoneType] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = , _pre_injected_sql: Union[str, NoneType] = None, contract: dbt.contracts.graph.nodes.Contract = )" }, "GenericTestNode": { "type": "object", "required": [ "test_metadata", "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ], "properties": { "test_metadata": { "$ref": "#/definitions/TestMetadata" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "test" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/TestConfig", "default": { "enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0" } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "group": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1696465994.4175282 }, "config_call_dict": { "type": "object", "default": {} }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "raw_code": { "type": "string", "default": "" }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "$ref": "#/definitions/RefArgs" }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "contract": { "$ref": "#/definitions/Contract", "default": { "enforced": false, "checksum": null } }, "column_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "file_key_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "attached_node": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "GenericTestNode(test_metadata: dbt.contracts.graph.nodes.TestMetadata, database: Union[str, NoneType], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.TestConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , group: Union[str, NoneType] = None, docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Union[str, NoneType] = None, raw_code: str = '', language: str = 'sql', refs: List[dbt.contracts.graph.nodes.RefArgs] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , compiled_path: Union[str, NoneType] = None, compiled: bool = False, compiled_code: Union[str, NoneType] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = , _pre_injected_sql: Union[str, NoneType] = None, contract: dbt.contracts.graph.nodes.Contract = , column_name: Union[str, NoneType] = None, file_key_name: Union[str, NoneType] = None, attached_node: Union[str, NoneType] = None)" }, "TestMetadata": { "type": "object", "required": [ "name" ], "properties": { "name": { "type": "string" }, "kwargs": { "type": "object", "default": {} }, "namespace": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "TestMetadata(name: str, kwargs: Dict[str, Any] = , namespace: Union[str, NoneType] = None)" }, "SnapshotNode": { "type": "object", "required": [ "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum", "config" ], "properties": { "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "snapshot" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/SnapshotConfig" }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "group": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1696465994.418854 }, "config_call_dict": { "type": "object", "default": {} }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "raw_code": { "type": "string", "default": "" }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "$ref": "#/definitions/RefArgs" }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "contract": { "$ref": "#/definitions/Contract", "default": { "enforced": false, "checksum": null } }, "defer_relation": { "oneOf": [ { "$ref": "#/definitions/DeferRelation" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "SnapshotNode(database: Union[str, NoneType], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.SnapshotConfig, _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , group: Union[str, NoneType] = None, docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Union[str, NoneType] = None, raw_code: str = '', language: str = 'sql', refs: List[dbt.contracts.graph.nodes.RefArgs] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , compiled_path: Union[str, NoneType] = None, compiled: bool = False, compiled_code: Union[str, NoneType] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = , _pre_injected_sql: Union[str, NoneType] = None, contract: dbt.contracts.graph.nodes.Contract = , defer_relation: Union[dbt.contracts.graph.nodes.DeferRelation, NoneType] = None)" }, "SnapshotConfig": { "type": "object", "required": [], "properties": { "enabled": { "type": "boolean", "default": true }, "alias": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "tags": { "oneOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ], "default": [] }, "meta": { "type": "object", "default": {} }, "group": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "materialized": { "type": "string", "default": "snapshot" }, "incremental_strategy": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "persist_docs": { "type": "object", "default": {} }, "post-hook": { "type": "array", "items": { "$ref": "#/definitions/Hook" }, "default": [] }, "pre-hook": { "type": "array", "items": { "$ref": "#/definitions/Hook" }, "default": [] }, "quoting": { "type": "object", "default": {} }, "column_types": { "type": "object", "default": {} }, "full_refresh": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "unique_key": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "on_schema_change": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "on_configuration_change": { "type": "string", "enum": [ "apply", "continue", "fail" ], "default": "apply" }, "grants": { "type": "object", "default": {} }, "packages": { "type": "array", "items": { "type": "string" }, "default": [] }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "contract": { "$ref": "#/definitions/ContractConfig", "default": { "enforced": false } }, "strategy": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "target_schema": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "target_database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "updated_at": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "check_cols": { "oneOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ] } }, "additionalProperties": true, "description": "SnapshotConfig(_extra: Dict[str, Any] = , enabled: bool = True, alias: Union[str, NoneType] = None, schema: Union[str, NoneType] = None, database: Union[str, NoneType] = None, tags: Union[List[str], str] = , meta: Dict[str, Any] = , group: Union[str, NoneType] = None, materialized: str = 'snapshot', incremental_strategy: Union[str, NoneType] = None, persist_docs: Dict[str, Any] = , post_hook: List[dbt.contracts.graph.model_config.Hook] = , pre_hook: List[dbt.contracts.graph.model_config.Hook] = , quoting: Dict[str, Any] = , column_types: Dict[str, Any] = , full_refresh: Union[bool, NoneType] = None, unique_key: Union[str, NoneType] = None, on_schema_change: Union[str, NoneType] = 'ignore', on_configuration_change: dbt.contracts.graph.model_config.OnConfigurationChangeOption = , grants: Dict[str, Any] = , packages: List[str] = , docs: dbt.contracts.graph.unparsed.Docs = , contract: dbt.contracts.graph.model_config.ContractConfig = , strategy: Union[str, NoneType] = None, target_schema: Union[str, NoneType] = None, target_database: Union[str, NoneType] = None, updated_at: Union[str, NoneType] = None, check_cols: Union[str, List[str], NoneType] = None)" }, "SeedNode": { "type": "object", "required": [ "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ], "properties": { "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "seed" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/SeedConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "seed", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "contract": { "enforced": false }, "quote_columns": null, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "group": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1696465994.420199 }, "config_call_dict": { "type": "object", "default": {} }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "raw_code": { "type": "string", "default": "" }, "root_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "depends_on": { "$ref": "#/definitions/MacroDependsOn", "default": { "macros": [] } }, "defer_relation": { "oneOf": [ { "$ref": "#/definitions/DeferRelation" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "SeedNode(database: Union[str, NoneType], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.SeedConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , group: Union[str, NoneType] = None, docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Union[str, NoneType] = None, raw_code: str = '', root_path: Union[str, NoneType] = None, depends_on: dbt.contracts.graph.nodes.MacroDependsOn = , defer_relation: Union[dbt.contracts.graph.nodes.DeferRelation, NoneType] = None)" }, "SeedConfig": { "type": "object", "required": [], "properties": { "enabled": { "type": "boolean", "default": true }, "alias": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "tags": { "oneOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ], "default": [] }, "meta": { "type": "object", "default": {} }, "group": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "materialized": { "type": "string", "default": "seed" }, "incremental_strategy": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "persist_docs": { "type": "object", "default": {} }, "post-hook": { "type": "array", "items": { "$ref": "#/definitions/Hook" }, "default": [] }, "pre-hook": { "type": "array", "items": { "$ref": "#/definitions/Hook" }, "default": [] }, "quoting": { "type": "object", "default": {} }, "column_types": { "type": "object", "default": {} }, "full_refresh": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "unique_key": { "oneOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ] }, "on_schema_change": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "on_configuration_change": { "type": "string", "enum": [ "apply", "continue", "fail" ], "default": "apply" }, "grants": { "type": "object", "default": {} }, "packages": { "type": "array", "items": { "type": "string" }, "default": [] }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "contract": { "$ref": "#/definitions/ContractConfig", "default": { "enforced": false } }, "quote_columns": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] } }, "additionalProperties": true, "description": "SeedConfig(_extra: Dict[str, Any] = , enabled: bool = True, alias: Union[str, NoneType] = None, schema: Union[str, NoneType] = None, database: Union[str, NoneType] = None, tags: Union[List[str], str] = , meta: Dict[str, Any] = , group: Union[str, NoneType] = None, materialized: str = 'seed', incremental_strategy: Union[str, NoneType] = None, persist_docs: Dict[str, Any] = , post_hook: List[dbt.contracts.graph.model_config.Hook] = , pre_hook: List[dbt.contracts.graph.model_config.Hook] = , quoting: Dict[str, Any] = , column_types: Dict[str, Any] = , full_refresh: Union[bool, NoneType] = None, unique_key: Union[str, List[str], NoneType] = None, on_schema_change: Union[str, NoneType] = 'ignore', on_configuration_change: dbt.contracts.graph.model_config.OnConfigurationChangeOption = , grants: Dict[str, Any] = , packages: List[str] = , docs: dbt.contracts.graph.unparsed.Docs = , contract: dbt.contracts.graph.model_config.ContractConfig = , quote_columns: Union[bool, NoneType] = None)" }, "MacroDependsOn": { "type": "object", "required": [], "properties": { "macros": { "type": "array", "items": { "type": "string" }, "default": [] } }, "additionalProperties": false, "description": "Used only in the Macro class" }, "SourceDefinition": { "type": "object", "required": [ "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "source_name", "source_description", "loader", "identifier" ], "properties": { "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "source" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "source_name": { "type": "string" }, "source_description": { "type": "string" }, "loader": { "type": "string" }, "identifier": { "type": "string" }, "quoting": { "$ref": "#/definitions/Quoting", "default": { "database": null, "schema": null, "identifier": null, "column": null } }, "loaded_at_field": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "freshness": { "oneOf": [ { "$ref": "#/definitions/FreshnessThreshold" }, { "type": "null" } ] }, "external": { "oneOf": [ { "$ref": "#/definitions/ExternalTable" }, { "type": "null" } ] }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "source_meta": { "type": "object", "default": {} }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "config": { "$ref": "#/definitions/SourceConfig", "default": { "enabled": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "unrendered_config": { "type": "object", "default": {} }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "created_at": { "type": "number", "default": 1696465994.421661 } }, "additionalProperties": false, "description": "SourceDefinition(database: Union[str, NoneType], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], source_name: str, source_description: str, loader: str, identifier: str, _event_status: Dict[str, Any] = , quoting: dbt.contracts.graph.unparsed.Quoting = , loaded_at_field: Union[str, NoneType] = None, freshness: Union[dbt.contracts.graph.unparsed.FreshnessThreshold, NoneType] = None, external: Union[dbt.contracts.graph.unparsed.ExternalTable, NoneType] = None, description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , source_meta: Dict[str, Any] = , tags: List[str] = , config: dbt.contracts.graph.model_config.SourceConfig = , patch_path: Union[str, NoneType] = None, unrendered_config: Dict[str, Any] = , relation_name: Union[str, NoneType] = None, created_at: float = )" }, "Quoting": { "type": "object", "required": [], "properties": { "database": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "schema": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "identifier": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "column": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "Quoting(database: Union[bool, NoneType] = None, schema: Union[bool, NoneType] = None, identifier: Union[bool, NoneType] = None, column: Union[bool, NoneType] = None)" }, "FreshnessThreshold": { "type": "object", "required": [], "properties": { "warn_after": { "oneOf": [ { "$ref": "#/definitions/Time" }, { "type": "null" } ], "default": { "count": null, "period": null } }, "error_after": { "oneOf": [ { "$ref": "#/definitions/Time" }, { "type": "null" } ], "default": { "count": null, "period": null } }, "filter": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "FreshnessThreshold(warn_after: Union[dbt.contracts.graph.unparsed.Time, NoneType] = , error_after: Union[dbt.contracts.graph.unparsed.Time, NoneType] = , filter: Union[str, NoneType] = None)" }, "Time": { "type": "object", "required": [], "properties": { "count": { "oneOf": [ { "type": "integer" }, { "type": "null" } ] }, "period": { "oneOf": [ { "type": "string", "enum": [ "minute", "hour", "day" ] }, { "type": "null" } ] } }, "additionalProperties": false, "description": "Time(count: Union[int, NoneType] = None, period: Union[dbt.contracts.graph.unparsed.TimePeriod, NoneType] = None)" }, "ExternalTable": { "type": "object", "required": [], "properties": { "location": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "file_format": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "row_format": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "tbl_properties": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "partitions": { "oneOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "array", "items": { "$ref": "#/definitions/ExternalPartition" } }, { "type": "null" } ] } }, "additionalProperties": true, "description": "ExternalTable(_extra: Dict[str, Any] = , location: Union[str, NoneType] = None, file_format: Union[str, NoneType] = None, row_format: Union[str, NoneType] = None, tbl_properties: Union[str, NoneType] = None, partitions: Union[List[str], List[dbt.contracts.graph.unparsed.ExternalPartition], NoneType] = None)" }, "ExternalPartition": { "type": "object", "required": [], "properties": { "name": { "type": "string", "default": "" }, "description": { "type": "string", "default": "" }, "data_type": { "type": "string", "default": "" }, "meta": { "type": "object", "default": {} } }, "additionalProperties": true, "description": "ExternalPartition(_extra: Dict[str, Any] = , name: str = '', description: str = '', data_type: str = '', meta: Dict[str, Any] = )" }, "SourceConfig": { "type": "object", "required": [], "properties": { "enabled": { "type": "boolean", "default": true } }, "additionalProperties": true, "description": "SourceConfig(_extra: Dict[str, Any] = , enabled: bool = True)" }, "Macro": { "type": "object", "required": [ "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "macro_sql" ], "properties": { "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "macro" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "macro_sql": { "type": "string" }, "depends_on": { "$ref": "#/definitions/MacroDependsOn", "default": { "macros": [] } }, "description": { "type": "string", "default": "" }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "arguments": { "type": "array", "items": { "$ref": "#/definitions/MacroArgument" }, "default": [] }, "created_at": { "type": "number", "default": 1696465994.421958 }, "supported_languages": { "oneOf": [ { "type": "array", "items": { "type": "string", "enum": [ "python", "sql" ] } }, { "type": "null" } ] } }, "additionalProperties": false, "description": "Macro(name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, macro_sql: str, depends_on: dbt.contracts.graph.nodes.MacroDependsOn = , description: str = '', meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, arguments: List[dbt.contracts.graph.unparsed.MacroArgument] = , created_at: float = , supported_languages: Union[List[dbt.node_types.ModelLanguage], NoneType] = None)" }, "MacroArgument": { "type": "object", "required": [ "name" ], "properties": { "name": { "type": "string" }, "type": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "description": { "type": "string", "default": "" } }, "additionalProperties": false, "description": "MacroArgument(name: str, type: Union[str, NoneType] = None, description: str = '')" }, "Documentation": { "type": "object", "required": [ "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "block_contents" ], "properties": { "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "doc" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "block_contents": { "type": "string" } }, "additionalProperties": false, "description": "Documentation(name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, block_contents: str)" }, "Exposure": { "type": "object", "required": [ "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "type", "owner" ], "properties": { "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "exposure" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "type": { "type": "string", "enum": [ "dashboard", "notebook", "analysis", "ml", "application" ] }, "owner": { "$ref": "#/definitions/Owner" }, "description": { "type": "string", "default": "" }, "label": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "maturity": { "oneOf": [ { "type": "string", "enum": [ "low", "medium", "high" ] }, { "type": "null" } ] }, "meta": { "type": "object", "default": {} }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "config": { "$ref": "#/definitions/ExposureConfig", "default": { "enabled": true } }, "unrendered_config": { "type": "object", "default": {} }, "url": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "refs": { "type": "array", "items": { "$ref": "#/definitions/RefArgs" }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "created_at": { "type": "number", "default": 1696465994.422623 } }, "additionalProperties": false, "description": "Exposure(name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], type: dbt.contracts.graph.unparsed.ExposureType, owner: dbt.contracts.graph.unparsed.Owner, description: str = '', label: Union[str, NoneType] = None, maturity: Union[dbt.contracts.graph.unparsed.MaturityType, NoneType] = None, meta: Dict[str, Any] = , tags: List[str] = , config: dbt.contracts.graph.model_config.ExposureConfig = , unrendered_config: Dict[str, Any] = , url: Union[str, NoneType] = None, depends_on: dbt.contracts.graph.nodes.DependsOn = , refs: List[dbt.contracts.graph.nodes.RefArgs] = , sources: List[List[str]] = , metrics: List[List[str]] = , created_at: float = )" }, "Owner": { "type": "object", "required": [], "properties": { "email": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": true, "description": "Owner(_extra: Dict[str, Any] = , email: Union[str, NoneType] = None, name: Union[str, NoneType] = None)" }, "ExposureConfig": { "type": "object", "required": [], "properties": { "enabled": { "type": "boolean", "default": true } }, "additionalProperties": true, "description": "ExposureConfig(_extra: Dict[str, Any] = , enabled: bool = True)" }, "Metric": { "type": "object", "required": [ "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "description", "label", "type", "type_params" ], "properties": { "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "metric" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "description": { "type": "string" }, "label": { "type": "string" }, "type": { "type": "string", "enum": [ "simple", "ratio", "cumulative", "derived" ] }, "type_params": { "$ref": "#/definitions/MetricTypeParams" }, "filter": { "oneOf": [ { "$ref": "#/definitions/WhereFilter" }, { "type": "null" } ] }, "metadata": { "oneOf": [ { "$ref": "#/definitions/SourceFileMetadata" }, { "type": "null" } ] }, "meta": { "type": "object", "default": {} }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "config": { "$ref": "#/definitions/MetricConfig", "default": { "enabled": true, "group": null } }, "unrendered_config": { "type": "object", "default": {} }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "refs": { "type": "array", "items": { "$ref": "#/definitions/RefArgs" }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "created_at": { "type": "number", "default": 1696465994.4238322 }, "group": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "Metric(name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], description: str, label: str, type: dbt_semantic_interfaces.type_enums.metric_type.MetricType, type_params: dbt.contracts.graph.nodes.MetricTypeParams, filter: Union[dbt.contracts.graph.nodes.WhereFilter, NoneType] = None, metadata: Union[dbt.contracts.graph.semantic_models.SourceFileMetadata, NoneType] = None, meta: Dict[str, Any] = , tags: List[str] = , config: dbt.contracts.graph.model_config.MetricConfig = , unrendered_config: Dict[str, Any] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , refs: List[dbt.contracts.graph.nodes.RefArgs] = , metrics: List[List[str]] = , created_at: float = , group: Union[str, NoneType] = None)" }, "MetricTypeParams": { "type": "object", "required": [], "properties": { "measure": { "oneOf": [ { "$ref": "#/definitions/MetricInputMeasure" }, { "type": "null" } ] }, "input_measures": { "type": "array", "items": { "$ref": "#/definitions/MetricInputMeasure" }, "default": [] }, "numerator": { "oneOf": [ { "$ref": "#/definitions/MetricInput" }, { "type": "null" } ] }, "denominator": { "oneOf": [ { "$ref": "#/definitions/MetricInput" }, { "type": "null" } ] }, "expr": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "window": { "oneOf": [ { "$ref": "#/definitions/MetricTimeWindow" }, { "type": "null" } ] }, "grain_to_date": { "oneOf": [ { "type": "string", "enum": [ "day", "week", "month", "quarter", "year" ] }, { "type": "null" } ] }, "metrics": { "oneOf": [ { "type": "array", "items": { "$ref": "#/definitions/MetricInput" } }, { "type": "null" } ] } }, "additionalProperties": false, "description": "MetricTypeParams(measure: Union[dbt.contracts.graph.nodes.MetricInputMeasure, NoneType] = None, input_measures: List[dbt.contracts.graph.nodes.MetricInputMeasure] = , numerator: Union[dbt.contracts.graph.nodes.MetricInput, NoneType] = None, denominator: Union[dbt.contracts.graph.nodes.MetricInput, NoneType] = None, expr: Union[str, NoneType] = None, window: Union[dbt.contracts.graph.nodes.MetricTimeWindow, NoneType] = None, grain_to_date: Union[dbt_semantic_interfaces.type_enums.time_granularity.TimeGranularity, NoneType] = None, metrics: Union[List[dbt.contracts.graph.nodes.MetricInput], NoneType] = None)" }, "MetricInputMeasure": { "type": "object", "required": [ "name" ], "properties": { "name": { "type": "string" }, "filter": { "oneOf": [ { "$ref": "#/definitions/WhereFilter" }, { "type": "null" } ] }, "alias": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "join_to_timespine": { "type": "boolean", "default": false }, "fill_nulls_with": { "oneOf": [ { "type": "integer" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "MetricInputMeasure(name: str, filter: Union[dbt.contracts.graph.nodes.WhereFilter, NoneType] = None, alias: Union[str, NoneType] = None, join_to_timespine: bool = False, fill_nulls_with: Union[int, NoneType] = None)" }, "WhereFilter": { "type": "object", "required": [ "where_sql_template" ], "properties": { "where_sql_template": { "type": "string" } }, "additionalProperties": false, "description": "WhereFilter(where_sql_template: str)" }, "MetricInput": { "type": "object", "required": [ "name" ], "properties": { "name": { "type": "string" }, "filter": { "oneOf": [ { "$ref": "#/definitions/WhereFilter" }, { "type": "null" } ] }, "alias": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "offset_window": { "oneOf": [ { "$ref": "#/definitions/MetricTimeWindow" }, { "type": "null" } ] }, "offset_to_grain": { "oneOf": [ { "type": "string", "enum": [ "day", "week", "month", "quarter", "year" ] }, { "type": "null" } ] } }, "additionalProperties": false, "description": "MetricInput(name: str, filter: Union[dbt.contracts.graph.nodes.WhereFilter, NoneType] = None, alias: Union[str, NoneType] = None, offset_window: Union[dbt.contracts.graph.nodes.MetricTimeWindow, NoneType] = None, offset_to_grain: Union[dbt_semantic_interfaces.type_enums.time_granularity.TimeGranularity, NoneType] = None)" }, "MetricTimeWindow": { "type": "object", "required": [ "count", "granularity" ], "properties": { "count": { "type": "integer" }, "granularity": { "type": "string", "enum": [ "day", "week", "month", "quarter", "year" ] } }, "additionalProperties": false, "description": "MetricTimeWindow(count: int, granularity: dbt_semantic_interfaces.type_enums.time_granularity.TimeGranularity)" }, "SourceFileMetadata": { "type": "object", "required": [ "repo_file_path", "file_slice" ], "properties": { "repo_file_path": { "type": "string" }, "file_slice": { "$ref": "#/definitions/FileSlice" } }, "additionalProperties": false, "description": "Provides file context about what something was created from.\n\n Implementation of the dbt-semantic-interfaces `Metadata` protocol\n " }, "FileSlice": { "type": "object", "required": [ "filename", "content", "start_line_number", "end_line_number" ], "properties": { "filename": { "type": "string" }, "content": { "type": "string" }, "start_line_number": { "type": "integer" }, "end_line_number": { "type": "integer" } }, "additionalProperties": false, "description": "Provides file slice level context about what something was created from.\n\n Implementation of the dbt-semantic-interfaces `FileSlice` protocol\n " }, "MetricConfig": { "type": "object", "required": [], "properties": { "enabled": { "type": "boolean", "default": true }, "group": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": true, "description": "MetricConfig(_extra: Dict[str, Any] = , enabled: bool = True, group: Union[str, NoneType] = None)" }, "Group": { "type": "object", "required": [ "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "owner" ], "properties": { "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "group" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "owner": { "$ref": "#/definitions/Owner" } }, "additionalProperties": false, "description": "Group(name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, owner: dbt.contracts.graph.unparsed.Owner)" }, "SemanticModel": { "type": "object", "required": [ "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "model" ], "properties": { "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "model", "analysis", "test", "snapshot", "operation", "seed", "rpc", "sql_operation", "doc", "source", "macro", "exposure", "metric", "group", "semantic_model" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "model": { "type": "string" }, "node_relation": { "oneOf": [ { "$ref": "#/definitions/NodeRelation" }, { "type": "null" } ] }, "description": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "label": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "defaults": { "oneOf": [ { "$ref": "#/definitions/Defaults" }, { "type": "null" } ] }, "entities": { "type": "array", "items": { "$ref": "#/definitions/Entity" }, "default": [] }, "measures": { "type": "array", "items": { "$ref": "#/definitions/Measure" }, "default": [] }, "dimensions": { "type": "array", "items": { "$ref": "#/definitions/Dimension" }, "default": [] }, "metadata": { "oneOf": [ { "$ref": "#/definitions/SourceFileMetadata" }, { "type": "null" } ] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "refs": { "type": "array", "items": { "$ref": "#/definitions/RefArgs" }, "default": [] }, "created_at": { "type": "number", "default": 1696465994.425479 }, "config": { "$ref": "#/definitions/SemanticModelConfig", "default": { "enabled": true } }, "primary_entity": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "SemanticModel(name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], model: str, node_relation: Union[dbt.contracts.graph.nodes.NodeRelation, NoneType], description: Union[str, NoneType] = None, label: Union[str, NoneType] = None, defaults: Union[dbt.contracts.graph.semantic_models.Defaults, NoneType] = None, entities: Sequence[dbt.contracts.graph.semantic_models.Entity] = , measures: Sequence[dbt.contracts.graph.semantic_models.Measure] = , dimensions: Sequence[dbt.contracts.graph.semantic_models.Dimension] = , metadata: Union[dbt.contracts.graph.semantic_models.SourceFileMetadata, NoneType] = None, depends_on: dbt.contracts.graph.nodes.DependsOn = , refs: List[dbt.contracts.graph.nodes.RefArgs] = , created_at: float = , config: dbt.contracts.graph.model_config.SemanticModelConfig = , primary_entity: Union[str, NoneType] = None)" }, "NodeRelation": { "type": "object", "required": [ "alias", "schema_name" ], "properties": { "alias": { "type": "string" }, "schema_name": { "type": "string" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "NodeRelation(alias: str, schema_name: str, database: Union[str, NoneType] = None, relation_name: Union[str, NoneType] = None)" }, "Defaults": { "type": "object", "required": [], "properties": { "agg_time_dimension": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "Defaults(agg_time_dimension: Union[str, NoneType] = None)" }, "Entity": { "type": "object", "required": [ "name", "type" ], "properties": { "name": { "type": "string" }, "type": { "type": "string", "enum": [ "foreign", "natural", "primary", "unique" ] }, "description": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "label": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "role": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "expr": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "Entity(name: str, type: dbt_semantic_interfaces.type_enums.entity_type.EntityType, description: Union[str, NoneType] = None, label: Union[str, NoneType] = None, role: Union[str, NoneType] = None, expr: Union[str, NoneType] = None)" }, "Measure": { "type": "object", "required": [ "name", "agg" ], "properties": { "name": { "type": "string" }, "agg": { "type": "string", "enum": [ "sum", "min", "max", "count_distinct", "sum_boolean", "average", "percentile", "median", "count" ] }, "description": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "label": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "create_metric": { "type": "boolean", "default": false }, "expr": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "agg_params": { "oneOf": [ { "$ref": "#/definitions/MeasureAggregationParameters" }, { "type": "null" } ] }, "non_additive_dimension": { "oneOf": [ { "$ref": "#/definitions/NonAdditiveDimension" }, { "type": "null" } ] }, "agg_time_dimension": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "Measure(name: str, agg: dbt_semantic_interfaces.type_enums.aggregation_type.AggregationType, description: Union[str, NoneType] = None, label: Union[str, NoneType] = None, create_metric: bool = False, expr: Union[str, NoneType] = None, agg_params: Union[dbt.contracts.graph.semantic_models.MeasureAggregationParameters, NoneType] = None, non_additive_dimension: Union[dbt.contracts.graph.semantic_models.NonAdditiveDimension, NoneType] = None, agg_time_dimension: Union[str, NoneType] = None)" }, "MeasureAggregationParameters": { "type": "object", "required": [], "properties": { "percentile": { "oneOf": [ { "type": "number" }, { "type": "null" } ] }, "use_discrete_percentile": { "type": "boolean", "default": false }, "use_approximate_percentile": { "type": "boolean", "default": false } }, "additionalProperties": false, "description": "MeasureAggregationParameters(percentile: Union[float, NoneType] = None, use_discrete_percentile: bool = False, use_approximate_percentile: bool = False)" }, "NonAdditiveDimension": { "type": "object", "required": [ "name", "window_choice", "window_groupings" ], "properties": { "name": { "type": "string" }, "window_choice": { "type": "string", "enum": [ "sum", "min", "max", "count_distinct", "sum_boolean", "average", "percentile", "median", "count" ] }, "window_groupings": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false, "description": "NonAdditiveDimension(name: str, window_choice: dbt_semantic_interfaces.type_enums.aggregation_type.AggregationType, window_groupings: List[str])" }, "Dimension": { "type": "object", "required": [ "name", "type" ], "properties": { "name": { "type": "string" }, "type": { "type": "string", "enum": [ "categorical", "time" ] }, "description": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "label": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "is_partition": { "type": "boolean", "default": false }, "type_params": { "oneOf": [ { "$ref": "#/definitions/DimensionTypeParams" }, { "type": "null" } ] }, "expr": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "metadata": { "oneOf": [ { "$ref": "#/definitions/SourceFileMetadata" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "Dimension(name: str, type: dbt_semantic_interfaces.type_enums.dimension_type.DimensionType, description: Union[str, NoneType] = None, label: Union[str, NoneType] = None, is_partition: bool = False, type_params: Union[dbt.contracts.graph.semantic_models.DimensionTypeParams, NoneType] = None, expr: Union[str, NoneType] = None, metadata: Union[dbt.contracts.graph.semantic_models.SourceFileMetadata, NoneType] = None)" }, "DimensionTypeParams": { "type": "object", "required": [ "time_granularity" ], "properties": { "time_granularity": { "type": "string", "enum": [ "day", "week", "month", "quarter", "year" ] }, "validity_params": { "oneOf": [ { "$ref": "#/definitions/DimensionValidityParams" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "DimensionTypeParams(time_granularity: dbt_semantic_interfaces.type_enums.time_granularity.TimeGranularity, validity_params: Union[dbt.contracts.graph.semantic_models.DimensionValidityParams, NoneType] = None)" }, "DimensionValidityParams": { "type": "object", "required": [], "properties": { "is_start": { "type": "boolean", "default": false }, "is_end": { "type": "boolean", "default": false } }, "additionalProperties": false, "description": "DimensionValidityParams(is_start: bool = False, is_end: bool = False)" }, "SemanticModelConfig": { "type": "object", "required": [], "properties": { "enabled": { "type": "boolean", "default": true } }, "additionalProperties": true, "description": "SemanticModelConfig(_extra: Dict[str, Any] = , enabled: bool = True)" } }, "$schema": "http://json-schema.org/draft-07/schema#", "$id": "https://schemas.getdbt.com/dbt/manifest/v10.json" } ================================================ FILE: schemas/dbt/manifest/v11.json ================================================ { "$ref": "#/$defs/WritableManifest", "$defs": { "ManifestMetadata": { "type": "object", "title": "ManifestMetadata", "properties": { "dbt_schema_version": { "type": "string" }, "dbt_version": { "type": "string", "default": "1.8.0a1" }, "generated_at": { "type": "string" }, "invocation_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "env": { "type": "object", "additionalProperties": { "type": "string" }, "propertyNames": { "type": "string" } }, "project_name": { "description": "Name of the root project", "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "project_id": { "description": "A unique identifier for the project, hashed from the project name", "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "user_id": { "description": "A unique identifier for the user", "anyOf": [ { "type": "string", "format": "uuid" }, { "type": "null" } ], "default": null }, "send_anonymous_usage_stats": { "description": "Whether dbt is configured to send anonymous usage statistics", "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "adapter_type": { "description": "The type name of the adapter", "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "FileHash": { "type": "object", "title": "FileHash", "properties": { "name": { "type": "string" }, "checksum": { "type": "string" } }, "additionalProperties": false, "required": [ "name", "checksum" ] }, "Hook": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] }, "Docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "ContractConfig": { "type": "object", "title": "ContractConfig", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true } }, "additionalProperties": false }, "NodeConfig": { "type": "object", "title": "NodeConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "tags": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "materialized": { "type": "string", "default": "view" }, "incremental_strategy": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "persist_docs": { "type": "object", "propertyNames": { "type": "string" } }, "post-hook": { "type": "array", "items": { "$ref": "#/$defs/Hook" } }, "pre-hook": { "type": "array", "items": { "$ref": "#/$defs/Hook" } }, "quoting": { "type": "object", "propertyNames": { "type": "string" } }, "column_types": { "type": "object", "propertyNames": { "type": "string" } }, "full_refresh": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "unique_key": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ], "default": null }, "on_schema_change": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "on_configuration_change": { "enum": [ "apply", "continue", "fail" ] }, "grants": { "type": "object", "propertyNames": { "type": "string" } }, "packages": { "type": "array", "items": { "type": "string" } }, "docs": { "$ref": "#/$defs/Docs" }, "contract": { "$ref": "#/$defs/ContractConfig" } }, "additionalProperties": true }, "ColumnLevelConstraint": { "type": "object", "title": "ColumnLevelConstraint", "properties": { "type": { "enum": [ "check", "not_null", "unique", "primary_key", "foreign_key", "custom" ] }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "expression": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "warn_unenforced": { "type": "boolean", "default": true }, "warn_unsupported": { "type": "boolean", "default": true } }, "additionalProperties": false, "required": [ "type" ] }, "ColumnInfo": { "type": "object", "title": "ColumnInfo", "properties": { "name": { "type": "string" }, "description": { "type": "string", "default": "" }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "data_type": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "constraints": { "type": "array", "items": { "$ref": "#/$defs/ColumnLevelConstraint" } }, "quote": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "tags": { "type": "array", "items": { "type": "string" } }, "_extra": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": true, "required": [ "name" ] }, "RefArgs": { "type": "object", "title": "RefArgs", "properties": { "name": { "type": "string" }, "package": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "version": { "anyOf": [ { "type": "string" }, { "type": "number" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] }, "DependsOn": { "type": "object", "title": "DependsOn", "properties": { "macros": { "type": "array", "items": { "type": "string" } }, "nodes": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false }, "InjectedCTE": { "type": "object", "title": "InjectedCTE", "properties": { "id": { "type": "string" }, "sql": { "type": "string" } }, "additionalProperties": false, "required": [ "id", "sql" ] }, "Contract": { "type": "object", "title": "Contract", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true }, "checksum": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "AnalysisNode": { "type": "object", "title": "AnalysisNode", "properties": { "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "const": "analysis" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "$ref": "#/$defs/FileHash" }, "config": { "$ref": "#/$defs/NodeConfig" }, "_event_status": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/$defs/ColumnInfo" }, "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "docs": { "$ref": "#/$defs/Docs" }, "patch_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "build_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "created_at": { "type": "number" }, "config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "raw_code": { "type": "string", "default": "" }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "$ref": "#/$defs/RefArgs" } }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "depends_on": { "$ref": "#/$defs/DependsOn" }, "compiled_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/$defs/InjectedCTE" } }, "_pre_injected_sql": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "contract": { "$ref": "#/$defs/Contract" } }, "additionalProperties": false, "required": [ "database", "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ] }, "TestConfig": { "type": "object", "title": "TestConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": "dbt_test__audit" }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "tags": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "materialized": { "type": "string", "default": "test" }, "severity": { "type": "string", "default": "ERROR", "pattern": "^([Ww][Aa][Rr][Nn]|[Ee][Rr][Rr][Oo][Rr])$" }, "store_failures": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "store_failures_as": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "where": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "limit": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null }, "fail_calc": { "type": "string", "default": "count(*)" }, "warn_if": { "type": "string", "default": "!= 0" }, "error_if": { "type": "string", "default": "!= 0" } }, "additionalProperties": true }, "SingularTestNode": { "type": "object", "title": "SingularTestNode", "properties": { "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "const": "test" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "$ref": "#/$defs/FileHash" }, "config": { "$ref": "#/$defs/TestConfig" }, "_event_status": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/$defs/ColumnInfo" }, "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "docs": { "$ref": "#/$defs/Docs" }, "patch_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "build_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "created_at": { "type": "number" }, "config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "raw_code": { "type": "string", "default": "" }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "$ref": "#/$defs/RefArgs" } }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "depends_on": { "$ref": "#/$defs/DependsOn" }, "compiled_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/$defs/InjectedCTE" } }, "_pre_injected_sql": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "contract": { "$ref": "#/$defs/Contract" } }, "additionalProperties": false, "required": [ "database", "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ] }, "HookNode": { "type": "object", "title": "HookNode", "properties": { "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "const": "operation" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "$ref": "#/$defs/FileHash" }, "config": { "$ref": "#/$defs/NodeConfig" }, "_event_status": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/$defs/ColumnInfo" }, "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "docs": { "$ref": "#/$defs/Docs" }, "patch_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "build_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "created_at": { "type": "number" }, "config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "raw_code": { "type": "string", "default": "" }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "$ref": "#/$defs/RefArgs" } }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "depends_on": { "$ref": "#/$defs/DependsOn" }, "compiled_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/$defs/InjectedCTE" } }, "_pre_injected_sql": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "contract": { "$ref": "#/$defs/Contract" }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "database", "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ] }, "ModelConfig": { "type": "object", "title": "ModelConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "tags": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "materialized": { "type": "string", "default": "view" }, "incremental_strategy": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "persist_docs": { "type": "object", "propertyNames": { "type": "string" } }, "post-hook": { "type": "array", "items": { "$ref": "#/$defs/Hook" } }, "pre-hook": { "type": "array", "items": { "$ref": "#/$defs/Hook" } }, "quoting": { "type": "object", "propertyNames": { "type": "string" } }, "column_types": { "type": "object", "propertyNames": { "type": "string" } }, "full_refresh": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "unique_key": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ], "default": null }, "on_schema_change": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "on_configuration_change": { "enum": [ "apply", "continue", "fail" ] }, "grants": { "type": "object", "propertyNames": { "type": "string" } }, "packages": { "type": "array", "items": { "type": "string" } }, "docs": { "$ref": "#/$defs/Docs" }, "contract": { "$ref": "#/$defs/ContractConfig" }, "access": { "enum": [ "private", "protected", "public" ], "default": "protected" } }, "additionalProperties": true }, "ModelLevelConstraint": { "type": "object", "title": "ModelLevelConstraint", "properties": { "type": { "enum": [ "check", "not_null", "unique", "primary_key", "foreign_key", "custom" ] }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "expression": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "warn_unenforced": { "type": "boolean", "default": true }, "warn_unsupported": { "type": "boolean", "default": true }, "columns": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false, "required": [ "type" ] }, "DeferRelation": { "type": "object", "title": "DeferRelation", "properties": { "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "alias": { "type": "string" }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "required": [ "database", "schema", "alias", "relation_name" ] }, "ModelNode": { "type": "object", "title": "ModelNode", "properties": { "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "const": "model" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "$ref": "#/$defs/FileHash" }, "config": { "$ref": "#/$defs/ModelConfig" }, "_event_status": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/$defs/ColumnInfo" }, "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "docs": { "$ref": "#/$defs/Docs" }, "patch_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "build_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "created_at": { "type": "number" }, "config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "raw_code": { "type": "string", "default": "" }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "$ref": "#/$defs/RefArgs" } }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "depends_on": { "$ref": "#/$defs/DependsOn" }, "compiled_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/$defs/InjectedCTE" } }, "_pre_injected_sql": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "contract": { "$ref": "#/$defs/Contract" }, "access": { "enum": [ "private", "protected", "public" ], "default": "protected" }, "constraints": { "type": "array", "items": { "$ref": "#/$defs/ModelLevelConstraint" } }, "version": { "anyOf": [ { "type": "string" }, { "type": "number" }, { "type": "null" } ], "default": null }, "latest_version": { "anyOf": [ { "type": "string" }, { "type": "number" }, { "type": "null" } ], "default": null }, "deprecation_date": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "defer_relation": { "anyOf": [ { "$ref": "#/$defs/DeferRelation" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "database", "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ] }, "RPCNode": { "type": "object", "title": "RPCNode", "properties": { "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "const": "rpc" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "$ref": "#/$defs/FileHash" }, "config": { "$ref": "#/$defs/NodeConfig" }, "_event_status": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/$defs/ColumnInfo" }, "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "docs": { "$ref": "#/$defs/Docs" }, "patch_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "build_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "created_at": { "type": "number" }, "config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "raw_code": { "type": "string", "default": "" }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "$ref": "#/$defs/RefArgs" } }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "depends_on": { "$ref": "#/$defs/DependsOn" }, "compiled_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/$defs/InjectedCTE" } }, "_pre_injected_sql": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "contract": { "$ref": "#/$defs/Contract" } }, "additionalProperties": false, "required": [ "database", "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ] }, "SqlNode": { "type": "object", "title": "SqlNode", "properties": { "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "const": "sql_operation" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "$ref": "#/$defs/FileHash" }, "config": { "$ref": "#/$defs/NodeConfig" }, "_event_status": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/$defs/ColumnInfo" }, "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "docs": { "$ref": "#/$defs/Docs" }, "patch_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "build_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "created_at": { "type": "number" }, "config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "raw_code": { "type": "string", "default": "" }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "$ref": "#/$defs/RefArgs" } }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "depends_on": { "$ref": "#/$defs/DependsOn" }, "compiled_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/$defs/InjectedCTE" } }, "_pre_injected_sql": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "contract": { "$ref": "#/$defs/Contract" } }, "additionalProperties": false, "required": [ "database", "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ] }, "TestMetadata": { "type": "object", "title": "TestMetadata", "properties": { "name": { "type": "string" }, "kwargs": { "type": "object", "propertyNames": { "type": "string" } }, "namespace": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] }, "GenericTestNode": { "type": "object", "title": "GenericTestNode", "properties": { "test_metadata": { "$ref": "#/$defs/TestMetadata" }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "const": "test" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "$ref": "#/$defs/FileHash" }, "config": { "$ref": "#/$defs/TestConfig" }, "_event_status": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/$defs/ColumnInfo" }, "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "docs": { "$ref": "#/$defs/Docs" }, "patch_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "build_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "created_at": { "type": "number" }, "config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "raw_code": { "type": "string", "default": "" }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "$ref": "#/$defs/RefArgs" } }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "depends_on": { "$ref": "#/$defs/DependsOn" }, "compiled_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/$defs/InjectedCTE" } }, "_pre_injected_sql": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "contract": { "$ref": "#/$defs/Contract" }, "column_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "file_key_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "attached_node": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "test_metadata", "database", "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ] }, "SnapshotConfig": { "type": "object", "title": "SnapshotConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "tags": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "materialized": { "type": "string", "default": "snapshot" }, "incremental_strategy": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "persist_docs": { "type": "object", "propertyNames": { "type": "string" } }, "post-hook": { "type": "array", "items": { "$ref": "#/$defs/Hook" } }, "pre-hook": { "type": "array", "items": { "$ref": "#/$defs/Hook" } }, "quoting": { "type": "object", "propertyNames": { "type": "string" } }, "column_types": { "type": "object", "propertyNames": { "type": "string" } }, "full_refresh": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "unique_key": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "on_schema_change": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "on_configuration_change": { "enum": [ "apply", "continue", "fail" ] }, "grants": { "type": "object", "propertyNames": { "type": "string" } }, "packages": { "type": "array", "items": { "type": "string" } }, "docs": { "$ref": "#/$defs/Docs" }, "contract": { "$ref": "#/$defs/ContractConfig" }, "strategy": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "target_schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "target_database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "updated_at": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "check_cols": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ], "default": null } }, "additionalProperties": true }, "SnapshotNode": { "type": "object", "title": "SnapshotNode", "properties": { "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "const": "snapshot" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "$ref": "#/$defs/FileHash" }, "config": { "$ref": "#/$defs/SnapshotConfig" }, "_event_status": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/$defs/ColumnInfo" }, "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "docs": { "$ref": "#/$defs/Docs" }, "patch_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "build_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "created_at": { "type": "number" }, "config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "raw_code": { "type": "string", "default": "" }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "$ref": "#/$defs/RefArgs" } }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "depends_on": { "$ref": "#/$defs/DependsOn" }, "compiled_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/$defs/InjectedCTE" } }, "_pre_injected_sql": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "contract": { "$ref": "#/$defs/Contract" }, "defer_relation": { "anyOf": [ { "$ref": "#/$defs/DeferRelation" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "database", "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum", "config" ] }, "UnitTestNodeConfig": { "type": "object", "title": "UnitTestNodeConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "tags": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "materialized": { "type": "string", "default": "view" }, "incremental_strategy": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "persist_docs": { "type": "object", "propertyNames": { "type": "string" } }, "post-hook": { "type": "array", "items": { "$ref": "#/$defs/Hook" } }, "pre-hook": { "type": "array", "items": { "$ref": "#/$defs/Hook" } }, "quoting": { "type": "object", "propertyNames": { "type": "string" } }, "column_types": { "type": "object", "propertyNames": { "type": "string" } }, "full_refresh": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "unique_key": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ], "default": null }, "on_schema_change": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "on_configuration_change": { "enum": [ "apply", "continue", "fail" ] }, "grants": { "type": "object", "propertyNames": { "type": "string" } }, "packages": { "type": "array", "items": { "type": "string" } }, "docs": { "$ref": "#/$defs/Docs" }, "contract": { "$ref": "#/$defs/ContractConfig" }, "expected_rows": { "type": "array", "items": { "type": "object", "propertyNames": { "type": "string" } } } }, "additionalProperties": true }, "UnitTestOverrides": { "type": "object", "title": "UnitTestOverrides", "properties": { "macros": { "type": "object", "propertyNames": { "type": "string" } }, "vars": { "type": "object", "propertyNames": { "type": "string" } }, "env_vars": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": false }, "UnitTestNode": { "type": "object", "title": "UnitTestNode", "properties": { "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "const": "unit_test" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "$ref": "#/$defs/FileHash" }, "config": { "$ref": "#/$defs/UnitTestNodeConfig" }, "_event_status": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/$defs/ColumnInfo" }, "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "docs": { "$ref": "#/$defs/Docs" }, "patch_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "build_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "created_at": { "type": "number" }, "config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "raw_code": { "type": "string", "default": "" }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "$ref": "#/$defs/RefArgs" } }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "depends_on": { "$ref": "#/$defs/DependsOn" }, "compiled_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/$defs/InjectedCTE" } }, "_pre_injected_sql": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "contract": { "$ref": "#/$defs/Contract" }, "tested_node_unique_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "this_input_node_unique_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "overrides": { "anyOf": [ { "$ref": "#/$defs/UnitTestOverrides" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "database", "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ] }, "SeedConfig": { "type": "object", "title": "SeedConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "tags": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "materialized": { "type": "string", "default": "seed" }, "incremental_strategy": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "persist_docs": { "type": "object", "propertyNames": { "type": "string" } }, "post-hook": { "type": "array", "items": { "$ref": "#/$defs/Hook" } }, "pre-hook": { "type": "array", "items": { "$ref": "#/$defs/Hook" } }, "quoting": { "type": "object", "propertyNames": { "type": "string" } }, "column_types": { "type": "object", "propertyNames": { "type": "string" } }, "full_refresh": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "unique_key": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ], "default": null }, "on_schema_change": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "on_configuration_change": { "enum": [ "apply", "continue", "fail" ] }, "grants": { "type": "object", "propertyNames": { "type": "string" } }, "packages": { "type": "array", "items": { "type": "string" } }, "docs": { "$ref": "#/$defs/Docs" }, "contract": { "$ref": "#/$defs/ContractConfig" }, "delimiter": { "type": "string", "default": "," }, "quote_columns": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null } }, "additionalProperties": true }, "MacroDependsOn": { "type": "object", "title": "MacroDependsOn", "properties": { "macros": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false }, "SeedNode": { "type": "object", "title": "SeedNode", "properties": { "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "const": "seed" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "$ref": "#/$defs/FileHash" }, "config": { "$ref": "#/$defs/SeedConfig" }, "_event_status": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/$defs/ColumnInfo" }, "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "docs": { "$ref": "#/$defs/Docs" }, "patch_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "build_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "created_at": { "type": "number" }, "config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "raw_code": { "type": "string", "default": "" }, "root_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "depends_on": { "$ref": "#/$defs/MacroDependsOn" }, "defer_relation": { "anyOf": [ { "$ref": "#/$defs/DeferRelation" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "database", "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ] }, "Quoting": { "type": "object", "title": "Quoting", "properties": { "database": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "identifier": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "column": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "Time": { "type": "object", "title": "Time", "properties": { "count": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null }, "period": { "anyOf": [ { "enum": [ "minute", "hour", "day" ] }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "FreshnessThreshold": { "type": "object", "title": "FreshnessThreshold", "properties": { "warn_after": { "anyOf": [ { "$ref": "#/$defs/Time" }, { "type": "null" } ] }, "error_after": { "anyOf": [ { "$ref": "#/$defs/Time" }, { "type": "null" } ] }, "filter": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "ExternalPartition": { "type": "object", "title": "ExternalPartition", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "name": { "type": "string", "default": "" }, "description": { "type": "string", "default": "" }, "data_type": { "type": "string", "default": "" }, "meta": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": true }, "ExternalTable": { "type": "object", "title": "ExternalTable", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "location": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "file_format": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "row_format": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "tbl_properties": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "partitions": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "array", "items": { "$ref": "#/$defs/ExternalPartition" } }, { "type": "null" } ], "default": null } }, "additionalProperties": true }, "SourceConfig": { "type": "object", "title": "SourceConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true } }, "additionalProperties": true }, "SourceDefinition": { "type": "object", "title": "SourceDefinition", "properties": { "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "const": "source" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "source_name": { "type": "string" }, "source_description": { "type": "string" }, "loader": { "type": "string" }, "identifier": { "type": "string" }, "_event_status": { "type": "object", "propertyNames": { "type": "string" } }, "quoting": { "$ref": "#/$defs/Quoting" }, "loaded_at_field": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "freshness": { "anyOf": [ { "$ref": "#/$defs/FreshnessThreshold" }, { "type": "null" } ], "default": null }, "external": { "anyOf": [ { "$ref": "#/$defs/ExternalTable" }, { "type": "null" } ], "default": null }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/$defs/ColumnInfo" }, "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "source_meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } }, "config": { "$ref": "#/$defs/SourceConfig" }, "patch_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "created_at": { "type": "number" } }, "additionalProperties": false, "required": [ "database", "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "source_name", "source_description", "loader", "identifier" ] }, "MacroArgument": { "type": "object", "title": "MacroArgument", "properties": { "name": { "type": "string" }, "type": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "description": { "type": "string", "default": "" } }, "additionalProperties": false, "required": [ "name" ] }, "Macro": { "type": "object", "title": "Macro", "properties": { "name": { "type": "string" }, "resource_type": { "const": "macro" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "macro_sql": { "type": "string" }, "depends_on": { "$ref": "#/$defs/MacroDependsOn" }, "description": { "type": "string", "default": "" }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "docs": { "$ref": "#/$defs/Docs" }, "patch_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "arguments": { "type": "array", "items": { "$ref": "#/$defs/MacroArgument" } }, "created_at": { "type": "number" }, "supported_languages": { "anyOf": [ { "type": "array", "items": { "enum": [ "python", "sql" ] } }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "macro_sql" ] }, "Documentation": { "type": "object", "title": "Documentation", "properties": { "name": { "type": "string" }, "resource_type": { "const": "doc" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "block_contents": { "type": "string" } }, "additionalProperties": false, "required": [ "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "block_contents" ] }, "Owner": { "type": "object", "title": "Owner", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "email": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": true }, "ExposureConfig": { "type": "object", "title": "ExposureConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true } }, "additionalProperties": true }, "Exposure": { "type": "object", "title": "Exposure", "properties": { "name": { "type": "string" }, "resource_type": { "const": "exposure" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "type": { "enum": [ "dashboard", "notebook", "analysis", "ml", "application" ] }, "owner": { "$ref": "#/$defs/Owner" }, "description": { "type": "string", "default": "" }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "maturity": { "anyOf": [ { "enum": [ "low", "medium", "high" ] }, { "type": "null" } ], "default": null }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } }, "config": { "$ref": "#/$defs/ExposureConfig" }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "url": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "depends_on": { "$ref": "#/$defs/DependsOn" }, "refs": { "type": "array", "items": { "$ref": "#/$defs/RefArgs" } }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "created_at": { "type": "number" } }, "additionalProperties": false, "required": [ "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "type", "owner" ] }, "WhereFilter": { "type": "object", "title": "WhereFilter", "properties": { "where_sql_template": { "type": "string" } }, "additionalProperties": false, "required": [ "where_sql_template" ] }, "WhereFilterIntersection": { "type": "object", "title": "WhereFilterIntersection", "properties": { "where_filters": { "type": "array", "items": { "$ref": "#/$defs/WhereFilter" } } }, "additionalProperties": false, "required": [ "where_filters" ] }, "MetricInputMeasure": { "type": "object", "title": "MetricInputMeasure", "properties": { "name": { "type": "string" }, "filter": { "anyOf": [ { "$ref": "#/$defs/WhereFilterIntersection" }, { "type": "null" } ], "default": null }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "join_to_timespine": { "type": "boolean", "default": false }, "fill_nulls_with": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] }, "MetricTimeWindow": { "type": "object", "title": "MetricTimeWindow", "properties": { "count": { "type": "integer" }, "granularity": { "enum": [ "day", "week", "month", "quarter", "year" ] } }, "additionalProperties": false, "required": [ "count", "granularity" ] }, "MetricInput": { "type": "object", "title": "MetricInput", "properties": { "name": { "type": "string" }, "filter": { "anyOf": [ { "$ref": "#/$defs/WhereFilterIntersection" }, { "type": "null" } ], "default": null }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "offset_window": { "anyOf": [ { "$ref": "#/$defs/MetricTimeWindow" }, { "type": "null" } ], "default": null }, "offset_to_grain": { "anyOf": [ { "enum": [ "day", "week", "month", "quarter", "year" ] }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] }, "MetricTypeParams": { "type": "object", "title": "MetricTypeParams", "properties": { "measure": { "anyOf": [ { "$ref": "#/$defs/MetricInputMeasure" }, { "type": "null" } ], "default": null }, "input_measures": { "type": "array", "items": { "$ref": "#/$defs/MetricInputMeasure" } }, "numerator": { "anyOf": [ { "$ref": "#/$defs/MetricInput" }, { "type": "null" } ], "default": null }, "denominator": { "anyOf": [ { "$ref": "#/$defs/MetricInput" }, { "type": "null" } ], "default": null }, "expr": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "window": { "anyOf": [ { "$ref": "#/$defs/MetricTimeWindow" }, { "type": "null" } ], "default": null }, "grain_to_date": { "anyOf": [ { "enum": [ "day", "week", "month", "quarter", "year" ] }, { "type": "null" } ], "default": null }, "metrics": { "anyOf": [ { "type": "array", "items": { "$ref": "#/$defs/MetricInput" } }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "FileSlice": { "type": "object", "title": "FileSlice", "properties": { "filename": { "type": "string" }, "content": { "type": "string" }, "start_line_number": { "type": "integer" }, "end_line_number": { "type": "integer" } }, "additionalProperties": false, "required": [ "filename", "content", "start_line_number", "end_line_number" ] }, "SourceFileMetadata": { "type": "object", "title": "SourceFileMetadata", "properties": { "repo_file_path": { "type": "string" }, "file_slice": { "$ref": "#/$defs/FileSlice" } }, "additionalProperties": false, "required": [ "repo_file_path", "file_slice" ] }, "MetricConfig": { "type": "object", "title": "MetricConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": true }, "Metric": { "type": "object", "title": "Metric", "properties": { "name": { "type": "string" }, "resource_type": { "const": "metric" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "description": { "type": "string" }, "label": { "type": "string" }, "type": { "enum": [ "simple", "ratio", "cumulative", "derived" ] }, "type_params": { "$ref": "#/$defs/MetricTypeParams" }, "filter": { "anyOf": [ { "$ref": "#/$defs/WhereFilterIntersection" }, { "type": "null" } ], "default": null }, "metadata": { "anyOf": [ { "$ref": "#/$defs/SourceFileMetadata" }, { "type": "null" } ], "default": null }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } }, "config": { "$ref": "#/$defs/MetricConfig" }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "depends_on": { "$ref": "#/$defs/DependsOn" }, "refs": { "type": "array", "items": { "$ref": "#/$defs/RefArgs" } }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "created_at": { "type": "number" }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "description", "label", "type", "type_params" ] }, "Group": { "type": "object", "title": "Group", "properties": { "name": { "type": "string" }, "resource_type": { "const": "group" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "owner": { "$ref": "#/$defs/Owner" } }, "additionalProperties": false, "required": [ "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "owner" ] }, "QueryParams": { "type": "object", "title": "QueryParams", "properties": { "metrics": { "type": "array", "items": { "type": "string" } }, "group_by": { "type": "array", "items": { "type": "string" } }, "where": { "anyOf": [ { "$ref": "#/$defs/WhereFilterIntersection" }, { "type": "null" } ] } }, "additionalProperties": false, "required": [ "metrics", "group_by", "where" ] }, "ExportConfig": { "type": "object", "title": "ExportConfig", "properties": { "export_as": { "enum": [ "table", "view" ] }, "schema_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "export_as" ] }, "Export": { "type": "object", "title": "Export", "properties": { "name": { "type": "string" }, "config": { "$ref": "#/$defs/ExportConfig" } }, "additionalProperties": false, "required": [ "name", "config" ] }, "SavedQueryConfig": { "type": "object", "title": "SavedQueryConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "export_as": { "anyOf": [ { "enum": [ "table", "view" ] }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": true }, "SavedQuery": { "type": "object", "title": "SavedQuery", "properties": { "name": { "type": "string" }, "resource_type": { "enum": [ "model", "analysis", "test", "snapshot", "operation", "seed", "rpc", "sql_operation", "doc", "source", "macro", "exposure", "metric", "group", "saved_query", "semantic_model", "unit_test" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "query_params": { "$ref": "#/$defs/QueryParams" }, "exports": { "type": "array", "items": { "$ref": "#/$defs/Export" } }, "_event_status": { "type": "object", "propertyNames": { "type": "string" } }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "metadata": { "anyOf": [ { "$ref": "#/$defs/SourceFileMetadata" }, { "type": "null" } ], "default": null }, "config": { "$ref": "#/$defs/SavedQueryConfig" }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "depends_on": { "$ref": "#/$defs/DependsOn" }, "created_at": { "type": "number" }, "refs": { "type": "array", "items": { "$ref": "#/$defs/RefArgs" } } }, "additionalProperties": false, "required": [ "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "query_params", "exports" ] }, "NodeRelation": { "type": "object", "title": "NodeRelation", "properties": { "alias": { "type": "string" }, "schema_name": { "type": "string" }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "alias", "schema_name" ] }, "Defaults": { "type": "object", "title": "Defaults", "properties": { "agg_time_dimension": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "Entity": { "type": "object", "title": "Entity", "properties": { "name": { "type": "string" }, "type": { "enum": [ "foreign", "natural", "primary", "unique" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "role": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "expr": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "type" ] }, "MeasureAggregationParameters": { "type": "object", "title": "MeasureAggregationParameters", "properties": { "percentile": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "default": null }, "use_discrete_percentile": { "type": "boolean", "default": false }, "use_approximate_percentile": { "type": "boolean", "default": false } }, "additionalProperties": false }, "NonAdditiveDimension": { "type": "object", "title": "NonAdditiveDimension", "properties": { "name": { "type": "string" }, "window_choice": { "enum": [ "sum", "min", "max", "count_distinct", "sum_boolean", "average", "percentile", "median", "count" ] }, "window_groupings": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false, "required": [ "name", "window_choice", "window_groupings" ] }, "Measure": { "type": "object", "title": "Measure", "properties": { "name": { "type": "string" }, "agg": { "enum": [ "sum", "min", "max", "count_distinct", "sum_boolean", "average", "percentile", "median", "count" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "create_metric": { "type": "boolean", "default": false }, "expr": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "agg_params": { "anyOf": [ { "$ref": "#/$defs/MeasureAggregationParameters" }, { "type": "null" } ], "default": null }, "non_additive_dimension": { "anyOf": [ { "$ref": "#/$defs/NonAdditiveDimension" }, { "type": "null" } ], "default": null }, "agg_time_dimension": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "agg" ] }, "DimensionValidityParams": { "type": "object", "title": "DimensionValidityParams", "properties": { "is_start": { "type": "boolean", "default": false }, "is_end": { "type": "boolean", "default": false } }, "additionalProperties": false }, "DimensionTypeParams": { "type": "object", "title": "DimensionTypeParams", "properties": { "time_granularity": { "enum": [ "day", "week", "month", "quarter", "year" ] }, "validity_params": { "anyOf": [ { "$ref": "#/$defs/DimensionValidityParams" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "time_granularity" ] }, "Dimension": { "type": "object", "title": "Dimension", "properties": { "name": { "type": "string" }, "type": { "enum": [ "categorical", "time" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "is_partition": { "type": "boolean", "default": false }, "type_params": { "anyOf": [ { "$ref": "#/$defs/DimensionTypeParams" }, { "type": "null" } ], "default": null }, "expr": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "metadata": { "anyOf": [ { "$ref": "#/$defs/SourceFileMetadata" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "type" ] }, "SemanticModelConfig": { "type": "object", "title": "SemanticModelConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "meta": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": true }, "SemanticModel": { "type": "object", "title": "SemanticModel", "properties": { "name": { "type": "string" }, "resource_type": { "enum": [ "model", "analysis", "test", "snapshot", "operation", "seed", "rpc", "sql_operation", "doc", "source", "macro", "exposure", "metric", "group", "saved_query", "semantic_model", "unit_test" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "model": { "type": "string" }, "node_relation": { "anyOf": [ { "$ref": "#/$defs/NodeRelation" }, { "type": "null" } ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "defaults": { "anyOf": [ { "$ref": "#/$defs/Defaults" }, { "type": "null" } ], "default": null }, "entities": { "type": "array", "items": { "$ref": "#/$defs/Entity" } }, "measures": { "type": "array", "items": { "$ref": "#/$defs/Measure" } }, "dimensions": { "type": "array", "items": { "$ref": "#/$defs/Dimension" } }, "metadata": { "anyOf": [ { "$ref": "#/$defs/SourceFileMetadata" }, { "type": "null" } ], "default": null }, "depends_on": { "$ref": "#/$defs/DependsOn" }, "refs": { "type": "array", "items": { "$ref": "#/$defs/RefArgs" } }, "created_at": { "type": "number" }, "config": { "$ref": "#/$defs/SemanticModelConfig" }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "primary_entity": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "model", "node_relation" ] }, "UnitTestInputFixture": { "type": "object", "title": "UnitTestInputFixture", "properties": { "input": { "type": "string" }, "rows": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "object", "propertyNames": { "type": "string" } } }, { "type": "null" } ], "default": null }, "format": { "enum": [ "csv", "dict" ], "default": "dict" }, "fixture": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "input" ] }, "UnitTestOutputFixture": { "type": "object", "title": "UnitTestOutputFixture", "properties": { "rows": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "object", "propertyNames": { "type": "string" } } }, { "type": "null" } ], "default": null }, "format": { "enum": [ "csv", "dict" ], "default": "dict" }, "fixture": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "UnitTestConfig": { "type": "object", "title": "UnitTestConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": true }, "UnitTestDefinition": { "type": "object", "title": "UnitTestDefinition", "properties": { "model": { "type": "string" }, "given": { "type": "array", "items": { "$ref": "#/$defs/UnitTestInputFixture" } }, "expect": { "$ref": "#/$defs/UnitTestOutputFixture" }, "name": { "type": "string" }, "resource_type": { "enum": [ "model", "analysis", "test", "snapshot", "operation", "seed", "rpc", "sql_operation", "doc", "source", "macro", "exposure", "metric", "group", "saved_query", "semantic_model", "unit_test" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "_event_status": { "type": "object", "propertyNames": { "type": "string" } }, "description": { "type": "string", "default": "" }, "overrides": { "anyOf": [ { "$ref": "#/$defs/UnitTestOverrides" }, { "type": "null" } ], "default": null }, "depends_on": { "$ref": "#/$defs/DependsOn" }, "config": { "$ref": "#/$defs/UnitTestConfig" }, "checksum": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "model", "given", "expect", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn" ] }, "WritableManifest": { "type": "object", "title": "WritableManifest", "properties": { "metadata": { "description": "Metadata about the manifest", "$ref": "#/$defs/ManifestMetadata" }, "nodes": { "type": "object", "description": "The nodes defined in the dbt project and its dependencies", "additionalProperties": { "anyOf": [ { "$ref": "#/$defs/AnalysisNode" }, { "$ref": "#/$defs/SingularTestNode" }, { "$ref": "#/$defs/HookNode" }, { "$ref": "#/$defs/ModelNode" }, { "$ref": "#/$defs/RPCNode" }, { "$ref": "#/$defs/SqlNode" }, { "$ref": "#/$defs/GenericTestNode" }, { "$ref": "#/$defs/SnapshotNode" }, { "$ref": "#/$defs/UnitTestNode" }, { "$ref": "#/$defs/SeedNode" } ] }, "propertyNames": { "type": "string" } }, "sources": { "type": "object", "description": "The sources defined in the dbt project and its dependencies", "additionalProperties": { "$ref": "#/$defs/SourceDefinition" }, "propertyNames": { "type": "string" } }, "macros": { "type": "object", "description": "The macros defined in the dbt project and its dependencies", "additionalProperties": { "$ref": "#/$defs/Macro" }, "propertyNames": { "type": "string" } }, "docs": { "type": "object", "description": "The docs defined in the dbt project and its dependencies", "additionalProperties": { "$ref": "#/$defs/Documentation" }, "propertyNames": { "type": "string" } }, "exposures": { "type": "object", "description": "The exposures defined in the dbt project and its dependencies", "additionalProperties": { "$ref": "#/$defs/Exposure" }, "propertyNames": { "type": "string" } }, "metrics": { "type": "object", "description": "The metrics defined in the dbt project and its dependencies", "additionalProperties": { "$ref": "#/$defs/Metric" }, "propertyNames": { "type": "string" } }, "groups": { "type": "object", "description": "The groups defined in the dbt project", "additionalProperties": { "$ref": "#/$defs/Group" }, "propertyNames": { "type": "string" } }, "selectors": { "type": "object", "description": "The selectors defined in selectors.yml", "propertyNames": { "type": "string" } }, "disabled": { "description": "A mapping of the disabled nodes in the target", "anyOf": [ { "type": "object", "additionalProperties": { "type": "array", "items": { "anyOf": [ { "$ref": "#/$defs/AnalysisNode" }, { "$ref": "#/$defs/SingularTestNode" }, { "$ref": "#/$defs/HookNode" }, { "$ref": "#/$defs/ModelNode" }, { "$ref": "#/$defs/RPCNode" }, { "$ref": "#/$defs/SqlNode" }, { "$ref": "#/$defs/GenericTestNode" }, { "$ref": "#/$defs/SnapshotNode" }, { "$ref": "#/$defs/UnitTestNode" }, { "$ref": "#/$defs/SeedNode" }, { "$ref": "#/$defs/SourceDefinition" }, { "$ref": "#/$defs/Exposure" }, { "$ref": "#/$defs/Metric" }, { "$ref": "#/$defs/SavedQuery" }, { "$ref": "#/$defs/SemanticModel" }, { "$ref": "#/$defs/UnitTestDefinition" } ] } }, "propertyNames": { "type": "string" } }, { "type": "null" } ] }, "parent_map": { "description": "A mapping from\u00a0child nodes to their dependencies", "anyOf": [ { "type": "object", "additionalProperties": { "type": "array", "items": { "type": "string" } }, "propertyNames": { "type": "string" } }, { "type": "null" } ] }, "child_map": { "description": "A mapping from parent nodes to their dependents", "anyOf": [ { "type": "object", "additionalProperties": { "type": "array", "items": { "type": "string" } }, "propertyNames": { "type": "string" } }, { "type": "null" } ] }, "group_map": { "description": "A mapping from group names to their nodes", "anyOf": [ { "type": "object", "additionalProperties": { "type": "array", "items": { "type": "string" } }, "propertyNames": { "type": "string" } }, { "type": "null" } ] }, "saved_queries": { "type": "object", "description": "The saved queries defined in the dbt project", "additionalProperties": { "$ref": "#/$defs/SavedQuery" }, "propertyNames": { "type": "string" } }, "semantic_models": { "type": "object", "description": "The semantic models defined in the dbt project", "additionalProperties": { "$ref": "#/$defs/SemanticModel" }, "propertyNames": { "type": "string" } }, "unit_tests": { "type": "object", "description": "The unit tests defined in the project", "additionalProperties": { "$ref": "#/$defs/UnitTestDefinition" }, "propertyNames": { "type": "string" } } }, "additionalProperties": false, "required": [ "metadata", "nodes", "sources", "macros", "docs", "exposures", "metrics", "groups", "selectors", "disabled", "parent_map", "child_map", "group_map", "saved_queries", "semantic_models", "unit_tests" ] } }, "$id": "https://schemas.getdbt.com/dbt/manifest/v11.json" } ================================================ FILE: schemas/dbt/manifest/v12.json ================================================ { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "title": "WritableManifest", "properties": { "metadata": { "type": "object", "title": "ManifestMetadata", "description": "Metadata about the manifest", "properties": { "dbt_schema_version": { "type": "string" }, "dbt_version": { "type": "string", "default": "1.12.0a1" }, "generated_at": { "type": "string" }, "invocation_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "invocation_started_at": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "env": { "type": "object", "additionalProperties": { "type": "string" }, "propertyNames": { "type": "string" } }, "project_name": { "description": "Name of the root project", "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "project_id": { "description": "A unique identifier for the project, hashed from the project name", "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "user_id": { "description": "A unique identifier for the user", "anyOf": [ { "type": "string", "format": "uuid" }, { "type": "null" } ], "default": null }, "send_anonymous_usage_stats": { "description": "Whether dbt is configured to send anonymous usage statistics", "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "adapter_type": { "description": "The type name of the adapter", "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "quoting": { "description": "The quoting configuration for the project", "anyOf": [ { "type": "object", "title": "Quoting", "properties": { "database": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "identifier": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "column": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, { "type": "null" } ] }, "run_started_at": { "description": "The timestamp when the run started", "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "nodes": { "type": "object", "description": "The nodes defined in the dbt project and its dependencies", "additionalProperties": { "anyOf": [ { "type": "object", "title": "Seed", "properties": { "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "const": "seed" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "type": "object", "title": "FileHash", "properties": { "name": { "type": "string" }, "checksum": { "type": "string" } }, "additionalProperties": false, "required": [ "name", "checksum" ] }, "config": { "type": "object", "title": "SeedConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "tags": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "materialized": { "type": "string", "default": "seed" }, "incremental_strategy": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "batch_size": { "default": null }, "lookback": { "default": 1 }, "begin": { "default": null }, "persist_docs": { "type": "object", "propertyNames": { "type": "string" } }, "post-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "pre-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "quoting": { "type": "object", "propertyNames": { "type": "string" } }, "column_types": { "type": "object", "propertyNames": { "type": "string" } }, "full_refresh": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "unique_key": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ], "default": null }, "on_schema_change": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "on_configuration_change": { "enum": [ "apply", "continue", "fail" ] }, "grants": { "type": "object", "propertyNames": { "type": "string" } }, "packages": { "type": "array", "items": { "type": "string" } }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "contract": { "type": "object", "title": "ContractConfig", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true } }, "additionalProperties": false }, "event_time": { "default": null }, "concurrent_batches": { "default": null }, "delimiter": { "type": "string", "default": "," }, "quote_columns": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null } }, "additionalProperties": true }, "tags": { "type": "array", "items": { "type": "string" } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "type": "object", "title": "ColumnInfo", "properties": { "name": { "type": "string" }, "description": { "type": "string", "default": "" }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "data_type": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "constraints": { "type": "array", "items": { "type": "object", "title": "ColumnLevelConstraint", "properties": { "type": { "enum": [ "check", "not_null", "unique", "primary_key", "foreign_key", "custom" ] }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "expression": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "warn_unenforced": { "type": "boolean", "default": true }, "warn_unsupported": { "type": "boolean", "default": true }, "to": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "to_columns": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false, "required": [ "type" ] } }, "quote": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "title": "ColumnConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true }, "tags": { "type": "array", "items": { "type": "string" } }, "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "granularity": { "anyOf": [ { "enum": [ "nanosecond", "microsecond", "millisecond", "second", "minute", "hour", "day", "week", "month", "quarter", "year" ] }, { "type": "null" } ], "default": null }, "dimension": { "anyOf": [ { "type": "object", "title": "ColumnDimension", "properties": { "name": { "type": "string" }, "type": { "enum": [ "categorical", "time" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "is_partition": { "type": "boolean", "default": false }, "config": { "type": "object", "propertyNames": { "type": "string" } }, "validity_params": { "anyOf": [ { "type": "object", "title": "ColumnDimensionValidityParams", "properties": { "is_start": { "type": "boolean", "default": false }, "is_end": { "type": "boolean", "default": false } }, "additionalProperties": false }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "categorical", "time" ] }, { "type": "null" } ], "default": null }, "entity": { "anyOf": [ { "type": "object", "title": "ColumnEntity", "properties": { "name": { "type": "string" }, "type": { "enum": [ "foreign", "natural", "primary", "unique" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "foreign", "natural", "primary", "unique" ] }, { "type": "null" } ], "default": null }, "doc_blocks": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true, "required": [ "name" ] }, "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "patch_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "build_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "created_at": { "type": "number" }, "config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "unrendered_config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "raw_code": { "type": "string", "default": "" }, "doc_blocks": { "type": "array", "items": { "type": "string" } }, "root_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "depends_on": { "type": "object", "title": "MacroDependsOn", "properties": { "macros": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false }, "defer_relation": { "anyOf": [ { "type": "object", "title": "DeferRelation", "properties": { "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "alias": { "type": "string" }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "resource_type": { "enum": [ "model", "analysis", "test", "snapshot", "operation", "seed", "rpc", "sql_operation", "doc", "source", "macro", "exposure", "metric", "group", "saved_query", "semantic_model", "unit_test", "fixture", "function" ] }, "name": { "type": "string" }, "description": { "type": "string" }, "compiled_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } }, "config": { "anyOf": [ { "type": "object", "title": "NodeConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "tags": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "materialized": { "type": "string", "default": "view" }, "incremental_strategy": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "batch_size": { "default": null }, "lookback": { "default": 1 }, "begin": { "default": null }, "persist_docs": { "type": "object", "propertyNames": { "type": "string" } }, "post-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "pre-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "quoting": { "type": "object", "propertyNames": { "type": "string" } }, "column_types": { "type": "object", "propertyNames": { "type": "string" } }, "full_refresh": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "unique_key": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ], "default": null }, "on_schema_change": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "on_configuration_change": { "enum": [ "apply", "continue", "fail" ] }, "grants": { "type": "object", "propertyNames": { "type": "string" } }, "packages": { "type": "array", "items": { "type": "string" } }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "contract": { "type": "object", "title": "ContractConfig", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true } }, "additionalProperties": false }, "event_time": { "default": null }, "concurrent_batches": { "default": null } }, "additionalProperties": true }, { "type": "null" } ] } }, "additionalProperties": false, "required": [ "database", "schema", "alias", "relation_name", "resource_type", "name", "description", "compiled_code", "meta", "tags", "config" ] }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "database", "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ] }, { "type": "object", "title": "Analysis", "properties": { "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "const": "analysis" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "type": "object", "title": "FileHash", "properties": { "name": { "type": "string" }, "checksum": { "type": "string" } }, "additionalProperties": false, "required": [ "name", "checksum" ] }, "config": { "type": "object", "title": "NodeConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "tags": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "materialized": { "type": "string", "default": "view" }, "incremental_strategy": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "batch_size": { "default": null }, "lookback": { "default": 1 }, "begin": { "default": null }, "persist_docs": { "type": "object", "propertyNames": { "type": "string" } }, "post-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "pre-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "quoting": { "type": "object", "propertyNames": { "type": "string" } }, "column_types": { "type": "object", "propertyNames": { "type": "string" } }, "full_refresh": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "unique_key": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ], "default": null }, "on_schema_change": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "on_configuration_change": { "enum": [ "apply", "continue", "fail" ] }, "grants": { "type": "object", "propertyNames": { "type": "string" } }, "packages": { "type": "array", "items": { "type": "string" } }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "contract": { "type": "object", "title": "ContractConfig", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true } }, "additionalProperties": false }, "event_time": { "default": null }, "concurrent_batches": { "default": null } }, "additionalProperties": true }, "tags": { "type": "array", "items": { "type": "string" } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "type": "object", "title": "ColumnInfo", "properties": { "name": { "type": "string" }, "description": { "type": "string", "default": "" }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "data_type": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "constraints": { "type": "array", "items": { "type": "object", "title": "ColumnLevelConstraint", "properties": { "type": { "enum": [ "check", "not_null", "unique", "primary_key", "foreign_key", "custom" ] }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "expression": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "warn_unenforced": { "type": "boolean", "default": true }, "warn_unsupported": { "type": "boolean", "default": true }, "to": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "to_columns": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false, "required": [ "type" ] } }, "quote": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "title": "ColumnConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true }, "tags": { "type": "array", "items": { "type": "string" } }, "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "granularity": { "anyOf": [ { "enum": [ "nanosecond", "microsecond", "millisecond", "second", "minute", "hour", "day", "week", "month", "quarter", "year" ] }, { "type": "null" } ], "default": null }, "dimension": { "anyOf": [ { "type": "object", "title": "ColumnDimension", "properties": { "name": { "type": "string" }, "type": { "enum": [ "categorical", "time" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "is_partition": { "type": "boolean", "default": false }, "config": { "type": "object", "propertyNames": { "type": "string" } }, "validity_params": { "anyOf": [ { "type": "object", "title": "ColumnDimensionValidityParams", "properties": { "is_start": { "type": "boolean", "default": false }, "is_end": { "type": "boolean", "default": false } }, "additionalProperties": false }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "categorical", "time" ] }, { "type": "null" } ], "default": null }, "entity": { "anyOf": [ { "type": "object", "title": "ColumnEntity", "properties": { "name": { "type": "string" }, "type": { "enum": [ "foreign", "natural", "primary", "unique" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "foreign", "natural", "primary", "unique" ] }, { "type": "null" } ], "default": null }, "doc_blocks": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true, "required": [ "name" ] }, "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "patch_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "build_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "created_at": { "type": "number" }, "config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "unrendered_config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "raw_code": { "type": "string", "default": "" }, "doc_blocks": { "type": "array", "items": { "type": "string" } }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "type": "object", "title": "RefArgs", "properties": { "name": { "type": "string" }, "package": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "version": { "anyOf": [ { "type": "string" }, { "type": "number" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] } }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "functions": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "depends_on": { "type": "object", "title": "DependsOn", "properties": { "macros": { "type": "array", "items": { "type": "string" } }, "nodes": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false }, "compiled_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "type": "object", "title": "InjectedCTE", "properties": { "id": { "type": "string" }, "sql": { "type": "string" } }, "additionalProperties": false, "required": [ "id", "sql" ] } }, "_pre_injected_sql": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "contract": { "type": "object", "title": "Contract", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true }, "checksum": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false } }, "additionalProperties": false, "required": [ "database", "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ] }, { "type": "object", "title": "SingularTest", "properties": { "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "const": "test" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "type": "object", "title": "FileHash", "properties": { "name": { "type": "string" }, "checksum": { "type": "string" } }, "additionalProperties": false, "required": [ "name", "checksum" ] }, "config": { "type": "object", "title": "TestConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": "dbt_test__audit" }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "tags": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "materialized": { "type": "string", "default": "test" }, "severity": { "type": "string", "default": "ERROR", "pattern": "^([Ww][Aa][Rr][Nn]|[Ee][Rr][Rr][Oo][Rr])$" }, "store_failures": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "store_failures_as": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "sql_header": { "default": null }, "where": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "limit": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null }, "fail_calc": { "type": "string", "default": "count(*)" }, "warn_if": { "type": "string", "default": "!= 0" }, "error_if": { "type": "string", "default": "!= 0" } }, "additionalProperties": true }, "tags": { "type": "array", "items": { "type": "string" } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "type": "object", "title": "ColumnInfo", "properties": { "name": { "type": "string" }, "description": { "type": "string", "default": "" }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "data_type": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "constraints": { "type": "array", "items": { "type": "object", "title": "ColumnLevelConstraint", "properties": { "type": { "enum": [ "check", "not_null", "unique", "primary_key", "foreign_key", "custom" ] }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "expression": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "warn_unenforced": { "type": "boolean", "default": true }, "warn_unsupported": { "type": "boolean", "default": true }, "to": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "to_columns": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false, "required": [ "type" ] } }, "quote": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "title": "ColumnConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true }, "tags": { "type": "array", "items": { "type": "string" } }, "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "granularity": { "anyOf": [ { "enum": [ "nanosecond", "microsecond", "millisecond", "second", "minute", "hour", "day", "week", "month", "quarter", "year" ] }, { "type": "null" } ], "default": null }, "dimension": { "anyOf": [ { "type": "object", "title": "ColumnDimension", "properties": { "name": { "type": "string" }, "type": { "enum": [ "categorical", "time" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "is_partition": { "type": "boolean", "default": false }, "config": { "type": "object", "propertyNames": { "type": "string" } }, "validity_params": { "anyOf": [ { "type": "object", "title": "ColumnDimensionValidityParams", "properties": { "is_start": { "type": "boolean", "default": false }, "is_end": { "type": "boolean", "default": false } }, "additionalProperties": false }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "categorical", "time" ] }, { "type": "null" } ], "default": null }, "entity": { "anyOf": [ { "type": "object", "title": "ColumnEntity", "properties": { "name": { "type": "string" }, "type": { "enum": [ "foreign", "natural", "primary", "unique" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "foreign", "natural", "primary", "unique" ] }, { "type": "null" } ], "default": null }, "doc_blocks": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true, "required": [ "name" ] }, "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "patch_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "build_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "created_at": { "type": "number" }, "config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "unrendered_config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "raw_code": { "type": "string", "default": "" }, "doc_blocks": { "type": "array", "items": { "type": "string" } }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "type": "object", "title": "RefArgs", "properties": { "name": { "type": "string" }, "package": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "version": { "anyOf": [ { "type": "string" }, { "type": "number" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] } }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "functions": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "depends_on": { "type": "object", "title": "DependsOn", "properties": { "macros": { "type": "array", "items": { "type": "string" } }, "nodes": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false }, "compiled_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "type": "object", "title": "InjectedCTE", "properties": { "id": { "type": "string" }, "sql": { "type": "string" } }, "additionalProperties": false, "required": [ "id", "sql" ] } }, "_pre_injected_sql": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "contract": { "type": "object", "title": "Contract", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true }, "checksum": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false } }, "additionalProperties": false, "required": [ "database", "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ] }, { "type": "object", "title": "HookNode", "properties": { "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "const": "operation" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "type": "object", "title": "FileHash", "properties": { "name": { "type": "string" }, "checksum": { "type": "string" } }, "additionalProperties": false, "required": [ "name", "checksum" ] }, "config": { "type": "object", "title": "NodeConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "tags": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "materialized": { "type": "string", "default": "view" }, "incremental_strategy": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "batch_size": { "default": null }, "lookback": { "default": 1 }, "begin": { "default": null }, "persist_docs": { "type": "object", "propertyNames": { "type": "string" } }, "post-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "pre-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "quoting": { "type": "object", "propertyNames": { "type": "string" } }, "column_types": { "type": "object", "propertyNames": { "type": "string" } }, "full_refresh": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "unique_key": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ], "default": null }, "on_schema_change": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "on_configuration_change": { "enum": [ "apply", "continue", "fail" ] }, "grants": { "type": "object", "propertyNames": { "type": "string" } }, "packages": { "type": "array", "items": { "type": "string" } }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "contract": { "type": "object", "title": "ContractConfig", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true } }, "additionalProperties": false }, "event_time": { "default": null }, "concurrent_batches": { "default": null } }, "additionalProperties": true }, "tags": { "type": "array", "items": { "type": "string" } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "type": "object", "title": "ColumnInfo", "properties": { "name": { "type": "string" }, "description": { "type": "string", "default": "" }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "data_type": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "constraints": { "type": "array", "items": { "type": "object", "title": "ColumnLevelConstraint", "properties": { "type": { "enum": [ "check", "not_null", "unique", "primary_key", "foreign_key", "custom" ] }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "expression": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "warn_unenforced": { "type": "boolean", "default": true }, "warn_unsupported": { "type": "boolean", "default": true }, "to": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "to_columns": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false, "required": [ "type" ] } }, "quote": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "title": "ColumnConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true }, "tags": { "type": "array", "items": { "type": "string" } }, "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "granularity": { "anyOf": [ { "enum": [ "nanosecond", "microsecond", "millisecond", "second", "minute", "hour", "day", "week", "month", "quarter", "year" ] }, { "type": "null" } ], "default": null }, "dimension": { "anyOf": [ { "type": "object", "title": "ColumnDimension", "properties": { "name": { "type": "string" }, "type": { "enum": [ "categorical", "time" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "is_partition": { "type": "boolean", "default": false }, "config": { "type": "object", "propertyNames": { "type": "string" } }, "validity_params": { "anyOf": [ { "type": "object", "title": "ColumnDimensionValidityParams", "properties": { "is_start": { "type": "boolean", "default": false }, "is_end": { "type": "boolean", "default": false } }, "additionalProperties": false }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "categorical", "time" ] }, { "type": "null" } ], "default": null }, "entity": { "anyOf": [ { "type": "object", "title": "ColumnEntity", "properties": { "name": { "type": "string" }, "type": { "enum": [ "foreign", "natural", "primary", "unique" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "foreign", "natural", "primary", "unique" ] }, { "type": "null" } ], "default": null }, "doc_blocks": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true, "required": [ "name" ] }, "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "patch_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "build_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "created_at": { "type": "number" }, "config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "unrendered_config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "raw_code": { "type": "string", "default": "" }, "doc_blocks": { "type": "array", "items": { "type": "string" } }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "type": "object", "title": "RefArgs", "properties": { "name": { "type": "string" }, "package": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "version": { "anyOf": [ { "type": "string" }, { "type": "number" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] } }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "functions": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "depends_on": { "type": "object", "title": "DependsOn", "properties": { "macros": { "type": "array", "items": { "type": "string" } }, "nodes": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false }, "compiled_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "type": "object", "title": "InjectedCTE", "properties": { "id": { "type": "string" }, "sql": { "type": "string" } }, "additionalProperties": false, "required": [ "id", "sql" ] } }, "_pre_injected_sql": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "contract": { "type": "object", "title": "Contract", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true }, "checksum": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "database", "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ] }, { "type": "object", "title": "Model", "properties": { "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "const": "model" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "type": "object", "title": "FileHash", "properties": { "name": { "type": "string" }, "checksum": { "type": "string" } }, "additionalProperties": false, "required": [ "name", "checksum" ] }, "config": { "type": "object", "title": "ModelConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "tags": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "materialized": { "type": "string", "default": "view" }, "incremental_strategy": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "batch_size": { "default": null }, "lookback": { "default": 1 }, "begin": { "default": null }, "persist_docs": { "type": "object", "propertyNames": { "type": "string" } }, "post-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "pre-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "quoting": { "type": "object", "propertyNames": { "type": "string" } }, "column_types": { "type": "object", "propertyNames": { "type": "string" } }, "full_refresh": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "unique_key": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ], "default": null }, "on_schema_change": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "on_configuration_change": { "enum": [ "apply", "continue", "fail" ] }, "grants": { "type": "object", "propertyNames": { "type": "string" } }, "packages": { "type": "array", "items": { "type": "string" } }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "contract": { "type": "object", "title": "ContractConfig", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true } }, "additionalProperties": false }, "event_time": { "default": null }, "concurrent_batches": { "default": null }, "access": { "enum": [ "private", "protected", "public" ], "default": "protected" }, "freshness": { "anyOf": [ { "type": "object", "title": "ModelFreshness", "properties": { "build_after": { "type": "object", "title": "ModelBuildAfter", "properties": { "count": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null }, "period": { "anyOf": [ { "enum": [ "minute", "hour", "day" ] }, { "type": "null" } ], "default": null }, "updates_on": { "enum": [ "all", "any" ], "default": "any" } }, "additionalProperties": true } }, "additionalProperties": true, "required": [ "build_after" ] }, { "type": "null" } ], "default": null } }, "additionalProperties": true }, "tags": { "type": "array", "items": { "type": "string" } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "type": "object", "title": "ColumnInfo", "properties": { "name": { "type": "string" }, "description": { "type": "string", "default": "" }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "data_type": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "constraints": { "type": "array", "items": { "type": "object", "title": "ColumnLevelConstraint", "properties": { "type": { "enum": [ "check", "not_null", "unique", "primary_key", "foreign_key", "custom" ] }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "expression": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "warn_unenforced": { "type": "boolean", "default": true }, "warn_unsupported": { "type": "boolean", "default": true }, "to": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "to_columns": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false, "required": [ "type" ] } }, "quote": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "title": "ColumnConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true }, "tags": { "type": "array", "items": { "type": "string" } }, "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "granularity": { "anyOf": [ { "enum": [ "nanosecond", "microsecond", "millisecond", "second", "minute", "hour", "day", "week", "month", "quarter", "year" ] }, { "type": "null" } ], "default": null }, "dimension": { "anyOf": [ { "type": "object", "title": "ColumnDimension", "properties": { "name": { "type": "string" }, "type": { "enum": [ "categorical", "time" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "is_partition": { "type": "boolean", "default": false }, "config": { "type": "object", "propertyNames": { "type": "string" } }, "validity_params": { "anyOf": [ { "type": "object", "title": "ColumnDimensionValidityParams", "properties": { "is_start": { "type": "boolean", "default": false }, "is_end": { "type": "boolean", "default": false } }, "additionalProperties": false }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "categorical", "time" ] }, { "type": "null" } ], "default": null }, "entity": { "anyOf": [ { "type": "object", "title": "ColumnEntity", "properties": { "name": { "type": "string" }, "type": { "enum": [ "foreign", "natural", "primary", "unique" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "foreign", "natural", "primary", "unique" ] }, { "type": "null" } ], "default": null }, "doc_blocks": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true, "required": [ "name" ] }, "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "patch_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "build_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "created_at": { "type": "number" }, "config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "unrendered_config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "raw_code": { "type": "string", "default": "" }, "doc_blocks": { "type": "array", "items": { "type": "string" } }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "type": "object", "title": "RefArgs", "properties": { "name": { "type": "string" }, "package": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "version": { "anyOf": [ { "type": "string" }, { "type": "number" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] } }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "functions": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "depends_on": { "type": "object", "title": "DependsOn", "properties": { "macros": { "type": "array", "items": { "type": "string" } }, "nodes": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false }, "compiled_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "type": "object", "title": "InjectedCTE", "properties": { "id": { "type": "string" }, "sql": { "type": "string" } }, "additionalProperties": false, "required": [ "id", "sql" ] } }, "_pre_injected_sql": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "contract": { "type": "object", "title": "Contract", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true }, "checksum": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "access": { "enum": [ "private", "protected", "public" ], "default": "protected" }, "constraints": { "type": "array", "items": { "type": "object", "title": "ModelLevelConstraint", "properties": { "type": { "enum": [ "check", "not_null", "unique", "primary_key", "foreign_key", "custom" ] }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "expression": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "warn_unenforced": { "type": "boolean", "default": true }, "warn_unsupported": { "type": "boolean", "default": true }, "to": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "to_columns": { "type": "array", "items": { "type": "string" } }, "columns": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false, "required": [ "type" ] } }, "version": { "anyOf": [ { "type": "string" }, { "type": "number" }, { "type": "null" } ], "default": null }, "latest_version": { "anyOf": [ { "type": "string" }, { "type": "number" }, { "type": "null" } ], "default": null }, "deprecation_date": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "defer_relation": { "anyOf": [ { "type": "object", "title": "DeferRelation", "properties": { "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "alias": { "type": "string" }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "resource_type": { "enum": [ "model", "analysis", "test", "snapshot", "operation", "seed", "rpc", "sql_operation", "doc", "source", "macro", "exposure", "metric", "group", "saved_query", "semantic_model", "unit_test", "fixture", "function" ] }, "name": { "type": "string" }, "description": { "type": "string" }, "compiled_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } }, "config": { "anyOf": [ { "type": "object", "title": "NodeConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "tags": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "materialized": { "type": "string", "default": "view" }, "incremental_strategy": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "batch_size": { "default": null }, "lookback": { "default": 1 }, "begin": { "default": null }, "persist_docs": { "type": "object", "propertyNames": { "type": "string" } }, "post-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "pre-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "quoting": { "type": "object", "propertyNames": { "type": "string" } }, "column_types": { "type": "object", "propertyNames": { "type": "string" } }, "full_refresh": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "unique_key": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ], "default": null }, "on_schema_change": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "on_configuration_change": { "enum": [ "apply", "continue", "fail" ] }, "grants": { "type": "object", "propertyNames": { "type": "string" } }, "packages": { "type": "array", "items": { "type": "string" } }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "contract": { "type": "object", "title": "ContractConfig", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true } }, "additionalProperties": false }, "event_time": { "default": null }, "concurrent_batches": { "default": null } }, "additionalProperties": true }, { "type": "null" } ] } }, "additionalProperties": false, "required": [ "database", "schema", "alias", "relation_name", "resource_type", "name", "description", "compiled_code", "meta", "tags", "config" ] }, { "type": "null" } ], "default": null }, "primary_key": { "type": "array", "items": { "type": "string" } }, "time_spine": { "anyOf": [ { "type": "object", "title": "TimeSpine", "properties": { "standard_granularity_column": { "type": "string" }, "custom_granularities": { "type": "array", "items": { "type": "object", "title": "CustomGranularity", "properties": { "name": { "type": "string" }, "column_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] } } }, "additionalProperties": false, "required": [ "standard_granularity_column" ] }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "database", "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ] }, { "type": "object", "title": "SqlOperation", "properties": { "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "const": "sql_operation" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "type": "object", "title": "FileHash", "properties": { "name": { "type": "string" }, "checksum": { "type": "string" } }, "additionalProperties": false, "required": [ "name", "checksum" ] }, "config": { "type": "object", "title": "NodeConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "tags": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "materialized": { "type": "string", "default": "view" }, "incremental_strategy": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "batch_size": { "default": null }, "lookback": { "default": 1 }, "begin": { "default": null }, "persist_docs": { "type": "object", "propertyNames": { "type": "string" } }, "post-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "pre-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "quoting": { "type": "object", "propertyNames": { "type": "string" } }, "column_types": { "type": "object", "propertyNames": { "type": "string" } }, "full_refresh": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "unique_key": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ], "default": null }, "on_schema_change": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "on_configuration_change": { "enum": [ "apply", "continue", "fail" ] }, "grants": { "type": "object", "propertyNames": { "type": "string" } }, "packages": { "type": "array", "items": { "type": "string" } }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "contract": { "type": "object", "title": "ContractConfig", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true } }, "additionalProperties": false }, "event_time": { "default": null }, "concurrent_batches": { "default": null } }, "additionalProperties": true }, "tags": { "type": "array", "items": { "type": "string" } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "type": "object", "title": "ColumnInfo", "properties": { "name": { "type": "string" }, "description": { "type": "string", "default": "" }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "data_type": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "constraints": { "type": "array", "items": { "type": "object", "title": "ColumnLevelConstraint", "properties": { "type": { "enum": [ "check", "not_null", "unique", "primary_key", "foreign_key", "custom" ] }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "expression": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "warn_unenforced": { "type": "boolean", "default": true }, "warn_unsupported": { "type": "boolean", "default": true }, "to": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "to_columns": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false, "required": [ "type" ] } }, "quote": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "title": "ColumnConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true }, "tags": { "type": "array", "items": { "type": "string" } }, "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "granularity": { "anyOf": [ { "enum": [ "nanosecond", "microsecond", "millisecond", "second", "minute", "hour", "day", "week", "month", "quarter", "year" ] }, { "type": "null" } ], "default": null }, "dimension": { "anyOf": [ { "type": "object", "title": "ColumnDimension", "properties": { "name": { "type": "string" }, "type": { "enum": [ "categorical", "time" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "is_partition": { "type": "boolean", "default": false }, "config": { "type": "object", "propertyNames": { "type": "string" } }, "validity_params": { "anyOf": [ { "type": "object", "title": "ColumnDimensionValidityParams", "properties": { "is_start": { "type": "boolean", "default": false }, "is_end": { "type": "boolean", "default": false } }, "additionalProperties": false }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "categorical", "time" ] }, { "type": "null" } ], "default": null }, "entity": { "anyOf": [ { "type": "object", "title": "ColumnEntity", "properties": { "name": { "type": "string" }, "type": { "enum": [ "foreign", "natural", "primary", "unique" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "foreign", "natural", "primary", "unique" ] }, { "type": "null" } ], "default": null }, "doc_blocks": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true, "required": [ "name" ] }, "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "patch_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "build_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "created_at": { "type": "number" }, "config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "unrendered_config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "raw_code": { "type": "string", "default": "" }, "doc_blocks": { "type": "array", "items": { "type": "string" } }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "type": "object", "title": "RefArgs", "properties": { "name": { "type": "string" }, "package": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "version": { "anyOf": [ { "type": "string" }, { "type": "number" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] } }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "functions": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "depends_on": { "type": "object", "title": "DependsOn", "properties": { "macros": { "type": "array", "items": { "type": "string" } }, "nodes": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false }, "compiled_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "type": "object", "title": "InjectedCTE", "properties": { "id": { "type": "string" }, "sql": { "type": "string" } }, "additionalProperties": false, "required": [ "id", "sql" ] } }, "_pre_injected_sql": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "contract": { "type": "object", "title": "Contract", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true }, "checksum": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false } }, "additionalProperties": false, "required": [ "database", "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ] }, { "type": "object", "title": "GenericTest", "properties": { "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "const": "test" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "type": "object", "title": "FileHash", "properties": { "name": { "type": "string" }, "checksum": { "type": "string" } }, "additionalProperties": false, "required": [ "name", "checksum" ] }, "config": { "type": "object", "title": "TestConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": "dbt_test__audit" }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "tags": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "materialized": { "type": "string", "default": "test" }, "severity": { "type": "string", "default": "ERROR", "pattern": "^([Ww][Aa][Rr][Nn]|[Ee][Rr][Rr][Oo][Rr])$" }, "store_failures": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "store_failures_as": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "sql_header": { "default": null }, "where": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "limit": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null }, "fail_calc": { "type": "string", "default": "count(*)" }, "warn_if": { "type": "string", "default": "!= 0" }, "error_if": { "type": "string", "default": "!= 0" } }, "additionalProperties": true }, "tags": { "type": "array", "items": { "type": "string" } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "type": "object", "title": "ColumnInfo", "properties": { "name": { "type": "string" }, "description": { "type": "string", "default": "" }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "data_type": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "constraints": { "type": "array", "items": { "type": "object", "title": "ColumnLevelConstraint", "properties": { "type": { "enum": [ "check", "not_null", "unique", "primary_key", "foreign_key", "custom" ] }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "expression": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "warn_unenforced": { "type": "boolean", "default": true }, "warn_unsupported": { "type": "boolean", "default": true }, "to": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "to_columns": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false, "required": [ "type" ] } }, "quote": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "title": "ColumnConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true }, "tags": { "type": "array", "items": { "type": "string" } }, "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "granularity": { "anyOf": [ { "enum": [ "nanosecond", "microsecond", "millisecond", "second", "minute", "hour", "day", "week", "month", "quarter", "year" ] }, { "type": "null" } ], "default": null }, "dimension": { "anyOf": [ { "type": "object", "title": "ColumnDimension", "properties": { "name": { "type": "string" }, "type": { "enum": [ "categorical", "time" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "is_partition": { "type": "boolean", "default": false }, "config": { "type": "object", "propertyNames": { "type": "string" } }, "validity_params": { "anyOf": [ { "type": "object", "title": "ColumnDimensionValidityParams", "properties": { "is_start": { "type": "boolean", "default": false }, "is_end": { "type": "boolean", "default": false } }, "additionalProperties": false }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "categorical", "time" ] }, { "type": "null" } ], "default": null }, "entity": { "anyOf": [ { "type": "object", "title": "ColumnEntity", "properties": { "name": { "type": "string" }, "type": { "enum": [ "foreign", "natural", "primary", "unique" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "foreign", "natural", "primary", "unique" ] }, { "type": "null" } ], "default": null }, "doc_blocks": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true, "required": [ "name" ] }, "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "patch_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "build_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "created_at": { "type": "number" }, "config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "unrendered_config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "raw_code": { "type": "string", "default": "" }, "doc_blocks": { "type": "array", "items": { "type": "string" } }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "type": "object", "title": "RefArgs", "properties": { "name": { "type": "string" }, "package": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "version": { "anyOf": [ { "type": "string" }, { "type": "number" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] } }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "functions": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "depends_on": { "type": "object", "title": "DependsOn", "properties": { "macros": { "type": "array", "items": { "type": "string" } }, "nodes": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false }, "compiled_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "type": "object", "title": "InjectedCTE", "properties": { "id": { "type": "string" }, "sql": { "type": "string" } }, "additionalProperties": false, "required": [ "id", "sql" ] } }, "_pre_injected_sql": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "contract": { "type": "object", "title": "Contract", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true }, "checksum": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "column_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "file_key_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "attached_node": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "test_metadata": { "type": "object", "title": "TestMetadata", "properties": { "name": { "type": "string", "default": "test" }, "kwargs": { "type": "object", "propertyNames": { "type": "string" } }, "namespace": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false } }, "additionalProperties": false, "required": [ "database", "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ] }, { "type": "object", "title": "Snapshot", "properties": { "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "const": "snapshot" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "type": "object", "title": "FileHash", "properties": { "name": { "type": "string" }, "checksum": { "type": "string" } }, "additionalProperties": false, "required": [ "name", "checksum" ] }, "config": { "type": "object", "title": "SnapshotConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "tags": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "materialized": { "type": "string", "default": "snapshot" }, "incremental_strategy": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "batch_size": { "default": null }, "lookback": { "default": 1 }, "begin": { "default": null }, "persist_docs": { "type": "object", "propertyNames": { "type": "string" } }, "post-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "pre-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "quoting": { "type": "object", "propertyNames": { "type": "string" } }, "column_types": { "type": "object", "propertyNames": { "type": "string" } }, "full_refresh": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "unique_key": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ], "default": null }, "on_schema_change": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "on_configuration_change": { "enum": [ "apply", "continue", "fail" ] }, "grants": { "type": "object", "propertyNames": { "type": "string" } }, "packages": { "type": "array", "items": { "type": "string" } }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "contract": { "type": "object", "title": "ContractConfig", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true } }, "additionalProperties": false }, "event_time": { "default": null }, "concurrent_batches": { "default": null }, "strategy": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "target_schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "target_database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "updated_at": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "check_cols": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ], "default": null }, "snapshot_meta_column_names": { "type": "object", "title": "SnapshotMetaColumnNames", "properties": { "dbt_valid_to": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "dbt_valid_from": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "dbt_scd_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "dbt_updated_at": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "dbt_is_deleted": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "dbt_valid_to_current": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": true }, "tags": { "type": "array", "items": { "type": "string" } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "type": "object", "title": "ColumnInfo", "properties": { "name": { "type": "string" }, "description": { "type": "string", "default": "" }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "data_type": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "constraints": { "type": "array", "items": { "type": "object", "title": "ColumnLevelConstraint", "properties": { "type": { "enum": [ "check", "not_null", "unique", "primary_key", "foreign_key", "custom" ] }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "expression": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "warn_unenforced": { "type": "boolean", "default": true }, "warn_unsupported": { "type": "boolean", "default": true }, "to": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "to_columns": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false, "required": [ "type" ] } }, "quote": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "title": "ColumnConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true }, "tags": { "type": "array", "items": { "type": "string" } }, "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "granularity": { "anyOf": [ { "enum": [ "nanosecond", "microsecond", "millisecond", "second", "minute", "hour", "day", "week", "month", "quarter", "year" ] }, { "type": "null" } ], "default": null }, "dimension": { "anyOf": [ { "type": "object", "title": "ColumnDimension", "properties": { "name": { "type": "string" }, "type": { "enum": [ "categorical", "time" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "is_partition": { "type": "boolean", "default": false }, "config": { "type": "object", "propertyNames": { "type": "string" } }, "validity_params": { "anyOf": [ { "type": "object", "title": "ColumnDimensionValidityParams", "properties": { "is_start": { "type": "boolean", "default": false }, "is_end": { "type": "boolean", "default": false } }, "additionalProperties": false }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "categorical", "time" ] }, { "type": "null" } ], "default": null }, "entity": { "anyOf": [ { "type": "object", "title": "ColumnEntity", "properties": { "name": { "type": "string" }, "type": { "enum": [ "foreign", "natural", "primary", "unique" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "foreign", "natural", "primary", "unique" ] }, { "type": "null" } ], "default": null }, "doc_blocks": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true, "required": [ "name" ] }, "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "patch_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "build_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "created_at": { "type": "number" }, "config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "unrendered_config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "raw_code": { "type": "string", "default": "" }, "doc_blocks": { "type": "array", "items": { "type": "string" } }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "type": "object", "title": "RefArgs", "properties": { "name": { "type": "string" }, "package": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "version": { "anyOf": [ { "type": "string" }, { "type": "number" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] } }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "functions": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "depends_on": { "type": "object", "title": "DependsOn", "properties": { "macros": { "type": "array", "items": { "type": "string" } }, "nodes": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false }, "compiled_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "type": "object", "title": "InjectedCTE", "properties": { "id": { "type": "string" }, "sql": { "type": "string" } }, "additionalProperties": false, "required": [ "id", "sql" ] } }, "_pre_injected_sql": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "contract": { "type": "object", "title": "Contract", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true }, "checksum": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "defer_relation": { "anyOf": [ { "type": "object", "title": "DeferRelation", "properties": { "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "alias": { "type": "string" }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "resource_type": { "enum": [ "model", "analysis", "test", "snapshot", "operation", "seed", "rpc", "sql_operation", "doc", "source", "macro", "exposure", "metric", "group", "saved_query", "semantic_model", "unit_test", "fixture", "function" ] }, "name": { "type": "string" }, "description": { "type": "string" }, "compiled_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } }, "config": { "anyOf": [ { "type": "object", "title": "NodeConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "tags": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "materialized": { "type": "string", "default": "view" }, "incremental_strategy": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "batch_size": { "default": null }, "lookback": { "default": 1 }, "begin": { "default": null }, "persist_docs": { "type": "object", "propertyNames": { "type": "string" } }, "post-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "pre-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "quoting": { "type": "object", "propertyNames": { "type": "string" } }, "column_types": { "type": "object", "propertyNames": { "type": "string" } }, "full_refresh": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "unique_key": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ], "default": null }, "on_schema_change": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "on_configuration_change": { "enum": [ "apply", "continue", "fail" ] }, "grants": { "type": "object", "propertyNames": { "type": "string" } }, "packages": { "type": "array", "items": { "type": "string" } }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "contract": { "type": "object", "title": "ContractConfig", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true } }, "additionalProperties": false }, "event_time": { "default": null }, "concurrent_batches": { "default": null } }, "additionalProperties": true }, { "type": "null" } ] } }, "additionalProperties": false, "required": [ "database", "schema", "alias", "relation_name", "resource_type", "name", "description", "compiled_code", "meta", "tags", "config" ] }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "database", "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum", "config" ] }, { "type": "object", "title": "Function", "properties": { "returns": { "type": "object", "title": "FunctionReturns", "properties": { "data_type": { "type": "string" }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "data_type" ] }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "const": "function" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "type": "object", "title": "FileHash", "properties": { "name": { "type": "string" }, "checksum": { "type": "string" } }, "additionalProperties": false, "required": [ "name", "checksum" ] }, "config": { "type": "object", "title": "FunctionConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "tags": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "materialized": { "type": "string", "default": "function" }, "incremental_strategy": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "batch_size": { "default": null }, "lookback": { "default": 1 }, "begin": { "default": null }, "persist_docs": { "type": "object", "propertyNames": { "type": "string" } }, "post-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "pre-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "quoting": { "type": "object", "propertyNames": { "type": "string" } }, "column_types": { "type": "object", "propertyNames": { "type": "string" } }, "full_refresh": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "unique_key": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ], "default": null }, "on_schema_change": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "on_configuration_change": { "enum": [ "apply", "continue", "fail" ] }, "grants": { "type": "object", "propertyNames": { "type": "string" } }, "packages": { "type": "array", "items": { "type": "string" } }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "contract": { "type": "object", "title": "ContractConfig", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true } }, "additionalProperties": false }, "event_time": { "default": null }, "concurrent_batches": { "default": null }, "type": { "enum": [ "scalar", "aggregate", "table" ], "default": "scalar" }, "volatility": { "anyOf": [ { "enum": [ "deterministic", "stable", "non-deterministic" ] }, { "type": "null" } ], "default": null }, "runtime_version": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "entry_point": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": true }, "tags": { "type": "array", "items": { "type": "string" } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "type": "object", "title": "ColumnInfo", "properties": { "name": { "type": "string" }, "description": { "type": "string", "default": "" }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "data_type": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "constraints": { "type": "array", "items": { "type": "object", "title": "ColumnLevelConstraint", "properties": { "type": { "enum": [ "check", "not_null", "unique", "primary_key", "foreign_key", "custom" ] }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "expression": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "warn_unenforced": { "type": "boolean", "default": true }, "warn_unsupported": { "type": "boolean", "default": true }, "to": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "to_columns": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false, "required": [ "type" ] } }, "quote": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "title": "ColumnConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true }, "tags": { "type": "array", "items": { "type": "string" } }, "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "granularity": { "anyOf": [ { "enum": [ "nanosecond", "microsecond", "millisecond", "second", "minute", "hour", "day", "week", "month", "quarter", "year" ] }, { "type": "null" } ], "default": null }, "dimension": { "anyOf": [ { "type": "object", "title": "ColumnDimension", "properties": { "name": { "type": "string" }, "type": { "enum": [ "categorical", "time" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "is_partition": { "type": "boolean", "default": false }, "config": { "type": "object", "propertyNames": { "type": "string" } }, "validity_params": { "anyOf": [ { "type": "object", "title": "ColumnDimensionValidityParams", "properties": { "is_start": { "type": "boolean", "default": false }, "is_end": { "type": "boolean", "default": false } }, "additionalProperties": false }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "categorical", "time" ] }, { "type": "null" } ], "default": null }, "entity": { "anyOf": [ { "type": "object", "title": "ColumnEntity", "properties": { "name": { "type": "string" }, "type": { "enum": [ "foreign", "natural", "primary", "unique" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "foreign", "natural", "primary", "unique" ] }, { "type": "null" } ], "default": null }, "doc_blocks": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true, "required": [ "name" ] }, "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "patch_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "build_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "created_at": { "type": "number" }, "config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "unrendered_config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "raw_code": { "type": "string", "default": "" }, "doc_blocks": { "type": "array", "items": { "type": "string" } }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "type": "object", "title": "RefArgs", "properties": { "name": { "type": "string" }, "package": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "version": { "anyOf": [ { "type": "string" }, { "type": "number" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] } }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "functions": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "depends_on": { "type": "object", "title": "DependsOn", "properties": { "macros": { "type": "array", "items": { "type": "string" } }, "nodes": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false }, "compiled_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "type": "object", "title": "InjectedCTE", "properties": { "id": { "type": "string" }, "sql": { "type": "string" } }, "additionalProperties": false, "required": [ "id", "sql" ] } }, "_pre_injected_sql": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "contract": { "type": "object", "title": "Contract", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true }, "checksum": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "arguments": { "type": "array", "items": { "type": "object", "title": "FunctionArgument", "properties": { "name": { "type": "string" }, "data_type": { "type": "string" }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "default_value": { "anyOf": [ {}, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "data_type" ] } }, "defer_function": { "anyOf": [ { "type": "object", "title": "DeferFunction", "properties": { "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "alias": { "type": "string" }, "resource_type": { "enum": [ "model", "analysis", "test", "snapshot", "operation", "seed", "rpc", "sql_operation", "doc", "source", "macro", "exposure", "metric", "group", "saved_query", "semantic_model", "unit_test", "fixture", "function" ] }, "name": { "type": "string" }, "description": { "type": "string" }, "compiled_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } }, "config": { "anyOf": [ { "type": "object", "title": "FunctionConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "tags": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "materialized": { "type": "string", "default": "function" }, "incremental_strategy": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "batch_size": { "default": null }, "lookback": { "default": 1 }, "begin": { "default": null }, "persist_docs": { "type": "object", "propertyNames": { "type": "string" } }, "post-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "pre-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "quoting": { "type": "object", "propertyNames": { "type": "string" } }, "column_types": { "type": "object", "propertyNames": { "type": "string" } }, "full_refresh": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "unique_key": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ], "default": null }, "on_schema_change": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "on_configuration_change": { "enum": [ "apply", "continue", "fail" ] }, "grants": { "type": "object", "propertyNames": { "type": "string" } }, "packages": { "type": "array", "items": { "type": "string" } }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "contract": { "type": "object", "title": "ContractConfig", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true } }, "additionalProperties": false }, "event_time": { "default": null }, "concurrent_batches": { "default": null }, "type": { "enum": [ "scalar", "aggregate", "table" ], "default": "scalar" }, "volatility": { "anyOf": [ { "enum": [ "deterministic", "stable", "non-deterministic" ] }, { "type": "null" } ], "default": null }, "runtime_version": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "entry_point": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": true }, { "type": "null" } ] }, "arguments": { "type": "array", "items": { "type": "object", "title": "FunctionArgument", "properties": { "name": { "type": "string" }, "data_type": { "type": "string" }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "default_value": { "anyOf": [ {}, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "data_type" ] } }, "returns": { "type": "object", "title": "FunctionReturns", "properties": { "data_type": { "type": "string" }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "data_type" ] } }, "additionalProperties": false, "required": [ "database", "schema", "alias", "resource_type", "name", "description", "compiled_code", "meta", "tags", "config", "arguments", "returns" ] }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "returns", "database", "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum", "config" ] } ] }, "propertyNames": { "type": "string" } }, "sources": { "type": "object", "description": "The sources defined in the dbt project and its dependencies", "additionalProperties": { "type": "object", "title": "SourceDefinition", "properties": { "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "const": "source" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "source_name": { "type": "string" }, "source_description": { "type": "string" }, "loader": { "type": "string" }, "identifier": { "type": "string" }, "quoting": { "type": "object", "title": "Quoting", "properties": { "database": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "identifier": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "column": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "loaded_at_field": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "loaded_at_query": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "freshness": { "anyOf": [ { "type": "object", "title": "FreshnessThreshold", "properties": { "warn_after": { "anyOf": [ { "type": "object", "title": "Time", "properties": { "count": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null }, "period": { "anyOf": [ { "enum": [ "minute", "hour", "day" ] }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, { "type": "null" } ] }, "error_after": { "anyOf": [ { "type": "object", "title": "Time", "properties": { "count": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null }, "period": { "anyOf": [ { "enum": [ "minute", "hour", "day" ] }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, { "type": "null" } ] }, "filter": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, { "type": "null" } ], "default": null }, "external": { "anyOf": [ { "type": "object", "title": "ExternalTable", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "location": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "file_format": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "row_format": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "tbl_properties": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "partitions": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "array", "items": { "type": "object", "title": "ExternalPartition", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "name": { "type": "string", "default": "" }, "description": { "type": "string", "default": "" }, "data_type": { "type": "string", "default": "" }, "meta": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": true } }, { "type": "null" } ], "default": null } }, "additionalProperties": true }, { "type": "null" } ], "default": null }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "type": "object", "title": "ColumnInfo", "properties": { "name": { "type": "string" }, "description": { "type": "string", "default": "" }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "data_type": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "constraints": { "type": "array", "items": { "type": "object", "title": "ColumnLevelConstraint", "properties": { "type": { "enum": [ "check", "not_null", "unique", "primary_key", "foreign_key", "custom" ] }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "expression": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "warn_unenforced": { "type": "boolean", "default": true }, "warn_unsupported": { "type": "boolean", "default": true }, "to": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "to_columns": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false, "required": [ "type" ] } }, "quote": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "title": "ColumnConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true }, "tags": { "type": "array", "items": { "type": "string" } }, "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "granularity": { "anyOf": [ { "enum": [ "nanosecond", "microsecond", "millisecond", "second", "minute", "hour", "day", "week", "month", "quarter", "year" ] }, { "type": "null" } ], "default": null }, "dimension": { "anyOf": [ { "type": "object", "title": "ColumnDimension", "properties": { "name": { "type": "string" }, "type": { "enum": [ "categorical", "time" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "is_partition": { "type": "boolean", "default": false }, "config": { "type": "object", "propertyNames": { "type": "string" } }, "validity_params": { "anyOf": [ { "type": "object", "title": "ColumnDimensionValidityParams", "properties": { "is_start": { "type": "boolean", "default": false }, "is_end": { "type": "boolean", "default": false } }, "additionalProperties": false }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "categorical", "time" ] }, { "type": "null" } ], "default": null }, "entity": { "anyOf": [ { "type": "object", "title": "ColumnEntity", "properties": { "name": { "type": "string" }, "type": { "enum": [ "foreign", "natural", "primary", "unique" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "foreign", "natural", "primary", "unique" ] }, { "type": "null" } ], "default": null }, "doc_blocks": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true, "required": [ "name" ] }, "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "source_meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } }, "config": { "type": "object", "title": "SourceConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "event_time": { "default": null }, "freshness": { "anyOf": [ { "type": "object", "title": "FreshnessThreshold", "properties": { "warn_after": { "anyOf": [ { "type": "object", "title": "Time", "properties": { "count": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null }, "period": { "anyOf": [ { "enum": [ "minute", "hour", "day" ] }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, { "type": "null" } ] }, "error_after": { "anyOf": [ { "type": "object", "title": "Time", "properties": { "count": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null }, "period": { "anyOf": [ { "enum": [ "minute", "hour", "day" ] }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, { "type": "null" } ] }, "filter": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, { "type": "null" } ] }, "loaded_at_field": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "loaded_at_query": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true }, "patch_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "created_at": { "type": "number" }, "unrendered_database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "unrendered_schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "doc_blocks": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false, "required": [ "database", "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "source_name", "source_description", "loader", "identifier" ] }, "propertyNames": { "type": "string" } }, "macros": { "type": "object", "description": "The macros defined in the dbt project and its dependencies", "additionalProperties": { "type": "object", "title": "Macro", "properties": { "name": { "type": "string" }, "resource_type": { "const": "macro" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "macro_sql": { "type": "string" }, "depends_on": { "type": "object", "title": "MacroDependsOn", "properties": { "macros": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false }, "description": { "type": "string", "default": "" }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "config": { "type": "object", "title": "MacroConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false } }, "additionalProperties": true }, "patch_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "arguments": { "type": "array", "items": { "type": "object", "title": "MacroArgument", "properties": { "name": { "type": "string" }, "type": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "description": { "type": "string", "default": "" } }, "additionalProperties": false, "required": [ "name" ] } }, "created_at": { "type": "number" }, "supported_languages": { "anyOf": [ { "type": "array", "items": { "enum": [ "python", "sql" ] } }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "macro_sql" ] }, "propertyNames": { "type": "string" } }, "docs": { "type": "object", "description": "The docs defined in the dbt project and its dependencies", "additionalProperties": { "type": "object", "title": "Documentation", "properties": { "name": { "type": "string" }, "resource_type": { "const": "doc" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "block_contents": { "type": "string" } }, "additionalProperties": false, "required": [ "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "block_contents" ] }, "propertyNames": { "type": "string" } }, "exposures": { "type": "object", "description": "The exposures defined in the dbt project and its dependencies", "additionalProperties": { "type": "object", "title": "Exposure", "properties": { "name": { "type": "string" }, "resource_type": { "const": "exposure" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "type": { "enum": [ "dashboard", "notebook", "analysis", "ml", "application" ] }, "owner": { "type": "object", "title": "Owner", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "email": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ], "default": null }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": true }, "description": { "type": "string", "default": "" }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "maturity": { "anyOf": [ { "enum": [ "low", "medium", "high" ] }, { "type": "null" } ], "default": null }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } }, "config": { "type": "object", "title": "ExposureConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "tags": { "type": "array", "items": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": true }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "url": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "depends_on": { "type": "object", "title": "DependsOn", "properties": { "macros": { "type": "array", "items": { "type": "string" } }, "nodes": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false }, "refs": { "type": "array", "items": { "type": "object", "title": "RefArgs", "properties": { "name": { "type": "string" }, "package": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "version": { "anyOf": [ { "type": "string" }, { "type": "number" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] } }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "created_at": { "type": "number" } }, "additionalProperties": false, "required": [ "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "type", "owner" ] }, "propertyNames": { "type": "string" } }, "metrics": { "type": "object", "description": "The metrics defined in the dbt project and its dependencies", "additionalProperties": { "type": "object", "title": "Metric", "properties": { "name": { "type": "string" }, "resource_type": { "const": "metric" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "description": { "type": "string" }, "label": { "type": "string" }, "type": { "enum": [ "simple", "ratio", "cumulative", "derived", "conversion" ] }, "type_params": { "type": "object", "title": "MetricTypeParams", "properties": { "measure": { "anyOf": [ { "type": "object", "title": "MetricInputMeasure", "properties": { "name": { "type": "string" }, "filter": { "anyOf": [ { "type": "object", "title": "WhereFilterIntersection", "properties": { "where_filters": { "type": "array", "items": { "type": "object", "title": "WhereFilter", "properties": { "where_sql_template": { "type": "string" } }, "additionalProperties": false, "required": [ "where_sql_template" ] } } }, "additionalProperties": false, "required": [ "where_filters" ] }, { "type": "null" } ], "default": null }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "join_to_timespine": { "type": "boolean", "default": false }, "fill_nulls_with": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] }, { "type": "null" } ], "default": null }, "input_measures": { "type": "array", "items": { "type": "object", "title": "MetricInputMeasure", "properties": { "name": { "type": "string" }, "filter": { "anyOf": [ { "type": "object", "title": "WhereFilterIntersection", "properties": { "where_filters": { "type": "array", "items": { "type": "object", "title": "WhereFilter", "properties": { "where_sql_template": { "type": "string" } }, "additionalProperties": false, "required": [ "where_sql_template" ] } } }, "additionalProperties": false, "required": [ "where_filters" ] }, { "type": "null" } ], "default": null }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "join_to_timespine": { "type": "boolean", "default": false }, "fill_nulls_with": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] } }, "numerator": { "anyOf": [ { "type": "object", "title": "MetricInput", "properties": { "name": { "type": "string" }, "filter": { "anyOf": [ { "type": "object", "title": "WhereFilterIntersection", "properties": { "where_filters": { "type": "array", "items": { "type": "object", "title": "WhereFilter", "properties": { "where_sql_template": { "type": "string" } }, "additionalProperties": false, "required": [ "where_sql_template" ] } } }, "additionalProperties": false, "required": [ "where_filters" ] }, { "type": "null" } ], "default": null }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "offset_window": { "anyOf": [ { "type": "object", "title": "MetricTimeWindow", "properties": { "count": { "type": "integer" }, "granularity": { "type": "string" } }, "additionalProperties": false, "required": [ "count", "granularity" ] }, { "type": "null" } ], "default": null }, "offset_to_grain": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] }, { "type": "null" } ], "default": null }, "denominator": { "anyOf": [ { "type": "object", "title": "MetricInput", "properties": { "name": { "type": "string" }, "filter": { "anyOf": [ { "type": "object", "title": "WhereFilterIntersection", "properties": { "where_filters": { "type": "array", "items": { "type": "object", "title": "WhereFilter", "properties": { "where_sql_template": { "type": "string" } }, "additionalProperties": false, "required": [ "where_sql_template" ] } } }, "additionalProperties": false, "required": [ "where_filters" ] }, { "type": "null" } ], "default": null }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "offset_window": { "anyOf": [ { "type": "object", "title": "MetricTimeWindow", "properties": { "count": { "type": "integer" }, "granularity": { "type": "string" } }, "additionalProperties": false, "required": [ "count", "granularity" ] }, { "type": "null" } ], "default": null }, "offset_to_grain": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] }, { "type": "null" } ], "default": null }, "expr": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "window": { "anyOf": [ { "type": "object", "title": "MetricTimeWindow", "properties": { "count": { "type": "integer" }, "granularity": { "type": "string" } }, "additionalProperties": false, "required": [ "count", "granularity" ] }, { "type": "null" } ], "default": null }, "grain_to_date": { "anyOf": [ { "enum": [ "nanosecond", "microsecond", "millisecond", "second", "minute", "hour", "day", "week", "month", "quarter", "year" ] }, { "type": "null" } ], "default": null }, "metrics": { "anyOf": [ { "type": "array", "items": { "type": "object", "title": "MetricInput", "properties": { "name": { "type": "string" }, "filter": { "anyOf": [ { "type": "object", "title": "WhereFilterIntersection", "properties": { "where_filters": { "type": "array", "items": { "type": "object", "title": "WhereFilter", "properties": { "where_sql_template": { "type": "string" } }, "additionalProperties": false, "required": [ "where_sql_template" ] } } }, "additionalProperties": false, "required": [ "where_filters" ] }, { "type": "null" } ], "default": null }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "offset_window": { "anyOf": [ { "type": "object", "title": "MetricTimeWindow", "properties": { "count": { "type": "integer" }, "granularity": { "type": "string" } }, "additionalProperties": false, "required": [ "count", "granularity" ] }, { "type": "null" } ], "default": null }, "offset_to_grain": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] } }, { "type": "null" } ], "default": null }, "conversion_type_params": { "anyOf": [ { "type": "object", "title": "ConversionTypeParams", "properties": { "entity": { "type": "string" }, "base_measure": { "anyOf": [ { "type": "object", "title": "MetricInputMeasure", "properties": { "name": { "type": "string" }, "filter": { "anyOf": [ { "type": "object", "title": "WhereFilterIntersection", "properties": { "where_filters": { "type": "array", "items": { "type": "object", "title": "WhereFilter", "properties": { "where_sql_template": { "type": "string" } }, "additionalProperties": false, "required": [ "where_sql_template" ] } } }, "additionalProperties": false, "required": [ "where_filters" ] }, { "type": "null" } ], "default": null }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "join_to_timespine": { "type": "boolean", "default": false }, "fill_nulls_with": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] }, { "type": "null" } ], "default": null }, "conversion_measure": { "anyOf": [ { "type": "object", "title": "MetricInputMeasure", "properties": { "name": { "type": "string" }, "filter": { "anyOf": [ { "type": "object", "title": "WhereFilterIntersection", "properties": { "where_filters": { "type": "array", "items": { "type": "object", "title": "WhereFilter", "properties": { "where_sql_template": { "type": "string" } }, "additionalProperties": false, "required": [ "where_sql_template" ] } } }, "additionalProperties": false, "required": [ "where_filters" ] }, { "type": "null" } ], "default": null }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "join_to_timespine": { "type": "boolean", "default": false }, "fill_nulls_with": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] }, { "type": "null" } ], "default": null }, "base_metric": { "anyOf": [ { "type": "object", "title": "MetricInput", "properties": { "name": { "type": "string" }, "filter": { "anyOf": [ { "type": "object", "title": "WhereFilterIntersection", "properties": { "where_filters": { "type": "array", "items": { "type": "object", "title": "WhereFilter", "properties": { "where_sql_template": { "type": "string" } }, "additionalProperties": false, "required": [ "where_sql_template" ] } } }, "additionalProperties": false, "required": [ "where_filters" ] }, { "type": "null" } ], "default": null }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "offset_window": { "anyOf": [ { "type": "object", "title": "MetricTimeWindow", "properties": { "count": { "type": "integer" }, "granularity": { "type": "string" } }, "additionalProperties": false, "required": [ "count", "granularity" ] }, { "type": "null" } ], "default": null }, "offset_to_grain": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] }, { "type": "null" } ], "default": null }, "conversion_metric": { "anyOf": [ { "type": "object", "title": "MetricInput", "properties": { "name": { "type": "string" }, "filter": { "anyOf": [ { "type": "object", "title": "WhereFilterIntersection", "properties": { "where_filters": { "type": "array", "items": { "type": "object", "title": "WhereFilter", "properties": { "where_sql_template": { "type": "string" } }, "additionalProperties": false, "required": [ "where_sql_template" ] } } }, "additionalProperties": false, "required": [ "where_filters" ] }, { "type": "null" } ], "default": null }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "offset_window": { "anyOf": [ { "type": "object", "title": "MetricTimeWindow", "properties": { "count": { "type": "integer" }, "granularity": { "type": "string" } }, "additionalProperties": false, "required": [ "count", "granularity" ] }, { "type": "null" } ], "default": null }, "offset_to_grain": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] }, { "type": "null" } ], "default": null }, "calculation": { "enum": [ "conversions", "conversion_rate" ], "default": "conversion_rate" }, "window": { "anyOf": [ { "type": "object", "title": "MetricTimeWindow", "properties": { "count": { "type": "integer" }, "granularity": { "type": "string" } }, "additionalProperties": false, "required": [ "count", "granularity" ] }, { "type": "null" } ], "default": null }, "constant_properties": { "anyOf": [ { "type": "array", "items": { "type": "object", "title": "ConstantPropertyInput", "properties": { "base_property": { "type": "string" }, "conversion_property": { "type": "string" } }, "additionalProperties": false, "required": [ "base_property", "conversion_property" ] } }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "entity" ] }, { "type": "null" } ], "default": null }, "cumulative_type_params": { "anyOf": [ { "type": "object", "title": "CumulativeTypeParams", "properties": { "window": { "anyOf": [ { "type": "object", "title": "MetricTimeWindow", "properties": { "count": { "type": "integer" }, "granularity": { "type": "string" } }, "additionalProperties": false, "required": [ "count", "granularity" ] }, { "type": "null" } ], "default": null }, "grain_to_date": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "period_agg": { "enum": [ "first", "last", "average" ], "default": "first" }, "metric": { "anyOf": [ { "type": "object", "title": "MetricInput", "properties": { "name": { "type": "string" }, "filter": { "anyOf": [ { "type": "object", "title": "WhereFilterIntersection", "properties": { "where_filters": { "type": "array", "items": { "type": "object", "title": "WhereFilter", "properties": { "where_sql_template": { "type": "string" } }, "additionalProperties": false, "required": [ "where_sql_template" ] } } }, "additionalProperties": false, "required": [ "where_filters" ] }, { "type": "null" } ], "default": null }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "offset_window": { "anyOf": [ { "type": "object", "title": "MetricTimeWindow", "properties": { "count": { "type": "integer" }, "granularity": { "type": "string" } }, "additionalProperties": false, "required": [ "count", "granularity" ] }, { "type": "null" } ], "default": null }, "offset_to_grain": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, { "type": "null" } ], "default": null }, "metric_aggregation_params": { "anyOf": [ { "type": "object", "title": "MetricAggregationParams", "properties": { "semantic_model": { "type": "string" }, "agg": { "enum": [ "sum", "min", "max", "count_distinct", "sum_boolean", "average", "percentile", "median", "count" ] }, "agg_params": { "anyOf": [ { "type": "object", "title": "MeasureAggregationParameters", "properties": { "percentile": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "default": null }, "use_discrete_percentile": { "type": "boolean", "default": false }, "use_approximate_percentile": { "type": "boolean", "default": false } }, "additionalProperties": false }, { "type": "null" } ], "default": null }, "agg_time_dimension": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "non_additive_dimension": { "anyOf": [ { "type": "object", "title": "NonAdditiveDimension", "properties": { "name": { "type": "string" }, "window_choice": { "enum": [ "sum", "min", "max", "count_distinct", "sum_boolean", "average", "percentile", "median", "count" ] }, "window_groupings": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false, "required": [ "name", "window_choice", "window_groupings" ] }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "semantic_model", "agg" ] }, { "type": "null" } ], "default": null }, "fill_nulls_with": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null }, "join_to_timespine": { "type": "boolean", "default": false }, "is_private": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "filter": { "anyOf": [ { "type": "object", "title": "WhereFilterIntersection", "properties": { "where_filters": { "type": "array", "items": { "type": "object", "title": "WhereFilter", "properties": { "where_sql_template": { "type": "string" } }, "additionalProperties": false, "required": [ "where_sql_template" ] } } }, "additionalProperties": false, "required": [ "where_filters" ] }, { "type": "null" } ], "default": null }, "metadata": { "anyOf": [ { "type": "object", "title": "SourceFileMetadata", "properties": { "repo_file_path": { "type": "string" }, "file_slice": { "type": "object", "title": "FileSlice", "properties": { "filename": { "type": "string" }, "content": { "type": "string" }, "start_line_number": { "type": "integer" }, "end_line_number": { "type": "integer" } }, "additionalProperties": false, "required": [ "filename", "content", "start_line_number", "end_line_number" ] } }, "additionalProperties": false, "required": [ "repo_file_path", "file_slice" ] }, { "type": "null" } ], "default": null }, "time_granularity": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "title": "MetricConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "meta": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": true }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "depends_on": { "type": "object", "title": "DependsOn", "properties": { "macros": { "type": "array", "items": { "type": "string" } }, "nodes": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false }, "refs": { "type": "array", "items": { "type": "object", "title": "RefArgs", "properties": { "name": { "type": "string" }, "package": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "version": { "anyOf": [ { "type": "string" }, { "type": "number" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] } }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "created_at": { "type": "number" }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false, "required": [ "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "description", "label", "type", "type_params" ] }, "propertyNames": { "type": "string" } }, "groups": { "type": "object", "description": "The groups defined in the dbt project", "additionalProperties": { "type": "object", "title": "Group", "properties": { "name": { "type": "string" }, "resource_type": { "const": "group" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "owner": { "type": "object", "title": "Owner", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "email": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ], "default": null }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": true }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "title": "GroupConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": true } }, "additionalProperties": false, "required": [ "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "owner" ] }, "propertyNames": { "type": "string" } }, "selectors": { "type": "object", "description": "The selectors defined in selectors.yml", "propertyNames": { "type": "string" } }, "disabled": { "description": "A mapping of the disabled nodes in the target", "anyOf": [ { "type": "object", "additionalProperties": { "type": "array", "items": { "anyOf": [ { "type": "object", "title": "Seed", "properties": { "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "const": "seed" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "type": "object", "title": "FileHash", "properties": { "name": { "type": "string" }, "checksum": { "type": "string" } }, "additionalProperties": false, "required": [ "name", "checksum" ] }, "config": { "type": "object", "title": "SeedConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "tags": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "materialized": { "type": "string", "default": "seed" }, "incremental_strategy": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "batch_size": { "default": null }, "lookback": { "default": 1 }, "begin": { "default": null }, "persist_docs": { "type": "object", "propertyNames": { "type": "string" } }, "post-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "pre-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "quoting": { "type": "object", "propertyNames": { "type": "string" } }, "column_types": { "type": "object", "propertyNames": { "type": "string" } }, "full_refresh": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "unique_key": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ], "default": null }, "on_schema_change": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "on_configuration_change": { "enum": [ "apply", "continue", "fail" ] }, "grants": { "type": "object", "propertyNames": { "type": "string" } }, "packages": { "type": "array", "items": { "type": "string" } }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "contract": { "type": "object", "title": "ContractConfig", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true } }, "additionalProperties": false }, "event_time": { "default": null }, "concurrent_batches": { "default": null }, "delimiter": { "type": "string", "default": "," }, "quote_columns": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null } }, "additionalProperties": true }, "tags": { "type": "array", "items": { "type": "string" } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "type": "object", "title": "ColumnInfo", "properties": { "name": { "type": "string" }, "description": { "type": "string", "default": "" }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "data_type": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "constraints": { "type": "array", "items": { "type": "object", "title": "ColumnLevelConstraint", "properties": { "type": { "enum": [ "check", "not_null", "unique", "primary_key", "foreign_key", "custom" ] }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "expression": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "warn_unenforced": { "type": "boolean", "default": true }, "warn_unsupported": { "type": "boolean", "default": true }, "to": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "to_columns": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false, "required": [ "type" ] } }, "quote": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "title": "ColumnConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true }, "tags": { "type": "array", "items": { "type": "string" } }, "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "granularity": { "anyOf": [ { "enum": [ "nanosecond", "microsecond", "millisecond", "second", "minute", "hour", "day", "week", "month", "quarter", "year" ] }, { "type": "null" } ], "default": null }, "dimension": { "anyOf": [ { "type": "object", "title": "ColumnDimension", "properties": { "name": { "type": "string" }, "type": { "enum": [ "categorical", "time" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "is_partition": { "type": "boolean", "default": false }, "config": { "type": "object", "propertyNames": { "type": "string" } }, "validity_params": { "anyOf": [ { "type": "object", "title": "ColumnDimensionValidityParams", "properties": { "is_start": { "type": "boolean", "default": false }, "is_end": { "type": "boolean", "default": false } }, "additionalProperties": false }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "categorical", "time" ] }, { "type": "null" } ], "default": null }, "entity": { "anyOf": [ { "type": "object", "title": "ColumnEntity", "properties": { "name": { "type": "string" }, "type": { "enum": [ "foreign", "natural", "primary", "unique" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "foreign", "natural", "primary", "unique" ] }, { "type": "null" } ], "default": null }, "doc_blocks": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true, "required": [ "name" ] }, "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "patch_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "build_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "created_at": { "type": "number" }, "config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "unrendered_config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "raw_code": { "type": "string", "default": "" }, "doc_blocks": { "type": "array", "items": { "type": "string" } }, "root_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "depends_on": { "type": "object", "title": "MacroDependsOn", "properties": { "macros": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false }, "defer_relation": { "anyOf": [ { "type": "object", "title": "DeferRelation", "properties": { "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "alias": { "type": "string" }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "resource_type": { "enum": [ "model", "analysis", "test", "snapshot", "operation", "seed", "rpc", "sql_operation", "doc", "source", "macro", "exposure", "metric", "group", "saved_query", "semantic_model", "unit_test", "fixture", "function" ] }, "name": { "type": "string" }, "description": { "type": "string" }, "compiled_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } }, "config": { "anyOf": [ { "type": "object", "title": "NodeConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "tags": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "materialized": { "type": "string", "default": "view" }, "incremental_strategy": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "batch_size": { "default": null }, "lookback": { "default": 1 }, "begin": { "default": null }, "persist_docs": { "type": "object", "propertyNames": { "type": "string" } }, "post-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "pre-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "quoting": { "type": "object", "propertyNames": { "type": "string" } }, "column_types": { "type": "object", "propertyNames": { "type": "string" } }, "full_refresh": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "unique_key": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ], "default": null }, "on_schema_change": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "on_configuration_change": { "enum": [ "apply", "continue", "fail" ] }, "grants": { "type": "object", "propertyNames": { "type": "string" } }, "packages": { "type": "array", "items": { "type": "string" } }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "contract": { "type": "object", "title": "ContractConfig", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true } }, "additionalProperties": false }, "event_time": { "default": null }, "concurrent_batches": { "default": null } }, "additionalProperties": true }, { "type": "null" } ] } }, "additionalProperties": false, "required": [ "database", "schema", "alias", "relation_name", "resource_type", "name", "description", "compiled_code", "meta", "tags", "config" ] }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "database", "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ] }, { "type": "object", "title": "Analysis", "properties": { "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "const": "analysis" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "type": "object", "title": "FileHash", "properties": { "name": { "type": "string" }, "checksum": { "type": "string" } }, "additionalProperties": false, "required": [ "name", "checksum" ] }, "config": { "type": "object", "title": "NodeConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "tags": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "materialized": { "type": "string", "default": "view" }, "incremental_strategy": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "batch_size": { "default": null }, "lookback": { "default": 1 }, "begin": { "default": null }, "persist_docs": { "type": "object", "propertyNames": { "type": "string" } }, "post-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "pre-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "quoting": { "type": "object", "propertyNames": { "type": "string" } }, "column_types": { "type": "object", "propertyNames": { "type": "string" } }, "full_refresh": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "unique_key": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ], "default": null }, "on_schema_change": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "on_configuration_change": { "enum": [ "apply", "continue", "fail" ] }, "grants": { "type": "object", "propertyNames": { "type": "string" } }, "packages": { "type": "array", "items": { "type": "string" } }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "contract": { "type": "object", "title": "ContractConfig", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true } }, "additionalProperties": false }, "event_time": { "default": null }, "concurrent_batches": { "default": null } }, "additionalProperties": true }, "tags": { "type": "array", "items": { "type": "string" } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "type": "object", "title": "ColumnInfo", "properties": { "name": { "type": "string" }, "description": { "type": "string", "default": "" }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "data_type": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "constraints": { "type": "array", "items": { "type": "object", "title": "ColumnLevelConstraint", "properties": { "type": { "enum": [ "check", "not_null", "unique", "primary_key", "foreign_key", "custom" ] }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "expression": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "warn_unenforced": { "type": "boolean", "default": true }, "warn_unsupported": { "type": "boolean", "default": true }, "to": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "to_columns": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false, "required": [ "type" ] } }, "quote": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "title": "ColumnConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true }, "tags": { "type": "array", "items": { "type": "string" } }, "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "granularity": { "anyOf": [ { "enum": [ "nanosecond", "microsecond", "millisecond", "second", "minute", "hour", "day", "week", "month", "quarter", "year" ] }, { "type": "null" } ], "default": null }, "dimension": { "anyOf": [ { "type": "object", "title": "ColumnDimension", "properties": { "name": { "type": "string" }, "type": { "enum": [ "categorical", "time" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "is_partition": { "type": "boolean", "default": false }, "config": { "type": "object", "propertyNames": { "type": "string" } }, "validity_params": { "anyOf": [ { "type": "object", "title": "ColumnDimensionValidityParams", "properties": { "is_start": { "type": "boolean", "default": false }, "is_end": { "type": "boolean", "default": false } }, "additionalProperties": false }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "categorical", "time" ] }, { "type": "null" } ], "default": null }, "entity": { "anyOf": [ { "type": "object", "title": "ColumnEntity", "properties": { "name": { "type": "string" }, "type": { "enum": [ "foreign", "natural", "primary", "unique" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "foreign", "natural", "primary", "unique" ] }, { "type": "null" } ], "default": null }, "doc_blocks": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true, "required": [ "name" ] }, "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "patch_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "build_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "created_at": { "type": "number" }, "config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "unrendered_config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "raw_code": { "type": "string", "default": "" }, "doc_blocks": { "type": "array", "items": { "type": "string" } }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "type": "object", "title": "RefArgs", "properties": { "name": { "type": "string" }, "package": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "version": { "anyOf": [ { "type": "string" }, { "type": "number" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] } }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "functions": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "depends_on": { "type": "object", "title": "DependsOn", "properties": { "macros": { "type": "array", "items": { "type": "string" } }, "nodes": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false }, "compiled_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "type": "object", "title": "InjectedCTE", "properties": { "id": { "type": "string" }, "sql": { "type": "string" } }, "additionalProperties": false, "required": [ "id", "sql" ] } }, "_pre_injected_sql": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "contract": { "type": "object", "title": "Contract", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true }, "checksum": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false } }, "additionalProperties": false, "required": [ "database", "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ] }, { "type": "object", "title": "SingularTest", "properties": { "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "const": "test" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "type": "object", "title": "FileHash", "properties": { "name": { "type": "string" }, "checksum": { "type": "string" } }, "additionalProperties": false, "required": [ "name", "checksum" ] }, "config": { "type": "object", "title": "TestConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": "dbt_test__audit" }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "tags": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "materialized": { "type": "string", "default": "test" }, "severity": { "type": "string", "default": "ERROR", "pattern": "^([Ww][Aa][Rr][Nn]|[Ee][Rr][Rr][Oo][Rr])$" }, "store_failures": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "store_failures_as": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "sql_header": { "default": null }, "where": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "limit": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null }, "fail_calc": { "type": "string", "default": "count(*)" }, "warn_if": { "type": "string", "default": "!= 0" }, "error_if": { "type": "string", "default": "!= 0" } }, "additionalProperties": true }, "tags": { "type": "array", "items": { "type": "string" } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "type": "object", "title": "ColumnInfo", "properties": { "name": { "type": "string" }, "description": { "type": "string", "default": "" }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "data_type": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "constraints": { "type": "array", "items": { "type": "object", "title": "ColumnLevelConstraint", "properties": { "type": { "enum": [ "check", "not_null", "unique", "primary_key", "foreign_key", "custom" ] }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "expression": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "warn_unenforced": { "type": "boolean", "default": true }, "warn_unsupported": { "type": "boolean", "default": true }, "to": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "to_columns": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false, "required": [ "type" ] } }, "quote": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "title": "ColumnConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true }, "tags": { "type": "array", "items": { "type": "string" } }, "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "granularity": { "anyOf": [ { "enum": [ "nanosecond", "microsecond", "millisecond", "second", "minute", "hour", "day", "week", "month", "quarter", "year" ] }, { "type": "null" } ], "default": null }, "dimension": { "anyOf": [ { "type": "object", "title": "ColumnDimension", "properties": { "name": { "type": "string" }, "type": { "enum": [ "categorical", "time" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "is_partition": { "type": "boolean", "default": false }, "config": { "type": "object", "propertyNames": { "type": "string" } }, "validity_params": { "anyOf": [ { "type": "object", "title": "ColumnDimensionValidityParams", "properties": { "is_start": { "type": "boolean", "default": false }, "is_end": { "type": "boolean", "default": false } }, "additionalProperties": false }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "categorical", "time" ] }, { "type": "null" } ], "default": null }, "entity": { "anyOf": [ { "type": "object", "title": "ColumnEntity", "properties": { "name": { "type": "string" }, "type": { "enum": [ "foreign", "natural", "primary", "unique" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "foreign", "natural", "primary", "unique" ] }, { "type": "null" } ], "default": null }, "doc_blocks": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true, "required": [ "name" ] }, "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "patch_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "build_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "created_at": { "type": "number" }, "config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "unrendered_config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "raw_code": { "type": "string", "default": "" }, "doc_blocks": { "type": "array", "items": { "type": "string" } }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "type": "object", "title": "RefArgs", "properties": { "name": { "type": "string" }, "package": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "version": { "anyOf": [ { "type": "string" }, { "type": "number" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] } }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "functions": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "depends_on": { "type": "object", "title": "DependsOn", "properties": { "macros": { "type": "array", "items": { "type": "string" } }, "nodes": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false }, "compiled_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "type": "object", "title": "InjectedCTE", "properties": { "id": { "type": "string" }, "sql": { "type": "string" } }, "additionalProperties": false, "required": [ "id", "sql" ] } }, "_pre_injected_sql": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "contract": { "type": "object", "title": "Contract", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true }, "checksum": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false } }, "additionalProperties": false, "required": [ "database", "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ] }, { "type": "object", "title": "HookNode", "properties": { "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "const": "operation" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "type": "object", "title": "FileHash", "properties": { "name": { "type": "string" }, "checksum": { "type": "string" } }, "additionalProperties": false, "required": [ "name", "checksum" ] }, "config": { "type": "object", "title": "NodeConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "tags": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "materialized": { "type": "string", "default": "view" }, "incremental_strategy": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "batch_size": { "default": null }, "lookback": { "default": 1 }, "begin": { "default": null }, "persist_docs": { "type": "object", "propertyNames": { "type": "string" } }, "post-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "pre-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "quoting": { "type": "object", "propertyNames": { "type": "string" } }, "column_types": { "type": "object", "propertyNames": { "type": "string" } }, "full_refresh": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "unique_key": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ], "default": null }, "on_schema_change": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "on_configuration_change": { "enum": [ "apply", "continue", "fail" ] }, "grants": { "type": "object", "propertyNames": { "type": "string" } }, "packages": { "type": "array", "items": { "type": "string" } }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "contract": { "type": "object", "title": "ContractConfig", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true } }, "additionalProperties": false }, "event_time": { "default": null }, "concurrent_batches": { "default": null } }, "additionalProperties": true }, "tags": { "type": "array", "items": { "type": "string" } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "type": "object", "title": "ColumnInfo", "properties": { "name": { "type": "string" }, "description": { "type": "string", "default": "" }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "data_type": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "constraints": { "type": "array", "items": { "type": "object", "title": "ColumnLevelConstraint", "properties": { "type": { "enum": [ "check", "not_null", "unique", "primary_key", "foreign_key", "custom" ] }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "expression": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "warn_unenforced": { "type": "boolean", "default": true }, "warn_unsupported": { "type": "boolean", "default": true }, "to": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "to_columns": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false, "required": [ "type" ] } }, "quote": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "title": "ColumnConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true }, "tags": { "type": "array", "items": { "type": "string" } }, "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "granularity": { "anyOf": [ { "enum": [ "nanosecond", "microsecond", "millisecond", "second", "minute", "hour", "day", "week", "month", "quarter", "year" ] }, { "type": "null" } ], "default": null }, "dimension": { "anyOf": [ { "type": "object", "title": "ColumnDimension", "properties": { "name": { "type": "string" }, "type": { "enum": [ "categorical", "time" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "is_partition": { "type": "boolean", "default": false }, "config": { "type": "object", "propertyNames": { "type": "string" } }, "validity_params": { "anyOf": [ { "type": "object", "title": "ColumnDimensionValidityParams", "properties": { "is_start": { "type": "boolean", "default": false }, "is_end": { "type": "boolean", "default": false } }, "additionalProperties": false }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "categorical", "time" ] }, { "type": "null" } ], "default": null }, "entity": { "anyOf": [ { "type": "object", "title": "ColumnEntity", "properties": { "name": { "type": "string" }, "type": { "enum": [ "foreign", "natural", "primary", "unique" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "foreign", "natural", "primary", "unique" ] }, { "type": "null" } ], "default": null }, "doc_blocks": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true, "required": [ "name" ] }, "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "patch_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "build_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "created_at": { "type": "number" }, "config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "unrendered_config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "raw_code": { "type": "string", "default": "" }, "doc_blocks": { "type": "array", "items": { "type": "string" } }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "type": "object", "title": "RefArgs", "properties": { "name": { "type": "string" }, "package": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "version": { "anyOf": [ { "type": "string" }, { "type": "number" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] } }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "functions": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "depends_on": { "type": "object", "title": "DependsOn", "properties": { "macros": { "type": "array", "items": { "type": "string" } }, "nodes": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false }, "compiled_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "type": "object", "title": "InjectedCTE", "properties": { "id": { "type": "string" }, "sql": { "type": "string" } }, "additionalProperties": false, "required": [ "id", "sql" ] } }, "_pre_injected_sql": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "contract": { "type": "object", "title": "Contract", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true }, "checksum": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "database", "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ] }, { "type": "object", "title": "Model", "properties": { "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "const": "model" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "type": "object", "title": "FileHash", "properties": { "name": { "type": "string" }, "checksum": { "type": "string" } }, "additionalProperties": false, "required": [ "name", "checksum" ] }, "config": { "type": "object", "title": "ModelConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "tags": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "materialized": { "type": "string", "default": "view" }, "incremental_strategy": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "batch_size": { "default": null }, "lookback": { "default": 1 }, "begin": { "default": null }, "persist_docs": { "type": "object", "propertyNames": { "type": "string" } }, "post-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "pre-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "quoting": { "type": "object", "propertyNames": { "type": "string" } }, "column_types": { "type": "object", "propertyNames": { "type": "string" } }, "full_refresh": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "unique_key": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ], "default": null }, "on_schema_change": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "on_configuration_change": { "enum": [ "apply", "continue", "fail" ] }, "grants": { "type": "object", "propertyNames": { "type": "string" } }, "packages": { "type": "array", "items": { "type": "string" } }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "contract": { "type": "object", "title": "ContractConfig", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true } }, "additionalProperties": false }, "event_time": { "default": null }, "concurrent_batches": { "default": null }, "access": { "enum": [ "private", "protected", "public" ], "default": "protected" }, "freshness": { "anyOf": [ { "type": "object", "title": "ModelFreshness", "properties": { "build_after": { "type": "object", "title": "ModelBuildAfter", "properties": { "count": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null }, "period": { "anyOf": [ { "enum": [ "minute", "hour", "day" ] }, { "type": "null" } ], "default": null }, "updates_on": { "enum": [ "all", "any" ], "default": "any" } }, "additionalProperties": true } }, "additionalProperties": true, "required": [ "build_after" ] }, { "type": "null" } ], "default": null } }, "additionalProperties": true }, "tags": { "type": "array", "items": { "type": "string" } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "type": "object", "title": "ColumnInfo", "properties": { "name": { "type": "string" }, "description": { "type": "string", "default": "" }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "data_type": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "constraints": { "type": "array", "items": { "type": "object", "title": "ColumnLevelConstraint", "properties": { "type": { "enum": [ "check", "not_null", "unique", "primary_key", "foreign_key", "custom" ] }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "expression": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "warn_unenforced": { "type": "boolean", "default": true }, "warn_unsupported": { "type": "boolean", "default": true }, "to": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "to_columns": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false, "required": [ "type" ] } }, "quote": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "title": "ColumnConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true }, "tags": { "type": "array", "items": { "type": "string" } }, "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "granularity": { "anyOf": [ { "enum": [ "nanosecond", "microsecond", "millisecond", "second", "minute", "hour", "day", "week", "month", "quarter", "year" ] }, { "type": "null" } ], "default": null }, "dimension": { "anyOf": [ { "type": "object", "title": "ColumnDimension", "properties": { "name": { "type": "string" }, "type": { "enum": [ "categorical", "time" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "is_partition": { "type": "boolean", "default": false }, "config": { "type": "object", "propertyNames": { "type": "string" } }, "validity_params": { "anyOf": [ { "type": "object", "title": "ColumnDimensionValidityParams", "properties": { "is_start": { "type": "boolean", "default": false }, "is_end": { "type": "boolean", "default": false } }, "additionalProperties": false }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "categorical", "time" ] }, { "type": "null" } ], "default": null }, "entity": { "anyOf": [ { "type": "object", "title": "ColumnEntity", "properties": { "name": { "type": "string" }, "type": { "enum": [ "foreign", "natural", "primary", "unique" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "foreign", "natural", "primary", "unique" ] }, { "type": "null" } ], "default": null }, "doc_blocks": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true, "required": [ "name" ] }, "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "patch_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "build_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "created_at": { "type": "number" }, "config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "unrendered_config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "raw_code": { "type": "string", "default": "" }, "doc_blocks": { "type": "array", "items": { "type": "string" } }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "type": "object", "title": "RefArgs", "properties": { "name": { "type": "string" }, "package": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "version": { "anyOf": [ { "type": "string" }, { "type": "number" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] } }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "functions": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "depends_on": { "type": "object", "title": "DependsOn", "properties": { "macros": { "type": "array", "items": { "type": "string" } }, "nodes": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false }, "compiled_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "type": "object", "title": "InjectedCTE", "properties": { "id": { "type": "string" }, "sql": { "type": "string" } }, "additionalProperties": false, "required": [ "id", "sql" ] } }, "_pre_injected_sql": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "contract": { "type": "object", "title": "Contract", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true }, "checksum": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "access": { "enum": [ "private", "protected", "public" ], "default": "protected" }, "constraints": { "type": "array", "items": { "type": "object", "title": "ModelLevelConstraint", "properties": { "type": { "enum": [ "check", "not_null", "unique", "primary_key", "foreign_key", "custom" ] }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "expression": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "warn_unenforced": { "type": "boolean", "default": true }, "warn_unsupported": { "type": "boolean", "default": true }, "to": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "to_columns": { "type": "array", "items": { "type": "string" } }, "columns": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false, "required": [ "type" ] } }, "version": { "anyOf": [ { "type": "string" }, { "type": "number" }, { "type": "null" } ], "default": null }, "latest_version": { "anyOf": [ { "type": "string" }, { "type": "number" }, { "type": "null" } ], "default": null }, "deprecation_date": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "defer_relation": { "anyOf": [ { "type": "object", "title": "DeferRelation", "properties": { "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "alias": { "type": "string" }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "resource_type": { "enum": [ "model", "analysis", "test", "snapshot", "operation", "seed", "rpc", "sql_operation", "doc", "source", "macro", "exposure", "metric", "group", "saved_query", "semantic_model", "unit_test", "fixture", "function" ] }, "name": { "type": "string" }, "description": { "type": "string" }, "compiled_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } }, "config": { "anyOf": [ { "type": "object", "title": "NodeConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "tags": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "materialized": { "type": "string", "default": "view" }, "incremental_strategy": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "batch_size": { "default": null }, "lookback": { "default": 1 }, "begin": { "default": null }, "persist_docs": { "type": "object", "propertyNames": { "type": "string" } }, "post-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "pre-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "quoting": { "type": "object", "propertyNames": { "type": "string" } }, "column_types": { "type": "object", "propertyNames": { "type": "string" } }, "full_refresh": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "unique_key": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ], "default": null }, "on_schema_change": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "on_configuration_change": { "enum": [ "apply", "continue", "fail" ] }, "grants": { "type": "object", "propertyNames": { "type": "string" } }, "packages": { "type": "array", "items": { "type": "string" } }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "contract": { "type": "object", "title": "ContractConfig", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true } }, "additionalProperties": false }, "event_time": { "default": null }, "concurrent_batches": { "default": null } }, "additionalProperties": true }, { "type": "null" } ] } }, "additionalProperties": false, "required": [ "database", "schema", "alias", "relation_name", "resource_type", "name", "description", "compiled_code", "meta", "tags", "config" ] }, { "type": "null" } ], "default": null }, "primary_key": { "type": "array", "items": { "type": "string" } }, "time_spine": { "anyOf": [ { "type": "object", "title": "TimeSpine", "properties": { "standard_granularity_column": { "type": "string" }, "custom_granularities": { "type": "array", "items": { "type": "object", "title": "CustomGranularity", "properties": { "name": { "type": "string" }, "column_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] } } }, "additionalProperties": false, "required": [ "standard_granularity_column" ] }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "database", "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ] }, { "type": "object", "title": "SqlOperation", "properties": { "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "const": "sql_operation" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "type": "object", "title": "FileHash", "properties": { "name": { "type": "string" }, "checksum": { "type": "string" } }, "additionalProperties": false, "required": [ "name", "checksum" ] }, "config": { "type": "object", "title": "NodeConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "tags": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "materialized": { "type": "string", "default": "view" }, "incremental_strategy": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "batch_size": { "default": null }, "lookback": { "default": 1 }, "begin": { "default": null }, "persist_docs": { "type": "object", "propertyNames": { "type": "string" } }, "post-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "pre-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "quoting": { "type": "object", "propertyNames": { "type": "string" } }, "column_types": { "type": "object", "propertyNames": { "type": "string" } }, "full_refresh": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "unique_key": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ], "default": null }, "on_schema_change": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "on_configuration_change": { "enum": [ "apply", "continue", "fail" ] }, "grants": { "type": "object", "propertyNames": { "type": "string" } }, "packages": { "type": "array", "items": { "type": "string" } }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "contract": { "type": "object", "title": "ContractConfig", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true } }, "additionalProperties": false }, "event_time": { "default": null }, "concurrent_batches": { "default": null } }, "additionalProperties": true }, "tags": { "type": "array", "items": { "type": "string" } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "type": "object", "title": "ColumnInfo", "properties": { "name": { "type": "string" }, "description": { "type": "string", "default": "" }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "data_type": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "constraints": { "type": "array", "items": { "type": "object", "title": "ColumnLevelConstraint", "properties": { "type": { "enum": [ "check", "not_null", "unique", "primary_key", "foreign_key", "custom" ] }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "expression": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "warn_unenforced": { "type": "boolean", "default": true }, "warn_unsupported": { "type": "boolean", "default": true }, "to": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "to_columns": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false, "required": [ "type" ] } }, "quote": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "title": "ColumnConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true }, "tags": { "type": "array", "items": { "type": "string" } }, "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "granularity": { "anyOf": [ { "enum": [ "nanosecond", "microsecond", "millisecond", "second", "minute", "hour", "day", "week", "month", "quarter", "year" ] }, { "type": "null" } ], "default": null }, "dimension": { "anyOf": [ { "type": "object", "title": "ColumnDimension", "properties": { "name": { "type": "string" }, "type": { "enum": [ "categorical", "time" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "is_partition": { "type": "boolean", "default": false }, "config": { "type": "object", "propertyNames": { "type": "string" } }, "validity_params": { "anyOf": [ { "type": "object", "title": "ColumnDimensionValidityParams", "properties": { "is_start": { "type": "boolean", "default": false }, "is_end": { "type": "boolean", "default": false } }, "additionalProperties": false }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "categorical", "time" ] }, { "type": "null" } ], "default": null }, "entity": { "anyOf": [ { "type": "object", "title": "ColumnEntity", "properties": { "name": { "type": "string" }, "type": { "enum": [ "foreign", "natural", "primary", "unique" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "foreign", "natural", "primary", "unique" ] }, { "type": "null" } ], "default": null }, "doc_blocks": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true, "required": [ "name" ] }, "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "patch_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "build_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "created_at": { "type": "number" }, "config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "unrendered_config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "raw_code": { "type": "string", "default": "" }, "doc_blocks": { "type": "array", "items": { "type": "string" } }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "type": "object", "title": "RefArgs", "properties": { "name": { "type": "string" }, "package": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "version": { "anyOf": [ { "type": "string" }, { "type": "number" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] } }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "functions": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "depends_on": { "type": "object", "title": "DependsOn", "properties": { "macros": { "type": "array", "items": { "type": "string" } }, "nodes": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false }, "compiled_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "type": "object", "title": "InjectedCTE", "properties": { "id": { "type": "string" }, "sql": { "type": "string" } }, "additionalProperties": false, "required": [ "id", "sql" ] } }, "_pre_injected_sql": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "contract": { "type": "object", "title": "Contract", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true }, "checksum": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false } }, "additionalProperties": false, "required": [ "database", "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ] }, { "type": "object", "title": "GenericTest", "properties": { "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "const": "test" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "type": "object", "title": "FileHash", "properties": { "name": { "type": "string" }, "checksum": { "type": "string" } }, "additionalProperties": false, "required": [ "name", "checksum" ] }, "config": { "type": "object", "title": "TestConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": "dbt_test__audit" }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "tags": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "materialized": { "type": "string", "default": "test" }, "severity": { "type": "string", "default": "ERROR", "pattern": "^([Ww][Aa][Rr][Nn]|[Ee][Rr][Rr][Oo][Rr])$" }, "store_failures": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "store_failures_as": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "sql_header": { "default": null }, "where": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "limit": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null }, "fail_calc": { "type": "string", "default": "count(*)" }, "warn_if": { "type": "string", "default": "!= 0" }, "error_if": { "type": "string", "default": "!= 0" } }, "additionalProperties": true }, "tags": { "type": "array", "items": { "type": "string" } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "type": "object", "title": "ColumnInfo", "properties": { "name": { "type": "string" }, "description": { "type": "string", "default": "" }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "data_type": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "constraints": { "type": "array", "items": { "type": "object", "title": "ColumnLevelConstraint", "properties": { "type": { "enum": [ "check", "not_null", "unique", "primary_key", "foreign_key", "custom" ] }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "expression": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "warn_unenforced": { "type": "boolean", "default": true }, "warn_unsupported": { "type": "boolean", "default": true }, "to": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "to_columns": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false, "required": [ "type" ] } }, "quote": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "title": "ColumnConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true }, "tags": { "type": "array", "items": { "type": "string" } }, "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "granularity": { "anyOf": [ { "enum": [ "nanosecond", "microsecond", "millisecond", "second", "minute", "hour", "day", "week", "month", "quarter", "year" ] }, { "type": "null" } ], "default": null }, "dimension": { "anyOf": [ { "type": "object", "title": "ColumnDimension", "properties": { "name": { "type": "string" }, "type": { "enum": [ "categorical", "time" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "is_partition": { "type": "boolean", "default": false }, "config": { "type": "object", "propertyNames": { "type": "string" } }, "validity_params": { "anyOf": [ { "type": "object", "title": "ColumnDimensionValidityParams", "properties": { "is_start": { "type": "boolean", "default": false }, "is_end": { "type": "boolean", "default": false } }, "additionalProperties": false }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "categorical", "time" ] }, { "type": "null" } ], "default": null }, "entity": { "anyOf": [ { "type": "object", "title": "ColumnEntity", "properties": { "name": { "type": "string" }, "type": { "enum": [ "foreign", "natural", "primary", "unique" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "foreign", "natural", "primary", "unique" ] }, { "type": "null" } ], "default": null }, "doc_blocks": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true, "required": [ "name" ] }, "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "patch_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "build_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "created_at": { "type": "number" }, "config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "unrendered_config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "raw_code": { "type": "string", "default": "" }, "doc_blocks": { "type": "array", "items": { "type": "string" } }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "type": "object", "title": "RefArgs", "properties": { "name": { "type": "string" }, "package": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "version": { "anyOf": [ { "type": "string" }, { "type": "number" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] } }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "functions": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "depends_on": { "type": "object", "title": "DependsOn", "properties": { "macros": { "type": "array", "items": { "type": "string" } }, "nodes": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false }, "compiled_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "type": "object", "title": "InjectedCTE", "properties": { "id": { "type": "string" }, "sql": { "type": "string" } }, "additionalProperties": false, "required": [ "id", "sql" ] } }, "_pre_injected_sql": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "contract": { "type": "object", "title": "Contract", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true }, "checksum": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "column_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "file_key_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "attached_node": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "test_metadata": { "type": "object", "title": "TestMetadata", "properties": { "name": { "type": "string", "default": "test" }, "kwargs": { "type": "object", "propertyNames": { "type": "string" } }, "namespace": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false } }, "additionalProperties": false, "required": [ "database", "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ] }, { "type": "object", "title": "Snapshot", "properties": { "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "const": "snapshot" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "type": "object", "title": "FileHash", "properties": { "name": { "type": "string" }, "checksum": { "type": "string" } }, "additionalProperties": false, "required": [ "name", "checksum" ] }, "config": { "type": "object", "title": "SnapshotConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "tags": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "materialized": { "type": "string", "default": "snapshot" }, "incremental_strategy": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "batch_size": { "default": null }, "lookback": { "default": 1 }, "begin": { "default": null }, "persist_docs": { "type": "object", "propertyNames": { "type": "string" } }, "post-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "pre-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "quoting": { "type": "object", "propertyNames": { "type": "string" } }, "column_types": { "type": "object", "propertyNames": { "type": "string" } }, "full_refresh": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "unique_key": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ], "default": null }, "on_schema_change": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "on_configuration_change": { "enum": [ "apply", "continue", "fail" ] }, "grants": { "type": "object", "propertyNames": { "type": "string" } }, "packages": { "type": "array", "items": { "type": "string" } }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "contract": { "type": "object", "title": "ContractConfig", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true } }, "additionalProperties": false }, "event_time": { "default": null }, "concurrent_batches": { "default": null }, "strategy": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "target_schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "target_database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "updated_at": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "check_cols": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ], "default": null }, "snapshot_meta_column_names": { "type": "object", "title": "SnapshotMetaColumnNames", "properties": { "dbt_valid_to": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "dbt_valid_from": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "dbt_scd_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "dbt_updated_at": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "dbt_is_deleted": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "dbt_valid_to_current": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": true }, "tags": { "type": "array", "items": { "type": "string" } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "type": "object", "title": "ColumnInfo", "properties": { "name": { "type": "string" }, "description": { "type": "string", "default": "" }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "data_type": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "constraints": { "type": "array", "items": { "type": "object", "title": "ColumnLevelConstraint", "properties": { "type": { "enum": [ "check", "not_null", "unique", "primary_key", "foreign_key", "custom" ] }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "expression": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "warn_unenforced": { "type": "boolean", "default": true }, "warn_unsupported": { "type": "boolean", "default": true }, "to": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "to_columns": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false, "required": [ "type" ] } }, "quote": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "title": "ColumnConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true }, "tags": { "type": "array", "items": { "type": "string" } }, "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "granularity": { "anyOf": [ { "enum": [ "nanosecond", "microsecond", "millisecond", "second", "minute", "hour", "day", "week", "month", "quarter", "year" ] }, { "type": "null" } ], "default": null }, "dimension": { "anyOf": [ { "type": "object", "title": "ColumnDimension", "properties": { "name": { "type": "string" }, "type": { "enum": [ "categorical", "time" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "is_partition": { "type": "boolean", "default": false }, "config": { "type": "object", "propertyNames": { "type": "string" } }, "validity_params": { "anyOf": [ { "type": "object", "title": "ColumnDimensionValidityParams", "properties": { "is_start": { "type": "boolean", "default": false }, "is_end": { "type": "boolean", "default": false } }, "additionalProperties": false }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "categorical", "time" ] }, { "type": "null" } ], "default": null }, "entity": { "anyOf": [ { "type": "object", "title": "ColumnEntity", "properties": { "name": { "type": "string" }, "type": { "enum": [ "foreign", "natural", "primary", "unique" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "foreign", "natural", "primary", "unique" ] }, { "type": "null" } ], "default": null }, "doc_blocks": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true, "required": [ "name" ] }, "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "patch_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "build_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "created_at": { "type": "number" }, "config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "unrendered_config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "raw_code": { "type": "string", "default": "" }, "doc_blocks": { "type": "array", "items": { "type": "string" } }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "type": "object", "title": "RefArgs", "properties": { "name": { "type": "string" }, "package": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "version": { "anyOf": [ { "type": "string" }, { "type": "number" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] } }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "functions": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "depends_on": { "type": "object", "title": "DependsOn", "properties": { "macros": { "type": "array", "items": { "type": "string" } }, "nodes": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false }, "compiled_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "type": "object", "title": "InjectedCTE", "properties": { "id": { "type": "string" }, "sql": { "type": "string" } }, "additionalProperties": false, "required": [ "id", "sql" ] } }, "_pre_injected_sql": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "contract": { "type": "object", "title": "Contract", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true }, "checksum": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "defer_relation": { "anyOf": [ { "type": "object", "title": "DeferRelation", "properties": { "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "alias": { "type": "string" }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "resource_type": { "enum": [ "model", "analysis", "test", "snapshot", "operation", "seed", "rpc", "sql_operation", "doc", "source", "macro", "exposure", "metric", "group", "saved_query", "semantic_model", "unit_test", "fixture", "function" ] }, "name": { "type": "string" }, "description": { "type": "string" }, "compiled_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } }, "config": { "anyOf": [ { "type": "object", "title": "NodeConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "tags": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "materialized": { "type": "string", "default": "view" }, "incremental_strategy": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "batch_size": { "default": null }, "lookback": { "default": 1 }, "begin": { "default": null }, "persist_docs": { "type": "object", "propertyNames": { "type": "string" } }, "post-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "pre-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "quoting": { "type": "object", "propertyNames": { "type": "string" } }, "column_types": { "type": "object", "propertyNames": { "type": "string" } }, "full_refresh": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "unique_key": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ], "default": null }, "on_schema_change": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "on_configuration_change": { "enum": [ "apply", "continue", "fail" ] }, "grants": { "type": "object", "propertyNames": { "type": "string" } }, "packages": { "type": "array", "items": { "type": "string" } }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "contract": { "type": "object", "title": "ContractConfig", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true } }, "additionalProperties": false }, "event_time": { "default": null }, "concurrent_batches": { "default": null } }, "additionalProperties": true }, { "type": "null" } ] } }, "additionalProperties": false, "required": [ "database", "schema", "alias", "relation_name", "resource_type", "name", "description", "compiled_code", "meta", "tags", "config" ] }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "database", "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum", "config" ] }, { "type": "object", "title": "Function", "properties": { "returns": { "type": "object", "title": "FunctionReturns", "properties": { "data_type": { "type": "string" }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "data_type" ] }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "const": "function" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "type": "object", "title": "FileHash", "properties": { "name": { "type": "string" }, "checksum": { "type": "string" } }, "additionalProperties": false, "required": [ "name", "checksum" ] }, "config": { "type": "object", "title": "FunctionConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "tags": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "materialized": { "type": "string", "default": "function" }, "incremental_strategy": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "batch_size": { "default": null }, "lookback": { "default": 1 }, "begin": { "default": null }, "persist_docs": { "type": "object", "propertyNames": { "type": "string" } }, "post-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "pre-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "quoting": { "type": "object", "propertyNames": { "type": "string" } }, "column_types": { "type": "object", "propertyNames": { "type": "string" } }, "full_refresh": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "unique_key": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ], "default": null }, "on_schema_change": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "on_configuration_change": { "enum": [ "apply", "continue", "fail" ] }, "grants": { "type": "object", "propertyNames": { "type": "string" } }, "packages": { "type": "array", "items": { "type": "string" } }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "contract": { "type": "object", "title": "ContractConfig", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true } }, "additionalProperties": false }, "event_time": { "default": null }, "concurrent_batches": { "default": null }, "type": { "enum": [ "scalar", "aggregate", "table" ], "default": "scalar" }, "volatility": { "anyOf": [ { "enum": [ "deterministic", "stable", "non-deterministic" ] }, { "type": "null" } ], "default": null }, "runtime_version": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "entry_point": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": true }, "tags": { "type": "array", "items": { "type": "string" } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "type": "object", "title": "ColumnInfo", "properties": { "name": { "type": "string" }, "description": { "type": "string", "default": "" }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "data_type": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "constraints": { "type": "array", "items": { "type": "object", "title": "ColumnLevelConstraint", "properties": { "type": { "enum": [ "check", "not_null", "unique", "primary_key", "foreign_key", "custom" ] }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "expression": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "warn_unenforced": { "type": "boolean", "default": true }, "warn_unsupported": { "type": "boolean", "default": true }, "to": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "to_columns": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false, "required": [ "type" ] } }, "quote": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "title": "ColumnConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true }, "tags": { "type": "array", "items": { "type": "string" } }, "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "granularity": { "anyOf": [ { "enum": [ "nanosecond", "microsecond", "millisecond", "second", "minute", "hour", "day", "week", "month", "quarter", "year" ] }, { "type": "null" } ], "default": null }, "dimension": { "anyOf": [ { "type": "object", "title": "ColumnDimension", "properties": { "name": { "type": "string" }, "type": { "enum": [ "categorical", "time" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "is_partition": { "type": "boolean", "default": false }, "config": { "type": "object", "propertyNames": { "type": "string" } }, "validity_params": { "anyOf": [ { "type": "object", "title": "ColumnDimensionValidityParams", "properties": { "is_start": { "type": "boolean", "default": false }, "is_end": { "type": "boolean", "default": false } }, "additionalProperties": false }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "categorical", "time" ] }, { "type": "null" } ], "default": null }, "entity": { "anyOf": [ { "type": "object", "title": "ColumnEntity", "properties": { "name": { "type": "string" }, "type": { "enum": [ "foreign", "natural", "primary", "unique" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "foreign", "natural", "primary", "unique" ] }, { "type": "null" } ], "default": null }, "doc_blocks": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true, "required": [ "name" ] }, "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "patch_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "build_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "created_at": { "type": "number" }, "config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "unrendered_config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "raw_code": { "type": "string", "default": "" }, "doc_blocks": { "type": "array", "items": { "type": "string" } }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "type": "object", "title": "RefArgs", "properties": { "name": { "type": "string" }, "package": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "version": { "anyOf": [ { "type": "string" }, { "type": "number" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] } }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "functions": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "depends_on": { "type": "object", "title": "DependsOn", "properties": { "macros": { "type": "array", "items": { "type": "string" } }, "nodes": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false }, "compiled_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "type": "object", "title": "InjectedCTE", "properties": { "id": { "type": "string" }, "sql": { "type": "string" } }, "additionalProperties": false, "required": [ "id", "sql" ] } }, "_pre_injected_sql": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "contract": { "type": "object", "title": "Contract", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true }, "checksum": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "arguments": { "type": "array", "items": { "type": "object", "title": "FunctionArgument", "properties": { "name": { "type": "string" }, "data_type": { "type": "string" }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "default_value": { "anyOf": [ {}, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "data_type" ] } }, "defer_function": { "anyOf": [ { "type": "object", "title": "DeferFunction", "properties": { "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "alias": { "type": "string" }, "resource_type": { "enum": [ "model", "analysis", "test", "snapshot", "operation", "seed", "rpc", "sql_operation", "doc", "source", "macro", "exposure", "metric", "group", "saved_query", "semantic_model", "unit_test", "fixture", "function" ] }, "name": { "type": "string" }, "description": { "type": "string" }, "compiled_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } }, "config": { "anyOf": [ { "type": "object", "title": "FunctionConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "tags": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "materialized": { "type": "string", "default": "function" }, "incremental_strategy": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "batch_size": { "default": null }, "lookback": { "default": 1 }, "begin": { "default": null }, "persist_docs": { "type": "object", "propertyNames": { "type": "string" } }, "post-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "pre-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "quoting": { "type": "object", "propertyNames": { "type": "string" } }, "column_types": { "type": "object", "propertyNames": { "type": "string" } }, "full_refresh": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "unique_key": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ], "default": null }, "on_schema_change": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "on_configuration_change": { "enum": [ "apply", "continue", "fail" ] }, "grants": { "type": "object", "propertyNames": { "type": "string" } }, "packages": { "type": "array", "items": { "type": "string" } }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "contract": { "type": "object", "title": "ContractConfig", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true } }, "additionalProperties": false }, "event_time": { "default": null }, "concurrent_batches": { "default": null }, "type": { "enum": [ "scalar", "aggregate", "table" ], "default": "scalar" }, "volatility": { "anyOf": [ { "enum": [ "deterministic", "stable", "non-deterministic" ] }, { "type": "null" } ], "default": null }, "runtime_version": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "entry_point": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": true }, { "type": "null" } ] }, "arguments": { "type": "array", "items": { "type": "object", "title": "FunctionArgument", "properties": { "name": { "type": "string" }, "data_type": { "type": "string" }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "default_value": { "anyOf": [ {}, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "data_type" ] } }, "returns": { "type": "object", "title": "FunctionReturns", "properties": { "data_type": { "type": "string" }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "data_type" ] } }, "additionalProperties": false, "required": [ "database", "schema", "alias", "resource_type", "name", "description", "compiled_code", "meta", "tags", "config", "arguments", "returns" ] }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "returns", "database", "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum", "config" ] }, { "type": "object", "title": "SourceDefinition", "properties": { "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "const": "source" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "source_name": { "type": "string" }, "source_description": { "type": "string" }, "loader": { "type": "string" }, "identifier": { "type": "string" }, "quoting": { "type": "object", "title": "Quoting", "properties": { "database": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "identifier": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "column": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "loaded_at_field": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "loaded_at_query": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "freshness": { "anyOf": [ { "type": "object", "title": "FreshnessThreshold", "properties": { "warn_after": { "anyOf": [ { "type": "object", "title": "Time", "properties": { "count": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null }, "period": { "anyOf": [ { "enum": [ "minute", "hour", "day" ] }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, { "type": "null" } ] }, "error_after": { "anyOf": [ { "type": "object", "title": "Time", "properties": { "count": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null }, "period": { "anyOf": [ { "enum": [ "minute", "hour", "day" ] }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, { "type": "null" } ] }, "filter": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, { "type": "null" } ], "default": null }, "external": { "anyOf": [ { "type": "object", "title": "ExternalTable", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "location": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "file_format": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "row_format": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "tbl_properties": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "partitions": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "array", "items": { "type": "object", "title": "ExternalPartition", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "name": { "type": "string", "default": "" }, "description": { "type": "string", "default": "" }, "data_type": { "type": "string", "default": "" }, "meta": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": true } }, { "type": "null" } ], "default": null } }, "additionalProperties": true }, { "type": "null" } ], "default": null }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "type": "object", "title": "ColumnInfo", "properties": { "name": { "type": "string" }, "description": { "type": "string", "default": "" }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "data_type": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "constraints": { "type": "array", "items": { "type": "object", "title": "ColumnLevelConstraint", "properties": { "type": { "enum": [ "check", "not_null", "unique", "primary_key", "foreign_key", "custom" ] }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "expression": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "warn_unenforced": { "type": "boolean", "default": true }, "warn_unsupported": { "type": "boolean", "default": true }, "to": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "to_columns": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false, "required": [ "type" ] } }, "quote": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "title": "ColumnConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true }, "tags": { "type": "array", "items": { "type": "string" } }, "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "granularity": { "anyOf": [ { "enum": [ "nanosecond", "microsecond", "millisecond", "second", "minute", "hour", "day", "week", "month", "quarter", "year" ] }, { "type": "null" } ], "default": null }, "dimension": { "anyOf": [ { "type": "object", "title": "ColumnDimension", "properties": { "name": { "type": "string" }, "type": { "enum": [ "categorical", "time" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "is_partition": { "type": "boolean", "default": false }, "config": { "type": "object", "propertyNames": { "type": "string" } }, "validity_params": { "anyOf": [ { "type": "object", "title": "ColumnDimensionValidityParams", "properties": { "is_start": { "type": "boolean", "default": false }, "is_end": { "type": "boolean", "default": false } }, "additionalProperties": false }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "categorical", "time" ] }, { "type": "null" } ], "default": null }, "entity": { "anyOf": [ { "type": "object", "title": "ColumnEntity", "properties": { "name": { "type": "string" }, "type": { "enum": [ "foreign", "natural", "primary", "unique" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "foreign", "natural", "primary", "unique" ] }, { "type": "null" } ], "default": null }, "doc_blocks": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true, "required": [ "name" ] }, "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "source_meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } }, "config": { "type": "object", "title": "SourceConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "event_time": { "default": null }, "freshness": { "anyOf": [ { "type": "object", "title": "FreshnessThreshold", "properties": { "warn_after": { "anyOf": [ { "type": "object", "title": "Time", "properties": { "count": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null }, "period": { "anyOf": [ { "enum": [ "minute", "hour", "day" ] }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, { "type": "null" } ] }, "error_after": { "anyOf": [ { "type": "object", "title": "Time", "properties": { "count": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null }, "period": { "anyOf": [ { "enum": [ "minute", "hour", "day" ] }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, { "type": "null" } ] }, "filter": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, { "type": "null" } ] }, "loaded_at_field": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "loaded_at_query": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true }, "patch_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "created_at": { "type": "number" }, "unrendered_database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "unrendered_schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "doc_blocks": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false, "required": [ "database", "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "source_name", "source_description", "loader", "identifier" ] }, { "type": "object", "title": "Exposure", "properties": { "name": { "type": "string" }, "resource_type": { "const": "exposure" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "type": { "enum": [ "dashboard", "notebook", "analysis", "ml", "application" ] }, "owner": { "type": "object", "title": "Owner", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "email": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ], "default": null }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": true }, "description": { "type": "string", "default": "" }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "maturity": { "anyOf": [ { "enum": [ "low", "medium", "high" ] }, { "type": "null" } ], "default": null }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } }, "config": { "type": "object", "title": "ExposureConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "tags": { "type": "array", "items": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": true }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "url": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "depends_on": { "type": "object", "title": "DependsOn", "properties": { "macros": { "type": "array", "items": { "type": "string" } }, "nodes": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false }, "refs": { "type": "array", "items": { "type": "object", "title": "RefArgs", "properties": { "name": { "type": "string" }, "package": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "version": { "anyOf": [ { "type": "string" }, { "type": "number" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] } }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "created_at": { "type": "number" } }, "additionalProperties": false, "required": [ "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "type", "owner" ] }, { "type": "object", "title": "Metric", "properties": { "name": { "type": "string" }, "resource_type": { "const": "metric" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "description": { "type": "string" }, "label": { "type": "string" }, "type": { "enum": [ "simple", "ratio", "cumulative", "derived", "conversion" ] }, "type_params": { "type": "object", "title": "MetricTypeParams", "properties": { "measure": { "anyOf": [ { "type": "object", "title": "MetricInputMeasure", "properties": { "name": { "type": "string" }, "filter": { "anyOf": [ { "type": "object", "title": "WhereFilterIntersection", "properties": { "where_filters": { "type": "array", "items": { "type": "object", "title": "WhereFilter", "properties": { "where_sql_template": { "type": "string" } }, "additionalProperties": false, "required": [ "where_sql_template" ] } } }, "additionalProperties": false, "required": [ "where_filters" ] }, { "type": "null" } ], "default": null }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "join_to_timespine": { "type": "boolean", "default": false }, "fill_nulls_with": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] }, { "type": "null" } ], "default": null }, "input_measures": { "type": "array", "items": { "type": "object", "title": "MetricInputMeasure", "properties": { "name": { "type": "string" }, "filter": { "anyOf": [ { "type": "object", "title": "WhereFilterIntersection", "properties": { "where_filters": { "type": "array", "items": { "type": "object", "title": "WhereFilter", "properties": { "where_sql_template": { "type": "string" } }, "additionalProperties": false, "required": [ "where_sql_template" ] } } }, "additionalProperties": false, "required": [ "where_filters" ] }, { "type": "null" } ], "default": null }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "join_to_timespine": { "type": "boolean", "default": false }, "fill_nulls_with": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] } }, "numerator": { "anyOf": [ { "type": "object", "title": "MetricInput", "properties": { "name": { "type": "string" }, "filter": { "anyOf": [ { "type": "object", "title": "WhereFilterIntersection", "properties": { "where_filters": { "type": "array", "items": { "type": "object", "title": "WhereFilter", "properties": { "where_sql_template": { "type": "string" } }, "additionalProperties": false, "required": [ "where_sql_template" ] } } }, "additionalProperties": false, "required": [ "where_filters" ] }, { "type": "null" } ], "default": null }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "offset_window": { "anyOf": [ { "type": "object", "title": "MetricTimeWindow", "properties": { "count": { "type": "integer" }, "granularity": { "type": "string" } }, "additionalProperties": false, "required": [ "count", "granularity" ] }, { "type": "null" } ], "default": null }, "offset_to_grain": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] }, { "type": "null" } ], "default": null }, "denominator": { "anyOf": [ { "type": "object", "title": "MetricInput", "properties": { "name": { "type": "string" }, "filter": { "anyOf": [ { "type": "object", "title": "WhereFilterIntersection", "properties": { "where_filters": { "type": "array", "items": { "type": "object", "title": "WhereFilter", "properties": { "where_sql_template": { "type": "string" } }, "additionalProperties": false, "required": [ "where_sql_template" ] } } }, "additionalProperties": false, "required": [ "where_filters" ] }, { "type": "null" } ], "default": null }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "offset_window": { "anyOf": [ { "type": "object", "title": "MetricTimeWindow", "properties": { "count": { "type": "integer" }, "granularity": { "type": "string" } }, "additionalProperties": false, "required": [ "count", "granularity" ] }, { "type": "null" } ], "default": null }, "offset_to_grain": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] }, { "type": "null" } ], "default": null }, "expr": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "window": { "anyOf": [ { "type": "object", "title": "MetricTimeWindow", "properties": { "count": { "type": "integer" }, "granularity": { "type": "string" } }, "additionalProperties": false, "required": [ "count", "granularity" ] }, { "type": "null" } ], "default": null }, "grain_to_date": { "anyOf": [ { "enum": [ "nanosecond", "microsecond", "millisecond", "second", "minute", "hour", "day", "week", "month", "quarter", "year" ] }, { "type": "null" } ], "default": null }, "metrics": { "anyOf": [ { "type": "array", "items": { "type": "object", "title": "MetricInput", "properties": { "name": { "type": "string" }, "filter": { "anyOf": [ { "type": "object", "title": "WhereFilterIntersection", "properties": { "where_filters": { "type": "array", "items": { "type": "object", "title": "WhereFilter", "properties": { "where_sql_template": { "type": "string" } }, "additionalProperties": false, "required": [ "where_sql_template" ] } } }, "additionalProperties": false, "required": [ "where_filters" ] }, { "type": "null" } ], "default": null }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "offset_window": { "anyOf": [ { "type": "object", "title": "MetricTimeWindow", "properties": { "count": { "type": "integer" }, "granularity": { "type": "string" } }, "additionalProperties": false, "required": [ "count", "granularity" ] }, { "type": "null" } ], "default": null }, "offset_to_grain": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] } }, { "type": "null" } ], "default": null }, "conversion_type_params": { "anyOf": [ { "type": "object", "title": "ConversionTypeParams", "properties": { "entity": { "type": "string" }, "base_measure": { "anyOf": [ { "type": "object", "title": "MetricInputMeasure", "properties": { "name": { "type": "string" }, "filter": { "anyOf": [ { "type": "object", "title": "WhereFilterIntersection", "properties": { "where_filters": { "type": "array", "items": { "type": "object", "title": "WhereFilter", "properties": { "where_sql_template": { "type": "string" } }, "additionalProperties": false, "required": [ "where_sql_template" ] } } }, "additionalProperties": false, "required": [ "where_filters" ] }, { "type": "null" } ], "default": null }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "join_to_timespine": { "type": "boolean", "default": false }, "fill_nulls_with": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] }, { "type": "null" } ], "default": null }, "conversion_measure": { "anyOf": [ { "type": "object", "title": "MetricInputMeasure", "properties": { "name": { "type": "string" }, "filter": { "anyOf": [ { "type": "object", "title": "WhereFilterIntersection", "properties": { "where_filters": { "type": "array", "items": { "type": "object", "title": "WhereFilter", "properties": { "where_sql_template": { "type": "string" } }, "additionalProperties": false, "required": [ "where_sql_template" ] } } }, "additionalProperties": false, "required": [ "where_filters" ] }, { "type": "null" } ], "default": null }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "join_to_timespine": { "type": "boolean", "default": false }, "fill_nulls_with": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] }, { "type": "null" } ], "default": null }, "base_metric": { "anyOf": [ { "type": "object", "title": "MetricInput", "properties": { "name": { "type": "string" }, "filter": { "anyOf": [ { "type": "object", "title": "WhereFilterIntersection", "properties": { "where_filters": { "type": "array", "items": { "type": "object", "title": "WhereFilter", "properties": { "where_sql_template": { "type": "string" } }, "additionalProperties": false, "required": [ "where_sql_template" ] } } }, "additionalProperties": false, "required": [ "where_filters" ] }, { "type": "null" } ], "default": null }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "offset_window": { "anyOf": [ { "type": "object", "title": "MetricTimeWindow", "properties": { "count": { "type": "integer" }, "granularity": { "type": "string" } }, "additionalProperties": false, "required": [ "count", "granularity" ] }, { "type": "null" } ], "default": null }, "offset_to_grain": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] }, { "type": "null" } ], "default": null }, "conversion_metric": { "anyOf": [ { "type": "object", "title": "MetricInput", "properties": { "name": { "type": "string" }, "filter": { "anyOf": [ { "type": "object", "title": "WhereFilterIntersection", "properties": { "where_filters": { "type": "array", "items": { "type": "object", "title": "WhereFilter", "properties": { "where_sql_template": { "type": "string" } }, "additionalProperties": false, "required": [ "where_sql_template" ] } } }, "additionalProperties": false, "required": [ "where_filters" ] }, { "type": "null" } ], "default": null }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "offset_window": { "anyOf": [ { "type": "object", "title": "MetricTimeWindow", "properties": { "count": { "type": "integer" }, "granularity": { "type": "string" } }, "additionalProperties": false, "required": [ "count", "granularity" ] }, { "type": "null" } ], "default": null }, "offset_to_grain": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] }, { "type": "null" } ], "default": null }, "calculation": { "enum": [ "conversions", "conversion_rate" ], "default": "conversion_rate" }, "window": { "anyOf": [ { "type": "object", "title": "MetricTimeWindow", "properties": { "count": { "type": "integer" }, "granularity": { "type": "string" } }, "additionalProperties": false, "required": [ "count", "granularity" ] }, { "type": "null" } ], "default": null }, "constant_properties": { "anyOf": [ { "type": "array", "items": { "type": "object", "title": "ConstantPropertyInput", "properties": { "base_property": { "type": "string" }, "conversion_property": { "type": "string" } }, "additionalProperties": false, "required": [ "base_property", "conversion_property" ] } }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "entity" ] }, { "type": "null" } ], "default": null }, "cumulative_type_params": { "anyOf": [ { "type": "object", "title": "CumulativeTypeParams", "properties": { "window": { "anyOf": [ { "type": "object", "title": "MetricTimeWindow", "properties": { "count": { "type": "integer" }, "granularity": { "type": "string" } }, "additionalProperties": false, "required": [ "count", "granularity" ] }, { "type": "null" } ], "default": null }, "grain_to_date": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "period_agg": { "enum": [ "first", "last", "average" ], "default": "first" }, "metric": { "anyOf": [ { "type": "object", "title": "MetricInput", "properties": { "name": { "type": "string" }, "filter": { "anyOf": [ { "type": "object", "title": "WhereFilterIntersection", "properties": { "where_filters": { "type": "array", "items": { "type": "object", "title": "WhereFilter", "properties": { "where_sql_template": { "type": "string" } }, "additionalProperties": false, "required": [ "where_sql_template" ] } } }, "additionalProperties": false, "required": [ "where_filters" ] }, { "type": "null" } ], "default": null }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "offset_window": { "anyOf": [ { "type": "object", "title": "MetricTimeWindow", "properties": { "count": { "type": "integer" }, "granularity": { "type": "string" } }, "additionalProperties": false, "required": [ "count", "granularity" ] }, { "type": "null" } ], "default": null }, "offset_to_grain": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, { "type": "null" } ], "default": null }, "metric_aggregation_params": { "anyOf": [ { "type": "object", "title": "MetricAggregationParams", "properties": { "semantic_model": { "type": "string" }, "agg": { "enum": [ "sum", "min", "max", "count_distinct", "sum_boolean", "average", "percentile", "median", "count" ] }, "agg_params": { "anyOf": [ { "type": "object", "title": "MeasureAggregationParameters", "properties": { "percentile": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "default": null }, "use_discrete_percentile": { "type": "boolean", "default": false }, "use_approximate_percentile": { "type": "boolean", "default": false } }, "additionalProperties": false }, { "type": "null" } ], "default": null }, "agg_time_dimension": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "non_additive_dimension": { "anyOf": [ { "type": "object", "title": "NonAdditiveDimension", "properties": { "name": { "type": "string" }, "window_choice": { "enum": [ "sum", "min", "max", "count_distinct", "sum_boolean", "average", "percentile", "median", "count" ] }, "window_groupings": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false, "required": [ "name", "window_choice", "window_groupings" ] }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "semantic_model", "agg" ] }, { "type": "null" } ], "default": null }, "fill_nulls_with": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null }, "join_to_timespine": { "type": "boolean", "default": false }, "is_private": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "filter": { "anyOf": [ { "type": "object", "title": "WhereFilterIntersection", "properties": { "where_filters": { "type": "array", "items": { "type": "object", "title": "WhereFilter", "properties": { "where_sql_template": { "type": "string" } }, "additionalProperties": false, "required": [ "where_sql_template" ] } } }, "additionalProperties": false, "required": [ "where_filters" ] }, { "type": "null" } ], "default": null }, "metadata": { "anyOf": [ { "type": "object", "title": "SourceFileMetadata", "properties": { "repo_file_path": { "type": "string" }, "file_slice": { "type": "object", "title": "FileSlice", "properties": { "filename": { "type": "string" }, "content": { "type": "string" }, "start_line_number": { "type": "integer" }, "end_line_number": { "type": "integer" } }, "additionalProperties": false, "required": [ "filename", "content", "start_line_number", "end_line_number" ] } }, "additionalProperties": false, "required": [ "repo_file_path", "file_slice" ] }, { "type": "null" } ], "default": null }, "time_granularity": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "title": "MetricConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "meta": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": true }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "depends_on": { "type": "object", "title": "DependsOn", "properties": { "macros": { "type": "array", "items": { "type": "string" } }, "nodes": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false }, "refs": { "type": "array", "items": { "type": "object", "title": "RefArgs", "properties": { "name": { "type": "string" }, "package": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "version": { "anyOf": [ { "type": "string" }, { "type": "number" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] } }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "created_at": { "type": "number" }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false, "required": [ "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "description", "label", "type", "type_params" ] }, { "type": "object", "title": "SavedQuery", "properties": { "name": { "type": "string" }, "resource_type": { "const": "saved_query" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "query_params": { "type": "object", "title": "QueryParams", "properties": { "metrics": { "type": "array", "items": { "type": "string" } }, "group_by": { "type": "array", "items": { "type": "string" } }, "where": { "anyOf": [ { "type": "object", "title": "WhereFilterIntersection", "properties": { "where_filters": { "type": "array", "items": { "type": "object", "title": "WhereFilter", "properties": { "where_sql_template": { "type": "string" } }, "additionalProperties": false, "required": [ "where_sql_template" ] } } }, "additionalProperties": false, "required": [ "where_filters" ] }, { "type": "null" } ] }, "order_by": { "type": "array", "items": { "type": "string" } }, "limit": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "metrics", "group_by", "where" ] }, "exports": { "type": "array", "items": { "type": "object", "title": "Export", "properties": { "name": { "type": "string" }, "config": { "type": "object", "title": "ExportConfig", "properties": { "export_as": { "enum": [ "table", "view" ] }, "schema_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "export_as" ] }, "unrendered_config": { "type": "object", "additionalProperties": { "type": "string" }, "propertyNames": { "type": "string" } } }, "additionalProperties": false, "required": [ "name", "config" ] } }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "metadata": { "anyOf": [ { "type": "object", "title": "SourceFileMetadata", "properties": { "repo_file_path": { "type": "string" }, "file_slice": { "type": "object", "title": "FileSlice", "properties": { "filename": { "type": "string" }, "content": { "type": "string" }, "start_line_number": { "type": "integer" }, "end_line_number": { "type": "integer" } }, "additionalProperties": false, "required": [ "filename", "content", "start_line_number", "end_line_number" ] } }, "additionalProperties": false, "required": [ "repo_file_path", "file_slice" ] }, { "type": "null" } ], "default": null }, "config": { "type": "object", "title": "SavedQueryConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "export_as": { "anyOf": [ { "enum": [ "table", "view" ] }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "cache": { "type": "object", "title": "SavedQueryCache", "properties": { "enabled": { "type": "boolean", "default": false } }, "additionalProperties": false } }, "additionalProperties": true }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "depends_on": { "type": "object", "title": "DependsOn", "properties": { "macros": { "type": "array", "items": { "type": "string" } }, "nodes": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false }, "created_at": { "type": "number" }, "refs": { "type": "array", "items": { "type": "object", "title": "RefArgs", "properties": { "name": { "type": "string" }, "package": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "version": { "anyOf": [ { "type": "string" }, { "type": "number" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] } }, "tags": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ] } }, "additionalProperties": false, "required": [ "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "query_params", "exports" ] }, { "type": "object", "title": "SemanticModel", "properties": { "name": { "type": "string" }, "resource_type": { "enum": [ "model", "analysis", "test", "snapshot", "operation", "seed", "rpc", "sql_operation", "doc", "source", "macro", "exposure", "metric", "group", "saved_query", "semantic_model", "unit_test", "fixture", "function" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "model": { "type": "string" }, "node_relation": { "anyOf": [ { "type": "object", "title": "NodeRelation", "properties": { "alias": { "type": "string" }, "schema_name": { "type": "string" }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": "" } }, "additionalProperties": false, "required": [ "alias", "schema_name" ] }, { "type": "null" } ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "defaults": { "anyOf": [ { "type": "object", "title": "Defaults", "properties": { "agg_time_dimension": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, { "type": "null" } ], "default": null }, "entities": { "type": "array", "items": { "type": "object", "title": "Entity", "properties": { "name": { "type": "string" }, "type": { "enum": [ "foreign", "natural", "primary", "unique" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "role": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "expr": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "config": { "anyOf": [ { "type": "object", "title": "SemanticLayerElementConfig", "properties": { "meta": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": false }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "type" ] } }, "measures": { "type": "array", "items": { "type": "object", "title": "Measure", "properties": { "name": { "type": "string" }, "agg": { "enum": [ "sum", "min", "max", "count_distinct", "sum_boolean", "average", "percentile", "median", "count" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "create_metric": { "type": "boolean", "default": false }, "expr": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "agg_params": { "anyOf": [ { "type": "object", "title": "MeasureAggregationParameters", "properties": { "percentile": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "default": null }, "use_discrete_percentile": { "type": "boolean", "default": false }, "use_approximate_percentile": { "type": "boolean", "default": false } }, "additionalProperties": false }, { "type": "null" } ], "default": null }, "non_additive_dimension": { "anyOf": [ { "type": "object", "title": "NonAdditiveDimension", "properties": { "name": { "type": "string" }, "window_choice": { "enum": [ "sum", "min", "max", "count_distinct", "sum_boolean", "average", "percentile", "median", "count" ] }, "window_groupings": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false, "required": [ "name", "window_choice", "window_groupings" ] }, { "type": "null" } ], "default": null }, "agg_time_dimension": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "config": { "anyOf": [ { "type": "object", "title": "SemanticLayerElementConfig", "properties": { "meta": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": false }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "agg" ] } }, "dimensions": { "type": "array", "items": { "type": "object", "title": "Dimension", "properties": { "name": { "type": "string" }, "type": { "enum": [ "categorical", "time" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "is_partition": { "type": "boolean", "default": false }, "type_params": { "anyOf": [ { "type": "object", "title": "DimensionTypeParams", "properties": { "time_granularity": { "enum": [ "nanosecond", "microsecond", "millisecond", "second", "minute", "hour", "day", "week", "month", "quarter", "year" ] }, "validity_params": { "anyOf": [ { "type": "object", "title": "DimensionValidityParams", "properties": { "is_start": { "type": "boolean", "default": false }, "is_end": { "type": "boolean", "default": false } }, "additionalProperties": false }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "time_granularity" ] }, { "type": "null" } ], "default": null }, "expr": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "metadata": { "anyOf": [ { "type": "object", "title": "SourceFileMetadata", "properties": { "repo_file_path": { "type": "string" }, "file_slice": { "type": "object", "title": "FileSlice", "properties": { "filename": { "type": "string" }, "content": { "type": "string" }, "start_line_number": { "type": "integer" }, "end_line_number": { "type": "integer" } }, "additionalProperties": false, "required": [ "filename", "content", "start_line_number", "end_line_number" ] } }, "additionalProperties": false, "required": [ "repo_file_path", "file_slice" ] }, { "type": "null" } ], "default": null }, "config": { "anyOf": [ { "type": "object", "title": "SemanticLayerElementConfig", "properties": { "meta": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": false }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "type" ] } }, "metadata": { "anyOf": [ { "type": "object", "title": "SourceFileMetadata", "properties": { "repo_file_path": { "type": "string" }, "file_slice": { "type": "object", "title": "FileSlice", "properties": { "filename": { "type": "string" }, "content": { "type": "string" }, "start_line_number": { "type": "integer" }, "end_line_number": { "type": "integer" } }, "additionalProperties": false, "required": [ "filename", "content", "start_line_number", "end_line_number" ] } }, "additionalProperties": false, "required": [ "repo_file_path", "file_slice" ] }, { "type": "null" } ], "default": null }, "depends_on": { "type": "object", "title": "DependsOn", "properties": { "macros": { "type": "array", "items": { "type": "string" } }, "nodes": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false }, "refs": { "type": "array", "items": { "type": "object", "title": "RefArgs", "properties": { "name": { "type": "string" }, "package": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "version": { "anyOf": [ { "type": "string" }, { "type": "number" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] } }, "created_at": { "type": "number" }, "config": { "type": "object", "title": "SemanticModelConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "meta": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": true }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "primary_entity": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "model", "node_relation" ] }, { "type": "object", "title": "UnitTestDefinition", "properties": { "model": { "type": "string" }, "given": { "type": "array", "items": { "type": "object", "title": "UnitTestInputFixture", "properties": { "input": { "type": "string" }, "rows": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "object", "propertyNames": { "type": "string" } } }, { "type": "null" } ], "default": null }, "format": { "enum": [ "csv", "dict", "sql" ], "default": "dict" }, "fixture": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "input" ] } }, "expect": { "type": "object", "title": "UnitTestOutputFixture", "properties": { "rows": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "object", "propertyNames": { "type": "string" } } }, { "type": "null" } ], "default": null }, "format": { "enum": [ "csv", "dict", "sql" ], "default": "dict" }, "fixture": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "name": { "type": "string" }, "resource_type": { "enum": [ "model", "analysis", "test", "snapshot", "operation", "seed", "rpc", "sql_operation", "doc", "source", "macro", "exposure", "metric", "group", "saved_query", "semantic_model", "unit_test", "fixture", "function" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "description": { "type": "string", "default": "" }, "overrides": { "anyOf": [ { "type": "object", "title": "UnitTestOverrides", "properties": { "macros": { "type": "object", "propertyNames": { "type": "string" } }, "vars": { "type": "object", "propertyNames": { "type": "string" } }, "env_vars": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": false }, { "type": "null" } ], "default": null }, "depends_on": { "type": "object", "title": "DependsOn", "properties": { "macros": { "type": "array", "items": { "type": "string" } }, "nodes": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false }, "config": { "type": "object", "title": "UnitTestConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true } }, "additionalProperties": true }, "checksum": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "created_at": { "type": "number" }, "versions": { "anyOf": [ { "type": "object", "title": "UnitTestNodeVersions", "properties": { "include": { "anyOf": [ { "type": "array", "items": { "anyOf": [ { "type": "string" }, { "type": "number" } ] } }, { "type": "null" } ], "default": null }, "exclude": { "anyOf": [ { "type": "array", "items": { "anyOf": [ { "type": "string" }, { "type": "number" } ] } }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, { "type": "null" } ], "default": null }, "version": { "anyOf": [ { "type": "string" }, { "type": "number" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "model", "given", "expect", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn" ] } ] } }, "propertyNames": { "type": "string" } }, { "type": "null" } ] }, "parent_map": { "description": "A mapping from\u00a0child nodes to their dependencies", "anyOf": [ { "type": "object", "additionalProperties": { "type": "array", "items": { "type": "string" } }, "propertyNames": { "type": "string" } }, { "type": "null" } ] }, "child_map": { "description": "A mapping from parent nodes to their dependents", "anyOf": [ { "type": "object", "additionalProperties": { "type": "array", "items": { "type": "string" } }, "propertyNames": { "type": "string" } }, { "type": "null" } ] }, "group_map": { "description": "A mapping from group names to their nodes", "anyOf": [ { "type": "object", "additionalProperties": { "type": "array", "items": { "type": "string" } }, "propertyNames": { "type": "string" } }, { "type": "null" } ] }, "saved_queries": { "type": "object", "description": "The saved queries defined in the dbt project", "additionalProperties": { "type": "object", "title": "SavedQuery", "properties": { "name": { "type": "string" }, "resource_type": { "const": "saved_query" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "query_params": { "type": "object", "title": "QueryParams", "properties": { "metrics": { "type": "array", "items": { "type": "string" } }, "group_by": { "type": "array", "items": { "type": "string" } }, "where": { "anyOf": [ { "type": "object", "title": "WhereFilterIntersection", "properties": { "where_filters": { "type": "array", "items": { "type": "object", "title": "WhereFilter", "properties": { "where_sql_template": { "type": "string" } }, "additionalProperties": false, "required": [ "where_sql_template" ] } } }, "additionalProperties": false, "required": [ "where_filters" ] }, { "type": "null" } ] }, "order_by": { "type": "array", "items": { "type": "string" } }, "limit": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "metrics", "group_by", "where" ] }, "exports": { "type": "array", "items": { "type": "object", "title": "Export", "properties": { "name": { "type": "string" }, "config": { "type": "object", "title": "ExportConfig", "properties": { "export_as": { "enum": [ "table", "view" ] }, "schema_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "export_as" ] }, "unrendered_config": { "type": "object", "additionalProperties": { "type": "string" }, "propertyNames": { "type": "string" } } }, "additionalProperties": false, "required": [ "name", "config" ] } }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "metadata": { "anyOf": [ { "type": "object", "title": "SourceFileMetadata", "properties": { "repo_file_path": { "type": "string" }, "file_slice": { "type": "object", "title": "FileSlice", "properties": { "filename": { "type": "string" }, "content": { "type": "string" }, "start_line_number": { "type": "integer" }, "end_line_number": { "type": "integer" } }, "additionalProperties": false, "required": [ "filename", "content", "start_line_number", "end_line_number" ] } }, "additionalProperties": false, "required": [ "repo_file_path", "file_slice" ] }, { "type": "null" } ], "default": null }, "config": { "type": "object", "title": "SavedQueryConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "export_as": { "anyOf": [ { "enum": [ "table", "view" ] }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "cache": { "type": "object", "title": "SavedQueryCache", "properties": { "enabled": { "type": "boolean", "default": false } }, "additionalProperties": false } }, "additionalProperties": true }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "depends_on": { "type": "object", "title": "DependsOn", "properties": { "macros": { "type": "array", "items": { "type": "string" } }, "nodes": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false }, "created_at": { "type": "number" }, "refs": { "type": "array", "items": { "type": "object", "title": "RefArgs", "properties": { "name": { "type": "string" }, "package": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "version": { "anyOf": [ { "type": "string" }, { "type": "number" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] } }, "tags": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ] } }, "additionalProperties": false, "required": [ "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "query_params", "exports" ] }, "propertyNames": { "type": "string" } }, "semantic_models": { "type": "object", "description": "The semantic models defined in the dbt project", "additionalProperties": { "type": "object", "title": "SemanticModel", "properties": { "name": { "type": "string" }, "resource_type": { "enum": [ "model", "analysis", "test", "snapshot", "operation", "seed", "rpc", "sql_operation", "doc", "source", "macro", "exposure", "metric", "group", "saved_query", "semantic_model", "unit_test", "fixture", "function" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "model": { "type": "string" }, "node_relation": { "anyOf": [ { "type": "object", "title": "NodeRelation", "properties": { "alias": { "type": "string" }, "schema_name": { "type": "string" }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": "" } }, "additionalProperties": false, "required": [ "alias", "schema_name" ] }, { "type": "null" } ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "defaults": { "anyOf": [ { "type": "object", "title": "Defaults", "properties": { "agg_time_dimension": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, { "type": "null" } ], "default": null }, "entities": { "type": "array", "items": { "type": "object", "title": "Entity", "properties": { "name": { "type": "string" }, "type": { "enum": [ "foreign", "natural", "primary", "unique" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "role": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "expr": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "config": { "anyOf": [ { "type": "object", "title": "SemanticLayerElementConfig", "properties": { "meta": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": false }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "type" ] } }, "measures": { "type": "array", "items": { "type": "object", "title": "Measure", "properties": { "name": { "type": "string" }, "agg": { "enum": [ "sum", "min", "max", "count_distinct", "sum_boolean", "average", "percentile", "median", "count" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "create_metric": { "type": "boolean", "default": false }, "expr": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "agg_params": { "anyOf": [ { "type": "object", "title": "MeasureAggregationParameters", "properties": { "percentile": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "default": null }, "use_discrete_percentile": { "type": "boolean", "default": false }, "use_approximate_percentile": { "type": "boolean", "default": false } }, "additionalProperties": false }, { "type": "null" } ], "default": null }, "non_additive_dimension": { "anyOf": [ { "type": "object", "title": "NonAdditiveDimension", "properties": { "name": { "type": "string" }, "window_choice": { "enum": [ "sum", "min", "max", "count_distinct", "sum_boolean", "average", "percentile", "median", "count" ] }, "window_groupings": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false, "required": [ "name", "window_choice", "window_groupings" ] }, { "type": "null" } ], "default": null }, "agg_time_dimension": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "config": { "anyOf": [ { "type": "object", "title": "SemanticLayerElementConfig", "properties": { "meta": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": false }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "agg" ] } }, "dimensions": { "type": "array", "items": { "type": "object", "title": "Dimension", "properties": { "name": { "type": "string" }, "type": { "enum": [ "categorical", "time" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "is_partition": { "type": "boolean", "default": false }, "type_params": { "anyOf": [ { "type": "object", "title": "DimensionTypeParams", "properties": { "time_granularity": { "enum": [ "nanosecond", "microsecond", "millisecond", "second", "minute", "hour", "day", "week", "month", "quarter", "year" ] }, "validity_params": { "anyOf": [ { "type": "object", "title": "DimensionValidityParams", "properties": { "is_start": { "type": "boolean", "default": false }, "is_end": { "type": "boolean", "default": false } }, "additionalProperties": false }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "time_granularity" ] }, { "type": "null" } ], "default": null }, "expr": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "metadata": { "anyOf": [ { "type": "object", "title": "SourceFileMetadata", "properties": { "repo_file_path": { "type": "string" }, "file_slice": { "type": "object", "title": "FileSlice", "properties": { "filename": { "type": "string" }, "content": { "type": "string" }, "start_line_number": { "type": "integer" }, "end_line_number": { "type": "integer" } }, "additionalProperties": false, "required": [ "filename", "content", "start_line_number", "end_line_number" ] } }, "additionalProperties": false, "required": [ "repo_file_path", "file_slice" ] }, { "type": "null" } ], "default": null }, "config": { "anyOf": [ { "type": "object", "title": "SemanticLayerElementConfig", "properties": { "meta": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": false }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "type" ] } }, "metadata": { "anyOf": [ { "type": "object", "title": "SourceFileMetadata", "properties": { "repo_file_path": { "type": "string" }, "file_slice": { "type": "object", "title": "FileSlice", "properties": { "filename": { "type": "string" }, "content": { "type": "string" }, "start_line_number": { "type": "integer" }, "end_line_number": { "type": "integer" } }, "additionalProperties": false, "required": [ "filename", "content", "start_line_number", "end_line_number" ] } }, "additionalProperties": false, "required": [ "repo_file_path", "file_slice" ] }, { "type": "null" } ], "default": null }, "depends_on": { "type": "object", "title": "DependsOn", "properties": { "macros": { "type": "array", "items": { "type": "string" } }, "nodes": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false }, "refs": { "type": "array", "items": { "type": "object", "title": "RefArgs", "properties": { "name": { "type": "string" }, "package": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "version": { "anyOf": [ { "type": "string" }, { "type": "number" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] } }, "created_at": { "type": "number" }, "config": { "type": "object", "title": "SemanticModelConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "meta": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": true }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "primary_entity": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "model", "node_relation" ] }, "propertyNames": { "type": "string" } }, "unit_tests": { "type": "object", "description": "The unit tests defined in the project", "additionalProperties": { "type": "object", "title": "UnitTestDefinition", "properties": { "model": { "type": "string" }, "given": { "type": "array", "items": { "type": "object", "title": "UnitTestInputFixture", "properties": { "input": { "type": "string" }, "rows": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "object", "propertyNames": { "type": "string" } } }, { "type": "null" } ], "default": null }, "format": { "enum": [ "csv", "dict", "sql" ], "default": "dict" }, "fixture": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "input" ] } }, "expect": { "type": "object", "title": "UnitTestOutputFixture", "properties": { "rows": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "object", "propertyNames": { "type": "string" } } }, { "type": "null" } ], "default": null }, "format": { "enum": [ "csv", "dict", "sql" ], "default": "dict" }, "fixture": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "name": { "type": "string" }, "resource_type": { "enum": [ "model", "analysis", "test", "snapshot", "operation", "seed", "rpc", "sql_operation", "doc", "source", "macro", "exposure", "metric", "group", "saved_query", "semantic_model", "unit_test", "fixture", "function" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "description": { "type": "string", "default": "" }, "overrides": { "anyOf": [ { "type": "object", "title": "UnitTestOverrides", "properties": { "macros": { "type": "object", "propertyNames": { "type": "string" } }, "vars": { "type": "object", "propertyNames": { "type": "string" } }, "env_vars": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": false }, { "type": "null" } ], "default": null }, "depends_on": { "type": "object", "title": "DependsOn", "properties": { "macros": { "type": "array", "items": { "type": "string" } }, "nodes": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false }, "config": { "type": "object", "title": "UnitTestConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true } }, "additionalProperties": true }, "checksum": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "created_at": { "type": "number" }, "versions": { "anyOf": [ { "type": "object", "title": "UnitTestNodeVersions", "properties": { "include": { "anyOf": [ { "type": "array", "items": { "anyOf": [ { "type": "string" }, { "type": "number" } ] } }, { "type": "null" } ], "default": null }, "exclude": { "anyOf": [ { "type": "array", "items": { "anyOf": [ { "type": "string" }, { "type": "number" } ] } }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, { "type": "null" } ], "default": null }, "version": { "anyOf": [ { "type": "string" }, { "type": "number" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "model", "given", "expect", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn" ] }, "propertyNames": { "type": "string" } }, "functions": { "type": "object", "description": "The functions defined in the dbt project", "additionalProperties": { "type": "object", "title": "Function", "properties": { "returns": { "type": "object", "title": "FunctionReturns", "properties": { "data_type": { "type": "string" }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "data_type" ] }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "const": "function" }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "type": "object", "title": "FileHash", "properties": { "name": { "type": "string" }, "checksum": { "type": "string" } }, "additionalProperties": false, "required": [ "name", "checksum" ] }, "config": { "type": "object", "title": "FunctionConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "tags": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "materialized": { "type": "string", "default": "function" }, "incremental_strategy": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "batch_size": { "default": null }, "lookback": { "default": 1 }, "begin": { "default": null }, "persist_docs": { "type": "object", "propertyNames": { "type": "string" } }, "post-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "pre-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "quoting": { "type": "object", "propertyNames": { "type": "string" } }, "column_types": { "type": "object", "propertyNames": { "type": "string" } }, "full_refresh": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "unique_key": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ], "default": null }, "on_schema_change": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "on_configuration_change": { "enum": [ "apply", "continue", "fail" ] }, "grants": { "type": "object", "propertyNames": { "type": "string" } }, "packages": { "type": "array", "items": { "type": "string" } }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "contract": { "type": "object", "title": "ContractConfig", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true } }, "additionalProperties": false }, "event_time": { "default": null }, "concurrent_batches": { "default": null }, "type": { "enum": [ "scalar", "aggregate", "table" ], "default": "scalar" }, "volatility": { "anyOf": [ { "enum": [ "deterministic", "stable", "non-deterministic" ] }, { "type": "null" } ], "default": null }, "runtime_version": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "entry_point": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": true }, "tags": { "type": "array", "items": { "type": "string" } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "type": "object", "title": "ColumnInfo", "properties": { "name": { "type": "string" }, "description": { "type": "string", "default": "" }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "data_type": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "constraints": { "type": "array", "items": { "type": "object", "title": "ColumnLevelConstraint", "properties": { "type": { "enum": [ "check", "not_null", "unique", "primary_key", "foreign_key", "custom" ] }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "expression": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "warn_unenforced": { "type": "boolean", "default": true }, "warn_unsupported": { "type": "boolean", "default": true }, "to": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "to_columns": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false, "required": [ "type" ] } }, "quote": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "title": "ColumnConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true }, "tags": { "type": "array", "items": { "type": "string" } }, "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "granularity": { "anyOf": [ { "enum": [ "nanosecond", "microsecond", "millisecond", "second", "minute", "hour", "day", "week", "month", "quarter", "year" ] }, { "type": "null" } ], "default": null }, "dimension": { "anyOf": [ { "type": "object", "title": "ColumnDimension", "properties": { "name": { "type": "string" }, "type": { "enum": [ "categorical", "time" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "is_partition": { "type": "boolean", "default": false }, "config": { "type": "object", "propertyNames": { "type": "string" } }, "validity_params": { "anyOf": [ { "type": "object", "title": "ColumnDimensionValidityParams", "properties": { "is_start": { "type": "boolean", "default": false }, "is_end": { "type": "boolean", "default": false } }, "additionalProperties": false }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "categorical", "time" ] }, { "type": "null" } ], "default": null }, "entity": { "anyOf": [ { "type": "object", "title": "ColumnEntity", "properties": { "name": { "type": "string" }, "type": { "enum": [ "foreign", "natural", "primary", "unique" ] }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "label": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "config": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": false, "required": [ "name", "type" ] }, { "enum": [ "foreign", "natural", "primary", "unique" ] }, { "type": "null" } ], "default": null }, "doc_blocks": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": true, "required": [ "name" ] }, "propertyNames": { "type": "string" } }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "patch_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "build_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "unrendered_config": { "type": "object", "propertyNames": { "type": "string" } }, "created_at": { "type": "number" }, "config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "unrendered_config_call_dict": { "type": "object", "propertyNames": { "type": "string" } }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "raw_code": { "type": "string", "default": "" }, "doc_blocks": { "type": "array", "items": { "type": "string" } }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "type": "object", "title": "RefArgs", "properties": { "name": { "type": "string" }, "package": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "version": { "anyOf": [ { "type": "string" }, { "type": "number" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] } }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "functions": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }, "depends_on": { "type": "object", "title": "DependsOn", "properties": { "macros": { "type": "array", "items": { "type": "string" } }, "nodes": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false }, "compiled_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "type": "object", "title": "InjectedCTE", "properties": { "id": { "type": "string" }, "sql": { "type": "string" } }, "additionalProperties": false, "required": [ "id", "sql" ] } }, "_pre_injected_sql": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "contract": { "type": "object", "title": "Contract", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true }, "checksum": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "arguments": { "type": "array", "items": { "type": "object", "title": "FunctionArgument", "properties": { "name": { "type": "string" }, "data_type": { "type": "string" }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "default_value": { "anyOf": [ {}, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "data_type" ] } }, "defer_function": { "anyOf": [ { "type": "object", "title": "DeferFunction", "properties": { "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "alias": { "type": "string" }, "resource_type": { "enum": [ "model", "analysis", "test", "snapshot", "operation", "seed", "rpc", "sql_operation", "doc", "source", "macro", "exposure", "metric", "group", "saved_query", "semantic_model", "unit_test", "fixture", "function" ] }, "name": { "type": "string" }, "description": { "type": "string" }, "compiled_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "tags": { "type": "array", "items": { "type": "string" } }, "config": { "anyOf": [ { "type": "object", "title": "FunctionConfig", "properties": { "_extra": { "type": "object", "propertyNames": { "type": "string" } }, "enabled": { "type": "boolean", "default": true }, "alias": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "database": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "tags": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ] }, "meta": { "type": "object", "propertyNames": { "type": "string" } }, "group": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "materialized": { "type": "string", "default": "function" }, "incremental_strategy": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "batch_size": { "default": null }, "lookback": { "default": 1 }, "begin": { "default": null }, "persist_docs": { "type": "object", "propertyNames": { "type": "string" } }, "post-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "pre-hook": { "type": "array", "items": { "type": "object", "title": "Hook", "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "sql" ] } }, "quoting": { "type": "object", "propertyNames": { "type": "string" } }, "column_types": { "type": "object", "propertyNames": { "type": "string" } }, "full_refresh": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null }, "unique_key": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ], "default": null }, "on_schema_change": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "on_configuration_change": { "enum": [ "apply", "continue", "fail" ] }, "grants": { "type": "object", "propertyNames": { "type": "string" } }, "packages": { "type": "array", "items": { "type": "string" } }, "docs": { "type": "object", "title": "Docs", "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "contract": { "type": "object", "title": "ContractConfig", "properties": { "enforced": { "type": "boolean", "default": false }, "alias_types": { "type": "boolean", "default": true } }, "additionalProperties": false }, "event_time": { "default": null }, "concurrent_batches": { "default": null }, "type": { "enum": [ "scalar", "aggregate", "table" ], "default": "scalar" }, "volatility": { "anyOf": [ { "enum": [ "deterministic", "stable", "non-deterministic" ] }, { "type": "null" } ], "default": null }, "runtime_version": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "entry_point": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": true }, { "type": "null" } ] }, "arguments": { "type": "array", "items": { "type": "object", "title": "FunctionArgument", "properties": { "name": { "type": "string" }, "data_type": { "type": "string" }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "default_value": { "anyOf": [ {}, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name", "data_type" ] } }, "returns": { "type": "object", "title": "FunctionReturns", "properties": { "data_type": { "type": "string" }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "data_type" ] } }, "additionalProperties": false, "required": [ "database", "schema", "alias", "resource_type", "name", "description", "compiled_code", "meta", "tags", "config", "arguments", "returns" ] }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "returns", "database", "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum", "config" ] }, "propertyNames": { "type": "string" } } }, "additionalProperties": false, "required": [ "metadata", "nodes", "sources", "macros", "docs", "exposures", "metrics", "groups", "selectors", "disabled", "parent_map", "child_map", "group_map", "saved_queries", "semantic_models", "unit_tests" ], "$id": "https://schemas.getdbt.com/dbt/manifest/v12.json" } ================================================ FILE: schemas/dbt/manifest/v5.json ================================================ { "type": "object", "required": [ "metadata", "nodes", "sources", "macros", "docs", "exposures", "metrics", "selectors" ], "properties": { "metadata": { "$ref": "#/definitions/ManifestMetadata", "description": "Metadata about the manifest" }, "nodes": { "type": "object", "additionalProperties": { "oneOf": [ { "$ref": "#/definitions/CompiledAnalysisNode" }, { "$ref": "#/definitions/CompiledSingularTestNode" }, { "$ref": "#/definitions/CompiledModelNode" }, { "$ref": "#/definitions/CompiledHookNode" }, { "$ref": "#/definitions/CompiledRPCNode" }, { "$ref": "#/definitions/CompiledSqlNode" }, { "$ref": "#/definitions/CompiledGenericTestNode" }, { "$ref": "#/definitions/CompiledSeedNode" }, { "$ref": "#/definitions/CompiledSnapshotNode" }, { "$ref": "#/definitions/ParsedAnalysisNode" }, { "$ref": "#/definitions/ParsedSingularTestNode" }, { "$ref": "#/definitions/ParsedHookNode" }, { "$ref": "#/definitions/ParsedModelNode" }, { "$ref": "#/definitions/ParsedRPCNode" }, { "$ref": "#/definitions/ParsedSqlNode" }, { "$ref": "#/definitions/ParsedGenericTestNode" }, { "$ref": "#/definitions/ParsedSeedNode" }, { "$ref": "#/definitions/ParsedSnapshotNode" } ] }, "description": "The nodes defined in the dbt project and its dependencies" }, "sources": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ParsedSourceDefinition" }, "description": "The sources defined in the dbt project and its dependencies" }, "macros": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ParsedMacro" }, "description": "The macros defined in the dbt project and its dependencies" }, "docs": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ParsedDocumentation" }, "description": "The docs defined in the dbt project and its dependencies" }, "exposures": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ParsedExposure" }, "description": "The exposures defined in the dbt project and its dependencies" }, "metrics": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ParsedMetric" }, "description": "The metrics defined in the dbt project and its dependencies" }, "selectors": { "type": "object", "description": "The selectors defined in selectors.yml" }, "disabled": { "oneOf": [ { "type": "object", "additionalProperties": { "type": "array", "items": { "oneOf": [ { "$ref": "#/definitions/CompiledAnalysisNode" }, { "$ref": "#/definitions/CompiledSingularTestNode" }, { "$ref": "#/definitions/CompiledModelNode" }, { "$ref": "#/definitions/CompiledHookNode" }, { "$ref": "#/definitions/CompiledRPCNode" }, { "$ref": "#/definitions/CompiledSqlNode" }, { "$ref": "#/definitions/CompiledGenericTestNode" }, { "$ref": "#/definitions/CompiledSeedNode" }, { "$ref": "#/definitions/CompiledSnapshotNode" }, { "$ref": "#/definitions/ParsedAnalysisNode" }, { "$ref": "#/definitions/ParsedSingularTestNode" }, { "$ref": "#/definitions/ParsedHookNode" }, { "$ref": "#/definitions/ParsedModelNode" }, { "$ref": "#/definitions/ParsedRPCNode" }, { "$ref": "#/definitions/ParsedSqlNode" }, { "$ref": "#/definitions/ParsedGenericTestNode" }, { "$ref": "#/definitions/ParsedSeedNode" }, { "$ref": "#/definitions/ParsedSnapshotNode" }, { "$ref": "#/definitions/ParsedSourceDefinition" } ] } } }, { "type": "null" } ], "description": "A mapping of the disabled nodes in the target" }, "parent_map": { "oneOf": [ { "type": "object", "additionalProperties": { "type": "array", "items": { "type": "string" } } }, { "type": "null" } ], "description": "A mapping from\u00a0child nodes to their dependencies" }, "child_map": { "oneOf": [ { "type": "object", "additionalProperties": { "type": "array", "items": { "type": "string" } } }, { "type": "null" } ], "description": "A mapping from parent nodes to their dependents" } }, "additionalProperties": false, "description": "WritableManifest(metadata: dbt.contracts.graph.manifest.ManifestMetadata, nodes: Mapping[str, Union[dbt.contracts.graph.compiled.CompiledAnalysisNode, dbt.contracts.graph.compiled.CompiledSingularTestNode, dbt.contracts.graph.compiled.CompiledModelNode, dbt.contracts.graph.compiled.CompiledHookNode, dbt.contracts.graph.compiled.CompiledRPCNode, dbt.contracts.graph.compiled.CompiledSqlNode, dbt.contracts.graph.compiled.CompiledGenericTestNode, dbt.contracts.graph.compiled.CompiledSeedNode, dbt.contracts.graph.compiled.CompiledSnapshotNode, dbt.contracts.graph.parsed.ParsedAnalysisNode, dbt.contracts.graph.parsed.ParsedSingularTestNode, dbt.contracts.graph.parsed.ParsedHookNode, dbt.contracts.graph.parsed.ParsedModelNode, dbt.contracts.graph.parsed.ParsedRPCNode, dbt.contracts.graph.parsed.ParsedSqlNode, dbt.contracts.graph.parsed.ParsedGenericTestNode, dbt.contracts.graph.parsed.ParsedSeedNode, dbt.contracts.graph.parsed.ParsedSnapshotNode]], sources: Mapping[str, dbt.contracts.graph.parsed.ParsedSourceDefinition], macros: Mapping[str, dbt.contracts.graph.parsed.ParsedMacro], docs: Mapping[str, dbt.contracts.graph.parsed.ParsedDocumentation], exposures: Mapping[str, dbt.contracts.graph.parsed.ParsedExposure], metrics: Mapping[str, dbt.contracts.graph.parsed.ParsedMetric], selectors: Mapping[str, Any], disabled: Union[Mapping[str, List[Union[dbt.contracts.graph.compiled.CompiledAnalysisNode, dbt.contracts.graph.compiled.CompiledSingularTestNode, dbt.contracts.graph.compiled.CompiledModelNode, dbt.contracts.graph.compiled.CompiledHookNode, dbt.contracts.graph.compiled.CompiledRPCNode, dbt.contracts.graph.compiled.CompiledSqlNode, dbt.contracts.graph.compiled.CompiledGenericTestNode, dbt.contracts.graph.compiled.CompiledSeedNode, dbt.contracts.graph.compiled.CompiledSnapshotNode, dbt.contracts.graph.parsed.ParsedAnalysisNode, dbt.contracts.graph.parsed.ParsedSingularTestNode, dbt.contracts.graph.parsed.ParsedHookNode, dbt.contracts.graph.parsed.ParsedModelNode, dbt.contracts.graph.parsed.ParsedRPCNode, dbt.contracts.graph.parsed.ParsedSqlNode, dbt.contracts.graph.parsed.ParsedGenericTestNode, dbt.contracts.graph.parsed.ParsedSeedNode, dbt.contracts.graph.parsed.ParsedSnapshotNode, dbt.contracts.graph.parsed.ParsedSourceDefinition]]], NoneType], parent_map: Union[Dict[str, List[str]], NoneType], child_map: Union[Dict[str, List[str]], NoneType])", "definitions": { "ManifestMetadata": { "type": "object", "required": [], "properties": { "dbt_schema_version": { "type": "string", "default": "https://schemas.getdbt.com/dbt/manifest/v5.json" }, "dbt_version": { "type": "string", "default": "1.2.0a1" }, "generated_at": { "type": "string", "format": "date-time", "default": "2022-04-15T20:38:22.703053Z" }, "invocation_id": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "default": "34abf75e-59d3-442f-920c-fa3843d98014" }, "env": { "type": "object", "additionalProperties": { "type": "string" }, "default": {} }, "project_id": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "description": "A unique identifier for the project" }, "user_id": { "oneOf": [ { "type": "string", "pattern": "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" }, { "type": "null" } ], "description": "A unique identifier for the user" }, "send_anonymous_usage_stats": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ], "description": "Whether dbt is configured to send anonymous usage statistics" }, "adapter_type": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "description": "The type name of the adapter" } }, "additionalProperties": false, "description": "Metadata for the manifest." }, "CompiledAnalysisNode": { "type": "object", "required": [ "raw_sql", "compiled", "schema", "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "raw_sql": { "type": "string" }, "compiled": { "type": "boolean" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "analysis" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1650055102.707036 }, "config_call_dict": { "type": "object", "default": {} }, "compiled_sql": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "CompiledAnalysisNode(raw_sql: str, compiled: bool, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , compiled_sql: Union[str, NoneType] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.compiled.InjectedCTE] = , relation_name: Union[str, NoneType] = None, _pre_injected_sql: Union[str, NoneType] = None)" }, "FileHash": { "type": "object", "required": [ "name", "checksum" ], "properties": { "name": { "type": "string" }, "checksum": { "type": "string" } }, "additionalProperties": false, "description": "FileHash(name: str, checksum: str)" }, "NodeConfig": { "type": "object", "required": [], "properties": { "enabled": { "type": "boolean", "default": true }, "alias": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "tags": { "oneOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ], "default": [] }, "meta": { "type": "object", "default": {} }, "materialized": { "type": "string", "default": "view" }, "persist_docs": { "type": "object", "default": {} }, "post-hook": { "type": "array", "items": { "$ref": "#/definitions/Hook" }, "default": [] }, "pre-hook": { "type": "array", "items": { "$ref": "#/definitions/Hook" }, "default": [] }, "quoting": { "type": "object", "default": {} }, "column_types": { "type": "object", "default": {} }, "full_refresh": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "unique_key": { "oneOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ] }, "on_schema_change": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" } }, "additionalProperties": true, "description": "NodeConfig(_extra: Dict[str, Any] = , enabled: bool = True, alias: Union[str, NoneType] = None, schema: Union[str, NoneType] = None, database: Union[str, NoneType] = None, tags: Union[List[str], str] = , meta: Dict[str, Any] = , materialized: str = 'view', persist_docs: Dict[str, Any] = , post_hook: List[dbt.contracts.graph.model_config.Hook] = , pre_hook: List[dbt.contracts.graph.model_config.Hook] = , quoting: Dict[str, Any] = , column_types: Dict[str, Any] = , full_refresh: Union[bool, NoneType] = None, unique_key: Union[str, List[str], NoneType] = None, on_schema_change: Union[str, NoneType] = 'ignore')" }, "Hook": { "type": "object", "required": [ "sql" ], "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "oneOf": [ { "type": "integer" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "Hook(sql: str, transaction: bool = True, index: Union[int, NoneType] = None)" }, "DependsOn": { "type": "object", "required": [], "properties": { "macros": { "type": "array", "items": { "type": "string" }, "default": [] }, "nodes": { "type": "array", "items": { "type": "string" }, "default": [] } }, "additionalProperties": false, "description": "DependsOn(macros: List[str] = , nodes: List[str] = )" }, "ColumnInfo": { "type": "object", "required": [ "name" ], "properties": { "name": { "type": "string" }, "description": { "type": "string", "default": "" }, "meta": { "type": "object", "default": {} }, "data_type": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "quote": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] } }, "additionalProperties": true, "description": "ColumnInfo(name: str, description: str = '', meta: Dict[str, Any] = , data_type: Union[str, NoneType] = None, quote: Union[bool, NoneType] = None, tags: List[str] = , _extra: Dict[str, Any] = )" }, "Docs": { "type": "object", "required": [], "properties": { "show": { "type": "boolean", "default": true } }, "additionalProperties": false, "description": "Docs(show: bool = True)" }, "InjectedCTE": { "type": "object", "required": [ "id", "sql" ], "properties": { "id": { "type": "string" }, "sql": { "type": "string" } }, "additionalProperties": false, "description": "InjectedCTE(id: str, sql: str)" }, "CompiledSingularTestNode": { "type": "object", "required": [ "raw_sql", "compiled", "schema", "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "raw_sql": { "type": "string" }, "compiled": { "type": "boolean" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "test" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/TestConfig", "default": { "enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0" } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1650055102.7093382 }, "config_call_dict": { "type": "object", "default": {} }, "compiled_sql": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "CompiledSingularTestNode(raw_sql: str, compiled: bool, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.TestConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , compiled_sql: Union[str, NoneType] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.compiled.InjectedCTE] = , relation_name: Union[str, NoneType] = None, _pre_injected_sql: Union[str, NoneType] = None)" }, "TestConfig": { "type": "object", "required": [], "properties": { "enabled": { "type": "boolean", "default": true }, "alias": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "default": "dbt_test__audit" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "tags": { "oneOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ], "default": [] }, "meta": { "type": "object", "default": {} }, "materialized": { "type": "string", "default": "test" }, "severity": { "type": "string", "pattern": "^([Ww][Aa][Rr][Nn]|[Ee][Rr][Rr][Oo][Rr])$", "default": "ERROR" }, "store_failures": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "where": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "limit": { "oneOf": [ { "type": "integer" }, { "type": "null" } ] }, "fail_calc": { "type": "string", "default": "count(*)" }, "warn_if": { "type": "string", "default": "!= 0" }, "error_if": { "type": "string", "default": "!= 0" } }, "additionalProperties": true, "description": "TestConfig(_extra: Dict[str, Any] = , enabled: bool = True, alias: Union[str, NoneType] = None, schema: Union[str, NoneType] = 'dbt_test__audit', database: Union[str, NoneType] = None, tags: Union[List[str], str] = , meta: Dict[str, Any] = , materialized: str = 'test', severity: dbt.contracts.graph.model_config.Severity = 'ERROR', store_failures: Union[bool, NoneType] = None, where: Union[str, NoneType] = None, limit: Union[int, NoneType] = None, fail_calc: str = 'count(*)', warn_if: str = '!= 0', error_if: str = '!= 0')" }, "CompiledModelNode": { "type": "object", "required": [ "raw_sql", "compiled", "schema", "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "raw_sql": { "type": "string" }, "compiled": { "type": "boolean" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "model" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1650055102.710903 }, "config_call_dict": { "type": "object", "default": {} }, "compiled_sql": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "CompiledModelNode(raw_sql: str, compiled: bool, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , compiled_sql: Union[str, NoneType] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.compiled.InjectedCTE] = , relation_name: Union[str, NoneType] = None, _pre_injected_sql: Union[str, NoneType] = None)" }, "CompiledHookNode": { "type": "object", "required": [ "raw_sql", "compiled", "schema", "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "raw_sql": { "type": "string" }, "compiled": { "type": "boolean" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "operation" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1650055102.712516 }, "config_call_dict": { "type": "object", "default": {} }, "compiled_sql": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "index": { "oneOf": [ { "type": "integer" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "CompiledHookNode(raw_sql: str, compiled: bool, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , compiled_sql: Union[str, NoneType] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.compiled.InjectedCTE] = , relation_name: Union[str, NoneType] = None, _pre_injected_sql: Union[str, NoneType] = None, index: Union[int, NoneType] = None)" }, "CompiledRPCNode": { "type": "object", "required": [ "raw_sql", "compiled", "schema", "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "raw_sql": { "type": "string" }, "compiled": { "type": "boolean" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "rpc" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1650055102.714043 }, "config_call_dict": { "type": "object", "default": {} }, "compiled_sql": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "CompiledRPCNode(raw_sql: str, compiled: bool, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , compiled_sql: Union[str, NoneType] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.compiled.InjectedCTE] = , relation_name: Union[str, NoneType] = None, _pre_injected_sql: Union[str, NoneType] = None)" }, "CompiledSqlNode": { "type": "object", "required": [ "raw_sql", "compiled", "schema", "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "raw_sql": { "type": "string" }, "compiled": { "type": "boolean" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "sql" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1650055102.715457 }, "config_call_dict": { "type": "object", "default": {} }, "compiled_sql": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "CompiledSqlNode(raw_sql: str, compiled: bool, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , compiled_sql: Union[str, NoneType] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.compiled.InjectedCTE] = , relation_name: Union[str, NoneType] = None, _pre_injected_sql: Union[str, NoneType] = None)" }, "CompiledGenericTestNode": { "type": "object", "required": [ "raw_sql", "test_metadata", "compiled", "schema", "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "raw_sql": { "type": "string" }, "test_metadata": { "$ref": "#/definitions/TestMetadata" }, "compiled": { "type": "boolean" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "test" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/TestConfig", "default": { "enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0" } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1650055102.717103 }, "config_call_dict": { "type": "object", "default": {} }, "compiled_sql": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "column_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "file_key_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "CompiledGenericTestNode(raw_sql: str, test_metadata: dbt.contracts.graph.parsed.TestMetadata, compiled: bool, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.TestConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , compiled_sql: Union[str, NoneType] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.compiled.InjectedCTE] = , relation_name: Union[str, NoneType] = None, _pre_injected_sql: Union[str, NoneType] = None, column_name: Union[str, NoneType] = None, file_key_name: Union[str, NoneType] = None)" }, "TestMetadata": { "type": "object", "required": [ "name" ], "properties": { "name": { "type": "string" }, "kwargs": { "type": "object", "default": {} }, "namespace": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "TestMetadata(name: str, kwargs: Dict[str, Any] = , namespace: Union[str, NoneType] = None)" }, "CompiledSeedNode": { "type": "object", "required": [ "raw_sql", "compiled", "schema", "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "raw_sql": { "type": "string" }, "compiled": { "type": "boolean" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "seed" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/SeedConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "seed", "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "quote_columns": null, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1650055102.719672 }, "config_call_dict": { "type": "object", "default": {} }, "compiled_sql": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "CompiledSeedNode(raw_sql: str, compiled: bool, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.SeedConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , compiled_sql: Union[str, NoneType] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.compiled.InjectedCTE] = , relation_name: Union[str, NoneType] = None, _pre_injected_sql: Union[str, NoneType] = None)" }, "SeedConfig": { "type": "object", "required": [], "properties": { "enabled": { "type": "boolean", "default": true }, "alias": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "tags": { "oneOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ], "default": [] }, "meta": { "type": "object", "default": {} }, "materialized": { "type": "string", "default": "seed" }, "persist_docs": { "type": "object", "default": {} }, "post-hook": { "type": "array", "items": { "$ref": "#/definitions/Hook" }, "default": [] }, "pre-hook": { "type": "array", "items": { "$ref": "#/definitions/Hook" }, "default": [] }, "quoting": { "type": "object", "default": {} }, "column_types": { "type": "object", "default": {} }, "full_refresh": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "unique_key": { "oneOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ] }, "on_schema_change": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "quote_columns": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] } }, "additionalProperties": true, "description": "SeedConfig(_extra: Dict[str, Any] = , enabled: bool = True, alias: Union[str, NoneType] = None, schema: Union[str, NoneType] = None, database: Union[str, NoneType] = None, tags: Union[List[str], str] = , meta: Dict[str, Any] = , materialized: str = 'seed', persist_docs: Dict[str, Any] = , post_hook: List[dbt.contracts.graph.model_config.Hook] = , pre_hook: List[dbt.contracts.graph.model_config.Hook] = , quoting: Dict[str, Any] = , column_types: Dict[str, Any] = , full_refresh: Union[bool, NoneType] = None, unique_key: Union[str, List[str], NoneType] = None, on_schema_change: Union[str, NoneType] = 'ignore', quote_columns: Union[bool, NoneType] = None)" }, "CompiledSnapshotNode": { "type": "object", "required": [ "raw_sql", "compiled", "schema", "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "raw_sql": { "type": "string" }, "compiled": { "type": "boolean" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "snapshot" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1650055102.72113 }, "config_call_dict": { "type": "object", "default": {} }, "compiled_sql": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "CompiledSnapshotNode(raw_sql: str, compiled: bool, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , compiled_sql: Union[str, NoneType] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.compiled.InjectedCTE] = , relation_name: Union[str, NoneType] = None, _pre_injected_sql: Union[str, NoneType] = None)" }, "ParsedAnalysisNode": { "type": "object", "required": [ "raw_sql", "schema", "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "raw_sql": { "type": "string" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "analysis" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1650055102.7224698 }, "config_call_dict": { "type": "object", "default": {} } }, "additionalProperties": false, "description": "ParsedAnalysisNode(raw_sql: str, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = )" }, "ParsedSingularTestNode": { "type": "object", "required": [ "raw_sql", "schema", "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "raw_sql": { "type": "string" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "test" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/TestConfig", "default": { "enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0" } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1650055102.723626 }, "config_call_dict": { "type": "object", "default": {} } }, "additionalProperties": false, "description": "ParsedSingularTestNode(raw_sql: str, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.TestConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = )" }, "ParsedHookNode": { "type": "object", "required": [ "raw_sql", "schema", "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "raw_sql": { "type": "string" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "operation" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1650055102.724789 }, "config_call_dict": { "type": "object", "default": {} }, "index": { "oneOf": [ { "type": "integer" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "ParsedHookNode(raw_sql: str, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , index: Union[int, NoneType] = None)" }, "ParsedModelNode": { "type": "object", "required": [ "raw_sql", "schema", "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "raw_sql": { "type": "string" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "model" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1650055102.725993 }, "config_call_dict": { "type": "object", "default": {} } }, "additionalProperties": false, "description": "ParsedModelNode(raw_sql: str, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = )" }, "ParsedRPCNode": { "type": "object", "required": [ "raw_sql", "schema", "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "raw_sql": { "type": "string" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "rpc" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1650055102.727343 }, "config_call_dict": { "type": "object", "default": {} } }, "additionalProperties": false, "description": "ParsedRPCNode(raw_sql: str, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = )" }, "ParsedSqlNode": { "type": "object", "required": [ "raw_sql", "schema", "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "raw_sql": { "type": "string" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "sql" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1650055102.728572 }, "config_call_dict": { "type": "object", "default": {} } }, "additionalProperties": false, "description": "ParsedSqlNode(raw_sql: str, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = )" }, "ParsedGenericTestNode": { "type": "object", "required": [ "raw_sql", "test_metadata", "schema", "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "raw_sql": { "type": "string" }, "test_metadata": { "$ref": "#/definitions/TestMetadata" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "test" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/TestConfig", "default": { "enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0" } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1650055102.729745 }, "config_call_dict": { "type": "object", "default": {} }, "column_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "file_key_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "ParsedGenericTestNode(raw_sql: str, test_metadata: dbt.contracts.graph.parsed.TestMetadata, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.TestConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , column_name: Union[str, NoneType] = None, file_key_name: Union[str, NoneType] = None)" }, "ParsedSeedNode": { "type": "object", "required": [ "raw_sql", "schema", "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "raw_sql": { "type": "string" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "seed" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/SeedConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "seed", "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "quote_columns": null, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1650055102.731012 }, "config_call_dict": { "type": "object", "default": {} } }, "additionalProperties": false, "description": "ParsedSeedNode(raw_sql: str, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.SeedConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = )" }, "ParsedSnapshotNode": { "type": "object", "required": [ "raw_sql", "schema", "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum", "config" ], "properties": { "raw_sql": { "type": "string" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "snapshot" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/SnapshotConfig" }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1650055102.733336 }, "config_call_dict": { "type": "object", "default": {} } }, "additionalProperties": false, "description": "ParsedSnapshotNode(raw_sql: str, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.SnapshotConfig, _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = )" }, "SnapshotConfig": { "type": "object", "required": [], "properties": { "enabled": { "type": "boolean", "default": true }, "alias": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "tags": { "oneOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ], "default": [] }, "meta": { "type": "object", "default": {} }, "materialized": { "type": "string", "default": "snapshot" }, "persist_docs": { "type": "object", "default": {} }, "post-hook": { "type": "array", "items": { "$ref": "#/definitions/Hook" }, "default": [] }, "pre-hook": { "type": "array", "items": { "$ref": "#/definitions/Hook" }, "default": [] }, "quoting": { "type": "object", "default": {} }, "column_types": { "type": "object", "default": {} }, "full_refresh": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "unique_key": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "on_schema_change": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "strategy": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "target_schema": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "target_database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "updated_at": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "check_cols": { "oneOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ] } }, "additionalProperties": true, "description": "SnapshotConfig(_extra: Dict[str, Any] = , enabled: bool = True, alias: Union[str, NoneType] = None, schema: Union[str, NoneType] = None, database: Union[str, NoneType] = None, tags: Union[List[str], str] = , meta: Dict[str, Any] = , materialized: str = 'snapshot', persist_docs: Dict[str, Any] = , post_hook: List[dbt.contracts.graph.model_config.Hook] = , pre_hook: List[dbt.contracts.graph.model_config.Hook] = , quoting: Dict[str, Any] = , column_types: Dict[str, Any] = , full_refresh: Union[bool, NoneType] = None, unique_key: Union[str, NoneType] = None, on_schema_change: Union[str, NoneType] = 'ignore', strategy: Union[str, NoneType] = None, target_schema: Union[str, NoneType] = None, target_database: Union[str, NoneType] = None, updated_at: Union[str, NoneType] = None, check_cols: Union[str, List[str], NoneType] = None)" }, "ParsedSourceDefinition": { "type": "object", "required": [ "fqn", "schema", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "source_name", "source_description", "loader", "identifier", "resource_type" ], "properties": { "fqn": { "type": "array", "items": { "type": "string" } }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "source_name": { "type": "string" }, "source_description": { "type": "string" }, "loader": { "type": "string" }, "identifier": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "source" ] }, "quoting": { "$ref": "#/definitions/Quoting", "default": { "database": null, "schema": null, "identifier": null, "column": null } }, "loaded_at_field": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "freshness": { "oneOf": [ { "$ref": "#/definitions/FreshnessThreshold" }, { "type": "null" } ] }, "external": { "oneOf": [ { "$ref": "#/definitions/ExternalTable" }, { "type": "null" } ] }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "source_meta": { "type": "object", "default": {} }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "config": { "$ref": "#/definitions/SourceConfig", "default": { "enabled": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "unrendered_config": { "type": "object", "default": {} }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "created_at": { "type": "number", "default": 1650055102.735436 } }, "additionalProperties": false, "description": "ParsedSourceDefinition(fqn: List[str], database: Union[str, NoneType], schema: str, unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, source_name: str, source_description: str, loader: str, identifier: str, resource_type: dbt.node_types.NodeType, _event_status: Dict[str, Any] = , quoting: dbt.contracts.graph.unparsed.Quoting = , loaded_at_field: Union[str, NoneType] = None, freshness: Union[dbt.contracts.graph.unparsed.FreshnessThreshold, NoneType] = None, external: Union[dbt.contracts.graph.unparsed.ExternalTable, NoneType] = None, description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , source_meta: Dict[str, Any] = , tags: List[str] = , config: dbt.contracts.graph.model_config.SourceConfig = , patch_path: Union[pathlib.Path, NoneType] = None, unrendered_config: Dict[str, Any] = , relation_name: Union[str, NoneType] = None, created_at: float = )" }, "Quoting": { "type": "object", "required": [], "properties": { "database": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "schema": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "identifier": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "column": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "Quoting(database: Union[bool, NoneType] = None, schema: Union[bool, NoneType] = None, identifier: Union[bool, NoneType] = None, column: Union[bool, NoneType] = None)" }, "FreshnessThreshold": { "type": "object", "required": [], "properties": { "warn_after": { "oneOf": [ { "$ref": "#/definitions/Time" }, { "type": "null" } ], "default": { "count": null, "period": null } }, "error_after": { "oneOf": [ { "$ref": "#/definitions/Time" }, { "type": "null" } ], "default": { "count": null, "period": null } }, "filter": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "FreshnessThreshold(warn_after: Union[dbt.contracts.graph.unparsed.Time, NoneType] = , error_after: Union[dbt.contracts.graph.unparsed.Time, NoneType] = , filter: Union[str, NoneType] = None)" }, "FreshnessMetadata": { "type": "object", "required": [], "properties": { "dbt_schema_version": { "type": "string", "default": "https://schemas.getdbt.com/dbt/sources/v3.json" }, "dbt_version": { "type": "string", "default": "1.2.0a1" }, "generated_at": { "type": "string", "format": "date-time", "default": "2022-04-15T20:38:22.697740Z" }, "invocation_id": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "default": "34abf75e-59d3-442f-920c-fa3843d98014" }, "env": { "type": "object", "additionalProperties": { "type": "string" }, "default": {} } }, "additionalProperties": false, "description": "FreshnessMetadata(dbt_schema_version: str = , dbt_version: str = '1.2.0a1', generated_at: datetime.datetime = , invocation_id: Union[str, NoneType] = , env: Dict[str, str] = )" }, "SourceFreshnessRuntimeError": { "type": "object", "required": [ "unique_id", "status" ], "properties": { "unique_id": { "type": "string" }, "error": { "oneOf": [ { "type": "string" }, { "type": "integer" }, { "type": "null" } ] }, "status": { "type": "string", "enum": [ "runtime error" ] } }, "additionalProperties": false, "description": "SourceFreshnessRuntimeError(unique_id: str, error: Union[str, int, NoneType], status: dbt.contracts.results.FreshnessErrorEnum)" }, "SourceFreshnessOutput": { "type": "object", "required": [ "unique_id", "max_loaded_at", "snapshotted_at", "max_loaded_at_time_ago_in_s", "status", "criteria", "adapter_response", "timing", "thread_id", "execution_time" ], "properties": { "unique_id": { "type": "string" }, "max_loaded_at": { "type": "string", "format": "date-time" }, "snapshotted_at": { "type": "string", "format": "date-time" }, "max_loaded_at_time_ago_in_s": { "type": "number" }, "status": { "type": "string", "enum": [ "pass", "warn", "error", "runtime error" ] }, "criteria": { "$ref": "#/definitions/FreshnessThreshold" }, "adapter_response": { "type": "object" }, "timing": { "type": "array", "items": { "$ref": "#/definitions/TimingInfo" } }, "thread_id": { "type": "string" }, "execution_time": { "type": "number" } }, "additionalProperties": false, "description": "SourceFreshnessOutput(unique_id: str, max_loaded_at: datetime.datetime, snapshotted_at: datetime.datetime, max_loaded_at_time_ago_in_s: float, status: dbt.contracts.results.FreshnessStatus, criteria: dbt.contracts.graph.unparsed.FreshnessThreshold, adapter_response: Dict[str, Any], timing: List[dbt.contracts.results.TimingInfo], thread_id: str, execution_time: float)" }, "Time": { "type": "object", "required": [], "properties": { "count": { "oneOf": [ { "type": "integer" }, { "type": "null" } ] }, "period": { "oneOf": [ { "type": "string", "enum": [ "minute", "hour", "day" ] }, { "type": "null" } ] } }, "additionalProperties": false, "description": "Time(count: Union[int, NoneType] = None, period: Union[dbt.contracts.graph.unparsed.TimePeriod, NoneType] = None)" }, "TimingInfo": { "type": "object", "required": [ "name" ], "properties": { "name": { "type": "string" }, "started_at": { "oneOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ] }, "completed_at": { "oneOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "TimingInfo(name: str, started_at: Union[datetime.datetime, NoneType] = None, completed_at: Union[datetime.datetime, NoneType] = None)" }, "ExternalTable": { "type": "object", "required": [], "properties": { "location": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "file_format": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "row_format": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "tbl_properties": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "partitions": { "oneOf": [ { "type": "array", "items": { "$ref": "#/definitions/ExternalPartition" } }, { "type": "null" } ] } }, "additionalProperties": true, "description": "ExternalTable(_extra: Dict[str, Any] = , location: Union[str, NoneType] = None, file_format: Union[str, NoneType] = None, row_format: Union[str, NoneType] = None, tbl_properties: Union[str, NoneType] = None, partitions: Union[List[dbt.contracts.graph.unparsed.ExternalPartition], NoneType] = None)" }, "ExternalPartition": { "type": "object", "required": [], "properties": { "name": { "type": "string", "default": "" }, "description": { "type": "string", "default": "" }, "data_type": { "type": "string", "default": "" }, "meta": { "type": "object", "default": {} } }, "additionalProperties": true, "description": "ExternalPartition(_extra: Dict[str, Any] = , name: str = '', description: str = '', data_type: str = '', meta: Dict[str, Any] = )" }, "SourceConfig": { "type": "object", "required": [], "properties": { "enabled": { "type": "boolean", "default": true } }, "additionalProperties": true, "description": "SourceConfig(_extra: Dict[str, Any] = , enabled: bool = True)" }, "ParsedMacro": { "type": "object", "required": [ "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "macro_sql", "resource_type" ], "properties": { "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "macro_sql": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "macro" ] }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "depends_on": { "$ref": "#/definitions/MacroDependsOn", "default": { "macros": [] } }, "description": { "type": "string", "default": "" }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "arguments": { "type": "array", "items": { "$ref": "#/definitions/MacroArgument" }, "default": [] }, "created_at": { "type": "number", "default": 1650055102.736266 } }, "additionalProperties": false, "description": "ParsedMacro(unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, macro_sql: str, resource_type: dbt.node_types.NodeType, tags: List[str] = , depends_on: dbt.contracts.graph.parsed.MacroDependsOn = , description: str = '', meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, arguments: List[dbt.contracts.graph.unparsed.MacroArgument] = , created_at: float = )" }, "MacroDependsOn": { "type": "object", "required": [], "properties": { "macros": { "type": "array", "items": { "type": "string" }, "default": [] } }, "additionalProperties": false, "description": "MacroDependsOn(macros: List[str] = )" }, "MacroArgument": { "type": "object", "required": [ "name" ], "properties": { "name": { "type": "string" }, "type": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "description": { "type": "string", "default": "" } }, "additionalProperties": false, "description": "MacroArgument(name: str, type: Union[str, NoneType] = None, description: str = '')" }, "ParsedDocumentation": { "type": "object", "required": [ "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "block_contents" ], "properties": { "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "block_contents": { "type": "string" } }, "additionalProperties": false, "description": "ParsedDocumentation(unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, block_contents: str)" }, "ParsedExposure": { "type": "object", "required": [ "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "type", "owner" ], "properties": { "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "type": { "type": "string", "enum": [ "dashboard", "notebook", "analysis", "ml", "application" ] }, "owner": { "$ref": "#/definitions/ExposureOwner" }, "resource_type": { "type": "string", "enum": [ "model", "analysis", "test", "snapshot", "operation", "seed", "rpc", "sql", "docs", "source", "macro", "exposure", "metric" ], "default": "exposure" }, "description": { "type": "string", "default": "" }, "maturity": { "oneOf": [ { "type": "string", "enum": [ "low", "medium", "high" ] }, { "type": "null" } ] }, "meta": { "type": "object", "default": {} }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "url": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "created_at": { "type": "number", "default": 1650055102.7375019 } }, "additionalProperties": false, "description": "ParsedExposure(fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, type: dbt.contracts.graph.unparsed.ExposureType, owner: dbt.contracts.graph.unparsed.ExposureOwner, resource_type: dbt.node_types.NodeType = , description: str = '', maturity: Union[dbt.contracts.graph.unparsed.MaturityType, NoneType] = None, meta: Dict[str, Any] = , tags: List[str] = , url: Union[str, NoneType] = None, depends_on: dbt.contracts.graph.parsed.DependsOn = , refs: List[List[str]] = , sources: List[List[str]] = , created_at: float = )" }, "ExposureOwner": { "type": "object", "required": [ "email" ], "properties": { "email": { "type": "string" }, "name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "ExposureOwner(email: str, name: Union[str, NoneType] = None)" }, "ParsedMetric": { "type": "object", "required": [ "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "model", "name", "description", "label", "type", "filters", "time_grains", "dimensions" ], "properties": { "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "model": { "type": "string" }, "name": { "type": "string" }, "description": { "type": "string" }, "label": { "type": "string" }, "type": { "type": "string" }, "sql": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "timestamp": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "filters": { "type": "array", "items": { "$ref": "#/definitions/MetricFilter" } }, "time_grains": { "type": "array", "items": { "type": "string" } }, "dimensions": { "type": "array", "items": { "type": "string" } }, "resource_type": { "type": "string", "enum": [ "model", "analysis", "test", "snapshot", "operation", "seed", "rpc", "sql", "docs", "source", "macro", "exposure", "metric" ], "default": "metric" }, "meta": { "type": "object", "default": {} }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "created_at": { "type": "number", "default": 1650055102.738508 } }, "additionalProperties": false, "description": "ParsedMetric(fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, model: str, name: str, description: str, label: str, type: str, sql: Union[str, NoneType], timestamp: Union[str, NoneType], filters: List[dbt.contracts.graph.unparsed.MetricFilter], time_grains: List[str], dimensions: List[str], resource_type: dbt.node_types.NodeType = , meta: Dict[str, Any] = , tags: List[str] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , refs: List[List[str]] = , created_at: float = )" }, "MetricFilter": { "type": "object", "required": [ "field", "operator", "value" ], "properties": { "field": { "type": "string" }, "operator": { "type": "string" }, "value": { "type": "string" } }, "additionalProperties": false, "description": "MetricFilter(field: str, operator: str, value: str)" } }, "$schema": "http://json-schema.org/draft-07/schema#", "$id": "https://schemas.getdbt.com/dbt/manifest/v5.json" } ================================================ FILE: schemas/dbt/manifest/v6.json ================================================ { "type": "object", "required": [ "metadata", "nodes", "sources", "macros", "docs", "exposures", "metrics", "selectors" ], "properties": { "metadata": { "$ref": "#/definitions/ManifestMetadata", "description": "Metadata about the manifest" }, "nodes": { "type": "object", "additionalProperties": { "oneOf": [ { "$ref": "#/definitions/CompiledAnalysisNode" }, { "$ref": "#/definitions/CompiledSingularTestNode" }, { "$ref": "#/definitions/CompiledModelNode" }, { "$ref": "#/definitions/CompiledHookNode" }, { "$ref": "#/definitions/CompiledRPCNode" }, { "$ref": "#/definitions/CompiledSqlNode" }, { "$ref": "#/definitions/CompiledGenericTestNode" }, { "$ref": "#/definitions/CompiledSeedNode" }, { "$ref": "#/definitions/CompiledSnapshotNode" }, { "$ref": "#/definitions/ParsedAnalysisNode" }, { "$ref": "#/definitions/ParsedSingularTestNode" }, { "$ref": "#/definitions/ParsedHookNode" }, { "$ref": "#/definitions/ParsedModelNode" }, { "$ref": "#/definitions/ParsedRPCNode" }, { "$ref": "#/definitions/ParsedSqlNode" }, { "$ref": "#/definitions/ParsedGenericTestNode" }, { "$ref": "#/definitions/ParsedSeedNode" }, { "$ref": "#/definitions/ParsedSnapshotNode" } ] }, "description": "The nodes defined in the dbt project and its dependencies" }, "sources": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ParsedSourceDefinition" }, "description": "The sources defined in the dbt project and its dependencies" }, "macros": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ParsedMacro" }, "description": "The macros defined in the dbt project and its dependencies" }, "docs": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ParsedDocumentation" }, "description": "The docs defined in the dbt project and its dependencies" }, "exposures": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ParsedExposure" }, "description": "The exposures defined in the dbt project and its dependencies" }, "metrics": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ParsedMetric" }, "description": "The metrics defined in the dbt project and its dependencies" }, "selectors": { "type": "object", "description": "The selectors defined in selectors.yml" }, "disabled": { "oneOf": [ { "type": "object", "additionalProperties": { "type": "array", "items": { "oneOf": [ { "$ref": "#/definitions/CompiledAnalysisNode" }, { "$ref": "#/definitions/CompiledSingularTestNode" }, { "$ref": "#/definitions/CompiledModelNode" }, { "$ref": "#/definitions/CompiledHookNode" }, { "$ref": "#/definitions/CompiledRPCNode" }, { "$ref": "#/definitions/CompiledSqlNode" }, { "$ref": "#/definitions/CompiledGenericTestNode" }, { "$ref": "#/definitions/CompiledSeedNode" }, { "$ref": "#/definitions/CompiledSnapshotNode" }, { "$ref": "#/definitions/ParsedAnalysisNode" }, { "$ref": "#/definitions/ParsedSingularTestNode" }, { "$ref": "#/definitions/ParsedHookNode" }, { "$ref": "#/definitions/ParsedModelNode" }, { "$ref": "#/definitions/ParsedRPCNode" }, { "$ref": "#/definitions/ParsedSqlNode" }, { "$ref": "#/definitions/ParsedGenericTestNode" }, { "$ref": "#/definitions/ParsedSeedNode" }, { "$ref": "#/definitions/ParsedSnapshotNode" }, { "$ref": "#/definitions/ParsedSourceDefinition" } ] } } }, { "type": "null" } ], "description": "A mapping of the disabled nodes in the target" }, "parent_map": { "oneOf": [ { "type": "object", "additionalProperties": { "type": "array", "items": { "type": "string" } } }, { "type": "null" } ], "description": "A mapping from\u00a0child nodes to their dependencies" }, "child_map": { "oneOf": [ { "type": "object", "additionalProperties": { "type": "array", "items": { "type": "string" } } }, { "type": "null" } ], "description": "A mapping from parent nodes to their dependents" } }, "additionalProperties": false, "description": "WritableManifest(metadata: dbt.contracts.graph.manifest.ManifestMetadata, nodes: Mapping[str, Union[dbt.contracts.graph.compiled.CompiledAnalysisNode, dbt.contracts.graph.compiled.CompiledSingularTestNode, dbt.contracts.graph.compiled.CompiledModelNode, dbt.contracts.graph.compiled.CompiledHookNode, dbt.contracts.graph.compiled.CompiledRPCNode, dbt.contracts.graph.compiled.CompiledSqlNode, dbt.contracts.graph.compiled.CompiledGenericTestNode, dbt.contracts.graph.compiled.CompiledSeedNode, dbt.contracts.graph.compiled.CompiledSnapshotNode, dbt.contracts.graph.parsed.ParsedAnalysisNode, dbt.contracts.graph.parsed.ParsedSingularTestNode, dbt.contracts.graph.parsed.ParsedHookNode, dbt.contracts.graph.parsed.ParsedModelNode, dbt.contracts.graph.parsed.ParsedRPCNode, dbt.contracts.graph.parsed.ParsedSqlNode, dbt.contracts.graph.parsed.ParsedGenericTestNode, dbt.contracts.graph.parsed.ParsedSeedNode, dbt.contracts.graph.parsed.ParsedSnapshotNode]], sources: Mapping[str, dbt.contracts.graph.parsed.ParsedSourceDefinition], macros: Mapping[str, dbt.contracts.graph.parsed.ParsedMacro], docs: Mapping[str, dbt.contracts.graph.parsed.ParsedDocumentation], exposures: Mapping[str, dbt.contracts.graph.parsed.ParsedExposure], metrics: Mapping[str, dbt.contracts.graph.parsed.ParsedMetric], selectors: Mapping[str, Any], disabled: Optional[Mapping[str, List[Union[dbt.contracts.graph.compiled.CompiledAnalysisNode, dbt.contracts.graph.compiled.CompiledSingularTestNode, dbt.contracts.graph.compiled.CompiledModelNode, dbt.contracts.graph.compiled.CompiledHookNode, dbt.contracts.graph.compiled.CompiledRPCNode, dbt.contracts.graph.compiled.CompiledSqlNode, dbt.contracts.graph.compiled.CompiledGenericTestNode, dbt.contracts.graph.compiled.CompiledSeedNode, dbt.contracts.graph.compiled.CompiledSnapshotNode, dbt.contracts.graph.parsed.ParsedAnalysisNode, dbt.contracts.graph.parsed.ParsedSingularTestNode, dbt.contracts.graph.parsed.ParsedHookNode, dbt.contracts.graph.parsed.ParsedModelNode, dbt.contracts.graph.parsed.ParsedRPCNode, dbt.contracts.graph.parsed.ParsedSqlNode, dbt.contracts.graph.parsed.ParsedGenericTestNode, dbt.contracts.graph.parsed.ParsedSeedNode, dbt.contracts.graph.parsed.ParsedSnapshotNode, dbt.contracts.graph.parsed.ParsedSourceDefinition]]]], parent_map: Optional[Dict[str, List[str]]], child_map: Optional[Dict[str, List[str]]])", "definitions": { "ManifestMetadata": { "type": "object", "required": [], "properties": { "dbt_schema_version": { "type": "string", "default": "https://schemas.getdbt.com/dbt/manifest/v6.json" }, "dbt_version": { "type": "string", "default": "1.2.0a1" }, "generated_at": { "type": "string", "format": "date-time", "default": "2022-07-05T17:06:02.022011Z" }, "invocation_id": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "default": "6cec400a-e8a4-4480-8ceb-d35589d12bab" }, "env": { "type": "object", "additionalProperties": { "type": "string" }, "default": {} }, "project_id": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "description": "A unique identifier for the project" }, "user_id": { "oneOf": [ { "type": "string", "pattern": "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" }, { "type": "null" } ], "description": "A unique identifier for the user" }, "send_anonymous_usage_stats": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ], "description": "Whether dbt is configured to send anonymous usage statistics" }, "adapter_type": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "description": "The type name of the adapter" } }, "additionalProperties": false, "description": "Metadata for the manifest." }, "CompiledAnalysisNode": { "type": "object", "required": [ "raw_sql", "compiled", "schema", "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "raw_sql": { "type": "string" }, "compiled": { "type": "boolean" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "analysis" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1657040762.0262258 }, "config_call_dict": { "type": "object", "default": {} }, "compiled_sql": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "CompiledAnalysisNode(raw_sql: str, compiled: bool, database: Optional[str], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, compiled_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , compiled_sql: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.compiled.InjectedCTE] = , relation_name: Optional[str] = None, _pre_injected_sql: Optional[str] = None)" }, "FileHash": { "type": "object", "required": [ "name", "checksum" ], "properties": { "name": { "type": "string" }, "checksum": { "type": "string" } }, "additionalProperties": false, "description": "FileHash(name: str, checksum: str)" }, "NodeConfig": { "type": "object", "required": [], "properties": { "enabled": { "type": "boolean", "default": true }, "alias": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "tags": { "oneOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ], "default": [] }, "meta": { "type": "object", "default": {} }, "materialized": { "type": "string", "default": "view" }, "persist_docs": { "type": "object", "default": {} }, "post-hook": { "type": "array", "items": { "$ref": "#/definitions/Hook" }, "default": [] }, "pre-hook": { "type": "array", "items": { "$ref": "#/definitions/Hook" }, "default": [] }, "quoting": { "type": "object", "default": {} }, "column_types": { "type": "object", "default": {} }, "full_refresh": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "unique_key": { "oneOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ] }, "on_schema_change": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "grants": { "type": "object", "default": {} } }, "additionalProperties": true, "description": "NodeConfig(_extra: Dict[str, Any] = , enabled: bool = True, alias: Optional[str] = None, schema: Optional[str] = None, database: Optional[str] = None, tags: Union[List[str], str] = , meta: Dict[str, Any] = , materialized: str = 'view', persist_docs: Dict[str, Any] = , post_hook: List[dbt.contracts.graph.model_config.Hook] = , pre_hook: List[dbt.contracts.graph.model_config.Hook] = , quoting: Dict[str, Any] = , column_types: Dict[str, Any] = , full_refresh: Optional[bool] = None, unique_key: Union[str, List[str], NoneType] = None, on_schema_change: Optional[str] = 'ignore', grants: Dict[str, Any] = )" }, "Hook": { "type": "object", "required": [ "sql" ], "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "oneOf": [ { "type": "integer" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "Hook(sql: str, transaction: bool = True, index: Optional[int] = None)" }, "DependsOn": { "type": "object", "required": [], "properties": { "macros": { "type": "array", "items": { "type": "string" }, "default": [] }, "nodes": { "type": "array", "items": { "type": "string" }, "default": [] } }, "additionalProperties": false, "description": "DependsOn(macros: List[str] = , nodes: List[str] = )" }, "ColumnInfo": { "type": "object", "required": [ "name" ], "properties": { "name": { "type": "string" }, "description": { "type": "string", "default": "" }, "meta": { "type": "object", "default": {} }, "data_type": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "quote": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] } }, "additionalProperties": true, "description": "ColumnInfo(name: str, description: str = '', meta: Dict[str, Any] = , data_type: Optional[str] = None, quote: Optional[bool] = None, tags: List[str] = , _extra: Dict[str, Any] = )" }, "Docs": { "type": "object", "required": [], "properties": { "show": { "type": "boolean", "default": true } }, "additionalProperties": false, "description": "Docs(show: bool = True)" }, "InjectedCTE": { "type": "object", "required": [ "id", "sql" ], "properties": { "id": { "type": "string" }, "sql": { "type": "string" } }, "additionalProperties": false, "description": "InjectedCTE(id: str, sql: str)" }, "CompiledSingularTestNode": { "type": "object", "required": [ "raw_sql", "compiled", "schema", "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "raw_sql": { "type": "string" }, "compiled": { "type": "boolean" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "test" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/TestConfig", "default": { "enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0" } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1657040762.028276 }, "config_call_dict": { "type": "object", "default": {} }, "compiled_sql": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "CompiledSingularTestNode(raw_sql: str, compiled: bool, database: Optional[str], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.TestConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, compiled_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , compiled_sql: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.compiled.InjectedCTE] = , relation_name: Optional[str] = None, _pre_injected_sql: Optional[str] = None)" }, "TestConfig": { "type": "object", "required": [], "properties": { "enabled": { "type": "boolean", "default": true }, "alias": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "default": "dbt_test__audit" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "tags": { "oneOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ], "default": [] }, "meta": { "type": "object", "default": {} }, "materialized": { "type": "string", "default": "test" }, "severity": { "type": "string", "pattern": "^([Ww][Aa][Rr][Nn]|[Ee][Rr][Rr][Oo][Rr])$", "default": "ERROR" }, "store_failures": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "where": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "limit": { "oneOf": [ { "type": "integer" }, { "type": "null" } ] }, "fail_calc": { "type": "string", "default": "count(*)" }, "warn_if": { "type": "string", "default": "!= 0" }, "error_if": { "type": "string", "default": "!= 0" } }, "additionalProperties": true, "description": "TestConfig(_extra: Dict[str, Any] = , enabled: bool = True, alias: Optional[str] = None, schema: Optional[str] = 'dbt_test__audit', database: Optional[str] = None, tags: Union[List[str], str] = , meta: Dict[str, Any] = , materialized: str = 'test', severity: dbt.contracts.graph.model_config.Severity = 'ERROR', store_failures: Optional[bool] = None, where: Optional[str] = None, limit: Optional[int] = None, fail_calc: str = 'count(*)', warn_if: str = '!= 0', error_if: str = '!= 0')" }, "CompiledModelNode": { "type": "object", "required": [ "raw_sql", "compiled", "schema", "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "raw_sql": { "type": "string" }, "compiled": { "type": "boolean" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "model" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1657040762.0294158 }, "config_call_dict": { "type": "object", "default": {} }, "compiled_sql": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "CompiledModelNode(raw_sql: str, compiled: bool, database: Optional[str], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, compiled_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , compiled_sql: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.compiled.InjectedCTE] = , relation_name: Optional[str] = None, _pre_injected_sql: Optional[str] = None)" }, "CompiledHookNode": { "type": "object", "required": [ "raw_sql", "compiled", "schema", "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "raw_sql": { "type": "string" }, "compiled": { "type": "boolean" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "operation" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1657040762.030488 }, "config_call_dict": { "type": "object", "default": {} }, "compiled_sql": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "index": { "oneOf": [ { "type": "integer" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "CompiledHookNode(raw_sql: str, compiled: bool, database: Optional[str], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, compiled_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , compiled_sql: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.compiled.InjectedCTE] = , relation_name: Optional[str] = None, _pre_injected_sql: Optional[str] = None, index: Optional[int] = None)" }, "CompiledRPCNode": { "type": "object", "required": [ "raw_sql", "compiled", "schema", "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "raw_sql": { "type": "string" }, "compiled": { "type": "boolean" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "rpc" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1657040762.031699 }, "config_call_dict": { "type": "object", "default": {} }, "compiled_sql": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "CompiledRPCNode(raw_sql: str, compiled: bool, database: Optional[str], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, compiled_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , compiled_sql: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.compiled.InjectedCTE] = , relation_name: Optional[str] = None, _pre_injected_sql: Optional[str] = None)" }, "CompiledSqlNode": { "type": "object", "required": [ "raw_sql", "compiled", "schema", "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "raw_sql": { "type": "string" }, "compiled": { "type": "boolean" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "sql" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1657040762.032803 }, "config_call_dict": { "type": "object", "default": {} }, "compiled_sql": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "CompiledSqlNode(raw_sql: str, compiled: bool, database: Optional[str], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, compiled_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , compiled_sql: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.compiled.InjectedCTE] = , relation_name: Optional[str] = None, _pre_injected_sql: Optional[str] = None)" }, "CompiledGenericTestNode": { "type": "object", "required": [ "raw_sql", "test_metadata", "compiled", "schema", "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "raw_sql": { "type": "string" }, "test_metadata": { "$ref": "#/definitions/TestMetadata" }, "compiled": { "type": "boolean" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "test" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/TestConfig", "default": { "enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0" } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1657040762.0341172 }, "config_call_dict": { "type": "object", "default": {} }, "compiled_sql": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "column_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "file_key_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "CompiledGenericTestNode(raw_sql: str, test_metadata: dbt.contracts.graph.parsed.TestMetadata, compiled: bool, database: Optional[str], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.TestConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, compiled_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , compiled_sql: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.compiled.InjectedCTE] = , relation_name: Optional[str] = None, _pre_injected_sql: Optional[str] = None, column_name: Optional[str] = None, file_key_name: Optional[str] = None)" }, "TestMetadata": { "type": "object", "required": [ "name" ], "properties": { "name": { "type": "string" }, "kwargs": { "type": "object", "default": {} }, "namespace": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "TestMetadata(name: str, kwargs: Dict[str, Any] = , namespace: Optional[str] = None)" }, "CompiledSeedNode": { "type": "object", "required": [ "raw_sql", "compiled", "schema", "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "raw_sql": { "type": "string" }, "compiled": { "type": "boolean" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "seed" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/SeedConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "seed", "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "quote_columns": null, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1657040762.035997 }, "config_call_dict": { "type": "object", "default": {} }, "compiled_sql": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "CompiledSeedNode(raw_sql: str, compiled: bool, database: Optional[str], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.SeedConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, compiled_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , compiled_sql: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.compiled.InjectedCTE] = , relation_name: Optional[str] = None, _pre_injected_sql: Optional[str] = None)" }, "SeedConfig": { "type": "object", "required": [], "properties": { "enabled": { "type": "boolean", "default": true }, "alias": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "tags": { "oneOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ], "default": [] }, "meta": { "type": "object", "default": {} }, "materialized": { "type": "string", "default": "seed" }, "persist_docs": { "type": "object", "default": {} }, "post-hook": { "type": "array", "items": { "$ref": "#/definitions/Hook" }, "default": [] }, "pre-hook": { "type": "array", "items": { "$ref": "#/definitions/Hook" }, "default": [] }, "quoting": { "type": "object", "default": {} }, "column_types": { "type": "object", "default": {} }, "full_refresh": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "unique_key": { "oneOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ] }, "on_schema_change": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "grants": { "type": "object", "default": {} }, "quote_columns": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] } }, "additionalProperties": true, "description": "SeedConfig(_extra: Dict[str, Any] = , enabled: bool = True, alias: Optional[str] = None, schema: Optional[str] = None, database: Optional[str] = None, tags: Union[List[str], str] = , meta: Dict[str, Any] = , materialized: str = 'seed', persist_docs: Dict[str, Any] = , post_hook: List[dbt.contracts.graph.model_config.Hook] = , pre_hook: List[dbt.contracts.graph.model_config.Hook] = , quoting: Dict[str, Any] = , column_types: Dict[str, Any] = , full_refresh: Optional[bool] = None, unique_key: Union[str, List[str], NoneType] = None, on_schema_change: Optional[str] = 'ignore', grants: Dict[str, Any] = , quote_columns: Optional[bool] = None)" }, "CompiledSnapshotNode": { "type": "object", "required": [ "raw_sql", "compiled", "schema", "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "raw_sql": { "type": "string" }, "compiled": { "type": "boolean" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "snapshot" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1657040762.037313 }, "config_call_dict": { "type": "object", "default": {} }, "compiled_sql": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "CompiledSnapshotNode(raw_sql: str, compiled: bool, database: Optional[str], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, compiled_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , compiled_sql: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.compiled.InjectedCTE] = , relation_name: Optional[str] = None, _pre_injected_sql: Optional[str] = None)" }, "ParsedAnalysisNode": { "type": "object", "required": [ "raw_sql", "schema", "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "raw_sql": { "type": "string" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "analysis" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1657040762.038346 }, "config_call_dict": { "type": "object", "default": {} } }, "additionalProperties": false, "description": "ParsedAnalysisNode(raw_sql: str, database: Optional[str], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, compiled_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = )" }, "ParsedSingularTestNode": { "type": "object", "required": [ "raw_sql", "schema", "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "raw_sql": { "type": "string" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "test" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/TestConfig", "default": { "enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0" } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1657040762.039365 }, "config_call_dict": { "type": "object", "default": {} } }, "additionalProperties": false, "description": "ParsedSingularTestNode(raw_sql: str, database: Optional[str], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.TestConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, compiled_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = )" }, "ParsedHookNode": { "type": "object", "required": [ "raw_sql", "schema", "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "raw_sql": { "type": "string" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "operation" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1657040762.0402749 }, "config_call_dict": { "type": "object", "default": {} }, "index": { "oneOf": [ { "type": "integer" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "ParsedHookNode(raw_sql: str, database: Optional[str], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, compiled_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , index: Optional[int] = None)" }, "ParsedModelNode": { "type": "object", "required": [ "raw_sql", "schema", "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "raw_sql": { "type": "string" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "model" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1657040762.041198 }, "config_call_dict": { "type": "object", "default": {} } }, "additionalProperties": false, "description": "ParsedModelNode(raw_sql: str, database: Optional[str], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, compiled_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = )" }, "ParsedRPCNode": { "type": "object", "required": [ "raw_sql", "schema", "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "raw_sql": { "type": "string" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "rpc" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1657040762.0420969 }, "config_call_dict": { "type": "object", "default": {} } }, "additionalProperties": false, "description": "ParsedRPCNode(raw_sql: str, database: Optional[str], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, compiled_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = )" }, "ParsedSqlNode": { "type": "object", "required": [ "raw_sql", "schema", "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "raw_sql": { "type": "string" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "sql" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1657040762.0429912 }, "config_call_dict": { "type": "object", "default": {} } }, "additionalProperties": false, "description": "ParsedSqlNode(raw_sql: str, database: Optional[str], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, compiled_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = )" }, "ParsedGenericTestNode": { "type": "object", "required": [ "raw_sql", "test_metadata", "schema", "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "raw_sql": { "type": "string" }, "test_metadata": { "$ref": "#/definitions/TestMetadata" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "test" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/TestConfig", "default": { "enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0" } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1657040762.043911 }, "config_call_dict": { "type": "object", "default": {} }, "column_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "file_key_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "ParsedGenericTestNode(raw_sql: str, test_metadata: dbt.contracts.graph.parsed.TestMetadata, database: Optional[str], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.TestConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, compiled_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , column_name: Optional[str] = None, file_key_name: Optional[str] = None)" }, "ParsedSeedNode": { "type": "object", "required": [ "raw_sql", "schema", "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "raw_sql": { "type": "string" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "seed" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/SeedConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "seed", "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "quote_columns": null, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1657040762.044916 }, "config_call_dict": { "type": "object", "default": {} } }, "additionalProperties": false, "description": "ParsedSeedNode(raw_sql: str, database: Optional[str], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.SeedConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, compiled_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = )" }, "ParsedSnapshotNode": { "type": "object", "required": [ "raw_sql", "schema", "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum", "config" ], "properties": { "raw_sql": { "type": "string" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "snapshot" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/SnapshotConfig" }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1657040762.04684 }, "config_call_dict": { "type": "object", "default": {} } }, "additionalProperties": false, "description": "ParsedSnapshotNode(raw_sql: str, database: Optional[str], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.SnapshotConfig, _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, compiled_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = )" }, "SnapshotConfig": { "type": "object", "required": [], "properties": { "enabled": { "type": "boolean", "default": true }, "alias": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "tags": { "oneOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ], "default": [] }, "meta": { "type": "object", "default": {} }, "materialized": { "type": "string", "default": "snapshot" }, "persist_docs": { "type": "object", "default": {} }, "post-hook": { "type": "array", "items": { "$ref": "#/definitions/Hook" }, "default": [] }, "pre-hook": { "type": "array", "items": { "$ref": "#/definitions/Hook" }, "default": [] }, "quoting": { "type": "object", "default": {} }, "column_types": { "type": "object", "default": {} }, "full_refresh": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "unique_key": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "on_schema_change": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "grants": { "type": "object", "default": {} }, "strategy": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "target_schema": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "target_database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "updated_at": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "check_cols": { "oneOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ] } }, "additionalProperties": true, "description": "SnapshotConfig(_extra: Dict[str, Any] = , enabled: bool = True, alias: Optional[str] = None, schema: Optional[str] = None, database: Optional[str] = None, tags: Union[List[str], str] = , meta: Dict[str, Any] = , materialized: str = 'snapshot', persist_docs: Dict[str, Any] = , post_hook: List[dbt.contracts.graph.model_config.Hook] = , pre_hook: List[dbt.contracts.graph.model_config.Hook] = , quoting: Dict[str, Any] = , column_types: Dict[str, Any] = , full_refresh: Optional[bool] = None, unique_key: Optional[str] = None, on_schema_change: Optional[str] = 'ignore', grants: Dict[str, Any] = , strategy: Optional[str] = None, target_schema: Optional[str] = None, target_database: Optional[str] = None, updated_at: Optional[str] = None, check_cols: Union[str, List[str], NoneType] = None)" }, "ParsedSourceDefinition": { "type": "object", "required": [ "fqn", "schema", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "source_name", "source_description", "loader", "identifier", "resource_type" ], "properties": { "fqn": { "type": "array", "items": { "type": "string" } }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "source_name": { "type": "string" }, "source_description": { "type": "string" }, "loader": { "type": "string" }, "identifier": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "source" ] }, "quoting": { "$ref": "#/definitions/Quoting", "default": { "database": null, "schema": null, "identifier": null, "column": null } }, "loaded_at_field": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "freshness": { "oneOf": [ { "$ref": "#/definitions/FreshnessThreshold" }, { "type": "null" } ] }, "external": { "oneOf": [ { "$ref": "#/definitions/ExternalTable" }, { "type": "null" } ] }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "source_meta": { "type": "object", "default": {} }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "config": { "$ref": "#/definitions/SourceConfig", "default": { "enabled": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "unrendered_config": { "type": "object", "default": {} }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "created_at": { "type": "number", "default": 1657040762.048556 } }, "additionalProperties": false, "description": "ParsedSourceDefinition(fqn: List[str], database: Optional[str], schema: str, unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, source_name: str, source_description: str, loader: str, identifier: str, resource_type: dbt.node_types.NodeType, _event_status: Dict[str, Any] = , quoting: dbt.contracts.graph.unparsed.Quoting = , loaded_at_field: Optional[str] = None, freshness: Optional[dbt.contracts.graph.unparsed.FreshnessThreshold] = None, external: Optional[dbt.contracts.graph.unparsed.ExternalTable] = None, description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , source_meta: Dict[str, Any] = , tags: List[str] = , config: dbt.contracts.graph.model_config.SourceConfig = , patch_path: Optional[pathlib.Path] = None, unrendered_config: Dict[str, Any] = , relation_name: Optional[str] = None, created_at: float = )" }, "Quoting": { "type": "object", "required": [], "properties": { "database": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "schema": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "identifier": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "column": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "Quoting(database: Optional[bool] = None, schema: Optional[bool] = None, identifier: Optional[bool] = None, column: Optional[bool] = None)" }, "FreshnessThreshold": { "type": "object", "required": [], "properties": { "warn_after": { "oneOf": [ { "$ref": "#/definitions/Time" }, { "type": "null" } ], "default": { "count": null, "period": null } }, "error_after": { "oneOf": [ { "$ref": "#/definitions/Time" }, { "type": "null" } ], "default": { "count": null, "period": null } }, "filter": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "FreshnessThreshold(warn_after: Optional[dbt.contracts.graph.unparsed.Time] = , error_after: Optional[dbt.contracts.graph.unparsed.Time] = , filter: Optional[str] = None)" }, "FreshnessMetadata": { "type": "object", "required": [], "properties": { "dbt_schema_version": { "type": "string", "default": "https://schemas.getdbt.com/dbt/sources/v3.json" }, "dbt_version": { "type": "string", "default": "1.2.0a1" }, "generated_at": { "type": "string", "format": "date-time", "default": "2022-07-05T17:06:02.017833Z" }, "invocation_id": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "default": "6cec400a-e8a4-4480-8ceb-d35589d12bab" }, "env": { "type": "object", "additionalProperties": { "type": "string" }, "default": {} } }, "additionalProperties": false, "description": "FreshnessMetadata(dbt_schema_version: str = , dbt_version: str = '1.2.0a1', generated_at: datetime.datetime = , invocation_id: Optional[str] = , env: Dict[str, str] = )" }, "SourceFreshnessRuntimeError": { "type": "object", "required": [ "unique_id", "status" ], "properties": { "unique_id": { "type": "string" }, "error": { "oneOf": [ { "type": "string" }, { "type": "integer" }, { "type": "null" } ] }, "status": { "type": "string", "enum": [ "runtime error" ] } }, "additionalProperties": false, "description": "SourceFreshnessRuntimeError(unique_id: str, error: Union[str, int, NoneType], status: dbt.contracts.results.FreshnessErrorEnum)" }, "SourceFreshnessOutput": { "type": "object", "required": [ "unique_id", "max_loaded_at", "snapshotted_at", "max_loaded_at_time_ago_in_s", "status", "criteria", "adapter_response", "timing", "thread_id", "execution_time" ], "properties": { "unique_id": { "type": "string" }, "max_loaded_at": { "type": "string", "format": "date-time" }, "snapshotted_at": { "type": "string", "format": "date-time" }, "max_loaded_at_time_ago_in_s": { "type": "number" }, "status": { "type": "string", "enum": [ "pass", "warn", "error", "runtime error" ] }, "criteria": { "$ref": "#/definitions/FreshnessThreshold" }, "adapter_response": { "type": "object" }, "timing": { "type": "array", "items": { "$ref": "#/definitions/TimingInfo" } }, "thread_id": { "type": "string" }, "execution_time": { "type": "number" } }, "additionalProperties": false, "description": "SourceFreshnessOutput(unique_id: str, max_loaded_at: datetime.datetime, snapshotted_at: datetime.datetime, max_loaded_at_time_ago_in_s: float, status: dbt.contracts.results.FreshnessStatus, criteria: dbt.contracts.graph.unparsed.FreshnessThreshold, adapter_response: Dict[str, Any], timing: List[dbt.contracts.results.TimingInfo], thread_id: str, execution_time: float)" }, "Time": { "type": "object", "required": [], "properties": { "count": { "oneOf": [ { "type": "integer" }, { "type": "null" } ] }, "period": { "oneOf": [ { "type": "string", "enum": [ "minute", "hour", "day" ] }, { "type": "null" } ] } }, "additionalProperties": false, "description": "Time(count: Optional[int] = None, period: Optional[dbt.contracts.graph.unparsed.TimePeriod] = None)" }, "TimingInfo": { "type": "object", "required": [ "name" ], "properties": { "name": { "type": "string" }, "started_at": { "oneOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ] }, "completed_at": { "oneOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "TimingInfo(name: str, started_at: Optional[datetime.datetime] = None, completed_at: Optional[datetime.datetime] = None)" }, "ExternalTable": { "type": "object", "required": [], "properties": { "location": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "file_format": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "row_format": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "tbl_properties": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "partitions": { "oneOf": [ { "type": "array", "items": { "$ref": "#/definitions/ExternalPartition" } }, { "type": "null" } ] } }, "additionalProperties": true, "description": "ExternalTable(_extra: Dict[str, Any] = , location: Optional[str] = None, file_format: Optional[str] = None, row_format: Optional[str] = None, tbl_properties: Optional[str] = None, partitions: Optional[List[dbt.contracts.graph.unparsed.ExternalPartition]] = None)" }, "ExternalPartition": { "type": "object", "required": [], "properties": { "name": { "type": "string", "default": "" }, "description": { "type": "string", "default": "" }, "data_type": { "type": "string", "default": "" }, "meta": { "type": "object", "default": {} } }, "additionalProperties": true, "description": "ExternalPartition(_extra: Dict[str, Any] = , name: str = '', description: str = '', data_type: str = '', meta: Dict[str, Any] = )" }, "SourceConfig": { "type": "object", "required": [], "properties": { "enabled": { "type": "boolean", "default": true } }, "additionalProperties": true, "description": "SourceConfig(_extra: Dict[str, Any] = , enabled: bool = True)" }, "ParsedMacro": { "type": "object", "required": [ "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "macro_sql", "resource_type" ], "properties": { "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "macro_sql": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "macro" ] }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "depends_on": { "$ref": "#/definitions/MacroDependsOn", "default": { "macros": [] } }, "description": { "type": "string", "default": "" }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "arguments": { "type": "array", "items": { "$ref": "#/definitions/MacroArgument" }, "default": [] }, "created_at": { "type": "number", "default": 1657040762.049289 } }, "additionalProperties": false, "description": "ParsedMacro(unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, macro_sql: str, resource_type: dbt.node_types.NodeType, tags: List[str] = , depends_on: dbt.contracts.graph.parsed.MacroDependsOn = , description: str = '', meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, arguments: List[dbt.contracts.graph.unparsed.MacroArgument] = , created_at: float = )" }, "MacroDependsOn": { "type": "object", "required": [], "properties": { "macros": { "type": "array", "items": { "type": "string" }, "default": [] } }, "additionalProperties": false, "description": "MacroDependsOn(macros: List[str] = )" }, "MacroArgument": { "type": "object", "required": [ "name" ], "properties": { "name": { "type": "string" }, "type": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "description": { "type": "string", "default": "" } }, "additionalProperties": false, "description": "MacroArgument(name: str, type: Optional[str] = None, description: str = '')" }, "ParsedDocumentation": { "type": "object", "required": [ "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "block_contents" ], "properties": { "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "block_contents": { "type": "string" } }, "additionalProperties": false, "description": "ParsedDocumentation(unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, block_contents: str)" }, "ParsedExposure": { "type": "object", "required": [ "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "type", "owner" ], "properties": { "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "type": { "type": "string", "enum": [ "dashboard", "notebook", "analysis", "ml", "application" ] }, "owner": { "$ref": "#/definitions/ExposureOwner" }, "resource_type": { "type": "string", "enum": [ "model", "analysis", "test", "snapshot", "operation", "seed", "rpc", "sql", "docs", "source", "macro", "exposure", "metric" ], "default": "exposure" }, "description": { "type": "string", "default": "" }, "maturity": { "oneOf": [ { "type": "string", "enum": [ "low", "medium", "high" ] }, { "type": "null" } ] }, "meta": { "type": "object", "default": {} }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "url": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "created_at": { "type": "number", "default": 1657040762.050287 } }, "additionalProperties": false, "description": "ParsedExposure(fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, type: dbt.contracts.graph.unparsed.ExposureType, owner: dbt.contracts.graph.unparsed.ExposureOwner, resource_type: dbt.node_types.NodeType = , description: str = '', maturity: Optional[dbt.contracts.graph.unparsed.MaturityType] = None, meta: Dict[str, Any] = , tags: List[str] = , url: Optional[str] = None, depends_on: dbt.contracts.graph.parsed.DependsOn = , refs: List[List[str]] = , sources: List[List[str]] = , created_at: float = )" }, "ExposureOwner": { "type": "object", "required": [ "email" ], "properties": { "email": { "type": "string" }, "name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "ExposureOwner(email: str, name: Optional[str] = None)" }, "ParsedMetric": { "type": "object", "required": [ "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "description", "label", "type", "sql", "filters", "time_grains", "dimensions" ], "properties": { "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "description": { "type": "string" }, "label": { "type": "string" }, "type": { "type": "string" }, "sql": { "type": "string" }, "timestamp": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "filters": { "type": "array", "items": { "$ref": "#/definitions/MetricFilter" } }, "time_grains": { "type": "array", "items": { "type": "string" } }, "dimensions": { "type": "array", "items": { "type": "string" } }, "model": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "model_unique_id": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "resource_type": { "type": "string", "enum": [ "model", "analysis", "test", "snapshot", "operation", "seed", "rpc", "sql", "docs", "source", "macro", "exposure", "metric" ], "default": "metric" }, "meta": { "type": "object", "default": {} }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "created_at": { "type": "number", "default": 1657040762.051152 } }, "additionalProperties": false, "description": "ParsedMetric(fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, description: str, label: str, type: str, sql: str, timestamp: Optional[str], filters: List[dbt.contracts.graph.unparsed.MetricFilter], time_grains: List[str], dimensions: List[str], model: Optional[str] = None, model_unique_id: Optional[str] = None, resource_type: dbt.node_types.NodeType = , meta: Dict[str, Any] = , tags: List[str] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , refs: List[List[str]] = , metrics: List[List[str]] = , created_at: float = )" }, "MetricFilter": { "type": "object", "required": [ "field", "operator", "value" ], "properties": { "field": { "type": "string" }, "operator": { "type": "string" }, "value": { "type": "string" } }, "additionalProperties": false, "description": "MetricFilter(field: str, operator: str, value: str)" } }, "$schema": "http://json-schema.org/draft-07/schema#", "$id": "https://schemas.getdbt.com/dbt/manifest/v6.json" } ================================================ FILE: schemas/dbt/manifest/v7.json ================================================ { "type": "object", "required": [ "metadata", "nodes", "sources", "macros", "docs", "exposures", "metrics", "selectors" ], "properties": { "metadata": { "$ref": "#/definitions/ManifestMetadata", "description": "Metadata about the manifest" }, "nodes": { "type": "object", "additionalProperties": { "oneOf": [ { "$ref": "#/definitions/CompiledAnalysisNode" }, { "$ref": "#/definitions/CompiledSingularTestNode" }, { "$ref": "#/definitions/CompiledModelNode" }, { "$ref": "#/definitions/CompiledHookNode" }, { "$ref": "#/definitions/CompiledRPCNode" }, { "$ref": "#/definitions/CompiledSqlNode" }, { "$ref": "#/definitions/CompiledGenericTestNode" }, { "$ref": "#/definitions/CompiledSeedNode" }, { "$ref": "#/definitions/CompiledSnapshotNode" }, { "$ref": "#/definitions/ParsedAnalysisNode" }, { "$ref": "#/definitions/ParsedSingularTestNode" }, { "$ref": "#/definitions/ParsedHookNode" }, { "$ref": "#/definitions/ParsedModelNode" }, { "$ref": "#/definitions/ParsedRPCNode" }, { "$ref": "#/definitions/ParsedSqlNode" }, { "$ref": "#/definitions/ParsedGenericTestNode" }, { "$ref": "#/definitions/ParsedSeedNode" }, { "$ref": "#/definitions/ParsedSnapshotNode" } ] }, "description": "The nodes defined in the dbt project and its dependencies" }, "sources": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ParsedSourceDefinition" }, "description": "The sources defined in the dbt project and its dependencies" }, "macros": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ParsedMacro" }, "description": "The macros defined in the dbt project and its dependencies" }, "docs": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ParsedDocumentation" }, "description": "The docs defined in the dbt project and its dependencies" }, "exposures": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ParsedExposure" }, "description": "The exposures defined in the dbt project and its dependencies" }, "metrics": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ParsedMetric" }, "description": "The metrics defined in the dbt project and its dependencies" }, "selectors": { "type": "object", "description": "The selectors defined in selectors.yml" }, "disabled": { "oneOf": [ { "type": "object", "additionalProperties": { "type": "array", "items": { "oneOf": [ { "$ref": "#/definitions/CompiledAnalysisNode" }, { "$ref": "#/definitions/CompiledSingularTestNode" }, { "$ref": "#/definitions/CompiledModelNode" }, { "$ref": "#/definitions/CompiledHookNode" }, { "$ref": "#/definitions/CompiledRPCNode" }, { "$ref": "#/definitions/CompiledSqlNode" }, { "$ref": "#/definitions/CompiledGenericTestNode" }, { "$ref": "#/definitions/CompiledSeedNode" }, { "$ref": "#/definitions/CompiledSnapshotNode" }, { "$ref": "#/definitions/ParsedAnalysisNode" }, { "$ref": "#/definitions/ParsedSingularTestNode" }, { "$ref": "#/definitions/ParsedHookNode" }, { "$ref": "#/definitions/ParsedModelNode" }, { "$ref": "#/definitions/ParsedRPCNode" }, { "$ref": "#/definitions/ParsedSqlNode" }, { "$ref": "#/definitions/ParsedGenericTestNode" }, { "$ref": "#/definitions/ParsedSeedNode" }, { "$ref": "#/definitions/ParsedSnapshotNode" }, { "$ref": "#/definitions/ParsedSourceDefinition" } ] } } }, { "type": "null" } ], "description": "A mapping of the disabled nodes in the target" }, "parent_map": { "oneOf": [ { "type": "object", "additionalProperties": { "type": "array", "items": { "type": "string" } } }, { "type": "null" } ], "description": "A mapping from\u00a0child nodes to their dependencies" }, "child_map": { "oneOf": [ { "type": "object", "additionalProperties": { "type": "array", "items": { "type": "string" } } }, { "type": "null" } ], "description": "A mapping from parent nodes to their dependents" } }, "additionalProperties": false, "description": "WritableManifest(metadata: dbt.contracts.graph.manifest.ManifestMetadata, nodes: Mapping[str, Union[dbt.contracts.graph.compiled.CompiledAnalysisNode, dbt.contracts.graph.compiled.CompiledSingularTestNode, dbt.contracts.graph.compiled.CompiledModelNode, dbt.contracts.graph.compiled.CompiledHookNode, dbt.contracts.graph.compiled.CompiledRPCNode, dbt.contracts.graph.compiled.CompiledSqlNode, dbt.contracts.graph.compiled.CompiledGenericTestNode, dbt.contracts.graph.compiled.CompiledSeedNode, dbt.contracts.graph.compiled.CompiledSnapshotNode, dbt.contracts.graph.parsed.ParsedAnalysisNode, dbt.contracts.graph.parsed.ParsedSingularTestNode, dbt.contracts.graph.parsed.ParsedHookNode, dbt.contracts.graph.parsed.ParsedModelNode, dbt.contracts.graph.parsed.ParsedRPCNode, dbt.contracts.graph.parsed.ParsedSqlNode, dbt.contracts.graph.parsed.ParsedGenericTestNode, dbt.contracts.graph.parsed.ParsedSeedNode, dbt.contracts.graph.parsed.ParsedSnapshotNode]], sources: Mapping[str, dbt.contracts.graph.parsed.ParsedSourceDefinition], macros: Mapping[str, dbt.contracts.graph.parsed.ParsedMacro], docs: Mapping[str, dbt.contracts.graph.parsed.ParsedDocumentation], exposures: Mapping[str, dbt.contracts.graph.parsed.ParsedExposure], metrics: Mapping[str, dbt.contracts.graph.parsed.ParsedMetric], selectors: Mapping[str, Any], disabled: Union[Mapping[str, List[Union[dbt.contracts.graph.compiled.CompiledAnalysisNode, dbt.contracts.graph.compiled.CompiledSingularTestNode, dbt.contracts.graph.compiled.CompiledModelNode, dbt.contracts.graph.compiled.CompiledHookNode, dbt.contracts.graph.compiled.CompiledRPCNode, dbt.contracts.graph.compiled.CompiledSqlNode, dbt.contracts.graph.compiled.CompiledGenericTestNode, dbt.contracts.graph.compiled.CompiledSeedNode, dbt.contracts.graph.compiled.CompiledSnapshotNode, dbt.contracts.graph.parsed.ParsedAnalysisNode, dbt.contracts.graph.parsed.ParsedSingularTestNode, dbt.contracts.graph.parsed.ParsedHookNode, dbt.contracts.graph.parsed.ParsedModelNode, dbt.contracts.graph.parsed.ParsedRPCNode, dbt.contracts.graph.parsed.ParsedSqlNode, dbt.contracts.graph.parsed.ParsedGenericTestNode, dbt.contracts.graph.parsed.ParsedSeedNode, dbt.contracts.graph.parsed.ParsedSnapshotNode, dbt.contracts.graph.parsed.ParsedSourceDefinition]]], NoneType], parent_map: Union[Dict[str, List[str]], NoneType], child_map: Union[Dict[str, List[str]], NoneType])", "definitions": { "ManifestMetadata": { "type": "object", "required": [], "properties": { "dbt_schema_version": { "type": "string", "default": "https://schemas.getdbt.com/dbt/manifest/v7.json" }, "dbt_version": { "type": "string", "default": "1.3.0b2" }, "generated_at": { "type": "string", "format": "date-time", "default": "2022-09-14T20:35:15.346636Z" }, "invocation_id": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "default": "c59a8269-533c-4b78-a709-5094045afd4d" }, "env": { "type": "object", "additionalProperties": { "type": "string" }, "default": {} }, "project_id": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "description": "A unique identifier for the project" }, "user_id": { "oneOf": [ { "type": "string", "pattern": "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" }, { "type": "null" } ], "description": "A unique identifier for the user" }, "send_anonymous_usage_stats": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ], "description": "Whether dbt is configured to send anonymous usage statistics" }, "adapter_type": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "description": "The type name of the adapter" } }, "additionalProperties": false, "description": "Metadata for the manifest." }, "CompiledAnalysisNode": { "type": "object", "required": [ "compiled", "schema", "fqn", "unique_id", "raw_code", "language", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "compiled": { "type": "boolean" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "raw_code": { "type": "string" }, "language": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "analysis" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1663187715.3517282 }, "config_call_dict": { "type": "object", "default": {} }, "compiled_code": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "CompiledAnalysisNode(compiled: bool, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, raw_code: str, language: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , compiled_code: Union[str, NoneType] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.compiled.InjectedCTE] = , relation_name: Union[str, NoneType] = None, _pre_injected_sql: Union[str, NoneType] = None)" }, "FileHash": { "type": "object", "required": [ "name", "checksum" ], "properties": { "name": { "type": "string" }, "checksum": { "type": "string" } }, "additionalProperties": false, "description": "FileHash(name: str, checksum: str)" }, "NodeConfig": { "type": "object", "required": [], "properties": { "enabled": { "type": "boolean", "default": true }, "alias": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "tags": { "oneOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ], "default": [] }, "meta": { "type": "object", "default": {} }, "materialized": { "type": "string", "default": "view" }, "incremental_strategy": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "persist_docs": { "type": "object", "default": {} }, "post-hook": { "type": "array", "items": { "$ref": "#/definitions/Hook" }, "default": [] }, "pre-hook": { "type": "array", "items": { "$ref": "#/definitions/Hook" }, "default": [] }, "quoting": { "type": "object", "default": {} }, "column_types": { "type": "object", "default": {} }, "full_refresh": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "unique_key": { "oneOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ] }, "on_schema_change": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "grants": { "type": "object", "default": {} }, "packages": { "type": "array", "items": { "type": "string" }, "default": [] }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } } }, "additionalProperties": true, "description": "NodeConfig(_extra: Dict[str, Any] = , enabled: bool = True, alias: Union[str, NoneType] = None, schema: Union[str, NoneType] = None, database: Union[str, NoneType] = None, tags: Union[List[str], str] = , meta: Dict[str, Any] = , materialized: str = 'view', incremental_strategy: Union[str, NoneType] = None, persist_docs: Dict[str, Any] = , post_hook: List[dbt.contracts.graph.model_config.Hook] = , pre_hook: List[dbt.contracts.graph.model_config.Hook] = , quoting: Dict[str, Any] = , column_types: Dict[str, Any] = , full_refresh: Union[bool, NoneType] = None, unique_key: Union[str, List[str], NoneType] = None, on_schema_change: Union[str, NoneType] = 'ignore', grants: Dict[str, Any] = , packages: List[str] = , docs: dbt.contracts.graph.unparsed.Docs = )" }, "Hook": { "type": "object", "required": [ "sql" ], "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "oneOf": [ { "type": "integer" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "Hook(sql: str, transaction: bool = True, index: Union[int, NoneType] = None)" }, "Docs": { "type": "object", "required": [], "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "Docs(show: bool = True, node_color: Union[str, NoneType] = None)" }, "DependsOn": { "type": "object", "required": [], "properties": { "macros": { "type": "array", "items": { "type": "string" }, "default": [] }, "nodes": { "type": "array", "items": { "type": "string" }, "default": [] } }, "additionalProperties": false, "description": "DependsOn(macros: List[str] = , nodes: List[str] = )" }, "ColumnInfo": { "type": "object", "required": [ "name" ], "properties": { "name": { "type": "string" }, "description": { "type": "string", "default": "" }, "meta": { "type": "object", "default": {} }, "data_type": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "quote": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] } }, "additionalProperties": true, "description": "ColumnInfo(name: str, description: str = '', meta: Dict[str, Any] = , data_type: Union[str, NoneType] = None, quote: Union[bool, NoneType] = None, tags: List[str] = , _extra: Dict[str, Any] = )" }, "InjectedCTE": { "type": "object", "required": [ "id", "sql" ], "properties": { "id": { "type": "string" }, "sql": { "type": "string" } }, "additionalProperties": false, "description": "InjectedCTE(id: str, sql: str)" }, "CompiledSingularTestNode": { "type": "object", "required": [ "compiled", "schema", "fqn", "unique_id", "raw_code", "language", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "compiled": { "type": "boolean" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "raw_code": { "type": "string" }, "language": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "test" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/TestConfig", "default": { "enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0" } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1663187715.35441 }, "config_call_dict": { "type": "object", "default": {} }, "compiled_code": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "CompiledSingularTestNode(compiled: bool, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, raw_code: str, language: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.TestConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , compiled_code: Union[str, NoneType] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.compiled.InjectedCTE] = , relation_name: Union[str, NoneType] = None, _pre_injected_sql: Union[str, NoneType] = None)" }, "TestConfig": { "type": "object", "required": [], "properties": { "enabled": { "type": "boolean", "default": true }, "alias": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "default": "dbt_test__audit" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "tags": { "oneOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ], "default": [] }, "meta": { "type": "object", "default": {} }, "materialized": { "type": "string", "default": "test" }, "severity": { "type": "string", "pattern": "^([Ww][Aa][Rr][Nn]|[Ee][Rr][Rr][Oo][Rr])$", "default": "ERROR" }, "store_failures": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "where": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "limit": { "oneOf": [ { "type": "integer" }, { "type": "null" } ] }, "fail_calc": { "type": "string", "default": "count(*)" }, "warn_if": { "type": "string", "default": "!= 0" }, "error_if": { "type": "string", "default": "!= 0" } }, "additionalProperties": true, "description": "TestConfig(_extra: Dict[str, Any] = , enabled: bool = True, alias: Union[str, NoneType] = None, schema: Union[str, NoneType] = 'dbt_test__audit', database: Union[str, NoneType] = None, tags: Union[List[str], str] = , meta: Dict[str, Any] = , materialized: str = 'test', severity: dbt.contracts.graph.model_config.Severity = 'ERROR', store_failures: Union[bool, NoneType] = None, where: Union[str, NoneType] = None, limit: Union[int, NoneType] = None, fail_calc: str = 'count(*)', warn_if: str = '!= 0', error_if: str = '!= 0')" }, "CompiledModelNode": { "type": "object", "required": [ "compiled", "schema", "fqn", "unique_id", "raw_code", "language", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "compiled": { "type": "boolean" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "raw_code": { "type": "string" }, "language": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "model" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1663187715.356541 }, "config_call_dict": { "type": "object", "default": {} }, "compiled_code": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "CompiledModelNode(compiled: bool, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, raw_code: str, language: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , compiled_code: Union[str, NoneType] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.compiled.InjectedCTE] = , relation_name: Union[str, NoneType] = None, _pre_injected_sql: Union[str, NoneType] = None)" }, "CompiledHookNode": { "type": "object", "required": [ "compiled", "schema", "fqn", "unique_id", "raw_code", "language", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "compiled": { "type": "boolean" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "raw_code": { "type": "string" }, "language": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "operation" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1663187715.3582149 }, "config_call_dict": { "type": "object", "default": {} }, "compiled_code": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "index": { "oneOf": [ { "type": "integer" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "CompiledHookNode(compiled: bool, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, raw_code: str, language: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , compiled_code: Union[str, NoneType] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.compiled.InjectedCTE] = , relation_name: Union[str, NoneType] = None, _pre_injected_sql: Union[str, NoneType] = None, index: Union[int, NoneType] = None)" }, "CompiledRPCNode": { "type": "object", "required": [ "compiled", "schema", "fqn", "unique_id", "raw_code", "language", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "compiled": { "type": "boolean" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "raw_code": { "type": "string" }, "language": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "rpc" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1663187715.359935 }, "config_call_dict": { "type": "object", "default": {} }, "compiled_code": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "CompiledRPCNode(compiled: bool, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, raw_code: str, language: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , compiled_code: Union[str, NoneType] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.compiled.InjectedCTE] = , relation_name: Union[str, NoneType] = None, _pre_injected_sql: Union[str, NoneType] = None)" }, "CompiledSqlNode": { "type": "object", "required": [ "compiled", "schema", "fqn", "unique_id", "raw_code", "language", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "compiled": { "type": "boolean" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "raw_code": { "type": "string" }, "language": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "sql operation" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1663187715.361423 }, "config_call_dict": { "type": "object", "default": {} }, "compiled_code": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "CompiledSqlNode(compiled: bool, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, raw_code: str, language: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , compiled_code: Union[str, NoneType] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.compiled.InjectedCTE] = , relation_name: Union[str, NoneType] = None, _pre_injected_sql: Union[str, NoneType] = None)" }, "CompiledGenericTestNode": { "type": "object", "required": [ "test_metadata", "compiled", "schema", "fqn", "unique_id", "raw_code", "language", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "test_metadata": { "$ref": "#/definitions/TestMetadata" }, "compiled": { "type": "boolean" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "raw_code": { "type": "string" }, "language": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "test" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/TestConfig", "default": { "enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0" } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1663187715.363411 }, "config_call_dict": { "type": "object", "default": {} }, "compiled_code": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "column_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "file_key_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "CompiledGenericTestNode(test_metadata: dbt.contracts.graph.parsed.TestMetadata, compiled: bool, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, raw_code: str, language: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.TestConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , compiled_code: Union[str, NoneType] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.compiled.InjectedCTE] = , relation_name: Union[str, NoneType] = None, _pre_injected_sql: Union[str, NoneType] = None, column_name: Union[str, NoneType] = None, file_key_name: Union[str, NoneType] = None)" }, "TestMetadata": { "type": "object", "required": [ "name" ], "properties": { "name": { "type": "string" }, "kwargs": { "type": "object", "default": {} }, "namespace": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "TestMetadata(name: str, kwargs: Dict[str, Any] = , namespace: Union[str, NoneType] = None)" }, "CompiledSeedNode": { "type": "object", "required": [ "compiled", "schema", "fqn", "unique_id", "raw_code", "language", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "compiled": { "type": "boolean" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "raw_code": { "type": "string" }, "language": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "seed" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/SeedConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "seed", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "quote_columns": null, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1663187715.366584 }, "config_call_dict": { "type": "object", "default": {} }, "compiled_code": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "CompiledSeedNode(compiled: bool, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, raw_code: str, language: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.SeedConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , compiled_code: Union[str, NoneType] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.compiled.InjectedCTE] = , relation_name: Union[str, NoneType] = None, _pre_injected_sql: Union[str, NoneType] = None)" }, "SeedConfig": { "type": "object", "required": [], "properties": { "enabled": { "type": "boolean", "default": true }, "alias": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "tags": { "oneOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ], "default": [] }, "meta": { "type": "object", "default": {} }, "materialized": { "type": "string", "default": "seed" }, "incremental_strategy": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "persist_docs": { "type": "object", "default": {} }, "post-hook": { "type": "array", "items": { "$ref": "#/definitions/Hook" }, "default": [] }, "pre-hook": { "type": "array", "items": { "$ref": "#/definitions/Hook" }, "default": [] }, "quoting": { "type": "object", "default": {} }, "column_types": { "type": "object", "default": {} }, "full_refresh": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "unique_key": { "oneOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ] }, "on_schema_change": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "grants": { "type": "object", "default": {} }, "packages": { "type": "array", "items": { "type": "string" }, "default": [] }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "quote_columns": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] } }, "additionalProperties": true, "description": "SeedConfig(_extra: Dict[str, Any] = , enabled: bool = True, alias: Union[str, NoneType] = None, schema: Union[str, NoneType] = None, database: Union[str, NoneType] = None, tags: Union[List[str], str] = , meta: Dict[str, Any] = , materialized: str = 'seed', incremental_strategy: Union[str, NoneType] = None, persist_docs: Dict[str, Any] = , post_hook: List[dbt.contracts.graph.model_config.Hook] = , pre_hook: List[dbt.contracts.graph.model_config.Hook] = , quoting: Dict[str, Any] = , column_types: Dict[str, Any] = , full_refresh: Union[bool, NoneType] = None, unique_key: Union[str, List[str], NoneType] = None, on_schema_change: Union[str, NoneType] = 'ignore', grants: Dict[str, Any] = , packages: List[str] = , docs: dbt.contracts.graph.unparsed.Docs = , quote_columns: Union[bool, NoneType] = None)" }, "CompiledSnapshotNode": { "type": "object", "required": [ "compiled", "schema", "fqn", "unique_id", "raw_code", "language", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "compiled": { "type": "boolean" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "raw_code": { "type": "string" }, "language": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "snapshot" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1663187715.3682182 }, "config_call_dict": { "type": "object", "default": {} }, "compiled_code": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "CompiledSnapshotNode(compiled: bool, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, raw_code: str, language: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , compiled_code: Union[str, NoneType] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.compiled.InjectedCTE] = , relation_name: Union[str, NoneType] = None, _pre_injected_sql: Union[str, NoneType] = None)" }, "ParsedAnalysisNode": { "type": "object", "required": [ "schema", "fqn", "unique_id", "raw_code", "language", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "raw_code": { "type": "string" }, "language": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "analysis" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1663187715.369675 }, "config_call_dict": { "type": "object", "default": {} } }, "additionalProperties": false, "description": "ParsedAnalysisNode(database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, raw_code: str, language: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = )" }, "ParsedSingularTestNode": { "type": "object", "required": [ "schema", "fqn", "unique_id", "raw_code", "language", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "raw_code": { "type": "string" }, "language": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "test" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/TestConfig", "default": { "enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0" } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1663187715.370925 }, "config_call_dict": { "type": "object", "default": {} } }, "additionalProperties": false, "description": "ParsedSingularTestNode(database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, raw_code: str, language: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.TestConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = )" }, "ParsedHookNode": { "type": "object", "required": [ "schema", "fqn", "unique_id", "raw_code", "language", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "raw_code": { "type": "string" }, "language": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "operation" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1663187715.372257 }, "config_call_dict": { "type": "object", "default": {} }, "index": { "oneOf": [ { "type": "integer" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "ParsedHookNode(database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, raw_code: str, language: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , index: Union[int, NoneType] = None)" }, "ParsedModelNode": { "type": "object", "required": [ "schema", "fqn", "unique_id", "raw_code", "language", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "raw_code": { "type": "string" }, "language": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "model" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1663187715.373705 }, "config_call_dict": { "type": "object", "default": {} } }, "additionalProperties": false, "description": "ParsedModelNode(database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, raw_code: str, language: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = )" }, "ParsedRPCNode": { "type": "object", "required": [ "schema", "fqn", "unique_id", "raw_code", "language", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "raw_code": { "type": "string" }, "language": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "rpc" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1663187715.375131 }, "config_call_dict": { "type": "object", "default": {} } }, "additionalProperties": false, "description": "ParsedRPCNode(database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, raw_code: str, language: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = )" }, "ParsedSqlNode": { "type": "object", "required": [ "schema", "fqn", "unique_id", "raw_code", "language", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "raw_code": { "type": "string" }, "language": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "sql operation" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1663187715.376405 }, "config_call_dict": { "type": "object", "default": {} } }, "additionalProperties": false, "description": "ParsedSqlNode(database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, raw_code: str, language: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = )" }, "ParsedGenericTestNode": { "type": "object", "required": [ "test_metadata", "schema", "fqn", "unique_id", "raw_code", "language", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "test_metadata": { "$ref": "#/definitions/TestMetadata" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "raw_code": { "type": "string" }, "language": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "test" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/TestConfig", "default": { "enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0" } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1663187715.3784761 }, "config_call_dict": { "type": "object", "default": {} }, "column_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "file_key_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "ParsedGenericTestNode(test_metadata: dbt.contracts.graph.parsed.TestMetadata, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, raw_code: str, language: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.TestConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , column_name: Union[str, NoneType] = None, file_key_name: Union[str, NoneType] = None)" }, "ParsedSeedNode": { "type": "object", "required": [ "schema", "fqn", "unique_id", "raw_code", "language", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum" ], "properties": { "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "raw_code": { "type": "string" }, "language": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "seed" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/SeedConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "seed", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "quote_columns": null, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1663187715.380325 }, "config_call_dict": { "type": "object", "default": {} } }, "additionalProperties": false, "description": "ParsedSeedNode(database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, raw_code: str, language: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.SeedConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = )" }, "ParsedSnapshotNode": { "type": "object", "required": [ "schema", "fqn", "unique_id", "raw_code", "language", "package_name", "root_path", "path", "original_file_path", "name", "resource_type", "alias", "checksum", "config" ], "properties": { "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "raw_code": { "type": "string" }, "language": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "snapshot" ] }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/SnapshotConfig" }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1663187715.3832462 }, "config_call_dict": { "type": "object", "default": {} } }, "additionalProperties": false, "description": "ParsedSnapshotNode(database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, raw_code: str, language: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.SnapshotConfig, _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = )" }, "SnapshotConfig": { "type": "object", "required": [], "properties": { "enabled": { "type": "boolean", "default": true }, "alias": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "tags": { "oneOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ], "default": [] }, "meta": { "type": "object", "default": {} }, "materialized": { "type": "string", "default": "snapshot" }, "incremental_strategy": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "persist_docs": { "type": "object", "default": {} }, "post-hook": { "type": "array", "items": { "$ref": "#/definitions/Hook" }, "default": [] }, "pre-hook": { "type": "array", "items": { "$ref": "#/definitions/Hook" }, "default": [] }, "quoting": { "type": "object", "default": {} }, "column_types": { "type": "object", "default": {} }, "full_refresh": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "unique_key": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "on_schema_change": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "grants": { "type": "object", "default": {} }, "packages": { "type": "array", "items": { "type": "string" }, "default": [] }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "strategy": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "target_schema": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "target_database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "updated_at": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "check_cols": { "oneOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ] } }, "additionalProperties": true, "description": "SnapshotConfig(_extra: Dict[str, Any] = , enabled: bool = True, alias: Union[str, NoneType] = None, schema: Union[str, NoneType] = None, database: Union[str, NoneType] = None, tags: Union[List[str], str] = , meta: Dict[str, Any] = , materialized: str = 'snapshot', incremental_strategy: Union[str, NoneType] = None, persist_docs: Dict[str, Any] = , post_hook: List[dbt.contracts.graph.model_config.Hook] = , pre_hook: List[dbt.contracts.graph.model_config.Hook] = , quoting: Dict[str, Any] = , column_types: Dict[str, Any] = , full_refresh: Union[bool, NoneType] = None, unique_key: Union[str, NoneType] = None, on_schema_change: Union[str, NoneType] = 'ignore', grants: Dict[str, Any] = , packages: List[str] = , docs: dbt.contracts.graph.unparsed.Docs = , strategy: Union[str, NoneType] = None, target_schema: Union[str, NoneType] = None, target_database: Union[str, NoneType] = None, updated_at: Union[str, NoneType] = None, check_cols: Union[str, List[str], NoneType] = None)" }, "ParsedSourceDefinition": { "type": "object", "required": [ "fqn", "schema", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "source_name", "source_description", "loader", "identifier", "resource_type" ], "properties": { "fqn": { "type": "array", "items": { "type": "string" } }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "source_name": { "type": "string" }, "source_description": { "type": "string" }, "loader": { "type": "string" }, "identifier": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "source" ] }, "quoting": { "$ref": "#/definitions/Quoting", "default": { "database": null, "schema": null, "identifier": null, "column": null } }, "loaded_at_field": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "freshness": { "oneOf": [ { "$ref": "#/definitions/FreshnessThreshold" }, { "type": "null" } ] }, "external": { "oneOf": [ { "$ref": "#/definitions/ExternalTable" }, { "type": "null" } ] }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "source_meta": { "type": "object", "default": {} }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "config": { "$ref": "#/definitions/SourceConfig", "default": { "enabled": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "unrendered_config": { "type": "object", "default": {} }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "created_at": { "type": "number", "default": 1663187715.3854342 } }, "additionalProperties": false, "description": "ParsedSourceDefinition(fqn: List[str], database: Union[str, NoneType], schema: str, unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, source_name: str, source_description: str, loader: str, identifier: str, resource_type: dbt.node_types.NodeType, _event_status: Dict[str, Any] = , quoting: dbt.contracts.graph.unparsed.Quoting = , loaded_at_field: Union[str, NoneType] = None, freshness: Union[dbt.contracts.graph.unparsed.FreshnessThreshold, NoneType] = None, external: Union[dbt.contracts.graph.unparsed.ExternalTable, NoneType] = None, description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , source_meta: Dict[str, Any] = , tags: List[str] = , config: dbt.contracts.graph.model_config.SourceConfig = , patch_path: Union[pathlib.Path, NoneType] = None, unrendered_config: Dict[str, Any] = , relation_name: Union[str, NoneType] = None, created_at: float = )" }, "Quoting": { "type": "object", "required": [], "properties": { "database": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "schema": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "identifier": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "column": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "Quoting(database: Union[bool, NoneType] = None, schema: Union[bool, NoneType] = None, identifier: Union[bool, NoneType] = None, column: Union[bool, NoneType] = None)" }, "FreshnessThreshold": { "type": "object", "required": [], "properties": { "warn_after": { "oneOf": [ { "$ref": "#/definitions/Time" }, { "type": "null" } ], "default": { "count": null, "period": null } }, "error_after": { "oneOf": [ { "$ref": "#/definitions/Time" }, { "type": "null" } ], "default": { "count": null, "period": null } }, "filter": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "FreshnessThreshold(warn_after: Union[dbt.contracts.graph.unparsed.Time, NoneType] = , error_after: Union[dbt.contracts.graph.unparsed.Time, NoneType] = , filter: Union[str, NoneType] = None)" }, "FreshnessMetadata": { "type": "object", "required": [], "properties": { "dbt_schema_version": { "type": "string", "default": "https://schemas.getdbt.com/dbt/sources/v3.json" }, "dbt_version": { "type": "string", "default": "1.3.0b2" }, "generated_at": { "type": "string", "format": "date-time", "default": "2022-09-14T20:35:15.341700Z" }, "invocation_id": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "default": "c59a8269-533c-4b78-a709-5094045afd4d" }, "env": { "type": "object", "additionalProperties": { "type": "string" }, "default": {} } }, "additionalProperties": false, "description": "FreshnessMetadata(dbt_schema_version: str = , dbt_version: str = '1.3.0b2', generated_at: datetime.datetime = , invocation_id: Union[str, NoneType] = , env: Dict[str, str] = )" }, "SourceFreshnessRuntimeError": { "type": "object", "required": [ "unique_id", "status" ], "properties": { "unique_id": { "type": "string" }, "error": { "oneOf": [ { "type": "string" }, { "type": "integer" }, { "type": "null" } ] }, "status": { "type": "string", "enum": [ "runtime error" ] } }, "additionalProperties": false, "description": "SourceFreshnessRuntimeError(unique_id: str, error: Union[str, int, NoneType], status: dbt.contracts.results.FreshnessErrorEnum)" }, "SourceFreshnessOutput": { "type": "object", "required": [ "unique_id", "max_loaded_at", "snapshotted_at", "max_loaded_at_time_ago_in_s", "status", "criteria", "adapter_response", "timing", "thread_id", "execution_time" ], "properties": { "unique_id": { "type": "string" }, "max_loaded_at": { "type": "string", "format": "date-time" }, "snapshotted_at": { "type": "string", "format": "date-time" }, "max_loaded_at_time_ago_in_s": { "type": "number" }, "status": { "type": "string", "enum": [ "pass", "warn", "error", "runtime error" ] }, "criteria": { "$ref": "#/definitions/FreshnessThreshold" }, "adapter_response": { "type": "object" }, "timing": { "type": "array", "items": { "$ref": "#/definitions/TimingInfo" } }, "thread_id": { "type": "string" }, "execution_time": { "type": "number" } }, "additionalProperties": false, "description": "SourceFreshnessOutput(unique_id: str, max_loaded_at: datetime.datetime, snapshotted_at: datetime.datetime, max_loaded_at_time_ago_in_s: float, status: dbt.contracts.results.FreshnessStatus, criteria: dbt.contracts.graph.unparsed.FreshnessThreshold, adapter_response: Dict[str, Any], timing: List[dbt.contracts.results.TimingInfo], thread_id: str, execution_time: float)" }, "Time": { "type": "object", "required": [], "properties": { "count": { "oneOf": [ { "type": "integer" }, { "type": "null" } ] }, "period": { "oneOf": [ { "type": "string", "enum": [ "minute", "hour", "day" ] }, { "type": "null" } ] } }, "additionalProperties": false, "description": "Time(count: Union[int, NoneType] = None, period: Union[dbt.contracts.graph.unparsed.TimePeriod, NoneType] = None)" }, "TimingInfo": { "type": "object", "required": [ "name" ], "properties": { "name": { "type": "string" }, "started_at": { "oneOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ] }, "completed_at": { "oneOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "TimingInfo(name: str, started_at: Union[datetime.datetime, NoneType] = None, completed_at: Union[datetime.datetime, NoneType] = None)" }, "ExternalTable": { "type": "object", "required": [], "properties": { "location": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "file_format": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "row_format": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "tbl_properties": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "partitions": { "oneOf": [ { "type": "array", "items": { "$ref": "#/definitions/ExternalPartition" } }, { "type": "null" } ] } }, "additionalProperties": true, "description": "ExternalTable(_extra: Dict[str, Any] = , location: Union[str, NoneType] = None, file_format: Union[str, NoneType] = None, row_format: Union[str, NoneType] = None, tbl_properties: Union[str, NoneType] = None, partitions: Union[List[dbt.contracts.graph.unparsed.ExternalPartition], NoneType] = None)" }, "ExternalPartition": { "type": "object", "required": [], "properties": { "name": { "type": "string", "default": "" }, "description": { "type": "string", "default": "" }, "data_type": { "type": "string", "default": "" }, "meta": { "type": "object", "default": {} } }, "additionalProperties": true, "description": "ExternalPartition(_extra: Dict[str, Any] = , name: str = '', description: str = '', data_type: str = '', meta: Dict[str, Any] = )" }, "SourceConfig": { "type": "object", "required": [], "properties": { "enabled": { "type": "boolean", "default": true } }, "additionalProperties": true, "description": "SourceConfig(_extra: Dict[str, Any] = , enabled: bool = True)" }, "ParsedMacro": { "type": "object", "required": [ "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "macro_sql", "resource_type" ], "properties": { "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "macro_sql": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "macro" ] }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "depends_on": { "$ref": "#/definitions/MacroDependsOn", "default": { "macros": [] } }, "description": { "type": "string", "default": "" }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "arguments": { "type": "array", "items": { "$ref": "#/definitions/MacroArgument" }, "default": [] }, "created_at": { "type": "number", "default": 1663187715.386226 }, "supported_languages": { "oneOf": [ { "type": "array", "items": { "type": "string", "enum": [ "python", "sql" ] } }, { "type": "null" } ] } }, "additionalProperties": false, "description": "ParsedMacro(unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, macro_sql: str, resource_type: dbt.node_types.NodeType, tags: List[str] = , depends_on: dbt.contracts.graph.parsed.MacroDependsOn = , description: str = '', meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, arguments: List[dbt.contracts.graph.unparsed.MacroArgument] = , created_at: float = , supported_languages: Union[List[dbt.node_types.ModelLanguage], NoneType] = None)" }, "MacroDependsOn": { "type": "object", "required": [], "properties": { "macros": { "type": "array", "items": { "type": "string" }, "default": [] } }, "additionalProperties": false, "description": "MacroDependsOn(macros: List[str] = )" }, "MacroArgument": { "type": "object", "required": [ "name" ], "properties": { "name": { "type": "string" }, "type": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "description": { "type": "string", "default": "" } }, "additionalProperties": false, "description": "MacroArgument(name: str, type: Union[str, NoneType] = None, description: str = '')" }, "ParsedDocumentation": { "type": "object", "required": [ "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "block_contents" ], "properties": { "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "block_contents": { "type": "string" } }, "additionalProperties": false, "description": "ParsedDocumentation(unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, block_contents: str)" }, "ParsedExposure": { "type": "object", "required": [ "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "type", "owner" ], "properties": { "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "type": { "type": "string", "enum": [ "dashboard", "notebook", "analysis", "ml", "application" ] }, "owner": { "$ref": "#/definitions/ExposureOwner" }, "resource_type": { "type": "string", "enum": [ "model", "analysis", "test", "snapshot", "operation", "seed", "rpc", "sql operation", "docs block", "source", "macro", "exposure", "metric" ], "default": "exposure" }, "description": { "type": "string", "default": "" }, "label": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "maturity": { "oneOf": [ { "type": "string", "enum": [ "low", "medium", "high" ] }, { "type": "null" } ] }, "meta": { "type": "object", "default": {} }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "config": { "$ref": "#/definitions/ExposureConfig", "default": { "enabled": true } }, "unrendered_config": { "type": "object", "default": {} }, "url": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "created_at": { "type": "number", "default": 1663187715.3878772 } }, "additionalProperties": false, "description": "ParsedExposure(fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, type: dbt.contracts.graph.unparsed.ExposureType, owner: dbt.contracts.graph.unparsed.ExposureOwner, resource_type: dbt.node_types.NodeType = , description: str = '', label: Union[str, NoneType] = None, maturity: Union[dbt.contracts.graph.unparsed.MaturityType, NoneType] = None, meta: Dict[str, Any] = , tags: List[str] = , config: dbt.contracts.graph.model_config.ExposureConfig = , unrendered_config: Dict[str, Any] = , url: Union[str, NoneType] = None, depends_on: dbt.contracts.graph.parsed.DependsOn = , refs: List[List[str]] = , sources: List[List[str]] = , created_at: float = )" }, "ExposureOwner": { "type": "object", "required": [ "email" ], "properties": { "email": { "type": "string" }, "name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "ExposureOwner(email: str, name: Union[str, NoneType] = None)" }, "ExposureConfig": { "type": "object", "required": [], "properties": { "enabled": { "type": "boolean", "default": true } }, "additionalProperties": true, "description": "ExposureConfig(_extra: Dict[str, Any] = , enabled: bool = True)" }, "ParsedMetric": { "type": "object", "required": [ "fqn", "unique_id", "package_name", "root_path", "path", "original_file_path", "name", "description", "label", "calculation_method", "expression", "filters", "time_grains", "dimensions" ], "properties": { "fqn": { "type": "array", "items": { "type": "string" } }, "unique_id": { "type": "string" }, "package_name": { "type": "string" }, "root_path": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "name": { "type": "string" }, "description": { "type": "string" }, "label": { "type": "string" }, "calculation_method": { "type": "string" }, "expression": { "type": "string" }, "timestamp": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "filters": { "type": "array", "items": { "$ref": "#/definitions/MetricFilter" } }, "time_grains": { "type": "array", "items": { "type": "string" } }, "dimensions": { "type": "array", "items": { "type": "string" } }, "window": { "oneOf": [ { "$ref": "#/definitions/MetricTime" }, { "type": "null" } ] }, "model": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "model_unique_id": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "resource_type": { "type": "string", "enum": [ "model", "analysis", "test", "snapshot", "operation", "seed", "rpc", "sql operation", "docs block", "source", "macro", "exposure", "metric" ], "default": "metric" }, "meta": { "type": "object", "default": {} }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "config": { "$ref": "#/definitions/MetricConfig", "default": { "enabled": true } }, "unrendered_config": { "type": "object", "default": {} }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "created_at": { "type": "number", "default": 1663187715.38939 } }, "additionalProperties": false, "description": "ParsedMetric(fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, description: str, label: str, calculation_method: str, expression: str, timestamp: Union[str, NoneType], filters: List[dbt.contracts.graph.unparsed.MetricFilter], time_grains: List[str], dimensions: List[str], window: Union[dbt.contracts.graph.unparsed.MetricTime, NoneType], model: Union[str, NoneType] = None, model_unique_id: Union[str, NoneType] = None, resource_type: dbt.node_types.NodeType = , meta: Dict[str, Any] = , tags: List[str] = , config: dbt.contracts.graph.model_config.MetricConfig = , unrendered_config: Dict[str, Any] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , refs: List[List[str]] = , metrics: List[List[str]] = , created_at: float = )" }, "MetricFilter": { "type": "object", "required": [ "field", "operator", "value" ], "properties": { "field": { "type": "string" }, "operator": { "type": "string" }, "value": { "type": "string" } }, "additionalProperties": false, "description": "MetricFilter(field: str, operator: str, value: str)" }, "MetricTime": { "type": "object", "required": [], "properties": { "count": { "oneOf": [ { "type": "integer" }, { "type": "null" } ] }, "period": { "oneOf": [ { "type": "string", "enum": [ "day", "week", "month", "year" ] }, { "type": "null" } ] } }, "additionalProperties": false, "description": "MetricTime(count: Union[int, NoneType] = None, period: Union[dbt.contracts.graph.unparsed.MetricTimePeriod, NoneType] = None)" }, "MetricConfig": { "type": "object", "required": [], "properties": { "enabled": { "type": "boolean", "default": true } }, "additionalProperties": true, "description": "MetricConfig(_extra: Dict[str, Any] = , enabled: bool = True)" } }, "$schema": "http://json-schema.org/draft-07/schema#", "$id": "https://schemas.getdbt.com/dbt/manifest/v7.json" } ================================================ FILE: schemas/dbt/manifest/v8.json ================================================ { "type": "object", "required": [ "metadata", "nodes", "sources", "macros", "docs", "exposures", "metrics", "selectors" ], "properties": { "metadata": { "$ref": "#/definitions/ManifestMetadata", "description": "Metadata about the manifest" }, "nodes": { "type": "object", "additionalProperties": { "oneOf": [ { "$ref": "#/definitions/AnalysisNode" }, { "$ref": "#/definitions/SingularTestNode" }, { "$ref": "#/definitions/HookNode" }, { "$ref": "#/definitions/ModelNode" }, { "$ref": "#/definitions/RPCNode" }, { "$ref": "#/definitions/SqlNode" }, { "$ref": "#/definitions/GenericTestNode" }, { "$ref": "#/definitions/SnapshotNode" }, { "$ref": "#/definitions/SeedNode" } ] }, "description": "The nodes defined in the dbt project and its dependencies" }, "sources": { "type": "object", "additionalProperties": { "$ref": "#/definitions/SourceDefinition" }, "description": "The sources defined in the dbt project and its dependencies" }, "macros": { "type": "object", "additionalProperties": { "$ref": "#/definitions/Macro" }, "description": "The macros defined in the dbt project and its dependencies" }, "docs": { "type": "object", "additionalProperties": { "$ref": "#/definitions/Documentation" }, "description": "The docs defined in the dbt project and its dependencies" }, "exposures": { "type": "object", "additionalProperties": { "$ref": "#/definitions/Exposure" }, "description": "The exposures defined in the dbt project and its dependencies" }, "metrics": { "type": "object", "additionalProperties": { "$ref": "#/definitions/Metric" }, "description": "The metrics defined in the dbt project and its dependencies" }, "selectors": { "type": "object", "description": "The selectors defined in selectors.yml" }, "disabled": { "oneOf": [ { "type": "object", "additionalProperties": { "type": "array", "items": { "oneOf": [ { "$ref": "#/definitions/AnalysisNode" }, { "$ref": "#/definitions/SingularTestNode" }, { "$ref": "#/definitions/HookNode" }, { "$ref": "#/definitions/ModelNode" }, { "$ref": "#/definitions/RPCNode" }, { "$ref": "#/definitions/SqlNode" }, { "$ref": "#/definitions/GenericTestNode" }, { "$ref": "#/definitions/SnapshotNode" }, { "$ref": "#/definitions/SeedNode" }, { "$ref": "#/definitions/SourceDefinition" } ] } } }, { "type": "null" } ], "description": "A mapping of the disabled nodes in the target" }, "parent_map": { "oneOf": [ { "type": "object", "additionalProperties": { "type": "array", "items": { "type": "string" } } }, { "type": "null" } ], "description": "A mapping from\u00a0child nodes to their dependencies" }, "child_map": { "oneOf": [ { "type": "object", "additionalProperties": { "type": "array", "items": { "type": "string" } } }, { "type": "null" } ], "description": "A mapping from parent nodes to their dependents" } }, "additionalProperties": false, "description": "WritableManifest(metadata: dbt.contracts.graph.manifest.ManifestMetadata, nodes: Mapping[str, Union[dbt.contracts.graph.nodes.AnalysisNode, dbt.contracts.graph.nodes.SingularTestNode, dbt.contracts.graph.nodes.HookNode, dbt.contracts.graph.nodes.ModelNode, dbt.contracts.graph.nodes.RPCNode, dbt.contracts.graph.nodes.SqlNode, dbt.contracts.graph.nodes.GenericTestNode, dbt.contracts.graph.nodes.SnapshotNode, dbt.contracts.graph.nodes.SeedNode]], sources: Mapping[str, dbt.contracts.graph.nodes.SourceDefinition], macros: Mapping[str, dbt.contracts.graph.nodes.Macro], docs: Mapping[str, dbt.contracts.graph.nodes.Documentation], exposures: Mapping[str, dbt.contracts.graph.nodes.Exposure], metrics: Mapping[str, dbt.contracts.graph.nodes.Metric], selectors: Mapping[str, Any], disabled: Optional[Mapping[str, List[Union[dbt.contracts.graph.nodes.AnalysisNode, dbt.contracts.graph.nodes.SingularTestNode, dbt.contracts.graph.nodes.HookNode, dbt.contracts.graph.nodes.ModelNode, dbt.contracts.graph.nodes.RPCNode, dbt.contracts.graph.nodes.SqlNode, dbt.contracts.graph.nodes.GenericTestNode, dbt.contracts.graph.nodes.SnapshotNode, dbt.contracts.graph.nodes.SeedNode, dbt.contracts.graph.nodes.SourceDefinition]]]], parent_map: Optional[Dict[str, List[str]]], child_map: Optional[Dict[str, List[str]]])", "definitions": { "ManifestMetadata": { "type": "object", "required": [], "properties": { "dbt_schema_version": { "type": "string", "default": "https://schemas.getdbt.com/dbt/manifest/v8.json" }, "dbt_version": { "type": "string", "default": "1.4.1" }, "generated_at": { "type": "string", "format": "date-time", "default": "2023-02-09T10:04:47.350768Z" }, "invocation_id": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "default": "f795bc66-f417-4007-af6e-f2e513d33790" }, "env": { "type": "object", "additionalProperties": { "type": "string" }, "default": {} }, "project_id": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "description": "A unique identifier for the project" }, "user_id": { "oneOf": [ { "type": "string", "pattern": "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" }, { "type": "null" } ], "description": "A unique identifier for the user" }, "send_anonymous_usage_stats": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ], "description": "Whether dbt is configured to send anonymous usage statistics" }, "adapter_type": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "description": "The type name of the adapter" } }, "additionalProperties": false, "description": "Metadata for the manifest." }, "AnalysisNode": { "type": "object", "required": [ "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ], "properties": { "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "analysis" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1675937087.353436 }, "config_call_dict": { "type": "object", "default": {} }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "raw_code": { "type": "string", "default": "" }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] } }, "additionalProperties": false, "description": "AnalysisNode(database: Optional[str], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Optional[str] = None, raw_code: str = '', language: str = 'sql', refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , compiled_path: Optional[str] = None, compiled: bool = False, compiled_code: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = , _pre_injected_sql: Optional[str] = None)" }, "FileHash": { "type": "object", "required": [ "name", "checksum" ], "properties": { "name": { "type": "string" }, "checksum": { "type": "string" } }, "additionalProperties": false, "description": "FileHash(name: str, checksum: str)" }, "NodeConfig": { "type": "object", "required": [], "properties": { "enabled": { "type": "boolean", "default": true }, "alias": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "tags": { "oneOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ], "default": [] }, "meta": { "type": "object", "default": {} }, "materialized": { "type": "string", "default": "view" }, "incremental_strategy": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "persist_docs": { "type": "object", "default": {} }, "post-hook": { "type": "array", "items": { "$ref": "#/definitions/Hook" }, "default": [] }, "pre-hook": { "type": "array", "items": { "$ref": "#/definitions/Hook" }, "default": [] }, "quoting": { "type": "object", "default": {} }, "column_types": { "type": "object", "default": {} }, "full_refresh": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "unique_key": { "oneOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ] }, "on_schema_change": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "grants": { "type": "object", "default": {} }, "packages": { "type": "array", "items": { "type": "string" }, "default": [] }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } } }, "additionalProperties": true, "description": "NodeConfig(_extra: Dict[str, Any] = , enabled: bool = True, alias: Optional[str] = None, schema: Optional[str] = None, database: Optional[str] = None, tags: Union[List[str], str] = , meta: Dict[str, Any] = , materialized: str = 'view', incremental_strategy: Optional[str] = None, persist_docs: Dict[str, Any] = , post_hook: List[dbt.contracts.graph.model_config.Hook] = , pre_hook: List[dbt.contracts.graph.model_config.Hook] = , quoting: Dict[str, Any] = , column_types: Dict[str, Any] = , full_refresh: Optional[bool] = None, unique_key: Union[str, List[str], NoneType] = None, on_schema_change: Optional[str] = 'ignore', grants: Dict[str, Any] = , packages: List[str] = , docs: dbt.contracts.graph.unparsed.Docs = )" }, "Hook": { "type": "object", "required": [ "sql" ], "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "oneOf": [ { "type": "integer" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "Hook(sql: str, transaction: bool = True, index: Optional[int] = None)" }, "Docs": { "type": "object", "required": [], "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "Docs(show: bool = True, node_color: Optional[str] = None)" }, "ColumnInfo": { "type": "object", "required": [ "name" ], "properties": { "name": { "type": "string" }, "description": { "type": "string", "default": "" }, "meta": { "type": "object", "default": {} }, "data_type": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "quote": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] } }, "additionalProperties": true, "description": "Used in all ManifestNodes and SourceDefinition" }, "DependsOn": { "type": "object", "required": [], "properties": { "macros": { "type": "array", "items": { "type": "string" }, "default": [] }, "nodes": { "type": "array", "items": { "type": "string" }, "default": [] } }, "additionalProperties": false, "description": "DependsOn(macros: List[str] = , nodes: List[str] = )" }, "InjectedCTE": { "type": "object", "required": [ "id", "sql" ], "properties": { "id": { "type": "string" }, "sql": { "type": "string" } }, "additionalProperties": false, "description": "Used in CompiledNodes as part of ephemeral model processing" }, "SingularTestNode": { "type": "object", "required": [ "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ], "properties": { "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "test" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/TestConfig", "default": { "enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0" } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1675937087.355371 }, "config_call_dict": { "type": "object", "default": {} }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "raw_code": { "type": "string", "default": "" }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] } }, "additionalProperties": false, "description": "SingularTestNode(database: Optional[str], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.TestConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Optional[str] = None, raw_code: str = '', language: str = 'sql', refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , compiled_path: Optional[str] = None, compiled: bool = False, compiled_code: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = , _pre_injected_sql: Optional[str] = None)" }, "TestConfig": { "type": "object", "required": [], "properties": { "enabled": { "type": "boolean", "default": true }, "alias": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "default": "dbt_test__audit" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "tags": { "oneOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ], "default": [] }, "meta": { "type": "object", "default": {} }, "materialized": { "type": "string", "default": "test" }, "severity": { "type": "string", "pattern": "^([Ww][Aa][Rr][Nn]|[Ee][Rr][Rr][Oo][Rr])$", "default": "ERROR" }, "store_failures": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "where": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "limit": { "oneOf": [ { "type": "integer" }, { "type": "null" } ] }, "fail_calc": { "type": "string", "default": "count(*)" }, "warn_if": { "type": "string", "default": "!= 0" }, "error_if": { "type": "string", "default": "!= 0" } }, "additionalProperties": true, "description": "TestConfig(_extra: Dict[str, Any] = , enabled: bool = True, alias: Optional[str] = None, schema: Optional[str] = 'dbt_test__audit', database: Optional[str] = None, tags: Union[List[str], str] = , meta: Dict[str, Any] = , materialized: str = 'test', severity: dbt.contracts.graph.model_config.Severity = 'ERROR', store_failures: Optional[bool] = None, where: Optional[str] = None, limit: Optional[int] = None, fail_calc: str = 'count(*)', warn_if: str = '!= 0', error_if: str = '!= 0')" }, "HookNode": { "type": "object", "required": [ "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ], "properties": { "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "operation" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1675937087.356482 }, "config_call_dict": { "type": "object", "default": {} }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "raw_code": { "type": "string", "default": "" }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "index": { "oneOf": [ { "type": "integer" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "HookNode(database: Optional[str], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Optional[str] = None, raw_code: str = '', language: str = 'sql', refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , compiled_path: Optional[str] = None, compiled: bool = False, compiled_code: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = , _pre_injected_sql: Optional[str] = None, index: Optional[int] = None)" }, "ModelNode": { "type": "object", "required": [ "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ], "properties": { "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "model" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1675937087.357701 }, "config_call_dict": { "type": "object", "default": {} }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "raw_code": { "type": "string", "default": "" }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] } }, "additionalProperties": false, "description": "ModelNode(database: Optional[str], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Optional[str] = None, raw_code: str = '', language: str = 'sql', refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , compiled_path: Optional[str] = None, compiled: bool = False, compiled_code: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = , _pre_injected_sql: Optional[str] = None)" }, "RPCNode": { "type": "object", "required": [ "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ], "properties": { "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "rpc" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1675937087.358761 }, "config_call_dict": { "type": "object", "default": {} }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "raw_code": { "type": "string", "default": "" }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] } }, "additionalProperties": false, "description": "RPCNode(database: Optional[str], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Optional[str] = None, raw_code: str = '', language: str = 'sql', refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , compiled_path: Optional[str] = None, compiled: bool = False, compiled_code: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = , _pre_injected_sql: Optional[str] = None)" }, "SqlNode": { "type": "object", "required": [ "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ], "properties": { "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "sql operation" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1675937087.359803 }, "config_call_dict": { "type": "object", "default": {} }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "raw_code": { "type": "string", "default": "" }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] } }, "additionalProperties": false, "description": "SqlNode(database: Optional[str], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Optional[str] = None, raw_code: str = '', language: str = 'sql', refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , compiled_path: Optional[str] = None, compiled: bool = False, compiled_code: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = , _pre_injected_sql: Optional[str] = None)" }, "GenericTestNode": { "type": "object", "required": [ "test_metadata", "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ], "properties": { "test_metadata": { "$ref": "#/definitions/TestMetadata" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "test" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/TestConfig", "default": { "enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0" } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1675937087.361009 }, "config_call_dict": { "type": "object", "default": {} }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "raw_code": { "type": "string", "default": "" }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "column_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "file_key_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "GenericTestNode(test_metadata: dbt.contracts.graph.nodes.TestMetadata, database: Optional[str], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.TestConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Optional[str] = None, raw_code: str = '', language: str = 'sql', refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , compiled_path: Optional[str] = None, compiled: bool = False, compiled_code: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = , _pre_injected_sql: Optional[str] = None, column_name: Optional[str] = None, file_key_name: Optional[str] = None)" }, "TestMetadata": { "type": "object", "required": [ "name" ], "properties": { "name": { "type": "string" }, "kwargs": { "type": "object", "default": {} }, "namespace": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "TestMetadata(name: str, kwargs: Dict[str, Any] = , namespace: Optional[str] = None)" }, "SnapshotNode": { "type": "object", "required": [ "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum", "config" ], "properties": { "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "snapshot" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/SnapshotConfig" }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1675937087.364386 }, "config_call_dict": { "type": "object", "default": {} }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "raw_code": { "type": "string", "default": "" }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] } }, "additionalProperties": false, "description": "SnapshotNode(database: Optional[str], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.SnapshotConfig, _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Optional[str] = None, raw_code: str = '', language: str = 'sql', refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , compiled_path: Optional[str] = None, compiled: bool = False, compiled_code: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = , _pre_injected_sql: Optional[str] = None)" }, "SnapshotConfig": { "type": "object", "required": [], "properties": { "enabled": { "type": "boolean", "default": true }, "alias": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "tags": { "oneOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ], "default": [] }, "meta": { "type": "object", "default": {} }, "materialized": { "type": "string", "default": "snapshot" }, "incremental_strategy": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "persist_docs": { "type": "object", "default": {} }, "post-hook": { "type": "array", "items": { "$ref": "#/definitions/Hook" }, "default": [] }, "pre-hook": { "type": "array", "items": { "$ref": "#/definitions/Hook" }, "default": [] }, "quoting": { "type": "object", "default": {} }, "column_types": { "type": "object", "default": {} }, "full_refresh": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "unique_key": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "on_schema_change": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "grants": { "type": "object", "default": {} }, "packages": { "type": "array", "items": { "type": "string" }, "default": [] }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "strategy": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "target_schema": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "target_database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "updated_at": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "check_cols": { "oneOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ] } }, "additionalProperties": true, "description": "SnapshotConfig(_extra: Dict[str, Any] = , enabled: bool = True, alias: Optional[str] = None, schema: Optional[str] = None, database: Optional[str] = None, tags: Union[List[str], str] = , meta: Dict[str, Any] = , materialized: str = 'snapshot', incremental_strategy: Optional[str] = None, persist_docs: Dict[str, Any] = , post_hook: List[dbt.contracts.graph.model_config.Hook] = , pre_hook: List[dbt.contracts.graph.model_config.Hook] = , quoting: Dict[str, Any] = , column_types: Dict[str, Any] = , full_refresh: Optional[bool] = None, unique_key: Optional[str] = None, on_schema_change: Optional[str] = 'ignore', grants: Dict[str, Any] = , packages: List[str] = , docs: dbt.contracts.graph.unparsed.Docs = , strategy: Optional[str] = None, target_schema: Optional[str] = None, target_database: Optional[str] = None, updated_at: Optional[str] = None, check_cols: Union[str, List[str], NoneType] = None)" }, "SeedNode": { "type": "object", "required": [ "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ], "properties": { "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "seed" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/SeedConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "seed", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "quote_columns": null, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1675937087.366245 }, "config_call_dict": { "type": "object", "default": {} }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "raw_code": { "type": "string", "default": "" }, "root_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "depends_on": { "$ref": "#/definitions/MacroDependsOn", "default": { "macros": [] } } }, "additionalProperties": false, "description": "SeedNode(database: Optional[str], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.SeedConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Optional[str] = None, raw_code: str = '', root_path: Optional[str] = None, depends_on: dbt.contracts.graph.nodes.MacroDependsOn = )" }, "SeedConfig": { "type": "object", "required": [], "properties": { "enabled": { "type": "boolean", "default": true }, "alias": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "tags": { "oneOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ], "default": [] }, "meta": { "type": "object", "default": {} }, "materialized": { "type": "string", "default": "seed" }, "incremental_strategy": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "persist_docs": { "type": "object", "default": {} }, "post-hook": { "type": "array", "items": { "$ref": "#/definitions/Hook" }, "default": [] }, "pre-hook": { "type": "array", "items": { "$ref": "#/definitions/Hook" }, "default": [] }, "quoting": { "type": "object", "default": {} }, "column_types": { "type": "object", "default": {} }, "full_refresh": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "unique_key": { "oneOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ] }, "on_schema_change": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "grants": { "type": "object", "default": {} }, "packages": { "type": "array", "items": { "type": "string" }, "default": [] }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "quote_columns": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] } }, "additionalProperties": true, "description": "SeedConfig(_extra: Dict[str, Any] = , enabled: bool = True, alias: Optional[str] = None, schema: Optional[str] = None, database: Optional[str] = None, tags: Union[List[str], str] = , meta: Dict[str, Any] = , materialized: str = 'seed', incremental_strategy: Optional[str] = None, persist_docs: Dict[str, Any] = , post_hook: List[dbt.contracts.graph.model_config.Hook] = , pre_hook: List[dbt.contracts.graph.model_config.Hook] = , quoting: Dict[str, Any] = , column_types: Dict[str, Any] = , full_refresh: Optional[bool] = None, unique_key: Union[str, List[str], NoneType] = None, on_schema_change: Optional[str] = 'ignore', grants: Dict[str, Any] = , packages: List[str] = , docs: dbt.contracts.graph.unparsed.Docs = , quote_columns: Optional[bool] = None)" }, "MacroDependsOn": { "type": "object", "required": [], "properties": { "macros": { "type": "array", "items": { "type": "string" }, "default": [] } }, "additionalProperties": false, "description": "Used only in the Macro class" }, "SourceDefinition": { "type": "object", "required": [ "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "source_name", "source_description", "loader", "identifier" ], "properties": { "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "source" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "source_name": { "type": "string" }, "source_description": { "type": "string" }, "loader": { "type": "string" }, "identifier": { "type": "string" }, "quoting": { "$ref": "#/definitions/Quoting", "default": { "database": null, "schema": null, "identifier": null, "column": null } }, "loaded_at_field": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "freshness": { "oneOf": [ { "$ref": "#/definitions/FreshnessThreshold" }, { "type": "null" } ] }, "external": { "oneOf": [ { "$ref": "#/definitions/ExternalTable" }, { "type": "null" } ] }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "source_meta": { "type": "object", "default": {} }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "config": { "$ref": "#/definitions/SourceConfig", "default": { "enabled": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "unrendered_config": { "type": "object", "default": {} }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "created_at": { "type": "number", "default": 1675937087.368067 } }, "additionalProperties": false, "description": "SourceDefinition(database: Optional[str], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], source_name: str, source_description: str, loader: str, identifier: str, _event_status: Dict[str, Any] = , quoting: dbt.contracts.graph.unparsed.Quoting = , loaded_at_field: Optional[str] = None, freshness: Optional[dbt.contracts.graph.unparsed.FreshnessThreshold] = None, external: Optional[dbt.contracts.graph.unparsed.ExternalTable] = None, description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , source_meta: Dict[str, Any] = , tags: List[str] = , config: dbt.contracts.graph.model_config.SourceConfig = , patch_path: Optional[str] = None, unrendered_config: Dict[str, Any] = , relation_name: Optional[str] = None, created_at: float = )" }, "Quoting": { "type": "object", "required": [], "properties": { "database": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "schema": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "identifier": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "column": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "Quoting(database: Optional[bool] = None, schema: Optional[bool] = None, identifier: Optional[bool] = None, column: Optional[bool] = None)" }, "FreshnessThreshold": { "type": "object", "required": [], "properties": { "warn_after": { "oneOf": [ { "$ref": "#/definitions/Time" }, { "type": "null" } ], "default": { "count": null, "period": null } }, "error_after": { "oneOf": [ { "$ref": "#/definitions/Time" }, { "type": "null" } ], "default": { "count": null, "period": null } }, "filter": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "FreshnessThreshold(warn_after: Optional[dbt.contracts.graph.unparsed.Time] = , error_after: Optional[dbt.contracts.graph.unparsed.Time] = , filter: Optional[str] = None)" }, "FreshnessMetadata": { "type": "object", "required": [], "properties": { "dbt_schema_version": { "type": "string", "default": "https://schemas.getdbt.com/dbt/sources/v3.json" }, "dbt_version": { "type": "string", "default": "1.4.1" }, "generated_at": { "type": "string", "format": "date-time", "default": "2023-02-09T10:04:47.347023Z" }, "invocation_id": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "default": "f795bc66-f417-4007-af6e-f2e513d33790" }, "env": { "type": "object", "additionalProperties": { "type": "string" }, "default": {} } }, "additionalProperties": false, "description": "FreshnessMetadata(dbt_schema_version: str = , dbt_version: str = '1.4.1', generated_at: datetime.datetime = , invocation_id: Optional[str] = , env: Dict[str, str] = )" }, "SourceFreshnessRuntimeError": { "type": "object", "required": [ "unique_id", "status" ], "properties": { "unique_id": { "type": "string" }, "error": { "oneOf": [ { "type": "string" }, { "type": "integer" }, { "type": "null" } ] }, "status": { "type": "string", "enum": [ "runtime error" ] } }, "additionalProperties": false, "description": "SourceFreshnessRuntimeError(unique_id: str, error: Union[str, int, NoneType], status: dbt.contracts.results.FreshnessErrorEnum)" }, "SourceFreshnessOutput": { "type": "object", "required": [ "unique_id", "max_loaded_at", "snapshotted_at", "max_loaded_at_time_ago_in_s", "status", "criteria", "adapter_response", "timing", "thread_id", "execution_time" ], "properties": { "unique_id": { "type": "string" }, "max_loaded_at": { "type": "string", "format": "date-time" }, "snapshotted_at": { "type": "string", "format": "date-time" }, "max_loaded_at_time_ago_in_s": { "type": "number" }, "status": { "type": "string", "enum": [ "pass", "warn", "error", "runtime error" ] }, "criteria": { "$ref": "#/definitions/FreshnessThreshold" }, "adapter_response": { "type": "object" }, "timing": { "type": "array", "items": { "$ref": "#/definitions/TimingInfo" } }, "thread_id": { "type": "string" }, "execution_time": { "type": "number" } }, "additionalProperties": false, "description": "SourceFreshnessOutput(unique_id: str, max_loaded_at: datetime.datetime, snapshotted_at: datetime.datetime, max_loaded_at_time_ago_in_s: float, status: dbt.contracts.results.FreshnessStatus, criteria: dbt.contracts.graph.unparsed.FreshnessThreshold, adapter_response: Dict[str, Any], timing: List[dbt.contracts.results.TimingInfo], thread_id: str, execution_time: float)" }, "Time": { "type": "object", "required": [], "properties": { "count": { "oneOf": [ { "type": "integer" }, { "type": "null" } ] }, "period": { "oneOf": [ { "type": "string", "enum": [ "minute", "hour", "day" ] }, { "type": "null" } ] } }, "additionalProperties": false, "description": "Time(count: Optional[int] = None, period: Optional[dbt.contracts.graph.unparsed.TimePeriod] = None)" }, "TimingInfo": { "type": "object", "required": [ "name" ], "properties": { "name": { "type": "string" }, "started_at": { "oneOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ] }, "completed_at": { "oneOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "TimingInfo(name: str, started_at: Optional[datetime.datetime] = None, completed_at: Optional[datetime.datetime] = None)" }, "ExternalTable": { "type": "object", "required": [], "properties": { "location": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "file_format": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "row_format": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "tbl_properties": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "partitions": { "oneOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "array", "items": { "$ref": "#/definitions/ExternalPartition" } }, { "type": "null" } ] } }, "additionalProperties": true, "description": "ExternalTable(_extra: Dict[str, Any] = , location: Optional[str] = None, file_format: Optional[str] = None, row_format: Optional[str] = None, tbl_properties: Optional[str] = None, partitions: Union[List[str], List[dbt.contracts.graph.unparsed.ExternalPartition], NoneType] = None)" }, "ExternalPartition": { "type": "object", "required": [], "properties": { "name": { "type": "string", "default": "" }, "description": { "type": "string", "default": "" }, "data_type": { "type": "string", "default": "" }, "meta": { "type": "object", "default": {} } }, "additionalProperties": true, "description": "ExternalPartition(_extra: Dict[str, Any] = , name: str = '', description: str = '', data_type: str = '', meta: Dict[str, Any] = )" }, "SourceConfig": { "type": "object", "required": [], "properties": { "enabled": { "type": "boolean", "default": true } }, "additionalProperties": true, "description": "SourceConfig(_extra: Dict[str, Any] = , enabled: bool = True)" }, "Macro": { "type": "object", "required": [ "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "macro_sql" ], "properties": { "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "macro" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "macro_sql": { "type": "string" }, "depends_on": { "$ref": "#/definitions/MacroDependsOn", "default": { "macros": [] } }, "description": { "type": "string", "default": "" }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "arguments": { "type": "array", "items": { "$ref": "#/definitions/MacroArgument" }, "default": [] }, "created_at": { "type": "number", "default": 1675937087.368656 }, "supported_languages": { "oneOf": [ { "type": "array", "items": { "type": "string", "enum": [ "python", "sql" ] } }, { "type": "null" } ] } }, "additionalProperties": false, "description": "Macro(name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, macro_sql: str, depends_on: dbt.contracts.graph.nodes.MacroDependsOn = , description: str = '', meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, arguments: List[dbt.contracts.graph.unparsed.MacroArgument] = , created_at: float = , supported_languages: Optional[List[dbt.node_types.ModelLanguage]] = None)" }, "MacroArgument": { "type": "object", "required": [ "name" ], "properties": { "name": { "type": "string" }, "type": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "description": { "type": "string", "default": "" } }, "additionalProperties": false, "description": "MacroArgument(name: str, type: Optional[str] = None, description: str = '')" }, "Documentation": { "type": "object", "required": [ "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "block_contents" ], "properties": { "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "doc" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "block_contents": { "type": "string" } }, "additionalProperties": false, "description": "Documentation(name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, block_contents: str)" }, "Exposure": { "type": "object", "required": [ "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "type", "owner" ], "properties": { "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "exposure" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "type": { "type": "string", "enum": [ "dashboard", "notebook", "analysis", "ml", "application" ] }, "owner": { "$ref": "#/definitions/ExposureOwner" }, "description": { "type": "string", "default": "" }, "label": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "maturity": { "oneOf": [ { "type": "string", "enum": [ "low", "medium", "high" ] }, { "type": "null" } ] }, "meta": { "type": "object", "default": {} }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "config": { "$ref": "#/definitions/ExposureConfig", "default": { "enabled": true } }, "unrendered_config": { "type": "object", "default": {} }, "url": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "created_at": { "type": "number", "default": 1675937087.369866 } }, "additionalProperties": false, "description": "Exposure(name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], type: dbt.contracts.graph.unparsed.ExposureType, owner: dbt.contracts.graph.unparsed.ExposureOwner, description: str = '', label: Optional[str] = None, maturity: Optional[dbt.contracts.graph.unparsed.MaturityType] = None, meta: Dict[str, Any] = , tags: List[str] = , config: dbt.contracts.graph.model_config.ExposureConfig = , unrendered_config: Dict[str, Any] = , url: Optional[str] = None, depends_on: dbt.contracts.graph.nodes.DependsOn = , refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , created_at: float = )" }, "ExposureOwner": { "type": "object", "required": [ "email" ], "properties": { "email": { "type": "string" }, "name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "ExposureOwner(email: str, name: Optional[str] = None)" }, "ExposureConfig": { "type": "object", "required": [], "properties": { "enabled": { "type": "boolean", "default": true } }, "additionalProperties": true, "description": "ExposureConfig(_extra: Dict[str, Any] = , enabled: bool = True)" }, "Metric": { "type": "object", "required": [ "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "description", "label", "calculation_method", "expression", "filters", "time_grains", "dimensions" ], "properties": { "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "metric" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "description": { "type": "string" }, "label": { "type": "string" }, "calculation_method": { "type": "string" }, "expression": { "type": "string" }, "filters": { "type": "array", "items": { "$ref": "#/definitions/MetricFilter" } }, "time_grains": { "type": "array", "items": { "type": "string" } }, "dimensions": { "type": "array", "items": { "type": "string" } }, "timestamp": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "window": { "oneOf": [ { "$ref": "#/definitions/MetricTime" }, { "type": "null" } ] }, "model": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "model_unique_id": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "meta": { "type": "object", "default": {} }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "config": { "$ref": "#/definitions/MetricConfig", "default": { "enabled": true } }, "unrendered_config": { "type": "object", "default": {} }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "refs": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "created_at": { "type": "number", "default": 1675937087.371092 } }, "additionalProperties": false, "description": "Metric(name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], description: str, label: str, calculation_method: str, expression: str, filters: List[dbt.contracts.graph.unparsed.MetricFilter], time_grains: List[str], dimensions: List[str], timestamp: Optional[str] = None, window: Optional[dbt.contracts.graph.unparsed.MetricTime] = None, model: Optional[str] = None, model_unique_id: Optional[str] = None, meta: Dict[str, Any] = , tags: List[str] = , config: dbt.contracts.graph.model_config.MetricConfig = , unrendered_config: Dict[str, Any] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , refs: List[List[str]] = , metrics: List[List[str]] = , created_at: float = )" }, "MetricFilter": { "type": "object", "required": [ "field", "operator", "value" ], "properties": { "field": { "type": "string" }, "operator": { "type": "string" }, "value": { "type": "string" } }, "additionalProperties": false, "description": "MetricFilter(field: str, operator: str, value: str)" }, "MetricTime": { "type": "object", "required": [], "properties": { "count": { "oneOf": [ { "type": "integer" }, { "type": "null" } ] }, "period": { "oneOf": [ { "type": "string", "enum": [ "day", "week", "month", "year" ] }, { "type": "null" } ] } }, "additionalProperties": false, "description": "MetricTime(count: Optional[int] = None, period: Optional[dbt.contracts.graph.unparsed.MetricTimePeriod] = None)" }, "MetricConfig": { "type": "object", "required": [], "properties": { "enabled": { "type": "boolean", "default": true } }, "additionalProperties": true, "description": "MetricConfig(_extra: Dict[str, Any] = , enabled: bool = True)" } }, "$schema": "http://json-schema.org/draft-07/schema#", "$id": "https://schemas.getdbt.com/dbt/manifest/v8.json" } ================================================ FILE: schemas/dbt/manifest/v9.json ================================================ { "type": "object", "required": [ "metadata", "nodes", "sources", "macros", "docs", "exposures", "metrics", "groups", "selectors" ], "properties": { "metadata": { "$ref": "#/definitions/ManifestMetadata", "description": "Metadata about the manifest" }, "nodes": { "type": "object", "additionalProperties": { "oneOf": [ { "$ref": "#/definitions/AnalysisNode" }, { "$ref": "#/definitions/SingularTestNode" }, { "$ref": "#/definitions/HookNode" }, { "$ref": "#/definitions/ModelNode" }, { "$ref": "#/definitions/RPCNode" }, { "$ref": "#/definitions/SqlNode" }, { "$ref": "#/definitions/GenericTestNode" }, { "$ref": "#/definitions/SnapshotNode" }, { "$ref": "#/definitions/SeedNode" } ] }, "description": "The nodes defined in the dbt project and its dependencies" }, "sources": { "type": "object", "additionalProperties": { "$ref": "#/definitions/SourceDefinition" }, "description": "The sources defined in the dbt project and its dependencies" }, "macros": { "type": "object", "additionalProperties": { "$ref": "#/definitions/Macro" }, "description": "The macros defined in the dbt project and its dependencies" }, "docs": { "type": "object", "additionalProperties": { "$ref": "#/definitions/Documentation" }, "description": "The docs defined in the dbt project and its dependencies" }, "exposures": { "type": "object", "additionalProperties": { "$ref": "#/definitions/Exposure" }, "description": "The exposures defined in the dbt project and its dependencies" }, "metrics": { "type": "object", "additionalProperties": { "$ref": "#/definitions/Metric" }, "description": "The metrics defined in the dbt project and its dependencies" }, "groups": { "type": "object", "additionalProperties": { "$ref": "#/definitions/Group" }, "description": "The groups defined in the dbt project" }, "selectors": { "type": "object", "description": "The selectors defined in selectors.yml" }, "disabled": { "oneOf": [ { "type": "object", "additionalProperties": { "type": "array", "items": { "oneOf": [ { "$ref": "#/definitions/AnalysisNode" }, { "$ref": "#/definitions/SingularTestNode" }, { "$ref": "#/definitions/HookNode" }, { "$ref": "#/definitions/ModelNode" }, { "$ref": "#/definitions/RPCNode" }, { "$ref": "#/definitions/SqlNode" }, { "$ref": "#/definitions/GenericTestNode" }, { "$ref": "#/definitions/SnapshotNode" }, { "$ref": "#/definitions/SeedNode" }, { "$ref": "#/definitions/SourceDefinition" }, { "$ref": "#/definitions/Exposure" }, { "$ref": "#/definitions/Metric" } ] } } }, { "type": "null" } ], "description": "A mapping of the disabled nodes in the target" }, "parent_map": { "oneOf": [ { "type": "object", "additionalProperties": { "type": "array", "items": { "type": "string" } } }, { "type": "null" } ], "description": "A mapping from\u00a0child nodes to their dependencies" }, "child_map": { "oneOf": [ { "type": "object", "additionalProperties": { "type": "array", "items": { "type": "string" } } }, { "type": "null" } ], "description": "A mapping from parent nodes to their dependents" }, "group_map": { "oneOf": [ { "type": "object", "additionalProperties": { "type": "array", "items": { "type": "string" } } }, { "type": "null" } ], "description": "A mapping from group names to their nodes" } }, "additionalProperties": false, "description": "WritableManifest(metadata: dbt.contracts.graph.manifest.ManifestMetadata, nodes: Mapping[str, Union[dbt.contracts.graph.nodes.AnalysisNode, dbt.contracts.graph.nodes.SingularTestNode, dbt.contracts.graph.nodes.HookNode, dbt.contracts.graph.nodes.ModelNode, dbt.contracts.graph.nodes.RPCNode, dbt.contracts.graph.nodes.SqlNode, dbt.contracts.graph.nodes.GenericTestNode, dbt.contracts.graph.nodes.SnapshotNode, dbt.contracts.graph.nodes.SeedNode]], sources: Mapping[str, dbt.contracts.graph.nodes.SourceDefinition], macros: Mapping[str, dbt.contracts.graph.nodes.Macro], docs: Mapping[str, dbt.contracts.graph.nodes.Documentation], exposures: Mapping[str, dbt.contracts.graph.nodes.Exposure], metrics: Mapping[str, dbt.contracts.graph.nodes.Metric], groups: Mapping[str, dbt.contracts.graph.nodes.Group], selectors: Mapping[str, Any], disabled: Optional[Mapping[str, List[Union[dbt.contracts.graph.nodes.AnalysisNode, dbt.contracts.graph.nodes.SingularTestNode, dbt.contracts.graph.nodes.HookNode, dbt.contracts.graph.nodes.ModelNode, dbt.contracts.graph.nodes.RPCNode, dbt.contracts.graph.nodes.SqlNode, dbt.contracts.graph.nodes.GenericTestNode, dbt.contracts.graph.nodes.SnapshotNode, dbt.contracts.graph.nodes.SeedNode, dbt.contracts.graph.nodes.SourceDefinition, dbt.contracts.graph.nodes.Exposure, dbt.contracts.graph.nodes.Metric]]]], parent_map: Optional[Dict[str, List[str]]], child_map: Optional[Dict[str, List[str]]], group_map: Optional[Dict[str, List[str]]])", "definitions": { "ManifestMetadata": { "type": "object", "required": [], "properties": { "dbt_schema_version": { "type": "string", "default": "https://schemas.getdbt.com/dbt/manifest/v9.json" }, "dbt_version": { "type": "string", "default": "1.5.0b5" }, "generated_at": { "type": "string", "format": "date-time", "default": "2023-04-12T03:35:01.188035Z" }, "invocation_id": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "default": "8aa1596d-f52f-40bc-ad4b-f5e48fc7e6c2" }, "env": { "type": "object", "additionalProperties": { "type": "string" }, "default": {} }, "project_id": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "description": "A unique identifier for the project" }, "user_id": { "oneOf": [ { "type": "string", "pattern": "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" }, { "type": "null" } ], "description": "A unique identifier for the user" }, "send_anonymous_usage_stats": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ], "description": "Whether dbt is configured to send anonymous usage statistics" }, "adapter_type": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "description": "The type name of the adapter" } }, "additionalProperties": false, "description": "Metadata for the manifest." }, "AnalysisNode": { "type": "object", "required": [ "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ], "properties": { "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "analysis" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "contract": { "enforced": false }, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "group": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1681270501.189703 }, "config_call_dict": { "type": "object", "default": {} }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "raw_code": { "type": "string", "default": "" }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "$ref": "#/definitions/RefArgs" }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "contract": { "$ref": "#/definitions/Contract", "default": { "enforced": false, "checksum": null } } }, "additionalProperties": false, "description": "AnalysisNode(database: Optional[str], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , group: Optional[str] = None, docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Optional[str] = None, raw_code: str = '', language: str = 'sql', refs: List[dbt.contracts.graph.nodes.RefArgs] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , compiled_path: Optional[str] = None, compiled: bool = False, compiled_code: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = , _pre_injected_sql: Optional[str] = None, contract: dbt.contracts.graph.nodes.Contract = )" }, "FileHash": { "type": "object", "required": [ "name", "checksum" ], "properties": { "name": { "type": "string" }, "checksum": { "type": "string" } }, "additionalProperties": false, "description": "FileHash(name: str, checksum: str)" }, "NodeConfig": { "type": "object", "required": [], "properties": { "enabled": { "type": "boolean", "default": true }, "alias": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "tags": { "oneOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ], "default": [] }, "meta": { "type": "object", "default": {} }, "group": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "materialized": { "type": "string", "default": "view" }, "incremental_strategy": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "persist_docs": { "type": "object", "default": {} }, "post-hook": { "type": "array", "items": { "$ref": "#/definitions/Hook" }, "default": [] }, "pre-hook": { "type": "array", "items": { "$ref": "#/definitions/Hook" }, "default": [] }, "quoting": { "type": "object", "default": {} }, "column_types": { "type": "object", "default": {} }, "full_refresh": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "unique_key": { "oneOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ] }, "on_schema_change": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "grants": { "type": "object", "default": {} }, "packages": { "type": "array", "items": { "type": "string" }, "default": [] }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "contract": { "$ref": "#/definitions/ContractConfig", "default": { "enforced": false } } }, "additionalProperties": true, "description": "NodeConfig(_extra: Dict[str, Any] = , enabled: bool = True, alias: Optional[str] = None, schema: Optional[str] = None, database: Optional[str] = None, tags: Union[List[str], str] = , meta: Dict[str, Any] = , group: Optional[str] = None, materialized: str = 'view', incremental_strategy: Optional[str] = None, persist_docs: Dict[str, Any] = , post_hook: List[dbt.contracts.graph.model_config.Hook] = , pre_hook: List[dbt.contracts.graph.model_config.Hook] = , quoting: Dict[str, Any] = , column_types: Dict[str, Any] = , full_refresh: Optional[bool] = None, unique_key: Union[str, List[str], NoneType] = None, on_schema_change: Optional[str] = 'ignore', grants: Dict[str, Any] = , packages: List[str] = , docs: dbt.contracts.graph.unparsed.Docs = , contract: dbt.contracts.graph.model_config.ContractConfig = )" }, "Hook": { "type": "object", "required": [ "sql" ], "properties": { "sql": { "type": "string" }, "transaction": { "type": "boolean", "default": true }, "index": { "oneOf": [ { "type": "integer" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "Hook(sql: str, transaction: bool = True, index: Optional[int] = None)" }, "Docs": { "type": "object", "required": [], "properties": { "show": { "type": "boolean", "default": true }, "node_color": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "Docs(show: bool = True, node_color: Optional[str] = None)" }, "ContractConfig": { "type": "object", "required": [], "properties": { "enforced": { "type": "boolean", "default": false } }, "additionalProperties": false, "description": "ContractConfig(enforced: bool = False)" }, "ColumnInfo": { "type": "object", "required": [ "name" ], "properties": { "name": { "type": "string" }, "description": { "type": "string", "default": "" }, "meta": { "type": "object", "default": {} }, "data_type": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "constraints": { "type": "array", "items": { "$ref": "#/definitions/ColumnLevelConstraint" }, "default": [] }, "quote": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] } }, "additionalProperties": true, "description": "Used in all ManifestNodes and SourceDefinition" }, "ColumnLevelConstraint": { "type": "object", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ "check", "not_null", "unique", "primary_key", "foreign_key", "custom" ] }, "name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "expression": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "warn_unenforced": { "type": "boolean", "default": true }, "warn_unsupported": { "type": "boolean", "default": true } }, "additionalProperties": false, "description": "ColumnLevelConstraint(type: dbt.contracts.graph.nodes.ConstraintType, name: Optional[str] = None, expression: Optional[str] = None, warn_unenforced: bool = True, warn_unsupported: bool = True)" }, "RefArgs": { "type": "object", "required": [ "name" ], "properties": { "name": { "type": "string" }, "package": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "version": { "oneOf": [ { "type": "string" }, { "type": "number" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "RefArgs(name: str, package: Optional[str] = None, version: Union[str, float, NoneType] = None)" }, "DependsOn": { "type": "object", "required": [], "properties": { "macros": { "type": "array", "items": { "type": "string" }, "default": [] }, "nodes": { "type": "array", "items": { "type": "string" }, "default": [] } }, "additionalProperties": false, "description": "DependsOn(macros: List[str] = , nodes: List[str] = )" }, "InjectedCTE": { "type": "object", "required": [ "id", "sql" ], "properties": { "id": { "type": "string" }, "sql": { "type": "string" } }, "additionalProperties": false, "description": "Used in CompiledNodes as part of ephemeral model processing" }, "Contract": { "type": "object", "required": [], "properties": { "enforced": { "type": "boolean", "default": false }, "checksum": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "Contract(enforced: bool = False, checksum: Optional[str] = None)" }, "SingularTestNode": { "type": "object", "required": [ "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ], "properties": { "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "test" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/TestConfig", "default": { "enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0" } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "group": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1681270501.19095 }, "config_call_dict": { "type": "object", "default": {} }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "raw_code": { "type": "string", "default": "" }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "$ref": "#/definitions/RefArgs" }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "contract": { "$ref": "#/definitions/Contract", "default": { "enforced": false, "checksum": null } } }, "additionalProperties": false, "description": "SingularTestNode(database: Optional[str], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.TestConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , group: Optional[str] = None, docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Optional[str] = None, raw_code: str = '', language: str = 'sql', refs: List[dbt.contracts.graph.nodes.RefArgs] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , compiled_path: Optional[str] = None, compiled: bool = False, compiled_code: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = , _pre_injected_sql: Optional[str] = None, contract: dbt.contracts.graph.nodes.Contract = )" }, "TestConfig": { "type": "object", "required": [], "properties": { "enabled": { "type": "boolean", "default": true }, "alias": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "default": "dbt_test__audit" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "tags": { "oneOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ], "default": [] }, "meta": { "type": "object", "default": {} }, "group": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "materialized": { "type": "string", "default": "test" }, "severity": { "type": "string", "pattern": "^([Ww][Aa][Rr][Nn]|[Ee][Rr][Rr][Oo][Rr])$", "default": "ERROR" }, "store_failures": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "where": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "limit": { "oneOf": [ { "type": "integer" }, { "type": "null" } ] }, "fail_calc": { "type": "string", "default": "count(*)" }, "warn_if": { "type": "string", "default": "!= 0" }, "error_if": { "type": "string", "default": "!= 0" } }, "additionalProperties": true, "description": "TestConfig(_extra: Dict[str, Any] = , enabled: bool = True, alias: Optional[str] = None, schema: Optional[str] = 'dbt_test__audit', database: Optional[str] = None, tags: Union[List[str], str] = , meta: Dict[str, Any] = , group: Optional[str] = None, materialized: str = 'test', severity: dbt.contracts.graph.model_config.Severity = 'ERROR', store_failures: Optional[bool] = None, where: Optional[str] = None, limit: Optional[int] = None, fail_calc: str = 'count(*)', warn_if: str = '!= 0', error_if: str = '!= 0')" }, "HookNode": { "type": "object", "required": [ "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ], "properties": { "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "operation" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "contract": { "enforced": false }, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "group": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1681270501.191555 }, "config_call_dict": { "type": "object", "default": {} }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "raw_code": { "type": "string", "default": "" }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "$ref": "#/definitions/RefArgs" }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "contract": { "$ref": "#/definitions/Contract", "default": { "enforced": false, "checksum": null } }, "index": { "oneOf": [ { "type": "integer" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "HookNode(database: Optional[str], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , group: Optional[str] = None, docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Optional[str] = None, raw_code: str = '', language: str = 'sql', refs: List[dbt.contracts.graph.nodes.RefArgs] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , compiled_path: Optional[str] = None, compiled: bool = False, compiled_code: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = , _pre_injected_sql: Optional[str] = None, contract: dbt.contracts.graph.nodes.Contract = , index: Optional[int] = None)" }, "ModelNode": { "type": "object", "required": [ "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ], "properties": { "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "model" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "contract": { "enforced": false }, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "group": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1681270501.192162 }, "config_call_dict": { "type": "object", "default": {} }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "raw_code": { "type": "string", "default": "" }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "$ref": "#/definitions/RefArgs" }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "contract": { "$ref": "#/definitions/Contract", "default": { "enforced": false, "checksum": null } }, "access": { "type": "string", "enum": [ "protected", "private", "public" ], "default": "protected" }, "constraints": { "type": "array", "items": { "$ref": "#/definitions/ModelLevelConstraint" }, "default": [] }, "version": { "oneOf": [ { "type": "string" }, { "type": "number" }, { "type": "null" } ] }, "latest_version": { "oneOf": [ { "type": "string" }, { "type": "number" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "ModelNode(database: Optional[str], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , group: Optional[str] = None, docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Optional[str] = None, raw_code: str = '', language: str = 'sql', refs: List[dbt.contracts.graph.nodes.RefArgs] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , compiled_path: Optional[str] = None, compiled: bool = False, compiled_code: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = , _pre_injected_sql: Optional[str] = None, contract: dbt.contracts.graph.nodes.Contract = , access: dbt.node_types.AccessType = , constraints: List[dbt.contracts.graph.nodes.ModelLevelConstraint] = , version: Union[str, float, NoneType] = None, latest_version: Union[str, float, NoneType] = None)" }, "ModelLevelConstraint": { "type": "object", "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ "check", "not_null", "unique", "primary_key", "foreign_key", "custom" ] }, "name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "expression": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "warn_unenforced": { "type": "boolean", "default": true }, "warn_unsupported": { "type": "boolean", "default": true }, "columns": { "type": "array", "items": { "type": "string" }, "default": [] } }, "additionalProperties": false, "description": "ModelLevelConstraint(type: dbt.contracts.graph.nodes.ConstraintType, name: Optional[str] = None, expression: Optional[str] = None, warn_unenforced: bool = True, warn_unsupported: bool = True, columns: List[str] = )" }, "RPCNode": { "type": "object", "required": [ "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ], "properties": { "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "rpc" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "contract": { "enforced": false }, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "group": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1681270501.192949 }, "config_call_dict": { "type": "object", "default": {} }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "raw_code": { "type": "string", "default": "" }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "$ref": "#/definitions/RefArgs" }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "contract": { "$ref": "#/definitions/Contract", "default": { "enforced": false, "checksum": null } } }, "additionalProperties": false, "description": "RPCNode(database: Optional[str], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , group: Optional[str] = None, docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Optional[str] = None, raw_code: str = '', language: str = 'sql', refs: List[dbt.contracts.graph.nodes.RefArgs] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , compiled_path: Optional[str] = None, compiled: bool = False, compiled_code: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = , _pre_injected_sql: Optional[str] = None, contract: dbt.contracts.graph.nodes.Contract = )" }, "SqlNode": { "type": "object", "required": [ "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ], "properties": { "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "sql operation" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/NodeConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "contract": { "enforced": false }, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "group": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1681270501.1935291 }, "config_call_dict": { "type": "object", "default": {} }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "raw_code": { "type": "string", "default": "" }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "$ref": "#/definitions/RefArgs" }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "contract": { "$ref": "#/definitions/Contract", "default": { "enforced": false, "checksum": null } } }, "additionalProperties": false, "description": "SqlNode(database: Optional[str], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , group: Optional[str] = None, docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Optional[str] = None, raw_code: str = '', language: str = 'sql', refs: List[dbt.contracts.graph.nodes.RefArgs] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , compiled_path: Optional[str] = None, compiled: bool = False, compiled_code: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = , _pre_injected_sql: Optional[str] = None, contract: dbt.contracts.graph.nodes.Contract = )" }, "GenericTestNode": { "type": "object", "required": [ "test_metadata", "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ], "properties": { "test_metadata": { "$ref": "#/definitions/TestMetadata" }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "test" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/TestConfig", "default": { "enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0" } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "group": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1681270501.19419 }, "config_call_dict": { "type": "object", "default": {} }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "raw_code": { "type": "string", "default": "" }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "$ref": "#/definitions/RefArgs" }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "contract": { "$ref": "#/definitions/Contract", "default": { "enforced": false, "checksum": null } }, "column_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "file_key_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "attached_node": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "GenericTestNode(test_metadata: dbt.contracts.graph.nodes.TestMetadata, database: Optional[str], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.TestConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , group: Optional[str] = None, docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Optional[str] = None, raw_code: str = '', language: str = 'sql', refs: List[dbt.contracts.graph.nodes.RefArgs] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , compiled_path: Optional[str] = None, compiled: bool = False, compiled_code: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = , _pre_injected_sql: Optional[str] = None, contract: dbt.contracts.graph.nodes.Contract = , column_name: Optional[str] = None, file_key_name: Optional[str] = None, attached_node: Optional[str] = None)" }, "TestMetadata": { "type": "object", "required": [ "name" ], "properties": { "name": { "type": "string" }, "kwargs": { "type": "object", "default": {} }, "namespace": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "TestMetadata(name: str, kwargs: Dict[str, Any] = , namespace: Optional[str] = None)" }, "SnapshotNode": { "type": "object", "required": [ "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum", "config" ], "properties": { "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "snapshot" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/SnapshotConfig" }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "group": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1681270501.1952698 }, "config_call_dict": { "type": "object", "default": {} }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "raw_code": { "type": "string", "default": "" }, "language": { "type": "string", "default": "sql" }, "refs": { "type": "array", "items": { "$ref": "#/definitions/RefArgs" }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "compiled_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "compiled": { "type": "boolean", "default": false }, "compiled_code": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "extra_ctes_injected": { "type": "boolean", "default": false }, "extra_ctes": { "type": "array", "items": { "$ref": "#/definitions/InjectedCTE" }, "default": [] }, "contract": { "$ref": "#/definitions/Contract", "default": { "enforced": false, "checksum": null } } }, "additionalProperties": false, "description": "SnapshotNode(database: Optional[str], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.SnapshotConfig, _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , group: Optional[str] = None, docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Optional[str] = None, raw_code: str = '', language: str = 'sql', refs: List[dbt.contracts.graph.nodes.RefArgs] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , compiled_path: Optional[str] = None, compiled: bool = False, compiled_code: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = , _pre_injected_sql: Optional[str] = None, contract: dbt.contracts.graph.nodes.Contract = )" }, "SnapshotConfig": { "type": "object", "required": [], "properties": { "enabled": { "type": "boolean", "default": true }, "alias": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "tags": { "oneOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ], "default": [] }, "meta": { "type": "object", "default": {} }, "group": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "materialized": { "type": "string", "default": "snapshot" }, "incremental_strategy": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "persist_docs": { "type": "object", "default": {} }, "post-hook": { "type": "array", "items": { "$ref": "#/definitions/Hook" }, "default": [] }, "pre-hook": { "type": "array", "items": { "$ref": "#/definitions/Hook" }, "default": [] }, "quoting": { "type": "object", "default": {} }, "column_types": { "type": "object", "default": {} }, "full_refresh": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "unique_key": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "on_schema_change": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "grants": { "type": "object", "default": {} }, "packages": { "type": "array", "items": { "type": "string" }, "default": [] }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "contract": { "$ref": "#/definitions/ContractConfig", "default": { "enforced": false } }, "strategy": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "target_schema": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "target_database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "updated_at": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "check_cols": { "oneOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ] } }, "additionalProperties": true, "description": "SnapshotConfig(_extra: Dict[str, Any] = , enabled: bool = True, alias: Optional[str] = None, schema: Optional[str] = None, database: Optional[str] = None, tags: Union[List[str], str] = , meta: Dict[str, Any] = , group: Optional[str] = None, materialized: str = 'snapshot', incremental_strategy: Optional[str] = None, persist_docs: Dict[str, Any] = , post_hook: List[dbt.contracts.graph.model_config.Hook] = , pre_hook: List[dbt.contracts.graph.model_config.Hook] = , quoting: Dict[str, Any] = , column_types: Dict[str, Any] = , full_refresh: Optional[bool] = None, unique_key: Optional[str] = None, on_schema_change: Optional[str] = 'ignore', grants: Dict[str, Any] = , packages: List[str] = , docs: dbt.contracts.graph.unparsed.Docs = , contract: dbt.contracts.graph.model_config.ContractConfig = , strategy: Optional[str] = None, target_schema: Optional[str] = None, target_database: Optional[str] = None, updated_at: Optional[str] = None, check_cols: Union[str, List[str], NoneType] = None)" }, "SeedNode": { "type": "object", "required": [ "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "alias", "checksum" ], "properties": { "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "seed" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "alias": { "type": "string" }, "checksum": { "$ref": "#/definitions/FileHash" }, "config": { "$ref": "#/definitions/SeedConfig", "default": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "seed", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "contract": { "enforced": false }, "quote_columns": null, "post-hook": [], "pre-hook": [] } }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "group": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "build_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "deferred": { "type": "boolean", "default": false }, "unrendered_config": { "type": "object", "default": {} }, "created_at": { "type": "number", "default": 1681270501.1968079 }, "config_call_dict": { "type": "object", "default": {} }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "raw_code": { "type": "string", "default": "" }, "root_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "depends_on": { "$ref": "#/definitions/MacroDependsOn", "default": { "macros": [] } } }, "additionalProperties": false, "description": "SeedNode(database: Optional[str], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.SeedConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , group: Optional[str] = None, docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Optional[str] = None, raw_code: str = '', root_path: Optional[str] = None, depends_on: dbt.contracts.graph.nodes.MacroDependsOn = )" }, "SeedConfig": { "type": "object", "required": [], "properties": { "enabled": { "type": "boolean", "default": true }, "alias": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "tags": { "oneOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "string" } ], "default": [] }, "meta": { "type": "object", "default": {} }, "group": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "materialized": { "type": "string", "default": "seed" }, "incremental_strategy": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "persist_docs": { "type": "object", "default": {} }, "post-hook": { "type": "array", "items": { "$ref": "#/definitions/Hook" }, "default": [] }, "pre-hook": { "type": "array", "items": { "$ref": "#/definitions/Hook" }, "default": [] }, "quoting": { "type": "object", "default": {} }, "column_types": { "type": "object", "default": {} }, "full_refresh": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "unique_key": { "oneOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } }, { "type": "null" } ] }, "on_schema_change": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "default": "ignore" }, "grants": { "type": "object", "default": {} }, "packages": { "type": "array", "items": { "type": "string" }, "default": [] }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "contract": { "$ref": "#/definitions/ContractConfig", "default": { "enforced": false } }, "quote_columns": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] } }, "additionalProperties": true, "description": "SeedConfig(_extra: Dict[str, Any] = , enabled: bool = True, alias: Optional[str] = None, schema: Optional[str] = None, database: Optional[str] = None, tags: Union[List[str], str] = , meta: Dict[str, Any] = , group: Optional[str] = None, materialized: str = 'seed', incremental_strategy: Optional[str] = None, persist_docs: Dict[str, Any] = , post_hook: List[dbt.contracts.graph.model_config.Hook] = , pre_hook: List[dbt.contracts.graph.model_config.Hook] = , quoting: Dict[str, Any] = , column_types: Dict[str, Any] = , full_refresh: Optional[bool] = None, unique_key: Union[str, List[str], NoneType] = None, on_schema_change: Optional[str] = 'ignore', grants: Dict[str, Any] = , packages: List[str] = , docs: dbt.contracts.graph.unparsed.Docs = , contract: dbt.contracts.graph.model_config.ContractConfig = , quote_columns: Optional[bool] = None)" }, "MacroDependsOn": { "type": "object", "required": [], "properties": { "macros": { "type": "array", "items": { "type": "string" }, "default": [] } }, "additionalProperties": false, "description": "Used only in the Macro class" }, "SourceDefinition": { "type": "object", "required": [ "schema", "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "source_name", "source_description", "loader", "identifier" ], "properties": { "database": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "schema": { "type": "string" }, "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "source" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "source_name": { "type": "string" }, "source_description": { "type": "string" }, "loader": { "type": "string" }, "identifier": { "type": "string" }, "quoting": { "$ref": "#/definitions/Quoting", "default": { "database": null, "schema": null, "identifier": null, "column": null } }, "loaded_at_field": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "freshness": { "oneOf": [ { "$ref": "#/definitions/FreshnessThreshold" }, { "type": "null" } ] }, "external": { "oneOf": [ { "$ref": "#/definitions/ExternalTable" }, { "type": "null" } ] }, "description": { "type": "string", "default": "" }, "columns": { "type": "object", "additionalProperties": { "$ref": "#/definitions/ColumnInfo" }, "default": {} }, "meta": { "type": "object", "default": {} }, "source_meta": { "type": "object", "default": {} }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "config": { "$ref": "#/definitions/SourceConfig", "default": { "enabled": true } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "unrendered_config": { "type": "object", "default": {} }, "relation_name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "created_at": { "type": "number", "default": 1681270501.197819 } }, "additionalProperties": false, "description": "SourceDefinition(database: Optional[str], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], source_name: str, source_description: str, loader: str, identifier: str, _event_status: Dict[str, Any] = , quoting: dbt.contracts.graph.unparsed.Quoting = , loaded_at_field: Optional[str] = None, freshness: Optional[dbt.contracts.graph.unparsed.FreshnessThreshold] = None, external: Optional[dbt.contracts.graph.unparsed.ExternalTable] = None, description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , source_meta: Dict[str, Any] = , tags: List[str] = , config: dbt.contracts.graph.model_config.SourceConfig = , patch_path: Optional[str] = None, unrendered_config: Dict[str, Any] = , relation_name: Optional[str] = None, created_at: float = )" }, "Quoting": { "type": "object", "required": [], "properties": { "database": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "schema": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "identifier": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] }, "column": { "oneOf": [ { "type": "boolean" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "Quoting(database: Optional[bool] = None, schema: Optional[bool] = None, identifier: Optional[bool] = None, column: Optional[bool] = None)" }, "FreshnessThreshold": { "type": "object", "required": [], "properties": { "warn_after": { "oneOf": [ { "$ref": "#/definitions/Time" }, { "type": "null" } ], "default": { "count": null, "period": null } }, "error_after": { "oneOf": [ { "$ref": "#/definitions/Time" }, { "type": "null" } ], "default": { "count": null, "period": null } }, "filter": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "FreshnessThreshold(warn_after: Optional[dbt.contracts.graph.unparsed.Time] = , error_after: Optional[dbt.contracts.graph.unparsed.Time] = , filter: Optional[str] = None)" }, "FreshnessMetadata": { "type": "object", "required": [], "properties": { "dbt_schema_version": { "type": "string", "default": "https://schemas.getdbt.com/dbt/sources/v3.json" }, "dbt_version": { "type": "string", "default": "1.5.0b5" }, "generated_at": { "type": "string", "format": "date-time", "default": "2023-04-12T03:35:01.185979Z" }, "invocation_id": { "oneOf": [ { "type": "string" }, { "type": "null" } ], "default": "8aa1596d-f52f-40bc-ad4b-f5e48fc7e6c2" }, "env": { "type": "object", "additionalProperties": { "type": "string" }, "default": {} } }, "additionalProperties": false, "description": "FreshnessMetadata(dbt_schema_version: str = , dbt_version: str = '1.5.0b5', generated_at: datetime.datetime = , invocation_id: Optional[str] = , env: Dict[str, str] = )" }, "SourceFreshnessRuntimeError": { "type": "object", "required": [ "unique_id", "status" ], "properties": { "unique_id": { "type": "string" }, "error": { "oneOf": [ { "type": "string" }, { "type": "integer" }, { "type": "null" } ] }, "status": { "type": "string", "enum": [ "runtime error" ] } }, "additionalProperties": false, "description": "SourceFreshnessRuntimeError(unique_id: str, error: Union[str, int, NoneType], status: dbt.contracts.results.FreshnessErrorEnum)" }, "SourceFreshnessOutput": { "type": "object", "required": [ "unique_id", "max_loaded_at", "snapshotted_at", "max_loaded_at_time_ago_in_s", "status", "criteria", "adapter_response", "timing", "thread_id", "execution_time" ], "properties": { "unique_id": { "type": "string" }, "max_loaded_at": { "type": "string", "format": "date-time" }, "snapshotted_at": { "type": "string", "format": "date-time" }, "max_loaded_at_time_ago_in_s": { "type": "number" }, "status": { "type": "string", "enum": [ "pass", "warn", "error", "runtime error" ] }, "criteria": { "$ref": "#/definitions/FreshnessThreshold" }, "adapter_response": { "type": "object" }, "timing": { "type": "array", "items": { "$ref": "#/definitions/TimingInfo" } }, "thread_id": { "type": "string" }, "execution_time": { "type": "number" } }, "additionalProperties": false, "description": "SourceFreshnessOutput(unique_id: str, max_loaded_at: datetime.datetime, snapshotted_at: datetime.datetime, max_loaded_at_time_ago_in_s: float, status: dbt.contracts.results.FreshnessStatus, criteria: dbt.contracts.graph.unparsed.FreshnessThreshold, adapter_response: Dict[str, Any], timing: List[dbt.contracts.results.TimingInfo], thread_id: str, execution_time: float)" }, "Time": { "type": "object", "required": [], "properties": { "count": { "oneOf": [ { "type": "integer" }, { "type": "null" } ] }, "period": { "oneOf": [ { "type": "string", "enum": [ "minute", "hour", "day" ] }, { "type": "null" } ] } }, "additionalProperties": false, "description": "Time(count: Optional[int] = None, period: Optional[dbt.contracts.graph.unparsed.TimePeriod] = None)" }, "TimingInfo": { "type": "object", "required": [ "name" ], "properties": { "name": { "type": "string" }, "started_at": { "oneOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ] }, "completed_at": { "oneOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "TimingInfo(name: str, started_at: Optional[datetime.datetime] = None, completed_at: Optional[datetime.datetime] = None)" }, "ExternalTable": { "type": "object", "required": [], "properties": { "location": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "file_format": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "row_format": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "tbl_properties": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "partitions": { "oneOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "array", "items": { "$ref": "#/definitions/ExternalPartition" } }, { "type": "null" } ] } }, "additionalProperties": true, "description": "ExternalTable(_extra: Dict[str, Any] = , location: Optional[str] = None, file_format: Optional[str] = None, row_format: Optional[str] = None, tbl_properties: Optional[str] = None, partitions: Union[List[str], List[dbt.contracts.graph.unparsed.ExternalPartition], NoneType] = None)" }, "ExternalPartition": { "type": "object", "required": [], "properties": { "name": { "type": "string", "default": "" }, "description": { "type": "string", "default": "" }, "data_type": { "type": "string", "default": "" }, "meta": { "type": "object", "default": {} } }, "additionalProperties": true, "description": "ExternalPartition(_extra: Dict[str, Any] = , name: str = '', description: str = '', data_type: str = '', meta: Dict[str, Any] = )" }, "SourceConfig": { "type": "object", "required": [], "properties": { "enabled": { "type": "boolean", "default": true } }, "additionalProperties": true, "description": "SourceConfig(_extra: Dict[str, Any] = , enabled: bool = True)" }, "Macro": { "type": "object", "required": [ "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "macro_sql" ], "properties": { "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "macro" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "macro_sql": { "type": "string" }, "depends_on": { "$ref": "#/definitions/MacroDependsOn", "default": { "macros": [] } }, "description": { "type": "string", "default": "" }, "meta": { "type": "object", "default": {} }, "docs": { "$ref": "#/definitions/Docs", "default": { "show": true, "node_color": null } }, "patch_path": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "arguments": { "type": "array", "items": { "$ref": "#/definitions/MacroArgument" }, "default": [] }, "created_at": { "type": "number", "default": 1681270501.198105 }, "supported_languages": { "oneOf": [ { "type": "array", "items": { "type": "string", "enum": [ "python", "sql" ] } }, { "type": "null" } ] } }, "additionalProperties": false, "description": "Macro(name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, macro_sql: str, depends_on: dbt.contracts.graph.nodes.MacroDependsOn = , description: str = '', meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, arguments: List[dbt.contracts.graph.unparsed.MacroArgument] = , created_at: float = , supported_languages: Optional[List[dbt.node_types.ModelLanguage]] = None)" }, "MacroArgument": { "type": "object", "required": [ "name" ], "properties": { "name": { "type": "string" }, "type": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "description": { "type": "string", "default": "" } }, "additionalProperties": false, "description": "MacroArgument(name: str, type: Optional[str] = None, description: str = '')" }, "Documentation": { "type": "object", "required": [ "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "block_contents" ], "properties": { "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "doc" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "block_contents": { "type": "string" } }, "additionalProperties": false, "description": "Documentation(name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, block_contents: str)" }, "Exposure": { "type": "object", "required": [ "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "type", "owner" ], "properties": { "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "exposure" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "type": { "type": "string", "enum": [ "dashboard", "notebook", "analysis", "ml", "application" ] }, "owner": { "$ref": "#/definitions/Owner" }, "description": { "type": "string", "default": "" }, "label": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "maturity": { "oneOf": [ { "type": "string", "enum": [ "low", "medium", "high" ] }, { "type": "null" } ] }, "meta": { "type": "object", "default": {} }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "config": { "$ref": "#/definitions/ExposureConfig", "default": { "enabled": true } }, "unrendered_config": { "type": "object", "default": {} }, "url": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "refs": { "type": "array", "items": { "$ref": "#/definitions/RefArgs" }, "default": [] }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "created_at": { "type": "number", "default": 1681270501.198782 } }, "additionalProperties": false, "description": "Exposure(name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], type: dbt.contracts.graph.unparsed.ExposureType, owner: dbt.contracts.graph.unparsed.Owner, description: str = '', label: Optional[str] = None, maturity: Optional[dbt.contracts.graph.unparsed.MaturityType] = None, meta: Dict[str, Any] = , tags: List[str] = , config: dbt.contracts.graph.model_config.ExposureConfig = , unrendered_config: Dict[str, Any] = , url: Optional[str] = None, depends_on: dbt.contracts.graph.nodes.DependsOn = , refs: List[dbt.contracts.graph.nodes.RefArgs] = , sources: List[List[str]] = , metrics: List[List[str]] = , created_at: float = )" }, "Owner": { "type": "object", "required": [], "properties": { "email": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "name": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": true, "description": "Owner(_extra: Dict[str, Any] = , email: Optional[str] = None, name: Optional[str] = None)" }, "ExposureConfig": { "type": "object", "required": [], "properties": { "enabled": { "type": "boolean", "default": true } }, "additionalProperties": true, "description": "ExposureConfig(_extra: Dict[str, Any] = , enabled: bool = True)" }, "Metric": { "type": "object", "required": [ "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "fqn", "description", "label", "calculation_method", "expression", "filters", "time_grains", "dimensions" ], "properties": { "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "metric" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "fqn": { "type": "array", "items": { "type": "string" } }, "description": { "type": "string" }, "label": { "type": "string" }, "calculation_method": { "type": "string" }, "expression": { "type": "string" }, "filters": { "type": "array", "items": { "$ref": "#/definitions/MetricFilter" } }, "time_grains": { "type": "array", "items": { "type": "string" } }, "dimensions": { "type": "array", "items": { "type": "string" } }, "timestamp": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "window": { "oneOf": [ { "$ref": "#/definitions/MetricTime" }, { "type": "null" } ] }, "model": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "model_unique_id": { "oneOf": [ { "type": "string" }, { "type": "null" } ] }, "meta": { "type": "object", "default": {} }, "tags": { "type": "array", "items": { "type": "string" }, "default": [] }, "config": { "$ref": "#/definitions/MetricConfig", "default": { "enabled": true, "group": null } }, "unrendered_config": { "type": "object", "default": {} }, "sources": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "depends_on": { "$ref": "#/definitions/DependsOn", "default": { "macros": [], "nodes": [] } }, "refs": { "type": "array", "items": { "$ref": "#/definitions/RefArgs" }, "default": [] }, "metrics": { "type": "array", "items": { "type": "array", "items": { "type": "string" } }, "default": [] }, "created_at": { "type": "number", "default": 1681270501.199492 }, "group": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "description": "Metric(name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], description: str, label: str, calculation_method: str, expression: str, filters: List[dbt.contracts.graph.unparsed.MetricFilter], time_grains: List[str], dimensions: List[str], timestamp: Optional[str] = None, window: Optional[dbt.contracts.graph.unparsed.MetricTime] = None, model: Optional[str] = None, model_unique_id: Optional[str] = None, meta: Dict[str, Any] = , tags: List[str] = , config: dbt.contracts.graph.model_config.MetricConfig = , unrendered_config: Dict[str, Any] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , refs: List[dbt.contracts.graph.nodes.RefArgs] = , metrics: List[List[str]] = , created_at: float = , group: Optional[str] = None)" }, "MetricFilter": { "type": "object", "required": [ "field", "operator", "value" ], "properties": { "field": { "type": "string" }, "operator": { "type": "string" }, "value": { "type": "string" } }, "additionalProperties": false, "description": "MetricFilter(field: str, operator: str, value: str)" }, "MetricTime": { "type": "object", "required": [], "properties": { "count": { "oneOf": [ { "type": "integer" }, { "type": "null" } ] }, "period": { "oneOf": [ { "type": "string", "enum": [ "day", "week", "month", "year" ] }, { "type": "null" } ] } }, "additionalProperties": false, "description": "MetricTime(count: Optional[int] = None, period: Optional[dbt.contracts.graph.unparsed.MetricTimePeriod] = None)" }, "MetricConfig": { "type": "object", "required": [], "properties": { "enabled": { "type": "boolean", "default": true }, "group": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": true, "description": "MetricConfig(_extra: Dict[str, Any] = , enabled: bool = True, group: Optional[str] = None)" }, "Group": { "type": "object", "required": [ "name", "resource_type", "package_name", "path", "original_file_path", "unique_id", "owner" ], "properties": { "name": { "type": "string" }, "resource_type": { "type": "string", "enum": [ "group" ] }, "package_name": { "type": "string" }, "path": { "type": "string" }, "original_file_path": { "type": "string" }, "unique_id": { "type": "string" }, "owner": { "$ref": "#/definitions/Owner" } }, "additionalProperties": false, "description": "Group(name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, owner: dbt.contracts.graph.unparsed.Owner)" } }, "$schema": "http://json-schema.org/draft-07/schema#", "$id": "https://schemas.getdbt.com/dbt/manifest/v9.json" } ================================================ FILE: schemas/dbt/run-results/v4.json ================================================ { "$ref": "#/$defs/RunResultsArtifact", "$defs": { "BaseArtifactMetadata": { "type": "object", "title": "BaseArtifactMetadata", "properties": { "dbt_schema_version": { "type": "string" }, "dbt_version": { "type": "string", "default": "1.7.0b1" }, "generated_at": { "type": "string" }, "invocation_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "env": { "type": "object", "additionalProperties": { "type": "string" }, "propertyNames": { "type": "string" } } }, "additionalProperties": false, "required": [ "dbt_schema_version" ] }, "TimingInfo": { "type": "object", "title": "TimingInfo", "properties": { "name": { "type": "string" }, "started_at": { "anyOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ], "default": null }, "completed_at": { "anyOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] }, "RunResultOutput": { "type": "object", "title": "RunResultOutput", "properties": { "status": { "anyOf": [ { "enum": [ "success", "error", "skipped" ] }, { "enum": [ "pass", "error", "fail", "warn", "skipped" ] }, { "enum": [ "pass", "warn", "error", "runtime error" ] } ] }, "timing": { "type": "array", "items": { "$ref": "#/$defs/TimingInfo" } }, "thread_id": { "type": "string" }, "execution_time": { "type": "number" }, "adapter_response": { "type": "object", "propertyNames": { "type": "string" } }, "message": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "failures": { "anyOf": [ { "type": "integer" }, { "type": "null" } ] }, "unique_id": { "type": "string" } }, "additionalProperties": false, "required": [ "status", "timing", "thread_id", "execution_time", "adapter_response", "message", "failures", "unique_id" ] }, "RunResultsArtifact": { "type": "object", "title": "RunResultsArtifact", "properties": { "metadata": { "$ref": "#/$defs/BaseArtifactMetadata" }, "results": { "type": "array", "items": { "$ref": "#/$defs/RunResultOutput" } }, "elapsed_time": { "type": "number" }, "args": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": false, "required": [ "metadata", "results", "elapsed_time" ] } }, "$id": "https://schemas.getdbt.com/dbt/run-results/v4.json" } ================================================ FILE: schemas/dbt/run-results/v5.json ================================================ { "$ref": "#/$defs/RunResultsArtifact", "$defs": { "BaseArtifactMetadata": { "type": "object", "title": "BaseArtifactMetadata", "properties": { "dbt_schema_version": { "type": "string" }, "dbt_version": { "type": "string", "default": "1.7.0b1" }, "generated_at": { "type": "string" }, "invocation_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "env": { "type": "object", "additionalProperties": { "type": "string" }, "propertyNames": { "type": "string" } } }, "additionalProperties": false, "required": [ "dbt_schema_version" ] }, "TimingInfo": { "type": "object", "title": "TimingInfo", "properties": { "name": { "type": "string" }, "started_at": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "completed_at": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] }, "RunResultOutput": { "type": "object", "title": "RunResultOutput", "properties": { "status": { "anyOf": [ { "enum": [ "success", "error", "skipped" ] }, { "enum": [ "pass", "error", "fail", "warn", "skipped" ] }, { "enum": [ "pass", "warn", "error", "runtime error" ] } ] }, "timing": { "type": "array", "items": { "$ref": "#/$defs/TimingInfo" } }, "thread_id": { "type": "string" }, "execution_time": { "type": "number" }, "adapter_response": { "type": "object", "propertyNames": { "type": "string" } }, "message": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "failures": { "anyOf": [ { "type": "integer" }, { "type": "null" } ] }, "unique_id": { "type": "string" }, "compiled": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "compiled_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ] } }, "additionalProperties": false, "required": [ "status", "timing", "thread_id", "execution_time", "adapter_response", "message", "failures", "unique_id", "compiled", "compiled_code", "relation_name" ] }, "RunResultsArtifact": { "type": "object", "title": "RunResultsArtifact", "properties": { "metadata": { "$ref": "#/$defs/BaseArtifactMetadata" }, "results": { "type": "array", "items": { "$ref": "#/$defs/RunResultOutput" } }, "elapsed_time": { "type": "number" }, "args": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": false, "required": [ "metadata", "results", "elapsed_time" ] } }, "$id": "https://schemas.getdbt.com/dbt/run-results/v5.json" } ================================================ FILE: schemas/dbt/run-results/v6.json ================================================ { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "title": "RunResultsArtifact", "properties": { "metadata": { "type": "object", "title": "BaseArtifactMetadata", "properties": { "dbt_schema_version": { "type": "string" }, "dbt_version": { "type": "string", "default": "1.12.0a1" }, "generated_at": { "type": "string" }, "invocation_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "invocation_started_at": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "env": { "type": "object", "additionalProperties": { "type": "string" }, "propertyNames": { "type": "string" } } }, "additionalProperties": false, "required": [ "dbt_schema_version" ] }, "results": { "type": "array", "items": { "type": "object", "title": "RunResultOutput", "properties": { "status": { "anyOf": [ { "enum": [ "success", "error", "skipped", "partial success", "no-op" ] }, { "enum": [ "pass", "error", "fail", "warn", "skipped" ] }, { "enum": [ "pass", "warn", "error", "runtime error" ] } ] }, "timing": { "type": "array", "items": { "type": "object", "title": "TimingInfo", "properties": { "name": { "type": "string" }, "started_at": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "completed_at": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] } }, "thread_id": { "type": "string" }, "execution_time": { "type": "number" }, "adapter_response": { "type": "object", "propertyNames": { "type": "string" } }, "message": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "failures": { "anyOf": [ { "type": "integer" }, { "type": "null" } ] }, "unique_id": { "type": "string" }, "compiled": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "compiled_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "relation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "batch_results": { "anyOf": [ { "type": "object", "title": "BatchResults", "properties": { "successful": { "type": "array", "items": { "type": "array", "prefixItems": [ { "type": "string" }, { "type": "string" } ], "maxItems": 2, "minItems": 2 } }, "failed": { "type": "array", "items": { "type": "array", "prefixItems": [ { "type": "string" }, { "type": "string" } ], "maxItems": 2, "minItems": 2 } } }, "additionalProperties": false }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "status", "timing", "thread_id", "execution_time", "adapter_response", "message", "failures", "unique_id", "compiled", "compiled_code", "relation_name" ] } }, "elapsed_time": { "type": "number" }, "args": { "type": "object", "propertyNames": { "type": "string" } } }, "additionalProperties": false, "required": [ "metadata", "results", "elapsed_time" ], "$id": "https://schemas.getdbt.com/dbt/run-results/v6.json" } ================================================ FILE: schemas/dbt/sources/v3.json ================================================ { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "title": "FreshnessExecutionResultArtifact", "properties": { "metadata": { "type": "object", "title": "FreshnessMetadata", "properties": { "dbt_schema_version": { "type": "string" }, "dbt_version": { "type": "string", "default": "1.12.0a1" }, "generated_at": { "type": "string" }, "invocation_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "invocation_started_at": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "env": { "type": "object", "additionalProperties": { "type": "string" }, "propertyNames": { "type": "string" } } }, "additionalProperties": false }, "results": { "type": "array", "items": { "anyOf": [ { "type": "object", "title": "SourceFreshnessRuntimeError", "properties": { "unique_id": { "type": "string" }, "error": { "anyOf": [ { "type": "string" }, { "type": "integer" }, { "type": "null" } ] }, "status": { "enum": [ "runtime error" ] } }, "additionalProperties": false, "required": [ "unique_id", "error", "status" ] }, { "type": "object", "title": "SourceFreshnessOutput", "properties": { "unique_id": { "type": "string" }, "max_loaded_at": { "type": "string" }, "snapshotted_at": { "type": "string" }, "max_loaded_at_time_ago_in_s": { "type": "number" }, "status": { "enum": [ "pass", "warn", "error", "runtime error" ] }, "criteria": { "type": "object", "title": "FreshnessThreshold", "properties": { "warn_after": { "anyOf": [ { "type": "object", "title": "Time", "properties": { "count": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null }, "period": { "anyOf": [ { "enum": [ "minute", "hour", "day" ] }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, { "type": "null" } ] }, "error_after": { "anyOf": [ { "type": "object", "title": "Time", "properties": { "count": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null }, "period": { "anyOf": [ { "enum": [ "minute", "hour", "day" ] }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, { "type": "null" } ] }, "filter": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false }, "adapter_response": { "type": "object", "propertyNames": { "type": "string" } }, "timing": { "type": "array", "items": { "type": "object", "title": "TimingInfo", "properties": { "name": { "type": "string" }, "started_at": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null }, "completed_at": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null } }, "additionalProperties": false, "required": [ "name" ] } }, "thread_id": { "type": "string" }, "execution_time": { "type": "number" } }, "additionalProperties": false, "required": [ "unique_id", "max_loaded_at", "snapshotted_at", "max_loaded_at_time_ago_in_s", "status", "criteria", "adapter_response", "timing", "thread_id", "execution_time" ] } ] } }, "elapsed_time": { "type": "number" } }, "additionalProperties": false, "required": [ "metadata", "results", "elapsed_time" ], "$id": "https://schemas.getdbt.com/dbt/sources/v3.json" } ================================================ FILE: scripts/check_libyaml.py ================================================ #!/usr/bin/env python try: from yaml import CDumper as Dumper from yaml import CLoader as Loader # noqa: F401 from yaml import CSafeLoader as SafeLoader except ImportError: from yaml import Dumper, Loader, SafeLoader # noqa: F401 if Loader.__name__ == "CLoader": print("libyaml is working") elif Loader.__name__ == "Loader": print("libyaml is not working") print("Check the python executable and pyyaml for libyaml support") ================================================ FILE: scripts/collect-artifact-schema.py ================================================ #!/usr/bin/env python import json from argparse import ArgumentParser from dataclasses import dataclass from pathlib import Path from typing import Any, Dict, Type from dbt.artifacts.schemas.base import VersionedSchema from dbt.artifacts.schemas.catalog import CatalogArtifact from dbt.artifacts.schemas.freshness import FreshnessExecutionResultArtifact from dbt.artifacts.schemas.run import RunResultsArtifact from dbt.contracts.graph.manifest import WritableManifest from dbt_common.clients.system import write_file @dataclass class ArtifactInfo: path: str name: str json_schema: Dict[str, Any] @classmethod def from_artifact_cls( cls, artifact_cls: Type[VersionedSchema], ) -> "ArtifactInfo": return cls( path=artifact_cls.dbt_schema_version.path, name=artifact_cls.dbt_schema_version.name, json_schema=artifact_cls.json_schema(), ) def write_schema(self, dest_dir: Path): write_file(str(dest_dir / self.path), json.dumps(self.json_schema, indent=2)) @dataclass class Arguments: artifact: str path: Path @classmethod def parse(cls) -> "Arguments": parser = ArgumentParser(prog="Collect and write dbt arfifact schema") parser.add_argument( "--path", type=Path, help="The dir to write artifact schema", ) parser.add_argument( "--artifact", type=str, choices=["manifest", "sources", "run-results", "catalog"], help="The name of the artifact to update", ) parsed = parser.parse_args() return cls(artifact=parsed.artifact, path=parsed.path) def collect_artifact_schema(args: Arguments): artifacts = [ FreshnessExecutionResultArtifact, RunResultsArtifact, CatalogArtifact, # WritableManifest introduces new definitions in hologram which are likely # getting persisted across invocations of json_schema and making their # way to other written artifacts - so write it last as a short-term fix. # https://github.com/dbt-labs/dbt-core/issues/7604 WritableManifest, ] filtered_artifacts = filter( lambda a: a.dbt_schema_version.name == args.artifact or args.artifact is None, artifacts ) artifact_infos = [] for artifact_cls in filtered_artifacts: artifact_infos.append(ArtifactInfo.from_artifact_cls(artifact_cls)) if args and args.path is not None: for artifact_info in artifact_infos: dest_dir = args.path.resolve() artifact_info.write_schema(dest_dir) else: artifacts_dict = { artifact_info.name: artifact_info.json_schema for artifact_info in artifact_infos } print(json.dumps(artifacts_dict)) def main(): parsed = Arguments.parse() collect_artifact_schema(parsed) if __name__ == "__main__": main() ================================================ FILE: scripts/collect-dbt-contexts.py ================================================ #!/usr/bin/env python import inspect import json from dataclasses import dataclass from typing import Any, Dict, Iterable, List, Optional, Union from dbt.context.base import BaseContext from dbt.context.providers import MacroContext, ModelContext from dbt.context.target import TargetContext from dbt_common.dataclass_schema import dbtClassMixin CONTEXTS_MAP = { "base": BaseContext, "target": TargetContext, "model": ModelContext, "macro": MacroContext, } @dataclass class ContextValue(dbtClassMixin): name: str value: str # a type description doc: Optional[str] @dataclass class MethodArgument(dbtClassMixin): name: str value: str # a type description @dataclass class ContextMethod(dbtClassMixin): name: str args: List[MethodArgument] result: str # a type description doc: Optional[str] @dataclass class Unknown(dbtClassMixin): name: str value: str doc: Optional[str] ContextMember = Union[ContextValue, ContextMethod, Unknown] def _get_args(func: inspect.Signature) -> Iterable[MethodArgument]: found_first = False for argname, arg in func.parameters.items(): if found_first is False and argname in {"self", "cls"}: continue if found_first is False: found_first = True yield MethodArgument( name=argname, value=inspect.formatannotation(arg.annotation), ) def collect(cls): values = [] for name, v in cls._context_members_.items(): attrname = cls._context_attrs_[name] attrdef = getattr(cls, attrname) doc = getattr(attrdef, "__doc__") if inspect.isfunction(attrdef): sig = inspect.signature(attrdef) result = inspect.formatannotation(sig.return_annotation) sig_good_part = ContextMethod( name=name, args=list(_get_args(sig)), result=result, doc=doc, ) elif isinstance(attrdef, property): sig = inspect.signature(attrdef.fget) sig_txt = inspect.formatannotation(sig.return_annotation) sig_good_part = ContextValue(name=name, value=sig_txt, doc=doc) else: sig_good_part = Unknown(name=name, value=repr(attrdef), doc=doc) values.append(sig_good_part) return values @dataclass class ContextCatalog(dbtClassMixin): base: List[ContextMember] target: List[ContextMember] model: List[ContextMember] macro: List[ContextMember] schema: Dict[str, Any] def main(): catalog = ContextCatalog( base=collect(BaseContext), target=collect(TargetContext), model=collect(ModelContext), macro=collect(MacroContext), schema=ContextCatalog.json_schema(), ) print(json.dumps(catalog.to_dict())) if __name__ == "__main__": main() ================================================ FILE: scripts/env-setup.sh ================================================ #!/bin/bash # Set environment variables required for integration tests # This is used in the release workflow to set the environment variables for the integration tests echo "DBT_INVOCATION_ENV=github-actions" >> $GITHUB_ENV echo "DBT_TEST_USER_1=dbt_test_user_1" >> $GITHUB_ENV echo "DBT_TEST_USER_2=dbt_test_user_2" >> $GITHUB_ENV echo "DBT_TEST_USER_3=dbt_test_user_3" >> $GITHUB_ENV ================================================ FILE: scripts/migrate-adapters.py ================================================ import argparse import ast from collections import namedtuple from pathlib import Path Import = namedtuple("Import", ["module", "name", "alias"]) def get_imports(path): with open(path) as fh: root = ast.parse(fh.read(), path) for node in ast.iter_child_nodes(root): if isinstance(node, ast.Import): module = [] elif isinstance(node, ast.ImportFrom): module = node.module.split(".") else: continue for n in node.names: yield Import(module, n.name.split("."), n.asname) parser = argparse.ArgumentParser("migrate_adapters") parser.add_argument("path", help="The path to run the migration tool over.", type=str) args = parser.parse_args() path = Path(args.path) pathlist = path.rglob("*.py") total_dbt_imports = 0 invalid_dbt_imports = 0 path_to_invalid_imports = {} for path in pathlist: path_to_invalid_imports[path] = [] for imported_module in get_imports(str(path)): if imported_module.module and imported_module.module[0] == "dbt": total_dbt_imports += 1 if imported_module.module[1] not in ("common", "adapters"): invalid_dbt_imports += 1 path_to_invalid_imports[path].append( f"{'.'.join(imported_module.module)}::{imported_module.name[0]}" ) migrated_imports = total_dbt_imports - invalid_dbt_imports migrated_imports_progress = round((migrated_imports / total_dbt_imports) * 100, 2) for path, invalid_imports in path_to_invalid_imports.items(): if invalid_imports: print() print(f"\033[92m{path}:\033[0m") for invalid_import in invalid_imports: print(f" - {invalid_import}") print() print( f"migration progress: {migrated_imports_progress}% of dbt imports are valid (from adapters or common)" ) print(f"remaining core imports: {invalid_dbt_imports}") ================================================ FILE: scripts/pre-commit-hooks/no_versioned_artifact_resource_imports.py ================================================ import os import re import sys from dbt_common.ui import red def normalize(path: str) -> str: """On windows, neither is enough on its own: >>> normcase('C:\\documents/ALL CAPS/subdir\\..') 'c:\\documents\\all caps\\subdir\\..' >>> normpath('C:\\documents/ALL CAPS/subdir\\..') 'C:\\documents\\ALL CAPS' >>> normpath(normcase('C:\\documents/ALL CAPS/subdir\\..')) 'c:\\documents\\all caps' """ return os.path.normcase(os.path.normpath(path)) def has_bad_artifact_resource_imports(filepath: str) -> bool: """ Checks for improper artifact resource imports outside of the artifacts module. Returns: True if a file imports from a versioned artifacts module False otherwise """ if not filepath.endswith(".py"): # skip non-python files return False elif normalize("core/dbt/artifacts") in filepath: # skip files in the artifacts module return False with open(filepath, "r") as f: lines = f.readlines() has_bad_imports = False for line_number, line in enumerate(lines): line_without_whitespace = line.strip() # get rid of leading and trailing whitespace # we only care about import/from statements if line_without_whitespace.startswith("from ") or line_without_whitespace.startswith( "import " ): import_path = line_without_whitespace.split(" ")[1] if re.match(r"^dbt\.artifacts\.resources\.v\d+", import_path): has_bad_imports = True print( f"{filepath}:{line_number}: [{red('ERROR')}] `{import_path}` is an import from a versioned resource. Instead import `from dbt.artifacts.resource` directly." ) return has_bad_imports def main(): all_passed = True for filepath in sys.argv[1:]: if has_bad_artifact_resource_imports(filepath): all_passed = False return 0 if all_passed else 1 if __name__ == "__main__": sys.exit(main()) ================================================ FILE: scripts/setup_db.sh ================================================ #!/bin/bash set -x if [ "${SKIP_HOMEBREW:-false}" = "false" ]; then brew install postgresql@16 brew link postgresql@16 --force export PATH="/opt/homebrew/opt/postgresql@16/bin:$PATH" # Start PostgreSQL using the full command instead of brew services pg_ctl -D /opt/homebrew/var/postgresql@16 start echo "Check PostgreSQL service is running" i=10 COMMAND='pg_isready' while [ $i -gt -1 ]; do if [ $i == 0 ]; then echo "PostgreSQL service not ready, all attempts exhausted" exit 1 fi echo "Check PostgreSQL service status" eval $COMMAND && break echo "PostgreSQL service not ready, wait 10 more sec, attempts left: $i" sleep 10 ((i--)) done createuser -s postgres env | grep '^PG' fi # If you want to run this script for your own postgresql (run with # docker-compose) it will look like this: # PGHOST=127.0.0.1 PGUSER=root PGPASSWORD=password PGDATABASE=postgres \ PGUSER="${PGUSER:-postgres}" export PGUSER PGPORT="${PGPORT:-5432}" export PGPORT PGHOST="${PGHOST:-localhost}" export PGHOST PGDATABASE="${PGDATABASE:-postgres}" export PGDATABASE : "${PGPASSWORD:=password}" export PGPASSWORD # Normalize localhost to IPv4 to avoid IPv6 (::1) surprises if [ "${PGHOST}" = "localhost" ]; then PGHOST="127.0.0.1" export PGHOST fi for i in {1..10}; do if pg_isready -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" ; then break fi echo "Waiting for postgres to be ready..." sleep 2; done; createdb dbt psql -c "SELECT version();" psql -c "CREATE ROLE root WITH PASSWORD 'password';" psql -c "ALTER ROLE root WITH LOGIN;" psql -c "GRANT CREATE, CONNECT ON DATABASE dbt TO root WITH GRANT OPTION;" psql -c "CREATE ROLE noaccess WITH PASSWORD 'password' NOSUPERUSER;" psql -c "ALTER ROLE noaccess WITH LOGIN;" psql -c "GRANT CONNECT ON DATABASE dbt TO noaccess;" psql -c "CREATE ROLE dbt_test_user_1;" psql -c "CREATE ROLE dbt_test_user_2;" psql -c "CREATE ROLE dbt_test_user_3;" psql -c 'CREATE DATABASE "dbtMixedCase";' psql -c 'GRANT CREATE, CONNECT ON DATABASE "dbtMixedCase" TO root WITH GRANT OPTION;' set +x ================================================ FILE: scripts/update_dev_packages.sh ================================================ #!/bin/bash -e set -e # this is used in dbt-common for CI repo=$1 ref=$2 target_req_file="core/hatch.toml" req_sed_pattern="s|${repo}.git@main|${repo}.git@${ref}|g" if [[ "$OSTYPE" == darwin* ]]; then # mac ships with a different version of sed that requires a delimiter arg sed -i "" "$req_sed_pattern" "$target_req_file" else sed -i "$req_sed_pattern" "$target_req_file" fi ================================================ FILE: tests/__init__.py ================================================ # tests directory ================================================ FILE: tests/conftest.py ================================================ # Import the fuctional fixtures as a plugin # Note: fixtures with session scope need to be local pytest_plugins = ["dbt.tests.fixtures.project"] ================================================ FILE: tests/data/__init__.py ================================================ # tests/data directory ================================================ FILE: tests/fixtures/__init__.py ================================================ # fixtures directory ================================================ FILE: tests/fixtures/dbt_integration_project.py ================================================ import pytest dbt_integration_project__my_macros_sql = """ {% macro do_something(foo, bar) %} select '{{ foo }}'::text as foo, '{{ bar }}'::text as bar {% endmacro %} """ dbt_integration_project__incremental_sql = """ -- TODO : add dist/sort keys {{ config( materialized = 'incremental', unique_key = 'id', ) }} select * from {{ this.schema }}.seed {% if is_incremental() %} where updated_at > (select max(updated_at) from {{ this }}) {% endif %} """ dbt_integration_project__schema_yml = """ version: 2 models: - name: table_model columns: - name: id data_tests: - unique """ dbt_integration_project__table_model_sql = """ -- TODO : add dist/sort keys {{ config( materialized = 'table', ) }} select * from {{ this.schema }}.seed """ dbt_integration_project__view_model_sql = """ {{ config( materialized = 'view', ) }} select * from {{ this.schema }}.seed """ dbt_integration_project__dbt_project_yml = """ name: dbt_integration_project version: '1.0' config-version: 2 model-paths: ["models"] # paths to models analysis-paths: ["analyses"] # path with analysis files which are compiled, but not run target-path: "target" # path for compiled code clean-targets: ["target"] # directories removed by the clean task test-paths: ["tests"] # where to store test results seed-paths: ["seeds"] # load CSVs from this directory with `dbt seed` macro-paths: ["macros"] # where to find macros profile: user models: dbt_integration_project: """ @pytest.fixture(scope="class") def dbt_integration_project(): return { "dbt_project.yml": dbt_integration_project__dbt_project_yml, "macros": {"my_macros.sql": dbt_integration_project__my_macros_sql}, "models": { "incremental.sql": dbt_integration_project__incremental_sql, "schema.yml": dbt_integration_project__schema_yml, "table_model.sql": dbt_integration_project__table_model_sql, "view_model.sql": dbt_integration_project__view_model_sql, }, } ================================================ FILE: tests/fixtures/jaffle_shop.py ================================================ import os import pytest from dbt.tests.util import read_file # models/customers.sql customers_sql = """ with customers as ( select * from {{ ref('stg_customers') }} ), orders as ( select * from {{ ref('stg_orders') }} ), payments as ( select * from {{ ref('stg_payments') }} ), customer_orders as ( select customer_id, min(order_date) as first_order, max(order_date) as most_recent_order, count(order_id) as number_of_orders from orders group by customer_id ), customer_payments as ( select orders.customer_id, sum(amount) as total_amount from payments left join orders on payments.order_id = orders.order_id group by orders.customer_id ), final as ( select customers.customer_id, customers.first_name, customers.last_name, customer_orders.first_order, customer_orders.most_recent_order, customer_orders.number_of_orders, customer_payments.total_amount as customer_lifetime_value from customers left join customer_orders on customers.customer_id = customer_orders.customer_id left join customer_payments on customers.customer_id = customer_payments.customer_id ) select * from final """ # models/docs.md docs_md = """ {% docs orders_status %} Orders can be one of the following statuses: | status | description | |----------------|------------------------------------------------------------------------------------------------------------------------| | placed | The order has been placed but has not yet left the warehouse | | shipped | The order has ben shipped to the customer and is currently in transit | | completed | The order has been received by the customer | | return_pending | The customer has indicated that they would like to return the order, but it has not yet been received at the warehouse | | returned | The order has been returned by the customer and received at the warehouse | {% enddocs %} """ # models/orders.sql orders_sql = """ {% set payment_methods = ['credit_card', 'coupon', 'bank_transfer', 'gift_card'] %} with orders as ( select * from {{ ref('stg_orders') }} ), payments as ( select * from {{ ref('stg_payments') }} ), order_payments as ( select order_id, {% for payment_method in payment_methods -%} sum(case when payment_method = '{{ payment_method }}' then amount else 0 end) as {{ payment_method }}_amount, {% endfor -%} sum(amount) as total_amount from payments group by order_id ), final as ( select orders.order_id, orders.customer_id, orders.order_date, orders.status, {% for payment_method in payment_methods -%} order_payments.{{ payment_method }}_amount, {% endfor -%} order_payments.total_amount as amount from orders left join order_payments on orders.order_id = order_payments.order_id ) select * from final """ # models/overview.md overview_md = """ {% docs __overview__ %} ## Data Documentation for Jaffle Shop `jaffle_shop` is a fictional ecommerce store. This [dbt](https://www.getdbt.com/) project is for testing out code. The source code can be found [here](https://github.com/clrcrl/jaffle_shop). {% enddocs %} """ # models/schema.yml schema_yml = """ version: 2 models: - name: customers description: This table has basic information about a customer, as well as some derived facts based on a customer's orders columns: - name: customer_id description: This is a unique identifier for a customer data_tests: - unique - not_null - name: first_name description: Customer's first name. PII. - name: last_name description: Customer's last name. PII. - name: first_order description: Date (UTC) of a customer's first order - name: most_recent_order description: Date (UTC) of a customer's most recent order - name: number_of_orders description: Count of the number of orders a customer has placed - name: total_order_amount description: Total value (AUD) of a customer's orders - name: orders description: This table has basic information about orders, as well as some derived facts based on payments columns: - name: order_id data_tests: - unique - not_null description: This is a unique identifier for an order - name: customer_id description: Foreign key to the customers table data_tests: - not_null - relationships: to: ref('customers') field: customer_id - name: order_date description: Date (UTC) that the order was placed - name: status description: '{{ doc("orders_status") }}' data_tests: - accepted_values: values: ['placed', 'shipped', 'completed', 'return_pending', 'returned'] - name: amount description: Total amount (AUD) of the order data_tests: - not_null - name: credit_card_amount description: Amount of the order (AUD) paid for by credit card data_tests: - not_null - name: coupon_amount description: Amount of the order (AUD) paid for by coupon data_tests: - not_null - name: bank_transfer_amount description: Amount of the order (AUD) paid for by bank transfer data_tests: - not_null - name: gift_card_amount description: Amount of the order (AUD) paid for by gift card data_tests: - not_null """ # models/staging/schema.yml staging_schema_yml = """ version: 2 models: - name: stg_customers columns: - name: customer_id data_tests: - unique - not_null - name: stg_orders columns: - name: order_id data_tests: - unique - not_null - name: status data_tests: - accepted_values: values: ['placed', 'shipped', 'completed', 'return_pending', 'returned'] - name: stg_payments columns: - name: payment_id data_tests: - unique - not_null - name: payment_method data_tests: - accepted_values: values: ['credit_card', 'coupon', 'bank_transfer', 'gift_card'] """ # models/staging/stg_customers.sql staging_stg_customers_sql = """ with source as ( {#- Normally we would select from the table here, but we are using seeds to load our data in this project #} select * from {{ ref('raw_customers') }} ), renamed as ( select id as customer_id, first_name, last_name from source ) select * from renamed """ # models/staging/stg_orders.sql staging_stg_orders_sql = """ with source as ( {#- Normally we would select from the table here, but we are using seeds to load our data in this project #} select * from {{ ref('raw_orders') }} ), renamed as ( select id as order_id, user_id as customer_id, order_date, status from source ) select * from renamed """ # models/staging/stg_payments.sql staging_stg_payments_sql = """ with source as ( {#- Normally we would select from the table here, but we are using seeds to load our data in this project #} select * from {{ ref('raw_payments') }} ), renamed as ( select id as payment_id, order_id, payment_method, -- `amount` is currently stored in cents, so we convert it to dollars amount / 100 as amount from source ) select * from renamed """ class JaffleShopProject: @pytest.fixture(scope="class") def models(self): return { "customers.sql": customers_sql, "docs.md": docs_md, "orders.sql": orders_sql, "ignored_model1.sql": "select 1 as id", "ignored_model2.sql": "select 1 as id", "overview.md": overview_md, "schema.yml": schema_yml, "ignore_folder": { "model1.sql": "select 1 as id", "model2.sql": "select 1 as id", }, "staging": { "schema.yml": staging_schema_yml, "stg_customers.sql": staging_stg_customers_sql, "stg_orders.sql": staging_stg_orders_sql, "stg_payments.sql": staging_stg_payments_sql, }, } @pytest.fixture(scope="class") def seeds(self): # Read seed file and return seeds = {} dir_path = os.path.dirname(os.path.realpath(__file__)) for file_name in ("raw_customers.csv", "raw_orders.csv", "raw_payments.csv"): contents = read_file(dir_path, "jaffle_shop_data", file_name) seeds[file_name] = contents return seeds @pytest.fixture(scope="class") def project_config_update(self): return { "name": "jaffle_shop", "models": { "jaffle_shop": { "materialized": "table", "staging": { "materialized": "view", }, } }, } ================================================ FILE: tests/fixtures/jaffle_shop_data/.gitkeep ================================================ ================================================ FILE: tests/fixtures/jaffle_shop_data/raw_customers.csv ================================================ id,first_name,last_name 1,Michael,P. 2,Shawn,M. 3,Kathleen,P. 4,Jimmy,C. 5,Katherine,R. 6,Sarah,R. 7,Martin,M. 8,Frank,R. 9,Jennifer,F. 10,Henry,W. 11,Fred,S. 12,Amy,D. 13,Kathleen,M. 14,Steve,F. 15,Teresa,H. 16,Amanda,H. 17,Kimberly,R. 18,Johnny,K. 19,Virginia,F. 20,Anna,A. 21,Willie,H. 22,Sean,H. 23,Mildred,A. 24,David,G. 25,Victor,H. 26,Aaron,R. 27,Benjamin,B. 28,Lisa,W. 29,Benjamin,K. 30,Christina,W. 31,Jane,G. 32,Thomas,O. 33,Katherine,M. 34,Jennifer,S. 35,Sara,T. 36,Harold,O. 37,Shirley,J. 38,Dennis,J. 39,Louise,W. 40,Maria,A. 41,Gloria,C. 42,Diana,S. 43,Kelly,N. 44,Jane,R. 45,Scott,B. 46,Norma,C. 47,Marie,P. 48,Lillian,C. 49,Judy,N. 50,Billy,L. 51,Howard,R. 52,Laura,F. 53,Anne,B. 54,Rose,M. 55,Nicholas,R. 56,Joshua,K. 57,Paul,W. 58,Kathryn,K. 59,Adam,A. 60,Norma,W. 61,Timothy,R. 62,Elizabeth,P. 63,Edward,G. 64,David,C. 65,Brenda,W. 66,Adam,W. 67,Michael,H. 68,Jesse,E. 69,Janet,P. 70,Helen,F. 71,Gerald,C. 72,Kathryn,O. 73,Alan,B. 74,Harry,A. 75,Andrea,H. 76,Barbara,W. 77,Anne,W. 78,Harry,H. 79,Jack,R. 80,Phillip,H. 81,Shirley,H. 82,Arthur,D. 83,Virginia,R. 84,Christina,R. 85,Theresa,M. 86,Jason,C. 87,Phillip,B. 88,Adam,T. 89,Margaret,J. 90,Paul,P. 91,Todd,W. 92,Willie,O. 93,Frances,R. 94,Gregory,H. 95,Lisa,P. 96,Jacqueline,A. 97,Shirley,D. 98,Nicole,M. 99,Mary,G. 100,Jean,M. ================================================ FILE: tests/fixtures/jaffle_shop_data/raw_orders.csv ================================================ id,user_id,order_date,status 1,1,2018-01-01,returned 2,3,2018-01-02,completed 3,94,2018-01-04,completed 4,50,2018-01-05,completed 5,64,2018-01-05,completed 6,54,2018-01-07,completed 7,88,2018-01-09,completed 8,2,2018-01-11,returned 9,53,2018-01-12,completed 10,7,2018-01-14,completed 11,99,2018-01-14,completed 12,59,2018-01-15,completed 13,84,2018-01-17,completed 14,40,2018-01-17,returned 15,25,2018-01-17,completed 16,39,2018-01-18,completed 17,71,2018-01-18,completed 18,64,2018-01-20,returned 19,54,2018-01-22,completed 20,20,2018-01-23,completed 21,71,2018-01-23,completed 22,86,2018-01-24,completed 23,22,2018-01-26,return_pending 24,3,2018-01-27,completed 25,51,2018-01-28,completed 26,32,2018-01-28,completed 27,94,2018-01-29,completed 28,8,2018-01-29,completed 29,57,2018-01-31,completed 30,69,2018-02-02,completed 31,16,2018-02-02,completed 32,28,2018-02-04,completed 33,42,2018-02-04,completed 34,38,2018-02-06,completed 35,80,2018-02-08,completed 36,85,2018-02-10,completed 37,1,2018-02-10,completed 38,51,2018-02-10,completed 39,26,2018-02-11,completed 40,33,2018-02-13,completed 41,99,2018-02-14,completed 42,92,2018-02-16,completed 43,31,2018-02-17,completed 44,66,2018-02-17,completed 45,22,2018-02-17,completed 46,6,2018-02-19,completed 47,50,2018-02-20,completed 48,27,2018-02-21,completed 49,35,2018-02-21,completed 50,51,2018-02-23,completed 51,71,2018-02-24,completed 52,54,2018-02-25,return_pending 53,34,2018-02-26,completed 54,54,2018-02-26,completed 55,18,2018-02-27,completed 56,79,2018-02-28,completed 57,93,2018-03-01,completed 58,22,2018-03-01,completed 59,30,2018-03-02,completed 60,12,2018-03-03,completed 61,63,2018-03-03,completed 62,57,2018-03-05,completed 63,70,2018-03-06,completed 64,13,2018-03-07,completed 65,26,2018-03-08,completed 66,36,2018-03-10,completed 67,79,2018-03-11,completed 68,53,2018-03-11,completed 69,3,2018-03-11,completed 70,8,2018-03-12,completed 71,42,2018-03-12,shipped 72,30,2018-03-14,shipped 73,19,2018-03-16,completed 74,9,2018-03-17,shipped 75,69,2018-03-18,completed 76,25,2018-03-20,completed 77,35,2018-03-21,shipped 78,90,2018-03-23,shipped 79,52,2018-03-23,shipped 80,11,2018-03-23,shipped 81,76,2018-03-23,shipped 82,46,2018-03-24,shipped 83,54,2018-03-24,shipped 84,70,2018-03-26,placed 85,47,2018-03-26,shipped 86,68,2018-03-26,placed 87,46,2018-03-27,placed 88,91,2018-03-27,shipped 89,21,2018-03-28,placed 90,66,2018-03-30,shipped 91,47,2018-03-31,placed 92,84,2018-04-02,placed 93,66,2018-04-03,placed 94,63,2018-04-03,placed 95,27,2018-04-04,placed 96,90,2018-04-06,placed 97,89,2018-04-07,placed 98,41,2018-04-07,placed 99,85,2018-04-09,placed ================================================ FILE: tests/fixtures/jaffle_shop_data/raw_payments.csv ================================================ id,order_id,payment_method,amount 1,1,credit_card,1000 2,2,credit_card,2000 3,3,coupon,100 4,4,coupon,2500 5,5,bank_transfer,1700 6,6,credit_card,600 7,7,credit_card,1600 8,8,credit_card,2300 9,9,gift_card,2300 10,9,bank_transfer,0 11,10,bank_transfer,2600 12,11,credit_card,2700 13,12,credit_card,100 14,13,credit_card,500 15,13,bank_transfer,1400 16,14,bank_transfer,300 17,15,coupon,2200 18,16,credit_card,1000 19,17,bank_transfer,200 20,18,credit_card,500 21,18,credit_card,800 22,19,gift_card,600 23,20,bank_transfer,1500 24,21,credit_card,1200 25,22,bank_transfer,800 26,23,gift_card,2300 27,24,coupon,2600 28,25,bank_transfer,2000 29,25,credit_card,2200 30,25,coupon,1600 31,26,credit_card,3000 32,27,credit_card,2300 33,28,bank_transfer,1900 34,29,bank_transfer,1200 35,30,credit_card,1300 36,31,credit_card,1200 37,32,credit_card,300 38,33,credit_card,2200 39,34,bank_transfer,1500 40,35,credit_card,2900 41,36,bank_transfer,900 42,37,credit_card,2300 43,38,credit_card,1500 44,39,bank_transfer,800 45,40,credit_card,1400 46,41,credit_card,1700 47,42,coupon,1700 48,43,gift_card,1800 49,44,gift_card,1100 50,45,bank_transfer,500 51,46,bank_transfer,800 52,47,credit_card,2200 53,48,bank_transfer,300 54,49,credit_card,600 55,49,credit_card,900 56,50,credit_card,2600 57,51,credit_card,2900 58,51,credit_card,100 59,52,bank_transfer,1500 60,53,credit_card,300 61,54,credit_card,1800 62,54,bank_transfer,1100 63,55,credit_card,2900 64,56,credit_card,400 65,57,bank_transfer,200 66,58,coupon,1800 67,58,gift_card,600 68,59,gift_card,2800 69,60,credit_card,400 70,61,bank_transfer,1600 71,62,gift_card,1400 72,63,credit_card,2900 73,64,bank_transfer,2600 74,65,credit_card,0 75,66,credit_card,2800 76,67,bank_transfer,400 77,67,credit_card,1900 78,68,credit_card,1600 79,69,credit_card,1900 80,70,credit_card,2600 81,71,credit_card,500 82,72,credit_card,2900 83,73,bank_transfer,300 84,74,credit_card,3000 85,75,credit_card,1900 86,76,coupon,200 87,77,credit_card,0 88,77,bank_transfer,1900 89,78,bank_transfer,2600 90,79,credit_card,1800 91,79,credit_card,900 92,80,gift_card,300 93,81,coupon,200 94,82,credit_card,800 95,83,credit_card,100 96,84,bank_transfer,2500 97,85,bank_transfer,1700 98,86,coupon,2300 99,87,gift_card,3000 100,87,credit_card,2600 101,88,credit_card,2900 102,89,bank_transfer,2200 103,90,bank_transfer,200 104,91,credit_card,1900 105,92,bank_transfer,1500 106,92,coupon,200 107,93,gift_card,2600 108,94,coupon,700 109,95,coupon,2400 110,96,gift_card,1700 111,97,bank_transfer,1400 112,98,bank_transfer,1000 113,99,credit_card,2400 ================================================ FILE: tests/functional/README.md ================================================ # This is where we are putting the pytest conversions of test/integration # Goals of moving tests to pytest * Readability * Modularity * Easier to create and debug * Ability to create a project for external debugging # TODO * Create the ability to export a project * Explore using: * https://github.com/pytest-docker-compose/pytest-docker-compose or * https://github.com/avast/pytest-docker for automatically managing a postgres instance running in a docker container * Track test coverage (https://pytest-cov.readthedocs.io/en/latest) ================================================ FILE: tests/functional/__init__.py ================================================ # Functional tests focus on the business requirements of an application. They # only verify the output of an action and do not check the intermediate states # of the system when performing that action. ================================================ FILE: tests/functional/access/test_access.py ================================================ import pytest from dbt.exceptions import DbtReferenceError, InvalidAccessTypeError from dbt.node_types import AccessType from dbt.tests.fixtures.project import write_project_files from dbt.tests.util import get_manifest, rm_file, run_dbt, write_file from tests.fixtures.dbt_integration_project import dbt_integration_project # noqa: F401 my_model_sql = "select 1 as fun" another_model_sql = "select 1234 as notfun" yet_another_model_sql = "select 999 as weird" schema_yml = """ models: - name: my_model description: "my model" access: public - name: another_model description: "yet another model" """ ephemeral_schema_yml = """ models: - name: my_model description: "my model" access: public config: materialized: ephemeral - name: another_model description: "yet another model" """ v2_schema_yml = """ models: - name: my_model description: "my model" access: public - name: another_model description: "another model" - name: yet_another_model description: "yet another model" access: unsupported """ ref_my_model_sql = """ select fun from {{ ref('my_model') }} """ groups_yml = """ groups: - name: analytics owner: name: analytics_owner - name: marts owner: name: marts_owner """ v3_schema_yml = """ models: - name: my_model description: "my model" access: private group: analytics - name: another_model description: "yet another model" - name: ref_my_model description: "a model that refs my_model" group: analytics """ v4_schema_yml = """ models: - name: my_model description: "my model" access: private group: analytics - name: another_model description: "yet another model" - name: ref_my_model description: "a model that refs my_model" group: marts """ simple_exposure_yml = """ exposures: - name: simple_exposure label: simple exposure label type: dashboard depends_on: - ref('my_model') owner: email: something@example.com """ v5_schema_yml = """ models: - name: my_model description: "my model" access: private group: analytics - name: another_model description: "yet another model" - name: ref_my_model description: "a model that refs my_model" group: analytics - name: people_model description: "some people" access: public group: analytics """ v6_schema_yml = """ models: - name: my_model description: "my model" config: access: private group: analytics - name: another_model description: "yet another model" - name: ref_my_model description: "a model that refs my_model" config: group: analytics - name: people_model description: "some people" config: access: public group: analytics """ people_model_sql = """ select 1 as id, 'Drew' as first_name, 'Banin' as last_name, 'yellow' as favorite_color, true as loves_dbt, 5 as tenure, current_timestamp as created_at union all select 1 as id, 'Jeremy' as first_name, 'Cohen' as last_name, 'indigo' as favorite_color, true as loves_dbt, 4 as tenure, current_timestamp as created_at union all select 1 as id, 'Callum' as first_name, 'McCann' as last_name, 'emerald' as favorite_color, true as loves_dbt, 0 as tenure, current_timestamp as created_at """ people_semantic_model_yml = """ semantic_models: - name: semantic_people model: ref('people_model') dimensions: - name: favorite_color type: categorical - name: created_at type: TIME type_params: time_granularity: day measures: - name: years_tenure agg: SUM expr: tenure - name: people agg: count expr: id entities: - name: id type: primary defaults: agg_time_dimension: created_at """ people_metric_yml = """ metrics: - name: number_of_people label: "Number of people" description: Total count of people type: simple type_params: measure: "people" meta: my_meta: 'testing' config: group: analytics """ v2_people_metric_yml = """ metrics: - name: number_of_people label: "Number of people" description: Total count of people type: simple type_params: measure: "people" meta: my_meta: 'testing' config: group: marts """ dbt_integration_project__dbt_project_yml_restrited_access = """ name: dbt_integration_project version: '1.0' config-version: 2 model-paths: ["models"] # paths to models analysis-paths: ["analyses"] # path with analysis files which are compiled, but not run target-path: "target" # path for compiled code clean-targets: ["target"] # directories removed by the clean task test-paths: ["tests"] # where to store test results seed-paths: ["seeds"] # load CSVs from this directory with `dbt seed` macro-paths: ["macros"] # where to find macros profile: user models: dbt_integration_project: restrict-access: True """ dbt_integration_project__schema_yml_protected_model = """ version: 2 models: - name: table_model access: protected """ dbt_integration_project__schema_yml_private_model = """ version: 2 models: - name: table_model access: private group: package """ ref_package_model_sql = """ select * from {{ ref('dbt_integration_project', 'table_model') }} """ schema_yml_ref_package_model = """ version: 2 models: - name: ref_package_model group: package """ metricflow_time_spine_sql = """ SELECT to_date('02/20/2023', 'mm/dd/yyyy') as date_day """ class TestAccess: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_model_sql, "another_model.sql": yet_another_model_sql, "schema.yml": schema_yml, } def test_access_attribute(self, project): manifest = run_dbt(["parse"]) assert len(manifest.nodes) == 2 my_model_id = "model.test.my_model" another_model_id = "model.test.another_model" assert my_model_id in manifest.nodes assert another_model_id in manifest.nodes assert manifest.nodes[my_model_id].access == AccessType.Public assert manifest.nodes[another_model_id].access == AccessType.Protected # write a file with invalid materialization for public access value write_file(ephemeral_schema_yml, project.project_root, "models", "schema.yml") with pytest.raises(InvalidAccessTypeError): run_dbt(["parse"]) # write a file with an invalid access value write_file(yet_another_model_sql, project.project_root, "models", "yet_another_model.sql") write_file(v2_schema_yml, project.project_root, "models", "schema.yml") with pytest.raises(InvalidAccessTypeError): run_dbt(["parse"]) write_file(v2_schema_yml, project.project_root, "models", "schema.yml") with pytest.raises(InvalidAccessTypeError): run_dbt(["parse"]) # Remove invalid access files and write out model that refs my_model rm_file(project.project_root, "models", "yet_another_model.sql") write_file(schema_yml, project.project_root, "models", "schema.yml") write_file(ref_my_model_sql, project.project_root, "models", "ref_my_model.sql") manifest = run_dbt(["parse"]) assert len(manifest.nodes) == 3 # make my_model private, set same group on my_model and ref_my_model write_file(groups_yml, project.project_root, "models", "groups.yml") write_file(v3_schema_yml, project.project_root, "models", "schema.yml") manifest = run_dbt(["parse"]) assert len(manifest.nodes) == 3 manifest = get_manifest(project.project_root) ref_my_model_id = "model.test.ref_my_model" assert manifest.nodes[my_model_id].group == "analytics" assert manifest.nodes[ref_my_model_id].group == "analytics" # Change group on ref_my_model and it should raise write_file(v4_schema_yml, project.project_root, "models", "schema.yml") with pytest.raises(DbtReferenceError): run_dbt(["parse"]) # put back group on ref_my_model, add exposure with ref to private model write_file(v3_schema_yml, project.project_root, "models", "schema.yml") # verify it works again manifest = run_dbt(["parse"]) assert len(manifest.nodes) == 3 # Write out exposure refing private my_model write_file(simple_exposure_yml, project.project_root, "models", "simple_exposure.yml") # Fails with reference error with pytest.raises(DbtReferenceError): run_dbt(["parse"]) # Remove exposure and add people model and metric file write_file(v5_schema_yml, project.project_root, "models", "schema.yml") rm_file(project.project_root, "models", "simple_exposure.yml") write_file(people_model_sql, "models", "people_model.sql") write_file(people_semantic_model_yml, "models", "people_semantic_model.yml") write_file(people_metric_yml, "models", "people_metric.yml") write_file(metricflow_time_spine_sql, "models", "metricflow_time_spine.sql") # Should succeed manifest = run_dbt(["parse"]) assert len(manifest.nodes) == 5 metric_id = "metric.test.number_of_people" assert manifest.metrics[metric_id].group == "analytics" # Use access and group in config write_file(v5_schema_yml, project.project_root, "models", "schema.yml") manifest = run_dbt(["parse"]) assert len(manifest.nodes) == 5 assert manifest.nodes["model.test.my_model"].access == AccessType.Private assert manifest.nodes["model.test.my_model"].group == "analytics" assert manifest.nodes["model.test.ref_my_model"].access == AccessType.Protected assert manifest.nodes["model.test.ref_my_model"].group == "analytics" assert manifest.nodes["model.test.people_model"].access == AccessType.Public assert manifest.nodes["model.test.people_model"].group == "analytics" assert manifest.nodes["model.test.another_model"].access == AccessType.Protected assert manifest.nodes["model.test.another_model"].group is None class TestUnrestrictedPackageAccess: @pytest.fixture(scope="class", autouse=True) def setUp(self, project_root, dbt_integration_project): # noqa: F811 write_project_files(project_root, "dbt_integration_project", dbt_integration_project) @pytest.fixture(scope="class") def packages(self): return {"packages": [{"local": "dbt_integration_project"}]} @pytest.fixture(scope="class") def models(self): return {"ref_protected_package_model.sql": ref_package_model_sql} def test_unrestricted_protected_ref(self, project): write_file( dbt_integration_project__schema_yml_protected_model, project.project_root, "dbt_integration_project", "models", "schema.yml", ) run_dbt(["deps"]) # Runs without issue because restrict-access defaults to False manifest = run_dbt(["parse"]) assert len(manifest.nodes) == 4 root_project_model = manifest.nodes["model.test.ref_protected_package_model"] assert root_project_model.depends_on_nodes == ["model.dbt_integration_project.table_model"] class TestRestrictedPackageAccess: @pytest.fixture(scope="class", autouse=True) def setUp(self, project_root, dbt_integration_project): # noqa: F811 write_project_files(project_root, "dbt_integration_project", dbt_integration_project) # Set table_model.access to protected write_file( dbt_integration_project__schema_yml_protected_model, project_root, "dbt_integration_project", "models", "schema.yml", ) # Set dbt_integration_project.restrict-access to True write_file( dbt_integration_project__dbt_project_yml_restrited_access, project_root, "dbt_integration_project", "dbt_project.yml", ) @pytest.fixture(scope="class") def packages(self): return {"packages": [{"local": "dbt_integration_project"}]} @pytest.fixture(scope="class") def models(self): return { "ref_package_model.sql": ref_package_model_sql, "schema.yml": schema_yml_ref_package_model, } def test_restricted_protected_ref(self, project): run_dbt(["deps"]) with pytest.raises(DbtReferenceError): run_dbt(["parse"]) def test_restricted_private_ref(self, project): run_dbt(["deps"]) # Set table_model.access to private write_file( dbt_integration_project__schema_yml_private_model, project.project_root, "dbt_integration_project", "models", "schema.yml", ) with pytest.raises(DbtReferenceError): run_dbt(["parse"]) dbt_project_yml = """ models: test: subdir_one: +group: analytics +access: private subdir_two: +group: marts +access: public """ class TestAccessDbtProjectConfig: @pytest.fixture(scope="class") def models(self): return { "model_one.sql": my_model_sql, "subdir_one": { "model_two.sql": my_model_sql, }, "subdir_two": { "model_three.sql": my_model_sql, }, } @pytest.fixture(scope="class") def project_config_update(self): return dbt_project_yml def test_dbt_project_access_config(self, project): write_file(groups_yml, project.project_root, "models", "groups.yml") manifest = run_dbt(["parse"]) model_one = manifest.nodes["model.test.model_one"] model_two = manifest.nodes["model.test.model_two"] model_three = manifest.nodes["model.test.model_three"] assert model_one.group is None assert model_one.access == AccessType.Protected assert model_two.group == "analytics" assert model_two.access == AccessType.Private assert model_three.group == "marts" assert model_three.access == AccessType.Public models_yml = """ models: - name: accounts description: > All accounts with whom we have done business. This is a very sensitive asset. access: private group: sales columns: - name: name description: Name of the account. tests: - not_null - unique groups: - name: sales owner: name: sales_owner """ accounts_sql = """ select 'Jane' as name """ class TestGenericTestRestrictAccess: @pytest.fixture(scope="class") def models(self): return { "models.yml": models_yml, "accounts.sql": accounts_sql, } @pytest.fixture(scope="class") def project_config_update(self): return { "restrict-access": True, } def test_generic_tests(self, project): run_dbt(["run"]) run_dbt(["test"]) ================================================ FILE: tests/functional/analysis/test_analyses.py ================================================ import os import pytest from dbt.tests.util import get_manifest, run_dbt my_model_sql = """ select 1 as id """ raw_stuff_sql = """ {% raw %} {% invalid jinja stuff %} {% endraw %} """ schema_yml = """ version: 2 analyses: - name: my_analysis description: "This is my analysis" """ my_analysis_sql = """ select * from {{ ref('my_model') }} """ class TestAnalyses: @pytest.fixture(scope="class") def models(self): return {"my_model.sql": my_model_sql} @pytest.fixture(scope="class") def analyses(self): return { "raw_stuff.sql": raw_stuff_sql, "schema.yml": schema_yml, "my_analysis.sql": my_analysis_sql, } def assert_contents_equal(self, path, expected): with open(path) as fp: assert fp.read().strip() == expected def test_postgres_analyses(self, project): compiled_analysis_path = os.path.normpath("target/compiled/test/analyses") path_1 = os.path.join(compiled_analysis_path, "my_analysis.sql") path_2 = os.path.join(compiled_analysis_path, "raw_stuff.sql") run_dbt(["clean"]) assert not (os.path.exists(compiled_analysis_path)) results = run_dbt(["compile"]) assert len(results) == 3 manifest = get_manifest(project.project_root) analysis_id = "analysis.test.my_analysis" assert analysis_id in manifest.nodes node = manifest.nodes[analysis_id] assert node.description == "This is my analysis" assert os.path.exists(path_1) assert os.path.exists(path_2) expected_sql = 'select * from "{}"."{}"."my_model"'.format( project.database, project.test_schema ) self.assert_contents_equal(path_1, expected_sql) self.assert_contents_equal(path_2, "{% invalid jinja stuff %}") ================================================ FILE: tests/functional/artifacts/data/results/v4/run_results.json ================================================ {"metadata": {"dbt_schema_version": "https://schemas.getdbt.com/dbt/run-results/v4.json", "dbt_version": "1.6.7", "generated_at": "2023-11-06T20:40:37.557735Z", "invocation_id": "42f85a60-4f7b-4cc1-a197-62687104fecc", "env": {}}, "results": [{"status": "success", "timing": [{"name": "compile", "started_at": "2023-11-06T20:40:37.486980Z", "completed_at": "2023-11-06T20:40:37.488837Z"}, {"name": "execute", "started_at": "2023-11-06T20:40:37.490290Z", "completed_at": "2023-11-06T20:40:37.539787Z"}], "thread_id": "Thread-9 (worker)", "execution_time": 0.0566411018371582, "adapter_response": {"_message": "CREATE VIEW", "code": "CREATE VIEW", "rows_affected": -1}, "message": "CREATE VIEW", "failures": null, "unique_id": "model.test.my_model"}, {"status": "success", "timing": [{"name": "compile", "started_at": "2023-11-06T20:40:37.485334Z", "completed_at": "2023-11-06T20:40:37.489266Z"}, {"name": "execute", "started_at": "2023-11-06T20:40:37.494545Z", "completed_at": "2023-11-06T20:40:37.542811Z"}], "thread_id": "Thread-8 (worker)", "execution_time": 0.060118675231933594, "adapter_response": {"_message": "CREATE VIEW", "code": "CREATE VIEW", "rows_affected": -1}, "message": "CREATE VIEW", "failures": null, "unique_id": "model.test.metricflow_time_spine"}], "elapsed_time": 0.18144583702087402, "args": {"defer": false, "indirect_selection": "eager", "select": [], "log_level_file": "debug", "use_colors": true, "cache_selected_only": false, "strict_mode": false, "use_colors_file": true, "partial_parse_file_diff": true, "static_parser": true, "write_json": true, "warn_error_options": {"include": [], "exclude": []}, "print": true, "log_level": "info", "profiles_dir": "/private/var/folders/7h/hj5_fw9j291c58hwfdvy5xbm0000gp/T/pytest-of-jerco/pytest-16/profile0", "log_path": "/Users/jerco/dev/product/dbt-core/logs/test16993032361853467608", "partial_parse": true, "quiet": false, "log_format_file": "debug", "version_check": true, "send_anonymous_usage_stats": false, "project_dir": "/private/var/folders/7h/hj5_fw9j291c58hwfdvy5xbm0000gp/T/pytest-of-jerco/pytest-16/project0", "log_format": "default", "enable_legacy_logger": false, "exclude": [], "populate_cache": true, "log_file_max_bytes": 10485760, "macro_debugging": false, "printer_width": 80, "invocation_command": "dbt tests/functional/artifacts/test_previous_version_state.py::TestPreviousVersionState", "which": "run", "favor_state": false, "introspect": true, "vars": {}}} ================================================ FILE: tests/functional/artifacts/data/results/v5/run_results.json ================================================ {"metadata": {"dbt_schema_version": "https://schemas.getdbt.com/dbt/run-results/v5.json", "dbt_version": "1.8.0a1", "generated_at": "2023-11-06T20:43:08.231028Z", "invocation_id": "a9238a29-6764-47f0-ba7d-f7d61ae5e6c0", "env": {}}, "results": [{"status": "success", "timing": [{"name": "compile", "started_at": "2023-11-06T20:43:08.146847Z", "completed_at": "2023-11-06T20:43:08.149862Z"}, {"name": "execute", "started_at": "2023-11-06T20:43:08.151676Z", "completed_at": "2023-11-06T20:43:08.206208Z"}], "thread_id": "Thread-9 (worker)", "execution_time": 0.06433510780334473, "adapter_response": {"_message": "CREATE VIEW", "code": "CREATE VIEW", "rows_affected": -1}, "message": "CREATE VIEW", "failures": null, "unique_id": "model.test.my_model", "compiled": true, "compiled_code": "select 1 as id", "relation_name": "\"dbt\".\"test16993033859513627134_test_previous_version_state\".\"my_model\""}, {"status": "success", "timing": [{"name": "compile", "started_at": "2023-11-06T20:43:08.144982Z", "completed_at": "2023-11-06T20:43:08.150320Z"}, {"name": "execute", "started_at": "2023-11-06T20:43:08.155222Z", "completed_at": "2023-11-06T20:43:08.209881Z"}], "thread_id": "Thread-8 (worker)", "execution_time": 0.06822013854980469, "adapter_response": {"_message": "CREATE VIEW", "code": "CREATE VIEW", "rows_affected": -1}, "message": "CREATE VIEW", "failures": null, "unique_id": "model.test.metricflow_time_spine", "compiled": true, "compiled_code": "SELECT to_date('02/20/2023', 'mm/dd/yyyy') as date_day", "relation_name": "\"dbt\".\"test16993033859513627134_test_previous_version_state\".\"metricflow_time_spine\""}], "elapsed_time": 0.18284392356872559, "args": {"send_anonymous_usage_stats": false, "profiles_dir": "/private/var/folders/7h/hj5_fw9j291c58hwfdvy5xbm0000gp/T/pytest-of-jerco/pytest-19/profile0", "static_parser": true, "partial_parse_file_diff": true, "printer_width": 80, "log_level_file": "debug", "project_dir": "/private/var/folders/7h/hj5_fw9j291c58hwfdvy5xbm0000gp/T/pytest-of-jerco/pytest-19/project0", "log_format": "default", "strict_mode": false, "macro_debugging": false, "indirect_selection": "eager", "version_check": true, "use_colors_file": true, "select": [], "log_file_max_bytes": 10485760, "warn_error_options": {"include": [], "exclude": []}, "log_format_file": "debug", "invocation_command": "dbt tests/functional/artifacts/test_previous_version_state.py::TestPreviousVersionState", "write_json": true, "log_level": "info", "cache_selected_only": false, "quiet": false, "favor_state": false, "enable_legacy_logger": false, "log_path": "/Users/jerco/dev/product/dbt-core/logs/test16993033859513627134", "which": "run", "partial_parse": true, "introspect": true, "show_resource_report": false, "exclude": [], "populate_cache": true, "vars": {}, "use_colors": true, "defer": false, "print": true}} ================================================ FILE: tests/functional/artifacts/data/results/v6/run_results.json ================================================ {"metadata": {"dbt_schema_version": "https://schemas.getdbt.com/dbt/run-results/v6.json", "dbt_version": "1.9.0a1", "generated_at": "2024-09-24T15:24:36.246406Z", "invocation_id": "ca07d2ca-054f-4ea1-be63-4acccfe4a6a7", "env": {}}, "results": [{"status": "success", "timing": [{"name": "compile", "started_at": "2024-09-24T15:24:36.181448Z", "completed_at": "2024-09-24T15:24:36.183192Z"}, {"name": "execute", "started_at": "2024-09-24T15:24:36.185536Z", "completed_at": "2024-09-24T15:24:36.231442Z"}], "thread_id": "Thread-9 (worker)", "execution_time": 0.0535128116607666, "adapter_response": {"_message": "CREATE VIEW", "code": "CREATE VIEW", "rows_affected": -1}, "message": "CREATE VIEW", "failures": null, "unique_id": "model.test.my_model", "compiled": true, "compiled_code": "select 1 as id", "relation_name": "\"dbt\".\"test17271914717676665882_test_previous_version_state\".\"my_model\"", "batch_results": null}, {"status": "success", "timing": [{"name": "compile", "started_at": "2024-09-24T15:24:36.179261Z", "completed_at": "2024-09-24T15:24:36.182955Z"}, {"name": "execute", "started_at": "2024-09-24T15:24:36.183455Z", "completed_at": "2024-09-24T15:24:36.232800Z"}], "thread_id": "Thread-8 (worker)", "execution_time": 0.055058956146240234, "adapter_response": {"_message": "CREATE VIEW", "code": "CREATE VIEW", "rows_affected": -1}, "message": "CREATE VIEW", "failures": null, "unique_id": "model.test.metricflow_time_spine", "compiled": true, "compiled_code": "SELECT to_date('02/20/2023', 'mm/dd/yyyy') as date_day", "relation_name": "\"dbt\".\"test17271914717676665882_test_previous_version_state\".\"metricflow_time_spine\"", "batch_results": null}], "elapsed_time": 0.6437027454376221, "args": {"profiles_dir": "/private/var/folders/79/5290gpvn3lx5jdryk4844rm80000gn/T/pytest-of-quigleymalcolm/pytest-139/profile0", "invocation_command": "dbt tests/functional/artifacts/test_previous_version_state.py", "strict_mode": false, "partial_parse_file_diff": true, "favor_state": false, "select": [], "log_level_file": "debug", "log_format_file": "debug", "which": "run", "introspect": true, "cache_selected_only": false, "log_level": "info", "defer": false, "static_parser": true, "macro_debugging": false, "write_json": true, "partial_parse": true, "version_check": true, "exclude": [], "use_colors_file": true, "indirect_selection": "eager", "project_dir": "/private/var/folders/79/5290gpvn3lx5jdryk4844rm80000gn/T/pytest-of-quigleymalcolm/pytest-139/project0", "require_resource_names_without_spaces": false, "warn_error_options": {"include": [], "exclude": []}, "log_path": "/Users/quigleymalcolm/Developer/dbt-labs/dbt-core/logs/test17271914717676665882", "printer_width": 80, "use_colors": true, "require_explicit_package_overrides_for_builtin_materializations": true, "show_resource_report": false, "quiet": false, "log_format": "default", "populate_cache": true, "send_anonymous_usage_stats": false, "source_freshness_run_project_hooks": false, "log_file_max_bytes": 10485760, "print": true, "vars": {}, "empty": false}} ================================================ FILE: tests/functional/artifacts/data/state/v1/manifest.json ================================================ { "metadata": { "dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v1.json", "dbt_version": "0.19.2", "generated_at": "2022-06-08T05:12:57.550908Z", "invocation_id": "57566e21-fbd4-4848-87ca-d05ddbd9012e", "env": {}, "project_id": "098f6bcd4621d373cade4e832627b4f6", "user_id": null, "send_anonymous_usage_stats": false, "adapter_type": "postgres" }, "nodes": { "model.test.my_model": { "raw_sql": "select 1 as id", "resource_type": "model", "depends_on": { "macros": [], "nodes": [] }, "config": { "enabled": true, "materialized": "view", "persist_docs": {}, "vars": {}, "quoting": {}, "column_types": {}, "alias": null, "schema": null, "database": null, "tags": [], "full_refresh": null, "post-hook": [], "pre-hook": [] }, "database": "jerco", "schema": "dbt_jcohen", "fqn": [ "test", "my_model" ], "unique_id": "model.test.my_model", "package_name": "test", "root_path": "/Users/jerco/dev/scratch/testy", "path": "my_model.sql", "original_file_path": "models/my_model.sql", "name": "my_model", "alias": "my_model", "checksum": { "name": "sha256", "checksum": "479636cb85ce8d3b0f8db5ff13cf338b61254ad98d905630eac61f963e719e9d" }, "tags": [], "refs": [], "sources": [], "description": "", "columns": {}, "meta": {}, "docs": { "show": true }, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {} } }, "sources": {}, "macros": { "macro.test.drop_relation": { "unique_id": "macro.test.drop_relation", "package_name": "test", "root_path": "/Users/jerco/dev/scratch/testy", "path": "macros/whatever.sql", "original_file_path": "macros/whatever.sql", "name": "drop_relation", "macro_sql": "{% macro drop_relation(relation) -%}\n {{ return(dbt_labs_materialized_views.drop_relation(relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.test.drop_relation" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.test.postgres__list_relations_without_caching": { "unique_id": "macro.test.postgres__list_relations_without_caching", "package_name": "test", "root_path": "/Users/jerco/dev/scratch/testy", "path": "macros/whatever.sql", "original_file_path": "macros/whatever.sql", "name": "postgres__list_relations_without_caching", "macro_sql": "{% macro postgres__list_relations_without_caching(schema_relation) %}\n {{ return(dbt_labs_materialized_views.postgres__list_relations_without_caching(schema_relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.test.postgres__list_relations_without_caching" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.test.postgres_get_relations": { "unique_id": "macro.test.postgres_get_relations", "package_name": "test", "root_path": "/Users/jerco/dev/scratch/testy", "path": "macros/whatever.sql", "original_file_path": "macros/whatever.sql", "name": "postgres_get_relations", "macro_sql": "{% macro postgres_get_relations() %}\n {{ return(dbt_labs_materialized_views.postgres_get_relations()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.test.postgres_get_relations" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.test.redshift__list_relations_without_caching": { "unique_id": "macro.test.redshift__list_relations_without_caching", "package_name": "test", "root_path": "/Users/jerco/dev/scratch/testy", "path": "macros/whatever.sql", "original_file_path": "macros/whatever.sql", "name": "redshift__list_relations_without_caching", "macro_sql": "{% macro redshift__list_relations_without_caching(schema_relation) %}\n {{ return(dbt_labs_materialized_views.redshift__list_relations_without_caching(schema_relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.test.redshift__list_relations_without_caching" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.test.load_relation": { "unique_id": "macro.test.load_relation", "package_name": "test", "root_path": "/Users/jerco/dev/scratch/testy", "path": "macros/whatever.sql", "original_file_path": "macros/whatever.sql", "name": "load_relation", "macro_sql": "{% macro load_relation(relation) %}\n {{ return(dbt_labs_materialized_views.redshift_load_relation_or_mv(relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt_postgres.postgres__get_catalog": { "unique_id": "macro.dbt_postgres.postgres__get_catalog", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/catalog.sql", "original_file_path": "macros/catalog.sql", "name": "postgres__get_catalog", "macro_sql": "{% macro postgres__get_catalog(information_schema, schemas) -%}\n\n {%- call statement('catalog', fetch_result=True) -%}\n {#\n If the user has multiple databases set and the first one is wrong, this will fail.\n But we won't fail in the case where there are multiple quoting-difference-only dbs, which is better.\n #}\n {% set database = information_schema.database %}\n {{ adapter.verify_database(database) }}\n\n select\n '{{ database }}' as table_database,\n sch.nspname as table_schema,\n tbl.relname as table_name,\n case tbl.relkind\n when 'v' then 'VIEW'\n else 'BASE TABLE'\n end as table_type,\n tbl_desc.description as table_comment,\n col.attname as column_name,\n col.attnum as column_index,\n pg_catalog.format_type(col.atttypid, col.atttypmod) as column_type,\n col_desc.description as column_comment,\n pg_get_userbyid(tbl.relowner) as table_owner\n\n from pg_catalog.pg_namespace sch\n join pg_catalog.pg_class tbl on tbl.relnamespace = sch.oid\n join pg_catalog.pg_attribute col on col.attrelid = tbl.oid\n left outer join pg_catalog.pg_description tbl_desc on (tbl_desc.objoid = tbl.oid and tbl_desc.objsubid = 0)\n left outer join pg_catalog.pg_description col_desc on (col_desc.objoid = tbl.oid and col_desc.objsubid = col.attnum)\n\n where (\n {%- for schema in schemas -%}\n upper(sch.nspname) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n )\n and not pg_is_other_temp_schema(sch.oid) -- not a temporary schema belonging to another session\n and tbl.relpersistence = 'p' -- [p]ermanent table. Other values are [u]nlogged table, [t]emporary table\n and tbl.relkind in ('r', 'v', 'f', 'p') -- o[r]dinary table, [v]iew, [f]oreign table, [p]artitioned table. Other values are [i]ndex, [S]equence, [c]omposite type, [t]OAST table, [m]aterialized view\n and col.attnum > 0 -- negative numbers are used for system columns such as oid\n and not col.attisdropped -- column as not been dropped\n\n order by\n sch.nspname,\n tbl.relname,\n col.attnum\n\n {%- endcall -%}\n\n {{ return(load_result('catalog').table) }}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.statement" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt_postgres.postgres_get_relations": { "unique_id": "macro.dbt_postgres.postgres_get_relations", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/relations.sql", "original_file_path": "macros/relations.sql", "name": "postgres_get_relations", "macro_sql": "{% macro postgres_get_relations () -%}\n\n {#\n -- in pg_depend, objid is the dependent, refobjid is the referenced object\n -- > a pg_depend entry indicates that the referenced object cannot be\n -- > dropped without also dropping the dependent object.\n #}\n\n {%- call statement('relations', fetch_result=True) -%}\n with relation as (\n select\n pg_rewrite.ev_class as class,\n pg_rewrite.oid as id\n from pg_rewrite\n ),\n class as (\n select\n oid as id,\n relname as name,\n relnamespace as schema,\n relkind as kind\n from pg_class\n ),\n dependency as (\n select\n pg_depend.objid as id,\n pg_depend.refobjid as ref\n from pg_depend\n ),\n schema as (\n select\n pg_namespace.oid as id,\n pg_namespace.nspname as name\n from pg_namespace\n where nspname != 'information_schema' and nspname not like 'pg\\_%'\n ),\n referenced as (\n select\n relation.id AS id,\n referenced_class.name ,\n referenced_class.schema ,\n referenced_class.kind\n from relation\n join class as referenced_class on relation.class=referenced_class.id\n where referenced_class.kind in ('r', 'v')\n ),\n relationships as (\n select\n referenced.name as referenced_name,\n referenced.schema as referenced_schema_id,\n dependent_class.name as dependent_name,\n dependent_class.schema as dependent_schema_id,\n referenced.kind as kind\n from referenced\n join dependency on referenced.id=dependency.id\n join class as dependent_class on dependency.ref=dependent_class.id\n where\n (referenced.name != dependent_class.name or\n referenced.schema != dependent_class.schema)\n )\n\n select\n referenced_schema.name as referenced_schema,\n relationships.referenced_name as referenced_name,\n dependent_schema.name as dependent_schema,\n relationships.dependent_name as dependent_name\n from relationships\n join schema as dependent_schema on relationships.dependent_schema_id=dependent_schema.id\n join schema as referenced_schema on relationships.referenced_schema_id=referenced_schema.id\n group by referenced_schema, referenced_name, dependent_schema, dependent_name\n order by referenced_schema, referenced_name, dependent_schema, dependent_name;\n\n {%- endcall -%}\n\n {{ return(load_result('relations').table) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.statement" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt_postgres.postgres__create_table_as": { "unique_id": "macro.dbt_postgres.postgres__create_table_as", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__create_table_as", "macro_sql": "{% macro postgres__create_table_as(temporary, relation, sql) -%}\n {%- set unlogged = config.get('unlogged', default=false) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary -%}\n temporary\n {%- elif unlogged -%}\n unlogged\n {%- endif %} table {{ relation }}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt_postgres.postgres__create_schema": { "unique_id": "macro.dbt_postgres.postgres__create_schema", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__create_schema", "macro_sql": "{% macro postgres__create_schema(relation) -%}\n {% if relation.database -%}\n {{ adapter.verify_database(relation.database) }}\n {%- endif -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier().include(database=False) }}\n {%- endcall -%}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.statement" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt_postgres.postgres__drop_schema": { "unique_id": "macro.dbt_postgres.postgres__drop_schema", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__drop_schema", "macro_sql": "{% macro postgres__drop_schema(relation) -%}\n {% if relation.database -%}\n {{ adapter.verify_database(relation.database) }}\n {%- endif -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier().include(database=False) }} cascade\n {%- endcall -%}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.statement" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt_postgres.postgres__get_columns_in_relation": { "unique_id": "macro.dbt_postgres.postgres__get_columns_in_relation", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__get_columns_in_relation", "macro_sql": "{% macro postgres__get_columns_in_relation(relation) -%}\n {% call statement('get_columns_in_relation', fetch_result=True) %}\n select\n column_name,\n data_type,\n character_maximum_length,\n numeric_precision,\n numeric_scale\n\n from {{ relation.information_schema('columns') }}\n where table_name = '{{ relation.identifier }}'\n {% if relation.schema %}\n and table_schema = '{{ relation.schema }}'\n {% endif %}\n order by ordinal_position\n\n {% endcall %}\n {% set table = load_result('get_columns_in_relation').table %}\n {{ return(sql_convert_columns_in_relation(table)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.statement", "macro.dbt.sql_convert_columns_in_relation" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt_postgres.postgres__list_relations_without_caching": { "unique_id": "macro.dbt_postgres.postgres__list_relations_without_caching", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__list_relations_without_caching", "macro_sql": "{% macro postgres__list_relations_without_caching(schema_relation) %}\n {% call statement('list_relations_without_caching', fetch_result=True) -%}\n select\n '{{ schema_relation.database }}' as database,\n tablename as name,\n schemaname as schema,\n 'table' as type\n from pg_tables\n where schemaname ilike '{{ schema_relation.schema }}'\n union all\n select\n '{{ schema_relation.database }}' as database,\n viewname as name,\n schemaname as schema,\n 'view' as type\n from pg_views\n where schemaname ilike '{{ schema_relation.schema }}'\n {% endcall %}\n {{ return(load_result('list_relations_without_caching').table) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.statement" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt_postgres.postgres__information_schema_name": { "unique_id": "macro.dbt_postgres.postgres__information_schema_name", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__information_schema_name", "macro_sql": "{% macro postgres__information_schema_name(database) -%}\n {% if database_name -%}\n {{ adapter.verify_database(database_name) }}\n {%- endif -%}\n information_schema\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt_postgres.postgres__list_schemas": { "unique_id": "macro.dbt_postgres.postgres__list_schemas", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__list_schemas", "macro_sql": "{% macro postgres__list_schemas(database) %}\n {% if database -%}\n {{ adapter.verify_database(database) }}\n {%- endif -%}\n {% call statement('list_schemas', fetch_result=True, auto_begin=False) %}\n select distinct nspname from pg_namespace\n {% endcall %}\n {{ return(load_result('list_schemas').table) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.statement" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt_postgres.postgres__check_schema_exists": { "unique_id": "macro.dbt_postgres.postgres__check_schema_exists", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__check_schema_exists", "macro_sql": "{% macro postgres__check_schema_exists(information_schema, schema) -%}\n {% if information_schema.database -%}\n {{ adapter.verify_database(information_schema.database) }}\n {%- endif -%}\n {% call statement('check_schema_exists', fetch_result=True, auto_begin=False) %}\n select count(*) from pg_namespace where nspname = '{{ schema }}'\n {% endcall %}\n {{ return(load_result('check_schema_exists').table) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.statement" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt_postgres.postgres__current_timestamp": { "unique_id": "macro.dbt_postgres.postgres__current_timestamp", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__current_timestamp", "macro_sql": "{% macro postgres__current_timestamp() -%}\n now()\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt_postgres.postgres__snapshot_string_as_time": { "unique_id": "macro.dbt_postgres.postgres__snapshot_string_as_time", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__snapshot_string_as_time", "macro_sql": "{% macro postgres__snapshot_string_as_time(timestamp) -%}\n {%- set result = \"'\" ~ timestamp ~ \"'::timestamp without time zone\" -%}\n {{ return(result) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt_postgres.postgres__snapshot_get_time": { "unique_id": "macro.dbt_postgres.postgres__snapshot_get_time", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__snapshot_get_time", "macro_sql": "{% macro postgres__snapshot_get_time() -%}\n {{ current_timestamp() }}::timestamp without time zone\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.current_timestamp" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt_postgres.postgres__make_temp_relation": { "unique_id": "macro.dbt_postgres.postgres__make_temp_relation", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__make_temp_relation", "macro_sql": "{% macro postgres__make_temp_relation(base_relation, suffix) %}\n {% set dt = modules.datetime.datetime.now() %}\n {% set dtstring = dt.strftime(\"%H%M%S%f\") %}\n {% set suffix_length = suffix|length + dtstring|length %}\n {% set relation_max_name_length = 63 %}\n {% if suffix_length > relation_max_name_length %}\n {% do exceptions.raise_compiler_error('Temp relation suffix is too long (' ~ suffix|length ~ ' characters). Maximum length is ' ~ (relation_max_name_length - dtstring|length) ~ ' characters.') %}\n {% endif %}\n {% set tmp_identifier = base_relation.identifier[:relation_max_name_length - suffix_length] ~ suffix ~ dtstring %}\n {% do return(base_relation.incorporate(\n path={\n \"identifier\": tmp_identifier,\n \"schema\": none,\n \"database\": none\n })) -%}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt_postgres.postgres_escape_comment": { "unique_id": "macro.dbt_postgres.postgres_escape_comment", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres_escape_comment", "macro_sql": "{% macro postgres_escape_comment(comment) -%}\n {% if comment is not string %}\n {% do exceptions.raise_compiler_error('cannot escape a non-string: ' ~ comment) %}\n {% endif %}\n {%- set magic = '$dbt_comment_literal_block$' -%}\n {%- if magic in comment -%}\n {%- do exceptions.raise_compiler_error('The string ' ~ magic ~ ' is not allowed in comments.') -%}\n {%- endif -%}\n {{ magic }}{{ comment }}{{ magic }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt_postgres.postgres__alter_relation_comment": { "unique_id": "macro.dbt_postgres.postgres__alter_relation_comment", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__alter_relation_comment", "macro_sql": "{% macro postgres__alter_relation_comment(relation, comment) %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n comment on {{ relation.type }} {{ relation }} is {{ escaped_comment }};\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt_postgres.postgres_escape_comment" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt_postgres.postgres__alter_column_comment": { "unique_id": "macro.dbt_postgres.postgres__alter_column_comment", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__alter_column_comment", "macro_sql": "{% macro postgres__alter_column_comment(relation, column_dict) %}\n {% for column_name in column_dict %}\n {% set comment = column_dict[column_name]['description'] %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n comment on column {{ relation }}.{{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }} is {{ escaped_comment }};\n {% endfor %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt_postgres.postgres_escape_comment" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt_postgres.postgres__snapshot_merge_sql": { "unique_id": "macro.dbt_postgres.postgres__snapshot_merge_sql", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/materializations/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshot_merge.sql", "name": "postgres__snapshot_merge_sql", "macro_sql": "{% macro postgres__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n update {{ target }}\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n from {{ source }} as DBT_INTERNAL_SOURCE\n where DBT_INTERNAL_SOURCE.dbt_scd_id::text = {{ target }}.dbt_scd_id::text\n and DBT_INTERNAL_SOURCE.dbt_change_type::text in ('update'::text, 'delete'::text)\n and {{ target }}.dbt_valid_to is null;\n\n insert into {{ target }} ({{ insert_cols_csv }})\n select {% for column in insert_cols -%}\n DBT_INTERNAL_SOURCE.{{ column }} {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n from {{ source }} as DBT_INTERNAL_SOURCE\n where DBT_INTERNAL_SOURCE.dbt_change_type::text = 'insert'::text;\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.statement": { "unique_id": "macro.dbt.statement", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/core.sql", "original_file_path": "macros/core.sql", "name": "statement", "macro_sql": "{% macro statement(name=None, fetch_result=False, auto_begin=True) -%}\n {%- if execute: -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- set res, table = adapter.execute(sql, auto_begin=auto_begin, fetch=fetch_result) -%}\n {%- if name is not none -%}\n {{ store_result(name, response=res, agate_table=table) }}\n {%- endif -%}\n\n {%- endif -%}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.noop_statement": { "unique_id": "macro.dbt.noop_statement", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/core.sql", "original_file_path": "macros/core.sql", "name": "noop_statement", "macro_sql": "{% macro noop_statement(name=None, message=None, code=None, rows_affected=None, res=None) -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_raw_result(name, message=message, code=code, rows_affected=rows_affected, agate_table=res) }}\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.run_hooks": { "unique_id": "macro.dbt.run_hooks", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/helpers.sql", "original_file_path": "macros/materializations/helpers.sql", "name": "run_hooks", "macro_sql": "{% macro run_hooks(hooks, inside_transaction=True) %}\n {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %}\n {% if not inside_transaction and loop.first %}\n {% call statement(auto_begin=inside_transaction) %}\n commit;\n {% endcall %}\n {% endif %}\n {% set rendered = render(hook.get('sql')) | trim %}\n {% if (rendered | length) > 0 %}\n {% call statement(auto_begin=inside_transaction) %}\n {{ rendered }}\n {% endcall %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.statement" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.column_list": { "unique_id": "macro.dbt.column_list", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/helpers.sql", "original_file_path": "macros/materializations/helpers.sql", "name": "column_list", "macro_sql": "{% macro column_list(columns) %}\n {%- for col in columns %}\n {{ col.name }} {% if not loop.last %},{% endif %}\n {% endfor -%}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.column_list_for_create_table": { "unique_id": "macro.dbt.column_list_for_create_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/helpers.sql", "original_file_path": "macros/materializations/helpers.sql", "name": "column_list_for_create_table", "macro_sql": "{% macro column_list_for_create_table(columns) %}\n {%- for col in columns %}\n {{ col.name }} {{ col.data_type }} {%- if not loop.last %},{% endif %}\n {% endfor -%}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.make_hook_config": { "unique_id": "macro.dbt.make_hook_config", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/helpers.sql", "original_file_path": "macros/materializations/helpers.sql", "name": "make_hook_config", "macro_sql": "{% macro make_hook_config(sql, inside_transaction) %}\n {{ tojson({\"sql\": sql, \"transaction\": inside_transaction}) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.before_begin": { "unique_id": "macro.dbt.before_begin", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/helpers.sql", "original_file_path": "macros/materializations/helpers.sql", "name": "before_begin", "macro_sql": "{% macro before_begin(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.make_hook_config" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.in_transaction": { "unique_id": "macro.dbt.in_transaction", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/helpers.sql", "original_file_path": "macros/materializations/helpers.sql", "name": "in_transaction", "macro_sql": "{% macro in_transaction(sql) %}\n {{ make_hook_config(sql, inside_transaction=True) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.make_hook_config" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.after_commit": { "unique_id": "macro.dbt.after_commit", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/helpers.sql", "original_file_path": "macros/materializations/helpers.sql", "name": "after_commit", "macro_sql": "{% macro after_commit(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.make_hook_config" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.drop_relation_if_exists": { "unique_id": "macro.dbt.drop_relation_if_exists", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/helpers.sql", "original_file_path": "macros/materializations/helpers.sql", "name": "drop_relation_if_exists", "macro_sql": "{% macro drop_relation_if_exists(relation) %}\n {% if relation is not none %}\n {{ adapter.drop_relation(relation) }}\n {% endif %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.load_relation": { "unique_id": "macro.dbt.load_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/helpers.sql", "original_file_path": "macros/materializations/helpers.sql", "name": "load_relation", "macro_sql": "{% macro load_relation(relation) %}\n {% do return(adapter.get_relation(\n database=relation.database,\n schema=relation.schema,\n identifier=relation.identifier\n )) -%}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.should_full_refresh": { "unique_id": "macro.dbt.should_full_refresh", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/helpers.sql", "original_file_path": "macros/materializations/helpers.sql", "name": "should_full_refresh", "macro_sql": "{% macro should_full_refresh() %}\n {% set config_full_refresh = config.get('full_refresh') %}\n {% if config_full_refresh is none %}\n {% set config_full_refresh = flags.FULL_REFRESH %}\n {% endif %}\n {% do return(config_full_refresh) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.snapshot_merge_sql": { "unique_id": "macro.dbt.snapshot_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshot/snapshot_merge.sql", "name": "snapshot_merge_sql", "macro_sql": "{% macro snapshot_merge_sql(target, source, insert_cols) -%}\n {{ adapter.dispatch('snapshot_merge_sql')(target, source, insert_cols) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt_postgres.postgres__snapshot_merge_sql" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.default__snapshot_merge_sql": { "unique_id": "macro.dbt.default__snapshot_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshot/snapshot_merge.sql", "name": "default__snapshot_merge_sql", "macro_sql": "{% macro default__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on DBT_INTERNAL_SOURCE.dbt_scd_id = DBT_INTERNAL_DEST.dbt_scd_id\n\n when matched\n and DBT_INTERNAL_DEST.dbt_valid_to is null\n and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')\n then update\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n\n when not matched\n and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'\n then insert ({{ insert_cols_csv }})\n values ({{ insert_cols_csv }})\n ;\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.strategy_dispatch": { "unique_id": "macro.dbt.strategy_dispatch", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/strategies.sql", "original_file_path": "macros/materializations/snapshot/strategies.sql", "name": "strategy_dispatch", "macro_sql": "{% macro strategy_dispatch(name) -%}\n{% set original_name = name %}\n {% if '.' in name %}\n {% set package_name, name = name.split(\".\", 1) %}\n {% else %}\n {% set package_name = none %}\n {% endif %}\n\n {% if package_name is none %}\n {% set package_context = context %}\n {% elif package_name in context %}\n {% set package_context = context[package_name] %}\n {% else %}\n {% set error_msg %}\n Could not find package '{{package_name}}', called with '{{original_name}}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n\n {%- set search_name = 'snapshot_' ~ name ~ '_strategy' -%}\n\n {% if search_name not in package_context %}\n {% set error_msg %}\n The specified strategy macro '{{name}}' was not found in package '{{ package_name }}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n {{ return(package_context[search_name]) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.snapshot_hash_arguments": { "unique_id": "macro.dbt.snapshot_hash_arguments", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/strategies.sql", "original_file_path": "macros/materializations/snapshot/strategies.sql", "name": "snapshot_hash_arguments", "macro_sql": "{% macro snapshot_hash_arguments(args) -%}\n {{ adapter.dispatch('snapshot_hash_arguments')(args) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.default__snapshot_hash_arguments" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.default__snapshot_hash_arguments": { "unique_id": "macro.dbt.default__snapshot_hash_arguments", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/strategies.sql", "original_file_path": "macros/materializations/snapshot/strategies.sql", "name": "default__snapshot_hash_arguments", "macro_sql": "{% macro default__snapshot_hash_arguments(args) -%}\n md5({%- for arg in args -%}\n coalesce(cast({{ arg }} as varchar ), '')\n {% if not loop.last %} || '|' || {% endif %}\n {%- endfor -%})\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.snapshot_get_time": { "unique_id": "macro.dbt.snapshot_get_time", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/strategies.sql", "original_file_path": "macros/materializations/snapshot/strategies.sql", "name": "snapshot_get_time", "macro_sql": "{% macro snapshot_get_time() -%}\n {{ adapter.dispatch('snapshot_get_time')() }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt_postgres.postgres__snapshot_get_time" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.default__snapshot_get_time": { "unique_id": "macro.dbt.default__snapshot_get_time", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/strategies.sql", "original_file_path": "macros/materializations/snapshot/strategies.sql", "name": "default__snapshot_get_time", "macro_sql": "{% macro default__snapshot_get_time() -%}\n {{ current_timestamp() }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.current_timestamp" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.snapshot_timestamp_strategy": { "unique_id": "macro.dbt.snapshot_timestamp_strategy", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/strategies.sql", "original_file_path": "macros/materializations/snapshot/strategies.sql", "name": "snapshot_timestamp_strategy", "macro_sql": "{% macro snapshot_timestamp_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}\n {% set primary_key = config['unique_key'] %}\n {% set updated_at = config['updated_at'] %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %}\n\n {#/*\n The snapshot relation might not have an {{ updated_at }} value if the\n snapshot strategy is changed from `check` to `timestamp`. We\n should use a dbt-created column for the comparison in the snapshot\n table instead of assuming that the user-supplied {{ updated_at }}\n will be present in the historical data.\n\n See https://github.com/fishtown-analytics/dbt/issues/2350\n */ #}\n {% set row_changed_expr -%}\n ({{ snapshotted_rel }}.dbt_valid_from < {{ current_rel }}.{{ updated_at }})\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.snapshot_hash_arguments" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.snapshot_string_as_time": { "unique_id": "macro.dbt.snapshot_string_as_time", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/strategies.sql", "original_file_path": "macros/materializations/snapshot/strategies.sql", "name": "snapshot_string_as_time", "macro_sql": "{% macro snapshot_string_as_time(timestamp) -%}\n {{ adapter.dispatch('snapshot_string_as_time')(timestamp) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt_postgres.postgres__snapshot_string_as_time" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.default__snapshot_string_as_time": { "unique_id": "macro.dbt.default__snapshot_string_as_time", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/strategies.sql", "original_file_path": "macros/materializations/snapshot/strategies.sql", "name": "default__snapshot_string_as_time", "macro_sql": "{% macro default__snapshot_string_as_time(timestamp) %}\n {% do exceptions.raise_not_implemented(\n 'snapshot_string_as_time macro not implemented for adapter '+adapter.type()\n ) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.snapshot_check_all_get_existing_columns": { "unique_id": "macro.dbt.snapshot_check_all_get_existing_columns", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/strategies.sql", "original_file_path": "macros/materializations/snapshot/strategies.sql", "name": "snapshot_check_all_get_existing_columns", "macro_sql": "{% macro snapshot_check_all_get_existing_columns(node, target_exists) -%}\n {%- set query_columns = get_columns_in_query(node['compiled_sql']) -%}\n {%- if not target_exists -%}\n {# no table yet -> return whatever the query does #}\n {{ return([false, query_columns]) }}\n {%- endif -%}\n {# handle any schema changes #}\n {%- set target_table = node.get('alias', node.get('name')) -%}\n {%- set target_relation = adapter.get_relation(database=node.database, schema=node.schema, identifier=target_table) -%}\n {%- set existing_cols = get_columns_in_query('select * from ' ~ target_relation) -%}\n {%- set ns = namespace() -%} {# handle for-loop scoping with a namespace #}\n {%- set ns.column_added = false -%}\n\n {%- set intersection = [] -%}\n {%- for col in query_columns -%}\n {%- if col in existing_cols -%}\n {%- do intersection.append(col) -%}\n {%- else -%}\n {% set ns.column_added = true %}\n {%- endif -%}\n {%- endfor -%}\n {{ return([ns.column_added, intersection]) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.get_columns_in_query" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.snapshot_check_strategy": { "unique_id": "macro.dbt.snapshot_check_strategy", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/strategies.sql", "original_file_path": "macros/materializations/snapshot/strategies.sql", "name": "snapshot_check_strategy", "macro_sql": "{% macro snapshot_check_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}\n {% set check_cols_config = config['check_cols'] %}\n {% set primary_key = config['unique_key'] %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %}\n \n {% set select_current_time -%}\n select {{ snapshot_get_time() }} as snapshot_start\n {%- endset %}\n\n {#-- don't access the column by name, to avoid dealing with casing issues on snowflake #}\n {%- set now = run_query(select_current_time)[0][0] -%}\n {% if now is none or now is undefined -%}\n {%- do exceptions.raise_compiler_error('Could not get a snapshot start time from the database') -%}\n {%- endif %}\n {% set updated_at = snapshot_string_as_time(now) %}\n\n {% set column_added = false %}\n\n {% if check_cols_config == 'all' %}\n {% set column_added, check_cols = snapshot_check_all_get_existing_columns(node, target_exists) %}\n {% elif check_cols_config is iterable and (check_cols_config | length) > 0 %}\n {% set check_cols = check_cols_config %}\n {% else %}\n {% do exceptions.raise_compiler_error(\"Invalid value for 'check_cols': \" ~ check_cols_config) %}\n {% endif %}\n\n {%- set row_changed_expr -%}\n (\n {%- if column_added -%}\n TRUE\n {%- else -%}\n {%- for col in check_cols -%}\n {{ snapshotted_rel }}.{{ col }} != {{ current_rel }}.{{ col }}\n or\n (\n (({{ snapshotted_rel }}.{{ col }} is null) and not ({{ current_rel }}.{{ col }} is null))\n or\n ((not {{ snapshotted_rel }}.{{ col }} is null) and ({{ current_rel }}.{{ col }} is null))\n )\n {%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n {%- endif -%}\n )\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.snapshot_get_time", "macro.dbt.run_query", "macro.dbt.snapshot_string_as_time", "macro.dbt.snapshot_check_all_get_existing_columns", "macro.dbt.snapshot_hash_arguments" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.create_columns": { "unique_id": "macro.dbt.create_columns", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/snapshot.sql", "original_file_path": "macros/materializations/snapshot/snapshot.sql", "name": "create_columns", "macro_sql": "{% macro create_columns(relation, columns) %}\n {{ adapter.dispatch('create_columns')(relation, columns) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.default__create_columns" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.default__create_columns": { "unique_id": "macro.dbt.default__create_columns", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/snapshot.sql", "original_file_path": "macros/materializations/snapshot/snapshot.sql", "name": "default__create_columns", "macro_sql": "{% macro default__create_columns(relation, columns) %}\n {% for column in columns %}\n {% call statement() %}\n alter table {{ relation }} add column \"{{ column.name }}\" {{ column.data_type }};\n {% endcall %}\n {% endfor %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.statement" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.post_snapshot": { "unique_id": "macro.dbt.post_snapshot", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/snapshot.sql", "original_file_path": "macros/materializations/snapshot/snapshot.sql", "name": "post_snapshot", "macro_sql": "{% macro post_snapshot(staging_relation) %}\n {{ adapter.dispatch('post_snapshot')(staging_relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.default__post_snapshot" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.default__post_snapshot": { "unique_id": "macro.dbt.default__post_snapshot", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/snapshot.sql", "original_file_path": "macros/materializations/snapshot/snapshot.sql", "name": "default__post_snapshot", "macro_sql": "{% macro default__post_snapshot(staging_relation) %}\n {# no-op #}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.snapshot_staging_table": { "unique_id": "macro.dbt.snapshot_staging_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/snapshot.sql", "original_file_path": "macros/materializations/snapshot/snapshot.sql", "name": "snapshot_staging_table", "macro_sql": "{% macro snapshot_staging_table(strategy, source_sql, target_relation) -%}\n\n with snapshot_query as (\n\n {{ source_sql }}\n\n ),\n\n snapshotted_data as (\n\n select *,\n {{ strategy.unique_key }} as dbt_unique_key\n\n from {{ target_relation }}\n where dbt_valid_to is null\n\n ),\n\n insertions_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to,\n {{ strategy.scd_id }} as dbt_scd_id\n\n from snapshot_query\n ),\n\n updates_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n {{ strategy.updated_at }} as dbt_valid_to\n\n from snapshot_query\n ),\n\n {%- if strategy.invalidate_hard_deletes %}\n\n deletes_source_data as (\n\n select \n *,\n {{ strategy.unique_key }} as dbt_unique_key\n from snapshot_query\n ),\n {% endif %}\n\n insertions as (\n\n select\n 'insert' as dbt_change_type,\n source_data.*\n\n from insertions_source_data as source_data\n left outer join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where snapshotted_data.dbt_unique_key is null\n or (\n snapshotted_data.dbt_unique_key is not null\n and (\n {{ strategy.row_changed }}\n )\n )\n\n ),\n\n updates as (\n\n select\n 'update' as dbt_change_type,\n source_data.*,\n snapshotted_data.dbt_scd_id\n\n from updates_source_data as source_data\n join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where (\n {{ strategy.row_changed }}\n )\n )\n\n {%- if strategy.invalidate_hard_deletes -%}\n ,\n\n deletes as (\n \n select\n 'delete' as dbt_change_type,\n source_data.*,\n {{ snapshot_get_time() }} as dbt_valid_from,\n {{ snapshot_get_time() }} as dbt_updated_at,\n {{ snapshot_get_time() }} as dbt_valid_to,\n snapshotted_data.dbt_scd_id\n \n from snapshotted_data\n left join deletes_source_data as source_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where source_data.dbt_unique_key is null\n )\n {%- endif %}\n\n select * from insertions\n union all\n select * from updates\n {%- if strategy.invalidate_hard_deletes %}\n union all\n select * from deletes\n {%- endif %}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.snapshot_get_time" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.build_snapshot_table": { "unique_id": "macro.dbt.build_snapshot_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/snapshot.sql", "original_file_path": "macros/materializations/snapshot/snapshot.sql", "name": "build_snapshot_table", "macro_sql": "{% macro build_snapshot_table(strategy, sql) %}\n\n select *,\n {{ strategy.scd_id }} as dbt_scd_id,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to\n from (\n {{ sql }}\n ) sbq\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.get_or_create_relation": { "unique_id": "macro.dbt.get_or_create_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/snapshot.sql", "original_file_path": "macros/materializations/snapshot/snapshot.sql", "name": "get_or_create_relation", "macro_sql": "{% macro get_or_create_relation(database, schema, identifier, type) %}\n {%- set target_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n\n {% if target_relation %}\n {% do return([true, target_relation]) %}\n {% endif %}\n\n {%- set new_relation = api.Relation.create(\n database=database,\n schema=schema,\n identifier=identifier,\n type=type\n ) -%}\n {% do return([false, new_relation]) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.build_snapshot_staging_table": { "unique_id": "macro.dbt.build_snapshot_staging_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/snapshot.sql", "original_file_path": "macros/materializations/snapshot/snapshot.sql", "name": "build_snapshot_staging_table", "macro_sql": "{% macro build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% set tmp_relation = make_temp_relation(target_relation) %}\n\n {% set select = snapshot_staging_table(strategy, sql, target_relation) %}\n\n {% call statement('build_snapshot_staging_relation') %}\n {{ create_table_as(True, tmp_relation, select) }}\n {% endcall %}\n\n {% do return(tmp_relation) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.make_temp_relation", "macro.dbt.snapshot_staging_table", "macro.dbt.statement", "macro.dbt.create_table_as" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.materialization_snapshot_default": { "unique_id": "macro.dbt.materialization_snapshot_default", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/snapshot.sql", "original_file_path": "macros/materializations/snapshot/snapshot.sql", "name": "materialization_snapshot_default", "macro_sql": "{% materialization snapshot, default %}\n {%- set config = model['config'] -%}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n\n {% if not adapter.check_schema_exists(model.database, model.schema) %}\n {% do create_schema(model.database, model.schema) %}\n {% endif %}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=model.database,\n schema=model.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", config, target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_sql']) %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% else %}\n\n {{ adapter.valid_snapshot_target(target_relation) }}\n\n {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% endif %}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if staging_table is defined %}\n {% do post_snapshot(staging_table) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.create_schema", "macro.dbt.get_or_create_relation", "macro.dbt.run_hooks", "macro.dbt.strategy_dispatch", "macro.dbt.build_snapshot_table", "macro.dbt.create_table_as", "macro.dbt.build_snapshot_staging_table", "macro.dbt.create_columns", "macro.dbt.snapshot_merge_sql", "macro.dbt.statement", "macro.dbt.persist_docs", "macro.dbt.post_snapshot" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.create_csv_table": { "unique_id": "macro.dbt.create_csv_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/seed/seed.sql", "original_file_path": "macros/materializations/seed/seed.sql", "name": "create_csv_table", "macro_sql": "{% macro create_csv_table(model, agate_table) -%}\n {{ adapter.dispatch('create_csv_table')(model, agate_table) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.default__create_csv_table" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.reset_csv_table": { "unique_id": "macro.dbt.reset_csv_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/seed/seed.sql", "original_file_path": "macros/materializations/seed/seed.sql", "name": "reset_csv_table", "macro_sql": "{% macro reset_csv_table(model, full_refresh, old_relation, agate_table) -%}\n {{ adapter.dispatch('reset_csv_table')(model, full_refresh, old_relation, agate_table) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.default__reset_csv_table" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.load_csv_rows": { "unique_id": "macro.dbt.load_csv_rows", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/seed/seed.sql", "original_file_path": "macros/materializations/seed/seed.sql", "name": "load_csv_rows", "macro_sql": "{% macro load_csv_rows(model, agate_table) -%}\n {{ adapter.dispatch('load_csv_rows')(model, agate_table) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.default__load_csv_rows" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.default__create_csv_table": { "unique_id": "macro.dbt.default__create_csv_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/seed/seed.sql", "original_file_path": "macros/materializations/seed/seed.sql", "name": "default__create_csv_table", "macro_sql": "{% macro default__create_csv_table(model, agate_table) %}\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n\n {% set sql %}\n create table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.statement" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.default__reset_csv_table": { "unique_id": "macro.dbt.default__reset_csv_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/seed/seed.sql", "original_file_path": "macros/materializations/seed/seed.sql", "name": "default__reset_csv_table", "macro_sql": "{% macro default__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% set sql = \"\" %}\n {% if full_refresh %}\n {{ adapter.drop_relation(old_relation) }}\n {% set sql = create_csv_table(model, agate_table) %}\n {% else %}\n {{ adapter.truncate_relation(old_relation) }}\n {% set sql = \"truncate table \" ~ old_relation %}\n {% endif %}\n\n {{ return(sql) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.create_csv_table" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.get_seed_column_quoted_csv": { "unique_id": "macro.dbt.get_seed_column_quoted_csv", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/seed/seed.sql", "original_file_path": "macros/materializations/seed/seed.sql", "name": "get_seed_column_quoted_csv", "macro_sql": "{% macro get_seed_column_quoted_csv(model, column_names) %}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote_seed_column(col, quote_seed_column)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.basic_load_csv_rows": { "unique_id": "macro.dbt.basic_load_csv_rows", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/seed/seed.sql", "original_file_path": "macros/materializations/seed/seed.sql", "name": "basic_load_csv_rows", "macro_sql": "{% macro basic_load_csv_rows(model, batch_size, agate_table) %}\n {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}\n {% set bindings = [] %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert into {{ this.render() }} ({{ cols_sql }}) values\n {% for row in chunk -%}\n ({%- for column in agate_table.column_names -%}\n %s\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.get_seed_column_quoted_csv" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.default__load_csv_rows": { "unique_id": "macro.dbt.default__load_csv_rows", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/seed/seed.sql", "original_file_path": "macros/materializations/seed/seed.sql", "name": "default__load_csv_rows", "macro_sql": "{% macro default__load_csv_rows(model, agate_table) %}\n {{ return(basic_load_csv_rows(model, 10000, agate_table) )}}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.basic_load_csv_rows" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.materialization_seed_default": { "unique_id": "macro.dbt.materialization_seed_default", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/seed/seed.sql", "original_file_path": "macros/materializations/seed/seed.sql", "name": "materialization_seed_default", "macro_sql": "{% materialization seed, default %}\n\n {%- set identifier = model['alias'] -%}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set agate_table = load_agate_table() -%}\n {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% set create_table_sql = \"\" %}\n {% if exists_as_view %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a view\".format(old_relation)) }}\n {% elif exists_as_table %}\n {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}\n {% else %}\n {% set create_table_sql = create_csv_table(model, agate_table) %}\n {% endif %}\n\n {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}\n {% set rows_affected = (agate_table.rows | length) %}\n {% set sql = load_csv_rows(model, agate_table) %}\n\n {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}\n {{ create_table_sql }};\n -- dbt seed --\n {{ sql }}\n {% endcall %}\n\n {% set target_relation = this.incorporate(type='table') %}\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.should_full_refresh", "macro.dbt.run_hooks", "macro.dbt.reset_csv_table", "macro.dbt.create_csv_table", "macro.dbt.load_csv_rows", "macro.dbt.noop_statement", "macro.dbt.persist_docs" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.incremental_upsert": { "unique_id": "macro.dbt.incremental_upsert", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/incremental/helpers.sql", "original_file_path": "macros/materializations/incremental/helpers.sql", "name": "incremental_upsert", "macro_sql": "{% macro incremental_upsert(tmp_relation, target_relation, unique_key=none, statement_name=\"main\") %}\n {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}\n\n {%- if unique_key is not none -%}\n delete\n from {{ target_relation }}\n where ({{ unique_key }}) in (\n select ({{ unique_key }})\n from {{ tmp_relation }}\n );\n {%- endif %}\n\n insert into {{ target_relation }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ tmp_relation }}\n );\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.materialization_incremental_default": { "unique_id": "macro.dbt.materialization_incremental_default", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/incremental/incremental.sql", "original_file_path": "macros/materializations/incremental/incremental.sql", "name": "materialization_incremental_default", "macro_sql": "{% materialization incremental, default -%}\n\n {% set unique_key = config.get('unique_key') %}\n\n {% set target_relation = this.incorporate(type='table') %}\n {% set existing_relation = load_relation(this) %}\n {% set tmp_relation = make_temp_relation(this) %}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set to_drop = [] %}\n {% if existing_relation is none %}\n {% set build_sql = create_table_as(False, target_relation, sql) %}\n {% elif existing_relation.is_view or should_full_refresh() %}\n {#-- Make sure the backup doesn't exist so we don't encounter issues with the rename below #}\n {% set backup_identifier = existing_relation.identifier ~ \"__dbt_backup\" %}\n {% set backup_relation = existing_relation.incorporate(path={\"identifier\": backup_identifier}) %}\n {% do adapter.drop_relation(backup_relation) %}\n\n {% do adapter.rename_relation(target_relation, backup_relation) %}\n {% set build_sql = create_table_as(False, target_relation, sql) %}\n {% do to_drop.append(backup_relation) %}\n {% else %}\n {% set tmp_relation = make_temp_relation(target_relation) %}\n {% do run_query(create_table_as(True, tmp_relation, sql)) %}\n {% do adapter.expand_target_column_types(\n from_relation=tmp_relation,\n to_relation=target_relation) %}\n {% set build_sql = incremental_upsert(tmp_relation, target_relation, unique_key=unique_key) %}\n {% endif %}\n\n {% call statement(\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {% do adapter.commit() %}\n\n {% for rel in to_drop %}\n {% do adapter.drop_relation(rel) %}\n {% endfor %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.test.load_relation", "macro.dbt.make_temp_relation", "macro.dbt.run_hooks", "macro.dbt.create_table_as", "macro.dbt.should_full_refresh", "macro.dbt.run_query", "macro.dbt.incremental_upsert", "macro.dbt.statement", "macro.dbt.persist_docs" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.get_merge_sql": { "unique_id": "macro.dbt.get_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/common/merge.sql", "original_file_path": "macros/materializations/common/merge.sql", "name": "get_merge_sql", "macro_sql": "{% macro get_merge_sql(target, source, unique_key, dest_columns, predicates=none) -%}\n {{ adapter.dispatch('get_merge_sql')(target, source, unique_key, dest_columns, predicates) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.default__get_merge_sql" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.get_delete_insert_merge_sql": { "unique_id": "macro.dbt.get_delete_insert_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/common/merge.sql", "original_file_path": "macros/materializations/common/merge.sql", "name": "get_delete_insert_merge_sql", "macro_sql": "{% macro get_delete_insert_merge_sql(target, source, unique_key, dest_columns) -%}\n {{ adapter.dispatch('get_delete_insert_merge_sql')(target, source, unique_key, dest_columns) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.default__get_delete_insert_merge_sql" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.get_insert_overwrite_merge_sql": { "unique_id": "macro.dbt.get_insert_overwrite_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/common/merge.sql", "original_file_path": "macros/materializations/common/merge.sql", "name": "get_insert_overwrite_merge_sql", "macro_sql": "{% macro get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header=false) -%}\n {{ adapter.dispatch('get_insert_overwrite_merge_sql')(target, source, dest_columns, predicates, include_sql_header) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.default__get_insert_overwrite_merge_sql" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.default__get_merge_sql": { "unique_id": "macro.dbt.default__get_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/common/merge.sql", "original_file_path": "macros/materializations/common/merge.sql", "name": "default__get_merge_sql", "macro_sql": "{% macro default__get_merge_sql(target, source, unique_key, dest_columns, predicates) -%}\n {%- set predicates = [] if predicates is none else [] + predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {% if unique_key %}\n {% set unique_key_match %}\n DBT_INTERNAL_SOURCE.{{ unique_key }} = DBT_INTERNAL_DEST.{{ unique_key }}\n {% endset %}\n {% do predicates.append(unique_key_match) %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n {{ sql_header if sql_header is not none }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{ predicates | join(' and ') }}\n\n {% if unique_key %}\n when matched then update set\n {% for column in dest_columns -%}\n {{ adapter.quote(column.name) }} = DBT_INTERNAL_SOURCE.{{ adapter.quote(column.name) }}\n {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n {% endif %}\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.get_quoted_csv" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.get_quoted_csv": { "unique_id": "macro.dbt.get_quoted_csv", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/common/merge.sql", "original_file_path": "macros/materializations/common/merge.sql", "name": "get_quoted_csv", "macro_sql": "{% macro get_quoted_csv(column_names) %}\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote(col)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.common_get_delete_insert_merge_sql": { "unique_id": "macro.dbt.common_get_delete_insert_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/common/merge.sql", "original_file_path": "macros/materializations/common/merge.sql", "name": "common_get_delete_insert_merge_sql", "macro_sql": "{% macro common_get_delete_insert_merge_sql(target, source, unique_key, dest_columns) -%}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n {% if unique_key is not none %}\n delete from {{ target }}\n where ({{ unique_key }}) in (\n select ({{ unique_key }})\n from {{ source }}\n );\n {% endif %}\n\n insert into {{ target }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ source }}\n );\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.get_quoted_csv" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.default__get_delete_insert_merge_sql": { "unique_id": "macro.dbt.default__get_delete_insert_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/common/merge.sql", "original_file_path": "macros/materializations/common/merge.sql", "name": "default__get_delete_insert_merge_sql", "macro_sql": "{% macro default__get_delete_insert_merge_sql(target, source, unique_key, dest_columns) -%}\n {{ common_get_delete_insert_merge_sql(target, source, unique_key, dest_columns) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.common_get_delete_insert_merge_sql" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.default__get_insert_overwrite_merge_sql": { "unique_id": "macro.dbt.default__get_insert_overwrite_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/common/merge.sql", "original_file_path": "macros/materializations/common/merge.sql", "name": "default__get_insert_overwrite_merge_sql", "macro_sql": "{% macro default__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) -%}\n {%- set predicates = [] if predicates is none else [] + predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none and include_sql_header }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on FALSE\n\n when not matched by source\n {% if predicates %} and {{ predicates | join(' and ') }} {% endif %}\n then delete\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.get_quoted_csv" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.materialization_table_default": { "unique_id": "macro.dbt.materialization_table_default", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/table/table.sql", "original_file_path": "macros/materializations/table/table.sql", "name": "materialization_table_default", "macro_sql": "{% materialization table, default %}\n {%- set identifier = model['alias'] -%}\n {%- set tmp_identifier = model['name'] + '__dbt_tmp' -%}\n {%- set backup_identifier = model['name'] + '__dbt_backup' -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set target_relation = api.Relation.create(identifier=identifier,\n schema=schema,\n database=database,\n type='table') -%}\n {%- set intermediate_relation = api.Relation.create(identifier=tmp_identifier,\n schema=schema,\n database=database,\n type='table') -%}\n\n /*\n See ../view/view.sql for more information about this relation.\n */\n {%- set backup_relation_type = 'table' if old_relation is none else old_relation.type -%}\n {%- set backup_relation = api.Relation.create(identifier=backup_identifier,\n schema=schema,\n database=database,\n type=backup_relation_type) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n\n -- drop the temp relations if they exists for some reason\n {{ adapter.drop_relation(intermediate_relation) }}\n {{ adapter.drop_relation(backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ create_table_as(False, intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n {% if old_relation is not none %}\n {{ adapter.rename_relation(target_relation, backup_relation) }}\n {% endif %}\n\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {% do persist_docs(target_relation, model) %}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n -- finally, drop the existing/backup relation after the commit\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n{% endmaterialization %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.create_table_as", "macro.dbt.persist_docs", "macro.dbt.drop_relation_if_exists" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.materialization_view_default": { "unique_id": "macro.dbt.materialization_view_default", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/view/view.sql", "original_file_path": "macros/materializations/view/view.sql", "name": "materialization_view_default", "macro_sql": "{%- materialization view, default -%}\n\n {%- set identifier = model['alias'] -%}\n {%- set tmp_identifier = model['name'] + '__dbt_tmp' -%}\n {%- set backup_identifier = model['name'] + '__dbt_backup' -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set target_relation = api.Relation.create(identifier=identifier, schema=schema, database=database,\n type='view') -%}\n {%- set intermediate_relation = api.Relation.create(identifier=tmp_identifier,\n schema=schema, database=database, type='view') -%}\n\n /*\n This relation (probably) doesn't exist yet. If it does exist, it's a leftover from\n a previous run, and we're going to try to drop it immediately. At the end of this\n materialization, we're going to rename the \"old_relation\" to this identifier,\n and then we're going to drop it. In order to make sure we run the correct one of:\n - drop view ...\n - drop table ...\n\n We need to set the type of this relation to be the type of the old_relation, if it exists,\n or else \"view\" as a sane default if it does not. Note that if the old_relation does not\n exist, then there is nothing to move out of the way and subsequentally drop. In that case,\n this relation will be effectively unused.\n */\n {%- set backup_relation_type = 'view' if old_relation is none else old_relation.type -%}\n {%- set backup_relation = api.Relation.create(identifier=backup_identifier,\n schema=schema, database=database,\n type=backup_relation_type) -%}\n\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- drop the temp relations if they exists for some reason\n {{ adapter.drop_relation(intermediate_relation) }}\n {{ adapter.drop_relation(backup_relation) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ create_view_as(intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n -- move the existing view out of the way\n {% if old_relation is not none %}\n {{ adapter.rename_relation(target_relation, backup_relation) }}\n {% endif %}\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization -%}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.create_view_as", "macro.dbt.persist_docs", "macro.dbt.drop_relation_if_exists" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.handle_existing_table": { "unique_id": "macro.dbt.handle_existing_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/view/create_or_replace_view.sql", "original_file_path": "macros/materializations/view/create_or_replace_view.sql", "name": "handle_existing_table", "macro_sql": "{% macro handle_existing_table(full_refresh, old_relation) %}\n {{ adapter.dispatch(\"handle_existing_table\", packages=['dbt'])(full_refresh, old_relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.default__handle_existing_table" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.default__handle_existing_table": { "unique_id": "macro.dbt.default__handle_existing_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/view/create_or_replace_view.sql", "original_file_path": "macros/materializations/view/create_or_replace_view.sql", "name": "default__handle_existing_table", "macro_sql": "{% macro default__handle_existing_table(full_refresh, old_relation) %}\n {{ adapter.drop_relation(old_relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.create_or_replace_view": { "unique_id": "macro.dbt.create_or_replace_view", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/view/create_or_replace_view.sql", "original_file_path": "macros/materializations/view/create_or_replace_view.sql", "name": "create_or_replace_view", "macro_sql": "{% macro create_or_replace_view(run_outside_transaction_hooks=True) %}\n {%- set identifier = model['alias'] -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database,\n type='view') -%}\n\n {% if run_outside_transaction_hooks %}\n -- no transactions on BigQuery\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n {% endif %}\n\n -- `BEGIN` happens here on Snowflake\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- If there's a table with the same name and we weren't told to full refresh,\n -- that's an error. If we were told to full refresh, drop it. This behavior differs\n -- for Snowflake and BigQuery, so multiple dispatch is used.\n {%- if old_relation is not none and old_relation.is_table -%}\n {{ handle_existing_table(should_full_refresh(), old_relation) }}\n {%- endif -%}\n\n -- build model\n {% call statement('main') -%}\n {{ create_view_as(target_relation, sql) }}\n {%- endcall %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if run_outside_transaction_hooks %}\n -- No transactions on BigQuery\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n {% endif %}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.run_hooks", "macro.dbt.handle_existing_table", "macro.dbt.should_full_refresh", "macro.dbt.statement", "macro.dbt.create_view_as" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.generate_alias_name": { "unique_id": "macro.dbt.generate_alias_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/etc/get_custom_alias.sql", "original_file_path": "macros/etc/get_custom_alias.sql", "name": "generate_alias_name", "macro_sql": "{% macro generate_alias_name(custom_alias_name=none, node=none) -%}\n\n {%- if custom_alias_name is none -%}\n\n {{ node.name }}\n\n {%- else -%}\n\n {{ custom_alias_name | trim }}\n\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.run_query": { "unique_id": "macro.dbt.run_query", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/etc/query.sql", "original_file_path": "macros/etc/query.sql", "name": "run_query", "macro_sql": "{% macro run_query(sql) %}\n {% call statement(\"run_query_statement\", fetch_result=true, auto_begin=false) %}\n {{ sql }}\n {% endcall %}\n\n {% do return(load_result(\"run_query_statement\").table) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.statement" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.is_incremental": { "unique_id": "macro.dbt.is_incremental", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/etc/is_incremental.sql", "original_file_path": "macros/etc/is_incremental.sql", "name": "is_incremental", "macro_sql": "{% macro is_incremental() %}\n {#-- do not run introspective queries in parsing #}\n {% if not execute %}\n {{ return(False) }}\n {% else %}\n {% set relation = adapter.get_relation(this.database, this.schema, this.table) %}\n {{ return(relation is not none\n and relation.type == 'table'\n and model.config.materialized == 'incremental'\n and not should_full_refresh()) }}\n {% endif %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.should_full_refresh" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.convert_datetime": { "unique_id": "macro.dbt.convert_datetime", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "name": "convert_datetime", "macro_sql": "{% macro convert_datetime(date_str, date_fmt) %}\n\n {% set error_msg -%}\n The provided partition date '{{ date_str }}' does not match the expected format '{{ date_fmt }}'\n {%- endset %}\n\n {% set res = try_or_compiler_error(error_msg, modules.datetime.datetime.strptime, date_str.strip(), date_fmt) %}\n {{ return(res) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.dates_in_range": { "unique_id": "macro.dbt.dates_in_range", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "name": "dates_in_range", "macro_sql": "{% macro dates_in_range(start_date_str, end_date_str=none, in_fmt=\"%Y%m%d\", out_fmt=\"%Y%m%d\") %}\n {% set end_date_str = start_date_str if end_date_str is none else end_date_str %}\n\n {% set start_date = convert_datetime(start_date_str, in_fmt) %}\n {% set end_date = convert_datetime(end_date_str, in_fmt) %}\n\n {% set day_count = (end_date - start_date).days %}\n {% if day_count < 0 %}\n {% set msg -%}\n Partition start date is after the end date ({{ start_date }}, {{ end_date }})\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg, model) }}\n {% endif %}\n\n {% set date_list = [] %}\n {% for i in range(0, day_count + 1) %}\n {% set the_date = (modules.datetime.timedelta(days=i) + start_date) %}\n {% if not out_fmt %}\n {% set _ = date_list.append(the_date) %}\n {% else %}\n {% set _ = date_list.append(the_date.strftime(out_fmt)) %}\n {% endif %}\n {% endfor %}\n\n {{ return(date_list) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.convert_datetime" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.partition_range": { "unique_id": "macro.dbt.partition_range", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "name": "partition_range", "macro_sql": "{% macro partition_range(raw_partition_date, date_fmt='%Y%m%d') %}\n {% set partition_range = (raw_partition_date | string).split(\",\") %}\n\n {% if (partition_range | length) == 1 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = none %}\n {% elif (partition_range | length) == 2 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = partition_range[1] %}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Invalid partition time. Expected format: {Start Date}[,{End Date}]. Got: \" ~ raw_partition_date) }}\n {% endif %}\n\n {{ return(dates_in_range(start_date, end_date, in_fmt=date_fmt)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.dates_in_range" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.py_current_timestring": { "unique_id": "macro.dbt.py_current_timestring", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "name": "py_current_timestring", "macro_sql": "{% macro py_current_timestring() %}\n {% set dt = modules.datetime.datetime.now() %}\n {% do return(dt.strftime(\"%Y%m%d%H%M%S%f\")) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.generate_schema_name": { "unique_id": "macro.dbt.generate_schema_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/etc/get_custom_schema.sql", "original_file_path": "macros/etc/get_custom_schema.sql", "name": "generate_schema_name", "macro_sql": "{% macro generate_schema_name(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if custom_schema_name is none -%}\n\n {{ default_schema }}\n\n {%- else -%}\n\n {{ default_schema }}_{{ custom_schema_name | trim }}\n\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.generate_schema_name_for_env": { "unique_id": "macro.dbt.generate_schema_name_for_env", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/etc/get_custom_schema.sql", "original_file_path": "macros/etc/get_custom_schema.sql", "name": "generate_schema_name_for_env", "macro_sql": "{% macro generate_schema_name_for_env(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if target.name == 'prod' and custom_schema_name is not none -%}\n\n {{ custom_schema_name | trim }}\n\n {%- else -%}\n\n {{ default_schema }}\n\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.generate_database_name": { "unique_id": "macro.dbt.generate_database_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/etc/get_custom_database.sql", "original_file_path": "macros/etc/get_custom_database.sql", "name": "generate_database_name", "macro_sql": "{% macro generate_database_name(custom_database_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_database_name')(custom_database_name, node)) %}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.default__generate_database_name" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.default__generate_database_name": { "unique_id": "macro.dbt.default__generate_database_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/etc/get_custom_database.sql", "original_file_path": "macros/etc/get_custom_database.sql", "name": "default__generate_database_name", "macro_sql": "{% macro default__generate_database_name(custom_database_name=none, node=none) -%}\n {%- set default_database = target.database -%}\n {%- if custom_database_name is none -%}\n\n {{ default_database }}\n\n {%- else -%}\n\n {{ custom_database_name }}\n\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.get_columns_in_query": { "unique_id": "macro.dbt.get_columns_in_query", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "get_columns_in_query", "macro_sql": "{% macro get_columns_in_query(select_sql) -%}\n {{ return(adapter.dispatch('get_columns_in_query')(select_sql)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.default__get_columns_in_query" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.default__get_columns_in_query": { "unique_id": "macro.dbt.default__get_columns_in_query", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__get_columns_in_query", "macro_sql": "{% macro default__get_columns_in_query(select_sql) %}\n {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%}\n select * from (\n {{ select_sql }}\n ) as __dbt_sbq\n where false\n limit 0\n {% endcall %}\n\n {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.statement" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.create_schema": { "unique_id": "macro.dbt.create_schema", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "create_schema", "macro_sql": "{% macro create_schema(relation) -%}\n {{ adapter.dispatch('create_schema')(relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt_postgres.postgres__create_schema" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.default__create_schema": { "unique_id": "macro.dbt.default__create_schema", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__create_schema", "macro_sql": "{% macro default__create_schema(relation) -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier() }}\n {% endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.statement" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.drop_schema": { "unique_id": "macro.dbt.drop_schema", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "drop_schema", "macro_sql": "{% macro drop_schema(relation) -%}\n {{ adapter.dispatch('drop_schema')(relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt_postgres.postgres__drop_schema" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.default__drop_schema": { "unique_id": "macro.dbt.default__drop_schema", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__drop_schema", "macro_sql": "{% macro default__drop_schema(relation) -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier() }} cascade\n {% endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.statement" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.create_table_as": { "unique_id": "macro.dbt.create_table_as", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "create_table_as", "macro_sql": "{% macro create_table_as(temporary, relation, sql) -%}\n {{ adapter.dispatch('create_table_as')(temporary, relation, sql) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt_postgres.postgres__create_table_as" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.default__create_table_as": { "unique_id": "macro.dbt.default__create_table_as", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__create_table_as", "macro_sql": "{% macro default__create_table_as(temporary, relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary: -%}temporary{%- endif %} table\n {{ relation.include(database=(not temporary), schema=(not temporary)) }}\n as (\n {{ sql }}\n );\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.create_view_as": { "unique_id": "macro.dbt.create_view_as", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "create_view_as", "macro_sql": "{% macro create_view_as(relation, sql) -%}\n {{ adapter.dispatch('create_view_as')(relation, sql) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.default__create_view_as" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.default__create_view_as": { "unique_id": "macro.dbt.default__create_view_as", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__create_view_as", "macro_sql": "{% macro default__create_view_as(relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n create view {{ relation }} as (\n {{ sql }}\n );\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.get_catalog": { "unique_id": "macro.dbt.get_catalog", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "get_catalog", "macro_sql": "{% macro get_catalog(information_schema, schemas) -%}\n {{ return(adapter.dispatch('get_catalog')(information_schema, schemas)) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_catalog" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.default__get_catalog": { "unique_id": "macro.dbt.default__get_catalog", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__get_catalog", "macro_sql": "{% macro default__get_catalog(information_schema, schemas) -%}\n\n {% set typename = adapter.type() %}\n {% set msg -%}\n get_catalog not implemented for {{ typename }}\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.get_columns_in_relation": { "unique_id": "macro.dbt.get_columns_in_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "get_columns_in_relation", "macro_sql": "{% macro get_columns_in_relation(relation) -%}\n {{ return(adapter.dispatch('get_columns_in_relation')(relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_columns_in_relation" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.sql_convert_columns_in_relation": { "unique_id": "macro.dbt.sql_convert_columns_in_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "sql_convert_columns_in_relation", "macro_sql": "{% macro sql_convert_columns_in_relation(table) -%}\n {% set columns = [] %}\n {% for row in table %}\n {% do columns.append(api.Column(*row)) %}\n {% endfor %}\n {{ return(columns) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.default__get_columns_in_relation": { "unique_id": "macro.dbt.default__get_columns_in_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__get_columns_in_relation", "macro_sql": "{% macro default__get_columns_in_relation(relation) -%}\n {{ exceptions.raise_not_implemented(\n 'get_columns_in_relation macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.alter_column_type": { "unique_id": "macro.dbt.alter_column_type", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "alter_column_type", "macro_sql": "{% macro alter_column_type(relation, column_name, new_column_type) -%}\n {{ return(adapter.dispatch('alter_column_type')(relation, column_name, new_column_type)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.default__alter_column_type" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.alter_column_comment": { "unique_id": "macro.dbt.alter_column_comment", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "alter_column_comment", "macro_sql": "{% macro alter_column_comment(relation, column_dict) -%}\n {{ return(adapter.dispatch('alter_column_comment')(relation, column_dict)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt_postgres.postgres__alter_column_comment" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.default__alter_column_comment": { "unique_id": "macro.dbt.default__alter_column_comment", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__alter_column_comment", "macro_sql": "{% macro default__alter_column_comment(relation, column_dict) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_column_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.alter_relation_comment": { "unique_id": "macro.dbt.alter_relation_comment", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "alter_relation_comment", "macro_sql": "{% macro alter_relation_comment(relation, relation_comment) -%}\n {{ return(adapter.dispatch('alter_relation_comment')(relation, relation_comment)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt_postgres.postgres__alter_relation_comment" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.default__alter_relation_comment": { "unique_id": "macro.dbt.default__alter_relation_comment", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__alter_relation_comment", "macro_sql": "{% macro default__alter_relation_comment(relation, relation_comment) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_relation_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.persist_docs": { "unique_id": "macro.dbt.persist_docs", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "persist_docs", "macro_sql": "{% macro persist_docs(relation, model, for_relation=true, for_columns=true) -%}\n {{ return(adapter.dispatch('persist_docs')(relation, model, for_relation, for_columns)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.default__persist_docs" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.default__persist_docs": { "unique_id": "macro.dbt.default__persist_docs", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__persist_docs", "macro_sql": "{% macro default__persist_docs(relation, model, for_relation, for_columns) -%}\n {% if for_relation and config.persist_relation_docs() and model.description %}\n {% do run_query(alter_relation_comment(relation, model.description)) %}\n {% endif %}\n\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {% do run_query(alter_column_comment(relation, model.columns)) %}\n {% endif %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.run_query", "macro.dbt.alter_relation_comment", "macro.dbt.alter_column_comment" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.default__alter_column_type": { "unique_id": "macro.dbt.default__alter_column_type", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__alter_column_type", "macro_sql": "{% macro default__alter_column_type(relation, column_name, new_column_type) -%}\n {#\n 1. Create a new column (w/ temp name and correct type)\n 2. Copy data over to it\n 3. Drop the existing column (cascade!)\n 4. Rename the new column to existing column\n #}\n {%- set tmp_column = column_name + \"__dbt_alter\" -%}\n\n {% call statement('alter_column_type') %}\n alter table {{ relation }} add column {{ adapter.quote(tmp_column) }} {{ new_column_type }};\n update {{ relation }} set {{ adapter.quote(tmp_column) }} = {{ adapter.quote(column_name) }};\n alter table {{ relation }} drop column {{ adapter.quote(column_name) }} cascade;\n alter table {{ relation }} rename column {{ adapter.quote(tmp_column) }} to {{ adapter.quote(column_name) }}\n {% endcall %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.statement" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.drop_relation": { "unique_id": "macro.dbt.drop_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "drop_relation", "macro_sql": "{% macro drop_relation(relation) -%}\n {{ return(adapter.dispatch('drop_relation')(relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.default__drop_relation" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.default__drop_relation": { "unique_id": "macro.dbt.default__drop_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__drop_relation", "macro_sql": "{% macro default__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n drop {{ relation.type }} if exists {{ relation }} cascade\n {%- endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.statement" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.truncate_relation": { "unique_id": "macro.dbt.truncate_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "truncate_relation", "macro_sql": "{% macro truncate_relation(relation) -%}\n {{ return(adapter.dispatch('truncate_relation')(relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.default__truncate_relation" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.default__truncate_relation": { "unique_id": "macro.dbt.default__truncate_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__truncate_relation", "macro_sql": "{% macro default__truncate_relation(relation) -%}\n {% call statement('truncate_relation') -%}\n truncate table {{ relation }}\n {%- endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.statement" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.rename_relation": { "unique_id": "macro.dbt.rename_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "rename_relation", "macro_sql": "{% macro rename_relation(from_relation, to_relation) -%}\n {{ return(adapter.dispatch('rename_relation')(from_relation, to_relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.default__rename_relation" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.default__rename_relation": { "unique_id": "macro.dbt.default__rename_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__rename_relation", "macro_sql": "{% macro default__rename_relation(from_relation, to_relation) -%}\n {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %}\n {% call statement('rename_relation') -%}\n alter table {{ from_relation }} rename to {{ target_name }}\n {%- endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.statement" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.information_schema_name": { "unique_id": "macro.dbt.information_schema_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "information_schema_name", "macro_sql": "{% macro information_schema_name(database) %}\n {{ return(adapter.dispatch('information_schema_name')(database)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt_postgres.postgres__information_schema_name" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.default__information_schema_name": { "unique_id": "macro.dbt.default__information_schema_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__information_schema_name", "macro_sql": "{% macro default__information_schema_name(database) -%}\n {%- if database -%}\n {{ database }}.INFORMATION_SCHEMA\n {%- else -%}\n INFORMATION_SCHEMA\n {%- endif -%}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.list_schemas": { "unique_id": "macro.dbt.list_schemas", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "list_schemas", "macro_sql": "{% macro list_schemas(database) -%}\n {{ return(adapter.dispatch('list_schemas')(database)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt_postgres.postgres__list_schemas" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.default__list_schemas": { "unique_id": "macro.dbt.default__list_schemas", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__list_schemas", "macro_sql": "{% macro default__list_schemas(database) -%}\n {% set sql %}\n select distinct schema_name\n from {{ information_schema_name(database) }}.SCHEMATA\n where catalog_name ilike '{{ database }}'\n {% endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.information_schema_name", "macro.dbt.run_query" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.check_schema_exists": { "unique_id": "macro.dbt.check_schema_exists", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "check_schema_exists", "macro_sql": "{% macro check_schema_exists(information_schema, schema) -%}\n {{ return(adapter.dispatch('check_schema_exists')(information_schema, schema)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt_postgres.postgres__check_schema_exists" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.default__check_schema_exists": { "unique_id": "macro.dbt.default__check_schema_exists", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__check_schema_exists", "macro_sql": "{% macro default__check_schema_exists(information_schema, schema) -%}\n {% set sql -%}\n select count(*)\n from {{ information_schema.replace(information_schema_view='SCHEMATA') }}\n where catalog_name='{{ information_schema.database }}'\n and schema_name='{{ schema }}'\n {%- endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.run_query" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.list_relations_without_caching": { "unique_id": "macro.dbt.list_relations_without_caching", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "list_relations_without_caching", "macro_sql": "{% macro list_relations_without_caching(schema_relation) %}\n {{ return(adapter.dispatch('list_relations_without_caching')(schema_relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.test.postgres__list_relations_without_caching" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.default__list_relations_without_caching": { "unique_id": "macro.dbt.default__list_relations_without_caching", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__list_relations_without_caching", "macro_sql": "{% macro default__list_relations_without_caching(schema_relation) %}\n {{ exceptions.raise_not_implemented(\n 'list_relations_without_caching macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.current_timestamp": { "unique_id": "macro.dbt.current_timestamp", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "current_timestamp", "macro_sql": "{% macro current_timestamp() -%}\n {{ adapter.dispatch('current_timestamp')() }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt_postgres.postgres__current_timestamp" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.default__current_timestamp": { "unique_id": "macro.dbt.default__current_timestamp", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__current_timestamp", "macro_sql": "{% macro default__current_timestamp() -%}\n {{ exceptions.raise_not_implemented(\n 'current_timestamp macro not implemented for adapter '+adapter.type()) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.collect_freshness": { "unique_id": "macro.dbt.collect_freshness", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "collect_freshness", "macro_sql": "{% macro collect_freshness(source, loaded_at_field, filter) %}\n {{ return(adapter.dispatch('collect_freshness')(source, loaded_at_field, filter))}}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.default__collect_freshness" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.default__collect_freshness": { "unique_id": "macro.dbt.default__collect_freshness", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__collect_freshness", "macro_sql": "{% macro default__collect_freshness(source, loaded_at_field, filter) %}\n {% call statement('collect_freshness', fetch_result=True, auto_begin=False) -%}\n select\n max({{ loaded_at_field }}) as max_loaded_at,\n {{ current_timestamp() }} as snapshotted_at\n from {{ source }}\n {% if filter %}\n where {{ filter }}\n {% endif %}\n {% endcall %}\n {{ return(load_result('collect_freshness').table) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.statement", "macro.dbt.current_timestamp" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.make_temp_relation": { "unique_id": "macro.dbt.make_temp_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "make_temp_relation", "macro_sql": "{% macro make_temp_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_temp_relation')(base_relation, suffix))}}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt_postgres.postgres__make_temp_relation" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.default__make_temp_relation": { "unique_id": "macro.dbt.default__make_temp_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__make_temp_relation", "macro_sql": "{% macro default__make_temp_relation(base_relation, suffix) %}\n {% set tmp_identifier = base_relation.identifier ~ suffix %}\n {% set tmp_relation = base_relation.incorporate(\n path={\"identifier\": tmp_identifier}) -%}\n\n {% do return(tmp_relation) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.set_sql_header": { "unique_id": "macro.dbt.set_sql_header", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "set_sql_header", "macro_sql": "{% macro set_sql_header(config) -%}\n {{ config.set('sql_header', caller()) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.default__test_relationships": { "unique_id": "macro.dbt.default__test_relationships", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/schema_tests/relationships.sql", "original_file_path": "macros/schema_tests/relationships.sql", "name": "default__test_relationships", "macro_sql": "{% macro default__test_relationships(model, to, field) %}\n\n{% set column_name = kwargs.get('column_name', kwargs.get('from')) %}\n\n\nselect count(*) as validation_errors\nfrom (\n select {{ column_name }} as id from {{ model }}\n) as child\nleft join (\n select {{ field }} as id from {{ to }}\n) as parent on parent.id = child.id\nwhere child.id is not null\n and parent.id is null\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.test_relationships": { "unique_id": "macro.dbt.test_relationships", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/schema_tests/relationships.sql", "original_file_path": "macros/schema_tests/relationships.sql", "name": "test_relationships", "macro_sql": "{% macro test_relationships(model, to, field) %}\n {% set macro = adapter.dispatch('test_relationships') %}\n {{ macro(model, to, field, **kwargs) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.default__test_relationships" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.default__test_not_null": { "unique_id": "macro.dbt.default__test_not_null", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/schema_tests/not_null.sql", "original_file_path": "macros/schema_tests/not_null.sql", "name": "default__test_not_null", "macro_sql": "{% macro default__test_not_null(model) %}\n\n{% set column_name = kwargs.get('column_name', kwargs.get('arg')) %}\n\nselect count(*) as validation_errors\nfrom {{ model }}\nwhere {{ column_name }} is null\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.test_not_null": { "unique_id": "macro.dbt.test_not_null", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/schema_tests/not_null.sql", "original_file_path": "macros/schema_tests/not_null.sql", "name": "test_not_null", "macro_sql": "{% macro test_not_null(model) %}\n {% set macro = adapter.dispatch('test_not_null') %}\n {{ macro(model, **kwargs) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.default__test_not_null" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.default__test_unique": { "unique_id": "macro.dbt.default__test_unique", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/schema_tests/unique.sql", "original_file_path": "macros/schema_tests/unique.sql", "name": "default__test_unique", "macro_sql": "{% macro default__test_unique(model) %}\n\n{% set column_name = kwargs.get('column_name', kwargs.get('arg')) %}\n\nselect count(*) as validation_errors\nfrom (\n\n select\n {{ column_name }}\n\n from {{ model }}\n where {{ column_name }} is not null\n group by {{ column_name }}\n having count(*) > 1\n\n) validation_errors\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.test_unique": { "unique_id": "macro.dbt.test_unique", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/schema_tests/unique.sql", "original_file_path": "macros/schema_tests/unique.sql", "name": "test_unique", "macro_sql": "{% macro test_unique(model) %}\n {% set macro = adapter.dispatch('test_unique') %}\n {{ macro(model, **kwargs) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.default__test_unique" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.default__test_accepted_values": { "unique_id": "macro.dbt.default__test_accepted_values", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/schema_tests/accepted_values.sql", "original_file_path": "macros/schema_tests/accepted_values.sql", "name": "default__test_accepted_values", "macro_sql": "{% macro default__test_accepted_values(model, values) %}\n\n{% set column_name = kwargs.get('column_name', kwargs.get('field')) %}\n{% set quote_values = kwargs.get('quote', True) %}\n\nwith all_values as (\n\n select distinct\n {{ column_name }} as value_field\n\n from {{ model }}\n\n),\n\nvalidation_errors as (\n\n select\n value_field\n\n from all_values\n where value_field not in (\n {% for value in values -%}\n {% if quote_values -%}\n '{{ value }}'\n {%- else -%}\n {{ value }}\n {%- endif -%}\n {%- if not loop.last -%},{%- endif %}\n {%- endfor %}\n )\n)\n\nselect count(*) as validation_errors\nfrom validation_errors\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] }, "macro.dbt.test_accepted_values": { "unique_id": "macro.dbt.test_accepted_values", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/schema_tests/accepted_values.sql", "original_file_path": "macros/schema_tests/accepted_values.sql", "name": "test_accepted_values", "macro_sql": "{% macro test_accepted_values(model, values) %}\n {% set macro = adapter.dispatch('test_accepted_values') %}\n {{ macro(model, values, **kwargs) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": { "macros": [ "macro.dbt.default__test_accepted_values" ] }, "description": "", "meta": {}, "docs": { "show": true }, "patch_path": null, "arguments": [] } }, "docs": { "dbt.__overview__": { "unique_id": "dbt.__overview__", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "overview.md", "original_file_path": "docs/overview.md", "name": "__overview__", "block_contents": "### Welcome!\n\nWelcome to the auto-generated documentation for your dbt project!\n\n### Navigation\n\nYou can use the `Project` and `Database` navigation tabs on the left side of the window to explore the models\nin your project.\n\n#### Project Tab\nThe `Project` tab mirrors the directory structure of your dbt project. In this tab, you can see all of the\nmodels defined in your dbt project, as well as models imported from dbt packages.\n\n#### Database Tab\nThe `Database` tab also exposes your models, but in a format that looks more like a database explorer. This view\nshows relations (tables and views) grouped into database schemas. Note that ephemeral models are _not_ shown\nin this interface, as they do not exist in the database.\n\n### Graph Exploration\nYou can click the blue icon on the bottom-right corner of the page to view the lineage graph of your models.\n\nOn model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the `Expand`\nbutton at the top-right of this lineage pane, you'll be able to see all of the models that are used to build,\nor are built from, the model you're exploring.\n\nOnce expanded, you'll be able to use the `--models` and `--exclude` model selection syntax to filter the\nmodels in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax).\n\nNote that you can also right-click on models to interactively filter and explore the graph.\n\n---\n\n### More information\n\n- [What is dbt](https://docs.getdbt.com/docs/overview)?\n- Read the [dbt viewpoint](https://docs.getdbt.com/docs/viewpoint)\n- [Installation](https://docs.getdbt.com/docs/installation)\n- Join the [chat](https://community.getdbt.com/) on Slack for live questions and support." } }, "exposures": {}, "selectors": {}, "disabled": [], "parent_map": { "model.test.my_model": [] }, "child_map": { "model.test.my_model": [] } } ================================================ FILE: tests/functional/artifacts/data/state/v10/manifest.json ================================================ {"metadata": {"dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v10.json", "dbt_version": "1.6.6", "generated_at": "2023-10-11T20:49:37.080431Z", "invocation_id": "e2f630c5-769a-47a2-89ce-294a00e14e1a", "env": {}, "project_name": "test", "project_id": "098f6bcd4621d373cade4e832627b4f6", "user_id": null, "send_anonymous_usage_stats": false, "adapter_type": "postgres"}, "nodes": {"model.test.my_model": {"database": "dbt", "schema": "test16970573770617803847_test_previous_version_state", "name": "my_model", "resource_type": "model", "package_name": "test", "path": "my_model.sql", "original_file_path": "models/my_model.sql", "unique_id": "model.test.my_model", "fqn": ["test", "my_model"], "alias": "my_model", "checksum": {"name": "sha256", "checksum": "3ea0f972fa1b56aa2dc2f56ee784b6a5796312f9a813d59ae70fd8855f10d16d"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false}, "post-hook": [], "pre-hook": []}, "tags": [], "description": "Example model", "columns": {"id": {"name": "id", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": "test://models/schema.yml", "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1697057377.543413, "relation_name": "\"dbt\".\"test16970573770617803847_test_previous_version_state\".\"my_model\"", "raw_code": "select 1 as id", "language": "sql", "refs": [], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": null, "contract": {"enforced": false, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "model.test.metricflow_time_spine": {"database": "dbt", "schema": "test16970573770617803847_test_previous_version_state", "name": "metricflow_time_spine", "resource_type": "model", "package_name": "test", "path": "metricflow_time_spine.sql", "original_file_path": "models/metricflow_time_spine.sql", "unique_id": "model.test.metricflow_time_spine", "fqn": ["test", "metricflow_time_spine"], "alias": "metricflow_time_spine", "checksum": {"name": "sha256", "checksum": "954d9b349821edb5558a373119a7d91eeac9e620aaa96cd112c0d14bab729fdb"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false}, "post-hook": [], "pre-hook": []}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1697057377.456355, "relation_name": "\"dbt\".\"test16970573770617803847_test_previous_version_state\".\"metricflow_time_spine\"", "raw_code": "SELECT to_date('02/20/2023', 'mm/dd/yyyy') as date_day", "language": "sql", "refs": [], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": null, "contract": {"enforced": false, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "snapshot.test.snapshot_seed": {"database": "dbt", "schema": "test16970573770617803847_test_previous_version_state", "name": "snapshot_seed", "resource_type": "snapshot", "package_name": "test", "path": "snapshot_seed.sql", "original_file_path": "snapshots/snapshot_seed.sql", "unique_id": "snapshot.test.snapshot_seed", "fqn": ["test", "snapshot_seed", "snapshot_seed"], "alias": "snapshot_seed", "checksum": {"name": "sha256", "checksum": "5fc998f39655f8fe52443a919e749b6e23883ef90202b040412baac13c6bfe18"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "snapshot", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": "id", "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false}, "strategy": "check", "target_schema": "test16970573770617803847_test_previous_version_state", "target_database": null, "updated_at": null, "check_cols": "all", "post-hook": [], "pre-hook": []}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {"unique_key": "id", "strategy": "check", "check_cols": "all", "target_schema": "test16970573770617803847_test_previous_version_state"}, "created_at": 1697057377.471309, "relation_name": "\"dbt\".\"test16970573770617803847_test_previous_version_state\".\"snapshot_seed\"", "raw_code": "\n{{\n config(\n unique_key='id',\n strategy='check',\n check_cols='all',\n target_schema=schema,\n )\n}}\nselect * from {{ ref('my_seed') }}\n", "language": "sql", "refs": [{"name": "my_seed", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["seed.test.my_seed"]}, "compiled_path": null, "contract": {"enforced": false, "checksum": null}}, "analysis.test.a": {"database": "dbt", "schema": "test16970573770617803847_test_previous_version_state", "name": "a", "resource_type": "analysis", "package_name": "test", "path": "analysis/a.sql", "original_file_path": "analyses/a.sql", "unique_id": "analysis.test.a", "fqn": ["test", "analysis", "a"], "alias": "a", "checksum": {"name": "sha256", "checksum": "a389c282f569f0bbdc2a8a4f174dea746c28582fdaf2048d31d9226af9feab23"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false}, "post-hook": [], "pre-hook": []}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1697057377.492032, "relation_name": null, "raw_code": "select 4 as id", "language": "sql", "refs": [], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": null, "contract": {"enforced": false, "checksum": null}}, "test.test.just_my": {"database": "dbt", "schema": "test16970573770617803847_test_previous_version_state_dbt_test__audit", "name": "just_my", "resource_type": "test", "package_name": "test", "path": "just_my.sql", "original_file_path": "tests/just_my.sql", "unique_id": "test.test.just_my", "fqn": ["test", "just_my"], "alias": "just_my", "checksum": {"name": "sha256", "checksum": "744889a2e2d9ce380619265e1217d7ccf6e6ca896c048d42ebe0f9cfb74d7156"}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": ["data_test_tag"], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": ["data_test_tag"], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {"tags": ["data_test_tag"]}, "created_at": 1697057377.508335, "relation_name": null, "raw_code": "{{ config(tags = ['data_test_tag']) }}\n\nselect * from {{ ref('my_model') }}\nwhere false", "language": "sql", "refs": [{"name": "my_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["model.test.my_model"]}, "compiled_path": null, "contract": {"enforced": false, "checksum": null}}, "seed.test.my_seed": {"database": "dbt", "schema": "test16970573770617803847_test_previous_version_state", "name": "my_seed", "resource_type": "seed", "package_name": "test", "path": "my_seed.csv", "original_file_path": "seeds/my_seed.csv", "unique_id": "seed.test.my_seed", "fqn": ["test", "my_seed"], "alias": "my_seed", "checksum": {"name": "sha256", "checksum": "f7ede83f36165ac6b7a047aa2c3f212dff385bfa9f35f395108cd06fc8e96943"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "seed", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false}, "quote_columns": null, "post-hook": [], "pre-hook": []}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1697057377.525708, "relation_name": "\"dbt\".\"test16970573770617803847_test_previous_version_state\".\"my_seed\"", "raw_code": "", "root_path": "/private/var/folders/79/5290gpvn3lx5jdryk4844rm80000gn/T/pytest-of-quigleymalcolm/pytest-271/project0", "depends_on": {"macros": []}}, "test.test.not_null_my_model_id.43e0e9183a": {"test_metadata": {"name": "not_null", "kwargs": {"column_name": "id", "model": "{{ get_where_subquery(ref('my_model')) }}"}, "namespace": null}, "database": "dbt", "schema": "test16970573770617803847_test_previous_version_state_dbt_test__audit", "name": "not_null_my_model_id", "resource_type": "test", "package_name": "test", "path": "not_null_my_model_id.sql", "original_file_path": "models/schema.yml", "unique_id": "test.test.not_null_my_model_id.43e0e9183a", "fqn": ["test", "not_null_my_model_id"], "alias": "not_null_my_model_id", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1697057377.552852, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "my_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.test.my_model"]}, "compiled_path": null, "contract": {"enforced": false, "checksum": null}, "column_name": "id", "file_key_name": "models.my_model", "attached_node": "model.test.my_model"}, "test.test.check_nothing_my_model_.d5a5e66110": {"test_metadata": {"name": "check_nothing", "kwargs": {"model": "{{ get_where_subquery(ref('my_model')) }}"}, "namespace": null}, "database": "dbt", "schema": "test16970573770617803847_test_previous_version_state_dbt_test__audit", "name": "check_nothing_my_model_", "resource_type": "test", "package_name": "test", "path": "check_nothing_my_model_.sql", "original_file_path": "models/schema.yml", "unique_id": "test.test.check_nothing_my_model_.d5a5e66110", "fqn": ["test", "check_nothing_my_model_"], "alias": "check_nothing_my_model_", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1697057377.553834, "relation_name": null, "raw_code": "{{ test_check_nothing(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "my_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.test.test_check_nothing", "macro.dbt.get_where_subquery"], "nodes": ["model.test.my_model"]}, "compiled_path": null, "contract": {"enforced": false, "checksum": null}, "column_name": null, "file_key_name": "models.my_model", "attached_node": "model.test.my_model"}}, "sources": {"source.test.my_source.my_table": {"database": "dbt", "schema": "my_source", "name": "my_table", "resource_type": "source", "package_name": "test", "path": "models/schema.yml", "original_file_path": "models/schema.yml", "unique_id": "source.test.my_source.my_table", "fqn": ["test", "my_source", "my_table"], "source_name": "my_source", "source_description": "My source", "loader": "a_loader", "identifier": "my_seed", "quoting": {"database": null, "schema": null, "identifier": null, "column": null}, "loaded_at_field": null, "freshness": {"warn_after": {"count": null, "period": null}, "error_after": {"count": null, "period": null}, "filter": null}, "external": null, "description": "My table", "columns": {}, "meta": {}, "source_meta": {}, "tags": [], "config": {"enabled": true}, "patch_path": null, "unrendered_config": {}, "relation_name": "\"dbt\".\"my_source\".\"my_seed\"", "created_at": 1697057377.594166}}, "macros": {"macro.test.test_check_nothing": {"name": "test_check_nothing", "resource_type": "macro", "package_name": "test", "path": "macros/dummy_test.sql", "original_file_path": "macros/dummy_test.sql", "unique_id": "macro.test.test_check_nothing", "macro_sql": "{% test check_nothing(model) %}\n-- a silly test to make sure that table-level tests show up in the manifest\n-- without a column_name field\n\nselect 0\n\n{% endtest %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.099874, "supported_languages": null}, "macro.test.test_disabled_check_nothing": {"name": "test_disabled_check_nothing", "resource_type": "macro", "package_name": "test", "path": "macros/disabled_dummy_test.sql", "original_file_path": "macros/disabled_dummy_test.sql", "unique_id": "macro.test.test_disabled_check_nothing", "macro_sql": "{% test disabled_check_nothing(model) %}\n-- a silly test to make sure that table-level tests show up in the manifest\n-- without a column_name field\n\n{{ config(enabled=False) }}\nselect 0\n\n{% endtest %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.1000938, "supported_languages": null}, "macro.test.do_nothing": {"name": "do_nothing", "resource_type": "macro", "package_name": "test", "path": "macros/do_nothing.sql", "original_file_path": "macros/do_nothing.sql", "unique_id": "macro.test.do_nothing", "macro_sql": "{% macro do_nothing(foo2, bar2) %}\n select\n '{{ foo2 }}' as foo2,\n '{{ bar2 }}' as bar2\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.1002848, "supported_languages": null}, "macro.dbt_postgres.postgres__current_timestamp": {"name": "postgres__current_timestamp", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/timestamps.sql", "original_file_path": "macros/timestamps.sql", "unique_id": "macro.dbt_postgres.postgres__current_timestamp", "macro_sql": "{% macro postgres__current_timestamp() -%}\n now()\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.1005828, "supported_languages": null}, "macro.dbt_postgres.postgres__snapshot_string_as_time": {"name": "postgres__snapshot_string_as_time", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/timestamps.sql", "original_file_path": "macros/timestamps.sql", "unique_id": "macro.dbt_postgres.postgres__snapshot_string_as_time", "macro_sql": "{% macro postgres__snapshot_string_as_time(timestamp) -%}\n {%- set result = \"'\" ~ timestamp ~ \"'::timestamp without time zone\" -%}\n {{ return(result) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.10079, "supported_languages": null}, "macro.dbt_postgres.postgres__snapshot_get_time": {"name": "postgres__snapshot_get_time", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/timestamps.sql", "original_file_path": "macros/timestamps.sql", "unique_id": "macro.dbt_postgres.postgres__snapshot_get_time", "macro_sql": "{% macro postgres__snapshot_get_time() -%}\n {{ current_timestamp() }}::timestamp without time zone\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.1009028, "supported_languages": null}, "macro.dbt_postgres.postgres__current_timestamp_backcompat": {"name": "postgres__current_timestamp_backcompat", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/timestamps.sql", "original_file_path": "macros/timestamps.sql", "unique_id": "macro.dbt_postgres.postgres__current_timestamp_backcompat", "macro_sql": "{% macro postgres__current_timestamp_backcompat() %}\n current_timestamp::{{ type_timestamp() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.type_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.101016, "supported_languages": null}, "macro.dbt_postgres.postgres__current_timestamp_in_utc_backcompat": {"name": "postgres__current_timestamp_in_utc_backcompat", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/timestamps.sql", "original_file_path": "macros/timestamps.sql", "unique_id": "macro.dbt_postgres.postgres__current_timestamp_in_utc_backcompat", "macro_sql": "{% macro postgres__current_timestamp_in_utc_backcompat() %}\n (current_timestamp at time zone 'utc')::{{ type_timestamp() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.type_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.101125, "supported_languages": null}, "macro.dbt_postgres.postgres__get_catalog": {"name": "postgres__get_catalog", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/catalog.sql", "original_file_path": "macros/catalog.sql", "unique_id": "macro.dbt_postgres.postgres__get_catalog", "macro_sql": "{% macro postgres__get_catalog(information_schema, schemas) -%}\n\n {%- call statement('catalog', fetch_result=True) -%}\n {#\n If the user has multiple databases set and the first one is wrong, this will fail.\n But we won't fail in the case where there are multiple quoting-difference-only dbs, which is better.\n #}\n {% set database = information_schema.database %}\n {{ adapter.verify_database(database) }}\n\n select\n '{{ database }}' as table_database,\n sch.nspname as table_schema,\n tbl.relname as table_name,\n case tbl.relkind\n when 'v' then 'VIEW'\n else 'BASE TABLE'\n end as table_type,\n tbl_desc.description as table_comment,\n col.attname as column_name,\n col.attnum as column_index,\n pg_catalog.format_type(col.atttypid, col.atttypmod) as column_type,\n col_desc.description as column_comment,\n pg_get_userbyid(tbl.relowner) as table_owner\n\n from pg_catalog.pg_namespace sch\n join pg_catalog.pg_class tbl on tbl.relnamespace = sch.oid\n join pg_catalog.pg_attribute col on col.attrelid = tbl.oid\n left outer join pg_catalog.pg_description tbl_desc on (tbl_desc.objoid = tbl.oid and tbl_desc.objsubid = 0)\n left outer join pg_catalog.pg_description col_desc on (col_desc.objoid = tbl.oid and col_desc.objsubid = col.attnum)\n\n where (\n {%- for schema in schemas -%}\n upper(sch.nspname) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n )\n and not pg_is_other_temp_schema(sch.oid) -- not a temporary schema belonging to another session\n and tbl.relpersistence in ('p', 'u') -- [p]ermanent table or [u]nlogged table. Exclude [t]emporary tables\n and tbl.relkind in ('r', 'v', 'f', 'p') -- o[r]dinary table, [v]iew, [f]oreign table, [p]artitioned table. Other values are [i]ndex, [S]equence, [c]omposite type, [t]OAST table, [m]aterialized view\n and col.attnum > 0 -- negative numbers are used for system columns such as oid\n and not col.attisdropped -- column as not been dropped\n\n order by\n sch.nspname,\n tbl.relname,\n col.attnum\n\n {%- endcall -%}\n\n {{ return(load_result('catalog').table) }}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.1022131, "supported_languages": null}, "macro.dbt_postgres.postgres_get_relations": {"name": "postgres_get_relations", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations.sql", "original_file_path": "macros/relations.sql", "unique_id": "macro.dbt_postgres.postgres_get_relations", "macro_sql": "{% macro postgres_get_relations () -%}\n\n {#\n -- in pg_depend, objid is the dependent, refobjid is the referenced object\n -- > a pg_depend entry indicates that the referenced object cannot be\n -- > dropped without also dropping the dependent object.\n #}\n\n {%- call statement('relations', fetch_result=True) -%}\n with relation as (\n select\n pg_rewrite.ev_class as class,\n pg_rewrite.oid as id\n from pg_rewrite\n ),\n class as (\n select\n oid as id,\n relname as name,\n relnamespace as schema,\n relkind as kind\n from pg_class\n ),\n dependency as (\n select distinct\n pg_depend.objid as id,\n pg_depend.refobjid as ref\n from pg_depend\n ),\n schema as (\n select\n pg_namespace.oid as id,\n pg_namespace.nspname as name\n from pg_namespace\n where nspname != 'information_schema' and nspname not like 'pg\\_%'\n ),\n referenced as (\n select\n relation.id AS id,\n referenced_class.name ,\n referenced_class.schema ,\n referenced_class.kind\n from relation\n join class as referenced_class on relation.class=referenced_class.id\n where referenced_class.kind in ('r', 'v', 'm')\n ),\n relationships as (\n select\n referenced.name as referenced_name,\n referenced.schema as referenced_schema_id,\n dependent_class.name as dependent_name,\n dependent_class.schema as dependent_schema_id,\n referenced.kind as kind\n from referenced\n join dependency on referenced.id=dependency.id\n join class as dependent_class on dependency.ref=dependent_class.id\n where\n (referenced.name != dependent_class.name or\n referenced.schema != dependent_class.schema)\n )\n\n select\n referenced_schema.name as referenced_schema,\n relationships.referenced_name as referenced_name,\n dependent_schema.name as dependent_schema,\n relationships.dependent_name as dependent_name\n from relationships\n join schema as dependent_schema on relationships.dependent_schema_id=dependent_schema.id\n join schema as referenced_schema on relationships.referenced_schema_id=referenced_schema.id\n group by referenced_schema, referenced_name, dependent_schema, dependent_name\n order by referenced_schema, referenced_name, dependent_schema, dependent_name;\n\n {%- endcall -%}\n\n {{ return(load_result('relations').table) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.1028638, "supported_languages": null}, "macro.dbt_postgres.postgres__create_table_as": {"name": "postgres__create_table_as", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__create_table_as", "macro_sql": "{% macro postgres__create_table_as(temporary, relation, sql) -%}\n {%- set unlogged = config.get('unlogged', default=false) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary -%}\n temporary\n {%- elif unlogged -%}\n unlogged\n {%- endif %} table {{ relation }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced and (not temporary) %}\n {{ get_assert_columns_equivalent(sql) }}\n {{ get_table_columns_and_constraints() }} ;\n insert into {{ relation }} (\n {{ adapter.dispatch('get_column_names', 'dbt')() }}\n )\n {%- set sql = get_select_subquery(sql) %}\n {% else %}\n as\n {% endif %}\n (\n {{ sql }}\n );\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_assert_columns_equivalent", "macro.dbt.get_table_columns_and_constraints", "macro.dbt.default__get_column_names", "macro.dbt.get_select_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.1119502, "supported_languages": null}, "macro.dbt_postgres.postgres__get_create_index_sql": {"name": "postgres__get_create_index_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__get_create_index_sql", "macro_sql": "{% macro postgres__get_create_index_sql(relation, index_dict) -%}\n {%- set index_config = adapter.parse_index(index_dict) -%}\n {%- set comma_separated_columns = \", \".join(index_config.columns) -%}\n {%- set index_name = index_config.render(relation) -%}\n\n create {% if index_config.unique -%}\n unique\n {%- endif %} index if not exists\n \"{{ index_name }}\"\n on {{ relation }} {% if index_config.type -%}\n using {{ index_config.type }}\n {%- endif %}\n ({{ comma_separated_columns }});\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.112461, "supported_languages": null}, "macro.dbt_postgres.postgres__create_schema": {"name": "postgres__create_schema", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__create_schema", "macro_sql": "{% macro postgres__create_schema(relation) -%}\n {% if relation.database -%}\n {{ adapter.verify_database(relation.database) }}\n {%- endif -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier().include(database=False) }}\n {%- endcall -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.112787, "supported_languages": null}, "macro.dbt_postgres.postgres__drop_schema": {"name": "postgres__drop_schema", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__drop_schema", "macro_sql": "{% macro postgres__drop_schema(relation) -%}\n {% if relation.database -%}\n {{ adapter.verify_database(relation.database) }}\n {%- endif -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier().include(database=False) }} cascade\n {%- endcall -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.113112, "supported_languages": null}, "macro.dbt_postgres.postgres__get_columns_in_relation": {"name": "postgres__get_columns_in_relation", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__get_columns_in_relation", "macro_sql": "{% macro postgres__get_columns_in_relation(relation) -%}\n {% call statement('get_columns_in_relation', fetch_result=True) %}\n select\n column_name,\n data_type,\n character_maximum_length,\n numeric_precision,\n numeric_scale\n\n from {{ relation.information_schema('columns') }}\n where table_name = '{{ relation.identifier }}'\n {% if relation.schema %}\n and table_schema = '{{ relation.schema }}'\n {% endif %}\n order by ordinal_position\n\n {% endcall %}\n {% set table = load_result('get_columns_in_relation').table %}\n {{ return(sql_convert_columns_in_relation(table)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.sql_convert_columns_in_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.113596, "supported_languages": null}, "macro.dbt_postgres.postgres__list_relations_without_caching": {"name": "postgres__list_relations_without_caching", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__list_relations_without_caching", "macro_sql": "{% macro postgres__list_relations_without_caching(schema_relation) %}\n {% call statement('list_relations_without_caching', fetch_result=True) -%}\n select\n '{{ schema_relation.database }}' as database,\n tablename as name,\n schemaname as schema,\n 'table' as type\n from pg_tables\n where schemaname ilike '{{ schema_relation.schema }}'\n union all\n select\n '{{ schema_relation.database }}' as database,\n viewname as name,\n schemaname as schema,\n 'view' as type\n from pg_views\n where schemaname ilike '{{ schema_relation.schema }}'\n union all\n select\n '{{ schema_relation.database }}' as database,\n matviewname as name,\n schemaname as schema,\n 'materialized_view' as type\n from pg_matviews\n where schemaname ilike '{{ schema_relation.schema }}'\n {% endcall %}\n {{ return(load_result('list_relations_without_caching').table) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.114043, "supported_languages": null}, "macro.dbt_postgres.postgres__information_schema_name": {"name": "postgres__information_schema_name", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__information_schema_name", "macro_sql": "{% macro postgres__information_schema_name(database) -%}\n {% if database_name -%}\n {{ adapter.verify_database(database_name) }}\n {%- endif -%}\n information_schema\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.114221, "supported_languages": null}, "macro.dbt_postgres.postgres__list_schemas": {"name": "postgres__list_schemas", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__list_schemas", "macro_sql": "{% macro postgres__list_schemas(database) %}\n {% if database -%}\n {{ adapter.verify_database(database) }}\n {%- endif -%}\n {% call statement('list_schemas', fetch_result=True, auto_begin=False) %}\n select distinct nspname from pg_namespace\n {% endcall %}\n {{ return(load_result('list_schemas').table) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.1145759, "supported_languages": null}, "macro.dbt_postgres.postgres__check_schema_exists": {"name": "postgres__check_schema_exists", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__check_schema_exists", "macro_sql": "{% macro postgres__check_schema_exists(information_schema, schema) -%}\n {% if information_schema.database -%}\n {{ adapter.verify_database(information_schema.database) }}\n {%- endif -%}\n {% call statement('check_schema_exists', fetch_result=True, auto_begin=False) %}\n select count(*) from pg_namespace where nspname = '{{ schema }}'\n {% endcall %}\n {{ return(load_result('check_schema_exists').table) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.1149912, "supported_languages": null}, "macro.dbt_postgres.postgres__make_relation_with_suffix": {"name": "postgres__make_relation_with_suffix", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__make_relation_with_suffix", "macro_sql": "{% macro postgres__make_relation_with_suffix(base_relation, suffix, dstring) %}\n {% if dstring %}\n {% set dt = modules.datetime.datetime.now() %}\n {% set dtstring = dt.strftime(\"%H%M%S%f\") %}\n {% set suffix = suffix ~ dtstring %}\n {% endif %}\n {% set suffix_length = suffix|length %}\n {% set relation_max_name_length = base_relation.relation_max_name_length() %}\n {% if suffix_length > relation_max_name_length %}\n {% do exceptions.raise_compiler_error('Relation suffix is too long (' ~ suffix_length ~ ' characters). Maximum length is ' ~ relation_max_name_length ~ ' characters.') %}\n {% endif %}\n {% set identifier = base_relation.identifier[:relation_max_name_length - suffix_length] ~ suffix %}\n\n {{ return(base_relation.incorporate(path={\"identifier\": identifier })) }}\n\n {% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.1158679, "supported_languages": null}, "macro.dbt_postgres.postgres__make_intermediate_relation": {"name": "postgres__make_intermediate_relation", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__make_intermediate_relation", "macro_sql": "{% macro postgres__make_intermediate_relation(base_relation, suffix) %}\n {{ return(postgres__make_relation_with_suffix(base_relation, suffix, dstring=False)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_relation_with_suffix"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.1160781, "supported_languages": null}, "macro.dbt_postgres.postgres__make_temp_relation": {"name": "postgres__make_temp_relation", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__make_temp_relation", "macro_sql": "{% macro postgres__make_temp_relation(base_relation, suffix) %}\n {% set temp_relation = postgres__make_relation_with_suffix(base_relation, suffix, dstring=True) %}\n {{ return(temp_relation.incorporate(path={\"schema\": none,\n \"database\": none})) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_relation_with_suffix"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.116409, "supported_languages": null}, "macro.dbt_postgres.postgres__make_backup_relation": {"name": "postgres__make_backup_relation", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__make_backup_relation", "macro_sql": "{% macro postgres__make_backup_relation(base_relation, backup_relation_type, suffix) %}\n {% set backup_relation = postgres__make_relation_with_suffix(base_relation, suffix, dstring=False) %}\n {{ return(backup_relation.incorporate(type=backup_relation_type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_relation_with_suffix"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.116695, "supported_languages": null}, "macro.dbt_postgres.postgres_escape_comment": {"name": "postgres_escape_comment", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres_escape_comment", "macro_sql": "{% macro postgres_escape_comment(comment) -%}\n {% if comment is not string %}\n {% do exceptions.raise_compiler_error('cannot escape a non-string: ' ~ comment) %}\n {% endif %}\n {%- set magic = '$dbt_comment_literal_block$' -%}\n {%- if magic in comment -%}\n {%- do exceptions.raise_compiler_error('The string ' ~ magic ~ ' is not allowed in comments.') -%}\n {%- endif -%}\n {{ magic }}{{ comment }}{{ magic }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.117132, "supported_languages": null}, "macro.dbt_postgres.postgres__alter_relation_comment": {"name": "postgres__alter_relation_comment", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__alter_relation_comment", "macro_sql": "{% macro postgres__alter_relation_comment(relation, comment) %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n comment on {{ relation.type }} {{ relation }} is {{ escaped_comment }};\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres_escape_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.117368, "supported_languages": null}, "macro.dbt_postgres.postgres__alter_column_comment": {"name": "postgres__alter_column_comment", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__alter_column_comment", "macro_sql": "{% macro postgres__alter_column_comment(relation, column_dict) %}\n {% set existing_columns = adapter.get_columns_in_relation(relation) | map(attribute=\"name\") | list %}\n {% for column_name in column_dict if (column_name in existing_columns) %}\n {% set comment = column_dict[column_name]['description'] %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n comment on column {{ relation }}.{{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }} is {{ escaped_comment }};\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres_escape_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.117985, "supported_languages": null}, "macro.dbt_postgres.postgres__get_show_grant_sql": {"name": "postgres__get_show_grant_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__get_show_grant_sql", "macro_sql": "\n\n{%- macro postgres__get_show_grant_sql(relation) -%}\n select grantee, privilege_type\n from {{ relation.information_schema('role_table_grants') }}\n where grantor = current_role\n and grantee != current_role\n and table_schema = '{{ relation.schema }}'\n and table_name = '{{ relation.identifier }}'\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.118195, "supported_languages": null}, "macro.dbt_postgres.postgres__copy_grants": {"name": "postgres__copy_grants", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__copy_grants", "macro_sql": "{% macro postgres__copy_grants() %}\n {{ return(False) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.118315, "supported_languages": null}, "macro.dbt_postgres.postgres__get_show_indexes_sql": {"name": "postgres__get_show_indexes_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__get_show_indexes_sql", "macro_sql": "{% macro postgres__get_show_indexes_sql(relation) %}\n select\n i.relname as name,\n m.amname as method,\n ix.indisunique as \"unique\",\n array_to_string(array_agg(a.attname), ',') as column_names\n from pg_index ix\n join pg_class i\n on i.oid = ix.indexrelid\n join pg_am m\n on m.oid=i.relam\n join pg_class t\n on t.oid = ix.indrelid\n join pg_namespace n\n on n.oid = t.relnamespace\n join pg_attribute a\n on a.attrelid = t.oid\n and a.attnum = ANY(ix.indkey)\n where t.relname = '{{ relation.identifier }}'\n and n.nspname = '{{ relation.schema }}'\n and t.relkind in ('r', 'm')\n group by 1, 2, 3\n order by 1, 2, 3\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.118505, "supported_languages": null}, "macro.dbt_postgres.postgres__get_drop_index_sql": {"name": "postgres__get_drop_index_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__get_drop_index_sql", "macro_sql": "\n\n\n{%- macro postgres__get_drop_index_sql(relation, index_name) -%}\n drop index if exists \"{{ relation.schema }}\".\"{{ index_name }}\"\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.118647, "supported_languages": null}, "macro.dbt_postgres.postgres__get_alter_materialized_view_as_sql": {"name": "postgres__get_alter_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/materializations/materialized_view.sql", "original_file_path": "macros/materializations/materialized_view.sql", "unique_id": "macro.dbt_postgres.postgres__get_alter_materialized_view_as_sql", "macro_sql": "{% macro postgres__get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n\n -- apply a full refresh immediately if needed\n {% if configuration_changes.requires_full_refresh %}\n\n {{ get_replace_materialized_view_as_sql(relation, sql, existing_relation, backup_relation, intermediate_relation) }}\n\n -- otherwise apply individual changes as needed\n {% else %}\n\n {{ postgres__update_indexes_on_materialized_view(relation, configuration_changes.indexes) }}\n\n {%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_replace_materialized_view_as_sql", "macro.dbt_postgres.postgres__update_indexes_on_materialized_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.120726, "supported_languages": null}, "macro.dbt_postgres.postgres__get_create_materialized_view_as_sql": {"name": "postgres__get_create_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/materializations/materialized_view.sql", "original_file_path": "macros/materializations/materialized_view.sql", "unique_id": "macro.dbt_postgres.postgres__get_create_materialized_view_as_sql", "macro_sql": "{% macro postgres__get_create_materialized_view_as_sql(relation, sql) %}\n create materialized view if not exists {{ relation }} as {{ sql }};\n\n {% for _index_dict in config.get('indexes', []) -%}\n {{- get_create_index_sql(relation, _index_dict) -}}\n {%- endfor -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_create_index_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.121023, "supported_languages": null}, "macro.dbt_postgres.postgres__get_replace_materialized_view_as_sql": {"name": "postgres__get_replace_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/materializations/materialized_view.sql", "original_file_path": "macros/materializations/materialized_view.sql", "unique_id": "macro.dbt_postgres.postgres__get_replace_materialized_view_as_sql", "macro_sql": "{% macro postgres__get_replace_materialized_view_as_sql(relation, sql, existing_relation, backup_relation, intermediate_relation) %}\n {{- get_create_materialized_view_as_sql(intermediate_relation, sql) -}}\n\n {% if existing_relation is not none %}\n alter materialized view {{ existing_relation }} rename to {{ backup_relation.include(database=False, schema=False) }};\n {% endif %}\n\n alter materialized view {{ intermediate_relation }} rename to {{ relation.include(database=False, schema=False) }};\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_create_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.121473, "supported_languages": null}, "macro.dbt_postgres.postgres__get_materialized_view_configuration_changes": {"name": "postgres__get_materialized_view_configuration_changes", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/materializations/materialized_view.sql", "original_file_path": "macros/materializations/materialized_view.sql", "unique_id": "macro.dbt_postgres.postgres__get_materialized_view_configuration_changes", "macro_sql": "{% macro postgres__get_materialized_view_configuration_changes(existing_relation, new_config) %}\n {% set _existing_materialized_view = postgres__describe_materialized_view(existing_relation) %}\n {% set _configuration_changes = existing_relation.get_materialized_view_config_change_collection(_existing_materialized_view, new_config) %}\n {% do return(_configuration_changes) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__describe_materialized_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.121773, "supported_languages": null}, "macro.dbt_postgres.postgres__refresh_materialized_view": {"name": "postgres__refresh_materialized_view", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/materializations/materialized_view.sql", "original_file_path": "macros/materializations/materialized_view.sql", "unique_id": "macro.dbt_postgres.postgres__refresh_materialized_view", "macro_sql": "{% macro postgres__refresh_materialized_view(relation) %}\n refresh materialized view {{ relation }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.1218822, "supported_languages": null}, "macro.dbt_postgres.postgres__update_indexes_on_materialized_view": {"name": "postgres__update_indexes_on_materialized_view", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/materializations/materialized_view.sql", "original_file_path": "macros/materializations/materialized_view.sql", "unique_id": "macro.dbt_postgres.postgres__update_indexes_on_materialized_view", "macro_sql": "\n\n\n{%- macro postgres__update_indexes_on_materialized_view(relation, index_changes) -%}\n {{- log(\"Applying UPDATE INDEXES to: \" ~ relation) -}}\n\n {%- for _index_change in index_changes -%}\n {%- set _index = _index_change.context -%}\n\n {%- if _index_change.action == \"drop\" -%}\n\n {{ postgres__get_drop_index_sql(relation, _index.name) }};\n\n {%- elif _index_change.action == \"create\" -%}\n\n {{ postgres__get_create_index_sql(relation, _index.as_node_config) }}\n\n {%- endif -%}\n\n {%- endfor -%}\n\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_drop_index_sql", "macro.dbt_postgres.postgres__get_create_index_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.122449, "supported_languages": null}, "macro.dbt_postgres.postgres__describe_materialized_view": {"name": "postgres__describe_materialized_view", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/materializations/materialized_view.sql", "original_file_path": "macros/materializations/materialized_view.sql", "unique_id": "macro.dbt_postgres.postgres__describe_materialized_view", "macro_sql": "{% macro postgres__describe_materialized_view(relation) %}\n -- for now just get the indexes, we don't need the name or the query yet\n {% set _indexes = run_query(get_show_indexes_sql(relation)) %}\n {% do return({'indexes': _indexes}) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query", "macro.dbt.get_show_indexes_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.122707, "supported_languages": null}, "macro.dbt_postgres.postgres__get_incremental_default_sql": {"name": "postgres__get_incremental_default_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/materializations/incremental_strategies.sql", "original_file_path": "macros/materializations/incremental_strategies.sql", "unique_id": "macro.dbt_postgres.postgres__get_incremental_default_sql", "macro_sql": "{% macro postgres__get_incremental_default_sql(arg_dict) %}\n\n {% if arg_dict[\"unique_key\"] %}\n {% do return(get_incremental_delete_insert_sql(arg_dict)) %}\n {% else %}\n {% do return(get_incremental_append_sql(arg_dict)) %}\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_incremental_delete_insert_sql", "macro.dbt.get_incremental_append_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.123094, "supported_languages": null}, "macro.dbt_postgres.postgres__snapshot_merge_sql": {"name": "postgres__snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/materializations/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshot_merge.sql", "unique_id": "macro.dbt_postgres.postgres__snapshot_merge_sql", "macro_sql": "{% macro postgres__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n update {{ target }}\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n from {{ source }} as DBT_INTERNAL_SOURCE\n where DBT_INTERNAL_SOURCE.dbt_scd_id::text = {{ target }}.dbt_scd_id::text\n and DBT_INTERNAL_SOURCE.dbt_change_type::text in ('update'::text, 'delete'::text)\n and {{ target }}.dbt_valid_to is null;\n\n insert into {{ target }} ({{ insert_cols_csv }})\n select {% for column in insert_cols -%}\n DBT_INTERNAL_SOURCE.{{ column }} {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n from {{ source }} as DBT_INTERNAL_SOURCE\n where DBT_INTERNAL_SOURCE.dbt_change_type::text = 'insert'::text;\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.123816, "supported_languages": null}, "macro.dbt_postgres.postgres__dateadd": {"name": "postgres__dateadd", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt_postgres.postgres__dateadd", "macro_sql": "{% macro postgres__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n {{ from_date_or_timestamp }} + ((interval '1 {{ datepart }}') * ({{ interval }}))\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.124038, "supported_languages": null}, "macro.dbt_postgres.postgres__listagg": {"name": "postgres__listagg", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "unique_id": "macro.dbt_postgres.postgres__listagg", "macro_sql": "{% macro postgres__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n {% if limit_num -%}\n array_to_string(\n (array_agg(\n {{ measure }}\n {% if order_by_clause -%}\n {{ order_by_clause }}\n {%- endif %}\n ))[1:{{ limit_num }}],\n {{ delimiter_text }}\n )\n {%- else %}\n string_agg(\n {{ measure }},\n {{ delimiter_text }}\n {% if order_by_clause -%}\n {{ order_by_clause }}\n {%- endif %}\n )\n {%- endif %}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.124678, "supported_languages": null}, "macro.dbt_postgres.postgres__datediff": {"name": "postgres__datediff", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt_postgres.postgres__datediff", "macro_sql": "{% macro postgres__datediff(first_date, second_date, datepart) -%}\n\n {% if datepart == 'year' %}\n (date_part('year', ({{second_date}})::date) - date_part('year', ({{first_date}})::date))\n {% elif datepart == 'quarter' %}\n ({{ datediff(first_date, second_date, 'year') }} * 4 + date_part('quarter', ({{second_date}})::date) - date_part('quarter', ({{first_date}})::date))\n {% elif datepart == 'month' %}\n ({{ datediff(first_date, second_date, 'year') }} * 12 + date_part('month', ({{second_date}})::date) - date_part('month', ({{first_date}})::date))\n {% elif datepart == 'day' %}\n (({{second_date}})::date - ({{first_date}})::date)\n {% elif datepart == 'week' %}\n ({{ datediff(first_date, second_date, 'day') }} / 7 + case\n when date_part('dow', ({{first_date}})::timestamp) <= date_part('dow', ({{second_date}})::timestamp) then\n case when {{first_date}} <= {{second_date}} then 0 else -1 end\n else\n case when {{first_date}} <= {{second_date}} then 1 else 0 end\n end)\n {% elif datepart == 'hour' %}\n ({{ datediff(first_date, second_date, 'day') }} * 24 + date_part('hour', ({{second_date}})::timestamp) - date_part('hour', ({{first_date}})::timestamp))\n {% elif datepart == 'minute' %}\n ({{ datediff(first_date, second_date, 'hour') }} * 60 + date_part('minute', ({{second_date}})::timestamp) - date_part('minute', ({{first_date}})::timestamp))\n {% elif datepart == 'second' %}\n ({{ datediff(first_date, second_date, 'minute') }} * 60 + floor(date_part('second', ({{second_date}})::timestamp)) - floor(date_part('second', ({{first_date}})::timestamp)))\n {% elif datepart == 'millisecond' %}\n ({{ datediff(first_date, second_date, 'minute') }} * 60000 + floor(date_part('millisecond', ({{second_date}})::timestamp)) - floor(date_part('millisecond', ({{first_date}})::timestamp)))\n {% elif datepart == 'microsecond' %}\n ({{ datediff(first_date, second_date, 'minute') }} * 60000000 + floor(date_part('microsecond', ({{second_date}})::timestamp)) - floor(date_part('microsecond', ({{first_date}})::timestamp)))\n {% else %}\n {{ exceptions.raise_compiler_error(\"Unsupported datepart for macro datediff in postgres: {!r}\".format(datepart)) }}\n {% endif %}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.datediff"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.127991, "supported_languages": null}, "macro.dbt_postgres.postgres__any_value": {"name": "postgres__any_value", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "unique_id": "macro.dbt_postgres.postgres__any_value", "macro_sql": "{% macro postgres__any_value(expression) -%}\n\n min({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.128149, "supported_languages": null}, "macro.dbt_postgres.postgres__last_day": {"name": "postgres__last_day", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt_postgres.postgres__last_day", "macro_sql": "{% macro postgres__last_day(date, datepart) -%}\n\n {%- if datepart == 'quarter' -%}\n -- postgres dateadd does not support quarter interval.\n cast(\n {{dbt.dateadd('day', '-1',\n dbt.dateadd('month', '3', dbt.date_trunc(datepart, date))\n )}}\n as date)\n {%- else -%}\n {{dbt.default_last_day(date, datepart)}}\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.dateadd", "macro.dbt.date_trunc", "macro.dbt.default_last_day"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.1286578, "supported_languages": null}, "macro.dbt_postgres.postgres__split_part": {"name": "postgres__split_part", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt_postgres.postgres__split_part", "macro_sql": "{% macro postgres__split_part(string_text, delimiter_text, part_number) %}\n\n {% if part_number >= 0 %}\n {{ dbt.default__split_part(string_text, delimiter_text, part_number) }}\n {% else %}\n {{ dbt._split_part_negative(string_text, delimiter_text, part_number) }}\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__split_part", "macro.dbt._split_part_negative"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.129076, "supported_languages": null}, "macro.dbt.run_hooks": {"name": "run_hooks", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.run_hooks", "macro_sql": "{% macro run_hooks(hooks, inside_transaction=True) %}\n {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %}\n {% if not inside_transaction and loop.first %}\n {% call statement(auto_begin=inside_transaction) %}\n commit;\n {% endcall %}\n {% endif %}\n {% set rendered = render(hook.get('sql')) | trim %}\n {% if (rendered | length) > 0 %}\n {% call statement(auto_begin=inside_transaction) %}\n {{ rendered }}\n {% endcall %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.130199, "supported_languages": null}, "macro.dbt.make_hook_config": {"name": "make_hook_config", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.make_hook_config", "macro_sql": "{% macro make_hook_config(sql, inside_transaction) %}\n {{ tojson({\"sql\": sql, \"transaction\": inside_transaction}) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.130403, "supported_languages": null}, "macro.dbt.before_begin": {"name": "before_begin", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.before_begin", "macro_sql": "{% macro before_begin(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.130552, "supported_languages": null}, "macro.dbt.in_transaction": {"name": "in_transaction", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.in_transaction", "macro_sql": "{% macro in_transaction(sql) %}\n {{ make_hook_config(sql, inside_transaction=True) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.1307, "supported_languages": null}, "macro.dbt.after_commit": {"name": "after_commit", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.after_commit", "macro_sql": "{% macro after_commit(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.1308448, "supported_languages": null}, "macro.dbt.set_sql_header": {"name": "set_sql_header", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "unique_id": "macro.dbt.set_sql_header", "macro_sql": "{% macro set_sql_header(config) -%}\n {{ config.set('sql_header', caller()) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.131226, "supported_languages": null}, "macro.dbt.should_full_refresh": {"name": "should_full_refresh", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "unique_id": "macro.dbt.should_full_refresh", "macro_sql": "{% macro should_full_refresh() %}\n {% set config_full_refresh = config.get('full_refresh') %}\n {% if config_full_refresh is none %}\n {% set config_full_refresh = flags.FULL_REFRESH %}\n {% endif %}\n {% do return(config_full_refresh) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.131537, "supported_languages": null}, "macro.dbt.should_store_failures": {"name": "should_store_failures", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "unique_id": "macro.dbt.should_store_failures", "macro_sql": "{% macro should_store_failures() %}\n {% set config_store_failures = config.get('store_failures') %}\n {% if config_store_failures is none %}\n {% set config_store_failures = flags.STORE_FAILURES %}\n {% endif %}\n {% do return(config_store_failures) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.1318662, "supported_languages": null}, "macro.dbt.snapshot_merge_sql": {"name": "snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", "unique_id": "macro.dbt.snapshot_merge_sql", "macro_sql": "{% macro snapshot_merge_sql(target, source, insert_cols) -%}\n {{ adapter.dispatch('snapshot_merge_sql', 'dbt')(target, source, insert_cols) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__snapshot_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.13231, "supported_languages": null}, "macro.dbt.default__snapshot_merge_sql": {"name": "default__snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", "unique_id": "macro.dbt.default__snapshot_merge_sql", "macro_sql": "{% macro default__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on DBT_INTERNAL_SOURCE.dbt_scd_id = DBT_INTERNAL_DEST.dbt_scd_id\n\n when matched\n and DBT_INTERNAL_DEST.dbt_valid_to is null\n and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')\n then update\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n\n when not matched\n and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'\n then insert ({{ insert_cols_csv }})\n values ({{ insert_cols_csv }})\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.132591, "supported_languages": null}, "macro.dbt.strategy_dispatch": {"name": "strategy_dispatch", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.strategy_dispatch", "macro_sql": "{% macro strategy_dispatch(name) -%}\n{% set original_name = name %}\n {% if '.' in name %}\n {% set package_name, name = name.split(\".\", 1) %}\n {% else %}\n {% set package_name = none %}\n {% endif %}\n\n {% if package_name is none %}\n {% set package_context = context %}\n {% elif package_name in context %}\n {% set package_context = context[package_name] %}\n {% else %}\n {% set error_msg %}\n Could not find package '{{package_name}}', called with '{{original_name}}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n\n {%- set search_name = 'snapshot_' ~ name ~ '_strategy' -%}\n\n {% if search_name not in package_context %}\n {% set error_msg %}\n The specified strategy macro '{{name}}' was not found in package '{{ package_name }}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n {{ return(package_context[search_name]) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.136333, "supported_languages": null}, "macro.dbt.snapshot_hash_arguments": {"name": "snapshot_hash_arguments", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_hash_arguments", "macro_sql": "{% macro snapshot_hash_arguments(args) -%}\n {{ adapter.dispatch('snapshot_hash_arguments', 'dbt')(args) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.13651, "supported_languages": null}, "macro.dbt.default__snapshot_hash_arguments": {"name": "default__snapshot_hash_arguments", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.default__snapshot_hash_arguments", "macro_sql": "{% macro default__snapshot_hash_arguments(args) -%}\n md5({%- for arg in args -%}\n coalesce(cast({{ arg }} as varchar ), '')\n {% if not loop.last %} || '|' || {% endif %}\n {%- endfor -%})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.1367402, "supported_languages": null}, "macro.dbt.snapshot_timestamp_strategy": {"name": "snapshot_timestamp_strategy", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_timestamp_strategy", "macro_sql": "{% macro snapshot_timestamp_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}\n {% set primary_key = config['unique_key'] %}\n {% set updated_at = config['updated_at'] %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %}\n\n {#/*\n The snapshot relation might not have an {{ updated_at }} value if the\n snapshot strategy is changed from `check` to `timestamp`. We\n should use a dbt-created column for the comparison in the snapshot\n table instead of assuming that the user-supplied {{ updated_at }}\n will be present in the historical data.\n\n See https://github.com/dbt-labs/dbt-core/issues/2350\n */ #}\n {% set row_changed_expr -%}\n ({{ snapshotted_rel }}.dbt_valid_from < {{ current_rel }}.{{ updated_at }})\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.1374788, "supported_languages": null}, "macro.dbt.snapshot_string_as_time": {"name": "snapshot_string_as_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_string_as_time", "macro_sql": "{% macro snapshot_string_as_time(timestamp) -%}\n {{ adapter.dispatch('snapshot_string_as_time', 'dbt')(timestamp) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__snapshot_string_as_time"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.13765, "supported_languages": null}, "macro.dbt.default__snapshot_string_as_time": {"name": "default__snapshot_string_as_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.default__snapshot_string_as_time", "macro_sql": "{% macro default__snapshot_string_as_time(timestamp) %}\n {% do exceptions.raise_not_implemented(\n 'snapshot_string_as_time macro not implemented for adapter '+adapter.type()\n ) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.137829, "supported_languages": null}, "macro.dbt.snapshot_check_all_get_existing_columns": {"name": "snapshot_check_all_get_existing_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_check_all_get_existing_columns", "macro_sql": "{% macro snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) -%}\n {%- if not target_exists -%}\n {#-- no table yet -> return whatever the query does --#}\n {{ return((false, query_columns)) }}\n {%- endif -%}\n\n {#-- handle any schema changes --#}\n {%- set target_relation = adapter.get_relation(database=node.database, schema=node.schema, identifier=node.alias) -%}\n\n {% if check_cols_config == 'all' %}\n {%- set query_columns = get_columns_in_query(node['compiled_code']) -%}\n\n {% elif check_cols_config is iterable and (check_cols_config | length) > 0 %}\n {#-- query for proper casing/quoting, to support comparison below --#}\n {%- set select_check_cols_from_target -%}\n {#-- N.B. The whitespace below is necessary to avoid edge case issue with comments --#}\n {#-- See: https://github.com/dbt-labs/dbt-core/issues/6781 --#}\n select {{ check_cols_config | join(', ') }} from (\n {{ node['compiled_code'] }}\n ) subq\n {%- endset -%}\n {% set query_columns = get_columns_in_query(select_check_cols_from_target) %}\n\n {% else %}\n {% do exceptions.raise_compiler_error(\"Invalid value for 'check_cols': \" ~ check_cols_config) %}\n {% endif %}\n\n {%- set existing_cols = adapter.get_columns_in_relation(target_relation) | map(attribute = 'name') | list -%}\n {%- set ns = namespace() -%} {#-- handle for-loop scoping with a namespace --#}\n {%- set ns.column_added = false -%}\n\n {%- set intersection = [] -%}\n {%- for col in query_columns -%}\n {%- if col in existing_cols -%}\n {%- do intersection.append(adapter.quote(col)) -%}\n {%- else -%}\n {% set ns.column_added = true %}\n {%- endif -%}\n {%- endfor -%}\n {{ return((ns.column_added, intersection)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_columns_in_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.1392791, "supported_languages": null}, "macro.dbt.snapshot_check_strategy": {"name": "snapshot_check_strategy", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_check_strategy", "macro_sql": "{% macro snapshot_check_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}\n {% set check_cols_config = config['check_cols'] %}\n {% set primary_key = config['unique_key'] %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %}\n {% set updated_at = config.get('updated_at', snapshot_get_time()) %}\n\n {% set column_added = false %}\n\n {% set column_added, check_cols = snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) %}\n\n {%- set row_changed_expr -%}\n (\n {%- if column_added -%}\n {{ get_true_sql() }}\n {%- else -%}\n {%- for col in check_cols -%}\n {{ snapshotted_rel }}.{{ col }} != {{ current_rel }}.{{ col }}\n or\n (\n (({{ snapshotted_rel }}.{{ col }} is null) and not ({{ current_rel }}.{{ col }} is null))\n or\n ((not {{ snapshotted_rel }}.{{ col }} is null) and ({{ current_rel }}.{{ col }} is null))\n )\n {%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n {%- endif -%}\n )\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.snapshot_get_time", "macro.dbt.snapshot_check_all_get_existing_columns", "macro.dbt.get_true_sql", "macro.dbt.snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.140613, "supported_languages": null}, "macro.dbt.create_columns": {"name": "create_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.create_columns", "macro_sql": "{% macro create_columns(relation, columns) %}\n {{ adapter.dispatch('create_columns', 'dbt')(relation, columns) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__create_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.144871, "supported_languages": null}, "macro.dbt.default__create_columns": {"name": "default__create_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__create_columns", "macro_sql": "{% macro default__create_columns(relation, columns) %}\n {% for column in columns %}\n {% call statement() %}\n alter table {{ relation }} add column \"{{ column.name }}\" {{ column.data_type }};\n {% endcall %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.14516, "supported_languages": null}, "macro.dbt.post_snapshot": {"name": "post_snapshot", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.post_snapshot", "macro_sql": "{% macro post_snapshot(staging_relation) %}\n {{ adapter.dispatch('post_snapshot', 'dbt')(staging_relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.145334, "supported_languages": null}, "macro.dbt.default__post_snapshot": {"name": "default__post_snapshot", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__post_snapshot", "macro_sql": "{% macro default__post_snapshot(staging_relation) %}\n {# no-op #}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.1454248, "supported_languages": null}, "macro.dbt.get_true_sql": {"name": "get_true_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.get_true_sql", "macro_sql": "{% macro get_true_sql() %}\n {{ adapter.dispatch('get_true_sql', 'dbt')() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_true_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.1455739, "supported_languages": null}, "macro.dbt.default__get_true_sql": {"name": "default__get_true_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__get_true_sql", "macro_sql": "{% macro default__get_true_sql() %}\n {{ return('TRUE') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.145694, "supported_languages": null}, "macro.dbt.snapshot_staging_table": {"name": "snapshot_staging_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.snapshot_staging_table", "macro_sql": "{% macro snapshot_staging_table(strategy, source_sql, target_relation) -%}\n {{ adapter.dispatch('snapshot_staging_table', 'dbt')(strategy, source_sql, target_relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__snapshot_staging_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.145904, "supported_languages": null}, "macro.dbt.default__snapshot_staging_table": {"name": "default__snapshot_staging_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__snapshot_staging_table", "macro_sql": "{% macro default__snapshot_staging_table(strategy, source_sql, target_relation) -%}\n\n with snapshot_query as (\n\n {{ source_sql }}\n\n ),\n\n snapshotted_data as (\n\n select *,\n {{ strategy.unique_key }} as dbt_unique_key\n\n from {{ target_relation }}\n where dbt_valid_to is null\n\n ),\n\n insertions_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to,\n {{ strategy.scd_id }} as dbt_scd_id\n\n from snapshot_query\n ),\n\n updates_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n {{ strategy.updated_at }} as dbt_valid_to\n\n from snapshot_query\n ),\n\n {%- if strategy.invalidate_hard_deletes %}\n\n deletes_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key\n from snapshot_query\n ),\n {% endif %}\n\n insertions as (\n\n select\n 'insert' as dbt_change_type,\n source_data.*\n\n from insertions_source_data as source_data\n left outer join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where snapshotted_data.dbt_unique_key is null\n or (\n snapshotted_data.dbt_unique_key is not null\n and (\n {{ strategy.row_changed }}\n )\n )\n\n ),\n\n updates as (\n\n select\n 'update' as dbt_change_type,\n source_data.*,\n snapshotted_data.dbt_scd_id\n\n from updates_source_data as source_data\n join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where (\n {{ strategy.row_changed }}\n )\n )\n\n {%- if strategy.invalidate_hard_deletes -%}\n ,\n\n deletes as (\n\n select\n 'delete' as dbt_change_type,\n source_data.*,\n {{ snapshot_get_time() }} as dbt_valid_from,\n {{ snapshot_get_time() }} as dbt_updated_at,\n {{ snapshot_get_time() }} as dbt_valid_to,\n snapshotted_data.dbt_scd_id\n\n from snapshotted_data\n left join deletes_source_data as source_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where source_data.dbt_unique_key is null\n )\n {%- endif %}\n\n select * from insertions\n union all\n select * from updates\n {%- if strategy.invalidate_hard_deletes %}\n union all\n select * from deletes\n {%- endif %}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.snapshot_get_time"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.146819, "supported_languages": null}, "macro.dbt.build_snapshot_table": {"name": "build_snapshot_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.build_snapshot_table", "macro_sql": "{% macro build_snapshot_table(strategy, sql) -%}\n {{ adapter.dispatch('build_snapshot_table', 'dbt')(strategy, sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__build_snapshot_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.1470149, "supported_languages": null}, "macro.dbt.default__build_snapshot_table": {"name": "default__build_snapshot_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__build_snapshot_table", "macro_sql": "{% macro default__build_snapshot_table(strategy, sql) %}\n\n select *,\n {{ strategy.scd_id }} as dbt_scd_id,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to\n from (\n {{ sql }}\n ) sbq\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.147279, "supported_languages": null}, "macro.dbt.build_snapshot_staging_table": {"name": "build_snapshot_staging_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.build_snapshot_staging_table", "macro_sql": "{% macro build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% set temp_relation = make_temp_relation(target_relation) %}\n\n {% set select = snapshot_staging_table(strategy, sql, target_relation) %}\n\n {% call statement('build_snapshot_staging_relation') %}\n {{ create_table_as(True, temp_relation, select) }}\n {% endcall %}\n\n {% do return(temp_relation) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_temp_relation", "macro.dbt.snapshot_staging_table", "macro.dbt.statement", "macro.dbt.create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.147718, "supported_languages": null}, "macro.dbt.materialization_snapshot_default": {"name": "materialization_snapshot_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/snapshot.sql", "original_file_path": "macros/materializations/snapshots/snapshot.sql", "unique_id": "macro.dbt.materialization_snapshot_default", "macro_sql": "{% materialization snapshot, default %}\n {%- set config = model['config'] -%}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n -- grab current tables grants config for comparision later on\n {%- set grant_config = config.get('grants') -%}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=model.database,\n schema=model.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", config, target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_code']) %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% else %}\n\n {{ adapter.valid_snapshot_target(target_relation) }}\n\n {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% endif %}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode=False) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if not target_relation_exists %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if staging_table is defined %}\n {% do post_snapshot(staging_table) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.get_or_create_relation", "macro.dbt.run_hooks", "macro.dbt.strategy_dispatch", "macro.dbt.build_snapshot_table", "macro.dbt.create_table_as", "macro.dbt.build_snapshot_staging_table", "macro.dbt.create_columns", "macro.dbt.snapshot_merge_sql", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes", "macro.dbt.post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.154073, "supported_languages": ["sql"]}, "macro.dbt.materialization_test_default": {"name": "materialization_test_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/test.sql", "original_file_path": "macros/materializations/tests/test.sql", "unique_id": "macro.dbt.materialization_test_default", "macro_sql": "{%- materialization test, default -%}\n\n {% set relations = [] %}\n\n {% if should_store_failures() %}\n\n {% set identifier = model['alias'] %}\n {% set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n {% set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database, type='table') -%} %}\n\n {% if old_relation %}\n {% do adapter.drop_relation(old_relation) %}\n {% endif %}\n\n {% call statement(auto_begin=True) %}\n {{ create_table_as(False, target_relation, sql) }}\n {% endcall %}\n\n {% do relations.append(target_relation) %}\n\n {% set main_sql %}\n select *\n from {{ target_relation }}\n {% endset %}\n\n {{ adapter.commit() }}\n\n {% else %}\n\n {% set main_sql = sql %}\n\n {% endif %}\n\n {% set limit = config.get('limit') %}\n {% set fail_calc = config.get('fail_calc') %}\n {% set warn_if = config.get('warn_if') %}\n {% set error_if = config.get('error_if') %}\n\n {% call statement('main', fetch_result=True) -%}\n\n {{ get_test_sql(main_sql, fail_calc, warn_if, error_if, limit)}}\n\n {%- endcall %}\n\n {{ return({'relations': relations}) }}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.should_store_failures", "macro.dbt.statement", "macro.dbt.create_table_as", "macro.dbt.get_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.156199, "supported_languages": ["sql"]}, "macro.dbt.get_test_sql": {"name": "get_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.get_test_sql", "macro_sql": "{% macro get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n {{ adapter.dispatch('get_test_sql', 'dbt')(main_sql, fail_calc, warn_if, error_if, limit) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.156668, "supported_languages": null}, "macro.dbt.default__get_test_sql": {"name": "default__get_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.default__get_test_sql", "macro_sql": "{% macro default__get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n select\n {{ fail_calc }} as failures,\n {{ fail_calc }} {{ warn_if }} as should_warn,\n {{ fail_calc }} {{ error_if }} as should_error\n from (\n {{ main_sql }}\n {{ \"limit \" ~ limit if limit != none }}\n ) dbt_internal_test\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.156987, "supported_languages": null}, "macro.dbt.get_where_subquery": {"name": "get_where_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/where_subquery.sql", "original_file_path": "macros/materializations/tests/where_subquery.sql", "unique_id": "macro.dbt.get_where_subquery", "macro_sql": "{% macro get_where_subquery(relation) -%}\n {% do return(adapter.dispatch('get_where_subquery', 'dbt')(relation)) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_where_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.157378, "supported_languages": null}, "macro.dbt.default__get_where_subquery": {"name": "default__get_where_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/where_subquery.sql", "original_file_path": "macros/materializations/tests/where_subquery.sql", "unique_id": "macro.dbt.default__get_where_subquery", "macro_sql": "{% macro default__get_where_subquery(relation) -%}\n {% set where = config.get('where', '') %}\n {% if where %}\n {%- set filtered -%}\n (select * from {{ relation }} where {{ where }}) dbt_subquery\n {%- endset -%}\n {% do return(filtered) %}\n {%- else -%}\n {% do return(relation) %}\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.157763, "supported_languages": null}, "macro.dbt.get_quoted_csv": {"name": "get_quoted_csv", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.get_quoted_csv", "macro_sql": "{% macro get_quoted_csv(column_names) %}\n\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote(col)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.159425, "supported_languages": null}, "macro.dbt.diff_columns": {"name": "diff_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.diff_columns", "macro_sql": "{% macro diff_columns(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% set source_names = source_columns | map(attribute = 'column') | list %}\n {% set target_names = target_columns | map(attribute = 'column') | list %}\n\n {# --check whether the name attribute exists in the target - this does not perform a data type check #}\n {% for sc in source_columns %}\n {% if sc.name not in target_names %}\n {{ result.append(sc) }}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.1599932, "supported_languages": null}, "macro.dbt.diff_column_data_types": {"name": "diff_column_data_types", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.diff_column_data_types", "macro_sql": "{% macro diff_column_data_types(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% for sc in source_columns %}\n {% set tc = target_columns | selectattr(\"name\", \"equalto\", sc.name) | list | first %}\n {% if tc %}\n {% if sc.data_type != tc.data_type and not sc.can_expand_to(other_column=tc) %}\n {{ result.append( { 'column_name': tc.name, 'new_type': sc.data_type } ) }}\n {% endif %}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.160686, "supported_languages": null}, "macro.dbt.get_merge_update_columns": {"name": "get_merge_update_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.get_merge_update_columns", "macro_sql": "{% macro get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {{ return(adapter.dispatch('get_merge_update_columns', 'dbt')(merge_update_columns, merge_exclude_columns, dest_columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_merge_update_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.1609302, "supported_languages": null}, "macro.dbt.default__get_merge_update_columns": {"name": "default__get_merge_update_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.default__get_merge_update_columns", "macro_sql": "{% macro default__get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {%- set default_cols = dest_columns | map(attribute=\"quoted\") | list -%}\n\n {%- if merge_update_columns and merge_exclude_columns -%}\n {{ exceptions.raise_compiler_error(\n 'Model cannot specify merge_update_columns and merge_exclude_columns. Please update model to use only one config'\n )}}\n {%- elif merge_update_columns -%}\n {%- set update_columns = merge_update_columns -%}\n {%- elif merge_exclude_columns -%}\n {%- set update_columns = [] -%}\n {%- for column in dest_columns -%}\n {% if column.column | lower not in merge_exclude_columns | map(\"lower\") | list %}\n {%- do update_columns.append(column.quoted) -%}\n {% endif %}\n {%- endfor -%}\n {%- else -%}\n {%- set update_columns = default_cols -%}\n {%- endif -%}\n\n {{ return(update_columns) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.1617038, "supported_languages": null}, "macro.dbt.get_merge_sql": {"name": "get_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.get_merge_sql", "macro_sql": "{% macro get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) -%}\n -- back compat for old kwarg name\n {% set incremental_predicates = kwargs.get('predicates', incremental_predicates) %}\n {{ adapter.dispatch('get_merge_sql', 'dbt')(target, source, unique_key, dest_columns, incremental_predicates) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.168492, "supported_languages": null}, "macro.dbt.default__get_merge_sql": {"name": "default__get_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.default__get_merge_sql", "macro_sql": "{% macro default__get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) -%}\n {%- set predicates = [] if incremental_predicates is none else [] + incremental_predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set merge_update_columns = config.get('merge_update_columns') -%}\n {%- set merge_exclude_columns = config.get('merge_exclude_columns') -%}\n {%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}\n {% for key in unique_key %}\n {% set this_key_match %}\n DBT_INTERNAL_SOURCE.{{ key }} = DBT_INTERNAL_DEST.{{ key }}\n {% endset %}\n {% do predicates.append(this_key_match) %}\n {% endfor %}\n {% else %}\n {% set unique_key_match %}\n DBT_INTERNAL_SOURCE.{{ unique_key }} = DBT_INTERNAL_DEST.{{ unique_key }}\n {% endset %}\n {% do predicates.append(unique_key_match) %}\n {% endif %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n {{ sql_header if sql_header is not none }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{\"(\" ~ predicates | join(\") and (\") ~ \")\"}}\n\n {% if unique_key %}\n when matched then update set\n {% for column_name in update_columns -%}\n {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}\n {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n {% endif %}\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv", "macro.dbt.get_merge_update_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.170215, "supported_languages": null}, "macro.dbt.get_delete_insert_merge_sql": {"name": "get_delete_insert_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.get_delete_insert_merge_sql", "macro_sql": "{% macro get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}\n {{ adapter.dispatch('get_delete_insert_merge_sql', 'dbt')(target, source, unique_key, dest_columns, incremental_predicates) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_delete_insert_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.170489, "supported_languages": null}, "macro.dbt.default__get_delete_insert_merge_sql": {"name": "default__get_delete_insert_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.default__get_delete_insert_merge_sql", "macro_sql": "{% macro default__get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not string %}\n delete from {{target }}\n using {{ source }}\n where (\n {% for key in unique_key %}\n {{ source }}.{{ key }} = {{ target }}.{{ key }}\n {{ \"and \" if not loop.last}}\n {% endfor %}\n {% if incremental_predicates %}\n {% for predicate in incremental_predicates %}\n and {{ predicate }}\n {% endfor %}\n {% endif %}\n );\n {% else %}\n delete from {{ target }}\n where (\n {{ unique_key }}) in (\n select ({{ unique_key }})\n from {{ source }}\n )\n {%- if incremental_predicates %}\n {% for predicate in incremental_predicates %}\n and {{ predicate }}\n {% endfor %}\n {%- endif -%};\n\n {% endif %}\n {% endif %}\n\n insert into {{ target }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ source }}\n )\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.171521, "supported_languages": null}, "macro.dbt.get_insert_overwrite_merge_sql": {"name": "get_insert_overwrite_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.get_insert_overwrite_merge_sql", "macro_sql": "{% macro get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header=false) -%}\n {{ adapter.dispatch('get_insert_overwrite_merge_sql', 'dbt')(target, source, dest_columns, predicates, include_sql_header) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_insert_overwrite_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.171801, "supported_languages": null}, "macro.dbt.default__get_insert_overwrite_merge_sql": {"name": "default__get_insert_overwrite_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.default__get_insert_overwrite_merge_sql", "macro_sql": "{% macro default__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) -%}\n {#-- The only time include_sql_header is True: --#}\n {#-- BigQuery + insert_overwrite strategy + \"static\" partitions config --#}\n {#-- We should consider including the sql header at the materialization level instead --#}\n\n {%- set predicates = [] if predicates is none else [] + predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none and include_sql_header }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on FALSE\n\n when not matched by source\n {% if predicates %} and {{ predicates | join(' and ') }} {% endif %}\n then delete\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.172469, "supported_languages": null}, "macro.dbt.is_incremental": {"name": "is_incremental", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/is_incremental.sql", "original_file_path": "macros/materializations/models/incremental/is_incremental.sql", "unique_id": "macro.dbt.is_incremental", "macro_sql": "{% macro is_incremental() %}\n {#-- do not run introspective queries in parsing #}\n {% if not execute %}\n {{ return(False) }}\n {% else %}\n {% set relation = adapter.get_relation(this.database, this.schema, this.table) %}\n {{ return(relation is not none\n and relation.type == 'table'\n and model.config.materialized == 'incremental'\n and not should_full_refresh()) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.173121, "supported_languages": null}, "macro.dbt.get_incremental_append_sql": {"name": "get_incremental_append_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_append_sql", "macro_sql": "{% macro get_incremental_append_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_append_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_append_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.17403, "supported_languages": null}, "macro.dbt.default__get_incremental_append_sql": {"name": "default__get_incremental_append_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_append_sql", "macro_sql": "{% macro default__get_incremental_append_sql(arg_dict) %}\n\n {% do return(get_insert_into_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"dest_columns\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_insert_into_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.174277, "supported_languages": null}, "macro.dbt.get_incremental_delete_insert_sql": {"name": "get_incremental_delete_insert_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_delete_insert_sql", "macro_sql": "{% macro get_incremental_delete_insert_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_delete_insert_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_delete_insert_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.174471, "supported_languages": null}, "macro.dbt.default__get_incremental_delete_insert_sql": {"name": "default__get_incremental_delete_insert_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_delete_insert_sql", "macro_sql": "{% macro default__get_incremental_delete_insert_sql(arg_dict) %}\n\n {% do return(get_delete_insert_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"unique_key\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_delete_insert_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.174779, "supported_languages": null}, "macro.dbt.get_incremental_merge_sql": {"name": "get_incremental_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_merge_sql", "macro_sql": "{% macro get_incremental_merge_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_merge_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.174974, "supported_languages": null}, "macro.dbt.default__get_incremental_merge_sql": {"name": "default__get_incremental_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_merge_sql", "macro_sql": "{% macro default__get_incremental_merge_sql(arg_dict) %}\n\n {% do return(get_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"unique_key\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.17528, "supported_languages": null}, "macro.dbt.get_incremental_insert_overwrite_sql": {"name": "get_incremental_insert_overwrite_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_insert_overwrite_sql", "macro_sql": "{% macro get_incremental_insert_overwrite_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_insert_overwrite_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_insert_overwrite_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.175472, "supported_languages": null}, "macro.dbt.default__get_incremental_insert_overwrite_sql": {"name": "default__get_incremental_insert_overwrite_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_insert_overwrite_sql", "macro_sql": "{% macro default__get_incremental_insert_overwrite_sql(arg_dict) %}\n\n {% do return(get_insert_overwrite_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_insert_overwrite_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.175742, "supported_languages": null}, "macro.dbt.get_incremental_default_sql": {"name": "get_incremental_default_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_default_sql", "macro_sql": "{% macro get_incremental_default_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_default_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_incremental_default_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.175937, "supported_languages": null}, "macro.dbt.default__get_incremental_default_sql": {"name": "default__get_incremental_default_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_default_sql", "macro_sql": "{% macro default__get_incremental_default_sql(arg_dict) %}\n\n {% do return(get_incremental_append_sql(arg_dict)) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_incremental_append_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.176092, "supported_languages": null}, "macro.dbt.get_insert_into_sql": {"name": "get_insert_into_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_insert_into_sql", "macro_sql": "{% macro get_insert_into_sql(target_relation, temp_relation, dest_columns) %}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n insert into {{ target_relation }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ temp_relation }}\n )\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.176376, "supported_languages": null}, "macro.dbt.materialization_incremental_default": {"name": "materialization_incremental_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/incremental.sql", "original_file_path": "macros/materializations/models/incremental/incremental.sql", "unique_id": "macro.dbt.materialization_incremental_default", "macro_sql": "{% materialization incremental, default -%}\n\n -- relations\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') -%}\n {%- set temp_relation = make_temp_relation(target_relation)-%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation)-%}\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n\n -- configs\n {%- set unique_key = config.get('unique_key') -%}\n {%- set full_refresh_mode = (should_full_refresh() or existing_relation.is_view) -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n\n -- the temp_ and backup_ relations should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation. This has to happen before\n -- BEGIN, in a separate transaction\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation)-%}\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set to_drop = [] %}\n\n {% if existing_relation is none %}\n {% set build_sql = get_create_table_as_sql(False, target_relation, sql) %}\n {% elif full_refresh_mode %}\n {% set build_sql = get_create_table_as_sql(False, intermediate_relation, sql) %}\n {% set need_swap = true %}\n {% else %}\n {% do run_query(get_create_table_as_sql(True, temp_relation, sql)) %}\n {% do adapter.expand_target_column_types(\n from_relation=temp_relation,\n to_relation=target_relation) %}\n {#-- Process schema changes. Returns dict of changes if successful. Use source columns for upserting/merging --#}\n {% set dest_columns = process_schema_changes(on_schema_change, temp_relation, existing_relation) %}\n {% if not dest_columns %}\n {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %}\n {% endif %}\n\n {#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#}\n {% set incremental_strategy = config.get('incremental_strategy') or 'default' %}\n {% set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) %}\n {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %}\n {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': temp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'incremental_predicates': incremental_predicates }) %}\n {% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %}\n\n {% endif %}\n\n {% call statement(\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% if need_swap %}\n {% do adapter.rename_relation(target_relation, backup_relation) %}\n {% do adapter.rename_relation(intermediate_relation, target_relation) %}\n {% do to_drop.append(backup_relation) %}\n {% endif %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if existing_relation is none or existing_relation.is_view or should_full_refresh() %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {% do adapter.commit() %}\n\n {% for rel in to_drop %}\n {% do adapter.drop_relation(rel) %}\n {% endfor %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_temp_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.should_full_refresh", "macro.dbt.incremental_validate_on_schema_change", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks", "macro.dbt.get_create_table_as_sql", "macro.dbt.run_query", "macro.dbt.process_schema_changes", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.1815941, "supported_languages": ["sql"]}, "macro.dbt.incremental_validate_on_schema_change": {"name": "incremental_validate_on_schema_change", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.incremental_validate_on_schema_change", "macro_sql": "{% macro incremental_validate_on_schema_change(on_schema_change, default='ignore') %}\n\n {% if on_schema_change not in ['sync_all_columns', 'append_new_columns', 'fail', 'ignore'] %}\n\n {% set log_message = 'Invalid value for on_schema_change (%s) specified. Setting default value of %s.' % (on_schema_change, default) %}\n {% do log(log_message) %}\n\n {{ return(default) }}\n\n {% else %}\n\n {{ return(on_schema_change) }}\n\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.1874628, "supported_languages": null}, "macro.dbt.check_for_schema_changes": {"name": "check_for_schema_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.check_for_schema_changes", "macro_sql": "{% macro check_for_schema_changes(source_relation, target_relation) %}\n\n {% set schema_changed = False %}\n\n {%- set source_columns = adapter.get_columns_in_relation(source_relation) -%}\n {%- set target_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set source_not_in_target = diff_columns(source_columns, target_columns) -%}\n {%- set target_not_in_source = diff_columns(target_columns, source_columns) -%}\n\n {% set new_target_types = diff_column_data_types(source_columns, target_columns) %}\n\n {% if source_not_in_target != [] %}\n {% set schema_changed = True %}\n {% elif target_not_in_source != [] or new_target_types != [] %}\n {% set schema_changed = True %}\n {% elif new_target_types != [] %}\n {% set schema_changed = True %}\n {% endif %}\n\n {% set changes_dict = {\n 'schema_changed': schema_changed,\n 'source_not_in_target': source_not_in_target,\n 'target_not_in_source': target_not_in_source,\n 'source_columns': source_columns,\n 'target_columns': target_columns,\n 'new_target_types': new_target_types\n } %}\n\n {% set msg %}\n In {{ target_relation }}:\n Schema changed: {{ schema_changed }}\n Source columns not in target: {{ source_not_in_target }}\n Target columns not in source: {{ target_not_in_source }}\n New column types: {{ new_target_types }}\n {% endset %}\n\n {% do log(msg) %}\n\n {{ return(changes_dict) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.diff_columns", "macro.dbt.diff_column_data_types"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.188724, "supported_languages": null}, "macro.dbt.sync_column_schemas": {"name": "sync_column_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.sync_column_schemas", "macro_sql": "{% macro sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {%- set add_to_target_arr = schema_changes_dict['source_not_in_target'] -%}\n\n {%- if on_schema_change == 'append_new_columns'-%}\n {%- if add_to_target_arr | length > 0 -%}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, none) -%}\n {%- endif -%}\n\n {% elif on_schema_change == 'sync_all_columns' %}\n {%- set remove_from_target_arr = schema_changes_dict['target_not_in_source'] -%}\n {%- set new_target_types = schema_changes_dict['new_target_types'] -%}\n\n {% if add_to_target_arr | length > 0 or remove_from_target_arr | length > 0 %}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, remove_from_target_arr) -%}\n {% endif %}\n\n {% if new_target_types != [] %}\n {% for ntt in new_target_types %}\n {% set column_name = ntt['column_name'] %}\n {% set new_type = ntt['new_type'] %}\n {% do alter_column_type(target_relation, column_name, new_type) %}\n {% endfor %}\n {% endif %}\n\n {% endif %}\n\n {% set schema_change_message %}\n In {{ target_relation }}:\n Schema change approach: {{ on_schema_change }}\n Columns added: {{ add_to_target_arr }}\n Columns removed: {{ remove_from_target_arr }}\n Data types changed: {{ new_target_types }}\n {% endset %}\n\n {% do log(schema_change_message) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.alter_relation_add_remove_columns", "macro.dbt.alter_column_type"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.189962, "supported_languages": null}, "macro.dbt.process_schema_changes": {"name": "process_schema_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.process_schema_changes", "macro_sql": "{% macro process_schema_changes(on_schema_change, source_relation, target_relation) %}\n\n {% if on_schema_change == 'ignore' %}\n\n {{ return({}) }}\n\n {% else %}\n\n {% set schema_changes_dict = check_for_schema_changes(source_relation, target_relation) %}\n\n {% if schema_changes_dict['schema_changed'] %}\n\n {% if on_schema_change == 'fail' %}\n\n {% set fail_msg %}\n The source and target schemas on this incremental model are out of sync!\n They can be reconciled in several ways:\n - set the `on_schema_change` config to either append_new_columns or sync_all_columns, depending on your situation.\n - Re-run the incremental model with `full_refresh: True` to update the target schema.\n - update the schema manually and re-run the process.\n\n Additional troubleshooting context:\n Source columns not in target: {{ schema_changes_dict['source_not_in_target'] }}\n Target columns not in source: {{ schema_changes_dict['target_not_in_source'] }}\n New column types: {{ schema_changes_dict['new_target_types'] }}\n {% endset %}\n\n {% do exceptions.raise_compiler_error(fail_msg) %}\n\n {# -- unless we ignore, run the sync operation per the config #}\n {% else %}\n\n {% do sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {% endif %}\n\n {% endif %}\n\n {{ return(schema_changes_dict['source_columns']) }}\n\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.check_for_schema_changes", "macro.dbt.sync_column_schemas"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.1908412, "supported_languages": null}, "macro.dbt.materialization_materialized_view_default": {"name": "materialization_materialized_view_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view/materialized_view.sql", "unique_id": "macro.dbt.materialization_materialized_view_default", "macro_sql": "{% materialization materialized_view, default %}\n {% set existing_relation = load_cached_relation(this) %}\n {% set target_relation = this.incorporate(type=this.MaterializedView) %}\n {% set intermediate_relation = make_intermediate_relation(target_relation) %}\n {% set backup_relation_type = target_relation.MaterializedView if existing_relation is none else existing_relation.type %}\n {% set backup_relation = make_backup_relation(target_relation, backup_relation_type) %}\n\n {{ materialized_view_setup(backup_relation, intermediate_relation, pre_hooks) }}\n\n {% set build_sql = materialized_view_get_build_sql(existing_relation, target_relation, backup_relation, intermediate_relation) %}\n\n {% if build_sql == '' %}\n {{ materialized_view_execute_no_op(target_relation) }}\n {% else %}\n {{ materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) }}\n {% endif %}\n\n {{ materialized_view_teardown(backup_relation, intermediate_relation, post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.materialized_view_setup", "macro.dbt.materialized_view_get_build_sql", "macro.dbt.materialized_view_execute_no_op", "macro.dbt.materialized_view_execute_build_sql", "macro.dbt.materialized_view_teardown"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.196028, "supported_languages": ["sql"]}, "macro.dbt.materialized_view_setup": {"name": "materialized_view_setup", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_setup", "macro_sql": "{% macro materialized_view_setup(backup_relation, intermediate_relation, pre_hooks) %}\n\n -- backup_relation and intermediate_relation should not already exist in the database\n -- it's possible these exist because of a previous run that exited unexpectedly\n {% set preexisting_backup_relation = load_cached_relation(backup_relation) %}\n {% set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) %}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.1964319, "supported_languages": null}, "macro.dbt.materialized_view_teardown": {"name": "materialized_view_teardown", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_teardown", "macro_sql": "{% macro materialized_view_teardown(backup_relation, intermediate_relation, post_hooks) %}\n\n -- drop the temp relations if they exist to leave the database clean for the next run\n {{ drop_relation_if_exists(backup_relation) }}\n {{ drop_relation_if_exists(intermediate_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.196692, "supported_languages": null}, "macro.dbt.materialized_view_get_build_sql": {"name": "materialized_view_get_build_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_get_build_sql", "macro_sql": "{% macro materialized_view_get_build_sql(existing_relation, target_relation, backup_relation, intermediate_relation) %}\n\n {% set full_refresh_mode = should_full_refresh() %}\n\n -- determine the scenario we're in: create, full_refresh, alter, refresh data\n {% if existing_relation is none %}\n {% set build_sql = get_create_materialized_view_as_sql(target_relation, sql) %}\n {% elif full_refresh_mode or not existing_relation.is_materialized_view %}\n {% set build_sql = get_replace_materialized_view_as_sql(target_relation, sql, existing_relation, backup_relation, intermediate_relation) %}\n {% else %}\n\n -- get config options\n {% set on_configuration_change = config.get('on_configuration_change') %}\n {% set configuration_changes = get_materialized_view_configuration_changes(existing_relation, config) %}\n\n {% if configuration_changes is none %}\n {% set build_sql = refresh_materialized_view(target_relation) %}\n\n {% elif on_configuration_change == 'apply' %}\n {% set build_sql = get_alter_materialized_view_as_sql(target_relation, configuration_changes, sql, existing_relation, backup_relation, intermediate_relation) %}\n {% elif on_configuration_change == 'continue' %}\n {% set build_sql = '' %}\n {{ exceptions.warn(\"Configuration changes were identified and `on_configuration_change` was set to `continue` for `\" ~ target_relation ~ \"`\") }}\n {% elif on_configuration_change == 'fail' %}\n {{ exceptions.raise_fail_fast_error(\"Configuration changes were identified and `on_configuration_change` was set to `fail` for `\" ~ target_relation ~ \"`\") }}\n\n {% else %}\n -- this only happens if the user provides a value other than `apply`, 'skip', 'fail'\n {{ exceptions.raise_compiler_error(\"Unexpected configuration scenario\") }}\n\n {% endif %}\n\n {% endif %}\n\n {% do return(build_sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.get_create_materialized_view_as_sql", "macro.dbt.get_replace_materialized_view_as_sql", "macro.dbt.get_materialized_view_configuration_changes", "macro.dbt.refresh_materialized_view", "macro.dbt.get_alter_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.1980631, "supported_languages": null}, "macro.dbt.materialized_view_execute_no_op": {"name": "materialized_view_execute_no_op", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_execute_no_op", "macro_sql": "{% macro materialized_view_execute_no_op(target_relation) %}\n {% do store_raw_result(\n name=\"main\",\n message=\"skip \" ~ target_relation,\n code=\"skip\",\n rows_affected=\"-1\"\n ) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.198303, "supported_languages": null}, "macro.dbt.materialized_view_execute_build_sql": {"name": "materialized_view_execute_build_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_execute_build_sql", "macro_sql": "{% macro materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) %}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set grant_config = config.get('grants') %}\n\n {% call statement(name=\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.198978, "supported_languages": null}, "macro.dbt.get_materialized_view_configuration_changes": {"name": "get_materialized_view_configuration_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view/get_materialized_view_configuration_changes.sql", "original_file_path": "macros/materializations/models/materialized_view/get_materialized_view_configuration_changes.sql", "unique_id": "macro.dbt.get_materialized_view_configuration_changes", "macro_sql": "{% macro get_materialized_view_configuration_changes(existing_relation, new_config) %}\n /* {#\n It's recommended that configuration changes be formatted as follows:\n {\"\": [{\"action\": \"\", \"context\": ...}]}\n\n For example:\n {\n \"indexes\": [\n {\"action\": \"drop\", \"context\": \"index_abc\"},\n {\"action\": \"create\", \"context\": {\"columns\": [\"column_1\", \"column_2\"], \"type\": \"hash\", \"unique\": True}},\n ],\n }\n\n Either way, `get_materialized_view_configuration_changes` needs to align with `get_alter_materialized_view_as_sql`.\n #} */\n {{- log('Determining configuration changes on: ' ~ existing_relation) -}}\n {%- do return(adapter.dispatch('get_materialized_view_configuration_changes', 'dbt')(existing_relation, new_config)) -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_materialized_view_configuration_changes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.19941, "supported_languages": null}, "macro.dbt.default__get_materialized_view_configuration_changes": {"name": "default__get_materialized_view_configuration_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view/get_materialized_view_configuration_changes.sql", "original_file_path": "macros/materializations/models/materialized_view/get_materialized_view_configuration_changes.sql", "unique_id": "macro.dbt.default__get_materialized_view_configuration_changes", "macro_sql": "{% macro default__get_materialized_view_configuration_changes(existing_relation, new_config) %}\n {{ exceptions.raise_compiler_error(\"Materialized views have not been implemented for this adapter.\") }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.19956, "supported_languages": null}, "macro.dbt.get_alter_materialized_view_as_sql": {"name": "get_alter_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view/alter_materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view/alter_materialized_view.sql", "unique_id": "macro.dbt.get_alter_materialized_view_as_sql", "macro_sql": "{% macro get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n {{- log('Applying ALTER to: ' ~ relation) -}}\n {{- adapter.dispatch('get_alter_materialized_view_as_sql', 'dbt')(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n ) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_alter_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.200053, "supported_languages": null}, "macro.dbt.default__get_alter_materialized_view_as_sql": {"name": "default__get_alter_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view/alter_materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view/alter_materialized_view.sql", "unique_id": "macro.dbt.default__get_alter_materialized_view_as_sql", "macro_sql": "{% macro default__get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n {{ exceptions.raise_compiler_error(\"Materialized views have not been implemented for this adapter.\") }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.200245, "supported_languages": null}, "macro.dbt.refresh_materialized_view": {"name": "refresh_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view/refresh_materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view/refresh_materialized_view.sql", "unique_id": "macro.dbt.refresh_materialized_view", "macro_sql": "{% macro refresh_materialized_view(relation) %}\n {{- log('Applying REFRESH to: ' ~ relation) -}}\n {{- adapter.dispatch('refresh_materialized_view', 'dbt')(relation) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__refresh_materialized_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2005591, "supported_languages": null}, "macro.dbt.default__refresh_materialized_view": {"name": "default__refresh_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view/refresh_materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view/refresh_materialized_view.sql", "unique_id": "macro.dbt.default__refresh_materialized_view", "macro_sql": "{% macro default__refresh_materialized_view(relation) %}\n {{ exceptions.raise_compiler_error(\"Materialized views have not been implemented for this adapter.\") }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2006972, "supported_languages": null}, "macro.dbt.get_replace_materialized_view_as_sql": {"name": "get_replace_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view/replace_materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view/replace_materialized_view.sql", "unique_id": "macro.dbt.get_replace_materialized_view_as_sql", "macro_sql": "{% macro get_replace_materialized_view_as_sql(relation, sql, existing_relation, backup_relation, intermediate_relation) %}\n {{- log('Applying REPLACE to: ' ~ relation) -}}\n {{- adapter.dispatch('get_replace_materialized_view_as_sql', 'dbt')(relation, sql, existing_relation, backup_relation, intermediate_relation) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_replace_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2011251, "supported_languages": null}, "macro.dbt.default__get_replace_materialized_view_as_sql": {"name": "default__get_replace_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view/replace_materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view/replace_materialized_view.sql", "unique_id": "macro.dbt.default__get_replace_materialized_view_as_sql", "macro_sql": "{% macro default__get_replace_materialized_view_as_sql(relation, sql, existing_relation, backup_relation, intermediate_relation) %}\n {{ exceptions.raise_compiler_error(\"Materialized views have not been implemented for this adapter.\") }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2013052, "supported_languages": null}, "macro.dbt.get_create_materialized_view_as_sql": {"name": "get_create_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view/create_materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view/create_materialized_view.sql", "unique_id": "macro.dbt.get_create_materialized_view_as_sql", "macro_sql": "{% macro get_create_materialized_view_as_sql(relation, sql) -%}\n {{- log('Applying CREATE to: ' ~ relation) -}}\n {{- adapter.dispatch('get_create_materialized_view_as_sql', 'dbt')(relation, sql) -}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_create_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.201658, "supported_languages": null}, "macro.dbt.default__get_create_materialized_view_as_sql": {"name": "default__get_create_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view/create_materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view/create_materialized_view.sql", "unique_id": "macro.dbt.default__get_create_materialized_view_as_sql", "macro_sql": "{% macro default__get_create_materialized_view_as_sql(relation, sql) -%}\n {{ exceptions.raise_compiler_error(\"Materialized views have not been implemented for this adapter.\") }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.201805, "supported_languages": null}, "macro.dbt.can_clone_table": {"name": "can_clone_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/can_clone_table.sql", "original_file_path": "macros/materializations/models/clone/can_clone_table.sql", "unique_id": "macro.dbt.can_clone_table", "macro_sql": "{% macro can_clone_table() %}\n {{ return(adapter.dispatch('can_clone_table', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__can_clone_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.202045, "supported_languages": null}, "macro.dbt.default__can_clone_table": {"name": "default__can_clone_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/can_clone_table.sql", "original_file_path": "macros/materializations/models/clone/can_clone_table.sql", "unique_id": "macro.dbt.default__can_clone_table", "macro_sql": "{% macro default__can_clone_table() %}\n {{ return(False) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.202231, "supported_languages": null}, "macro.dbt.create_or_replace_clone": {"name": "create_or_replace_clone", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/create_or_replace_clone.sql", "original_file_path": "macros/materializations/models/clone/create_or_replace_clone.sql", "unique_id": "macro.dbt.create_or_replace_clone", "macro_sql": "{% macro create_or_replace_clone(this_relation, defer_relation) %}\n {{ return(adapter.dispatch('create_or_replace_clone', 'dbt')(this_relation, defer_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__create_or_replace_clone"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.202538, "supported_languages": null}, "macro.dbt.default__create_or_replace_clone": {"name": "default__create_or_replace_clone", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/create_or_replace_clone.sql", "original_file_path": "macros/materializations/models/clone/create_or_replace_clone.sql", "unique_id": "macro.dbt.default__create_or_replace_clone", "macro_sql": "{% macro default__create_or_replace_clone(this_relation, defer_relation) %}\n create or replace table {{ this_relation }} clone {{ defer_relation }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.202677, "supported_languages": null}, "macro.dbt.materialization_clone_default": {"name": "materialization_clone_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/clone.sql", "original_file_path": "macros/materializations/models/clone/clone.sql", "unique_id": "macro.dbt.materialization_clone_default", "macro_sql": "{%- materialization clone, default -%}\n\n {%- set relations = {'relations': []} -%}\n\n {%- if not defer_relation -%}\n -- nothing to do\n {{ log(\"No relation found in state manifest for \" ~ model.unique_id, info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n\n {%- if existing_relation and not flags.FULL_REFRESH -%}\n -- noop!\n {{ log(\"Relation \" ~ existing_relation ~ \" already exists\", info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set other_existing_relation = load_cached_relation(defer_relation) -%}\n\n -- If this is a database that can do zero-copy cloning of tables, and the other relation is a table, then this will be a table\n -- Otherwise, this will be a view\n\n {% set can_clone_table = can_clone_table() %}\n\n {%- if other_existing_relation and other_existing_relation.type == 'table' and can_clone_table -%}\n\n {%- set target_relation = this.incorporate(type='table') -%}\n {% if existing_relation is not none and not existing_relation.is_table %}\n {{ log(\"Dropping relation \" ~ existing_relation ~ \" because it is of type \" ~ existing_relation.type) }}\n {{ drop_relation_if_exists(existing_relation) }}\n {% endif %}\n\n -- as a general rule, data platforms that can clone tables can also do atomic 'create or replace'\n {% call statement('main') %}\n {{ create_or_replace_clone(target_relation, defer_relation) }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n {% do persist_docs(target_relation, model) %}\n\n {{ return({'relations': [target_relation]}) }}\n\n {%- else -%}\n\n {%- set target_relation = this.incorporate(type='view') -%}\n\n -- reuse the view materialization\n -- TODO: support actual dispatch for materialization macros\n -- Tracking ticket: https://github.com/dbt-labs/dbt-core/issues/7799\n {% set search_name = \"materialization_view_\" ~ adapter.type() %}\n {% if not search_name in context %}\n {% set search_name = \"materialization_view_default\" %}\n {% endif %}\n {% set materialization_macro = context[search_name] %}\n {% set relations = materialization_macro() %}\n {{ return(relations) }}\n\n {%- endif -%}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.can_clone_table", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.create_or_replace_clone", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2059531, "supported_languages": ["sql"]}, "macro.dbt.get_table_columns_and_constraints": {"name": "get_table_columns_and_constraints", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table/columns_spec_ddl.sql", "original_file_path": "macros/materializations/models/table/columns_spec_ddl.sql", "unique_id": "macro.dbt.get_table_columns_and_constraints", "macro_sql": "{%- macro get_table_columns_and_constraints() -%}\n {{ adapter.dispatch('get_table_columns_and_constraints', 'dbt')() }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__get_table_columns_and_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2070122, "supported_languages": null}, "macro.dbt.default__get_table_columns_and_constraints": {"name": "default__get_table_columns_and_constraints", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table/columns_spec_ddl.sql", "original_file_path": "macros/materializations/models/table/columns_spec_ddl.sql", "unique_id": "macro.dbt.default__get_table_columns_and_constraints", "macro_sql": "{% macro default__get_table_columns_and_constraints() -%}\n {{ return(table_columns_and_constraints()) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.table_columns_and_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2071402, "supported_languages": null}, "macro.dbt.table_columns_and_constraints": {"name": "table_columns_and_constraints", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table/columns_spec_ddl.sql", "original_file_path": "macros/materializations/models/table/columns_spec_ddl.sql", "unique_id": "macro.dbt.table_columns_and_constraints", "macro_sql": "{% macro table_columns_and_constraints() %}\n {# loop through user_provided_columns to create DDL with data types and constraints #}\n {%- set raw_column_constraints = adapter.render_raw_columns_constraints(raw_columns=model['columns']) -%}\n {%- set raw_model_constraints = adapter.render_raw_model_constraints(raw_constraints=model['constraints']) -%}\n (\n {% for c in raw_column_constraints -%}\n {{ c }}{{ \",\" if not loop.last or raw_model_constraints }}\n {% endfor %}\n {% for c in raw_model_constraints -%}\n {{ c }}{{ \",\" if not loop.last }}\n {% endfor -%}\n )\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.207693, "supported_languages": null}, "macro.dbt.get_assert_columns_equivalent": {"name": "get_assert_columns_equivalent", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table/columns_spec_ddl.sql", "original_file_path": "macros/materializations/models/table/columns_spec_ddl.sql", "unique_id": "macro.dbt.get_assert_columns_equivalent", "macro_sql": "\n\n{%- macro get_assert_columns_equivalent(sql) -%}\n {{ adapter.dispatch('get_assert_columns_equivalent', 'dbt')(sql) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__get_assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2078662, "supported_languages": null}, "macro.dbt.default__get_assert_columns_equivalent": {"name": "default__get_assert_columns_equivalent", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table/columns_spec_ddl.sql", "original_file_path": "macros/materializations/models/table/columns_spec_ddl.sql", "unique_id": "macro.dbt.default__get_assert_columns_equivalent", "macro_sql": "{% macro default__get_assert_columns_equivalent(sql) -%}\n {{ return(assert_columns_equivalent(sql)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2080052, "supported_languages": null}, "macro.dbt.assert_columns_equivalent": {"name": "assert_columns_equivalent", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table/columns_spec_ddl.sql", "original_file_path": "macros/materializations/models/table/columns_spec_ddl.sql", "unique_id": "macro.dbt.assert_columns_equivalent", "macro_sql": "{% macro assert_columns_equivalent(sql) %}\n\n {#-- First ensure the user has defined 'columns' in yaml specification --#}\n {%- set user_defined_columns = model['columns'] -%}\n {%- if not user_defined_columns -%}\n {{ exceptions.raise_contract_error([], []) }}\n {%- endif -%}\n\n {#-- Obtain the column schema provided by sql file. #}\n {%- set sql_file_provided_columns = get_column_schema_from_query(sql, config.get('sql_header', none)) -%}\n {#--Obtain the column schema provided by the schema file by generating an 'empty schema' query from the model's columns. #}\n {%- set schema_file_provided_columns = get_column_schema_from_query(get_empty_schema_sql(user_defined_columns)) -%}\n\n {#-- create dictionaries with name and formatted data type and strings for exception #}\n {%- set sql_columns = format_columns(sql_file_provided_columns) -%}\n {%- set yaml_columns = format_columns(schema_file_provided_columns) -%}\n\n {%- if sql_columns|length != yaml_columns|length -%}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n\n {%- for sql_col in sql_columns -%}\n {%- set yaml_col = [] -%}\n {%- for this_col in yaml_columns -%}\n {%- if this_col['name'] == sql_col['name'] -%}\n {%- do yaml_col.append(this_col) -%}\n {%- break -%}\n {%- endif -%}\n {%- endfor -%}\n {%- if not yaml_col -%}\n {#-- Column with name not found in yaml #}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n {%- if sql_col['formatted'] != yaml_col[0]['formatted'] -%}\n {#-- Column data types don't match #}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n {%- endfor -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_column_schema_from_query", "macro.dbt.get_empty_schema_sql", "macro.dbt.format_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.209325, "supported_languages": null}, "macro.dbt.format_columns": {"name": "format_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table/columns_spec_ddl.sql", "original_file_path": "macros/materializations/models/table/columns_spec_ddl.sql", "unique_id": "macro.dbt.format_columns", "macro_sql": "{% macro format_columns(columns) %}\n {% set formatted_columns = [] %}\n {% for column in columns %}\n {%- set formatted_column = adapter.dispatch('format_column', 'dbt')(column) -%}\n {%- do formatted_columns.append(formatted_column) -%}\n {% endfor %}\n {{ return(formatted_columns) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__format_column"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.209718, "supported_languages": null}, "macro.dbt.default__format_column": {"name": "default__format_column", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table/columns_spec_ddl.sql", "original_file_path": "macros/materializations/models/table/columns_spec_ddl.sql", "unique_id": "macro.dbt.default__format_column", "macro_sql": "{% macro default__format_column(column) -%}\n {% set data_type = column.dtype %}\n {% set formatted = column.column.lower() ~ \" \" ~ data_type %}\n {{ return({'name': column.name, 'data_type': data_type, 'formatted': formatted}) }}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.210073, "supported_languages": null}, "macro.dbt.materialization_table_default": {"name": "materialization_table_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table/table.sql", "original_file_path": "macros/materializations/models/table/table.sql", "unique_id": "macro.dbt.materialization_table_default", "macro_sql": "{% materialization table, default %}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') %}\n {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}\n /*\n See ../view/view.sql for more information about this relation.\n */\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_table_as_sql(False, intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n {% if existing_relation is not none %}\n /* Do the equivalent of rename_if_exists. 'existing_relation' could have been dropped\n since the variable was first set. */\n {% set existing_relation = load_cached_relation(existing_relation) %}\n {% if existing_relation is not none %}\n {{ adapter.rename_relation(existing_relation, backup_relation) }}\n {% endif %}\n {% endif %}\n\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% do create_indexes(target_relation) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n -- finally, drop the existing/backup relation after the commit\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.get_create_table_as_sql", "macro.dbt.create_indexes", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2131069, "supported_languages": ["sql"]}, "macro.dbt.get_create_table_as_sql": {"name": "get_create_table_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table/create_table_as.sql", "original_file_path": "macros/materializations/models/table/create_table_as.sql", "unique_id": "macro.dbt.get_create_table_as_sql", "macro_sql": "{% macro get_create_table_as_sql(temporary, relation, sql) -%}\n {{ adapter.dispatch('get_create_table_as_sql', 'dbt')(temporary, relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_create_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.214011, "supported_languages": null}, "macro.dbt.default__get_create_table_as_sql": {"name": "default__get_create_table_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table/create_table_as.sql", "original_file_path": "macros/materializations/models/table/create_table_as.sql", "unique_id": "macro.dbt.default__get_create_table_as_sql", "macro_sql": "{% macro default__get_create_table_as_sql(temporary, relation, sql) -%}\n {{ return(create_table_as(temporary, relation, sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2142022, "supported_languages": null}, "macro.dbt.create_table_as": {"name": "create_table_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table/create_table_as.sql", "original_file_path": "macros/materializations/models/table/create_table_as.sql", "unique_id": "macro.dbt.create_table_as", "macro_sql": "{% macro create_table_as(temporary, relation, compiled_code, language='sql') -%}\n {# backward compatibility for create_table_as that does not support language #}\n {% if language == \"sql\" %}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, compiled_code)}}\n {% else %}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, compiled_code, language) }}\n {% endif %}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.214653, "supported_languages": null}, "macro.dbt.default__create_table_as": {"name": "default__create_table_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table/create_table_as.sql", "original_file_path": "macros/materializations/models/table/create_table_as.sql", "unique_id": "macro.dbt.default__create_table_as", "macro_sql": "{% macro default__create_table_as(temporary, relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary: -%}temporary{%- endif %} table\n {{ relation.include(database=(not temporary), schema=(not temporary)) }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced and (not temporary) %}\n {{ get_assert_columns_equivalent(sql) }}\n {{ get_table_columns_and_constraints() }}\n {%- set sql = get_select_subquery(sql) %}\n {% endif %}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_assert_columns_equivalent", "macro.dbt.get_table_columns_and_constraints", "macro.dbt.get_select_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2153769, "supported_languages": null}, "macro.dbt.default__get_column_names": {"name": "default__get_column_names", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table/create_table_as.sql", "original_file_path": "macros/materializations/models/table/create_table_as.sql", "unique_id": "macro.dbt.default__get_column_names", "macro_sql": "{% macro default__get_column_names() %}\n {#- loop through user_provided_columns to get column names -#}\n {%- set user_provided_columns = model['columns'] -%}\n {%- for i in user_provided_columns %}\n {%- set col = user_provided_columns[i] -%}\n {%- set col_name = adapter.quote(col['name']) if col.get('quote') else col['name'] -%}\n {{ col_name }}{{ \", \" if not loop.last }}\n {%- endfor -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2158551, "supported_languages": null}, "macro.dbt.get_select_subquery": {"name": "get_select_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table/create_table_as.sql", "original_file_path": "macros/materializations/models/table/create_table_as.sql", "unique_id": "macro.dbt.get_select_subquery", "macro_sql": "{% macro get_select_subquery(sql) %}\n {{ return(adapter.dispatch('get_select_subquery', 'dbt')(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_select_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2160509, "supported_languages": null}, "macro.dbt.default__get_select_subquery": {"name": "default__get_select_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table/create_table_as.sql", "original_file_path": "macros/materializations/models/table/create_table_as.sql", "unique_id": "macro.dbt.default__get_select_subquery", "macro_sql": "{% macro default__get_select_subquery(sql) %}\n select {{ adapter.dispatch('get_column_names', 'dbt')() }}\n from (\n {{ sql }}\n ) as model_subq\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_column_names"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.216245, "supported_languages": null}, "macro.dbt.materialization_view_default": {"name": "materialization_view_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/view/view.sql", "original_file_path": "macros/materializations/models/view/view.sql", "unique_id": "macro.dbt.materialization_view_default", "macro_sql": "{%- materialization view, default -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='view') -%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}\n\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}\n /*\n This relation (probably) doesn't exist yet. If it does exist, it's a leftover from\n a previous run, and we're going to try to drop it immediately. At the end of this\n materialization, we're going to rename the \"existing_relation\" to this identifier,\n and then we're going to drop it. In order to make sure we run the correct one of:\n - drop view ...\n - drop table ...\n\n We need to set the type of this relation to be the type of the existing_relation, if it exists,\n or else \"view\" as a sane default if it does not. Note that if the existing_relation does not\n exist, then there is nothing to move out of the way and subsequentally drop. In that case,\n this relation will be effectively unused.\n */\n {%- set backup_relation_type = 'view' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n -- move the existing view out of the way\n {% if existing_relation is not none %}\n /* Do the equivalent of rename_if_exists. 'existing_relation' could have been dropped\n since the variable was first set. */\n {% set existing_relation = load_cached_relation(existing_relation) %}\n {% if existing_relation is not none %}\n {{ adapter.rename_relation(existing_relation, backup_relation) }}\n {% endif %}\n {% endif %}\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.run_hooks", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.get_create_view_as_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.219257, "supported_languages": ["sql"]}, "macro.dbt.handle_existing_table": {"name": "handle_existing_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/view/helpers.sql", "original_file_path": "macros/materializations/models/view/helpers.sql", "unique_id": "macro.dbt.handle_existing_table", "macro_sql": "{% macro handle_existing_table(full_refresh, old_relation) %}\n {{ adapter.dispatch('handle_existing_table', 'dbt')(full_refresh, old_relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__handle_existing_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.219564, "supported_languages": null}, "macro.dbt.default__handle_existing_table": {"name": "default__handle_existing_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/view/helpers.sql", "original_file_path": "macros/materializations/models/view/helpers.sql", "unique_id": "macro.dbt.default__handle_existing_table", "macro_sql": "{% macro default__handle_existing_table(full_refresh, old_relation) %}\n {{ log(\"Dropping relation \" ~ old_relation ~ \" because it is of type \" ~ old_relation.type) }}\n {{ adapter.drop_relation(old_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.219792, "supported_languages": null}, "macro.dbt.create_or_replace_view": {"name": "create_or_replace_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/view/create_or_replace_view.sql", "original_file_path": "macros/materializations/models/view/create_or_replace_view.sql", "unique_id": "macro.dbt.create_or_replace_view", "macro_sql": "{% macro create_or_replace_view() %}\n {%- set identifier = model['alias'] -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database,\n type='view') -%}\n {% set grant_config = config.get('grants') %}\n\n {{ run_hooks(pre_hooks) }}\n\n -- If there's a table with the same name and we weren't told to full refresh,\n -- that's an error. If we were told to full refresh, drop it. This behavior differs\n -- for Snowflake and BigQuery, so multiple dispatch is used.\n {%- if old_relation is not none and old_relation.is_table -%}\n {{ handle_existing_table(should_full_refresh(), old_relation) }}\n {%- endif -%}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(target_relation, sql) }}\n {%- endcall %}\n\n {% set should_revoke = should_revoke(exists_as_view, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.handle_existing_table", "macro.dbt.should_full_refresh", "macro.dbt.statement", "macro.dbt.get_create_view_as_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2213418, "supported_languages": null}, "macro.dbt.get_create_view_as_sql": {"name": "get_create_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/view/create_view_as.sql", "original_file_path": "macros/materializations/models/view/create_view_as.sql", "unique_id": "macro.dbt.get_create_view_as_sql", "macro_sql": "{% macro get_create_view_as_sql(relation, sql) -%}\n {{ adapter.dispatch('get_create_view_as_sql', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_create_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.221772, "supported_languages": null}, "macro.dbt.default__get_create_view_as_sql": {"name": "default__get_create_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/view/create_view_as.sql", "original_file_path": "macros/materializations/models/view/create_view_as.sql", "unique_id": "macro.dbt.default__get_create_view_as_sql", "macro_sql": "{% macro default__get_create_view_as_sql(relation, sql) -%}\n {{ return(create_view_as(relation, sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2219388, "supported_languages": null}, "macro.dbt.create_view_as": {"name": "create_view_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/view/create_view_as.sql", "original_file_path": "macros/materializations/models/view/create_view_as.sql", "unique_id": "macro.dbt.create_view_as", "macro_sql": "{% macro create_view_as(relation, sql) -%}\n {{ adapter.dispatch('create_view_as', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.222131, "supported_languages": null}, "macro.dbt.default__create_view_as": {"name": "default__create_view_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/view/create_view_as.sql", "original_file_path": "macros/materializations/models/view/create_view_as.sql", "unique_id": "macro.dbt.default__create_view_as", "macro_sql": "{% macro default__create_view_as(relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n create view {{ relation }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {%- endif %}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.222568, "supported_languages": null}, "macro.dbt.materialization_seed_default": {"name": "materialization_seed_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/seed.sql", "original_file_path": "macros/materializations/seeds/seed.sql", "unique_id": "macro.dbt.materialization_seed_default", "macro_sql": "{% materialization seed, default %}\n\n {%- set identifier = model['alias'] -%}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set grant_config = config.get('grants') -%}\n {%- set agate_table = load_agate_table() -%}\n -- grab current tables grants config for comparison later on\n\n {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% set create_table_sql = \"\" %}\n {% if exists_as_view %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a view\".format(old_relation)) }}\n {% elif exists_as_table %}\n {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}\n {% else %}\n {% set create_table_sql = create_csv_table(model, agate_table) %}\n {% endif %}\n\n {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}\n {% set rows_affected = (agate_table.rows | length) %}\n {% set sql = load_csv_rows(model, agate_table) %}\n\n {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}\n {{ get_csv_sql(create_table_sql, sql) }};\n {% endcall %}\n\n {% set target_relation = this.incorporate(type='table') %}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if full_refresh_mode or not exists_as_table %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.run_hooks", "macro.dbt.reset_csv_table", "macro.dbt.create_csv_table", "macro.dbt.load_csv_rows", "macro.dbt.noop_statement", "macro.dbt.get_csv_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.225961, "supported_languages": ["sql"]}, "macro.dbt.create_csv_table": {"name": "create_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.create_csv_table", "macro_sql": "{% macro create_csv_table(model, agate_table) -%}\n {{ adapter.dispatch('create_csv_table', 'dbt')(model, agate_table) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.231348, "supported_languages": null}, "macro.dbt.default__create_csv_table": {"name": "default__create_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__create_csv_table", "macro_sql": "{% macro default__create_csv_table(model, agate_table) %}\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n\n {% set sql %}\n create table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.232269, "supported_languages": null}, "macro.dbt.reset_csv_table": {"name": "reset_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.reset_csv_table", "macro_sql": "{% macro reset_csv_table(model, full_refresh, old_relation, agate_table) -%}\n {{ adapter.dispatch('reset_csv_table', 'dbt')(model, full_refresh, old_relation, agate_table) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__reset_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.232512, "supported_languages": null}, "macro.dbt.default__reset_csv_table": {"name": "default__reset_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__reset_csv_table", "macro_sql": "{% macro default__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% set sql = \"\" %}\n {% if full_refresh %}\n {{ adapter.drop_relation(old_relation) }}\n {% set sql = create_csv_table(model, agate_table) %}\n {% else %}\n {{ adapter.truncate_relation(old_relation) }}\n {% set sql = \"truncate table \" ~ old_relation %}\n {% endif %}\n\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.233001, "supported_languages": null}, "macro.dbt.get_csv_sql": {"name": "get_csv_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_csv_sql", "macro_sql": "{% macro get_csv_sql(create_or_truncate_sql, insert_sql) %}\n {{ adapter.dispatch('get_csv_sql', 'dbt')(create_or_truncate_sql, insert_sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_csv_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2331991, "supported_languages": null}, "macro.dbt.default__get_csv_sql": {"name": "default__get_csv_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__get_csv_sql", "macro_sql": "{% macro default__get_csv_sql(create_or_truncate_sql, insert_sql) %}\n {{ create_or_truncate_sql }};\n -- dbt seed --\n {{ insert_sql }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.233336, "supported_languages": null}, "macro.dbt.get_binding_char": {"name": "get_binding_char", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_binding_char", "macro_sql": "{% macro get_binding_char() -%}\n {{ adapter.dispatch('get_binding_char', 'dbt')() }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.233479, "supported_languages": null}, "macro.dbt.default__get_binding_char": {"name": "default__get_binding_char", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__get_binding_char", "macro_sql": "{% macro default__get_binding_char() %}\n {{ return('%s') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.233599, "supported_languages": null}, "macro.dbt.get_batch_size": {"name": "get_batch_size", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_batch_size", "macro_sql": "{% macro get_batch_size() -%}\n {{ return(adapter.dispatch('get_batch_size', 'dbt')()) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_batch_size"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.233762, "supported_languages": null}, "macro.dbt.default__get_batch_size": {"name": "default__get_batch_size", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__get_batch_size", "macro_sql": "{% macro default__get_batch_size() %}\n {{ return(10000) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.233883, "supported_languages": null}, "macro.dbt.get_seed_column_quoted_csv": {"name": "get_seed_column_quoted_csv", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_seed_column_quoted_csv", "macro_sql": "{% macro get_seed_column_quoted_csv(model, column_names) %}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote_seed_column(col, quote_seed_column)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2343712, "supported_languages": null}, "macro.dbt.load_csv_rows": {"name": "load_csv_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.load_csv_rows", "macro_sql": "{% macro load_csv_rows(model, agate_table) -%}\n {{ adapter.dispatch('load_csv_rows', 'dbt')(model, agate_table) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__load_csv_rows"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2345622, "supported_languages": null}, "macro.dbt.default__load_csv_rows": {"name": "default__load_csv_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__load_csv_rows", "macro_sql": "{% macro default__load_csv_rows(model, agate_table) %}\n\n {% set batch_size = get_batch_size() %}\n\n {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}\n {% set bindings = [] %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert into {{ this.render() }} ({{ cols_sql }}) values\n {% for row in chunk -%}\n ({%- for column in agate_table.column_names -%}\n {{ get_binding_char() }}\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_batch_size", "macro.dbt.get_seed_column_quoted_csv", "macro.dbt.get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.236012, "supported_languages": null}, "macro.dbt.generate_alias_name": {"name": "generate_alias_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_alias.sql", "original_file_path": "macros/get_custom_name/get_custom_alias.sql", "unique_id": "macro.dbt.generate_alias_name", "macro_sql": "{% macro generate_alias_name(custom_alias_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_alias_name', 'dbt')(custom_alias_name, node)) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__generate_alias_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2364511, "supported_languages": null}, "macro.dbt.default__generate_alias_name": {"name": "default__generate_alias_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_alias.sql", "original_file_path": "macros/get_custom_name/get_custom_alias.sql", "unique_id": "macro.dbt.default__generate_alias_name", "macro_sql": "{% macro default__generate_alias_name(custom_alias_name=none, node=none) -%}\n\n {%- if custom_alias_name -%}\n\n {{ custom_alias_name | trim }}\n\n {%- elif node.version -%}\n\n {{ return(node.name ~ \"_v\" ~ (node.version | replace(\".\", \"_\"))) }}\n\n {%- else -%}\n\n {{ node.name }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.236854, "supported_languages": null}, "macro.dbt.generate_schema_name": {"name": "generate_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "unique_id": "macro.dbt.generate_schema_name", "macro_sql": "{% macro generate_schema_name(custom_schema_name=none, node=none) -%}\n {{ return(adapter.dispatch('generate_schema_name', 'dbt')(custom_schema_name, node)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__generate_schema_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.237386, "supported_languages": null}, "macro.dbt.default__generate_schema_name": {"name": "default__generate_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "unique_id": "macro.dbt.default__generate_schema_name", "macro_sql": "{% macro default__generate_schema_name(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if custom_schema_name is none -%}\n\n {{ default_schema }}\n\n {%- else -%}\n\n {{ default_schema }}_{{ custom_schema_name | trim }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.237652, "supported_languages": null}, "macro.dbt.generate_schema_name_for_env": {"name": "generate_schema_name_for_env", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "unique_id": "macro.dbt.generate_schema_name_for_env", "macro_sql": "{% macro generate_schema_name_for_env(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if target.name == 'prod' and custom_schema_name is not none -%}\n\n {{ custom_schema_name | trim }}\n\n {%- else -%}\n\n {{ default_schema }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.237949, "supported_languages": null}, "macro.dbt.generate_database_name": {"name": "generate_database_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "unique_id": "macro.dbt.generate_database_name", "macro_sql": "{% macro generate_database_name(custom_database_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_database_name', 'dbt')(custom_database_name, node)) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__generate_database_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.238351, "supported_languages": null}, "macro.dbt.default__generate_database_name": {"name": "default__generate_database_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "unique_id": "macro.dbt.default__generate_database_name", "macro_sql": "{% macro default__generate_database_name(custom_database_name=none, node=none) -%}\n {%- set default_database = target.database -%}\n {%- if custom_database_name is none -%}\n\n {{ default_database }}\n\n {%- else -%}\n\n {{ custom_database_name }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.238611, "supported_languages": null}, "macro.dbt.default__test_relationships": {"name": "default__test_relationships", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/relationships.sql", "original_file_path": "macros/generic_test_sql/relationships.sql", "unique_id": "macro.dbt.default__test_relationships", "macro_sql": "{% macro default__test_relationships(model, column_name, to, field) %}\n\nwith child as (\n select {{ column_name }} as from_field\n from {{ model }}\n where {{ column_name }} is not null\n),\n\nparent as (\n select {{ field }} as to_field\n from {{ to }}\n)\n\nselect\n from_field\n\nfrom child\nleft join parent\n on child.from_field = parent.to_field\n\nwhere parent.to_field is null\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.238965, "supported_languages": null}, "macro.dbt.default__test_not_null": {"name": "default__test_not_null", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/not_null.sql", "original_file_path": "macros/generic_test_sql/not_null.sql", "unique_id": "macro.dbt.default__test_not_null", "macro_sql": "{% macro default__test_not_null(model, column_name) %}\n\n{% set column_list = '*' if should_store_failures() else column_name %}\n\nselect {{ column_list }}\nfrom {{ model }}\nwhere {{ column_name }} is null\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_store_failures"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2392662, "supported_languages": null}, "macro.dbt.default__test_unique": {"name": "default__test_unique", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/unique.sql", "original_file_path": "macros/generic_test_sql/unique.sql", "unique_id": "macro.dbt.default__test_unique", "macro_sql": "{% macro default__test_unique(model, column_name) %}\n\nselect\n {{ column_name }} as unique_field,\n count(*) as n_records\n\nfrom {{ model }}\nwhere {{ column_name }} is not null\ngroup by {{ column_name }}\nhaving count(*) > 1\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.239514, "supported_languages": null}, "macro.dbt.default__test_accepted_values": {"name": "default__test_accepted_values", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/accepted_values.sql", "original_file_path": "macros/generic_test_sql/accepted_values.sql", "unique_id": "macro.dbt.default__test_accepted_values", "macro_sql": "{% macro default__test_accepted_values(model, column_name, values, quote=True) %}\n\nwith all_values as (\n\n select\n {{ column_name }} as value_field,\n count(*) as n_records\n\n from {{ model }}\n group by {{ column_name }}\n\n)\n\nselect *\nfrom all_values\nwhere value_field not in (\n {% for value in values -%}\n {% if quote -%}\n '{{ value }}'\n {%- else -%}\n {{ value }}\n {%- endif -%}\n {%- if not loop.last -%},{%- endif %}\n {%- endfor %}\n)\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2400918, "supported_languages": null}, "macro.dbt.statement": {"name": "statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt.statement", "macro_sql": "\n{%- macro statement(name=None, fetch_result=False, auto_begin=True, language='sql') -%}\n {%- if execute: -%}\n {%- set compiled_code = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime {} for node \"{}\"'.format(language, model['unique_id'])) }}\n {{ write(compiled_code) }}\n {%- endif -%}\n {%- if language == 'sql'-%}\n {%- set res, table = adapter.execute(compiled_code, auto_begin=auto_begin, fetch=fetch_result) -%}\n {%- elif language == 'python' -%}\n {%- set res = submit_python_job(model, compiled_code) -%}\n {#-- TODO: What should table be for python models? --#}\n {%- set table = None -%}\n {%- else -%}\n {% do exceptions.raise_compiler_error(\"statement macro didn't get supported language\") %}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_result(name, response=res, agate_table=table) }}\n {%- endif -%}\n\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.241607, "supported_languages": null}, "macro.dbt.noop_statement": {"name": "noop_statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt.noop_statement", "macro_sql": "{% macro noop_statement(name=None, message=None, code=None, rows_affected=None, res=None) -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_raw_result(name, message=message, code=code, rows_affected=rows_affected, agate_table=res) }}\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.242203, "supported_languages": null}, "macro.dbt.run_query": {"name": "run_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt.run_query", "macro_sql": "{% macro run_query(sql) %}\n {% call statement(\"run_query_statement\", fetch_result=true, auto_begin=false) %}\n {{ sql }}\n {% endcall %}\n\n {% do return(load_result(\"run_query_statement\").table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2425091, "supported_languages": null}, "macro.dbt.convert_datetime": {"name": "convert_datetime", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.convert_datetime", "macro_sql": "{% macro convert_datetime(date_str, date_fmt) %}\n\n {% set error_msg -%}\n The provided partition date '{{ date_str }}' does not match the expected format '{{ date_fmt }}'\n {%- endset %}\n\n {% set res = try_or_compiler_error(error_msg, modules.datetime.datetime.strptime, date_str.strip(), date_fmt) %}\n {{ return(res) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2444658, "supported_languages": null}, "macro.dbt.dates_in_range": {"name": "dates_in_range", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.dates_in_range", "macro_sql": "{% macro dates_in_range(start_date_str, end_date_str=none, in_fmt=\"%Y%m%d\", out_fmt=\"%Y%m%d\") %}\n {% set end_date_str = start_date_str if end_date_str is none else end_date_str %}\n\n {% set start_date = convert_datetime(start_date_str, in_fmt) %}\n {% set end_date = convert_datetime(end_date_str, in_fmt) %}\n\n {% set day_count = (end_date - start_date).days %}\n {% if day_count < 0 %}\n {% set msg -%}\n Partiton start date is after the end date ({{ start_date }}, {{ end_date }})\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg, model) }}\n {% endif %}\n\n {% set date_list = [] %}\n {% for i in range(0, day_count + 1) %}\n {% set the_date = (modules.datetime.timedelta(days=i) + start_date) %}\n {% if not out_fmt %}\n {% set _ = date_list.append(the_date) %}\n {% else %}\n {% set _ = date_list.append(the_date.strftime(out_fmt)) %}\n {% endif %}\n {% endfor %}\n\n {{ return(date_list) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.convert_datetime"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.245748, "supported_languages": null}, "macro.dbt.partition_range": {"name": "partition_range", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.partition_range", "macro_sql": "{% macro partition_range(raw_partition_date, date_fmt='%Y%m%d') %}\n {% set partition_range = (raw_partition_date | string).split(\",\") %}\n\n {% if (partition_range | length) == 1 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = none %}\n {% elif (partition_range | length) == 2 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = partition_range[1] %}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Invalid partition time. Expected format: {Start Date}[,{End Date}]. Got: \" ~ raw_partition_date) }}\n {% endif %}\n\n {{ return(dates_in_range(start_date, end_date, in_fmt=date_fmt)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.dates_in_range"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.246516, "supported_languages": null}, "macro.dbt.py_current_timestring": {"name": "py_current_timestring", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.py_current_timestring", "macro_sql": "{% macro py_current_timestring() %}\n {% set dt = modules.datetime.datetime.now() %}\n {% do return(dt.strftime(\"%Y%m%d%H%M%S%f\")) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2467651, "supported_languages": null}, "macro.dbt.except": {"name": "except", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/except.sql", "original_file_path": "macros/utils/except.sql", "unique_id": "macro.dbt.except", "macro_sql": "{% macro except() %}\n {{ return(adapter.dispatch('except', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__except"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2469969, "supported_languages": null}, "macro.dbt.default__except": {"name": "default__except", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/except.sql", "original_file_path": "macros/utils/except.sql", "unique_id": "macro.dbt.default__except", "macro_sql": "{% macro default__except() %}\n\n except\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.247075, "supported_languages": null}, "macro.dbt.replace": {"name": "replace", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/replace.sql", "original_file_path": "macros/utils/replace.sql", "unique_id": "macro.dbt.replace", "macro_sql": "{% macro replace(field, old_chars, new_chars) -%}\n {{ return(adapter.dispatch('replace', 'dbt') (field, old_chars, new_chars)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__replace"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2474089, "supported_languages": null}, "macro.dbt.default__replace": {"name": "default__replace", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/replace.sql", "original_file_path": "macros/utils/replace.sql", "unique_id": "macro.dbt.default__replace", "macro_sql": "{% macro default__replace(field, old_chars, new_chars) %}\n\n replace(\n {{ field }},\n {{ old_chars }},\n {{ new_chars }}\n )\n\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2475772, "supported_languages": null}, "macro.dbt.concat": {"name": "concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/concat.sql", "original_file_path": "macros/utils/concat.sql", "unique_id": "macro.dbt.concat", "macro_sql": "{% macro concat(fields) -%}\n {{ return(adapter.dispatch('concat', 'dbt')(fields)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__concat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.247827, "supported_languages": null}, "macro.dbt.default__concat": {"name": "default__concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/concat.sql", "original_file_path": "macros/utils/concat.sql", "unique_id": "macro.dbt.default__concat", "macro_sql": "{% macro default__concat(fields) -%}\n {{ fields|join(' || ') }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.247957, "supported_languages": null}, "macro.dbt.length": {"name": "length", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/length.sql", "original_file_path": "macros/utils/length.sql", "unique_id": "macro.dbt.length", "macro_sql": "{% macro length(expression) -%}\n {{ return(adapter.dispatch('length', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__length"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.248211, "supported_languages": null}, "macro.dbt.default__length": {"name": "default__length", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/length.sql", "original_file_path": "macros/utils/length.sql", "unique_id": "macro.dbt.default__length", "macro_sql": "{% macro default__length(expression) %}\n\n length(\n {{ expression }}\n )\n\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.248318, "supported_languages": null}, "macro.dbt.dateadd": {"name": "dateadd", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt.dateadd", "macro_sql": "{% macro dateadd(datepart, interval, from_date_or_timestamp) %}\n {{ return(adapter.dispatch('dateadd', 'dbt')(datepart, interval, from_date_or_timestamp)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__dateadd"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2486641, "supported_languages": null}, "macro.dbt.default__dateadd": {"name": "default__dateadd", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt.default__dateadd", "macro_sql": "{% macro default__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n dateadd(\n {{ datepart }},\n {{ interval }},\n {{ from_date_or_timestamp }}\n )\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.248833, "supported_languages": null}, "macro.dbt.intersect": {"name": "intersect", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/intersect.sql", "original_file_path": "macros/utils/intersect.sql", "unique_id": "macro.dbt.intersect", "macro_sql": "{% macro intersect() %}\n {{ return(adapter.dispatch('intersect', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__intersect"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2490602, "supported_languages": null}, "macro.dbt.default__intersect": {"name": "default__intersect", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/intersect.sql", "original_file_path": "macros/utils/intersect.sql", "unique_id": "macro.dbt.default__intersect", "macro_sql": "{% macro default__intersect() %}\n\n intersect\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2491379, "supported_languages": null}, "macro.dbt.escape_single_quotes": {"name": "escape_single_quotes", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/escape_single_quotes.sql", "original_file_path": "macros/utils/escape_single_quotes.sql", "unique_id": "macro.dbt.escape_single_quotes", "macro_sql": "{% macro escape_single_quotes(expression) %}\n {{ return(adapter.dispatch('escape_single_quotes', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__escape_single_quotes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.249404, "supported_languages": null}, "macro.dbt.default__escape_single_quotes": {"name": "default__escape_single_quotes", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/escape_single_quotes.sql", "original_file_path": "macros/utils/escape_single_quotes.sql", "unique_id": "macro.dbt.default__escape_single_quotes", "macro_sql": "{% macro default__escape_single_quotes(expression) -%}\n{{ expression | replace(\"'\",\"''\") }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2495492, "supported_languages": null}, "macro.dbt.right": {"name": "right", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/right.sql", "original_file_path": "macros/utils/right.sql", "unique_id": "macro.dbt.right", "macro_sql": "{% macro right(string_text, length_expression) -%}\n {{ return(adapter.dispatch('right', 'dbt') (string_text, length_expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__right"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.249847, "supported_languages": null}, "macro.dbt.default__right": {"name": "default__right", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/right.sql", "original_file_path": "macros/utils/right.sql", "unique_id": "macro.dbt.default__right", "macro_sql": "{% macro default__right(string_text, length_expression) %}\n\n right(\n {{ string_text }},\n {{ length_expression }}\n )\n\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.249984, "supported_languages": null}, "macro.dbt.listagg": {"name": "listagg", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "unique_id": "macro.dbt.listagg", "macro_sql": "{% macro listagg(measure, delimiter_text=\"','\", order_by_clause=none, limit_num=none) -%}\n {{ return(adapter.dispatch('listagg', 'dbt') (measure, delimiter_text, order_by_clause, limit_num)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__listagg"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2506409, "supported_languages": null}, "macro.dbt.default__listagg": {"name": "default__listagg", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "unique_id": "macro.dbt.default__listagg", "macro_sql": "{% macro default__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n {% if limit_num -%}\n array_to_string(\n array_slice(\n array_agg(\n {{ measure }}\n ){% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n ,0\n ,{{ limit_num }}\n ),\n {{ delimiter_text }}\n )\n {%- else %}\n listagg(\n {{ measure }},\n {{ delimiter_text }}\n )\n {% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n {%- endif %}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.251061, "supported_languages": null}, "macro.dbt.datediff": {"name": "datediff", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt.datediff", "macro_sql": "{% macro datediff(first_date, second_date, datepart) %}\n {{ return(adapter.dispatch('datediff', 'dbt')(first_date, second_date, datepart)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__datediff"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.251404, "supported_languages": null}, "macro.dbt.default__datediff": {"name": "default__datediff", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt.default__datediff", "macro_sql": "{% macro default__datediff(first_date, second_date, datepart) -%}\n\n datediff(\n {{ datepart }},\n {{ first_date }},\n {{ second_date }}\n )\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.251571, "supported_languages": null}, "macro.dbt.safe_cast": {"name": "safe_cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "unique_id": "macro.dbt.safe_cast", "macro_sql": "{% macro safe_cast(field, type) %}\n {{ return(adapter.dispatch('safe_cast', 'dbt') (field, type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__safe_cast"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.25186, "supported_languages": null}, "macro.dbt.default__safe_cast": {"name": "default__safe_cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "unique_id": "macro.dbt.default__safe_cast", "macro_sql": "{% macro default__safe_cast(field, type) %}\n {# most databases don't support this function yet\n so we just need to use cast #}\n cast({{field}} as {{type}})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.252075, "supported_languages": null}, "macro.dbt.hash": {"name": "hash", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/hash.sql", "original_file_path": "macros/utils/hash.sql", "unique_id": "macro.dbt.hash", "macro_sql": "{% macro hash(field) -%}\n {{ return(adapter.dispatch('hash', 'dbt') (field)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__hash"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.252336, "supported_languages": null}, "macro.dbt.default__hash": {"name": "default__hash", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/hash.sql", "original_file_path": "macros/utils/hash.sql", "unique_id": "macro.dbt.default__hash", "macro_sql": "{% macro default__hash(field) -%}\n md5(cast({{ field }} as {{ api.Column.translate_type('string') }}))\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.252498, "supported_languages": null}, "macro.dbt.cast_bool_to_text": {"name": "cast_bool_to_text", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast_bool_to_text.sql", "original_file_path": "macros/utils/cast_bool_to_text.sql", "unique_id": "macro.dbt.cast_bool_to_text", "macro_sql": "{% macro cast_bool_to_text(field) %}\n {{ adapter.dispatch('cast_bool_to_text', 'dbt') (field) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__cast_bool_to_text"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2527459, "supported_languages": null}, "macro.dbt.default__cast_bool_to_text": {"name": "default__cast_bool_to_text", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast_bool_to_text.sql", "original_file_path": "macros/utils/cast_bool_to_text.sql", "unique_id": "macro.dbt.default__cast_bool_to_text", "macro_sql": "{% macro default__cast_bool_to_text(field) %}\n cast({{ field }} as {{ api.Column.translate_type('string') }})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.252909, "supported_languages": null}, "macro.dbt.any_value": {"name": "any_value", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "unique_id": "macro.dbt.any_value", "macro_sql": "{% macro any_value(expression) -%}\n {{ return(adapter.dispatch('any_value', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__any_value"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.25316, "supported_languages": null}, "macro.dbt.default__any_value": {"name": "default__any_value", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "unique_id": "macro.dbt.default__any_value", "macro_sql": "{% macro default__any_value(expression) -%}\n\n any_value({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.253269, "supported_languages": null}, "macro.dbt.position": {"name": "position", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/position.sql", "original_file_path": "macros/utils/position.sql", "unique_id": "macro.dbt.position", "macro_sql": "{% macro position(substring_text, string_text) -%}\n {{ return(adapter.dispatch('position', 'dbt') (substring_text, string_text)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__position"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2535648, "supported_languages": null}, "macro.dbt.default__position": {"name": "default__position", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/position.sql", "original_file_path": "macros/utils/position.sql", "unique_id": "macro.dbt.default__position", "macro_sql": "{% macro default__position(substring_text, string_text) %}\n\n position(\n {{ substring_text }} in {{ string_text }}\n )\n\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2537038, "supported_languages": null}, "macro.dbt.string_literal": {"name": "string_literal", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/literal.sql", "original_file_path": "macros/utils/literal.sql", "unique_id": "macro.dbt.string_literal", "macro_sql": "{%- macro string_literal(value) -%}\n {{ return(adapter.dispatch('string_literal', 'dbt') (value)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__string_literal"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.253951, "supported_languages": null}, "macro.dbt.default__string_literal": {"name": "default__string_literal", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/literal.sql", "original_file_path": "macros/utils/literal.sql", "unique_id": "macro.dbt.default__string_literal", "macro_sql": "{% macro default__string_literal(value) -%}\n '{{ value }}'\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.254057, "supported_languages": null}, "macro.dbt.type_string": {"name": "type_string", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_string", "macro_sql": "\n\n{%- macro type_string() -%}\n {{ return(adapter.dispatch('type_string', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_string"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.254996, "supported_languages": null}, "macro.dbt.default__type_string": {"name": "default__type_string", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_string", "macro_sql": "{% macro default__type_string() %}\n {{ return(api.Column.translate_type(\"string\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2551548, "supported_languages": null}, "macro.dbt.type_timestamp": {"name": "type_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_timestamp", "macro_sql": "\n\n{%- macro type_timestamp() -%}\n {{ return(adapter.dispatch('type_timestamp', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2553222, "supported_languages": null}, "macro.dbt.default__type_timestamp": {"name": "default__type_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_timestamp", "macro_sql": "{% macro default__type_timestamp() %}\n {{ return(api.Column.translate_type(\"timestamp\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.255476, "supported_languages": null}, "macro.dbt.type_float": {"name": "type_float", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_float", "macro_sql": "\n\n{%- macro type_float() -%}\n {{ return(adapter.dispatch('type_float', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_float"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.255645, "supported_languages": null}, "macro.dbt.default__type_float": {"name": "default__type_float", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_float", "macro_sql": "{% macro default__type_float() %}\n {{ return(api.Column.translate_type(\"float\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2558029, "supported_languages": null}, "macro.dbt.type_numeric": {"name": "type_numeric", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_numeric", "macro_sql": "\n\n{%- macro type_numeric() -%}\n {{ return(adapter.dispatch('type_numeric', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_numeric"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.255967, "supported_languages": null}, "macro.dbt.default__type_numeric": {"name": "default__type_numeric", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_numeric", "macro_sql": "{% macro default__type_numeric() %}\n {{ return(api.Column.numeric_type(\"numeric\", 28, 6)) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.256154, "supported_languages": null}, "macro.dbt.type_bigint": {"name": "type_bigint", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_bigint", "macro_sql": "\n\n{%- macro type_bigint() -%}\n {{ return(adapter.dispatch('type_bigint', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_bigint"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.256317, "supported_languages": null}, "macro.dbt.default__type_bigint": {"name": "default__type_bigint", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_bigint", "macro_sql": "{% macro default__type_bigint() %}\n {{ return(api.Column.translate_type(\"bigint\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.256474, "supported_languages": null}, "macro.dbt.type_int": {"name": "type_int", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_int", "macro_sql": "\n\n{%- macro type_int() -%}\n {{ return(adapter.dispatch('type_int', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_int"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.256723, "supported_languages": null}, "macro.dbt.default__type_int": {"name": "default__type_int", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_int", "macro_sql": "{%- macro default__type_int() -%}\n {{ return(api.Column.translate_type(\"integer\")) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2568839, "supported_languages": null}, "macro.dbt.type_boolean": {"name": "type_boolean", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_boolean", "macro_sql": "\n\n{%- macro type_boolean() -%}\n {{ return(adapter.dispatch('type_boolean', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_boolean"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.257049, "supported_languages": null}, "macro.dbt.default__type_boolean": {"name": "default__type_boolean", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_boolean", "macro_sql": "{%- macro default__type_boolean() -%}\n {{ return(api.Column.translate_type(\"boolean\")) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2572002, "supported_languages": null}, "macro.dbt.array_concat": {"name": "array_concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_concat.sql", "original_file_path": "macros/utils/array_concat.sql", "unique_id": "macro.dbt.array_concat", "macro_sql": "{% macro array_concat(array_1, array_2) -%}\n {{ return(adapter.dispatch('array_concat', 'dbt')(array_1, array_2)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__array_concat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2574809, "supported_languages": null}, "macro.dbt.default__array_concat": {"name": "default__array_concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_concat.sql", "original_file_path": "macros/utils/array_concat.sql", "unique_id": "macro.dbt.default__array_concat", "macro_sql": "{% macro default__array_concat(array_1, array_2) -%}\n array_cat({{ array_1 }}, {{ array_2 }})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.257617, "supported_languages": null}, "macro.dbt.bool_or": {"name": "bool_or", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/bool_or.sql", "original_file_path": "macros/utils/bool_or.sql", "unique_id": "macro.dbt.bool_or", "macro_sql": "{% macro bool_or(expression) -%}\n {{ return(adapter.dispatch('bool_or', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__bool_or"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2578712, "supported_languages": null}, "macro.dbt.default__bool_or": {"name": "default__bool_or", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/bool_or.sql", "original_file_path": "macros/utils/bool_or.sql", "unique_id": "macro.dbt.default__bool_or", "macro_sql": "{% macro default__bool_or(expression) -%}\n\n bool_or({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.257977, "supported_languages": null}, "macro.dbt.last_day": {"name": "last_day", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt.last_day", "macro_sql": "{% macro last_day(date, datepart) %}\n {{ return(adapter.dispatch('last_day', 'dbt') (date, datepart)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__last_day"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.258348, "supported_languages": null}, "macro.dbt.default_last_day": {"name": "default_last_day", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt.default_last_day", "macro_sql": "\n\n{%- macro default_last_day(date, datepart) -%}\n cast(\n {{dbt.dateadd('day', '-1',\n dbt.dateadd(datepart, '1', dbt.date_trunc(datepart, date))\n )}}\n as date)\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.dateadd", "macro.dbt.date_trunc"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.258624, "supported_languages": null}, "macro.dbt.default__last_day": {"name": "default__last_day", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt.default__last_day", "macro_sql": "{% macro default__last_day(date, datepart) -%}\n {{dbt.default_last_day(date, datepart)}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default_last_day"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.258776, "supported_languages": null}, "macro.dbt.split_part": {"name": "split_part", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt.split_part", "macro_sql": "{% macro split_part(string_text, delimiter_text, part_number) %}\n {{ return(adapter.dispatch('split_part', 'dbt') (string_text, delimiter_text, part_number)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__split_part"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.259321, "supported_languages": null}, "macro.dbt.default__split_part": {"name": "default__split_part", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt.default__split_part", "macro_sql": "{% macro default__split_part(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n {{ part_number }}\n )\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.259488, "supported_languages": null}, "macro.dbt._split_part_negative": {"name": "_split_part_negative", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt._split_part_negative", "macro_sql": "{% macro _split_part_negative(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n length({{ string_text }})\n - length(\n replace({{ string_text }}, {{ delimiter_text }}, '')\n ) + 2 + {{ part_number }}\n )\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.259715, "supported_languages": null}, "macro.dbt.date_trunc": {"name": "date_trunc", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_trunc.sql", "original_file_path": "macros/utils/date_trunc.sql", "unique_id": "macro.dbt.date_trunc", "macro_sql": "{% macro date_trunc(datepart, date) -%}\n {{ return(adapter.dispatch('date_trunc', 'dbt') (datepart, date)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__date_trunc"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.259997, "supported_languages": null}, "macro.dbt.default__date_trunc": {"name": "default__date_trunc", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_trunc.sql", "original_file_path": "macros/utils/date_trunc.sql", "unique_id": "macro.dbt.default__date_trunc", "macro_sql": "{% macro default__date_trunc(datepart, date) -%}\n date_trunc('{{datepart}}', {{date}})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.260129, "supported_languages": null}, "macro.dbt.array_construct": {"name": "array_construct", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_construct.sql", "original_file_path": "macros/utils/array_construct.sql", "unique_id": "macro.dbt.array_construct", "macro_sql": "{% macro array_construct(inputs=[], data_type=api.Column.translate_type('integer')) -%}\n {{ return(adapter.dispatch('array_construct', 'dbt')(inputs, data_type)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__array_construct"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.26051, "supported_languages": null}, "macro.dbt.default__array_construct": {"name": "default__array_construct", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_construct.sql", "original_file_path": "macros/utils/array_construct.sql", "unique_id": "macro.dbt.default__array_construct", "macro_sql": "{% macro default__array_construct(inputs, data_type) -%}\n {% if inputs|length > 0 %}\n array[ {{ inputs|join(' , ') }} ]\n {% else %}\n array[]::{{data_type}}[]\n {% endif %}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.260766, "supported_languages": null}, "macro.dbt.array_append": {"name": "array_append", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_append.sql", "original_file_path": "macros/utils/array_append.sql", "unique_id": "macro.dbt.array_append", "macro_sql": "{% macro array_append(array, new_element) -%}\n {{ return(adapter.dispatch('array_append', 'dbt')(array, new_element)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__array_append"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2610502, "supported_languages": null}, "macro.dbt.default__array_append": {"name": "default__array_append", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_append.sql", "original_file_path": "macros/utils/array_append.sql", "unique_id": "macro.dbt.default__array_append", "macro_sql": "{% macro default__array_append(array, new_element) -%}\n array_append({{ array }}, {{ new_element }})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.261183, "supported_languages": null}, "macro.dbt.create_schema": {"name": "create_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.create_schema", "macro_sql": "{% macro create_schema(relation) -%}\n {{ adapter.dispatch('create_schema', 'dbt')(relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__create_schema"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.261564, "supported_languages": null}, "macro.dbt.default__create_schema": {"name": "default__create_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.default__create_schema", "macro_sql": "{% macro default__create_schema(relation) -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier() }}\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.261753, "supported_languages": null}, "macro.dbt.drop_schema": {"name": "drop_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.drop_schema", "macro_sql": "{% macro drop_schema(relation) -%}\n {{ adapter.dispatch('drop_schema', 'dbt')(relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__drop_schema"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.261918, "supported_languages": null}, "macro.dbt.default__drop_schema": {"name": "default__drop_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.default__drop_schema", "macro_sql": "{% macro default__drop_schema(relation) -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier() }} cascade\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.262178, "supported_languages": null}, "macro.dbt.current_timestamp": {"name": "current_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.current_timestamp", "macro_sql": "{%- macro current_timestamp() -%}\n {{ adapter.dispatch('current_timestamp', 'dbt')() }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_postgres.postgres__current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2626739, "supported_languages": null}, "macro.dbt.default__current_timestamp": {"name": "default__current_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__current_timestamp", "macro_sql": "{% macro default__current_timestamp() -%}\n {{ exceptions.raise_not_implemented(\n 'current_timestamp macro not implemented for adapter ' + adapter.type()) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.262827, "supported_languages": null}, "macro.dbt.snapshot_get_time": {"name": "snapshot_get_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.snapshot_get_time", "macro_sql": "\n\n{%- macro snapshot_get_time() -%}\n {{ adapter.dispatch('snapshot_get_time', 'dbt')() }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_postgres.postgres__snapshot_get_time"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.262975, "supported_languages": null}, "macro.dbt.default__snapshot_get_time": {"name": "default__snapshot_get_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__snapshot_get_time", "macro_sql": "{% macro default__snapshot_get_time() %}\n {{ current_timestamp() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.263083, "supported_languages": null}, "macro.dbt.current_timestamp_backcompat": {"name": "current_timestamp_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.current_timestamp_backcompat", "macro_sql": "{% macro current_timestamp_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_backcompat', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__current_timestamp_backcompat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.263253, "supported_languages": null}, "macro.dbt.default__current_timestamp_backcompat": {"name": "default__current_timestamp_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__current_timestamp_backcompat", "macro_sql": "{% macro default__current_timestamp_backcompat() %}\n current_timestamp::timestamp\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.26333, "supported_languages": null}, "macro.dbt.current_timestamp_in_utc_backcompat": {"name": "current_timestamp_in_utc_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.current_timestamp_in_utc_backcompat", "macro_sql": "{% macro current_timestamp_in_utc_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_in_utc_backcompat', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__current_timestamp_in_utc_backcompat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2635038, "supported_languages": null}, "macro.dbt.default__current_timestamp_in_utc_backcompat": {"name": "default__current_timestamp_in_utc_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__current_timestamp_in_utc_backcompat", "macro_sql": "{% macro default__current_timestamp_in_utc_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_backcompat', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.current_timestamp_backcompat", "macro.dbt_postgres.postgres__current_timestamp_backcompat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.263676, "supported_languages": null}, "macro.dbt.get_create_index_sql": {"name": "get_create_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.get_create_index_sql", "macro_sql": "{% macro get_create_index_sql(relation, index_dict) -%}\n {{ return(adapter.dispatch('get_create_index_sql', 'dbt')(relation, index_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_create_index_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.264548, "supported_languages": null}, "macro.dbt.default__get_create_index_sql": {"name": "default__get_create_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__get_create_index_sql", "macro_sql": "{% macro default__get_create_index_sql(relation, index_dict) -%}\n {% do return(None) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.264692, "supported_languages": null}, "macro.dbt.create_indexes": {"name": "create_indexes", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.create_indexes", "macro_sql": "{% macro create_indexes(relation) -%}\n {{ adapter.dispatch('create_indexes', 'dbt')(relation) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.264854, "supported_languages": null}, "macro.dbt.default__create_indexes": {"name": "default__create_indexes", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__create_indexes", "macro_sql": "{% macro default__create_indexes(relation) -%}\n {%- set _indexes = config.get('indexes', default=[]) -%}\n\n {% for _index_dict in _indexes %}\n {% set create_index_sql = get_create_index_sql(relation, _index_dict) %}\n {% if create_index_sql %}\n {% do run_query(create_index_sql) %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_create_index_sql", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.265263, "supported_languages": null}, "macro.dbt.get_drop_index_sql": {"name": "get_drop_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.get_drop_index_sql", "macro_sql": "{% macro get_drop_index_sql(relation, index_name) -%}\n {{ adapter.dispatch('get_drop_index_sql', 'dbt')(relation, index_name) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_drop_index_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.265459, "supported_languages": null}, "macro.dbt.default__get_drop_index_sql": {"name": "default__get_drop_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__get_drop_index_sql", "macro_sql": "{% macro default__get_drop_index_sql(relation, index_name) -%}\n {{ exceptions.raise_compiler_error(\"`get_drop_index_sql has not been implemented for this adapter.\") }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.265602, "supported_languages": null}, "macro.dbt.get_show_indexes_sql": {"name": "get_show_indexes_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.get_show_indexes_sql", "macro_sql": "{% macro get_show_indexes_sql(relation) -%}\n {{ adapter.dispatch('get_show_indexes_sql', 'dbt')(relation) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_show_indexes_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.265763, "supported_languages": null}, "macro.dbt.default__get_show_indexes_sql": {"name": "default__get_show_indexes_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__get_show_indexes_sql", "macro_sql": "{% macro default__get_show_indexes_sql(relation) -%}\n {{ exceptions.raise_compiler_error(\"`get_show_indexes_sql has not been implemented for this adapter.\") }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.265894, "supported_languages": null}, "macro.dbt.make_intermediate_relation": {"name": "make_intermediate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.make_intermediate_relation", "macro_sql": "{% macro make_intermediate_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_intermediate_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_intermediate_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.268777, "supported_languages": null}, "macro.dbt.default__make_intermediate_relation": {"name": "default__make_intermediate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__make_intermediate_relation", "macro_sql": "{% macro default__make_intermediate_relation(base_relation, suffix) %}\n {{ return(default__make_temp_relation(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__make_temp_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.268948, "supported_languages": null}, "macro.dbt.make_temp_relation": {"name": "make_temp_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.make_temp_relation", "macro_sql": "{% macro make_temp_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_temp_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_temp_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.269174, "supported_languages": null}, "macro.dbt.default__make_temp_relation": {"name": "default__make_temp_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__make_temp_relation", "macro_sql": "{% macro default__make_temp_relation(base_relation, suffix) %}\n {%- set temp_identifier = base_relation.identifier ~ suffix -%}\n {%- set temp_relation = base_relation.incorporate(\n path={\"identifier\": temp_identifier}) -%}\n\n {{ return(temp_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.269548, "supported_languages": null}, "macro.dbt.make_backup_relation": {"name": "make_backup_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.make_backup_relation", "macro_sql": "{% macro make_backup_relation(base_relation, backup_relation_type, suffix='__dbt_backup') %}\n {{ return(adapter.dispatch('make_backup_relation', 'dbt')(base_relation, backup_relation_type, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_backup_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.269803, "supported_languages": null}, "macro.dbt.default__make_backup_relation": {"name": "default__make_backup_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__make_backup_relation", "macro_sql": "{% macro default__make_backup_relation(base_relation, backup_relation_type, suffix) %}\n {%- set backup_identifier = base_relation.identifier ~ suffix -%}\n {%- set backup_relation = base_relation.incorporate(\n path={\"identifier\": backup_identifier},\n type=backup_relation_type\n ) -%}\n {{ return(backup_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.270133, "supported_languages": null}, "macro.dbt.truncate_relation": {"name": "truncate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.truncate_relation", "macro_sql": "{% macro truncate_relation(relation) -%}\n {{ return(adapter.dispatch('truncate_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__truncate_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.270325, "supported_languages": null}, "macro.dbt.default__truncate_relation": {"name": "default__truncate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__truncate_relation", "macro_sql": "{% macro default__truncate_relation(relation) -%}\n {% call statement('truncate_relation') -%}\n truncate table {{ relation }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2704918, "supported_languages": null}, "macro.dbt.rename_relation": {"name": "rename_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.rename_relation", "macro_sql": "{% macro rename_relation(from_relation, to_relation) -%}\n {{ return(adapter.dispatch('rename_relation', 'dbt')(from_relation, to_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__rename_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.270706, "supported_languages": null}, "macro.dbt.default__rename_relation": {"name": "default__rename_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__rename_relation", "macro_sql": "{% macro default__rename_relation(from_relation, to_relation) -%}\n {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %}\n {% call statement('rename_relation') -%}\n alter table {{ from_relation }} rename to {{ target_name }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.270998, "supported_languages": null}, "macro.dbt.get_or_create_relation": {"name": "get_or_create_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.get_or_create_relation", "macro_sql": "{% macro get_or_create_relation(database, schema, identifier, type) -%}\n {{ return(adapter.dispatch('get_or_create_relation', 'dbt')(database, schema, identifier, type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_or_create_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.271255, "supported_languages": null}, "macro.dbt.default__get_or_create_relation": {"name": "default__get_or_create_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__get_or_create_relation", "macro_sql": "{% macro default__get_or_create_relation(database, schema, identifier, type) %}\n {%- set target_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n\n {% if target_relation %}\n {% do return([true, target_relation]) %}\n {% endif %}\n\n {%- set new_relation = api.Relation.create(\n database=database,\n schema=schema,\n identifier=identifier,\n type=type\n ) -%}\n {% do return([false, new_relation]) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.271829, "supported_languages": null}, "macro.dbt.load_cached_relation": {"name": "load_cached_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.load_cached_relation", "macro_sql": "{% macro load_cached_relation(relation) %}\n {% do return(adapter.get_relation(\n database=relation.database,\n schema=relation.schema,\n identifier=relation.identifier\n )) -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.27207, "supported_languages": null}, "macro.dbt.load_relation": {"name": "load_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.load_relation", "macro_sql": "{% macro load_relation(relation) %}\n {{ return(load_cached_relation(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.27222, "supported_languages": null}, "macro.dbt.drop_relation_if_exists": {"name": "drop_relation_if_exists", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.drop_relation_if_exists", "macro_sql": "{% macro drop_relation_if_exists(relation) %}\n {% if relation is not none %}\n {{ adapter.drop_relation(relation) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.272419, "supported_languages": null}, "macro.dbt.collect_freshness": {"name": "collect_freshness", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "unique_id": "macro.dbt.collect_freshness", "macro_sql": "{% macro collect_freshness(source, loaded_at_field, filter) %}\n {{ return(adapter.dispatch('collect_freshness', 'dbt')(source, loaded_at_field, filter))}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__collect_freshness"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.272854, "supported_languages": null}, "macro.dbt.default__collect_freshness": {"name": "default__collect_freshness", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "unique_id": "macro.dbt.default__collect_freshness", "macro_sql": "{% macro default__collect_freshness(source, loaded_at_field, filter) %}\n {% call statement('collect_freshness', fetch_result=True, auto_begin=False) -%}\n select\n max({{ loaded_at_field }}) as max_loaded_at,\n {{ current_timestamp() }} as snapshotted_at\n from {{ source }}\n {% if filter %}\n where {{ filter }}\n {% endif %}\n {% endcall %}\n {{ return(load_result('collect_freshness')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.273275, "supported_languages": null}, "macro.dbt.validate_sql": {"name": "validate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/validate_sql.sql", "original_file_path": "macros/adapters/validate_sql.sql", "unique_id": "macro.dbt.validate_sql", "macro_sql": "{% macro validate_sql(sql) -%}\n {{ return(adapter.dispatch('validate_sql', 'dbt')(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__validate_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2735639, "supported_languages": null}, "macro.dbt.default__validate_sql": {"name": "default__validate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/validate_sql.sql", "original_file_path": "macros/adapters/validate_sql.sql", "unique_id": "macro.dbt.default__validate_sql", "macro_sql": "{% macro default__validate_sql(sql) -%}\n {% call statement('validate_sql') -%}\n explain {{ sql }}\n {% endcall %}\n {{ return(load_result('validate_sql')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.273793, "supported_languages": null}, "macro.dbt.copy_grants": {"name": "copy_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.copy_grants", "macro_sql": "{% macro copy_grants() %}\n {{ return(adapter.dispatch('copy_grants', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__copy_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.275553, "supported_languages": null}, "macro.dbt.default__copy_grants": {"name": "default__copy_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__copy_grants", "macro_sql": "{% macro default__copy_grants() %}\n {{ return(True) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2756748, "supported_languages": null}, "macro.dbt.support_multiple_grantees_per_dcl_statement": {"name": "support_multiple_grantees_per_dcl_statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.support_multiple_grantees_per_dcl_statement", "macro_sql": "{% macro support_multiple_grantees_per_dcl_statement() %}\n {{ return(adapter.dispatch('support_multiple_grantees_per_dcl_statement', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__support_multiple_grantees_per_dcl_statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2758532, "supported_languages": null}, "macro.dbt.default__support_multiple_grantees_per_dcl_statement": {"name": "default__support_multiple_grantees_per_dcl_statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__support_multiple_grantees_per_dcl_statement", "macro_sql": "\n\n{%- macro default__support_multiple_grantees_per_dcl_statement() -%}\n {{ return(True) }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.275969, "supported_languages": null}, "macro.dbt.should_revoke": {"name": "should_revoke", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.should_revoke", "macro_sql": "{% macro should_revoke(existing_relation, full_refresh_mode=True) %}\n\n {% if not existing_relation %}\n {#-- The table doesn't already exist, so no grants to copy over --#}\n {{ return(False) }}\n {% elif full_refresh_mode %}\n {#-- The object is being REPLACED -- whether grants are copied over depends on the value of user config --#}\n {{ return(copy_grants()) }}\n {% else %}\n {#-- The table is being merged/upserted/inserted -- grants will be carried over --#}\n {{ return(True) }}\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.copy_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.276321, "supported_languages": null}, "macro.dbt.get_show_grant_sql": {"name": "get_show_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_show_grant_sql", "macro_sql": "{% macro get_show_grant_sql(relation) %}\n {{ return(adapter.dispatch(\"get_show_grant_sql\", \"dbt\")(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_show_grant_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.276514, "supported_languages": null}, "macro.dbt.default__get_show_grant_sql": {"name": "default__get_show_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_show_grant_sql", "macro_sql": "{% macro default__get_show_grant_sql(relation) %}\n show grants on {{ relation }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2766201, "supported_languages": null}, "macro.dbt.get_grant_sql": {"name": "get_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_grant_sql", "macro_sql": "{% macro get_grant_sql(relation, privilege, grantees) %}\n {{ return(adapter.dispatch('get_grant_sql', 'dbt')(relation, privilege, grantees)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_grant_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.276926, "supported_languages": null}, "macro.dbt.default__get_grant_sql": {"name": "default__get_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_grant_sql", "macro_sql": "\n\n{%- macro default__get_grant_sql(relation, privilege, grantees) -%}\n grant {{ privilege }} on {{ relation }} to {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.277123, "supported_languages": null}, "macro.dbt.get_revoke_sql": {"name": "get_revoke_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_revoke_sql", "macro_sql": "{% macro get_revoke_sql(relation, privilege, grantees) %}\n {{ return(adapter.dispatch('get_revoke_sql', 'dbt')(relation, privilege, grantees)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_revoke_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.277358, "supported_languages": null}, "macro.dbt.default__get_revoke_sql": {"name": "default__get_revoke_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_revoke_sql", "macro_sql": "\n\n{%- macro default__get_revoke_sql(relation, privilege, grantees) -%}\n revoke {{ privilege }} on {{ relation }} from {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2775512, "supported_languages": null}, "macro.dbt.get_dcl_statement_list": {"name": "get_dcl_statement_list", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_dcl_statement_list", "macro_sql": "{% macro get_dcl_statement_list(relation, grant_config, get_dcl_macro) %}\n {{ return(adapter.dispatch('get_dcl_statement_list', 'dbt')(relation, grant_config, get_dcl_macro)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_dcl_statement_list"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.277785, "supported_languages": null}, "macro.dbt.default__get_dcl_statement_list": {"name": "default__get_dcl_statement_list", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_dcl_statement_list", "macro_sql": "\n\n{%- macro default__get_dcl_statement_list(relation, grant_config, get_dcl_macro) -%}\n {#\n -- Unpack grant_config into specific privileges and the set of users who need them granted/revoked.\n -- Depending on whether this database supports multiple grantees per statement, pass in the list of\n -- all grantees per privilege, or (if not) template one statement per privilege-grantee pair.\n -- `get_dcl_macro` will be either `get_grant_sql` or `get_revoke_sql`\n #}\n {%- set dcl_statements = [] -%}\n {%- for privilege, grantees in grant_config.items() %}\n {%- if support_multiple_grantees_per_dcl_statement() and grantees -%}\n {%- set dcl = get_dcl_macro(relation, privilege, grantees) -%}\n {%- do dcl_statements.append(dcl) -%}\n {%- else -%}\n {%- for grantee in grantees -%}\n {% set dcl = get_dcl_macro(relation, privilege, [grantee]) %}\n {%- do dcl_statements.append(dcl) -%}\n {% endfor -%}\n {%- endif -%}\n {%- endfor -%}\n {{ return(dcl_statements) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.support_multiple_grantees_per_dcl_statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.278485, "supported_languages": null}, "macro.dbt.call_dcl_statements": {"name": "call_dcl_statements", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.call_dcl_statements", "macro_sql": "{% macro call_dcl_statements(dcl_statement_list) %}\n {{ return(adapter.dispatch(\"call_dcl_statements\", \"dbt\")(dcl_statement_list)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__call_dcl_statements"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.278682, "supported_languages": null}, "macro.dbt.default__call_dcl_statements": {"name": "default__call_dcl_statements", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__call_dcl_statements", "macro_sql": "{% macro default__call_dcl_statements(dcl_statement_list) %}\n {#\n -- By default, supply all grant + revoke statements in a single semicolon-separated block,\n -- so that they're all processed together.\n\n -- Some databases do not support this. Those adapters will need to override this macro\n -- to run each statement individually.\n #}\n {% call statement('grants') %}\n {% for dcl_statement in dcl_statement_list %}\n {{ dcl_statement }};\n {% endfor %}\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2789361, "supported_languages": null}, "macro.dbt.apply_grants": {"name": "apply_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.apply_grants", "macro_sql": "{% macro apply_grants(relation, grant_config, should_revoke) %}\n {{ return(adapter.dispatch(\"apply_grants\", \"dbt\")(relation, grant_config, should_revoke)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__apply_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2791739, "supported_languages": null}, "macro.dbt.default__apply_grants": {"name": "default__apply_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__apply_grants", "macro_sql": "{% macro default__apply_grants(relation, grant_config, should_revoke=True) %}\n {#-- If grant_config is {} or None, this is a no-op --#}\n {% if grant_config %}\n {% if should_revoke %}\n {#-- We think previous grants may have carried over --#}\n {#-- Show current grants and calculate diffs --#}\n {% set current_grants_table = run_query(get_show_grant_sql(relation)) %}\n {% set current_grants_dict = adapter.standardize_grants_dict(current_grants_table) %}\n {% set needs_granting = diff_of_two_dicts(grant_config, current_grants_dict) %}\n {% set needs_revoking = diff_of_two_dicts(current_grants_dict, grant_config) %}\n {% if not (needs_granting or needs_revoking) %}\n {{ log('On ' ~ relation ~': All grants are in place, no revocation or granting needed.')}}\n {% endif %}\n {% else %}\n {#-- We don't think there's any chance of previous grants having carried over. --#}\n {#-- Jump straight to granting what the user has configured. --#}\n {% set needs_revoking = {} %}\n {% set needs_granting = grant_config %}\n {% endif %}\n {% if needs_granting or needs_revoking %}\n {% set revoke_statement_list = get_dcl_statement_list(relation, needs_revoking, get_revoke_sql) %}\n {% set grant_statement_list = get_dcl_statement_list(relation, needs_granting, get_grant_sql) %}\n {% set dcl_statement_list = revoke_statement_list + grant_statement_list %}\n {% if dcl_statement_list %}\n {{ call_dcl_statements(dcl_statement_list) }}\n {% endif %}\n {% endif %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query", "macro.dbt.get_show_grant_sql", "macro.dbt.get_dcl_statement_list", "macro.dbt.call_dcl_statements"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2803478, "supported_languages": null}, "macro.dbt.get_show_sql": {"name": "get_show_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/show.sql", "original_file_path": "macros/adapters/show.sql", "unique_id": "macro.dbt.get_show_sql", "macro_sql": "{% macro get_show_sql(compiled_code, sql_header, limit) -%}\n {%- if sql_header -%}\n {{ sql_header }}\n {%- endif -%}\n {%- if limit is not none -%}\n {{ get_limit_subquery_sql(compiled_code, limit) }}\n {%- else -%}\n {{ compiled_code }}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_limit_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2808928, "supported_languages": null}, "macro.dbt.get_limit_subquery_sql": {"name": "get_limit_subquery_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/show.sql", "original_file_path": "macros/adapters/show.sql", "unique_id": "macro.dbt.get_limit_subquery_sql", "macro_sql": "{% macro get_limit_subquery_sql(sql, limit) %}\n {{ adapter.dispatch('get_limit_subquery_sql', 'dbt')(sql, limit) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_limit_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.281089, "supported_languages": null}, "macro.dbt.default__get_limit_subquery_sql": {"name": "default__get_limit_subquery_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/show.sql", "original_file_path": "macros/adapters/show.sql", "unique_id": "macro.dbt.default__get_limit_subquery_sql", "macro_sql": "{% macro default__get_limit_subquery_sql(sql, limit) %}\n select *\n from (\n {{ sql }}\n ) as model_limit_subq\n limit {{ limit }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.281228, "supported_languages": null}, "macro.dbt.alter_column_comment": {"name": "alter_column_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.alter_column_comment", "macro_sql": "{% macro alter_column_comment(relation, column_dict) -%}\n {{ return(adapter.dispatch('alter_column_comment', 'dbt')(relation, column_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.281921, "supported_languages": null}, "macro.dbt.default__alter_column_comment": {"name": "default__alter_column_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.default__alter_column_comment", "macro_sql": "{% macro default__alter_column_comment(relation, column_dict) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_column_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2820952, "supported_languages": null}, "macro.dbt.alter_relation_comment": {"name": "alter_relation_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.alter_relation_comment", "macro_sql": "{% macro alter_relation_comment(relation, relation_comment) -%}\n {{ return(adapter.dispatch('alter_relation_comment', 'dbt')(relation, relation_comment)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__alter_relation_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.282303, "supported_languages": null}, "macro.dbt.default__alter_relation_comment": {"name": "default__alter_relation_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.default__alter_relation_comment", "macro_sql": "{% macro default__alter_relation_comment(relation, relation_comment) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_relation_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.282475, "supported_languages": null}, "macro.dbt.persist_docs": {"name": "persist_docs", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.persist_docs", "macro_sql": "{% macro persist_docs(relation, model, for_relation=true, for_columns=true) -%}\n {{ return(adapter.dispatch('persist_docs', 'dbt')(relation, model, for_relation, for_columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.282752, "supported_languages": null}, "macro.dbt.default__persist_docs": {"name": "default__persist_docs", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.default__persist_docs", "macro_sql": "{% macro default__persist_docs(relation, model, for_relation, for_columns) -%}\n {% if for_relation and config.persist_relation_docs() and model.description %}\n {% do run_query(alter_relation_comment(relation, model.description)) %}\n {% endif %}\n\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {% do run_query(alter_column_comment(relation, model.columns)) %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query", "macro.dbt.alter_relation_comment", "macro.dbt.alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.283254, "supported_languages": null}, "macro.dbt.get_catalog": {"name": "get_catalog", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_catalog", "macro_sql": "{% macro get_catalog(information_schema, schemas) -%}\n {{ return(adapter.dispatch('get_catalog', 'dbt')(information_schema, schemas)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_catalog"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2848241, "supported_languages": null}, "macro.dbt.default__get_catalog": {"name": "default__get_catalog", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_catalog", "macro_sql": "{% macro default__get_catalog(information_schema, schemas) -%}\n\n {% set typename = adapter.type() %}\n {% set msg -%}\n get_catalog not implemented for {{ typename }}\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.285086, "supported_languages": null}, "macro.dbt.information_schema_name": {"name": "information_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.information_schema_name", "macro_sql": "{% macro information_schema_name(database) %}\n {{ return(adapter.dispatch('information_schema_name', 'dbt')(database)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__information_schema_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2852778, "supported_languages": null}, "macro.dbt.default__information_schema_name": {"name": "default__information_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__information_schema_name", "macro_sql": "{% macro default__information_schema_name(database) -%}\n {%- if database -%}\n {{ database }}.INFORMATION_SCHEMA\n {%- else -%}\n INFORMATION_SCHEMA\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.285444, "supported_languages": null}, "macro.dbt.list_schemas": {"name": "list_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.list_schemas", "macro_sql": "{% macro list_schemas(database) -%}\n {{ return(adapter.dispatch('list_schemas', 'dbt')(database)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__list_schemas"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.285694, "supported_languages": null}, "macro.dbt.default__list_schemas": {"name": "default__list_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__list_schemas", "macro_sql": "{% macro default__list_schemas(database) -%}\n {% set sql %}\n select distinct schema_name\n from {{ information_schema_name(database) }}.SCHEMATA\n where catalog_name ilike '{{ database }}'\n {% endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.information_schema_name", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.28595, "supported_languages": null}, "macro.dbt.check_schema_exists": {"name": "check_schema_exists", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.check_schema_exists", "macro_sql": "{% macro check_schema_exists(information_schema, schema) -%}\n {{ return(adapter.dispatch('check_schema_exists', 'dbt')(information_schema, schema)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__check_schema_exists"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2861598, "supported_languages": null}, "macro.dbt.default__check_schema_exists": {"name": "default__check_schema_exists", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__check_schema_exists", "macro_sql": "{% macro default__check_schema_exists(information_schema, schema) -%}\n {% set sql -%}\n select count(*)\n from {{ information_schema.replace(information_schema_view='SCHEMATA') }}\n where catalog_name='{{ information_schema.database }}'\n and schema_name='{{ schema }}'\n {%- endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.replace", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2864761, "supported_languages": null}, "macro.dbt.list_relations_without_caching": {"name": "list_relations_without_caching", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.list_relations_without_caching", "macro_sql": "{% macro list_relations_without_caching(schema_relation) %}\n {{ return(adapter.dispatch('list_relations_without_caching', 'dbt')(schema_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__list_relations_without_caching"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.286675, "supported_languages": null}, "macro.dbt.default__list_relations_without_caching": {"name": "default__list_relations_without_caching", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__list_relations_without_caching", "macro_sql": "{% macro default__list_relations_without_caching(schema_relation) %}\n {{ exceptions.raise_not_implemented(\n 'list_relations_without_caching macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2868428, "supported_languages": null}, "macro.dbt.drop_relation": {"name": "drop_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/drop_relation.sql", "original_file_path": "macros/adapters/drop_relation.sql", "unique_id": "macro.dbt.drop_relation", "macro_sql": "{% macro drop_relation(relation) -%}\n {{ return(adapter.dispatch('drop_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__drop_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2877948, "supported_languages": null}, "macro.dbt.default__drop_relation": {"name": "default__drop_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/drop_relation.sql", "original_file_path": "macros/adapters/drop_relation.sql", "unique_id": "macro.dbt.default__drop_relation", "macro_sql": "{% macro default__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n {%- if relation.is_table -%}\n {{- drop_table(relation) -}}\n {%- elif relation.is_view -%}\n {{- drop_view(relation) -}}\n {%- elif relation.is_materialized_view -%}\n {{- drop_materialized_view(relation) -}}\n {%- else -%}\n drop {{ relation.type }} if exists {{ relation }} cascade\n {%- endif -%}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.drop_table", "macro.dbt.drop_view", "macro.dbt.drop_materialized_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.28826, "supported_languages": null}, "macro.dbt.drop_table": {"name": "drop_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/drop_relation.sql", "original_file_path": "macros/adapters/drop_relation.sql", "unique_id": "macro.dbt.drop_table", "macro_sql": "{% macro drop_table(relation) -%}\n {{ return(adapter.dispatch('drop_table', 'dbt')(relation)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__drop_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.288448, "supported_languages": null}, "macro.dbt.default__drop_table": {"name": "default__drop_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/drop_relation.sql", "original_file_path": "macros/adapters/drop_relation.sql", "unique_id": "macro.dbt.default__drop_table", "macro_sql": "{% macro default__drop_table(relation) -%}\n drop table if exists {{ relation }} cascade\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.288556, "supported_languages": null}, "macro.dbt.drop_view": {"name": "drop_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/drop_relation.sql", "original_file_path": "macros/adapters/drop_relation.sql", "unique_id": "macro.dbt.drop_view", "macro_sql": "{% macro drop_view(relation) -%}\n {{ return(adapter.dispatch('drop_view', 'dbt')(relation)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__drop_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.288737, "supported_languages": null}, "macro.dbt.default__drop_view": {"name": "default__drop_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/drop_relation.sql", "original_file_path": "macros/adapters/drop_relation.sql", "unique_id": "macro.dbt.default__drop_view", "macro_sql": "{% macro default__drop_view(relation) -%}\n drop view if exists {{ relation }} cascade\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.288845, "supported_languages": null}, "macro.dbt.drop_materialized_view": {"name": "drop_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/drop_relation.sql", "original_file_path": "macros/adapters/drop_relation.sql", "unique_id": "macro.dbt.drop_materialized_view", "macro_sql": "{% macro drop_materialized_view(relation) -%}\n {{ return(adapter.dispatch('drop_materialized_view', 'dbt')(relation)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__drop_materialized_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.289025, "supported_languages": null}, "macro.dbt.default__drop_materialized_view": {"name": "default__drop_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/drop_relation.sql", "original_file_path": "macros/adapters/drop_relation.sql", "unique_id": "macro.dbt.default__drop_materialized_view", "macro_sql": "{% macro default__drop_materialized_view(relation) -%}\n drop materialized view if exists {{ relation }} cascade\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.289133, "supported_languages": null}, "macro.dbt.get_columns_in_relation": {"name": "get_columns_in_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_columns_in_relation", "macro_sql": "{% macro get_columns_in_relation(relation) -%}\n {{ return(adapter.dispatch('get_columns_in_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_columns_in_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.291446, "supported_languages": null}, "macro.dbt.default__get_columns_in_relation": {"name": "default__get_columns_in_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_columns_in_relation", "macro_sql": "{% macro default__get_columns_in_relation(relation) -%}\n {{ exceptions.raise_not_implemented(\n 'get_columns_in_relation macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2916129, "supported_languages": null}, "macro.dbt.sql_convert_columns_in_relation": {"name": "sql_convert_columns_in_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.sql_convert_columns_in_relation", "macro_sql": "{% macro sql_convert_columns_in_relation(table) -%}\n {% set columns = [] %}\n {% for row in table %}\n {% do columns.append(api.Column(*row)) %}\n {% endfor %}\n {{ return(columns) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2919302, "supported_languages": null}, "macro.dbt.get_empty_subquery_sql": {"name": "get_empty_subquery_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_empty_subquery_sql", "macro_sql": "{% macro get_empty_subquery_sql(select_sql, select_sql_header=none) -%}\n {{ return(adapter.dispatch('get_empty_subquery_sql', 'dbt')(select_sql, select_sql_header)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_empty_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2921631, "supported_languages": null}, "macro.dbt.default__get_empty_subquery_sql": {"name": "default__get_empty_subquery_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_empty_subquery_sql", "macro_sql": "{% macro default__get_empty_subquery_sql(select_sql, select_sql_header=none) %}\n {%- if select_sql_header is not none -%}\n {{ select_sql_header }}\n {%- endif -%}\n select * from (\n {{ select_sql }}\n ) as __dbt_sbq\n where false\n limit 0\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.292375, "supported_languages": null}, "macro.dbt.get_empty_schema_sql": {"name": "get_empty_schema_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_empty_schema_sql", "macro_sql": "{% macro get_empty_schema_sql(columns) -%}\n {{ return(adapter.dispatch('get_empty_schema_sql', 'dbt')(columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_empty_schema_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.292562, "supported_languages": null}, "macro.dbt.default__get_empty_schema_sql": {"name": "default__get_empty_schema_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_empty_schema_sql", "macro_sql": "{% macro default__get_empty_schema_sql(columns) %}\n {%- set col_err = [] -%}\n select\n {% for i in columns %}\n {%- set col = columns[i] -%}\n {%- if col['data_type'] is not defined -%}\n {{ col_err.append(col['name']) }}\n {%- endif -%}\n {% set col_name = adapter.quote(col['name']) if col.get('quote') else col['name'] %}\n cast(null as {{ col['data_type'] }}) as {{ col_name }}{{ \", \" if not loop.last }}\n {%- endfor -%}\n {%- if (col_err | length) > 0 -%}\n {{ exceptions.column_type_missing(column_names=col_err) }}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.293418, "supported_languages": null}, "macro.dbt.get_column_schema_from_query": {"name": "get_column_schema_from_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_column_schema_from_query", "macro_sql": "{% macro get_column_schema_from_query(select_sql, select_sql_header=none) -%}\n {% set columns = [] %}\n {# -- Using an 'empty subquery' here to get the same schema as the given select_sql statement, without necessitating a data scan.#}\n {% set sql = get_empty_subquery_sql(select_sql, select_sql_header) %}\n {% set column_schema = adapter.get_column_schema_from_query(sql) %}\n {{ return(column_schema) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_empty_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2937758, "supported_languages": null}, "macro.dbt.get_columns_in_query": {"name": "get_columns_in_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_columns_in_query", "macro_sql": "{% macro get_columns_in_query(select_sql) -%}\n {{ return(adapter.dispatch('get_columns_in_query', 'dbt')(select_sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_columns_in_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2939641, "supported_languages": null}, "macro.dbt.default__get_columns_in_query": {"name": "default__get_columns_in_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_columns_in_query", "macro_sql": "{% macro default__get_columns_in_query(select_sql) %}\n {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%}\n {{ get_empty_subquery_sql(select_sql) }}\n {% endcall %}\n {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.get_empty_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.294328, "supported_languages": null}, "macro.dbt.alter_column_type": {"name": "alter_column_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.alter_column_type", "macro_sql": "{% macro alter_column_type(relation, column_name, new_column_type) -%}\n {{ return(adapter.dispatch('alter_column_type', 'dbt')(relation, column_name, new_column_type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__alter_column_type"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2945652, "supported_languages": null}, "macro.dbt.default__alter_column_type": {"name": "default__alter_column_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__alter_column_type", "macro_sql": "{% macro default__alter_column_type(relation, column_name, new_column_type) -%}\n {#\n 1. Create a new column (w/ temp name and correct type)\n 2. Copy data over to it\n 3. Drop the existing column (cascade!)\n 4. Rename the new column to existing column\n #}\n {%- set tmp_column = column_name + \"__dbt_alter\" -%}\n\n {% call statement('alter_column_type') %}\n alter table {{ relation }} add column {{ adapter.quote(tmp_column) }} {{ new_column_type }};\n update {{ relation }} set {{ adapter.quote(tmp_column) }} = {{ adapter.quote(column_name) }};\n alter table {{ relation }} drop column {{ adapter.quote(column_name) }} cascade;\n alter table {{ relation }} rename column {{ adapter.quote(tmp_column) }} to {{ adapter.quote(column_name) }}\n {% endcall %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.295179, "supported_languages": null}, "macro.dbt.alter_relation_add_remove_columns": {"name": "alter_relation_add_remove_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.alter_relation_add_remove_columns", "macro_sql": "{% macro alter_relation_add_remove_columns(relation, add_columns = none, remove_columns = none) -%}\n {{ return(adapter.dispatch('alter_relation_add_remove_columns', 'dbt')(relation, add_columns, remove_columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__alter_relation_add_remove_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.295451, "supported_languages": null}, "macro.dbt.default__alter_relation_add_remove_columns": {"name": "default__alter_relation_add_remove_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__alter_relation_add_remove_columns", "macro_sql": "{% macro default__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}\n\n {% if add_columns is none %}\n {% set add_columns = [] %}\n {% endif %}\n {% if remove_columns is none %}\n {% set remove_columns = [] %}\n {% endif %}\n\n {% set sql -%}\n\n alter {{ relation.type }} {{ relation }}\n\n {% for column in add_columns %}\n add column {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }}\n {% endfor %}{{ ',' if add_columns and remove_columns }}\n\n {% for column in remove_columns %}\n drop column {{ column.name }}{{ ',' if not loop.last }}\n {% endfor %}\n\n {%- endset -%}\n\n {% do run_query(sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.296254, "supported_languages": null}, "macro.dbt.resolve_model_name": {"name": "resolve_model_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.resolve_model_name", "macro_sql": "{% macro resolve_model_name(input_model_name) %}\n {{ return(adapter.dispatch('resolve_model_name', 'dbt')(input_model_name)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__resolve_model_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.29796, "supported_languages": null}, "macro.dbt.default__resolve_model_name": {"name": "default__resolve_model_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.default__resolve_model_name", "macro_sql": "\n\n{%- macro default__resolve_model_name(input_model_name) -%}\n {{ input_model_name | string | replace('\"', '\\\"') }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.2981188, "supported_languages": null}, "macro.dbt.build_ref_function": {"name": "build_ref_function", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.build_ref_function", "macro_sql": "{% macro build_ref_function(model) %}\n\n {%- set ref_dict = {} -%}\n {%- for _ref in model.refs -%}\n {% set _ref_args = [_ref.get('package'), _ref['name']] if _ref.get('package') else [_ref['name'],] %}\n {%- set resolved = ref(*_ref_args, v=_ref.get('version')) -%}\n {%- if _ref.get('version') -%}\n {% do _ref_args.extend([\"v\" ~ _ref['version']]) %}\n {%- endif -%}\n {%- do ref_dict.update({_ref_args | join('.'): resolve_model_name(resolved)}) -%}\n {%- endfor -%}\n\ndef ref(*args, **kwargs):\n refs = {{ ref_dict | tojson }}\n key = '.'.join(args)\n version = kwargs.get(\"v\") or kwargs.get(\"version\")\n if version:\n key += f\".v{version}\"\n dbt_load_df_function = kwargs.get(\"dbt_load_df_function\")\n return dbt_load_df_function(refs[key])\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.resolve_model_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.298965, "supported_languages": null}, "macro.dbt.build_source_function": {"name": "build_source_function", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.build_source_function", "macro_sql": "{% macro build_source_function(model) %}\n\n {%- set source_dict = {} -%}\n {%- for _source in model.sources -%}\n {%- set resolved = source(*_source) -%}\n {%- do source_dict.update({_source | join('.'): resolve_model_name(resolved)}) -%}\n {%- endfor -%}\n\ndef source(*args, dbt_load_df_function):\n sources = {{ source_dict | tojson }}\n key = '.'.join(args)\n return dbt_load_df_function(sources[key])\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.resolve_model_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.299389, "supported_languages": null}, "macro.dbt.build_config_dict": {"name": "build_config_dict", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.build_config_dict", "macro_sql": "{% macro build_config_dict(model) %}\n {%- set config_dict = {} -%}\n {% set config_dbt_used = zip(model.config.config_keys_used, model.config.config_keys_defaults) | list %}\n {%- for key, default in config_dbt_used -%}\n {# weird type testing with enum, would be much easier to write this logic in Python! #}\n {%- if key == \"language\" -%}\n {%- set value = \"python\" -%}\n {%- endif -%}\n {%- set value = model.config.get(key, default) -%}\n {%- do config_dict.update({key: value}) -%}\n {%- endfor -%}\nconfig_dict = {{ config_dict }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.299982, "supported_languages": null}, "macro.dbt.py_script_postfix": {"name": "py_script_postfix", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.py_script_postfix", "macro_sql": "{% macro py_script_postfix(model) %}\n# This part is user provided model code\n# you will need to copy the next section to run the code\n# COMMAND ----------\n# this part is dbt logic for get ref work, do not modify\n\n{{ build_ref_function(model ) }}\n{{ build_source_function(model ) }}\n{{ build_config_dict(model) }}\n\nclass config:\n def __init__(self, *args, **kwargs):\n pass\n\n @staticmethod\n def get(key, default=None):\n return config_dict.get(key, default)\n\nclass this:\n \"\"\"dbt.this() or dbt.this.identifier\"\"\"\n database = \"{{ this.database }}\"\n schema = \"{{ this.schema }}\"\n identifier = \"{{ this.identifier }}\"\n {% set this_relation_name = resolve_model_name(this) %}\n def __repr__(self):\n return '{{ this_relation_name }}'\n\n\nclass dbtObj:\n def __init__(self, load_df_function) -> None:\n self.source = lambda *args: source(*args, dbt_load_df_function=load_df_function)\n self.ref = lambda *args, **kwargs: ref(*args, **kwargs, dbt_load_df_function=load_df_function)\n self.config = config\n self.this = this()\n self.is_incremental = {{ is_incremental() }}\n\n# COMMAND ----------\n{{py_script_comment()}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.build_ref_function", "macro.dbt.build_source_function", "macro.dbt.build_config_dict", "macro.dbt.resolve_model_name", "macro.dbt.is_incremental", "macro.dbt.py_script_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.30047, "supported_languages": null}, "macro.dbt.py_script_comment": {"name": "py_script_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.py_script_comment", "macro_sql": "{%macro py_script_comment()%}\n{%endmacro%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.3005419, "supported_languages": null}, "macro.dbt.test_unique": {"name": "test_unique", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_unique", "macro_sql": "{% test unique(model, column_name) %}\n {% set macro = adapter.dispatch('test_unique', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_unique"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.3010602, "supported_languages": null}, "macro.dbt.test_not_null": {"name": "test_not_null", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_not_null", "macro_sql": "{% test not_null(model, column_name) %}\n {% set macro = adapter.dispatch('test_not_null', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_not_null"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.3013, "supported_languages": null}, "macro.dbt.test_accepted_values": {"name": "test_accepted_values", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_accepted_values", "macro_sql": "{% test accepted_values(model, column_name, values, quote=True) %}\n {% set macro = adapter.dispatch('test_accepted_values', 'dbt') %}\n {{ macro(model, column_name, values, quote) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_accepted_values"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.301594, "supported_languages": null}, "macro.dbt.test_relationships": {"name": "test_relationships", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_relationships", "macro_sql": "{% test relationships(model, column_name, to, field) %}\n {% set macro = adapter.dispatch('test_relationships', 'dbt') %}\n {{ macro(model, column_name, to, field) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_relationships"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1697057377.301873, "supported_languages": null}}, "docs": {"doc.test.somedoc": {"name": "somedoc", "resource_type": "doc", "package_name": "test", "path": "somedoc.md", "original_file_path": "models/somedoc.md", "unique_id": "doc.test.somedoc", "block_contents": "Testing, testing"}, "doc.dbt.__overview__": {"name": "__overview__", "resource_type": "doc", "package_name": "dbt", "path": "overview.md", "original_file_path": "docs/overview.md", "unique_id": "doc.dbt.__overview__", "block_contents": "### Welcome!\n\nWelcome to the auto-generated documentation for your dbt project!\n\n### Navigation\n\nYou can use the `Project` and `Database` navigation tabs on the left side of the window to explore the models\nin your project.\n\n#### Project Tab\nThe `Project` tab mirrors the directory structure of your dbt project. In this tab, you can see all of the\nmodels defined in your dbt project, as well as models imported from dbt packages.\n\n#### Database Tab\nThe `Database` tab also exposes your models, but in a format that looks more like a database explorer. This view\nshows relations (tables and views) grouped into database schemas. Note that ephemeral models are _not_ shown\nin this interface, as they do not exist in the database.\n\n### Graph Exploration\nYou can click the blue icon on the bottom-right corner of the page to view the lineage graph of your models.\n\nOn model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the `Expand`\nbutton at the top-right of this lineage pane, you'll be able to see all of the models that are used to build,\nor are built from, the model you're exploring.\n\nOnce expanded, you'll be able to use the `--select` and `--exclude` model selection syntax to filter the\nmodels in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax).\n\nNote that you can also right-click on models to interactively filter and explore the graph.\n\n---\n\n### More information\n\n- [What is dbt](https://docs.getdbt.com/docs/introduction)?\n- Read the [dbt viewpoint](https://docs.getdbt.com/docs/viewpoint)\n- [Installation](https://docs.getdbt.com/docs/installation)\n- Join the [dbt Community](https://www.getdbt.com/community/) for questions and discussion"}}, "exposures": {"exposure.test.simple_exposure": {"name": "simple_exposure", "resource_type": "exposure", "package_name": "test", "path": "schema.yml", "original_file_path": "models/schema.yml", "unique_id": "exposure.test.simple_exposure", "fqn": ["test", "simple_exposure"], "type": "dashboard", "owner": {"email": "something@example.com", "name": null}, "description": "", "label": null, "maturity": null, "meta": {}, "tags": [], "config": {"enabled": true}, "unrendered_config": {}, "url": null, "depends_on": {"macros": [], "nodes": ["source.test.my_source.my_table", "model.test.my_model"]}, "refs": [{"name": "my_model", "package": null, "version": null}], "sources": [["my_source", "my_table"]], "metrics": [], "created_at": 1697057377.578206}}, "metrics": {"metric.test.blue_customers_post_2010": {"name": "blue_customers_post_2010", "resource_type": "metric", "package_name": "test", "path": "schema.yml", "original_file_path": "models/schema.yml", "unique_id": "metric.test.blue_customers_post_2010", "fqn": ["test", "blue_customers_post_2010"], "description": "", "label": "Blue Customers since 2010", "type": "simple", "type_params": {"measure": {"name": "customers", "filter": {"where_sql_template": "{{ Dimension('id__favorite_color') }} = 'blue'"}, "alias": null, "join_to_timespine": false, "fill_nulls_with": null}, "input_measures": [{"name": "customers", "filter": {"where_sql_template": "{{ Dimension('id__favorite_color') }} = 'blue'"}, "alias": null, "join_to_timespine": false, "fill_nulls_with": null}], "numerator": null, "denominator": null, "expr": null, "window": null, "grain_to_date": null, "metrics": []}, "filter": {"where_sql_template": "{{ TimeDimension('id__created_at', 'day') }} > '2010-01-01'"}, "metadata": null, "meta": {}, "tags": [], "config": {"enabled": true, "group": null}, "unrendered_config": {}, "sources": [], "depends_on": {"macros": [], "nodes": ["semantic_model.test.semantic_people"]}, "refs": [], "metrics": [], "created_at": 1697057377.583621, "group": null}, "metric.test.customers": {"name": "customers", "resource_type": "metric", "package_name": "test", "path": "schema.yml", "original_file_path": "models/schema.yml", "unique_id": "metric.test.customers", "fqn": ["test", "customers"], "description": "", "label": "Customers Metric", "type": "simple", "type_params": {"measure": {"name": "customers", "filter": null, "alias": null, "join_to_timespine": false, "fill_nulls_with": null}, "input_measures": [{"name": "customers", "filter": null, "alias": null, "join_to_timespine": false, "fill_nulls_with": null}], "numerator": null, "denominator": null, "expr": null, "window": null, "grain_to_date": null, "metrics": []}, "filter": null, "metadata": null, "meta": {}, "tags": [], "config": {"enabled": true, "group": null}, "unrendered_config": {}, "sources": [], "depends_on": {"macros": [], "nodes": ["semantic_model.test.semantic_people"]}, "refs": [], "metrics": [], "created_at": 1697057377.5840042, "group": null}, "metric.test.ratio_of_blue_customers_to_red_customers": {"name": "ratio_of_blue_customers_to_red_customers", "resource_type": "metric", "package_name": "test", "path": "schema.yml", "original_file_path": "models/schema.yml", "unique_id": "metric.test.ratio_of_blue_customers_to_red_customers", "fqn": ["test", "ratio_of_blue_customers_to_red_customers"], "description": "", "label": "Very Important Customer Color Ratio", "type": "ratio", "type_params": {"measure": null, "input_measures": [{"name": "customers", "filter": null, "alias": null, "join_to_timespine": false, "fill_nulls_with": null}, {"name": "customers", "filter": null, "alias": null, "join_to_timespine": false, "fill_nulls_with": null}], "numerator": {"name": "customers", "filter": {"where_sql_template": "{{ Dimension('id__favorite_color')}} = 'blue'"}, "alias": null, "offset_window": null, "offset_to_grain": null}, "denominator": {"name": "customers", "filter": {"where_sql_template": "{{ Dimension('id__favorite_color')}} = 'red'"}, "alias": null, "offset_window": null, "offset_to_grain": null}, "expr": null, "window": null, "grain_to_date": null, "metrics": []}, "filter": null, "metadata": null, "meta": {}, "tags": [], "config": {"enabled": true, "group": null}, "unrendered_config": {}, "sources": [], "depends_on": {"macros": [], "nodes": ["metric.test.customers"]}, "refs": [], "metrics": [], "created_at": 1697057377.585288, "group": null}, "metric.test.doubled_blue_customers": {"name": "doubled_blue_customers", "resource_type": "metric", "package_name": "test", "path": "schema.yml", "original_file_path": "models/schema.yml", "unique_id": "metric.test.doubled_blue_customers", "fqn": ["test", "doubled_blue_customers"], "description": "", "label": "Inflated blue customer numbers", "type": "derived", "type_params": {"measure": null, "input_measures": [{"name": "customers", "filter": null, "alias": null, "join_to_timespine": false, "fill_nulls_with": null}], "numerator": null, "denominator": null, "expr": "customers * 2", "window": null, "grain_to_date": null, "metrics": [{"name": "customers", "filter": {"where_sql_template": "{{ Dimension('id__favorite_color')}} = 'blue'"}, "alias": null, "offset_window": null, "offset_to_grain": null}]}, "filter": null, "metadata": null, "meta": {}, "tags": [], "config": {"enabled": true, "group": null}, "unrendered_config": {}, "sources": [], "depends_on": {"macros": [], "nodes": ["metric.test.customers"]}, "refs": [], "metrics": [], "created_at": 1697057377.5861351, "group": null}}, "groups": {}, "selectors": {}, "disabled": {"model.test.disabled_model": [{"database": "dbt", "schema": "test16970573770617803847_test_previous_version_state", "name": "disabled_model", "resource_type": "model", "package_name": "test", "path": "disabled_model.sql", "original_file_path": "models/disabled_model.sql", "unique_id": "model.test.disabled_model", "fqn": ["test", "disabled_model"], "alias": "disabled_model", "checksum": {"name": "sha256", "checksum": "597106d23ce34e3cd2430588e5c1cf474ebdd138fc47e09b925a4ab258a27acc"}, "config": {"enabled": false, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false}, "post-hook": [], "pre-hook": []}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {"enabled": false}, "created_at": 1697057377.4547698, "config_call_dict": {"enabled": false}, "relation_name": "\"dbt\".\"test16970573770617803847_test_previous_version_state\".\"disabled_model\"", "raw_code": "{{ config(enabled=False) }}\nselect 2 as id", "language": "sql", "refs": [], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": null, "contract": {"enforced": false, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null, "defer_relation": null}], "snapshot.test.disabled_snapshot_seed": [{"database": "dbt", "schema": "test16970573770617803847_test_previous_version_state", "name": "disabled_snapshot_seed", "resource_type": "snapshot", "package_name": "test", "path": "disabled_snapshot_seed.sql", "original_file_path": "snapshots/disabled_snapshot_seed.sql", "unique_id": "snapshot.test.disabled_snapshot_seed", "fqn": ["test", "disabled_snapshot_seed", "disabled_snapshot_seed"], "alias": "disabled_snapshot_seed", "checksum": {"name": "sha256", "checksum": "fe76c9dd437341c9e82a0f2a8baf3148f961b768eaa0a4410cd27d3c071bd617"}, "config": {"enabled": false, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "snapshot", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": "id", "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false}, "strategy": "check", "target_schema": "test16970573770617803847_test_previous_version_state", "target_database": null, "updated_at": null, "check_cols": "all", "post-hook": [], "pre-hook": []}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {"unique_key": "id", "strategy": "check", "check_cols": "all", "target_schema": "test16970573770617803847_test_previous_version_state", "enabled": false}, "created_at": 1697057377.4774349, "config_call_dict": {"unique_key": "id", "strategy": "check", "check_cols": "all", "target_schema": "test16970573770617803847_test_previous_version_state", "enabled": false}, "relation_name": "\"dbt\".\"test16970573770617803847_test_previous_version_state\".\"disabled_snapshot_seed\"", "raw_code": "\n{{\n config(\n unique_key='id',\n strategy='check',\n check_cols='all',\n target_schema=schema,\n enabled=False,\n )\n}}\nselect * from {{ ref('my_seed') }}\n", "language": "sql", "refs": [{"name": "my_seed", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": null, "contract": {"enforced": false, "checksum": null}, "defer_relation": null}], "analysis.test.disabled_al": [{"database": "dbt", "schema": "test16970573770617803847_test_previous_version_state", "name": "disabled_al", "resource_type": "analysis", "package_name": "test", "path": "analysis/disabled_al.sql", "original_file_path": "analyses/disabled_al.sql", "unique_id": "analysis.test.disabled_al", "fqn": ["test", "analysis", "disabled_al"], "alias": "disabled_al", "checksum": {"name": "sha256", "checksum": "32d36ad6cff0786eb562440ba60ef6c9b9a7f4c282dfb7a52eaf19d36370f0e1"}, "config": {"enabled": false, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false}, "post-hook": [], "pre-hook": []}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {"enabled": false}, "created_at": 1697057377.489575, "config_call_dict": {"enabled": false}, "relation_name": null, "raw_code": "{{ config(enabled=False) }}\nselect 9 as id", "language": "sql", "refs": [], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": null, "contract": {"enforced": false, "checksum": null}}], "test.test.disabled_just_my": [{"database": "dbt", "schema": "test16970573770617803847_test_previous_version_state_dbt_test__audit", "name": "disabled_just_my", "resource_type": "test", "package_name": "test", "path": "disabled_just_my.sql", "original_file_path": "tests/disabled_just_my.sql", "unique_id": "test.test.disabled_just_my", "fqn": ["test", "disabled_just_my"], "alias": "disabled_just_my", "checksum": {"name": "sha256", "checksum": "4f2268fd89a3b4ef899264ada6d7aa33603671cbc5d5acead7dc2eadf1add985"}, "config": {"enabled": false, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {"enabled": false}, "created_at": 1697057377.5060952, "config_call_dict": {"enabled": false}, "relation_name": null, "raw_code": "{{ config(enabled=False) }}\n\nselect * from {{ ref('my_model') }}\nwhere false", "language": "sql", "refs": [{"name": "my_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": null, "contract": {"enforced": false, "checksum": null}}], "test.test.disabled_check_nothing_my_model_.f2c6a72d37": [{"test_metadata": {"name": "disabled_check_nothing", "kwargs": {"model": "{{ get_where_subquery(ref('my_model')) }}"}, "namespace": null}, "database": "dbt", "schema": "test16970573770617803847_test_previous_version_state_dbt_test__audit", "name": "disabled_check_nothing_my_model_", "resource_type": "test", "package_name": "test", "path": "disabled_check_nothing_my_model_.sql", "original_file_path": "models/schema.yml", "unique_id": "test.test.disabled_check_nothing_my_model_.f2c6a72d37", "fqn": ["test", "disabled_check_nothing_my_model_"], "alias": "disabled_check_nothing_my_model_", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": false, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {"enabled": false}, "created_at": 1697057377.558094, "config_call_dict": {"enabled": false}, "relation_name": null, "raw_code": "{{ test_disabled_check_nothing(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "my_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.test.test_disabled_check_nothing", "macro.dbt.get_where_subquery"], "nodes": []}, "compiled_path": null, "contract": {"enforced": false, "checksum": null}, "column_name": null, "file_key_name": "models.my_model", "attached_node": "model.test.my_model"}], "exposure.test.disabled_exposure": [{"name": "disabled_exposure", "resource_type": "exposure", "package_name": "test", "path": "schema.yml", "original_file_path": "models/schema.yml", "unique_id": "exposure.test.disabled_exposure", "fqn": ["test", "disabled_exposure"], "type": "dashboard", "owner": {"email": "something@example.com", "name": null}, "description": "", "label": null, "maturity": null, "meta": {}, "tags": [], "config": {"enabled": false}, "unrendered_config": {"enabled": false}, "url": null, "depends_on": {"macros": [], "nodes": []}, "refs": [{"name": "my_model", "package": null, "version": null}], "sources": [], "metrics": [], "created_at": 1697057377.5790222}], "metric.test.disabled_metric": [{"name": "disabled_metric", "resource_type": "metric", "package_name": "test", "path": "schema.yml", "original_file_path": "models/schema.yml", "unique_id": "metric.test.disabled_metric", "fqn": ["test", "disabled_metric"], "description": "", "label": "Count records", "type": "simple", "type_params": {"measure": {"name": "customers", "filter": null, "alias": null, "join_to_timespine": false, "fill_nulls_with": null}, "input_measures": [], "numerator": null, "denominator": null, "expr": null, "window": null, "grain_to_date": null, "metrics": []}, "filter": {"where_sql_template": "{{ Dimension('id__favorite_color') }} = 'blue'"}, "metadata": null, "meta": {}, "tags": [], "config": {"enabled": false, "group": null}, "unrendered_config": {"enabled": false}, "sources": [], "depends_on": {"macros": [], "nodes": []}, "refs": [], "metrics": [], "created_at": 1697057377.584552, "group": null}], "seed.test.disabled_seed": [{"database": "dbt", "schema": "test16970573770617803847_test_previous_version_state", "name": "disabled_seed", "resource_type": "seed", "package_name": "test", "path": "disabled_seed.csv", "original_file_path": "seeds/disabled_seed.csv", "unique_id": "seed.test.disabled_seed", "fqn": ["test", "disabled_seed"], "alias": "disabled_seed", "checksum": {"name": "sha256", "checksum": "31fddd8ec40c6aba6a3a8e7d83fedea2fd0a56c47b64ea3df1847ec1b018e2d1"}, "config": {"enabled": false, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "seed", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false}, "quote_columns": null, "post-hook": [], "pre-hook": []}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": "test://models/schema.yml", "build_path": null, "deferred": false, "unrendered_config": {"enabled": false}, "created_at": 1697057377.5646772, "config_call_dict": {}, "relation_name": "\"dbt\".\"test16970573770617803847_test_previous_version_state\".\"disabled_seed\"", "raw_code": "", "root_path": "/private/var/folders/79/5290gpvn3lx5jdryk4844rm80000gn/T/pytest-of-quigleymalcolm/pytest-271/project0", "depends_on": {"macros": []}, "defer_relation": null}], "source.test.my_source.disabled_table": [{"database": "dbt", "schema": "my_source", "name": "disabled_table", "resource_type": "source", "package_name": "test", "path": "models/schema.yml", "original_file_path": "models/schema.yml", "unique_id": "source.test.my_source.disabled_table", "fqn": ["test", "my_source", "disabled_table"], "source_name": "my_source", "source_description": "My source", "loader": "a_loader", "identifier": "disabled_table", "quoting": {"database": null, "schema": null, "identifier": null, "column": null}, "loaded_at_field": null, "freshness": {"warn_after": {"count": null, "period": null}, "error_after": {"count": null, "period": null}, "filter": null}, "external": null, "description": "Disabled table", "columns": {}, "meta": {}, "source_meta": {}, "tags": [], "config": {"enabled": false}, "patch_path": null, "unrendered_config": {"enabled": false}, "relation_name": "\"dbt\".\"my_source\".\"disabled_table\"", "created_at": 1697057377.5942788}]}, "parent_map": {"model.test.my_model": [], "model.test.metricflow_time_spine": [], "snapshot.test.snapshot_seed": ["seed.test.my_seed"], "analysis.test.a": [], "test.test.just_my": ["model.test.my_model"], "seed.test.my_seed": [], "test.test.not_null_my_model_id.43e0e9183a": ["model.test.my_model"], "test.test.check_nothing_my_model_.d5a5e66110": ["model.test.my_model"], "source.test.my_source.my_table": [], "exposure.test.simple_exposure": ["model.test.my_model", "source.test.my_source.my_table"], "metric.test.blue_customers_post_2010": ["semantic_model.test.semantic_people"], "metric.test.customers": ["semantic_model.test.semantic_people"], "metric.test.ratio_of_blue_customers_to_red_customers": ["metric.test.customers"], "metric.test.doubled_blue_customers": ["metric.test.customers"], "semantic_model.test.semantic_people": ["model.test.my_model"]}, "child_map": {"model.test.my_model": ["exposure.test.simple_exposure", "semantic_model.test.semantic_people", "test.test.check_nothing_my_model_.d5a5e66110", "test.test.just_my", "test.test.not_null_my_model_id.43e0e9183a"], "model.test.metricflow_time_spine": [], "snapshot.test.snapshot_seed": [], "analysis.test.a": [], "test.test.just_my": [], "seed.test.my_seed": ["snapshot.test.snapshot_seed"], "test.test.not_null_my_model_id.43e0e9183a": [], "test.test.check_nothing_my_model_.d5a5e66110": [], "source.test.my_source.my_table": ["exposure.test.simple_exposure"], "exposure.test.simple_exposure": [], "metric.test.blue_customers_post_2010": [], "metric.test.customers": ["metric.test.doubled_blue_customers", "metric.test.ratio_of_blue_customers_to_red_customers"], "metric.test.ratio_of_blue_customers_to_red_customers": [], "metric.test.doubled_blue_customers": [], "semantic_model.test.semantic_people": ["metric.test.blue_customers_post_2010", "metric.test.customers"]}, "group_map": {}, "semantic_models": {"semantic_model.test.semantic_people": {"name": "semantic_people", "resource_type": "semantic_model", "package_name": "test", "path": "schema.yml", "original_file_path": "models/schema.yml", "unique_id": "semantic_model.test.semantic_people", "fqn": ["test", "semantic_people"], "model": "ref('my_model')", "node_relation": {"alias": "my_model", "schema_name": "test16970573770617803847_test_previous_version_state", "database": "dbt", "relation_name": "\"dbt\".\"test16970573770617803847_test_previous_version_state\".\"my_model\""}, "description": null, "label": null, "defaults": {"agg_time_dimension": "created_at"}, "entities": [{"name": "id", "type": "primary", "description": null, "label": null, "role": null, "expr": null}], "measures": [{"name": "years_tenure", "agg": "sum", "description": null, "label": null, "create_metric": false, "expr": "tenure", "agg_params": null, "non_additive_dimension": null, "agg_time_dimension": null}, {"name": "people", "agg": "count", "description": null, "label": null, "create_metric": false, "expr": "id", "agg_params": null, "non_additive_dimension": null, "agg_time_dimension": null}, {"name": "customers", "agg": "count", "description": null, "label": null, "create_metric": false, "expr": "id", "agg_params": null, "non_additive_dimension": null, "agg_time_dimension": null}], "dimensions": [{"name": "favorite_color", "type": "categorical", "description": null, "label": null, "is_partition": false, "type_params": null, "expr": null, "metadata": null}, {"name": "created_at", "type": "time", "description": null, "label": null, "is_partition": false, "type_params": {"time_granularity": "day", "validity_params": null}, "expr": null, "metadata": null}], "metadata": null, "depends_on": {"macros": [], "nodes": ["model.test.my_model"]}, "refs": [{"name": "my_model", "package": null, "version": null}], "created_at": 1697057377.5929039, "config": {"enabled": true}, "primary_entity": null}}} ================================================ FILE: tests/functional/artifacts/data/state/v11/manifest.json ================================================ {"metadata": {"dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v11.json", "dbt_version": "1.7.6", "generated_at": "2024-01-30T11:27:16.415331Z", "invocation_id": "05b2865a-32e1-4aae-bf19-bace9c34ecf6", "env": {}, "project_name": "test", "project_id": "098f6bcd4621d373cade4e832627b4f6", "user_id": null, "send_anonymous_usage_stats": false, "adapter_type": "postgres"}, "nodes": {"model.test.my_model": {"database": "dbt", "schema": "test17066140326720908695_test_previous_version_state", "name": "my_model", "resource_type": "model", "package_name": "test", "path": "my_model.sql", "original_file_path": "models/my_model.sql", "unique_id": "model.test.my_model", "fqn": ["test", "my_model"], "alias": "my_model", "checksum": {"name": "sha256", "checksum": "3ea0f972fa1b56aa2dc2f56ee784b6a5796312f9a813d59ae70fd8855f10d16d"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "access": "protected"}, "tags": [], "description": "Example model", "columns": {"id": {"name": "id", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": "test://models/schema.yml", "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1706614035.6203048, "relation_name": "\"dbt\".\"test17066140326720908695_test_previous_version_state\".\"my_model\"", "raw_code": "select 1 as id", "language": "sql", "refs": [], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "model.test.metricflow_time_spine": {"database": "dbt", "schema": "test17066140326720908695_test_previous_version_state", "name": "metricflow_time_spine", "resource_type": "model", "package_name": "test", "path": "metricflow_time_spine.sql", "original_file_path": "models/metricflow_time_spine.sql", "unique_id": "model.test.metricflow_time_spine", "fqn": ["test", "metricflow_time_spine"], "alias": "metricflow_time_spine", "checksum": {"name": "sha256", "checksum": "954d9b349821edb5558a373119a7d91eeac9e620aaa96cd112c0d14bab729fdb"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "access": "protected"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1706614035.2472441, "relation_name": "\"dbt\".\"test17066140326720908695_test_previous_version_state\".\"metricflow_time_spine\"", "raw_code": "SELECT to_date('02/20/2023', 'mm/dd/yyyy') as date_day", "language": "sql", "refs": [], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "snapshot.test.snapshot_seed": {"database": "dbt", "schema": "test17066140326720908695_test_previous_version_state", "name": "snapshot_seed", "resource_type": "snapshot", "package_name": "test", "path": "snapshot_seed.sql", "original_file_path": "snapshots/snapshot_seed.sql", "unique_id": "snapshot.test.snapshot_seed", "fqn": ["test", "snapshot_seed", "snapshot_seed"], "alias": "snapshot_seed", "checksum": {"name": "sha256", "checksum": "5fc998f39655f8fe52443a919e749b6e23883ef90202b040412baac13c6bfe18"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "snapshot", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": "id", "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "strategy": "check", "target_schema": "test17066140326720908695_test_previous_version_state", "target_database": null, "updated_at": null, "check_cols": "all"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {"unique_key": "id", "strategy": "check", "check_cols": "all", "target_schema": "test17066140326720908695_test_previous_version_state"}, "created_at": 1706614035.331658, "relation_name": "\"dbt\".\"test17066140326720908695_test_previous_version_state\".\"snapshot_seed\"", "raw_code": "\n{{\n config(\n unique_key='id',\n strategy='check',\n check_cols='all',\n target_schema=schema,\n )\n}}\nselect * from {{ ref('my_seed') }}\n", "language": "sql", "refs": [{"name": "my_seed", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["seed.test.my_seed"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}}, "analysis.test.a": {"database": "dbt", "schema": "test17066140326720908695_test_previous_version_state", "name": "a", "resource_type": "analysis", "package_name": "test", "path": "analysis/a.sql", "original_file_path": "analyses/a.sql", "unique_id": "analysis.test.a", "fqn": ["test", "analysis", "a"], "alias": "a", "checksum": {"name": "sha256", "checksum": "a389c282f569f0bbdc2a8a4f174dea746c28582fdaf2048d31d9226af9feab23"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1706614035.426197, "relation_name": null, "raw_code": "select 4 as id", "language": "sql", "refs": [], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}}, "test.test.just_my": {"database": "dbt", "schema": "test17066140326720908695_test_previous_version_state_dbt_test__audit", "name": "just_my", "resource_type": "test", "package_name": "test", "path": "just_my.sql", "original_file_path": "tests/just_my.sql", "unique_id": "test.test.just_my", "fqn": ["test", "just_my"], "alias": "just_my", "checksum": {"name": "sha256", "checksum": "744889a2e2d9ce380619265e1217d7ccf6e6ca896c048d42ebe0f9cfb74d7156"}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": ["data_test_tag"], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": ["data_test_tag"], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {"tags": ["data_test_tag"]}, "created_at": 1706614035.493541, "relation_name": null, "raw_code": "{{ config(tags = ['data_test_tag']) }}\n\nselect * from {{ ref('my_model') }}\nwhere false", "language": "sql", "refs": [{"name": "my_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["model.test.my_model"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}}, "seed.test.my_seed": {"database": "dbt", "schema": "test17066140326720908695_test_previous_version_state", "name": "my_seed", "resource_type": "seed", "package_name": "test", "path": "my_seed.csv", "original_file_path": "seeds/my_seed.csv", "unique_id": "seed.test.my_seed", "fqn": ["test", "my_seed"], "alias": "my_seed", "checksum": {"name": "sha256", "checksum": "f7ede83f36165ac6b7a047aa2c3f212dff385bfa9f35f395108cd06fc8e96943"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "seed", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "delimiter": ",", "quote_columns": null}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1706614035.5665338, "relation_name": "\"dbt\".\"test17066140326720908695_test_previous_version_state\".\"my_seed\"", "raw_code": "", "root_path": "/private/var/folders/7h/hj5_fw9j291c58hwfdvy5xbm0000gp/T/pytest-of-jerco/pytest-10/project0", "depends_on": {"macros": []}}, "test.test.not_null_my_model_id.43e0e9183a": {"test_metadata": {"name": "not_null", "kwargs": {"column_name": "id", "model": "{{ get_where_subquery(ref('my_model')) }}"}, "namespace": null}, "database": "dbt", "schema": "test17066140326720908695_test_previous_version_state_dbt_test__audit", "name": "not_null_my_model_id", "resource_type": "test", "package_name": "test", "path": "not_null_my_model_id.sql", "original_file_path": "models/schema.yml", "unique_id": "test.test.not_null_my_model_id.43e0e9183a", "fqn": ["test", "not_null_my_model_id"], "alias": "not_null_my_model_id", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1706614035.668334, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "my_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.test.my_model"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "id", "file_key_name": "models.my_model", "attached_node": "model.test.my_model"}, "test.test.check_nothing_my_model_.d5a5e66110": {"test_metadata": {"name": "check_nothing", "kwargs": {"model": "{{ get_where_subquery(ref('my_model')) }}"}, "namespace": null}, "database": "dbt", "schema": "test17066140326720908695_test_previous_version_state_dbt_test__audit", "name": "check_nothing_my_model_", "resource_type": "test", "package_name": "test", "path": "check_nothing_my_model_.sql", "original_file_path": "models/schema.yml", "unique_id": "test.test.check_nothing_my_model_.d5a5e66110", "fqn": ["test", "check_nothing_my_model_"], "alias": "check_nothing_my_model_", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1706614035.6698759, "relation_name": null, "raw_code": "{{ test_check_nothing(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "my_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.test.test_check_nothing", "macro.dbt.get_where_subquery"], "nodes": ["model.test.my_model"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": null, "file_key_name": "models.my_model", "attached_node": "model.test.my_model"}}, "sources": {"source.test.my_source.my_table": {"database": "dbt", "schema": "my_source", "name": "my_table", "resource_type": "source", "package_name": "test", "path": "models/schema.yml", "original_file_path": "models/schema.yml", "unique_id": "source.test.my_source.my_table", "fqn": ["test", "my_source", "my_table"], "source_name": "my_source", "source_description": "My source", "loader": "a_loader", "identifier": "my_seed", "quoting": {"database": null, "schema": null, "identifier": null, "column": null}, "loaded_at_field": null, "freshness": {"warn_after": {"count": null, "period": null}, "error_after": {"count": null, "period": null}, "filter": null}, "external": null, "description": "My table", "columns": {}, "meta": {}, "source_meta": {}, "tags": [], "config": {"enabled": true}, "patch_path": null, "unrendered_config": {}, "relation_name": "\"dbt\".\"my_source\".\"my_seed\"", "created_at": 1706614035.938528}}, "macros": {"macro.test.test_check_nothing": {"name": "test_check_nothing", "resource_type": "macro", "package_name": "test", "path": "macros/dummy_test.sql", "original_file_path": "macros/dummy_test.sql", "unique_id": "macro.test.test_check_nothing", "macro_sql": "{% test check_nothing(model) %}\n-- a silly test to make sure that table-level tests show up in the manifest\n-- without a column_name field\n\nselect 0\n\n{% endtest %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.49347, "supported_languages": null}, "macro.test.test_disabled_check_nothing": {"name": "test_disabled_check_nothing", "resource_type": "macro", "package_name": "test", "path": "macros/disabled_dummy_test.sql", "original_file_path": "macros/disabled_dummy_test.sql", "unique_id": "macro.test.test_disabled_check_nothing", "macro_sql": "{% test disabled_check_nothing(model) %}\n-- a silly test to make sure that table-level tests show up in the manifest\n-- without a column_name field\n\n{{ config(enabled=False) }}\nselect 0\n\n{% endtest %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.493846, "supported_languages": null}, "macro.test.do_nothing": {"name": "do_nothing", "resource_type": "macro", "package_name": "test", "path": "macros/do_nothing.sql", "original_file_path": "macros/do_nothing.sql", "unique_id": "macro.test.do_nothing", "macro_sql": "{% macro do_nothing(foo2, bar2) %}\n select\n '{{ foo2 }}' as foo2,\n '{{ bar2 }}' as bar2\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.494155, "supported_languages": null}, "macro.dbt_postgres.postgres__current_timestamp": {"name": "postgres__current_timestamp", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/timestamps.sql", "original_file_path": "macros/timestamps.sql", "unique_id": "macro.dbt_postgres.postgres__current_timestamp", "macro_sql": "{% macro postgres__current_timestamp() -%}\n now()\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.4946299, "supported_languages": null}, "macro.dbt_postgres.postgres__snapshot_string_as_time": {"name": "postgres__snapshot_string_as_time", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/timestamps.sql", "original_file_path": "macros/timestamps.sql", "unique_id": "macro.dbt_postgres.postgres__snapshot_string_as_time", "macro_sql": "{% macro postgres__snapshot_string_as_time(timestamp) -%}\n {%- set result = \"'\" ~ timestamp ~ \"'::timestamp without time zone\" -%}\n {{ return(result) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.494959, "supported_languages": null}, "macro.dbt_postgres.postgres__snapshot_get_time": {"name": "postgres__snapshot_get_time", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/timestamps.sql", "original_file_path": "macros/timestamps.sql", "unique_id": "macro.dbt_postgres.postgres__snapshot_get_time", "macro_sql": "{% macro postgres__snapshot_get_time() -%}\n {{ current_timestamp() }}::timestamp without time zone\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.495212, "supported_languages": null}, "macro.dbt_postgres.postgres__current_timestamp_backcompat": {"name": "postgres__current_timestamp_backcompat", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/timestamps.sql", "original_file_path": "macros/timestamps.sql", "unique_id": "macro.dbt_postgres.postgres__current_timestamp_backcompat", "macro_sql": "{% macro postgres__current_timestamp_backcompat() %}\n current_timestamp::{{ type_timestamp() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.type_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.4953868, "supported_languages": null}, "macro.dbt_postgres.postgres__current_timestamp_in_utc_backcompat": {"name": "postgres__current_timestamp_in_utc_backcompat", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/timestamps.sql", "original_file_path": "macros/timestamps.sql", "unique_id": "macro.dbt_postgres.postgres__current_timestamp_in_utc_backcompat", "macro_sql": "{% macro postgres__current_timestamp_in_utc_backcompat() %}\n (current_timestamp at time zone 'utc')::{{ type_timestamp() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.type_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.495561, "supported_languages": null}, "macro.dbt_postgres.postgres__get_catalog_relations": {"name": "postgres__get_catalog_relations", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/catalog.sql", "original_file_path": "macros/catalog.sql", "unique_id": "macro.dbt_postgres.postgres__get_catalog_relations", "macro_sql": "{% macro postgres__get_catalog_relations(information_schema, relations) -%}\n {%- call statement('catalog', fetch_result=True) -%}\n\n {#\n If the user has multiple databases set and the first one is wrong, this will fail.\n But we won't fail in the case where there are multiple quoting-difference-only dbs, which is better.\n #}\n {% set database = information_schema.database %}\n {{ adapter.verify_database(database) }}\n\n select\n '{{ database }}' as table_database,\n sch.nspname as table_schema,\n tbl.relname as table_name,\n case tbl.relkind\n when 'v' then 'VIEW'\n when 'm' then 'MATERIALIZED VIEW'\n else 'BASE TABLE'\n end as table_type,\n tbl_desc.description as table_comment,\n col.attname as column_name,\n col.attnum as column_index,\n pg_catalog.format_type(col.atttypid, col.atttypmod) as column_type,\n col_desc.description as column_comment,\n pg_get_userbyid(tbl.relowner) as table_owner\n\n from pg_catalog.pg_namespace sch\n join pg_catalog.pg_class tbl on tbl.relnamespace = sch.oid\n join pg_catalog.pg_attribute col on col.attrelid = tbl.oid\n left outer join pg_catalog.pg_description tbl_desc on (tbl_desc.objoid = tbl.oid and tbl_desc.objsubid = 0)\n left outer join pg_catalog.pg_description col_desc on (col_desc.objoid = tbl.oid and col_desc.objsubid = col.attnum)\n where (\n {%- for relation in relations -%}\n {%- if relation.identifier -%}\n (upper(sch.nspname) = upper('{{ relation.schema }}') and\n upper(tbl.relname) = upper('{{ relation.identifier }}'))\n {%- else-%}\n upper(sch.nspname) = upper('{{ relation.schema }}')\n {%- endif -%}\n {%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n )\n and not pg_is_other_temp_schema(sch.oid) -- not a temporary schema belonging to another session\n and tbl.relpersistence in ('p', 'u') -- [p]ermanent table or [u]nlogged table. Exclude [t]emporary tables\n and tbl.relkind in ('r', 'v', 'f', 'p', 'm') -- o[r]dinary table, [v]iew, [f]oreign table, [p]artitioned table, [m]aterialized view. Other values are [i]ndex, [S]equence, [c]omposite type, [t]OAST table\n and col.attnum > 0 -- negative numbers are used for system columns such as oid\n and not col.attisdropped -- column as not been dropped\n\n order by\n sch.nspname,\n tbl.relname,\n col.attnum\n\n {%- endcall -%}\n\n {{ return(load_result('catalog').table) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.498032, "supported_languages": null}, "macro.dbt_postgres.postgres__get_catalog": {"name": "postgres__get_catalog", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/catalog.sql", "original_file_path": "macros/catalog.sql", "unique_id": "macro.dbt_postgres.postgres__get_catalog", "macro_sql": "{% macro postgres__get_catalog(information_schema, schemas) -%}\n {%- set relations = [] -%}\n {%- for schema in schemas -%}\n {%- set dummy = relations.append({'schema': schema}) -%}\n {%- endfor -%}\n {{ return(postgres__get_catalog_relations(information_schema, relations)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_catalog_relations"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.498621, "supported_languages": null}, "macro.dbt_postgres.postgres__get_relations": {"name": "postgres__get_relations", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations.sql", "original_file_path": "macros/relations.sql", "unique_id": "macro.dbt_postgres.postgres__get_relations", "macro_sql": "{% macro postgres__get_relations() -%}\n\n {#\n -- in pg_depend, objid is the dependent, refobjid is the referenced object\n -- > a pg_depend entry indicates that the referenced object cannot be\n -- > dropped without also dropping the dependent object.\n #}\n\n {%- call statement('relations', fetch_result=True) -%}\n with relation as (\n select\n pg_rewrite.ev_class as class,\n pg_rewrite.oid as id\n from pg_rewrite\n ),\n class as (\n select\n oid as id,\n relname as name,\n relnamespace as schema,\n relkind as kind\n from pg_class\n ),\n dependency as (\n select distinct\n pg_depend.objid as id,\n pg_depend.refobjid as ref\n from pg_depend\n ),\n schema as (\n select\n pg_namespace.oid as id,\n pg_namespace.nspname as name\n from pg_namespace\n where nspname != 'information_schema' and nspname not like 'pg\\_%'\n ),\n referenced as (\n select\n relation.id AS id,\n referenced_class.name ,\n referenced_class.schema ,\n referenced_class.kind\n from relation\n join class as referenced_class on relation.class=referenced_class.id\n where referenced_class.kind in ('r', 'v', 'm')\n ),\n relationships as (\n select\n referenced.name as referenced_name,\n referenced.schema as referenced_schema_id,\n dependent_class.name as dependent_name,\n dependent_class.schema as dependent_schema_id,\n referenced.kind as kind\n from referenced\n join dependency on referenced.id=dependency.id\n join class as dependent_class on dependency.ref=dependent_class.id\n where\n (referenced.name != dependent_class.name or\n referenced.schema != dependent_class.schema)\n )\n\n select\n referenced_schema.name as referenced_schema,\n relationships.referenced_name as referenced_name,\n dependent_schema.name as dependent_schema,\n relationships.dependent_name as dependent_name\n from relationships\n join schema as dependent_schema on relationships.dependent_schema_id=dependent_schema.id\n join schema as referenced_schema on relationships.referenced_schema_id=referenced_schema.id\n group by referenced_schema, referenced_name, dependent_schema, dependent_name\n order by referenced_schema, referenced_name, dependent_schema, dependent_name;\n\n {%- endcall -%}\n\n {{ return(load_result('relations').table) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.4997098, "supported_languages": null}, "macro.dbt_postgres.postgres_get_relations": {"name": "postgres_get_relations", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations.sql", "original_file_path": "macros/relations.sql", "unique_id": "macro.dbt_postgres.postgres_get_relations", "macro_sql": "{% macro postgres_get_relations() %}\n {{ return(postgres__get_relations()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_relations"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.499924, "supported_languages": null}, "macro.dbt_postgres.postgres__create_table_as": {"name": "postgres__create_table_as", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__create_table_as", "macro_sql": "{% macro postgres__create_table_as(temporary, relation, sql) -%}\n {%- set unlogged = config.get('unlogged', default=false) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary -%}\n temporary\n {%- elif unlogged -%}\n unlogged\n {%- endif %} table {{ relation }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {% endif -%}\n {% if contract_config.enforced and (not temporary) -%}\n {{ get_table_columns_and_constraints() }} ;\n insert into {{ relation }} (\n {{ adapter.dispatch('get_column_names', 'dbt')() }}\n )\n {%- set sql = get_select_subquery(sql) %}\n {% else %}\n as\n {% endif %}\n (\n {{ sql }}\n );\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_assert_columns_equivalent", "macro.dbt.get_table_columns_and_constraints", "macro.dbt.default__get_column_names", "macro.dbt.get_select_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.5139258, "supported_languages": null}, "macro.dbt_postgres.postgres__get_create_index_sql": {"name": "postgres__get_create_index_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__get_create_index_sql", "macro_sql": "{% macro postgres__get_create_index_sql(relation, index_dict) -%}\n {%- set index_config = adapter.parse_index(index_dict) -%}\n {%- set comma_separated_columns = \", \".join(index_config.columns) -%}\n {%- set index_name = index_config.render(relation) -%}\n\n create {% if index_config.unique -%}\n unique\n {%- endif %} index if not exists\n \"{{ index_name }}\"\n on {{ relation }} {% if index_config.type -%}\n using {{ index_config.type }}\n {%- endif %}\n ({{ comma_separated_columns }});\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.514729, "supported_languages": null}, "macro.dbt_postgres.postgres__create_schema": {"name": "postgres__create_schema", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__create_schema", "macro_sql": "{% macro postgres__create_schema(relation) -%}\n {% if relation.database -%}\n {{ adapter.verify_database(relation.database) }}\n {%- endif -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier().include(database=False) }}\n {%- endcall -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.515243, "supported_languages": null}, "macro.dbt_postgres.postgres__drop_schema": {"name": "postgres__drop_schema", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__drop_schema", "macro_sql": "{% macro postgres__drop_schema(relation) -%}\n {% if relation.database -%}\n {{ adapter.verify_database(relation.database) }}\n {%- endif -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier().include(database=False) }} cascade\n {%- endcall -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.5157518, "supported_languages": null}, "macro.dbt_postgres.postgres__get_columns_in_relation": {"name": "postgres__get_columns_in_relation", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__get_columns_in_relation", "macro_sql": "{% macro postgres__get_columns_in_relation(relation) -%}\n {% call statement('get_columns_in_relation', fetch_result=True) %}\n select\n column_name,\n data_type,\n character_maximum_length,\n numeric_precision,\n numeric_scale\n\n from {{ relation.information_schema('columns') }}\n where table_name = '{{ relation.identifier }}'\n {% if relation.schema %}\n and table_schema = '{{ relation.schema }}'\n {% endif %}\n order by ordinal_position\n\n {% endcall %}\n {% set table = load_result('get_columns_in_relation').table %}\n {{ return(sql_convert_columns_in_relation(table)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.sql_convert_columns_in_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.5165281, "supported_languages": null}, "macro.dbt_postgres.postgres__list_relations_without_caching": {"name": "postgres__list_relations_without_caching", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__list_relations_without_caching", "macro_sql": "{% macro postgres__list_relations_without_caching(schema_relation) %}\n {% call statement('list_relations_without_caching', fetch_result=True) -%}\n select\n '{{ schema_relation.database }}' as database,\n tablename as name,\n schemaname as schema,\n 'table' as type\n from pg_tables\n where schemaname ilike '{{ schema_relation.schema }}'\n union all\n select\n '{{ schema_relation.database }}' as database,\n viewname as name,\n schemaname as schema,\n 'view' as type\n from pg_views\n where schemaname ilike '{{ schema_relation.schema }}'\n union all\n select\n '{{ schema_relation.database }}' as database,\n matviewname as name,\n schemaname as schema,\n 'materialized_view' as type\n from pg_matviews\n where schemaname ilike '{{ schema_relation.schema }}'\n {% endcall %}\n {{ return(load_result('list_relations_without_caching').table) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.5172532, "supported_languages": null}, "macro.dbt_postgres.postgres__information_schema_name": {"name": "postgres__information_schema_name", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__information_schema_name", "macro_sql": "{% macro postgres__information_schema_name(database) -%}\n {% if database_name -%}\n {{ adapter.verify_database(database_name) }}\n {%- endif -%}\n information_schema\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.5175521, "supported_languages": null}, "macro.dbt_postgres.postgres__list_schemas": {"name": "postgres__list_schemas", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__list_schemas", "macro_sql": "{% macro postgres__list_schemas(database) %}\n {% if database -%}\n {{ adapter.verify_database(database) }}\n {%- endif -%}\n {% call statement('list_schemas', fetch_result=True, auto_begin=False) %}\n select distinct nspname from pg_namespace\n {% endcall %}\n {{ return(load_result('list_schemas').table) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.518124, "supported_languages": null}, "macro.dbt_postgres.postgres__check_schema_exists": {"name": "postgres__check_schema_exists", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__check_schema_exists", "macro_sql": "{% macro postgres__check_schema_exists(information_schema, schema) -%}\n {% if information_schema.database -%}\n {{ adapter.verify_database(information_schema.database) }}\n {%- endif -%}\n {% call statement('check_schema_exists', fetch_result=True, auto_begin=False) %}\n select count(*) from pg_namespace where nspname = '{{ schema }}'\n {% endcall %}\n {{ return(load_result('check_schema_exists').table) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.518896, "supported_languages": null}, "macro.dbt_postgres.postgres__make_relation_with_suffix": {"name": "postgres__make_relation_with_suffix", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__make_relation_with_suffix", "macro_sql": "{% macro postgres__make_relation_with_suffix(base_relation, suffix, dstring) %}\n {% if dstring %}\n {% set dt = modules.datetime.datetime.now() %}\n {% set dtstring = dt.strftime(\"%H%M%S%f\") %}\n {% set suffix = suffix ~ dtstring %}\n {% endif %}\n {% set suffix_length = suffix|length %}\n {% set relation_max_name_length = base_relation.relation_max_name_length() %}\n {% if suffix_length > relation_max_name_length %}\n {% do exceptions.raise_compiler_error('Relation suffix is too long (' ~ suffix_length ~ ' characters). Maximum length is ' ~ relation_max_name_length ~ ' characters.') %}\n {% endif %}\n {% set identifier = base_relation.identifier[:relation_max_name_length - suffix_length] ~ suffix %}\n\n {{ return(base_relation.incorporate(path={\"identifier\": identifier })) }}\n\n {% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.520294, "supported_languages": null}, "macro.dbt_postgres.postgres__make_intermediate_relation": {"name": "postgres__make_intermediate_relation", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__make_intermediate_relation", "macro_sql": "{% macro postgres__make_intermediate_relation(base_relation, suffix) %}\n {{ return(postgres__make_relation_with_suffix(base_relation, suffix, dstring=False)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_relation_with_suffix"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.520627, "supported_languages": null}, "macro.dbt_postgres.postgres__make_temp_relation": {"name": "postgres__make_temp_relation", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__make_temp_relation", "macro_sql": "{% macro postgres__make_temp_relation(base_relation, suffix) %}\n {% set temp_relation = postgres__make_relation_with_suffix(base_relation, suffix, dstring=True) %}\n {{ return(temp_relation.incorporate(path={\"schema\": none,\n \"database\": none})) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_relation_with_suffix"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.521166, "supported_languages": null}, "macro.dbt_postgres.postgres__make_backup_relation": {"name": "postgres__make_backup_relation", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__make_backup_relation", "macro_sql": "{% macro postgres__make_backup_relation(base_relation, backup_relation_type, suffix) %}\n {% set backup_relation = postgres__make_relation_with_suffix(base_relation, suffix, dstring=False) %}\n {{ return(backup_relation.incorporate(type=backup_relation_type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_relation_with_suffix"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.521612, "supported_languages": null}, "macro.dbt_postgres.postgres_escape_comment": {"name": "postgres_escape_comment", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres_escape_comment", "macro_sql": "{% macro postgres_escape_comment(comment) -%}\n {% if comment is not string %}\n {% do exceptions.raise_compiler_error('cannot escape a non-string: ' ~ comment) %}\n {% endif %}\n {%- set magic = '$dbt_comment_literal_block$' -%}\n {%- if magic in comment -%}\n {%- do exceptions.raise_compiler_error('The string ' ~ magic ~ ' is not allowed in comments.') -%}\n {%- endif -%}\n {{ magic }}{{ comment }}{{ magic }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.5223072, "supported_languages": null}, "macro.dbt_postgres.postgres__alter_relation_comment": {"name": "postgres__alter_relation_comment", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__alter_relation_comment", "macro_sql": "{% macro postgres__alter_relation_comment(relation, comment) %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n comment on {{ relation.type }} {{ relation }} is {{ escaped_comment }};\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres_escape_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.522679, "supported_languages": null}, "macro.dbt_postgres.postgres__alter_column_comment": {"name": "postgres__alter_column_comment", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__alter_column_comment", "macro_sql": "{% macro postgres__alter_column_comment(relation, column_dict) %}\n {% set existing_columns = adapter.get_columns_in_relation(relation) | map(attribute=\"name\") | list %}\n {% for column_name in column_dict if (column_name in existing_columns) %}\n {% set comment = column_dict[column_name]['description'] %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n comment on column {{ relation }}.{{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }} is {{ escaped_comment }};\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres_escape_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.52365, "supported_languages": null}, "macro.dbt_postgres.postgres__get_show_grant_sql": {"name": "postgres__get_show_grant_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__get_show_grant_sql", "macro_sql": "\n\n{%- macro postgres__get_show_grant_sql(relation) -%}\n select grantee, privilege_type\n from {{ relation.information_schema('role_table_grants') }}\n where grantor = current_role\n and grantee != current_role\n and table_schema = '{{ relation.schema }}'\n and table_name = '{{ relation.identifier }}'\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.523999, "supported_languages": null}, "macro.dbt_postgres.postgres__copy_grants": {"name": "postgres__copy_grants", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__copy_grants", "macro_sql": "{% macro postgres__copy_grants() %}\n {{ return(False) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.524189, "supported_languages": null}, "macro.dbt_postgres.postgres__get_show_indexes_sql": {"name": "postgres__get_show_indexes_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__get_show_indexes_sql", "macro_sql": "{% macro postgres__get_show_indexes_sql(relation) %}\n select\n i.relname as name,\n m.amname as method,\n ix.indisunique as \"unique\",\n array_to_string(array_agg(a.attname), ',') as column_names\n from pg_index ix\n join pg_class i\n on i.oid = ix.indexrelid\n join pg_am m\n on m.oid=i.relam\n join pg_class t\n on t.oid = ix.indrelid\n join pg_namespace n\n on n.oid = t.relnamespace\n join pg_attribute a\n on a.attrelid = t.oid\n and a.attnum = ANY(ix.indkey)\n where t.relname = '{{ relation.identifier }}'\n and n.nspname = '{{ relation.schema }}'\n and t.relkind in ('r', 'm')\n group by 1, 2, 3\n order by 1, 2, 3\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.524483, "supported_languages": null}, "macro.dbt_postgres.postgres__get_drop_index_sql": {"name": "postgres__get_drop_index_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__get_drop_index_sql", "macro_sql": "\n\n\n{%- macro postgres__get_drop_index_sql(relation, index_name) -%}\n drop index if exists \"{{ relation.schema }}\".\"{{ index_name }}\"\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.524713, "supported_languages": null}, "macro.dbt_postgres.postgres__get_incremental_default_sql": {"name": "postgres__get_incremental_default_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/materializations/incremental_strategies.sql", "original_file_path": "macros/materializations/incremental_strategies.sql", "unique_id": "macro.dbt_postgres.postgres__get_incremental_default_sql", "macro_sql": "{% macro postgres__get_incremental_default_sql(arg_dict) %}\n\n {% if arg_dict[\"unique_key\"] %}\n {% do return(get_incremental_delete_insert_sql(arg_dict)) %}\n {% else %}\n {% do return(get_incremental_append_sql(arg_dict)) %}\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_incremental_delete_insert_sql", "macro.dbt.get_incremental_append_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.525357, "supported_languages": null}, "macro.dbt_postgres.postgres__snapshot_merge_sql": {"name": "postgres__snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/materializations/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshot_merge.sql", "unique_id": "macro.dbt_postgres.postgres__snapshot_merge_sql", "macro_sql": "{% macro postgres__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n update {{ target }}\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n from {{ source }} as DBT_INTERNAL_SOURCE\n where DBT_INTERNAL_SOURCE.dbt_scd_id::text = {{ target }}.dbt_scd_id::text\n and DBT_INTERNAL_SOURCE.dbt_change_type::text in ('update'::text, 'delete'::text)\n and {{ target }}.dbt_valid_to is null;\n\n insert into {{ target }} ({{ insert_cols_csv }})\n select {% for column in insert_cols -%}\n DBT_INTERNAL_SOURCE.{{ column }} {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n from {{ source }} as DBT_INTERNAL_SOURCE\n where DBT_INTERNAL_SOURCE.dbt_change_type::text = 'insert'::text;\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.526551, "supported_languages": null}, "macro.dbt_postgres.postgres__drop_materialized_view": {"name": "postgres__drop_materialized_view", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/materialized_view/drop.sql", "original_file_path": "macros/relations/materialized_view/drop.sql", "unique_id": "macro.dbt_postgres.postgres__drop_materialized_view", "macro_sql": "{% macro postgres__drop_materialized_view(relation) -%}\n drop materialized view if exists {{ relation }} cascade\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.5267892, "supported_languages": null}, "macro.dbt_postgres.postgres__describe_materialized_view": {"name": "postgres__describe_materialized_view", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/materialized_view/describe.sql", "original_file_path": "macros/relations/materialized_view/describe.sql", "unique_id": "macro.dbt_postgres.postgres__describe_materialized_view", "macro_sql": "{% macro postgres__describe_materialized_view(relation) %}\n -- for now just get the indexes, we don't need the name or the query yet\n {% set _indexes = run_query(get_show_indexes_sql(relation)) %}\n {% do return({'indexes': _indexes}) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query", "macro.dbt.get_show_indexes_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.527286, "supported_languages": null}, "macro.dbt_postgres.postgres__refresh_materialized_view": {"name": "postgres__refresh_materialized_view", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/materialized_view/refresh.sql", "original_file_path": "macros/relations/materialized_view/refresh.sql", "unique_id": "macro.dbt_postgres.postgres__refresh_materialized_view", "macro_sql": "{% macro postgres__refresh_materialized_view(relation) %}\n refresh materialized view {{ relation }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.527518, "supported_languages": null}, "macro.dbt_postgres.postgres__get_rename_materialized_view_sql": {"name": "postgres__get_rename_materialized_view_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/materialized_view/rename.sql", "original_file_path": "macros/relations/materialized_view/rename.sql", "unique_id": "macro.dbt_postgres.postgres__get_rename_materialized_view_sql", "macro_sql": "{% macro postgres__get_rename_materialized_view_sql(relation, new_name) %}\n alter materialized view {{ relation }} rename to {{ new_name }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.527804, "supported_languages": null}, "macro.dbt_postgres.postgres__get_alter_materialized_view_as_sql": {"name": "postgres__get_alter_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt_postgres.postgres__get_alter_materialized_view_as_sql", "macro_sql": "{% macro postgres__get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n\n -- apply a full refresh immediately if needed\n {% if configuration_changes.requires_full_refresh %}\n\n {{ get_replace_sql(existing_relation, relation, sql) }}\n\n -- otherwise apply individual changes as needed\n {% else %}\n\n {{ postgres__update_indexes_on_materialized_view(relation, configuration_changes.indexes) }}\n\n {%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_replace_sql", "macro.dbt_postgres.postgres__update_indexes_on_materialized_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.529207, "supported_languages": null}, "macro.dbt_postgres.postgres__update_indexes_on_materialized_view": {"name": "postgres__update_indexes_on_materialized_view", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt_postgres.postgres__update_indexes_on_materialized_view", "macro_sql": "\n\n\n{%- macro postgres__update_indexes_on_materialized_view(relation, index_changes) -%}\n {{- log(\"Applying UPDATE INDEXES to: \" ~ relation) -}}\n\n {%- for _index_change in index_changes -%}\n {%- set _index = _index_change.context -%}\n\n {%- if _index_change.action == \"drop\" -%}\n\n {{ postgres__get_drop_index_sql(relation, _index.name) }};\n\n {%- elif _index_change.action == \"create\" -%}\n\n {{ postgres__get_create_index_sql(relation, _index.as_node_config) }}\n\n {%- endif -%}\n\n {%- endfor -%}\n\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_drop_index_sql", "macro.dbt_postgres.postgres__get_create_index_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.530157, "supported_languages": null}, "macro.dbt_postgres.postgres__get_materialized_view_configuration_changes": {"name": "postgres__get_materialized_view_configuration_changes", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt_postgres.postgres__get_materialized_view_configuration_changes", "macro_sql": "{% macro postgres__get_materialized_view_configuration_changes(existing_relation, new_config) %}\n {% set _existing_materialized_view = postgres__describe_materialized_view(existing_relation) %}\n {% set _configuration_changes = existing_relation.get_materialized_view_config_change_collection(_existing_materialized_view, new_config) %}\n {% do return(_configuration_changes) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__describe_materialized_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.530636, "supported_languages": null}, "macro.dbt_postgres.postgres__get_create_materialized_view_as_sql": {"name": "postgres__get_create_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/materialized_view/create.sql", "original_file_path": "macros/relations/materialized_view/create.sql", "unique_id": "macro.dbt_postgres.postgres__get_create_materialized_view_as_sql", "macro_sql": "{% macro postgres__get_create_materialized_view_as_sql(relation, sql) %}\n create materialized view if not exists {{ relation }} as {{ sql }};\n\n {% for _index_dict in config.get('indexes', []) -%}\n {{- get_create_index_sql(relation, _index_dict) -}}\n {%- endfor -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_create_index_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.5312471, "supported_languages": null}, "macro.dbt_postgres.postgres__drop_table": {"name": "postgres__drop_table", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/table/drop.sql", "original_file_path": "macros/relations/table/drop.sql", "unique_id": "macro.dbt_postgres.postgres__drop_table", "macro_sql": "{% macro postgres__drop_table(relation) -%}\n drop table if exists {{ relation }} cascade\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.5314791, "supported_languages": null}, "macro.dbt_postgres.postgres__get_replace_table_sql": {"name": "postgres__get_replace_table_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/table/replace.sql", "original_file_path": "macros/relations/table/replace.sql", "unique_id": "macro.dbt_postgres.postgres__get_replace_table_sql", "macro_sql": "{% macro postgres__get_replace_table_sql(relation, sql) -%}\n\n {%- set sql_header = config.get('sql_header', none) -%}\n {{ sql_header if sql_header is not none }}\n\n create or replace table {{ relation }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {{ get_table_columns_and_constraints() }}\n {%- set sql = get_select_subquery(sql) %}\n {% endif %}\n as (\n {{ sql }}\n );\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_assert_columns_equivalent", "macro.dbt.get_table_columns_and_constraints", "macro.dbt.get_select_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.532609, "supported_languages": null}, "macro.dbt_postgres.postgres__get_rename_table_sql": {"name": "postgres__get_rename_table_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/table/rename.sql", "original_file_path": "macros/relations/table/rename.sql", "unique_id": "macro.dbt_postgres.postgres__get_rename_table_sql", "macro_sql": "{% macro postgres__get_rename_table_sql(relation, new_name) %}\n alter table {{ relation }} rename to {{ new_name }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.5328991, "supported_languages": null}, "macro.dbt_postgres.postgres__drop_view": {"name": "postgres__drop_view", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/view/drop.sql", "original_file_path": "macros/relations/view/drop.sql", "unique_id": "macro.dbt_postgres.postgres__drop_view", "macro_sql": "{% macro postgres__drop_view(relation) -%}\n drop view if exists {{ relation }} cascade\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.5331168, "supported_languages": null}, "macro.dbt_postgres.postgres__get_replace_view_sql": {"name": "postgres__get_replace_view_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt_postgres.postgres__get_replace_view_sql", "macro_sql": "{% macro postgres__get_replace_view_sql(relation, sql) -%}\n\n {%- set sql_header = config.get('sql_header', none) -%}\n {{ sql_header if sql_header is not none }}\n\n create or replace view {{ relation }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {%- endif %}\n as (\n {{ sql }}\n );\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.5340202, "supported_languages": null}, "macro.dbt_postgres.postgres__get_rename_view_sql": {"name": "postgres__get_rename_view_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/view/rename.sql", "original_file_path": "macros/relations/view/rename.sql", "unique_id": "macro.dbt_postgres.postgres__get_rename_view_sql", "macro_sql": "{% macro postgres__get_rename_view_sql(relation, new_name) %}\n alter view {{ relation }} rename to {{ new_name }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.534313, "supported_languages": null}, "macro.dbt_postgres.postgres__dateadd": {"name": "postgres__dateadd", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt_postgres.postgres__dateadd", "macro_sql": "{% macro postgres__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n {{ from_date_or_timestamp }} + ((interval '1 {{ datepart }}') * ({{ interval }}))\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.534663, "supported_languages": null}, "macro.dbt_postgres.postgres__listagg": {"name": "postgres__listagg", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "unique_id": "macro.dbt_postgres.postgres__listagg", "macro_sql": "{% macro postgres__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n {% if limit_num -%}\n array_to_string(\n (array_agg(\n {{ measure }}\n {% if order_by_clause -%}\n {{ order_by_clause }}\n {%- endif %}\n ))[1:{{ limit_num }}],\n {{ delimiter_text }}\n )\n {%- else %}\n string_agg(\n {{ measure }},\n {{ delimiter_text }}\n {% if order_by_clause -%}\n {{ order_by_clause }}\n {%- endif %}\n )\n {%- endif %}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.5356429, "supported_languages": null}, "macro.dbt_postgres.postgres__datediff": {"name": "postgres__datediff", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt_postgres.postgres__datediff", "macro_sql": "{% macro postgres__datediff(first_date, second_date, datepart) -%}\n\n {% if datepart == 'year' %}\n (date_part('year', ({{second_date}})::date) - date_part('year', ({{first_date}})::date))\n {% elif datepart == 'quarter' %}\n ({{ datediff(first_date, second_date, 'year') }} * 4 + date_part('quarter', ({{second_date}})::date) - date_part('quarter', ({{first_date}})::date))\n {% elif datepart == 'month' %}\n ({{ datediff(first_date, second_date, 'year') }} * 12 + date_part('month', ({{second_date}})::date) - date_part('month', ({{first_date}})::date))\n {% elif datepart == 'day' %}\n (({{second_date}})::date - ({{first_date}})::date)\n {% elif datepart == 'week' %}\n ({{ datediff(first_date, second_date, 'day') }} / 7 + case\n when date_part('dow', ({{first_date}})::timestamp) <= date_part('dow', ({{second_date}})::timestamp) then\n case when {{first_date}} <= {{second_date}} then 0 else -1 end\n else\n case when {{first_date}} <= {{second_date}} then 1 else 0 end\n end)\n {% elif datepart == 'hour' %}\n ({{ datediff(first_date, second_date, 'day') }} * 24 + date_part('hour', ({{second_date}})::timestamp) - date_part('hour', ({{first_date}})::timestamp))\n {% elif datepart == 'minute' %}\n ({{ datediff(first_date, second_date, 'hour') }} * 60 + date_part('minute', ({{second_date}})::timestamp) - date_part('minute', ({{first_date}})::timestamp))\n {% elif datepart == 'second' %}\n ({{ datediff(first_date, second_date, 'minute') }} * 60 + floor(date_part('second', ({{second_date}})::timestamp)) - floor(date_part('second', ({{first_date}})::timestamp)))\n {% elif datepart == 'millisecond' %}\n ({{ datediff(first_date, second_date, 'minute') }} * 60000 + floor(date_part('millisecond', ({{second_date}})::timestamp)) - floor(date_part('millisecond', ({{first_date}})::timestamp)))\n {% elif datepart == 'microsecond' %}\n ({{ datediff(first_date, second_date, 'minute') }} * 60000000 + floor(date_part('microsecond', ({{second_date}})::timestamp)) - floor(date_part('microsecond', ({{first_date}})::timestamp)))\n {% else %}\n {{ exceptions.raise_compiler_error(\"Unsupported datepart for macro datediff in postgres: {!r}\".format(datepart)) }}\n {% endif %}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.datediff"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.5409238, "supported_languages": null}, "macro.dbt_postgres.postgres__any_value": {"name": "postgres__any_value", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "unique_id": "macro.dbt_postgres.postgres__any_value", "macro_sql": "{% macro postgres__any_value(expression) -%}\n\n min({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.541182, "supported_languages": null}, "macro.dbt_postgres.postgres__last_day": {"name": "postgres__last_day", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt_postgres.postgres__last_day", "macro_sql": "{% macro postgres__last_day(date, datepart) -%}\n\n {%- if datepart == 'quarter' -%}\n -- postgres dateadd does not support quarter interval.\n cast(\n {{dbt.dateadd('day', '-1',\n dbt.dateadd('month', '3', dbt.date_trunc(datepart, date))\n )}}\n as date)\n {%- else -%}\n {{dbt.default_last_day(date, datepart)}}\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.dateadd", "macro.dbt.date_trunc", "macro.dbt.default_last_day"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.541986, "supported_languages": null}, "macro.dbt_postgres.postgres__split_part": {"name": "postgres__split_part", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt_postgres.postgres__split_part", "macro_sql": "{% macro postgres__split_part(string_text, delimiter_text, part_number) %}\n\n {% if part_number >= 0 %}\n {{ dbt.default__split_part(string_text, delimiter_text, part_number) }}\n {% else %}\n {{ dbt._split_part_negative(string_text, delimiter_text, part_number) }}\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__split_part", "macro.dbt._split_part_negative"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.542672, "supported_languages": null}, "macro.dbt.run_hooks": {"name": "run_hooks", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.run_hooks", "macro_sql": "{% macro run_hooks(hooks, inside_transaction=True) %}\n {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %}\n {% if not inside_transaction and loop.first %}\n {% call statement(auto_begin=inside_transaction) %}\n commit;\n {% endcall %}\n {% endif %}\n {% set rendered = render(hook.get('sql')) | trim %}\n {% if (rendered | length) > 0 %}\n {% call statement(auto_begin=inside_transaction) %}\n {{ rendered }}\n {% endcall %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.544426, "supported_languages": null}, "macro.dbt.make_hook_config": {"name": "make_hook_config", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.make_hook_config", "macro_sql": "{% macro make_hook_config(sql, inside_transaction) %}\n {{ tojson({\"sql\": sql, \"transaction\": inside_transaction}) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.5447628, "supported_languages": null}, "macro.dbt.before_begin": {"name": "before_begin", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.before_begin", "macro_sql": "{% macro before_begin(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.545001, "supported_languages": null}, "macro.dbt.in_transaction": {"name": "in_transaction", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.in_transaction", "macro_sql": "{% macro in_transaction(sql) %}\n {{ make_hook_config(sql, inside_transaction=True) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.545242, "supported_languages": null}, "macro.dbt.after_commit": {"name": "after_commit", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.after_commit", "macro_sql": "{% macro after_commit(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.545506, "supported_languages": null}, "macro.dbt.set_sql_header": {"name": "set_sql_header", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "unique_id": "macro.dbt.set_sql_header", "macro_sql": "{% macro set_sql_header(config) -%}\n {{ config.set('sql_header', caller()) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.5461252, "supported_languages": null}, "macro.dbt.should_full_refresh": {"name": "should_full_refresh", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "unique_id": "macro.dbt.should_full_refresh", "macro_sql": "{% macro should_full_refresh() %}\n {% set config_full_refresh = config.get('full_refresh') %}\n {% if config_full_refresh is none %}\n {% set config_full_refresh = flags.FULL_REFRESH %}\n {% endif %}\n {% do return(config_full_refresh) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.5466251, "supported_languages": null}, "macro.dbt.should_store_failures": {"name": "should_store_failures", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "unique_id": "macro.dbt.should_store_failures", "macro_sql": "{% macro should_store_failures() %}\n {% set config_store_failures = config.get('store_failures') %}\n {% if config_store_failures is none %}\n {% set config_store_failures = flags.STORE_FAILURES %}\n {% endif %}\n {% do return(config_store_failures) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.547284, "supported_languages": null}, "macro.dbt.snapshot_merge_sql": {"name": "snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", "unique_id": "macro.dbt.snapshot_merge_sql", "macro_sql": "{% macro snapshot_merge_sql(target, source, insert_cols) -%}\n {{ adapter.dispatch('snapshot_merge_sql', 'dbt')(target, source, insert_cols) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__snapshot_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.548049, "supported_languages": null}, "macro.dbt.default__snapshot_merge_sql": {"name": "default__snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", "unique_id": "macro.dbt.default__snapshot_merge_sql", "macro_sql": "{% macro default__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on DBT_INTERNAL_SOURCE.dbt_scd_id = DBT_INTERNAL_DEST.dbt_scd_id\n\n when matched\n and DBT_INTERNAL_DEST.dbt_valid_to is null\n and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')\n then update\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n\n when not matched\n and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'\n then insert ({{ insert_cols_csv }})\n values ({{ insert_cols_csv }})\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.548515, "supported_languages": null}, "macro.dbt.strategy_dispatch": {"name": "strategy_dispatch", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.strategy_dispatch", "macro_sql": "{% macro strategy_dispatch(name) -%}\n{% set original_name = name %}\n {% if '.' in name %}\n {% set package_name, name = name.split(\".\", 1) %}\n {% else %}\n {% set package_name = none %}\n {% endif %}\n\n {% if package_name is none %}\n {% set package_context = context %}\n {% elif package_name in context %}\n {% set package_context = context[package_name] %}\n {% else %}\n {% set error_msg %}\n Could not find package '{{package_name}}', called with '{{original_name}}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n\n {%- set search_name = 'snapshot_' ~ name ~ '_strategy' -%}\n\n {% if search_name not in package_context %}\n {% set error_msg %}\n The specified strategy macro '{{name}}' was not found in package '{{ package_name }}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n {{ return(package_context[search_name]) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.5546088, "supported_languages": null}, "macro.dbt.snapshot_hash_arguments": {"name": "snapshot_hash_arguments", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_hash_arguments", "macro_sql": "{% macro snapshot_hash_arguments(args) -%}\n {{ adapter.dispatch('snapshot_hash_arguments', 'dbt')(args) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.554924, "supported_languages": null}, "macro.dbt.default__snapshot_hash_arguments": {"name": "default__snapshot_hash_arguments", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.default__snapshot_hash_arguments", "macro_sql": "{% macro default__snapshot_hash_arguments(args) -%}\n md5({%- for arg in args -%}\n coalesce(cast({{ arg }} as varchar ), '')\n {% if not loop.last %} || '|' || {% endif %}\n {%- endfor -%})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.55531, "supported_languages": null}, "macro.dbt.snapshot_timestamp_strategy": {"name": "snapshot_timestamp_strategy", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_timestamp_strategy", "macro_sql": "{% macro snapshot_timestamp_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}\n {% set primary_key = config['unique_key'] %}\n {% set updated_at = config['updated_at'] %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %}\n\n {#/*\n The snapshot relation might not have an {{ updated_at }} value if the\n snapshot strategy is changed from `check` to `timestamp`. We\n should use a dbt-created column for the comparison in the snapshot\n table instead of assuming that the user-supplied {{ updated_at }}\n will be present in the historical data.\n\n See https://github.com/dbt-labs/dbt-core/issues/2350\n */ #}\n {% set row_changed_expr -%}\n ({{ snapshotted_rel }}.dbt_valid_from < {{ current_rel }}.{{ updated_at }})\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.556515, "supported_languages": null}, "macro.dbt.snapshot_string_as_time": {"name": "snapshot_string_as_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_string_as_time", "macro_sql": "{% macro snapshot_string_as_time(timestamp) -%}\n {{ adapter.dispatch('snapshot_string_as_time', 'dbt')(timestamp) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__snapshot_string_as_time"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.5568, "supported_languages": null}, "macro.dbt.default__snapshot_string_as_time": {"name": "default__snapshot_string_as_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.default__snapshot_string_as_time", "macro_sql": "{% macro default__snapshot_string_as_time(timestamp) %}\n {% do exceptions.raise_not_implemented(\n 'snapshot_string_as_time macro not implemented for adapter '+adapter.type()\n ) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.557099, "supported_languages": null}, "macro.dbt.snapshot_check_all_get_existing_columns": {"name": "snapshot_check_all_get_existing_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_check_all_get_existing_columns", "macro_sql": "{% macro snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) -%}\n {%- if not target_exists -%}\n {#-- no table yet -> return whatever the query does --#}\n {{ return((false, query_columns)) }}\n {%- endif -%}\n\n {#-- handle any schema changes --#}\n {%- set target_relation = adapter.get_relation(database=node.database, schema=node.schema, identifier=node.alias) -%}\n\n {% if check_cols_config == 'all' %}\n {%- set query_columns = get_columns_in_query(node['compiled_code']) -%}\n\n {% elif check_cols_config is iterable and (check_cols_config | length) > 0 %}\n {#-- query for proper casing/quoting, to support comparison below --#}\n {%- set select_check_cols_from_target -%}\n {#-- N.B. The whitespace below is necessary to avoid edge case issue with comments --#}\n {#-- See: https://github.com/dbt-labs/dbt-core/issues/6781 --#}\n select {{ check_cols_config | join(', ') }} from (\n {{ node['compiled_code'] }}\n ) subq\n {%- endset -%}\n {% set query_columns = get_columns_in_query(select_check_cols_from_target) %}\n\n {% else %}\n {% do exceptions.raise_compiler_error(\"Invalid value for 'check_cols': \" ~ check_cols_config) %}\n {% endif %}\n\n {%- set existing_cols = adapter.get_columns_in_relation(target_relation) | map(attribute = 'name') | list -%}\n {%- set ns = namespace() -%} {#-- handle for-loop scoping with a namespace --#}\n {%- set ns.column_added = false -%}\n\n {%- set intersection = [] -%}\n {%- for col in query_columns -%}\n {%- if col in existing_cols -%}\n {%- do intersection.append(adapter.quote(col)) -%}\n {%- else -%}\n {% set ns.column_added = true %}\n {%- endif -%}\n {%- endfor -%}\n {{ return((ns.column_added, intersection)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_columns_in_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.559448, "supported_languages": null}, "macro.dbt.snapshot_check_strategy": {"name": "snapshot_check_strategy", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_check_strategy", "macro_sql": "{% macro snapshot_check_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}\n {% set check_cols_config = config['check_cols'] %}\n {% set primary_key = config['unique_key'] %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %}\n {% set updated_at = config.get('updated_at', snapshot_get_time()) %}\n\n {% set column_added = false %}\n\n {% set column_added, check_cols = snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) %}\n\n {%- set row_changed_expr -%}\n (\n {%- if column_added -%}\n {{ get_true_sql() }}\n {%- else -%}\n {%- for col in check_cols -%}\n {{ snapshotted_rel }}.{{ col }} != {{ current_rel }}.{{ col }}\n or\n (\n (({{ snapshotted_rel }}.{{ col }} is null) and not ({{ current_rel }}.{{ col }} is null))\n or\n ((not {{ snapshotted_rel }}.{{ col }} is null) and ({{ current_rel }}.{{ col }} is null))\n )\n {%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n {%- endif -%}\n )\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.snapshot_get_time", "macro.dbt.snapshot_check_all_get_existing_columns", "macro.dbt.get_true_sql", "macro.dbt.snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.561697, "supported_languages": null}, "macro.dbt.create_columns": {"name": "create_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.create_columns", "macro_sql": "{% macro create_columns(relation, columns) %}\n {{ adapter.dispatch('create_columns', 'dbt')(relation, columns) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__create_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.567965, "supported_languages": null}, "macro.dbt.default__create_columns": {"name": "default__create_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__create_columns", "macro_sql": "{% macro default__create_columns(relation, columns) %}\n {% for column in columns %}\n {% call statement() %}\n alter table {{ relation }} add column \"{{ column.name }}\" {{ column.data_type }};\n {% endcall %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.568439, "supported_languages": null}, "macro.dbt.post_snapshot": {"name": "post_snapshot", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.post_snapshot", "macro_sql": "{% macro post_snapshot(staging_relation) %}\n {{ adapter.dispatch('post_snapshot', 'dbt')(staging_relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.568725, "supported_languages": null}, "macro.dbt.default__post_snapshot": {"name": "default__post_snapshot", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__post_snapshot", "macro_sql": "{% macro default__post_snapshot(staging_relation) %}\n {# no-op #}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.568873, "supported_languages": null}, "macro.dbt.get_true_sql": {"name": "get_true_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.get_true_sql", "macro_sql": "{% macro get_true_sql() %}\n {{ adapter.dispatch('get_true_sql', 'dbt')() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_true_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.569119, "supported_languages": null}, "macro.dbt.default__get_true_sql": {"name": "default__get_true_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__get_true_sql", "macro_sql": "{% macro default__get_true_sql() %}\n {{ return('TRUE') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.569309, "supported_languages": null}, "macro.dbt.snapshot_staging_table": {"name": "snapshot_staging_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.snapshot_staging_table", "macro_sql": "{% macro snapshot_staging_table(strategy, source_sql, target_relation) -%}\n {{ adapter.dispatch('snapshot_staging_table', 'dbt')(strategy, source_sql, target_relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__snapshot_staging_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.569651, "supported_languages": null}, "macro.dbt.default__snapshot_staging_table": {"name": "default__snapshot_staging_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__snapshot_staging_table", "macro_sql": "{% macro default__snapshot_staging_table(strategy, source_sql, target_relation) -%}\n\n with snapshot_query as (\n\n {{ source_sql }}\n\n ),\n\n snapshotted_data as (\n\n select *,\n {{ strategy.unique_key }} as dbt_unique_key\n\n from {{ target_relation }}\n where dbt_valid_to is null\n\n ),\n\n insertions_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to,\n {{ strategy.scd_id }} as dbt_scd_id\n\n from snapshot_query\n ),\n\n updates_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n {{ strategy.updated_at }} as dbt_valid_to\n\n from snapshot_query\n ),\n\n {%- if strategy.invalidate_hard_deletes %}\n\n deletes_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key\n from snapshot_query\n ),\n {% endif %}\n\n insertions as (\n\n select\n 'insert' as dbt_change_type,\n source_data.*\n\n from insertions_source_data as source_data\n left outer join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where snapshotted_data.dbt_unique_key is null\n or (\n snapshotted_data.dbt_unique_key is not null\n and (\n {{ strategy.row_changed }}\n )\n )\n\n ),\n\n updates as (\n\n select\n 'update' as dbt_change_type,\n source_data.*,\n snapshotted_data.dbt_scd_id\n\n from updates_source_data as source_data\n join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where (\n {{ strategy.row_changed }}\n )\n )\n\n {%- if strategy.invalidate_hard_deletes -%}\n ,\n\n deletes as (\n\n select\n 'delete' as dbt_change_type,\n source_data.*,\n {{ snapshot_get_time() }} as dbt_valid_from,\n {{ snapshot_get_time() }} as dbt_updated_at,\n {{ snapshot_get_time() }} as dbt_valid_to,\n snapshotted_data.dbt_scd_id\n\n from snapshotted_data\n left join deletes_source_data as source_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where source_data.dbt_unique_key is null\n )\n {%- endif %}\n\n select * from insertions\n union all\n select * from updates\n {%- if strategy.invalidate_hard_deletes %}\n union all\n select * from deletes\n {%- endif %}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.snapshot_get_time"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.571101, "supported_languages": null}, "macro.dbt.build_snapshot_table": {"name": "build_snapshot_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.build_snapshot_table", "macro_sql": "{% macro build_snapshot_table(strategy, sql) -%}\n {{ adapter.dispatch('build_snapshot_table', 'dbt')(strategy, sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__build_snapshot_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.571421, "supported_languages": null}, "macro.dbt.default__build_snapshot_table": {"name": "default__build_snapshot_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__build_snapshot_table", "macro_sql": "{% macro default__build_snapshot_table(strategy, sql) %}\n\n select *,\n {{ strategy.scd_id }} as dbt_scd_id,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to\n from (\n {{ sql }}\n ) sbq\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.571839, "supported_languages": null}, "macro.dbt.build_snapshot_staging_table": {"name": "build_snapshot_staging_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.build_snapshot_staging_table", "macro_sql": "{% macro build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% set temp_relation = make_temp_relation(target_relation) %}\n\n {% set select = snapshot_staging_table(strategy, sql, target_relation) %}\n\n {% call statement('build_snapshot_staging_relation') %}\n {{ create_table_as(True, temp_relation, select) }}\n {% endcall %}\n\n {% do return(temp_relation) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_temp_relation", "macro.dbt.snapshot_staging_table", "macro.dbt.statement", "macro.dbt.create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.5725482, "supported_languages": null}, "macro.dbt.materialization_snapshot_default": {"name": "materialization_snapshot_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/snapshot.sql", "original_file_path": "macros/materializations/snapshots/snapshot.sql", "unique_id": "macro.dbt.materialization_snapshot_default", "macro_sql": "{% materialization snapshot, default %}\n {%- set config = model['config'] -%}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n -- grab current tables grants config for comparision later on\n {%- set grant_config = config.get('grants') -%}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=model.database,\n schema=model.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", config, target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_code']) %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% else %}\n\n {{ adapter.valid_snapshot_target(target_relation) }}\n\n {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% endif %}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode=False) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if not target_relation_exists %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if staging_table is defined %}\n {% do post_snapshot(staging_table) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.get_or_create_relation", "macro.dbt.run_hooks", "macro.dbt.strategy_dispatch", "macro.dbt.build_snapshot_table", "macro.dbt.create_table_as", "macro.dbt.build_snapshot_staging_table", "macro.dbt.create_columns", "macro.dbt.snapshot_merge_sql", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes", "macro.dbt.post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.582187, "supported_languages": ["sql"]}, "macro.dbt.materialization_test_default": {"name": "materialization_test_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/test.sql", "original_file_path": "macros/materializations/tests/test.sql", "unique_id": "macro.dbt.materialization_test_default", "macro_sql": "{%- materialization test, default -%}\n\n {% set relations = [] %}\n\n {% if should_store_failures() %}\n\n {% set identifier = model['alias'] %}\n {% set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n\n {% set store_failures_as = config.get('store_failures_as') %}\n -- if `--store-failures` is invoked via command line and `store_failures_as` is not set,\n -- config.get('store_failures_as', 'table') returns None, not 'table'\n {% if store_failures_as == none %}{% set store_failures_as = 'table' %}{% endif %}\n {% if store_failures_as not in ['table', 'view'] %}\n {{ exceptions.raise_compiler_error(\n \"'\" ~ store_failures_as ~ \"' is not a valid value for `store_failures_as`. \"\n \"Accepted values are: ['ephemeral', 'table', 'view']\"\n ) }}\n {% endif %}\n\n {% set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database, type=store_failures_as) -%} %}\n\n {% if old_relation %}\n {% do adapter.drop_relation(old_relation) %}\n {% endif %}\n\n {% call statement(auto_begin=True) %}\n {{ get_create_sql(target_relation, sql) }}\n {% endcall %}\n\n {% do relations.append(target_relation) %}\n\n {% set main_sql %}\n select *\n from {{ target_relation }}\n {% endset %}\n\n {{ adapter.commit() }}\n\n {% else %}\n\n {% set main_sql = sql %}\n\n {% endif %}\n\n {% set limit = config.get('limit') %}\n {% set fail_calc = config.get('fail_calc') %}\n {% set warn_if = config.get('warn_if') %}\n {% set error_if = config.get('error_if') %}\n\n {% call statement('main', fetch_result=True) -%}\n\n {{ get_test_sql(main_sql, fail_calc, warn_if, error_if, limit)}}\n\n {%- endcall %}\n\n {{ return({'relations': relations}) }}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.should_store_failures", "macro.dbt.statement", "macro.dbt.get_create_sql", "macro.dbt.get_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.586581, "supported_languages": ["sql"]}, "macro.dbt.get_test_sql": {"name": "get_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.get_test_sql", "macro_sql": "{% macro get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n {{ adapter.dispatch('get_test_sql', 'dbt')(main_sql, fail_calc, warn_if, error_if, limit) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.587331, "supported_languages": null}, "macro.dbt.default__get_test_sql": {"name": "default__get_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.default__get_test_sql", "macro_sql": "{% macro default__get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n select\n {{ fail_calc }} as failures,\n {{ fail_calc }} {{ warn_if }} as should_warn,\n {{ fail_calc }} {{ error_if }} as should_error\n from (\n {{ main_sql }}\n {{ \"limit \" ~ limit if limit != none }}\n ) dbt_internal_test\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.587847, "supported_languages": null}, "macro.dbt.get_where_subquery": {"name": "get_where_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/where_subquery.sql", "original_file_path": "macros/materializations/tests/where_subquery.sql", "unique_id": "macro.dbt.get_where_subquery", "macro_sql": "{% macro get_where_subquery(relation) -%}\n {% do return(adapter.dispatch('get_where_subquery', 'dbt')(relation)) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_where_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.5884538, "supported_languages": null}, "macro.dbt.default__get_where_subquery": {"name": "default__get_where_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/where_subquery.sql", "original_file_path": "macros/materializations/tests/where_subquery.sql", "unique_id": "macro.dbt.default__get_where_subquery", "macro_sql": "{% macro default__get_where_subquery(relation) -%}\n {% set where = config.get('where', '') %}\n {% if where %}\n {%- set filtered -%}\n (select * from {{ relation }} where {{ where }}) dbt_subquery\n {%- endset -%}\n {% do return(filtered) %}\n {%- else -%}\n {% do return(relation) %}\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.589078, "supported_languages": null}, "macro.dbt.materialization_materialized_view_default": {"name": "materialization_materialized_view_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialization_materialized_view_default", "macro_sql": "{% materialization materialized_view, default %}\n {% set existing_relation = load_cached_relation(this) %}\n {% set target_relation = this.incorporate(type=this.MaterializedView) %}\n {% set intermediate_relation = make_intermediate_relation(target_relation) %}\n {% set backup_relation_type = target_relation.MaterializedView if existing_relation is none else existing_relation.type %}\n {% set backup_relation = make_backup_relation(target_relation, backup_relation_type) %}\n\n {{ materialized_view_setup(backup_relation, intermediate_relation, pre_hooks) }}\n\n {% set build_sql = materialized_view_get_build_sql(existing_relation, target_relation, backup_relation, intermediate_relation) %}\n\n {% if build_sql == '' %}\n {{ materialized_view_execute_no_op(target_relation) }}\n {% else %}\n {{ materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) }}\n {% endif %}\n\n {{ materialized_view_teardown(backup_relation, intermediate_relation, post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.materialized_view_setup", "macro.dbt.materialized_view_get_build_sql", "macro.dbt.materialized_view_execute_no_op", "macro.dbt.materialized_view_execute_build_sql", "macro.dbt.materialized_view_teardown"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.5966778, "supported_languages": ["sql"]}, "macro.dbt.materialized_view_setup": {"name": "materialized_view_setup", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_setup", "macro_sql": "{% macro materialized_view_setup(backup_relation, intermediate_relation, pre_hooks) %}\n\n -- backup_relation and intermediate_relation should not already exist in the database\n -- it's possible these exist because of a previous run that exited unexpectedly\n {% set preexisting_backup_relation = load_cached_relation(backup_relation) %}\n {% set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) %}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.5973198, "supported_languages": null}, "macro.dbt.materialized_view_teardown": {"name": "materialized_view_teardown", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_teardown", "macro_sql": "{% macro materialized_view_teardown(backup_relation, intermediate_relation, post_hooks) %}\n\n -- drop the temp relations if they exist to leave the database clean for the next run\n {{ drop_relation_if_exists(backup_relation) }}\n {{ drop_relation_if_exists(intermediate_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.597737, "supported_languages": null}, "macro.dbt.materialized_view_get_build_sql": {"name": "materialized_view_get_build_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_get_build_sql", "macro_sql": "{% macro materialized_view_get_build_sql(existing_relation, target_relation, backup_relation, intermediate_relation) %}\n\n {% set full_refresh_mode = should_full_refresh() %}\n\n -- determine the scenario we're in: create, full_refresh, alter, refresh data\n {% if existing_relation is none %}\n {% set build_sql = get_create_materialized_view_as_sql(target_relation, sql) %}\n {% elif full_refresh_mode or not existing_relation.is_materialized_view %}\n {% set build_sql = get_replace_sql(existing_relation, target_relation, sql) %}\n {% else %}\n\n -- get config options\n {% set on_configuration_change = config.get('on_configuration_change') %}\n {% set configuration_changes = get_materialized_view_configuration_changes(existing_relation, config) %}\n\n {% if configuration_changes is none %}\n {% set build_sql = refresh_materialized_view(target_relation) %}\n\n {% elif on_configuration_change == 'apply' %}\n {% set build_sql = get_alter_materialized_view_as_sql(target_relation, configuration_changes, sql, existing_relation, backup_relation, intermediate_relation) %}\n {% elif on_configuration_change == 'continue' %}\n {% set build_sql = '' %}\n {{ exceptions.warn(\"Configuration changes were identified and `on_configuration_change` was set to `continue` for `\" ~ target_relation ~ \"`\") }}\n {% elif on_configuration_change == 'fail' %}\n {{ exceptions.raise_fail_fast_error(\"Configuration changes were identified and `on_configuration_change` was set to `fail` for `\" ~ target_relation ~ \"`\") }}\n\n {% else %}\n -- this only happens if the user provides a value other than `apply`, 'skip', 'fail'\n {{ exceptions.raise_compiler_error(\"Unexpected configuration scenario\") }}\n\n {% endif %}\n\n {% endif %}\n\n {% do return(build_sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.get_create_materialized_view_as_sql", "macro.dbt.get_replace_sql", "macro.dbt.get_materialized_view_configuration_changes", "macro.dbt.refresh_materialized_view", "macro.dbt.get_alter_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.599859, "supported_languages": null}, "macro.dbt.materialized_view_execute_no_op": {"name": "materialized_view_execute_no_op", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_execute_no_op", "macro_sql": "{% macro materialized_view_execute_no_op(target_relation) %}\n {% do store_raw_result(\n name=\"main\",\n message=\"skip \" ~ target_relation,\n code=\"skip\",\n rows_affected=\"-1\"\n ) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.600245, "supported_languages": null}, "macro.dbt.materialized_view_execute_build_sql": {"name": "materialized_view_execute_build_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_execute_build_sql", "macro_sql": "{% macro materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) %}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set grant_config = config.get('grants') %}\n\n {% call statement(name=\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.601315, "supported_languages": null}, "macro.dbt.materialization_view_default": {"name": "materialization_view_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/view.sql", "original_file_path": "macros/materializations/models/view.sql", "unique_id": "macro.dbt.materialization_view_default", "macro_sql": "{%- materialization view, default -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='view') -%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}\n\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}\n /*\n This relation (probably) doesn't exist yet. If it does exist, it's a leftover from\n a previous run, and we're going to try to drop it immediately. At the end of this\n materialization, we're going to rename the \"existing_relation\" to this identifier,\n and then we're going to drop it. In order to make sure we run the correct one of:\n - drop view ...\n - drop table ...\n\n We need to set the type of this relation to be the type of the existing_relation, if it exists,\n or else \"view\" as a sane default if it does not. Note that if the existing_relation does not\n exist, then there is nothing to move out of the way and subsequentally drop. In that case,\n this relation will be effectively unused.\n */\n {%- set backup_relation_type = 'view' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n -- move the existing view out of the way\n {% if existing_relation is not none %}\n /* Do the equivalent of rename_if_exists. 'existing_relation' could have been dropped\n since the variable was first set. */\n {% set existing_relation = load_cached_relation(existing_relation) %}\n {% if existing_relation is not none %}\n {{ adapter.rename_relation(existing_relation, backup_relation) }}\n {% endif %}\n {% endif %}\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.run_hooks", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.get_create_view_as_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.6060789, "supported_languages": ["sql"]}, "macro.dbt.materialization_table_default": {"name": "materialization_table_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table.sql", "original_file_path": "macros/materializations/models/table.sql", "unique_id": "macro.dbt.materialization_table_default", "macro_sql": "{% materialization table, default %}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') %}\n {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}\n /*\n See ../view/view.sql for more information about this relation.\n */\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_table_as_sql(False, intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n {% if existing_relation is not none %}\n /* Do the equivalent of rename_if_exists. 'existing_relation' could have been dropped\n since the variable was first set. */\n {% set existing_relation = load_cached_relation(existing_relation) %}\n {% if existing_relation is not none %}\n {{ adapter.rename_relation(existing_relation, backup_relation) }}\n {% endif %}\n {% endif %}\n\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% do create_indexes(target_relation) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n -- finally, drop the existing/backup relation after the commit\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.get_create_table_as_sql", "macro.dbt.create_indexes", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.610741, "supported_languages": ["sql"]}, "macro.dbt.get_quoted_csv": {"name": "get_quoted_csv", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.get_quoted_csv", "macro_sql": "{% macro get_quoted_csv(column_names) %}\n\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote(col)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.613347, "supported_languages": null}, "macro.dbt.diff_columns": {"name": "diff_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.diff_columns", "macro_sql": "{% macro diff_columns(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% set source_names = source_columns | map(attribute = 'column') | list %}\n {% set target_names = target_columns | map(attribute = 'column') | list %}\n\n {# --check whether the name attribute exists in the target - this does not perform a data type check #}\n {% for sc in source_columns %}\n {% if sc.name not in target_names %}\n {{ result.append(sc) }}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.6142552, "supported_languages": null}, "macro.dbt.diff_column_data_types": {"name": "diff_column_data_types", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.diff_column_data_types", "macro_sql": "{% macro diff_column_data_types(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% for sc in source_columns %}\n {% set tc = target_columns | selectattr(\"name\", \"equalto\", sc.name) | list | first %}\n {% if tc %}\n {% if sc.data_type != tc.data_type and not sc.can_expand_to(other_column=tc) %}\n {{ result.append( { 'column_name': tc.name, 'new_type': sc.data_type } ) }}\n {% endif %}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.615365, "supported_languages": null}, "macro.dbt.get_merge_update_columns": {"name": "get_merge_update_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.get_merge_update_columns", "macro_sql": "{% macro get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {{ return(adapter.dispatch('get_merge_update_columns', 'dbt')(merge_update_columns, merge_exclude_columns, dest_columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_merge_update_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.615764, "supported_languages": null}, "macro.dbt.default__get_merge_update_columns": {"name": "default__get_merge_update_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.default__get_merge_update_columns", "macro_sql": "{% macro default__get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {%- set default_cols = dest_columns | map(attribute=\"quoted\") | list -%}\n\n {%- if merge_update_columns and merge_exclude_columns -%}\n {{ exceptions.raise_compiler_error(\n 'Model cannot specify merge_update_columns and merge_exclude_columns. Please update model to use only one config'\n )}}\n {%- elif merge_update_columns -%}\n {%- set update_columns = merge_update_columns -%}\n {%- elif merge_exclude_columns -%}\n {%- set update_columns = [] -%}\n {%- for column in dest_columns -%}\n {% if column.column | lower not in merge_exclude_columns | map(\"lower\") | list %}\n {%- do update_columns.append(column.quoted) -%}\n {% endif %}\n {%- endfor -%}\n {%- else -%}\n {%- set update_columns = default_cols -%}\n {%- endif -%}\n\n {{ return(update_columns) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.6169448, "supported_languages": null}, "macro.dbt.get_merge_sql": {"name": "get_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.get_merge_sql", "macro_sql": "{% macro get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) -%}\n -- back compat for old kwarg name\n {% set incremental_predicates = kwargs.get('predicates', incremental_predicates) %}\n {{ adapter.dispatch('get_merge_sql', 'dbt')(target, source, unique_key, dest_columns, incremental_predicates) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.6264052, "supported_languages": null}, "macro.dbt.default__get_merge_sql": {"name": "default__get_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.default__get_merge_sql", "macro_sql": "{% macro default__get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) -%}\n {%- set predicates = [] if incremental_predicates is none else [] + incremental_predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set merge_update_columns = config.get('merge_update_columns') -%}\n {%- set merge_exclude_columns = config.get('merge_exclude_columns') -%}\n {%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}\n {% for key in unique_key %}\n {% set this_key_match %}\n DBT_INTERNAL_SOURCE.{{ key }} = DBT_INTERNAL_DEST.{{ key }}\n {% endset %}\n {% do predicates.append(this_key_match) %}\n {% endfor %}\n {% else %}\n {% set unique_key_match %}\n DBT_INTERNAL_SOURCE.{{ unique_key }} = DBT_INTERNAL_DEST.{{ unique_key }}\n {% endset %}\n {% do predicates.append(unique_key_match) %}\n {% endif %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n {{ sql_header if sql_header is not none }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{\"(\" ~ predicates | join(\") and (\") ~ \")\"}}\n\n {% if unique_key %}\n when matched then update set\n {% for column_name in update_columns -%}\n {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}\n {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n {% endif %}\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv", "macro.dbt.get_merge_update_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.6290119, "supported_languages": null}, "macro.dbt.get_delete_insert_merge_sql": {"name": "get_delete_insert_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.get_delete_insert_merge_sql", "macro_sql": "{% macro get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}\n {{ adapter.dispatch('get_delete_insert_merge_sql', 'dbt')(target, source, unique_key, dest_columns, incremental_predicates) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_delete_insert_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.6294458, "supported_languages": null}, "macro.dbt.default__get_delete_insert_merge_sql": {"name": "default__get_delete_insert_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.default__get_delete_insert_merge_sql", "macro_sql": "{% macro default__get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not string %}\n delete from {{target }}\n using {{ source }}\n where (\n {% for key in unique_key %}\n {{ source }}.{{ key }} = {{ target }}.{{ key }}\n {{ \"and \" if not loop.last}}\n {% endfor %}\n {% if incremental_predicates %}\n {% for predicate in incremental_predicates %}\n and {{ predicate }}\n {% endfor %}\n {% endif %}\n );\n {% else %}\n delete from {{ target }}\n where (\n {{ unique_key }}) in (\n select ({{ unique_key }})\n from {{ source }}\n )\n {%- if incremental_predicates %}\n {% for predicate in incremental_predicates %}\n and {{ predicate }}\n {% endfor %}\n {%- endif -%};\n\n {% endif %}\n {% endif %}\n\n insert into {{ target }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ source }}\n )\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.63107, "supported_languages": null}, "macro.dbt.get_insert_overwrite_merge_sql": {"name": "get_insert_overwrite_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.get_insert_overwrite_merge_sql", "macro_sql": "{% macro get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header=false) -%}\n {{ adapter.dispatch('get_insert_overwrite_merge_sql', 'dbt')(target, source, dest_columns, predicates, include_sql_header) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_insert_overwrite_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.631517, "supported_languages": null}, "macro.dbt.default__get_insert_overwrite_merge_sql": {"name": "default__get_insert_overwrite_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.default__get_insert_overwrite_merge_sql", "macro_sql": "{% macro default__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) -%}\n {#-- The only time include_sql_header is True: --#}\n {#-- BigQuery + insert_overwrite strategy + \"static\" partitions config --#}\n {#-- We should consider including the sql header at the materialization level instead --#}\n\n {%- set predicates = [] if predicates is none else [] + predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none and include_sql_header }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on FALSE\n\n when not matched by source\n {% if predicates %} and {{ predicates | join(' and ') }} {% endif %}\n then delete\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.632575, "supported_languages": null}, "macro.dbt.is_incremental": {"name": "is_incremental", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/is_incremental.sql", "original_file_path": "macros/materializations/models/incremental/is_incremental.sql", "unique_id": "macro.dbt.is_incremental", "macro_sql": "{% macro is_incremental() %}\n {#-- do not run introspective queries in parsing #}\n {% if not execute %}\n {{ return(False) }}\n {% else %}\n {% set relation = adapter.get_relation(this.database, this.schema, this.table) %}\n {{ return(relation is not none\n and relation.type == 'table'\n and model.config.materialized == 'incremental'\n and not should_full_refresh()) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.633599, "supported_languages": null}, "macro.dbt.get_incremental_append_sql": {"name": "get_incremental_append_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_append_sql", "macro_sql": "{% macro get_incremental_append_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_append_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_append_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.635051, "supported_languages": null}, "macro.dbt.default__get_incremental_append_sql": {"name": "default__get_incremental_append_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_append_sql", "macro_sql": "{% macro default__get_incremental_append_sql(arg_dict) %}\n\n {% do return(get_insert_into_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"dest_columns\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_insert_into_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.635458, "supported_languages": null}, "macro.dbt.get_incremental_delete_insert_sql": {"name": "get_incremental_delete_insert_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_delete_insert_sql", "macro_sql": "{% macro get_incremental_delete_insert_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_delete_insert_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_delete_insert_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.63577, "supported_languages": null}, "macro.dbt.default__get_incremental_delete_insert_sql": {"name": "default__get_incremental_delete_insert_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_delete_insert_sql", "macro_sql": "{% macro default__get_incremental_delete_insert_sql(arg_dict) %}\n\n {% do return(get_delete_insert_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"unique_key\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_delete_insert_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.6362581, "supported_languages": null}, "macro.dbt.get_incremental_merge_sql": {"name": "get_incremental_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_merge_sql", "macro_sql": "{% macro get_incremental_merge_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_merge_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.636678, "supported_languages": null}, "macro.dbt.default__get_incremental_merge_sql": {"name": "default__get_incremental_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_merge_sql", "macro_sql": "{% macro default__get_incremental_merge_sql(arg_dict) %}\n\n {% do return(get_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"unique_key\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.637161, "supported_languages": null}, "macro.dbt.get_incremental_insert_overwrite_sql": {"name": "get_incremental_insert_overwrite_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_insert_overwrite_sql", "macro_sql": "{% macro get_incremental_insert_overwrite_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_insert_overwrite_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_insert_overwrite_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.637471, "supported_languages": null}, "macro.dbt.default__get_incremental_insert_overwrite_sql": {"name": "default__get_incremental_insert_overwrite_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_insert_overwrite_sql", "macro_sql": "{% macro default__get_incremental_insert_overwrite_sql(arg_dict) %}\n\n {% do return(get_insert_overwrite_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_insert_overwrite_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.6379, "supported_languages": null}, "macro.dbt.get_incremental_default_sql": {"name": "get_incremental_default_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_default_sql", "macro_sql": "{% macro get_incremental_default_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_default_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_incremental_default_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.6382148, "supported_languages": null}, "macro.dbt.default__get_incremental_default_sql": {"name": "default__get_incremental_default_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_default_sql", "macro_sql": "{% macro default__get_incremental_default_sql(arg_dict) %}\n\n {% do return(get_incremental_append_sql(arg_dict)) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_incremental_append_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.63846, "supported_languages": null}, "macro.dbt.get_insert_into_sql": {"name": "get_insert_into_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_insert_into_sql", "macro_sql": "{% macro get_insert_into_sql(target_relation, temp_relation, dest_columns) %}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n insert into {{ target_relation }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ temp_relation }}\n )\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.6389172, "supported_languages": null}, "macro.dbt.materialization_incremental_default": {"name": "materialization_incremental_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/incremental.sql", "original_file_path": "macros/materializations/models/incremental/incremental.sql", "unique_id": "macro.dbt.materialization_incremental_default", "macro_sql": "{% materialization incremental, default -%}\n\n -- relations\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') -%}\n {%- set temp_relation = make_temp_relation(target_relation)-%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation)-%}\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n\n -- configs\n {%- set unique_key = config.get('unique_key') -%}\n {%- set full_refresh_mode = (should_full_refresh() or existing_relation.is_view) -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n\n -- the temp_ and backup_ relations should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation. This has to happen before\n -- BEGIN, in a separate transaction\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation)-%}\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set to_drop = [] %}\n\n {% if existing_relation is none %}\n {% set build_sql = get_create_table_as_sql(False, target_relation, sql) %}\n {% elif full_refresh_mode %}\n {% set build_sql = get_create_table_as_sql(False, intermediate_relation, sql) %}\n {% set need_swap = true %}\n {% else %}\n {% do run_query(get_create_table_as_sql(True, temp_relation, sql)) %}\n {% do adapter.expand_target_column_types(\n from_relation=temp_relation,\n to_relation=target_relation) %}\n {#-- Process schema changes. Returns dict of changes if successful. Use source columns for upserting/merging --#}\n {% set dest_columns = process_schema_changes(on_schema_change, temp_relation, existing_relation) %}\n {% if not dest_columns %}\n {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %}\n {% endif %}\n\n {#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#}\n {% set incremental_strategy = config.get('incremental_strategy') or 'default' %}\n {% set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) %}\n {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %}\n {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': temp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'incremental_predicates': incremental_predicates }) %}\n {% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %}\n\n {% endif %}\n\n {% call statement(\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% if need_swap %}\n {% do adapter.rename_relation(target_relation, backup_relation) %}\n {% do adapter.rename_relation(intermediate_relation, target_relation) %}\n {% do to_drop.append(backup_relation) %}\n {% endif %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if existing_relation is none or existing_relation.is_view or should_full_refresh() %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {% do adapter.commit() %}\n\n {% for rel in to_drop %}\n {% do adapter.drop_relation(rel) %}\n {% endfor %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_temp_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.should_full_refresh", "macro.dbt.incremental_validate_on_schema_change", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks", "macro.dbt.get_create_table_as_sql", "macro.dbt.run_query", "macro.dbt.process_schema_changes", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.647358, "supported_languages": ["sql"]}, "macro.dbt.incremental_validate_on_schema_change": {"name": "incremental_validate_on_schema_change", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.incremental_validate_on_schema_change", "macro_sql": "{% macro incremental_validate_on_schema_change(on_schema_change, default='ignore') %}\n\n {% if on_schema_change not in ['sync_all_columns', 'append_new_columns', 'fail', 'ignore'] %}\n\n {% set log_message = 'Invalid value for on_schema_change (%s) specified. Setting default value of %s.' % (on_schema_change, default) %}\n {% do log(log_message) %}\n\n {{ return(default) }}\n\n {% else %}\n\n {{ return(on_schema_change) }}\n\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.655884, "supported_languages": null}, "macro.dbt.check_for_schema_changes": {"name": "check_for_schema_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.check_for_schema_changes", "macro_sql": "{% macro check_for_schema_changes(source_relation, target_relation) %}\n\n {% set schema_changed = False %}\n\n {%- set source_columns = adapter.get_columns_in_relation(source_relation) -%}\n {%- set target_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set source_not_in_target = diff_columns(source_columns, target_columns) -%}\n {%- set target_not_in_source = diff_columns(target_columns, source_columns) -%}\n\n {% set new_target_types = diff_column_data_types(source_columns, target_columns) %}\n\n {% if source_not_in_target != [] %}\n {% set schema_changed = True %}\n {% elif target_not_in_source != [] or new_target_types != [] %}\n {% set schema_changed = True %}\n {% elif new_target_types != [] %}\n {% set schema_changed = True %}\n {% endif %}\n\n {% set changes_dict = {\n 'schema_changed': schema_changed,\n 'source_not_in_target': source_not_in_target,\n 'target_not_in_source': target_not_in_source,\n 'source_columns': source_columns,\n 'target_columns': target_columns,\n 'new_target_types': new_target_types\n } %}\n\n {% set msg %}\n In {{ target_relation }}:\n Schema changed: {{ schema_changed }}\n Source columns not in target: {{ source_not_in_target }}\n Target columns not in source: {{ target_not_in_source }}\n New column types: {{ new_target_types }}\n {% endset %}\n\n {% do log(msg) %}\n\n {{ return(changes_dict) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.diff_columns", "macro.dbt.diff_column_data_types"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.657853, "supported_languages": null}, "macro.dbt.sync_column_schemas": {"name": "sync_column_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.sync_column_schemas", "macro_sql": "{% macro sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {%- set add_to_target_arr = schema_changes_dict['source_not_in_target'] -%}\n\n {%- if on_schema_change == 'append_new_columns'-%}\n {%- if add_to_target_arr | length > 0 -%}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, none) -%}\n {%- endif -%}\n\n {% elif on_schema_change == 'sync_all_columns' %}\n {%- set remove_from_target_arr = schema_changes_dict['target_not_in_source'] -%}\n {%- set new_target_types = schema_changes_dict['new_target_types'] -%}\n\n {% if add_to_target_arr | length > 0 or remove_from_target_arr | length > 0 %}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, remove_from_target_arr) -%}\n {% endif %}\n\n {% if new_target_types != [] %}\n {% for ntt in new_target_types %}\n {% set column_name = ntt['column_name'] %}\n {% set new_type = ntt['new_type'] %}\n {% do alter_column_type(target_relation, column_name, new_type) %}\n {% endfor %}\n {% endif %}\n\n {% endif %}\n\n {% set schema_change_message %}\n In {{ target_relation }}:\n Schema change approach: {{ on_schema_change }}\n Columns added: {{ add_to_target_arr }}\n Columns removed: {{ remove_from_target_arr }}\n Data types changed: {{ new_target_types }}\n {% endset %}\n\n {% do log(schema_change_message) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.alter_relation_add_remove_columns", "macro.dbt.alter_column_type"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.659792, "supported_languages": null}, "macro.dbt.process_schema_changes": {"name": "process_schema_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.process_schema_changes", "macro_sql": "{% macro process_schema_changes(on_schema_change, source_relation, target_relation) %}\n\n {% if on_schema_change == 'ignore' %}\n\n {{ return({}) }}\n\n {% else %}\n\n {% set schema_changes_dict = check_for_schema_changes(source_relation, target_relation) %}\n\n {% if schema_changes_dict['schema_changed'] %}\n\n {% if on_schema_change == 'fail' %}\n\n {% set fail_msg %}\n The source and target schemas on this incremental model are out of sync!\n They can be reconciled in several ways:\n - set the `on_schema_change` config to either append_new_columns or sync_all_columns, depending on your situation.\n - Re-run the incremental model with `full_refresh: True` to update the target schema.\n - update the schema manually and re-run the process.\n\n Additional troubleshooting context:\n Source columns not in target: {{ schema_changes_dict['source_not_in_target'] }}\n Target columns not in source: {{ schema_changes_dict['target_not_in_source'] }}\n New column types: {{ schema_changes_dict['new_target_types'] }}\n {% endset %}\n\n {% do exceptions.raise_compiler_error(fail_msg) %}\n\n {# -- unless we ignore, run the sync operation per the config #}\n {% else %}\n\n {% do sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {% endif %}\n\n {% endif %}\n\n {{ return(schema_changes_dict['source_columns']) }}\n\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.check_for_schema_changes", "macro.dbt.sync_column_schemas"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.6611788, "supported_languages": null}, "macro.dbt.can_clone_table": {"name": "can_clone_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/can_clone_table.sql", "original_file_path": "macros/materializations/models/clone/can_clone_table.sql", "unique_id": "macro.dbt.can_clone_table", "macro_sql": "{% macro can_clone_table() %}\n {{ return(adapter.dispatch('can_clone_table', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__can_clone_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.661578, "supported_languages": null}, "macro.dbt.default__can_clone_table": {"name": "default__can_clone_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/can_clone_table.sql", "original_file_path": "macros/materializations/models/clone/can_clone_table.sql", "unique_id": "macro.dbt.default__can_clone_table", "macro_sql": "{% macro default__can_clone_table() %}\n {{ return(False) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.6617699, "supported_languages": null}, "macro.dbt.create_or_replace_clone": {"name": "create_or_replace_clone", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/create_or_replace_clone.sql", "original_file_path": "macros/materializations/models/clone/create_or_replace_clone.sql", "unique_id": "macro.dbt.create_or_replace_clone", "macro_sql": "{% macro create_or_replace_clone(this_relation, defer_relation) %}\n {{ return(adapter.dispatch('create_or_replace_clone', 'dbt')(this_relation, defer_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__create_or_replace_clone"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.662251, "supported_languages": null}, "macro.dbt.default__create_or_replace_clone": {"name": "default__create_or_replace_clone", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/create_or_replace_clone.sql", "original_file_path": "macros/materializations/models/clone/create_or_replace_clone.sql", "unique_id": "macro.dbt.default__create_or_replace_clone", "macro_sql": "{% macro default__create_or_replace_clone(this_relation, defer_relation) %}\n create or replace table {{ this_relation }} clone {{ defer_relation }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.662475, "supported_languages": null}, "macro.dbt.materialization_clone_default": {"name": "materialization_clone_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/clone.sql", "original_file_path": "macros/materializations/models/clone/clone.sql", "unique_id": "macro.dbt.materialization_clone_default", "macro_sql": "{%- materialization clone, default -%}\n\n {%- set relations = {'relations': []} -%}\n\n {%- if not defer_relation -%}\n -- nothing to do\n {{ log(\"No relation found in state manifest for \" ~ model.unique_id, info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n\n {%- if existing_relation and not flags.FULL_REFRESH -%}\n -- noop!\n {{ log(\"Relation \" ~ existing_relation ~ \" already exists\", info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set other_existing_relation = load_cached_relation(defer_relation) -%}\n\n -- If this is a database that can do zero-copy cloning of tables, and the other relation is a table, then this will be a table\n -- Otherwise, this will be a view\n\n {% set can_clone_table = can_clone_table() %}\n\n {%- if other_existing_relation and other_existing_relation.type == 'table' and can_clone_table -%}\n\n {%- set target_relation = this.incorporate(type='table') -%}\n {% if existing_relation is not none and not existing_relation.is_table %}\n {{ log(\"Dropping relation \" ~ existing_relation ~ \" because it is of type \" ~ existing_relation.type) }}\n {{ drop_relation_if_exists(existing_relation) }}\n {% endif %}\n\n -- as a general rule, data platforms that can clone tables can also do atomic 'create or replace'\n {% call statement('main') %}\n {% if target_relation and defer_relation and target_relation == defer_relation %}\n {{ log(\"Target relation and defer relation are the same, skipping clone for relation: \" ~ target_relation) }}\n {% else %}\n {{ create_or_replace_clone(target_relation, defer_relation) }}\n {% endif %}\n\n {% endcall %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n {% do persist_docs(target_relation, model) %}\n\n {{ return({'relations': [target_relation]}) }}\n\n {%- else -%}\n\n {%- set target_relation = this.incorporate(type='view') -%}\n\n -- reuse the view materialization\n -- TODO: support actual dispatch for materialization macros\n -- Tracking ticket: https://github.com/dbt-labs/dbt-core/issues/7799\n {% set search_name = \"materialization_view_\" ~ adapter.type() %}\n {% if not search_name in context %}\n {% set search_name = \"materialization_view_default\" %}\n {% endif %}\n {% set materialization_macro = context[search_name] %}\n {% set relations = materialization_macro() %}\n {{ return(relations) }}\n\n {%- endif -%}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.can_clone_table", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.create_or_replace_clone", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.668271, "supported_languages": ["sql"]}, "macro.dbt.materialization_seed_default": {"name": "materialization_seed_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/seed.sql", "original_file_path": "macros/materializations/seeds/seed.sql", "unique_id": "macro.dbt.materialization_seed_default", "macro_sql": "{% materialization seed, default %}\n\n {%- set identifier = model['alias'] -%}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set grant_config = config.get('grants') -%}\n {%- set agate_table = load_agate_table() -%}\n -- grab current tables grants config for comparison later on\n\n {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% set create_table_sql = \"\" %}\n {% if exists_as_view %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a view\".format(old_relation)) }}\n {% elif exists_as_table %}\n {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}\n {% else %}\n {% set create_table_sql = create_csv_table(model, agate_table) %}\n {% endif %}\n\n {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}\n {% set rows_affected = (agate_table.rows | length) %}\n {% set sql = load_csv_rows(model, agate_table) %}\n\n {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}\n {{ get_csv_sql(create_table_sql, sql) }};\n {% endcall %}\n\n {% set target_relation = this.incorporate(type='table') %}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if full_refresh_mode or not exists_as_table %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.run_hooks", "macro.dbt.reset_csv_table", "macro.dbt.create_csv_table", "macro.dbt.load_csv_rows", "macro.dbt.noop_statement", "macro.dbt.get_csv_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.673449, "supported_languages": ["sql"]}, "macro.dbt.create_csv_table": {"name": "create_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.create_csv_table", "macro_sql": "{% macro create_csv_table(model, agate_table) -%}\n {{ adapter.dispatch('create_csv_table', 'dbt')(model, agate_table) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.681057, "supported_languages": null}, "macro.dbt.default__create_csv_table": {"name": "default__create_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__create_csv_table", "macro_sql": "{% macro default__create_csv_table(model, agate_table) %}\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n\n {% set sql %}\n create table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.682514, "supported_languages": null}, "macro.dbt.reset_csv_table": {"name": "reset_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.reset_csv_table", "macro_sql": "{% macro reset_csv_table(model, full_refresh, old_relation, agate_table) -%}\n {{ adapter.dispatch('reset_csv_table', 'dbt')(model, full_refresh, old_relation, agate_table) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__reset_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.6828969, "supported_languages": null}, "macro.dbt.default__reset_csv_table": {"name": "default__reset_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__reset_csv_table", "macro_sql": "{% macro default__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% set sql = \"\" %}\n {% if full_refresh %}\n {{ adapter.drop_relation(old_relation) }}\n {% set sql = create_csv_table(model, agate_table) %}\n {% else %}\n {{ adapter.truncate_relation(old_relation) }}\n {% set sql = \"truncate table \" ~ old_relation %}\n {% endif %}\n\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.6836748, "supported_languages": null}, "macro.dbt.get_csv_sql": {"name": "get_csv_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_csv_sql", "macro_sql": "{% macro get_csv_sql(create_or_truncate_sql, insert_sql) %}\n {{ adapter.dispatch('get_csv_sql', 'dbt')(create_or_truncate_sql, insert_sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_csv_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.6839921, "supported_languages": null}, "macro.dbt.default__get_csv_sql": {"name": "default__get_csv_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__get_csv_sql", "macro_sql": "{% macro default__get_csv_sql(create_or_truncate_sql, insert_sql) %}\n {{ create_or_truncate_sql }};\n -- dbt seed --\n {{ insert_sql }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.684214, "supported_languages": null}, "macro.dbt.get_binding_char": {"name": "get_binding_char", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_binding_char", "macro_sql": "{% macro get_binding_char() -%}\n {{ adapter.dispatch('get_binding_char', 'dbt')() }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.684445, "supported_languages": null}, "macro.dbt.default__get_binding_char": {"name": "default__get_binding_char", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__get_binding_char", "macro_sql": "{% macro default__get_binding_char() %}\n {{ return('%s') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.684631, "supported_languages": null}, "macro.dbt.get_batch_size": {"name": "get_batch_size", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_batch_size", "macro_sql": "{% macro get_batch_size() -%}\n {{ return(adapter.dispatch('get_batch_size', 'dbt')()) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_batch_size"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.684887, "supported_languages": null}, "macro.dbt.default__get_batch_size": {"name": "default__get_batch_size", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__get_batch_size", "macro_sql": "{% macro default__get_batch_size() %}\n {{ return(10000) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.68508, "supported_languages": null}, "macro.dbt.get_seed_column_quoted_csv": {"name": "get_seed_column_quoted_csv", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_seed_column_quoted_csv", "macro_sql": "{% macro get_seed_column_quoted_csv(model, column_names) %}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote_seed_column(col, quote_seed_column)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.6858501, "supported_languages": null}, "macro.dbt.load_csv_rows": {"name": "load_csv_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.load_csv_rows", "macro_sql": "{% macro load_csv_rows(model, agate_table) -%}\n {{ adapter.dispatch('load_csv_rows', 'dbt')(model, agate_table) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__load_csv_rows"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.686155, "supported_languages": null}, "macro.dbt.default__load_csv_rows": {"name": "default__load_csv_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__load_csv_rows", "macro_sql": "{% macro default__load_csv_rows(model, agate_table) %}\n\n {% set batch_size = get_batch_size() %}\n\n {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}\n {% set bindings = [] %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert into {{ this.render() }} ({{ cols_sql }}) values\n {% for row in chunk -%}\n ({%- for column in agate_table.column_names -%}\n {{ get_binding_char() }}\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_batch_size", "macro.dbt.get_seed_column_quoted_csv", "macro.dbt.get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.688233, "supported_languages": null}, "macro.dbt.generate_alias_name": {"name": "generate_alias_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_alias.sql", "original_file_path": "macros/get_custom_name/get_custom_alias.sql", "unique_id": "macro.dbt.generate_alias_name", "macro_sql": "{% macro generate_alias_name(custom_alias_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_alias_name', 'dbt')(custom_alias_name, node)) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__generate_alias_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.68893, "supported_languages": null}, "macro.dbt.default__generate_alias_name": {"name": "default__generate_alias_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_alias.sql", "original_file_path": "macros/get_custom_name/get_custom_alias.sql", "unique_id": "macro.dbt.default__generate_alias_name", "macro_sql": "{% macro default__generate_alias_name(custom_alias_name=none, node=none) -%}\n\n {%- if custom_alias_name -%}\n\n {{ custom_alias_name | trim }}\n\n {%- elif node.version -%}\n\n {{ return(node.name ~ \"_v\" ~ (node.version | replace(\".\", \"_\"))) }}\n\n {%- else -%}\n\n {{ node.name }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.6895602, "supported_languages": null}, "macro.dbt.generate_schema_name": {"name": "generate_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "unique_id": "macro.dbt.generate_schema_name", "macro_sql": "{% macro generate_schema_name(custom_schema_name=none, node=none) -%}\n {{ return(adapter.dispatch('generate_schema_name', 'dbt')(custom_schema_name, node)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__generate_schema_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.69043, "supported_languages": null}, "macro.dbt.default__generate_schema_name": {"name": "default__generate_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "unique_id": "macro.dbt.default__generate_schema_name", "macro_sql": "{% macro default__generate_schema_name(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if custom_schema_name is none -%}\n\n {{ default_schema }}\n\n {%- else -%}\n\n {{ default_schema }}_{{ custom_schema_name | trim }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.690857, "supported_languages": null}, "macro.dbt.generate_schema_name_for_env": {"name": "generate_schema_name_for_env", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "unique_id": "macro.dbt.generate_schema_name_for_env", "macro_sql": "{% macro generate_schema_name_for_env(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if target.name == 'prod' and custom_schema_name is not none -%}\n\n {{ custom_schema_name | trim }}\n\n {%- else -%}\n\n {{ default_schema }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.6913202, "supported_languages": null}, "macro.dbt.generate_database_name": {"name": "generate_database_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "unique_id": "macro.dbt.generate_database_name", "macro_sql": "{% macro generate_database_name(custom_database_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_database_name', 'dbt')(custom_database_name, node)) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__generate_database_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.691966, "supported_languages": null}, "macro.dbt.default__generate_database_name": {"name": "default__generate_database_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "unique_id": "macro.dbt.default__generate_database_name", "macro_sql": "{% macro default__generate_database_name(custom_database_name=none, node=none) -%}\n {%- set default_database = target.database -%}\n {%- if custom_database_name is none -%}\n\n {{ default_database }}\n\n {%- else -%}\n\n {{ custom_database_name }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.692378, "supported_languages": null}, "macro.dbt.get_drop_sql": {"name": "get_drop_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.get_drop_sql", "macro_sql": "{%- macro get_drop_sql(relation) -%}\n {{- log('Applying DROP to: ' ~ relation) -}}\n {{- adapter.dispatch('get_drop_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_drop_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.693515, "supported_languages": null}, "macro.dbt.default__get_drop_sql": {"name": "default__get_drop_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.default__get_drop_sql", "macro_sql": "{%- macro default__get_drop_sql(relation) -%}\n\n {%- if relation.is_view -%}\n {{ drop_view(relation) }}\n\n {%- elif relation.is_table -%}\n {{ drop_table(relation) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ drop_materialized_view(relation) }}\n\n {%- else -%}\n drop {{ relation.type }} if exists {{ relation }} cascade\n\n {%- endif -%}\n\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.drop_view", "macro.dbt.drop_table", "macro.dbt.drop_materialized_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.694123, "supported_languages": null}, "macro.dbt.drop_relation": {"name": "drop_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.drop_relation", "macro_sql": "{% macro drop_relation(relation) -%}\n {{ return(adapter.dispatch('drop_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__drop_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.694428, "supported_languages": null}, "macro.dbt.default__drop_relation": {"name": "default__drop_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.default__drop_relation", "macro_sql": "{% macro default__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n {{ get_drop_sql(relation) }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.get_drop_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.694753, "supported_languages": null}, "macro.dbt.drop_relation_if_exists": {"name": "drop_relation_if_exists", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.drop_relation_if_exists", "macro_sql": "{% macro drop_relation_if_exists(relation) %}\n {% if relation is not none %}\n {{ adapter.drop_relation(relation) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.695075, "supported_languages": null}, "macro.dbt.get_replace_sql": {"name": "get_replace_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt.get_replace_sql", "macro_sql": "{% macro get_replace_sql(existing_relation, target_relation, sql) %}\n {{- log('Applying REPLACE to: ' ~ existing_relation) -}}\n {{- adapter.dispatch('get_replace_sql', 'dbt')(existing_relation, target_relation, sql) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_replace_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.6964202, "supported_languages": null}, "macro.dbt.default__get_replace_sql": {"name": "default__get_replace_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt.default__get_replace_sql", "macro_sql": "{% macro default__get_replace_sql(existing_relation, target_relation, sql) %}\n\n {# /* use a create or replace statement if possible */ #}\n\n {% set is_replaceable = existing_relation.type == target_relation_type and existing_relation.can_be_replaced %}\n\n {% if is_replaceable and existing_relation.is_view %}\n {{ get_replace_view_sql(target_relation, sql) }}\n\n {% elif is_replaceable and existing_relation.is_table %}\n {{ get_replace_table_sql(target_relation, sql) }}\n\n {% elif is_replaceable and existing_relation.is_materialized_view %}\n {{ get_replace_materialized_view_sql(target_relation, sql) }}\n\n {# /* a create or replace statement is not possible, so try to stage and/or backup to be safe */ #}\n\n {# /* create target_relation as an intermediate relation, then swap it out with the existing one using a backup */ #}\n {%- elif target_relation.can_be_renamed and existing_relation.can_be_renamed -%}\n {{ get_create_intermediate_sql(target_relation, sql) }};\n {{ get_create_backup_sql(existing_relation) }};\n {{ get_rename_intermediate_sql(target_relation) }};\n {{ get_drop_backup_sql(existing_relation) }}\n\n {# /* create target_relation as an intermediate relation, then swap it out with the existing one without using a backup */ #}\n {%- elif target_relation.can_be_renamed -%}\n {{ get_create_intermediate_sql(target_relation, sql) }};\n {{ get_drop_sql(existing_relation) }};\n {{ get_rename_intermediate_sql(target_relation) }}\n\n {# /* create target_relation in place by first backing up the existing relation */ #}\n {%- elif existing_relation.can_be_renamed -%}\n {{ get_create_backup_sql(existing_relation) }};\n {{ get_create_sql(target_relation, sql) }};\n {{ get_drop_backup_sql(existing_relation) }}\n\n {# /* no renaming is allowed, so just drop and create */ #}\n {%- else -%}\n {{ get_drop_sql(existing_relation) }};\n {{ get_create_sql(target_relation, sql) }}\n\n {%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_replace_view_sql", "macro.dbt.get_replace_table_sql", "macro.dbt.get_replace_materialized_view_sql", "macro.dbt.get_create_intermediate_sql", "macro.dbt.get_create_backup_sql", "macro.dbt.get_rename_intermediate_sql", "macro.dbt.get_drop_backup_sql", "macro.dbt.get_drop_sql", "macro.dbt.get_create_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.698515, "supported_languages": null}, "macro.dbt.get_create_intermediate_sql": {"name": "get_create_intermediate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create_intermediate.sql", "original_file_path": "macros/relations/create_intermediate.sql", "unique_id": "macro.dbt.get_create_intermediate_sql", "macro_sql": "{%- macro get_create_intermediate_sql(relation, sql) -%}\n {{- log('Applying CREATE INTERMEDIATE to: ' ~ relation) -}}\n {{- adapter.dispatch('get_create_intermediate_sql', 'dbt')(relation, sql) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_create_intermediate_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.699143, "supported_languages": null}, "macro.dbt.default__get_create_intermediate_sql": {"name": "default__get_create_intermediate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create_intermediate.sql", "original_file_path": "macros/relations/create_intermediate.sql", "unique_id": "macro.dbt.default__get_create_intermediate_sql", "macro_sql": "{%- macro default__get_create_intermediate_sql(relation, sql) -%}\n\n -- get the standard intermediate name\n {% set intermediate_relation = make_intermediate_relation(relation) %}\n\n -- drop any pre-existing intermediate\n {{ get_drop_sql(intermediate_relation) }};\n\n {{ get_create_sql(intermediate_relation, sql) }}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.make_intermediate_relation", "macro.dbt.get_drop_sql", "macro.dbt.get_create_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.699562, "supported_languages": null}, "macro.dbt.get_drop_backup_sql": {"name": "get_drop_backup_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop_backup.sql", "original_file_path": "macros/relations/drop_backup.sql", "unique_id": "macro.dbt.get_drop_backup_sql", "macro_sql": "{%- macro get_drop_backup_sql(relation) -%}\n {{- log('Applying DROP BACKUP to: ' ~ relation) -}}\n {{- adapter.dispatch('get_drop_backup_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_drop_backup_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.700088, "supported_languages": null}, "macro.dbt.default__get_drop_backup_sql": {"name": "default__get_drop_backup_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop_backup.sql", "original_file_path": "macros/relations/drop_backup.sql", "unique_id": "macro.dbt.default__get_drop_backup_sql", "macro_sql": "{%- macro default__get_drop_backup_sql(relation) -%}\n\n -- get the standard backup name\n {% set backup_relation = make_backup_relation(relation, relation.type) %}\n\n {{ get_drop_sql(backup_relation) }}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.make_backup_relation", "macro.dbt.get_drop_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7004328, "supported_languages": null}, "macro.dbt.get_rename_sql": {"name": "get_rename_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename.sql", "original_file_path": "macros/relations/rename.sql", "unique_id": "macro.dbt.get_rename_sql", "macro_sql": "{%- macro get_rename_sql(relation, new_name) -%}\n {{- log('Applying RENAME to: ' ~ relation) -}}\n {{- adapter.dispatch('get_rename_sql', 'dbt')(relation, new_name) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_rename_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.701566, "supported_languages": null}, "macro.dbt.default__get_rename_sql": {"name": "default__get_rename_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename.sql", "original_file_path": "macros/relations/rename.sql", "unique_id": "macro.dbt.default__get_rename_sql", "macro_sql": "{%- macro default__get_rename_sql(relation, new_name) -%}\n\n {%- if relation.is_view -%}\n {{ get_rename_view_sql(relation, new_name) }}\n\n {%- elif relation.is_table -%}\n {{ get_rename_table_sql(relation, new_name) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ get_rename_materialized_view_sql(relation, new_name) }}\n\n {%- else -%}\n {{- exceptions.raise_compiler_error(\"`get_rename_sql` has not been implemented for: \" ~ relation.type ) -}}\n\n {%- endif -%}\n\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.get_rename_view_sql", "macro.dbt.get_rename_table_sql", "macro.dbt.get_rename_materialized_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.702281, "supported_languages": null}, "macro.dbt.rename_relation": {"name": "rename_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename.sql", "original_file_path": "macros/relations/rename.sql", "unique_id": "macro.dbt.rename_relation", "macro_sql": "{% macro rename_relation(from_relation, to_relation) -%}\n {{ return(adapter.dispatch('rename_relation', 'dbt')(from_relation, to_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__rename_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.702626, "supported_languages": null}, "macro.dbt.default__rename_relation": {"name": "default__rename_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename.sql", "original_file_path": "macros/relations/rename.sql", "unique_id": "macro.dbt.default__rename_relation", "macro_sql": "{% macro default__rename_relation(from_relation, to_relation) -%}\n {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %}\n {% call statement('rename_relation') -%}\n alter table {{ from_relation }} rename to {{ target_name }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.703085, "supported_languages": null}, "macro.dbt.get_create_backup_sql": {"name": "get_create_backup_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create_backup.sql", "original_file_path": "macros/relations/create_backup.sql", "unique_id": "macro.dbt.get_create_backup_sql", "macro_sql": "{%- macro get_create_backup_sql(relation) -%}\n {{- log('Applying CREATE BACKUP to: ' ~ relation) -}}\n {{- adapter.dispatch('get_create_backup_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_create_backup_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.703655, "supported_languages": null}, "macro.dbt.default__get_create_backup_sql": {"name": "default__get_create_backup_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create_backup.sql", "original_file_path": "macros/relations/create_backup.sql", "unique_id": "macro.dbt.default__get_create_backup_sql", "macro_sql": "{%- macro default__get_create_backup_sql(relation) -%}\n\n -- get the standard backup name\n {% set backup_relation = make_backup_relation(relation, relation.type) %}\n\n -- drop any pre-existing backup\n {{ get_drop_sql(backup_relation) }};\n\n {{ get_rename_sql(relation, backup_relation.identifier) }}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.make_backup_relation", "macro.dbt.get_drop_sql", "macro.dbt.get_rename_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.704103, "supported_languages": null}, "macro.dbt.get_create_sql": {"name": "get_create_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create.sql", "original_file_path": "macros/relations/create.sql", "unique_id": "macro.dbt.get_create_sql", "macro_sql": "{%- macro get_create_sql(relation, sql) -%}\n {{- log('Applying CREATE to: ' ~ relation) -}}\n {{- adapter.dispatch('get_create_sql', 'dbt')(relation, sql) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_create_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.704864, "supported_languages": null}, "macro.dbt.default__get_create_sql": {"name": "default__get_create_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create.sql", "original_file_path": "macros/relations/create.sql", "unique_id": "macro.dbt.default__get_create_sql", "macro_sql": "{%- macro default__get_create_sql(relation, sql) -%}\n\n {%- if relation.is_view -%}\n {{ get_create_view_as_sql(relation, sql) }}\n\n {%- elif relation.is_table -%}\n {{ get_create_table_as_sql(False, relation, sql) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ get_create_materialized_view_as_sql(relation, sql) }}\n\n {%- else -%}\n {{- exceptions.raise_compiler_error(\"`get_create_sql` has not been implemented for: \" ~ relation.type ) -}}\n\n {%- endif -%}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.get_create_view_as_sql", "macro.dbt.get_create_table_as_sql", "macro.dbt.get_create_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7055922, "supported_languages": null}, "macro.dbt.get_rename_intermediate_sql": {"name": "get_rename_intermediate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename_intermediate.sql", "original_file_path": "macros/relations/rename_intermediate.sql", "unique_id": "macro.dbt.get_rename_intermediate_sql", "macro_sql": "{%- macro get_rename_intermediate_sql(relation) -%}\n {{- log('Applying RENAME INTERMEDIATE to: ' ~ relation) -}}\n {{- adapter.dispatch('get_rename_intermediate_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_rename_intermediate_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.706131, "supported_languages": null}, "macro.dbt.default__get_rename_intermediate_sql": {"name": "default__get_rename_intermediate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename_intermediate.sql", "original_file_path": "macros/relations/rename_intermediate.sql", "unique_id": "macro.dbt.default__get_rename_intermediate_sql", "macro_sql": "{%- macro default__get_rename_intermediate_sql(relation) -%}\n\n -- get the standard intermediate name\n {% set intermediate_relation = make_intermediate_relation(relation) %}\n\n {{ get_rename_sql(intermediate_relation, relation.identifier) }}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.make_intermediate_relation", "macro.dbt.get_rename_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.706486, "supported_languages": null}, "macro.dbt.drop_materialized_view": {"name": "drop_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/drop.sql", "original_file_path": "macros/relations/materialized_view/drop.sql", "unique_id": "macro.dbt.drop_materialized_view", "macro_sql": "{% macro drop_materialized_view(relation) -%}\n {{ return(adapter.dispatch('drop_materialized_view', 'dbt')(relation)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__drop_materialized_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.706933, "supported_languages": null}, "macro.dbt.default__drop_materialized_view": {"name": "default__drop_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/drop.sql", "original_file_path": "macros/relations/materialized_view/drop.sql", "unique_id": "macro.dbt.default__drop_materialized_view", "macro_sql": "{% macro default__drop_materialized_view(relation) -%}\n drop materialized view if exists {{ relation }} cascade\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.707106, "supported_languages": null}, "macro.dbt.get_replace_materialized_view_sql": {"name": "get_replace_materialized_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/replace.sql", "original_file_path": "macros/relations/materialized_view/replace.sql", "unique_id": "macro.dbt.get_replace_materialized_view_sql", "macro_sql": "{% macro get_replace_materialized_view_sql(relation, sql) %}\n {{- adapter.dispatch('get_replace_materialized_view_sql', 'dbt')(relation, sql) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_replace_materialized_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.707551, "supported_languages": null}, "macro.dbt.default__get_replace_materialized_view_sql": {"name": "default__get_replace_materialized_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/replace.sql", "original_file_path": "macros/relations/materialized_view/replace.sql", "unique_id": "macro.dbt.default__get_replace_materialized_view_sql", "macro_sql": "{% macro default__get_replace_materialized_view_sql(relation, sql) %}\n {{ exceptions.raise_compiler_error(\n \"`get_replace_materialized_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.707791, "supported_languages": null}, "macro.dbt.refresh_materialized_view": {"name": "refresh_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/refresh.sql", "original_file_path": "macros/relations/materialized_view/refresh.sql", "unique_id": "macro.dbt.refresh_materialized_view", "macro_sql": "{% macro refresh_materialized_view(relation) %}\n {{- log('Applying REFRESH to: ' ~ relation) -}}\n {{- adapter.dispatch('refresh_materialized_view', 'dbt')(relation) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__refresh_materialized_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.708298, "supported_languages": null}, "macro.dbt.default__refresh_materialized_view": {"name": "default__refresh_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/refresh.sql", "original_file_path": "macros/relations/materialized_view/refresh.sql", "unique_id": "macro.dbt.default__refresh_materialized_view", "macro_sql": "{% macro default__refresh_materialized_view(relation) %}\n {{ exceptions.raise_compiler_error(\"`refresh_materialized_view` has not been implemented for this adapter.\") }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.708629, "supported_languages": null}, "macro.dbt.get_rename_materialized_view_sql": {"name": "get_rename_materialized_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/rename.sql", "original_file_path": "macros/relations/materialized_view/rename.sql", "unique_id": "macro.dbt.get_rename_materialized_view_sql", "macro_sql": "{% macro get_rename_materialized_view_sql(relation, new_name) %}\n {{- adapter.dispatch('get_rename_materialized_view_sql', 'dbt')(relation, new_name) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_rename_materialized_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.709074, "supported_languages": null}, "macro.dbt.default__get_rename_materialized_view_sql": {"name": "default__get_rename_materialized_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/rename.sql", "original_file_path": "macros/relations/materialized_view/rename.sql", "unique_id": "macro.dbt.default__get_rename_materialized_view_sql", "macro_sql": "{% macro default__get_rename_materialized_view_sql(relation, new_name) %}\n {{ exceptions.raise_compiler_error(\n \"`get_rename_materialized_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7093172, "supported_languages": null}, "macro.dbt.get_alter_materialized_view_as_sql": {"name": "get_alter_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt.get_alter_materialized_view_as_sql", "macro_sql": "{% macro get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n {{- log('Applying ALTER to: ' ~ relation) -}}\n {{- adapter.dispatch('get_alter_materialized_view_as_sql', 'dbt')(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n ) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_alter_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.710311, "supported_languages": null}, "macro.dbt.default__get_alter_materialized_view_as_sql": {"name": "default__get_alter_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt.default__get_alter_materialized_view_as_sql", "macro_sql": "{% macro default__get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n {{ exceptions.raise_compiler_error(\"Materialized views have not been implemented for this adapter.\") }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7106209, "supported_languages": null}, "macro.dbt.get_materialized_view_configuration_changes": {"name": "get_materialized_view_configuration_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt.get_materialized_view_configuration_changes", "macro_sql": "{% macro get_materialized_view_configuration_changes(existing_relation, new_config) %}\n /* {#\n It's recommended that configuration changes be formatted as follows:\n {\"\": [{\"action\": \"\", \"context\": ...}]}\n\n For example:\n {\n \"indexes\": [\n {\"action\": \"drop\", \"context\": \"index_abc\"},\n {\"action\": \"create\", \"context\": {\"columns\": [\"column_1\", \"column_2\"], \"type\": \"hash\", \"unique\": True}},\n ],\n }\n\n Either way, `get_materialized_view_configuration_changes` needs to align with `get_alter_materialized_view_as_sql`.\n #} */\n {{- log('Determining configuration changes on: ' ~ existing_relation) -}}\n {%- do return(adapter.dispatch('get_materialized_view_configuration_changes', 'dbt')(existing_relation, new_config)) -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_materialized_view_configuration_changes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.711103, "supported_languages": null}, "macro.dbt.default__get_materialized_view_configuration_changes": {"name": "default__get_materialized_view_configuration_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt.default__get_materialized_view_configuration_changes", "macro_sql": "{% macro default__get_materialized_view_configuration_changes(existing_relation, new_config) %}\n {{ exceptions.raise_compiler_error(\"Materialized views have not been implemented for this adapter.\") }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7113469, "supported_languages": null}, "macro.dbt.get_create_materialized_view_as_sql": {"name": "get_create_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/create.sql", "original_file_path": "macros/relations/materialized_view/create.sql", "unique_id": "macro.dbt.get_create_materialized_view_as_sql", "macro_sql": "{% macro get_create_materialized_view_as_sql(relation, sql) -%}\n {{- adapter.dispatch('get_create_materialized_view_as_sql', 'dbt')(relation, sql) -}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_create_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7117891, "supported_languages": null}, "macro.dbt.default__get_create_materialized_view_as_sql": {"name": "default__get_create_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/create.sql", "original_file_path": "macros/relations/materialized_view/create.sql", "unique_id": "macro.dbt.default__get_create_materialized_view_as_sql", "macro_sql": "{% macro default__get_create_materialized_view_as_sql(relation, sql) -%}\n {{ exceptions.raise_compiler_error(\n \"`get_create_materialized_view_as_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.712033, "supported_languages": null}, "macro.dbt.get_table_columns_and_constraints": {"name": "get_table_columns_and_constraints", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.get_table_columns_and_constraints", "macro_sql": "{%- macro get_table_columns_and_constraints() -%}\n {{ adapter.dispatch('get_table_columns_and_constraints', 'dbt')() }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__get_table_columns_and_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.713679, "supported_languages": null}, "macro.dbt.default__get_table_columns_and_constraints": {"name": "default__get_table_columns_and_constraints", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.default__get_table_columns_and_constraints", "macro_sql": "{% macro default__get_table_columns_and_constraints() -%}\n {{ return(table_columns_and_constraints()) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.table_columns_and_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.713873, "supported_languages": null}, "macro.dbt.table_columns_and_constraints": {"name": "table_columns_and_constraints", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.table_columns_and_constraints", "macro_sql": "{% macro table_columns_and_constraints() %}\n {# loop through user_provided_columns to create DDL with data types and constraints #}\n {%- set raw_column_constraints = adapter.render_raw_columns_constraints(raw_columns=model['columns']) -%}\n {%- set raw_model_constraints = adapter.render_raw_model_constraints(raw_constraints=model['constraints']) -%}\n (\n {% for c in raw_column_constraints -%}\n {{ c }}{{ \",\" if not loop.last or raw_model_constraints }}\n {% endfor %}\n {% for c in raw_model_constraints -%}\n {{ c }}{{ \",\" if not loop.last }}\n {% endfor -%}\n )\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7147489, "supported_languages": null}, "macro.dbt.get_assert_columns_equivalent": {"name": "get_assert_columns_equivalent", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.get_assert_columns_equivalent", "macro_sql": "\n\n{%- macro get_assert_columns_equivalent(sql) -%}\n {{ adapter.dispatch('get_assert_columns_equivalent', 'dbt')(sql) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__get_assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.715026, "supported_languages": null}, "macro.dbt.default__get_assert_columns_equivalent": {"name": "default__get_assert_columns_equivalent", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.default__get_assert_columns_equivalent", "macro_sql": "{% macro default__get_assert_columns_equivalent(sql) -%}\n {{ return(assert_columns_equivalent(sql)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7152472, "supported_languages": null}, "macro.dbt.assert_columns_equivalent": {"name": "assert_columns_equivalent", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.assert_columns_equivalent", "macro_sql": "{% macro assert_columns_equivalent(sql) %}\n\n {#-- First ensure the user has defined 'columns' in yaml specification --#}\n {%- set user_defined_columns = model['columns'] -%}\n {%- if not user_defined_columns -%}\n {{ exceptions.raise_contract_error([], []) }}\n {%- endif -%}\n\n {#-- Obtain the column schema provided by sql file. #}\n {%- set sql_file_provided_columns = get_column_schema_from_query(sql, config.get('sql_header', none)) -%}\n {#--Obtain the column schema provided by the schema file by generating an 'empty schema' query from the model's columns. #}\n {%- set schema_file_provided_columns = get_column_schema_from_query(get_empty_schema_sql(user_defined_columns)) -%}\n\n {#-- create dictionaries with name and formatted data type and strings for exception #}\n {%- set sql_columns = format_columns(sql_file_provided_columns) -%}\n {%- set yaml_columns = format_columns(schema_file_provided_columns) -%}\n\n {%- if sql_columns|length != yaml_columns|length -%}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n\n {%- for sql_col in sql_columns -%}\n {%- set yaml_col = [] -%}\n {%- for this_col in yaml_columns -%}\n {%- if this_col['name'] == sql_col['name'] -%}\n {%- do yaml_col.append(this_col) -%}\n {%- break -%}\n {%- endif -%}\n {%- endfor -%}\n {%- if not yaml_col -%}\n {#-- Column with name not found in yaml #}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n {%- if sql_col['formatted'] != yaml_col[0]['formatted'] -%}\n {#-- Column data types don't match #}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n {%- endfor -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_column_schema_from_query", "macro.dbt.get_empty_schema_sql", "macro.dbt.format_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.717453, "supported_languages": null}, "macro.dbt.format_columns": {"name": "format_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.format_columns", "macro_sql": "{% macro format_columns(columns) %}\n {% set formatted_columns = [] %}\n {% for column in columns %}\n {%- set formatted_column = adapter.dispatch('format_column', 'dbt')(column) -%}\n {%- do formatted_columns.append(formatted_column) -%}\n {% endfor %}\n {{ return(formatted_columns) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__format_column"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.718085, "supported_languages": null}, "macro.dbt.default__format_column": {"name": "default__format_column", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.default__format_column", "macro_sql": "{% macro default__format_column(column) -%}\n {% set data_type = column.dtype %}\n {% set formatted = column.column.lower() ~ \" \" ~ data_type %}\n {{ return({'name': column.name, 'data_type': data_type, 'formatted': formatted}) }}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.718652, "supported_languages": null}, "macro.dbt.drop_table": {"name": "drop_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/drop.sql", "original_file_path": "macros/relations/table/drop.sql", "unique_id": "macro.dbt.drop_table", "macro_sql": "{% macro drop_table(relation) -%}\n {{ return(adapter.dispatch('drop_table', 'dbt')(relation)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__drop_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.719095, "supported_languages": null}, "macro.dbt.default__drop_table": {"name": "default__drop_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/drop.sql", "original_file_path": "macros/relations/table/drop.sql", "unique_id": "macro.dbt.default__drop_table", "macro_sql": "{% macro default__drop_table(relation) -%}\n drop table if exists {{ relation }} cascade\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.719268, "supported_languages": null}, "macro.dbt.get_replace_table_sql": {"name": "get_replace_table_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/replace.sql", "original_file_path": "macros/relations/table/replace.sql", "unique_id": "macro.dbt.get_replace_table_sql", "macro_sql": "{% macro get_replace_table_sql(relation, sql) %}\n {{- adapter.dispatch('get_replace_table_sql', 'dbt')(relation, sql) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_replace_table_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.719711, "supported_languages": null}, "macro.dbt.default__get_replace_table_sql": {"name": "default__get_replace_table_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/replace.sql", "original_file_path": "macros/relations/table/replace.sql", "unique_id": "macro.dbt.default__get_replace_table_sql", "macro_sql": "{% macro default__get_replace_table_sql(relation, sql) %}\n {{ exceptions.raise_compiler_error(\n \"`get_replace_table_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.719948, "supported_languages": null}, "macro.dbt.get_rename_table_sql": {"name": "get_rename_table_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/rename.sql", "original_file_path": "macros/relations/table/rename.sql", "unique_id": "macro.dbt.get_rename_table_sql", "macro_sql": "{% macro get_rename_table_sql(relation, new_name) %}\n {{- adapter.dispatch('get_rename_table_sql', 'dbt')(relation, new_name) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_rename_table_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7203882, "supported_languages": null}, "macro.dbt.default__get_rename_table_sql": {"name": "default__get_rename_table_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/rename.sql", "original_file_path": "macros/relations/table/rename.sql", "unique_id": "macro.dbt.default__get_rename_table_sql", "macro_sql": "{% macro default__get_rename_table_sql(relation, new_name) %}\n {{ exceptions.raise_compiler_error(\n \"`get_rename_table_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7206268, "supported_languages": null}, "macro.dbt.get_create_table_as_sql": {"name": "get_create_table_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.get_create_table_as_sql", "macro_sql": "{% macro get_create_table_as_sql(temporary, relation, sql) -%}\n {{ adapter.dispatch('get_create_table_as_sql', 'dbt')(temporary, relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_create_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.722004, "supported_languages": null}, "macro.dbt.default__get_create_table_as_sql": {"name": "default__get_create_table_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.default__get_create_table_as_sql", "macro_sql": "{% macro default__get_create_table_as_sql(temporary, relation, sql) -%}\n {{ return(create_table_as(temporary, relation, sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.722313, "supported_languages": null}, "macro.dbt.create_table_as": {"name": "create_table_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.create_table_as", "macro_sql": "{% macro create_table_as(temporary, relation, compiled_code, language='sql') -%}\n {# backward compatibility for create_table_as that does not support language #}\n {% if language == \"sql\" %}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, compiled_code)}}\n {% else %}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, compiled_code, language) }}\n {% endif %}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.723018, "supported_languages": null}, "macro.dbt.default__create_table_as": {"name": "default__create_table_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.default__create_table_as", "macro_sql": "{% macro default__create_table_as(temporary, relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary: -%}temporary{%- endif %} table\n {{ relation.include(database=(not temporary), schema=(not temporary)) }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced and (not temporary) %}\n {{ get_assert_columns_equivalent(sql) }}\n {{ get_table_columns_and_constraints() }}\n {%- set sql = get_select_subquery(sql) %}\n {% endif %}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_assert_columns_equivalent", "macro.dbt.get_table_columns_and_constraints", "macro.dbt.get_select_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.724143, "supported_languages": null}, "macro.dbt.default__get_column_names": {"name": "default__get_column_names", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.default__get_column_names", "macro_sql": "{% macro default__get_column_names() %}\n {#- loop through user_provided_columns to get column names -#}\n {%- set user_provided_columns = model['columns'] -%}\n {%- for i in user_provided_columns %}\n {%- set col = user_provided_columns[i] -%}\n {%- set col_name = adapter.quote(col['name']) if col.get('quote') else col['name'] -%}\n {{ col_name }}{{ \", \" if not loop.last }}\n {%- endfor -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.724905, "supported_languages": null}, "macro.dbt.get_select_subquery": {"name": "get_select_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.get_select_subquery", "macro_sql": "{% macro get_select_subquery(sql) %}\n {{ return(adapter.dispatch('get_select_subquery', 'dbt')(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_select_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.725209, "supported_languages": null}, "macro.dbt.default__get_select_subquery": {"name": "default__get_select_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.default__get_select_subquery", "macro_sql": "{% macro default__get_select_subquery(sql) %}\n select {{ adapter.dispatch('get_column_names', 'dbt')() }}\n from (\n {{ sql }}\n ) as model_subq\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_column_names"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7255082, "supported_languages": null}, "macro.dbt.drop_view": {"name": "drop_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/drop.sql", "original_file_path": "macros/relations/view/drop.sql", "unique_id": "macro.dbt.drop_view", "macro_sql": "{% macro drop_view(relation) -%}\n {{ return(adapter.dispatch('drop_view', 'dbt')(relation)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__drop_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.72594, "supported_languages": null}, "macro.dbt.default__drop_view": {"name": "default__drop_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/drop.sql", "original_file_path": "macros/relations/view/drop.sql", "unique_id": "macro.dbt.default__drop_view", "macro_sql": "{% macro default__drop_view(relation) -%}\n drop view if exists {{ relation }} cascade\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.726114, "supported_languages": null}, "macro.dbt.get_replace_view_sql": {"name": "get_replace_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.get_replace_view_sql", "macro_sql": "{% macro get_replace_view_sql(relation, sql) %}\n {{- adapter.dispatch('get_replace_view_sql', 'dbt')(relation, sql) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_replace_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.72751, "supported_languages": null}, "macro.dbt.default__get_replace_view_sql": {"name": "default__get_replace_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.default__get_replace_view_sql", "macro_sql": "{% macro default__get_replace_view_sql(relation, sql) %}\n {{ exceptions.raise_compiler_error(\n \"`get_replace_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7277539, "supported_languages": null}, "macro.dbt.create_or_replace_view": {"name": "create_or_replace_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.create_or_replace_view", "macro_sql": "{% macro create_or_replace_view() %}\n {%- set identifier = model['alias'] -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database,\n type='view') -%}\n {% set grant_config = config.get('grants') %}\n\n {{ run_hooks(pre_hooks) }}\n\n -- If there's a table with the same name and we weren't told to full refresh,\n -- that's an error. If we were told to full refresh, drop it. This behavior differs\n -- for Snowflake and BigQuery, so multiple dispatch is used.\n {%- if old_relation is not none and old_relation.is_table -%}\n {{ handle_existing_table(should_full_refresh(), old_relation) }}\n {%- endif -%}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(target_relation, sql) }}\n {%- endcall %}\n\n {% set should_revoke = should_revoke(exists_as_view, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.handle_existing_table", "macro.dbt.should_full_refresh", "macro.dbt.statement", "macro.dbt.get_create_view_as_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.729735, "supported_languages": null}, "macro.dbt.handle_existing_table": {"name": "handle_existing_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.handle_existing_table", "macro_sql": "{% macro handle_existing_table(full_refresh, old_relation) %}\n {{ adapter.dispatch('handle_existing_table', 'dbt')(full_refresh, old_relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__handle_existing_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.730062, "supported_languages": null}, "macro.dbt.default__handle_existing_table": {"name": "default__handle_existing_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.default__handle_existing_table", "macro_sql": "{% macro default__handle_existing_table(full_refresh, old_relation) %}\n {{ log(\"Dropping relation \" ~ old_relation ~ \" because it is of type \" ~ old_relation.type) }}\n {{ adapter.drop_relation(old_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7304268, "supported_languages": null}, "macro.dbt.get_rename_view_sql": {"name": "get_rename_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/rename.sql", "original_file_path": "macros/relations/view/rename.sql", "unique_id": "macro.dbt.get_rename_view_sql", "macro_sql": "{% macro get_rename_view_sql(relation, new_name) %}\n {{- adapter.dispatch('get_rename_view_sql', 'dbt')(relation, new_name) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_rename_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.730877, "supported_languages": null}, "macro.dbt.default__get_rename_view_sql": {"name": "default__get_rename_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/rename.sql", "original_file_path": "macros/relations/view/rename.sql", "unique_id": "macro.dbt.default__get_rename_view_sql", "macro_sql": "{% macro default__get_rename_view_sql(relation, new_name) %}\n {{ exceptions.raise_compiler_error(\n \"`get_rename_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7311208, "supported_languages": null}, "macro.dbt.get_create_view_as_sql": {"name": "get_create_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt.get_create_view_as_sql", "macro_sql": "{% macro get_create_view_as_sql(relation, sql) -%}\n {{ adapter.dispatch('get_create_view_as_sql', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_create_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7317948, "supported_languages": null}, "macro.dbt.default__get_create_view_as_sql": {"name": "default__get_create_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt.default__get_create_view_as_sql", "macro_sql": "{% macro default__get_create_view_as_sql(relation, sql) -%}\n {{ return(create_view_as(relation, sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.732067, "supported_languages": null}, "macro.dbt.create_view_as": {"name": "create_view_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt.create_view_as", "macro_sql": "{% macro create_view_as(relation, sql) -%}\n {{ adapter.dispatch('create_view_as', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7323651, "supported_languages": null}, "macro.dbt.default__create_view_as": {"name": "default__create_view_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt.default__create_view_as", "macro_sql": "{% macro default__create_view_as(relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n create view {{ relation }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {%- endif %}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7330492, "supported_languages": null}, "macro.dbt.default__test_relationships": {"name": "default__test_relationships", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/relationships.sql", "original_file_path": "macros/generic_test_sql/relationships.sql", "unique_id": "macro.dbt.default__test_relationships", "macro_sql": "{% macro default__test_relationships(model, column_name, to, field) %}\n\nwith child as (\n select {{ column_name }} as from_field\n from {{ model }}\n where {{ column_name }} is not null\n),\n\nparent as (\n select {{ field }} as to_field\n from {{ to }}\n)\n\nselect\n from_field\n\nfrom child\nleft join parent\n on child.from_field = parent.to_field\n\nwhere parent.to_field is null\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7336361, "supported_languages": null}, "macro.dbt.default__test_not_null": {"name": "default__test_not_null", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/not_null.sql", "original_file_path": "macros/generic_test_sql/not_null.sql", "unique_id": "macro.dbt.default__test_not_null", "macro_sql": "{% macro default__test_not_null(model, column_name) %}\n\n{% set column_list = '*' if should_store_failures() else column_name %}\n\nselect {{ column_list }}\nfrom {{ model }}\nwhere {{ column_name }} is null\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_store_failures"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.734132, "supported_languages": null}, "macro.dbt.default__test_unique": {"name": "default__test_unique", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/unique.sql", "original_file_path": "macros/generic_test_sql/unique.sql", "unique_id": "macro.dbt.default__test_unique", "macro_sql": "{% macro default__test_unique(model, column_name) %}\n\nselect\n {{ column_name }} as unique_field,\n count(*) as n_records\n\nfrom {{ model }}\nwhere {{ column_name }} is not null\ngroup by {{ column_name }}\nhaving count(*) > 1\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7345579, "supported_languages": null}, "macro.dbt.default__test_accepted_values": {"name": "default__test_accepted_values", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/accepted_values.sql", "original_file_path": "macros/generic_test_sql/accepted_values.sql", "unique_id": "macro.dbt.default__test_accepted_values", "macro_sql": "{% macro default__test_accepted_values(model, column_name, values, quote=True) %}\n\nwith all_values as (\n\n select\n {{ column_name }} as value_field,\n count(*) as n_records\n\n from {{ model }}\n group by {{ column_name }}\n\n)\n\nselect *\nfrom all_values\nwhere value_field not in (\n {% for value in values -%}\n {% if quote -%}\n '{{ value }}'\n {%- else -%}\n {{ value }}\n {%- endif -%}\n {%- if not loop.last -%},{%- endif %}\n {%- endfor %}\n)\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.735542, "supported_languages": null}, "macro.dbt.statement": {"name": "statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt.statement", "macro_sql": "\n{%- macro statement(name=None, fetch_result=False, auto_begin=True, language='sql') -%}\n {%- if execute: -%}\n {%- set compiled_code = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime {} for node \"{}\"'.format(language, model['unique_id'])) }}\n {{ write(compiled_code) }}\n {%- endif -%}\n {%- if language == 'sql'-%}\n {%- set res, table = adapter.execute(compiled_code, auto_begin=auto_begin, fetch=fetch_result) -%}\n {%- elif language == 'python' -%}\n {%- set res = submit_python_job(model, compiled_code) -%}\n {#-- TODO: What should table be for python models? --#}\n {%- set table = None -%}\n {%- else -%}\n {% do exceptions.raise_compiler_error(\"statement macro didn't get supported language\") %}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_result(name, response=res, agate_table=table) }}\n {%- endif -%}\n\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7379699, "supported_languages": null}, "macro.dbt.noop_statement": {"name": "noop_statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt.noop_statement", "macro_sql": "{% macro noop_statement(name=None, message=None, code=None, rows_affected=None, res=None) -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_raw_result(name, message=message, code=code, rows_affected=rows_affected, agate_table=res) }}\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7389178, "supported_languages": null}, "macro.dbt.run_query": {"name": "run_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt.run_query", "macro_sql": "{% macro run_query(sql) %}\n {% call statement(\"run_query_statement\", fetch_result=true, auto_begin=false) %}\n {{ sql }}\n {% endcall %}\n\n {% do return(load_result(\"run_query_statement\").table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.739396, "supported_languages": null}, "macro.dbt.convert_datetime": {"name": "convert_datetime", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.convert_datetime", "macro_sql": "{% macro convert_datetime(date_str, date_fmt) %}\n\n {% set error_msg -%}\n The provided partition date '{{ date_str }}' does not match the expected format '{{ date_fmt }}'\n {%- endset %}\n\n {% set res = try_or_compiler_error(error_msg, modules.datetime.datetime.strptime, date_str.strip(), date_fmt) %}\n {{ return(res) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7423072, "supported_languages": null}, "macro.dbt.dates_in_range": {"name": "dates_in_range", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.dates_in_range", "macro_sql": "{% macro dates_in_range(start_date_str, end_date_str=none, in_fmt=\"%Y%m%d\", out_fmt=\"%Y%m%d\") %}\n {% set end_date_str = start_date_str if end_date_str is none else end_date_str %}\n\n {% set start_date = convert_datetime(start_date_str, in_fmt) %}\n {% set end_date = convert_datetime(end_date_str, in_fmt) %}\n\n {% set day_count = (end_date - start_date).days %}\n {% if day_count < 0 %}\n {% set msg -%}\n Partition start date is after the end date ({{ start_date }}, {{ end_date }})\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg, model) }}\n {% endif %}\n\n {% set date_list = [] %}\n {% for i in range(0, day_count + 1) %}\n {% set the_date = (modules.datetime.timedelta(days=i) + start_date) %}\n {% if not out_fmt %}\n {% set _ = date_list.append(the_date) %}\n {% else %}\n {% set _ = date_list.append(the_date.strftime(out_fmt)) %}\n {% endif %}\n {% endfor %}\n\n {{ return(date_list) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.convert_datetime"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7443829, "supported_languages": null}, "macro.dbt.partition_range": {"name": "partition_range", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.partition_range", "macro_sql": "{% macro partition_range(raw_partition_date, date_fmt='%Y%m%d') %}\n {% set partition_range = (raw_partition_date | string).split(\",\") %}\n\n {% if (partition_range | length) == 1 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = none %}\n {% elif (partition_range | length) == 2 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = partition_range[1] %}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Invalid partition time. Expected format: {Start Date}[,{End Date}]. Got: \" ~ raw_partition_date) }}\n {% endif %}\n\n {{ return(dates_in_range(start_date, end_date, in_fmt=date_fmt)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.dates_in_range"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.745625, "supported_languages": null}, "macro.dbt.py_current_timestring": {"name": "py_current_timestring", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.py_current_timestring", "macro_sql": "{% macro py_current_timestring() %}\n {% set dt = modules.datetime.datetime.now() %}\n {% do return(dt.strftime(\"%Y%m%d%H%M%S%f\")) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.746016, "supported_languages": null}, "macro.dbt.except": {"name": "except", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/except.sql", "original_file_path": "macros/utils/except.sql", "unique_id": "macro.dbt.except", "macro_sql": "{% macro except() %}\n {{ return(adapter.dispatch('except', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__except"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7463942, "supported_languages": null}, "macro.dbt.default__except": {"name": "default__except", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/except.sql", "original_file_path": "macros/utils/except.sql", "unique_id": "macro.dbt.default__except", "macro_sql": "{% macro default__except() %}\n\n except\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7465222, "supported_languages": null}, "macro.dbt.get_intervals_between": {"name": "get_intervals_between", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_spine.sql", "original_file_path": "macros/utils/date_spine.sql", "unique_id": "macro.dbt.get_intervals_between", "macro_sql": "{% macro get_intervals_between(start_date, end_date, datepart) -%}\n {{ return(adapter.dispatch('get_intervals_between', 'dbt')(start_date, end_date, datepart)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_intervals_between"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7477372, "supported_languages": null}, "macro.dbt.default__get_intervals_between": {"name": "default__get_intervals_between", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_spine.sql", "original_file_path": "macros/utils/date_spine.sql", "unique_id": "macro.dbt.default__get_intervals_between", "macro_sql": "{% macro default__get_intervals_between(start_date, end_date, datepart) -%}\n {%- call statement('get_intervals_between', fetch_result=True) %}\n\n select {{ dbt.datediff(start_date, end_date, datepart) }}\n\n {%- endcall -%}\n\n {%- set value_list = load_result('get_intervals_between') -%}\n\n {%- if value_list and value_list['data'] -%}\n {%- set values = value_list['data'] | map(attribute=0) | list %}\n {{ return(values[0]) }}\n {%- else -%}\n {{ return(1) }}\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.datediff"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.748717, "supported_languages": null}, "macro.dbt.date_spine": {"name": "date_spine", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_spine.sql", "original_file_path": "macros/utils/date_spine.sql", "unique_id": "macro.dbt.date_spine", "macro_sql": "{% macro date_spine(datepart, start_date, end_date) %}\n {{ return(adapter.dispatch('date_spine', 'dbt')(datepart, start_date, end_date)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__date_spine"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.749108, "supported_languages": null}, "macro.dbt.default__date_spine": {"name": "default__date_spine", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_spine.sql", "original_file_path": "macros/utils/date_spine.sql", "unique_id": "macro.dbt.default__date_spine", "macro_sql": "{% macro default__date_spine(datepart, start_date, end_date) %}\n\n\n {# call as follows:\n\n date_spine(\n \"day\",\n \"to_date('01/01/2016', 'mm/dd/yyyy')\",\n \"dbt.dateadd(week, 1, current_date)\"\n ) #}\n\n\n with rawdata as (\n\n {{dbt.generate_series(\n dbt.get_intervals_between(start_date, end_date, datepart)\n )}}\n\n ),\n\n all_periods as (\n\n select (\n {{\n dbt.dateadd(\n datepart,\n \"row_number() over (order by 1) - 1\",\n start_date\n )\n }}\n ) as date_{{datepart}}\n from rawdata\n\n ),\n\n filtered as (\n\n select *\n from all_periods\n where date_{{datepart}} <= {{ end_date }}\n\n )\n\n select * from filtered\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.generate_series", "macro.dbt.get_intervals_between", "macro.dbt.dateadd"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.749716, "supported_languages": null}, "macro.dbt.replace": {"name": "replace", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/replace.sql", "original_file_path": "macros/utils/replace.sql", "unique_id": "macro.dbt.replace", "macro_sql": "{% macro replace(field, old_chars, new_chars) -%}\n {{ return(adapter.dispatch('replace', 'dbt') (field, old_chars, new_chars)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__replace"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.750282, "supported_languages": null}, "macro.dbt.default__replace": {"name": "default__replace", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/replace.sql", "original_file_path": "macros/utils/replace.sql", "unique_id": "macro.dbt.default__replace", "macro_sql": "{% macro default__replace(field, old_chars, new_chars) %}\n\n replace(\n {{ field }},\n {{ old_chars }},\n {{ new_chars }}\n )\n\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.750558, "supported_languages": null}, "macro.dbt.concat": {"name": "concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/concat.sql", "original_file_path": "macros/utils/concat.sql", "unique_id": "macro.dbt.concat", "macro_sql": "{% macro concat(fields) -%}\n {{ return(adapter.dispatch('concat', 'dbt')(fields)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__concat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.75096, "supported_languages": null}, "macro.dbt.default__concat": {"name": "default__concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/concat.sql", "original_file_path": "macros/utils/concat.sql", "unique_id": "macro.dbt.default__concat", "macro_sql": "{% macro default__concat(fields) -%}\n {{ fields|join(' || ') }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7511628, "supported_languages": null}, "macro.dbt.get_powers_of_two": {"name": "get_powers_of_two", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/generate_series.sql", "original_file_path": "macros/utils/generate_series.sql", "unique_id": "macro.dbt.get_powers_of_two", "macro_sql": "{% macro get_powers_of_two(upper_bound) %}\n {{ return(adapter.dispatch('get_powers_of_two', 'dbt')(upper_bound)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_powers_of_two"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7524989, "supported_languages": null}, "macro.dbt.default__get_powers_of_two": {"name": "default__get_powers_of_two", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/generate_series.sql", "original_file_path": "macros/utils/generate_series.sql", "unique_id": "macro.dbt.default__get_powers_of_two", "macro_sql": "{% macro default__get_powers_of_two(upper_bound) %}\n\n {% if upper_bound <= 0 %}\n {{ exceptions.raise_compiler_error(\"upper bound must be positive\") }}\n {% endif %}\n\n {% for _ in range(1, 100) %}\n {% if upper_bound <= 2 ** loop.index %}{{ return(loop.index) }}{% endif %}\n {% endfor %}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.753182, "supported_languages": null}, "macro.dbt.generate_series": {"name": "generate_series", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/generate_series.sql", "original_file_path": "macros/utils/generate_series.sql", "unique_id": "macro.dbt.generate_series", "macro_sql": "{% macro generate_series(upper_bound) %}\n {{ return(adapter.dispatch('generate_series', 'dbt')(upper_bound)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__generate_series"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7535, "supported_languages": null}, "macro.dbt.default__generate_series": {"name": "default__generate_series", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/generate_series.sql", "original_file_path": "macros/utils/generate_series.sql", "unique_id": "macro.dbt.default__generate_series", "macro_sql": "{% macro default__generate_series(upper_bound) %}\n\n {% set n = dbt.get_powers_of_two(upper_bound) %}\n\n with p as (\n select 0 as generated_number union all select 1\n ), unioned as (\n\n select\n\n {% for i in range(n) %}\n p{{i}}.generated_number * power(2, {{i}})\n {% if not loop.last %} + {% endif %}\n {% endfor %}\n + 1\n as generated_number\n\n from\n\n {% for i in range(n) %}\n p as p{{i}}\n {% if not loop.last %} cross join {% endif %}\n {% endfor %}\n\n )\n\n select *\n from unioned\n where generated_number <= {{upper_bound}}\n order by generated_number\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_powers_of_two"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.754335, "supported_languages": null}, "macro.dbt.length": {"name": "length", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/length.sql", "original_file_path": "macros/utils/length.sql", "unique_id": "macro.dbt.length", "macro_sql": "{% macro length(expression) -%}\n {{ return(adapter.dispatch('length', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__length"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7547672, "supported_languages": null}, "macro.dbt.default__length": {"name": "default__length", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/length.sql", "original_file_path": "macros/utils/length.sql", "unique_id": "macro.dbt.default__length", "macro_sql": "{% macro default__length(expression) %}\n\n length(\n {{ expression }}\n )\n\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7549481, "supported_languages": null}, "macro.dbt.dateadd": {"name": "dateadd", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt.dateadd", "macro_sql": "{% macro dateadd(datepart, interval, from_date_or_timestamp) %}\n {{ return(adapter.dispatch('dateadd', 'dbt')(datepart, interval, from_date_or_timestamp)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__dateadd"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7555032, "supported_languages": null}, "macro.dbt.default__dateadd": {"name": "default__dateadd", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt.default__dateadd", "macro_sql": "{% macro default__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n dateadd(\n {{ datepart }},\n {{ interval }},\n {{ from_date_or_timestamp }}\n )\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7557771, "supported_languages": null}, "macro.dbt.intersect": {"name": "intersect", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/intersect.sql", "original_file_path": "macros/utils/intersect.sql", "unique_id": "macro.dbt.intersect", "macro_sql": "{% macro intersect() %}\n {{ return(adapter.dispatch('intersect', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__intersect"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.75615, "supported_languages": null}, "macro.dbt.default__intersect": {"name": "default__intersect", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/intersect.sql", "original_file_path": "macros/utils/intersect.sql", "unique_id": "macro.dbt.default__intersect", "macro_sql": "{% macro default__intersect() %}\n\n intersect\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7564, "supported_languages": null}, "macro.dbt.escape_single_quotes": {"name": "escape_single_quotes", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/escape_single_quotes.sql", "original_file_path": "macros/utils/escape_single_quotes.sql", "unique_id": "macro.dbt.escape_single_quotes", "macro_sql": "{% macro escape_single_quotes(expression) %}\n {{ return(adapter.dispatch('escape_single_quotes', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__escape_single_quotes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.756841, "supported_languages": null}, "macro.dbt.default__escape_single_quotes": {"name": "default__escape_single_quotes", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/escape_single_quotes.sql", "original_file_path": "macros/utils/escape_single_quotes.sql", "unique_id": "macro.dbt.default__escape_single_quotes", "macro_sql": "{% macro default__escape_single_quotes(expression) -%}\n{{ expression | replace(\"'\",\"''\") }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7570739, "supported_languages": null}, "macro.dbt.right": {"name": "right", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/right.sql", "original_file_path": "macros/utils/right.sql", "unique_id": "macro.dbt.right", "macro_sql": "{% macro right(string_text, length_expression) -%}\n {{ return(adapter.dispatch('right', 'dbt') (string_text, length_expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__right"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.75756, "supported_languages": null}, "macro.dbt.default__right": {"name": "default__right", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/right.sql", "original_file_path": "macros/utils/right.sql", "unique_id": "macro.dbt.default__right", "macro_sql": "{% macro default__right(string_text, length_expression) %}\n\n right(\n {{ string_text }},\n {{ length_expression }}\n )\n\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.757789, "supported_languages": null}, "macro.dbt.listagg": {"name": "listagg", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "unique_id": "macro.dbt.listagg", "macro_sql": "{% macro listagg(measure, delimiter_text=\"','\", order_by_clause=none, limit_num=none) -%}\n {{ return(adapter.dispatch('listagg', 'dbt') (measure, delimiter_text, order_by_clause, limit_num)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__listagg"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.758796, "supported_languages": null}, "macro.dbt.default__listagg": {"name": "default__listagg", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "unique_id": "macro.dbt.default__listagg", "macro_sql": "{% macro default__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n {% if limit_num -%}\n array_to_string(\n array_slice(\n array_agg(\n {{ measure }}\n ){% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n ,0\n ,{{ limit_num }}\n ),\n {{ delimiter_text }}\n )\n {%- else %}\n listagg(\n {{ measure }},\n {{ delimiter_text }}\n )\n {% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n {%- endif %}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7594528, "supported_languages": null}, "macro.dbt.datediff": {"name": "datediff", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt.datediff", "macro_sql": "{% macro datediff(first_date, second_date, datepart) %}\n {{ return(adapter.dispatch('datediff', 'dbt')(first_date, second_date, datepart)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__datediff"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.759997, "supported_languages": null}, "macro.dbt.default__datediff": {"name": "default__datediff", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt.default__datediff", "macro_sql": "{% macro default__datediff(first_date, second_date, datepart) -%}\n\n datediff(\n {{ datepart }},\n {{ first_date }},\n {{ second_date }}\n )\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7602718, "supported_languages": null}, "macro.dbt.safe_cast": {"name": "safe_cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "unique_id": "macro.dbt.safe_cast", "macro_sql": "{% macro safe_cast(field, type) %}\n {{ return(adapter.dispatch('safe_cast', 'dbt') (field, type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__safe_cast"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.760739, "supported_languages": null}, "macro.dbt.default__safe_cast": {"name": "default__safe_cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "unique_id": "macro.dbt.default__safe_cast", "macro_sql": "{% macro default__safe_cast(field, type) %}\n {# most databases don't support this function yet\n so we just need to use cast #}\n cast({{field}} as {{type}})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.760973, "supported_languages": null}, "macro.dbt.hash": {"name": "hash", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/hash.sql", "original_file_path": "macros/utils/hash.sql", "unique_id": "macro.dbt.hash", "macro_sql": "{% macro hash(field) -%}\n {{ return(adapter.dispatch('hash', 'dbt') (field)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__hash"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.76143, "supported_languages": null}, "macro.dbt.default__hash": {"name": "default__hash", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/hash.sql", "original_file_path": "macros/utils/hash.sql", "unique_id": "macro.dbt.default__hash", "macro_sql": "{% macro default__hash(field) -%}\n md5(cast({{ field }} as {{ api.Column.translate_type('string') }}))\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.761698, "supported_languages": null}, "macro.dbt.cast_bool_to_text": {"name": "cast_bool_to_text", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast_bool_to_text.sql", "original_file_path": "macros/utils/cast_bool_to_text.sql", "unique_id": "macro.dbt.cast_bool_to_text", "macro_sql": "{% macro cast_bool_to_text(field) %}\n {{ adapter.dispatch('cast_bool_to_text', 'dbt') (field) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__cast_bool_to_text"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.762109, "supported_languages": null}, "macro.dbt.default__cast_bool_to_text": {"name": "default__cast_bool_to_text", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast_bool_to_text.sql", "original_file_path": "macros/utils/cast_bool_to_text.sql", "unique_id": "macro.dbt.default__cast_bool_to_text", "macro_sql": "{% macro default__cast_bool_to_text(field) %}\n cast({{ field }} as {{ api.Column.translate_type('string') }})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.762371, "supported_languages": null}, "macro.dbt.any_value": {"name": "any_value", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "unique_id": "macro.dbt.any_value", "macro_sql": "{% macro any_value(expression) -%}\n {{ return(adapter.dispatch('any_value', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__any_value"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.762775, "supported_languages": null}, "macro.dbt.default__any_value": {"name": "default__any_value", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "unique_id": "macro.dbt.default__any_value", "macro_sql": "{% macro default__any_value(expression) -%}\n\n any_value({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7629561, "supported_languages": null}, "macro.dbt.position": {"name": "position", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/position.sql", "original_file_path": "macros/utils/position.sql", "unique_id": "macro.dbt.position", "macro_sql": "{% macro position(substring_text, string_text) -%}\n {{ return(adapter.dispatch('position', 'dbt') (substring_text, string_text)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__position"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7634618, "supported_languages": null}, "macro.dbt.default__position": {"name": "default__position", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/position.sql", "original_file_path": "macros/utils/position.sql", "unique_id": "macro.dbt.default__position", "macro_sql": "{% macro default__position(substring_text, string_text) %}\n\n position(\n {{ substring_text }} in {{ string_text }}\n )\n\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.763691, "supported_languages": null}, "macro.dbt.string_literal": {"name": "string_literal", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/literal.sql", "original_file_path": "macros/utils/literal.sql", "unique_id": "macro.dbt.string_literal", "macro_sql": "{%- macro string_literal(value) -%}\n {{ return(adapter.dispatch('string_literal', 'dbt') (value)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__string_literal"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.764083, "supported_languages": null}, "macro.dbt.default__string_literal": {"name": "default__string_literal", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/literal.sql", "original_file_path": "macros/utils/literal.sql", "unique_id": "macro.dbt.default__string_literal", "macro_sql": "{% macro default__string_literal(value) -%}\n '{{ value }}'\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.764363, "supported_languages": null}, "macro.dbt.type_string": {"name": "type_string", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_string", "macro_sql": "\n\n{%- macro type_string() -%}\n {{ return(adapter.dispatch('type_string', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_string"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.765888, "supported_languages": null}, "macro.dbt.default__type_string": {"name": "default__type_string", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_string", "macro_sql": "{% macro default__type_string() %}\n {{ return(api.Column.translate_type(\"string\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.766141, "supported_languages": null}, "macro.dbt.type_timestamp": {"name": "type_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_timestamp", "macro_sql": "\n\n{%- macro type_timestamp() -%}\n {{ return(adapter.dispatch('type_timestamp', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.766407, "supported_languages": null}, "macro.dbt.default__type_timestamp": {"name": "default__type_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_timestamp", "macro_sql": "{% macro default__type_timestamp() %}\n {{ return(api.Column.translate_type(\"timestamp\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7666519, "supported_languages": null}, "macro.dbt.type_float": {"name": "type_float", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_float", "macro_sql": "\n\n{%- macro type_float() -%}\n {{ return(adapter.dispatch('type_float', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_float"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7669199, "supported_languages": null}, "macro.dbt.default__type_float": {"name": "default__type_float", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_float", "macro_sql": "{% macro default__type_float() %}\n {{ return(api.Column.translate_type(\"float\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.767169, "supported_languages": null}, "macro.dbt.type_numeric": {"name": "type_numeric", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_numeric", "macro_sql": "\n\n{%- macro type_numeric() -%}\n {{ return(adapter.dispatch('type_numeric', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_numeric"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.767443, "supported_languages": null}, "macro.dbt.default__type_numeric": {"name": "default__type_numeric", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_numeric", "macro_sql": "{% macro default__type_numeric() %}\n {{ return(api.Column.numeric_type(\"numeric\", 28, 6)) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7677372, "supported_languages": null}, "macro.dbt.type_bigint": {"name": "type_bigint", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_bigint", "macro_sql": "\n\n{%- macro type_bigint() -%}\n {{ return(adapter.dispatch('type_bigint', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_bigint"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7680051, "supported_languages": null}, "macro.dbt.default__type_bigint": {"name": "default__type_bigint", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_bigint", "macro_sql": "{% macro default__type_bigint() %}\n {{ return(api.Column.translate_type(\"bigint\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.768262, "supported_languages": null}, "macro.dbt.type_int": {"name": "type_int", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_int", "macro_sql": "\n\n{%- macro type_int() -%}\n {{ return(adapter.dispatch('type_int', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_int"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.768537, "supported_languages": null}, "macro.dbt.default__type_int": {"name": "default__type_int", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_int", "macro_sql": "{%- macro default__type_int() -%}\n {{ return(api.Column.translate_type(\"integer\")) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.768776, "supported_languages": null}, "macro.dbt.type_boolean": {"name": "type_boolean", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_boolean", "macro_sql": "\n\n{%- macro type_boolean() -%}\n {{ return(adapter.dispatch('type_boolean', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_boolean"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.769038, "supported_languages": null}, "macro.dbt.default__type_boolean": {"name": "default__type_boolean", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_boolean", "macro_sql": "{%- macro default__type_boolean() -%}\n {{ return(api.Column.translate_type(\"boolean\")) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.769278, "supported_languages": null}, "macro.dbt.array_concat": {"name": "array_concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_concat.sql", "original_file_path": "macros/utils/array_concat.sql", "unique_id": "macro.dbt.array_concat", "macro_sql": "{% macro array_concat(array_1, array_2) -%}\n {{ return(adapter.dispatch('array_concat', 'dbt')(array_1, array_2)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__array_concat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.769752, "supported_languages": null}, "macro.dbt.default__array_concat": {"name": "default__array_concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_concat.sql", "original_file_path": "macros/utils/array_concat.sql", "unique_id": "macro.dbt.default__array_concat", "macro_sql": "{% macro default__array_concat(array_1, array_2) -%}\n array_cat({{ array_1 }}, {{ array_2 }})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.769978, "supported_languages": null}, "macro.dbt.bool_or": {"name": "bool_or", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/bool_or.sql", "original_file_path": "macros/utils/bool_or.sql", "unique_id": "macro.dbt.bool_or", "macro_sql": "{% macro bool_or(expression) -%}\n {{ return(adapter.dispatch('bool_or', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__bool_or"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.770388, "supported_languages": null}, "macro.dbt.default__bool_or": {"name": "default__bool_or", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/bool_or.sql", "original_file_path": "macros/utils/bool_or.sql", "unique_id": "macro.dbt.default__bool_or", "macro_sql": "{% macro default__bool_or(expression) -%}\n\n bool_or({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.770564, "supported_languages": null}, "macro.dbt.last_day": {"name": "last_day", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt.last_day", "macro_sql": "{% macro last_day(date, datepart) %}\n {{ return(adapter.dispatch('last_day', 'dbt') (date, datepart)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__last_day"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.771231, "supported_languages": null}, "macro.dbt.default_last_day": {"name": "default_last_day", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt.default_last_day", "macro_sql": "\n\n{%- macro default_last_day(date, datepart) -%}\n cast(\n {{dbt.dateadd('day', '-1',\n dbt.dateadd(datepart, '1', dbt.date_trunc(datepart, date))\n )}}\n as date)\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.dateadd", "macro.dbt.date_trunc"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.771678, "supported_languages": null}, "macro.dbt.default__last_day": {"name": "default__last_day", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt.default__last_day", "macro_sql": "{% macro default__last_day(date, datepart) -%}\n {{dbt.default_last_day(date, datepart)}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default_last_day"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.771925, "supported_languages": null}, "macro.dbt.split_part": {"name": "split_part", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt.split_part", "macro_sql": "{% macro split_part(string_text, delimiter_text, part_number) %}\n {{ return(adapter.dispatch('split_part', 'dbt') (string_text, delimiter_text, part_number)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__split_part"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.772767, "supported_languages": null}, "macro.dbt.default__split_part": {"name": "default__split_part", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt.default__split_part", "macro_sql": "{% macro default__split_part(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n {{ part_number }}\n )\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.773036, "supported_languages": null}, "macro.dbt._split_part_negative": {"name": "_split_part_negative", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt._split_part_negative", "macro_sql": "{% macro _split_part_negative(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n length({{ string_text }})\n - length(\n replace({{ string_text }}, {{ delimiter_text }}, '')\n ) + 2 + {{ part_number }}\n )\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7733939, "supported_languages": null}, "macro.dbt.date_trunc": {"name": "date_trunc", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_trunc.sql", "original_file_path": "macros/utils/date_trunc.sql", "unique_id": "macro.dbt.date_trunc", "macro_sql": "{% macro date_trunc(datepart, date) -%}\n {{ return(adapter.dispatch('date_trunc', 'dbt') (datepart, date)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__date_trunc"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.773841, "supported_languages": null}, "macro.dbt.default__date_trunc": {"name": "default__date_trunc", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_trunc.sql", "original_file_path": "macros/utils/date_trunc.sql", "unique_id": "macro.dbt.default__date_trunc", "macro_sql": "{% macro default__date_trunc(datepart, date) -%}\n date_trunc('{{datepart}}', {{date}})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.774057, "supported_languages": null}, "macro.dbt.array_construct": {"name": "array_construct", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_construct.sql", "original_file_path": "macros/utils/array_construct.sql", "unique_id": "macro.dbt.array_construct", "macro_sql": "{% macro array_construct(inputs=[], data_type=api.Column.translate_type('integer')) -%}\n {{ return(adapter.dispatch('array_construct', 'dbt')(inputs, data_type)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__array_construct"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7746718, "supported_languages": null}, "macro.dbt.default__array_construct": {"name": "default__array_construct", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_construct.sql", "original_file_path": "macros/utils/array_construct.sql", "unique_id": "macro.dbt.default__array_construct", "macro_sql": "{% macro default__array_construct(inputs, data_type) -%}\n {% if inputs|length > 0 %}\n array[ {{ inputs|join(' , ') }} ]\n {% else %}\n array[]::{{data_type}}[]\n {% endif %}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.77508, "supported_languages": null}, "macro.dbt.array_append": {"name": "array_append", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_append.sql", "original_file_path": "macros/utils/array_append.sql", "unique_id": "macro.dbt.array_append", "macro_sql": "{% macro array_append(array, new_element) -%}\n {{ return(adapter.dispatch('array_append', 'dbt')(array, new_element)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__array_append"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.775543, "supported_languages": null}, "macro.dbt.default__array_append": {"name": "default__array_append", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_append.sql", "original_file_path": "macros/utils/array_append.sql", "unique_id": "macro.dbt.default__array_append", "macro_sql": "{% macro default__array_append(array, new_element) -%}\n array_append({{ array }}, {{ new_element }})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.775769, "supported_languages": null}, "macro.dbt.create_schema": {"name": "create_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.create_schema", "macro_sql": "{% macro create_schema(relation) -%}\n {{ adapter.dispatch('create_schema', 'dbt')(relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__create_schema"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.776381, "supported_languages": null}, "macro.dbt.default__create_schema": {"name": "default__create_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.default__create_schema", "macro_sql": "{% macro default__create_schema(relation) -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier() }}\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.776683, "supported_languages": null}, "macro.dbt.drop_schema": {"name": "drop_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.drop_schema", "macro_sql": "{% macro drop_schema(relation) -%}\n {{ adapter.dispatch('drop_schema', 'dbt')(relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__drop_schema"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.776957, "supported_languages": null}, "macro.dbt.default__drop_schema": {"name": "default__drop_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.default__drop_schema", "macro_sql": "{% macro default__drop_schema(relation) -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier() }} cascade\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.777257, "supported_languages": null}, "macro.dbt.current_timestamp": {"name": "current_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.current_timestamp", "macro_sql": "{%- macro current_timestamp() -%}\n {{ adapter.dispatch('current_timestamp', 'dbt')() }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_postgres.postgres__current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.778059, "supported_languages": null}, "macro.dbt.default__current_timestamp": {"name": "default__current_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__current_timestamp", "macro_sql": "{% macro default__current_timestamp() -%}\n {{ exceptions.raise_not_implemented(\n 'current_timestamp macro not implemented for adapter ' + adapter.type()) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.778315, "supported_languages": null}, "macro.dbt.snapshot_get_time": {"name": "snapshot_get_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.snapshot_get_time", "macro_sql": "\n\n{%- macro snapshot_get_time() -%}\n {{ adapter.dispatch('snapshot_get_time', 'dbt')() }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_postgres.postgres__snapshot_get_time"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.778553, "supported_languages": null}, "macro.dbt.default__snapshot_get_time": {"name": "default__snapshot_get_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__snapshot_get_time", "macro_sql": "{% macro default__snapshot_get_time() %}\n {{ current_timestamp() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7787268, "supported_languages": null}, "macro.dbt.current_timestamp_backcompat": {"name": "current_timestamp_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.current_timestamp_backcompat", "macro_sql": "{% macro current_timestamp_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_backcompat', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__current_timestamp_backcompat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7791228, "supported_languages": null}, "macro.dbt.default__current_timestamp_backcompat": {"name": "default__current_timestamp_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__current_timestamp_backcompat", "macro_sql": "{% macro default__current_timestamp_backcompat() %}\n current_timestamp::timestamp\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.779256, "supported_languages": null}, "macro.dbt.current_timestamp_in_utc_backcompat": {"name": "current_timestamp_in_utc_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.current_timestamp_in_utc_backcompat", "macro_sql": "{% macro current_timestamp_in_utc_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_in_utc_backcompat', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__current_timestamp_in_utc_backcompat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.779534, "supported_languages": null}, "macro.dbt.default__current_timestamp_in_utc_backcompat": {"name": "default__current_timestamp_in_utc_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__current_timestamp_in_utc_backcompat", "macro_sql": "{% macro default__current_timestamp_in_utc_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_backcompat', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.current_timestamp_backcompat", "macro.dbt_postgres.postgres__current_timestamp_backcompat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.779809, "supported_languages": null}, "macro.dbt.get_create_index_sql": {"name": "get_create_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.get_create_index_sql", "macro_sql": "{% macro get_create_index_sql(relation, index_dict) -%}\n {{ return(adapter.dispatch('get_create_index_sql', 'dbt')(relation, index_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_create_index_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.781184, "supported_languages": null}, "macro.dbt.default__get_create_index_sql": {"name": "default__get_create_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__get_create_index_sql", "macro_sql": "{% macro default__get_create_index_sql(relation, index_dict) -%}\n {% do return(None) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7814138, "supported_languages": null}, "macro.dbt.create_indexes": {"name": "create_indexes", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.create_indexes", "macro_sql": "{% macro create_indexes(relation) -%}\n {{ adapter.dispatch('create_indexes', 'dbt')(relation) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.781677, "supported_languages": null}, "macro.dbt.default__create_indexes": {"name": "default__create_indexes", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__create_indexes", "macro_sql": "{% macro default__create_indexes(relation) -%}\n {%- set _indexes = config.get('indexes', default=[]) -%}\n\n {% for _index_dict in _indexes %}\n {% set create_index_sql = get_create_index_sql(relation, _index_dict) %}\n {% if create_index_sql %}\n {% do run_query(create_index_sql) %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_create_index_sql", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.782335, "supported_languages": null}, "macro.dbt.get_drop_index_sql": {"name": "get_drop_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.get_drop_index_sql", "macro_sql": "{% macro get_drop_index_sql(relation, index_name) -%}\n {{ adapter.dispatch('get_drop_index_sql', 'dbt')(relation, index_name) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_drop_index_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.782665, "supported_languages": null}, "macro.dbt.default__get_drop_index_sql": {"name": "default__get_drop_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__get_drop_index_sql", "macro_sql": "{% macro default__get_drop_index_sql(relation, index_name) -%}\n {{ exceptions.raise_compiler_error(\"`get_drop_index_sql has not been implemented for this adapter.\") }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.782913, "supported_languages": null}, "macro.dbt.get_show_indexes_sql": {"name": "get_show_indexes_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.get_show_indexes_sql", "macro_sql": "{% macro get_show_indexes_sql(relation) -%}\n {{ adapter.dispatch('get_show_indexes_sql', 'dbt')(relation) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_show_indexes_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.783176, "supported_languages": null}, "macro.dbt.default__get_show_indexes_sql": {"name": "default__get_show_indexes_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__get_show_indexes_sql", "macro_sql": "{% macro default__get_show_indexes_sql(relation) -%}\n {{ exceptions.raise_compiler_error(\"`get_show_indexes_sql has not been implemented for this adapter.\") }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.783384, "supported_languages": null}, "macro.dbt.make_intermediate_relation": {"name": "make_intermediate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.make_intermediate_relation", "macro_sql": "{% macro make_intermediate_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_intermediate_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_intermediate_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7865632, "supported_languages": null}, "macro.dbt.default__make_intermediate_relation": {"name": "default__make_intermediate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__make_intermediate_relation", "macro_sql": "{% macro default__make_intermediate_relation(base_relation, suffix) %}\n {{ return(default__make_temp_relation(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__make_temp_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7868469, "supported_languages": null}, "macro.dbt.make_temp_relation": {"name": "make_temp_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.make_temp_relation", "macro_sql": "{% macro make_temp_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_temp_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_temp_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7872078, "supported_languages": null}, "macro.dbt.default__make_temp_relation": {"name": "default__make_temp_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__make_temp_relation", "macro_sql": "{% macro default__make_temp_relation(base_relation, suffix) %}\n {%- set temp_identifier = base_relation.identifier ~ suffix -%}\n {%- set temp_relation = base_relation.incorporate(\n path={\"identifier\": temp_identifier}) -%}\n\n {{ return(temp_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7876961, "supported_languages": null}, "macro.dbt.make_backup_relation": {"name": "make_backup_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.make_backup_relation", "macro_sql": "{% macro make_backup_relation(base_relation, backup_relation_type, suffix='__dbt_backup') %}\n {{ return(adapter.dispatch('make_backup_relation', 'dbt')(base_relation, backup_relation_type, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_backup_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.788102, "supported_languages": null}, "macro.dbt.default__make_backup_relation": {"name": "default__make_backup_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__make_backup_relation", "macro_sql": "{% macro default__make_backup_relation(base_relation, backup_relation_type, suffix) %}\n {%- set backup_identifier = base_relation.identifier ~ suffix -%}\n {%- set backup_relation = base_relation.incorporate(\n path={\"identifier\": backup_identifier},\n type=backup_relation_type\n ) -%}\n {{ return(backup_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.788644, "supported_languages": null}, "macro.dbt.truncate_relation": {"name": "truncate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.truncate_relation", "macro_sql": "{% macro truncate_relation(relation) -%}\n {{ return(adapter.dispatch('truncate_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__truncate_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.788949, "supported_languages": null}, "macro.dbt.default__truncate_relation": {"name": "default__truncate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__truncate_relation", "macro_sql": "{% macro default__truncate_relation(relation) -%}\n {% call statement('truncate_relation') -%}\n truncate table {{ relation }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.789593, "supported_languages": null}, "macro.dbt.get_or_create_relation": {"name": "get_or_create_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.get_or_create_relation", "macro_sql": "{% macro get_or_create_relation(database, schema, identifier, type) -%}\n {{ return(adapter.dispatch('get_or_create_relation', 'dbt')(database, schema, identifier, type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_or_create_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.790008, "supported_languages": null}, "macro.dbt.default__get_or_create_relation": {"name": "default__get_or_create_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__get_or_create_relation", "macro_sql": "{% macro default__get_or_create_relation(database, schema, identifier, type) %}\n {%- set target_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n\n {% if target_relation %}\n {% do return([true, target_relation]) %}\n {% endif %}\n\n {%- set new_relation = api.Relation.create(\n database=database,\n schema=schema,\n identifier=identifier,\n type=type\n ) -%}\n {% do return([false, new_relation]) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7909272, "supported_languages": null}, "macro.dbt.load_cached_relation": {"name": "load_cached_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.load_cached_relation", "macro_sql": "{% macro load_cached_relation(relation) %}\n {% do return(adapter.get_relation(\n database=relation.database,\n schema=relation.schema,\n identifier=relation.identifier\n )) -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.791318, "supported_languages": null}, "macro.dbt.load_relation": {"name": "load_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.load_relation", "macro_sql": "{% macro load_relation(relation) %}\n {{ return(load_cached_relation(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.791553, "supported_languages": null}, "macro.dbt.collect_freshness": {"name": "collect_freshness", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "unique_id": "macro.dbt.collect_freshness", "macro_sql": "{% macro collect_freshness(source, loaded_at_field, filter) %}\n {{ return(adapter.dispatch('collect_freshness', 'dbt')(source, loaded_at_field, filter))}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__collect_freshness"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7922392, "supported_languages": null}, "macro.dbt.default__collect_freshness": {"name": "default__collect_freshness", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "unique_id": "macro.dbt.default__collect_freshness", "macro_sql": "{% macro default__collect_freshness(source, loaded_at_field, filter) %}\n {% call statement('collect_freshness', fetch_result=True, auto_begin=False) -%}\n select\n max({{ loaded_at_field }}) as max_loaded_at,\n {{ current_timestamp() }} as snapshotted_at\n from {{ source }}\n {% if filter %}\n where {{ filter }}\n {% endif %}\n {% endcall %}\n {{ return(load_result('collect_freshness')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7929611, "supported_languages": null}, "macro.dbt.validate_sql": {"name": "validate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/validate_sql.sql", "original_file_path": "macros/adapters/validate_sql.sql", "unique_id": "macro.dbt.validate_sql", "macro_sql": "{% macro validate_sql(sql) -%}\n {{ return(adapter.dispatch('validate_sql', 'dbt')(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__validate_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.793429, "supported_languages": null}, "macro.dbt.default__validate_sql": {"name": "default__validate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/validate_sql.sql", "original_file_path": "macros/adapters/validate_sql.sql", "unique_id": "macro.dbt.default__validate_sql", "macro_sql": "{% macro default__validate_sql(sql) -%}\n {% call statement('validate_sql') -%}\n explain {{ sql }}\n {% endcall %}\n {{ return(load_result('validate_sql')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.793799, "supported_languages": null}, "macro.dbt.copy_grants": {"name": "copy_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.copy_grants", "macro_sql": "{% macro copy_grants() %}\n {{ return(adapter.dispatch('copy_grants', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__copy_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.796642, "supported_languages": null}, "macro.dbt.default__copy_grants": {"name": "default__copy_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__copy_grants", "macro_sql": "{% macro default__copy_grants() %}\n {{ return(True) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7968378, "supported_languages": null}, "macro.dbt.support_multiple_grantees_per_dcl_statement": {"name": "support_multiple_grantees_per_dcl_statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.support_multiple_grantees_per_dcl_statement", "macro_sql": "{% macro support_multiple_grantees_per_dcl_statement() %}\n {{ return(adapter.dispatch('support_multiple_grantees_per_dcl_statement', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__support_multiple_grantees_per_dcl_statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.797121, "supported_languages": null}, "macro.dbt.default__support_multiple_grantees_per_dcl_statement": {"name": "default__support_multiple_grantees_per_dcl_statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__support_multiple_grantees_per_dcl_statement", "macro_sql": "\n\n{%- macro default__support_multiple_grantees_per_dcl_statement() -%}\n {{ return(True) }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.797313, "supported_languages": null}, "macro.dbt.should_revoke": {"name": "should_revoke", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.should_revoke", "macro_sql": "{% macro should_revoke(existing_relation, full_refresh_mode=True) %}\n\n {% if not existing_relation %}\n {#-- The table doesn't already exist, so no grants to copy over --#}\n {{ return(False) }}\n {% elif full_refresh_mode %}\n {#-- The object is being REPLACED -- whether grants are copied over depends on the value of user config --#}\n {{ return(copy_grants()) }}\n {% else %}\n {#-- The table is being merged/upserted/inserted -- grants will be carried over --#}\n {{ return(True) }}\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.copy_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.797886, "supported_languages": null}, "macro.dbt.get_show_grant_sql": {"name": "get_show_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_show_grant_sql", "macro_sql": "{% macro get_show_grant_sql(relation) %}\n {{ return(adapter.dispatch(\"get_show_grant_sql\", \"dbt\")(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_show_grant_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7982059, "supported_languages": null}, "macro.dbt.default__get_show_grant_sql": {"name": "default__get_show_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_show_grant_sql", "macro_sql": "{% macro default__get_show_grant_sql(relation) %}\n show grants on {{ relation }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.798379, "supported_languages": null}, "macro.dbt.get_grant_sql": {"name": "get_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_grant_sql", "macro_sql": "{% macro get_grant_sql(relation, privilege, grantees) %}\n {{ return(adapter.dispatch('get_grant_sql', 'dbt')(relation, privilege, grantees)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_grant_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7987661, "supported_languages": null}, "macro.dbt.default__get_grant_sql": {"name": "default__get_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_grant_sql", "macro_sql": "\n\n{%- macro default__get_grant_sql(relation, privilege, grantees) -%}\n grant {{ privilege }} on {{ relation }} to {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.799092, "supported_languages": null}, "macro.dbt.get_revoke_sql": {"name": "get_revoke_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_revoke_sql", "macro_sql": "{% macro get_revoke_sql(relation, privilege, grantees) %}\n {{ return(adapter.dispatch('get_revoke_sql', 'dbt')(relation, privilege, grantees)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_revoke_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.799478, "supported_languages": null}, "macro.dbt.default__get_revoke_sql": {"name": "default__get_revoke_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_revoke_sql", "macro_sql": "\n\n{%- macro default__get_revoke_sql(relation, privilege, grantees) -%}\n revoke {{ privilege }} on {{ relation }} from {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.7999208, "supported_languages": null}, "macro.dbt.get_dcl_statement_list": {"name": "get_dcl_statement_list", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_dcl_statement_list", "macro_sql": "{% macro get_dcl_statement_list(relation, grant_config, get_dcl_macro) %}\n {{ return(adapter.dispatch('get_dcl_statement_list', 'dbt')(relation, grant_config, get_dcl_macro)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_dcl_statement_list"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.8003159, "supported_languages": null}, "macro.dbt.default__get_dcl_statement_list": {"name": "default__get_dcl_statement_list", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_dcl_statement_list", "macro_sql": "\n\n{%- macro default__get_dcl_statement_list(relation, grant_config, get_dcl_macro) -%}\n {#\n -- Unpack grant_config into specific privileges and the set of users who need them granted/revoked.\n -- Depending on whether this database supports multiple grantees per statement, pass in the list of\n -- all grantees per privilege, or (if not) template one statement per privilege-grantee pair.\n -- `get_dcl_macro` will be either `get_grant_sql` or `get_revoke_sql`\n #}\n {%- set dcl_statements = [] -%}\n {%- for privilege, grantees in grant_config.items() %}\n {%- if support_multiple_grantees_per_dcl_statement() and grantees -%}\n {%- set dcl = get_dcl_macro(relation, privilege, grantees) -%}\n {%- do dcl_statements.append(dcl) -%}\n {%- else -%}\n {%- for grantee in grantees -%}\n {% set dcl = get_dcl_macro(relation, privilege, [grantee]) %}\n {%- do dcl_statements.append(dcl) -%}\n {% endfor -%}\n {%- endif -%}\n {%- endfor -%}\n {{ return(dcl_statements) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.support_multiple_grantees_per_dcl_statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.80143, "supported_languages": null}, "macro.dbt.call_dcl_statements": {"name": "call_dcl_statements", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.call_dcl_statements", "macro_sql": "{% macro call_dcl_statements(dcl_statement_list) %}\n {{ return(adapter.dispatch(\"call_dcl_statements\", \"dbt\")(dcl_statement_list)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__call_dcl_statements"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.801748, "supported_languages": null}, "macro.dbt.default__call_dcl_statements": {"name": "default__call_dcl_statements", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__call_dcl_statements", "macro_sql": "{% macro default__call_dcl_statements(dcl_statement_list) %}\n {#\n -- By default, supply all grant + revoke statements in a single semicolon-separated block,\n -- so that they're all processed together.\n\n -- Some databases do not support this. Those adapters will need to override this macro\n -- to run each statement individually.\n #}\n {% call statement('grants') %}\n {% for dcl_statement in dcl_statement_list %}\n {{ dcl_statement }};\n {% endfor %}\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.802148, "supported_languages": null}, "macro.dbt.apply_grants": {"name": "apply_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.apply_grants", "macro_sql": "{% macro apply_grants(relation, grant_config, should_revoke) %}\n {{ return(adapter.dispatch(\"apply_grants\", \"dbt\")(relation, grant_config, should_revoke)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__apply_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.802525, "supported_languages": null}, "macro.dbt.default__apply_grants": {"name": "default__apply_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__apply_grants", "macro_sql": "{% macro default__apply_grants(relation, grant_config, should_revoke=True) %}\n {#-- If grant_config is {} or None, this is a no-op --#}\n {% if grant_config %}\n {% if should_revoke %}\n {#-- We think previous grants may have carried over --#}\n {#-- Show current grants and calculate diffs --#}\n {% set current_grants_table = run_query(get_show_grant_sql(relation)) %}\n {% set current_grants_dict = adapter.standardize_grants_dict(current_grants_table) %}\n {% set needs_granting = diff_of_two_dicts(grant_config, current_grants_dict) %}\n {% set needs_revoking = diff_of_two_dicts(current_grants_dict, grant_config) %}\n {% if not (needs_granting or needs_revoking) %}\n {{ log('On ' ~ relation ~': All grants are in place, no revocation or granting needed.')}}\n {% endif %}\n {% else %}\n {#-- We don't think there's any chance of previous grants having carried over. --#}\n {#-- Jump straight to granting what the user has configured. --#}\n {% set needs_revoking = {} %}\n {% set needs_granting = grant_config %}\n {% endif %}\n {% if needs_granting or needs_revoking %}\n {% set revoke_statement_list = get_dcl_statement_list(relation, needs_revoking, get_revoke_sql) %}\n {% set grant_statement_list = get_dcl_statement_list(relation, needs_granting, get_grant_sql) %}\n {% set dcl_statement_list = revoke_statement_list + grant_statement_list %}\n {% if dcl_statement_list %}\n {{ call_dcl_statements(dcl_statement_list) }}\n {% endif %}\n {% endif %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query", "macro.dbt.get_show_grant_sql", "macro.dbt.get_dcl_statement_list", "macro.dbt.call_dcl_statements"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.804396, "supported_languages": null}, "macro.dbt.get_show_sql": {"name": "get_show_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/show.sql", "original_file_path": "macros/adapters/show.sql", "unique_id": "macro.dbt.get_show_sql", "macro_sql": "{% macro get_show_sql(compiled_code, sql_header, limit) -%}\n {%- if sql_header -%}\n {{ sql_header }}\n {%- endif -%}\n {%- if limit is not none -%}\n {{ get_limit_subquery_sql(compiled_code, limit) }}\n {%- else -%}\n {{ compiled_code }}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_limit_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.805258, "supported_languages": null}, "macro.dbt.get_limit_subquery_sql": {"name": "get_limit_subquery_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/show.sql", "original_file_path": "macros/adapters/show.sql", "unique_id": "macro.dbt.get_limit_subquery_sql", "macro_sql": "{% macro get_limit_subquery_sql(sql, limit) %}\n {{ adapter.dispatch('get_limit_subquery_sql', 'dbt')(sql, limit) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_limit_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.805578, "supported_languages": null}, "macro.dbt.default__get_limit_subquery_sql": {"name": "default__get_limit_subquery_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/show.sql", "original_file_path": "macros/adapters/show.sql", "unique_id": "macro.dbt.default__get_limit_subquery_sql", "macro_sql": "{% macro default__get_limit_subquery_sql(sql, limit) %}\n select *\n from (\n {{ sql }}\n ) as model_limit_subq\n limit {{ limit }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.805804, "supported_languages": null}, "macro.dbt.alter_column_comment": {"name": "alter_column_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.alter_column_comment", "macro_sql": "{% macro alter_column_comment(relation, column_dict) -%}\n {{ return(adapter.dispatch('alter_column_comment', 'dbt')(relation, column_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.80692, "supported_languages": null}, "macro.dbt.default__alter_column_comment": {"name": "default__alter_column_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.default__alter_column_comment", "macro_sql": "{% macro default__alter_column_comment(relation, column_dict) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_column_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.8072052, "supported_languages": null}, "macro.dbt.alter_relation_comment": {"name": "alter_relation_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.alter_relation_comment", "macro_sql": "{% macro alter_relation_comment(relation, relation_comment) -%}\n {{ return(adapter.dispatch('alter_relation_comment', 'dbt')(relation, relation_comment)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__alter_relation_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.80755, "supported_languages": null}, "macro.dbt.default__alter_relation_comment": {"name": "default__alter_relation_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.default__alter_relation_comment", "macro_sql": "{% macro default__alter_relation_comment(relation, relation_comment) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_relation_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.807842, "supported_languages": null}, "macro.dbt.persist_docs": {"name": "persist_docs", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.persist_docs", "macro_sql": "{% macro persist_docs(relation, model, for_relation=true, for_columns=true) -%}\n {{ return(adapter.dispatch('persist_docs', 'dbt')(relation, model, for_relation, for_columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.8083148, "supported_languages": null}, "macro.dbt.default__persist_docs": {"name": "default__persist_docs", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.default__persist_docs", "macro_sql": "{% macro default__persist_docs(relation, model, for_relation, for_columns) -%}\n {% if for_relation and config.persist_relation_docs() and model.description %}\n {% do run_query(alter_relation_comment(relation, model.description)) %}\n {% endif %}\n\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {% do run_query(alter_column_comment(relation, model.columns)) %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query", "macro.dbt.alter_relation_comment", "macro.dbt.alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.80914, "supported_languages": null}, "macro.dbt.get_catalog_relations": {"name": "get_catalog_relations", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_catalog_relations", "macro_sql": "{% macro get_catalog_relations(information_schema, relations) -%}\n {{ return(adapter.dispatch('get_catalog_relations', 'dbt')(information_schema, relations)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_catalog_relations"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.813777, "supported_languages": null}, "macro.dbt.default__get_catalog_relations": {"name": "default__get_catalog_relations", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_catalog_relations", "macro_sql": "{% macro default__get_catalog_relations(information_schema, relations) -%}\n {% set typename = adapter.type() %}\n {% set msg -%}\n get_catalog_relations not implemented for {{ typename }}\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.814204, "supported_languages": null}, "macro.dbt.get_catalog": {"name": "get_catalog", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_catalog", "macro_sql": "{% macro get_catalog(information_schema, schemas) -%}\n {{ return(adapter.dispatch('get_catalog', 'dbt')(information_schema, schemas)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_catalog"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.814544, "supported_languages": null}, "macro.dbt.default__get_catalog": {"name": "default__get_catalog", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_catalog", "macro_sql": "{% macro default__get_catalog(information_schema, schemas) -%}\n\n {% set typename = adapter.type() %}\n {% set msg -%}\n get_catalog not implemented for {{ typename }}\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.815083, "supported_languages": null}, "macro.dbt.information_schema_name": {"name": "information_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.information_schema_name", "macro_sql": "{% macro information_schema_name(database) %}\n {{ return(adapter.dispatch('information_schema_name', 'dbt')(database)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__information_schema_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.815398, "supported_languages": null}, "macro.dbt.default__information_schema_name": {"name": "default__information_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__information_schema_name", "macro_sql": "{% macro default__information_schema_name(database) -%}\n {%- if database -%}\n {{ database }}.INFORMATION_SCHEMA\n {%- else -%}\n INFORMATION_SCHEMA\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.8156521, "supported_languages": null}, "macro.dbt.list_schemas": {"name": "list_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.list_schemas", "macro_sql": "{% macro list_schemas(database) -%}\n {{ return(adapter.dispatch('list_schemas', 'dbt')(database)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__list_schemas"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.8159559, "supported_languages": null}, "macro.dbt.default__list_schemas": {"name": "default__list_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__list_schemas", "macro_sql": "{% macro default__list_schemas(database) -%}\n {% set sql %}\n select distinct schema_name\n from {{ information_schema_name(database) }}.SCHEMATA\n where catalog_name ilike '{{ database }}'\n {% endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.information_schema_name", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.816355, "supported_languages": null}, "macro.dbt.check_schema_exists": {"name": "check_schema_exists", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.check_schema_exists", "macro_sql": "{% macro check_schema_exists(information_schema, schema) -%}\n {{ return(adapter.dispatch('check_schema_exists', 'dbt')(information_schema, schema)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__check_schema_exists"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.816694, "supported_languages": null}, "macro.dbt.default__check_schema_exists": {"name": "default__check_schema_exists", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__check_schema_exists", "macro_sql": "{% macro default__check_schema_exists(information_schema, schema) -%}\n {% set sql -%}\n select count(*)\n from {{ information_schema.replace(information_schema_view='SCHEMATA') }}\n where catalog_name='{{ information_schema.database }}'\n and schema_name='{{ schema }}'\n {%- endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.replace", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.817198, "supported_languages": null}, "macro.dbt.list_relations_without_caching": {"name": "list_relations_without_caching", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.list_relations_without_caching", "macro_sql": "{% macro list_relations_without_caching(schema_relation) %}\n {{ return(adapter.dispatch('list_relations_without_caching', 'dbt')(schema_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__list_relations_without_caching"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.817512, "supported_languages": null}, "macro.dbt.default__list_relations_without_caching": {"name": "default__list_relations_without_caching", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__list_relations_without_caching", "macro_sql": "{% macro default__list_relations_without_caching(schema_relation) %}\n {{ exceptions.raise_not_implemented(\n 'list_relations_without_caching macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.8177779, "supported_languages": null}, "macro.dbt.get_relations": {"name": "get_relations", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_relations", "macro_sql": "{% macro get_relations() %}\n {{ return(adapter.dispatch('get_relations', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_relations"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.818053, "supported_languages": null}, "macro.dbt.default__get_relations": {"name": "default__get_relations", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_relations", "macro_sql": "{% macro default__get_relations() %}\n {{ exceptions.raise_not_implemented(\n 'get_relations macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.818312, "supported_languages": null}, "macro.dbt.get_relation_last_modified": {"name": "get_relation_last_modified", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_relation_last_modified", "macro_sql": "{% macro get_relation_last_modified(information_schema, relations) %}\n {{ return(adapter.dispatch('get_relation_last_modified', 'dbt')(information_schema, relations)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_relation_last_modified"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.818652, "supported_languages": null}, "macro.dbt.default__get_relation_last_modified": {"name": "default__get_relation_last_modified", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_relation_last_modified", "macro_sql": "{% macro default__get_relation_last_modified(information_schema, relations) %}\n {{ exceptions.raise_not_implemented(\n 'get_relation_last_modified macro not implemented for adapter ' + adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.818933, "supported_languages": null}, "macro.dbt.get_columns_in_relation": {"name": "get_columns_in_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_columns_in_relation", "macro_sql": "{% macro get_columns_in_relation(relation) -%}\n {{ return(adapter.dispatch('get_columns_in_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_columns_in_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.822517, "supported_languages": null}, "macro.dbt.default__get_columns_in_relation": {"name": "default__get_columns_in_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_columns_in_relation", "macro_sql": "{% macro default__get_columns_in_relation(relation) -%}\n {{ exceptions.raise_not_implemented(\n 'get_columns_in_relation macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.822782, "supported_languages": null}, "macro.dbt.sql_convert_columns_in_relation": {"name": "sql_convert_columns_in_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.sql_convert_columns_in_relation", "macro_sql": "{% macro sql_convert_columns_in_relation(table) -%}\n {% set columns = [] %}\n {% for row in table %}\n {% do columns.append(api.Column(*row)) %}\n {% endfor %}\n {{ return(columns) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.82329, "supported_languages": null}, "macro.dbt.get_empty_subquery_sql": {"name": "get_empty_subquery_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_empty_subquery_sql", "macro_sql": "{% macro get_empty_subquery_sql(select_sql, select_sql_header=none) -%}\n {{ return(adapter.dispatch('get_empty_subquery_sql', 'dbt')(select_sql, select_sql_header)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_empty_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.823643, "supported_languages": null}, "macro.dbt.default__get_empty_subquery_sql": {"name": "default__get_empty_subquery_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_empty_subquery_sql", "macro_sql": "{% macro default__get_empty_subquery_sql(select_sql, select_sql_header=none) %}\n {%- if select_sql_header is not none -%}\n {{ select_sql_header }}\n {%- endif -%}\n select * from (\n {{ select_sql }}\n ) as __dbt_sbq\n where false\n limit 0\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.823992, "supported_languages": null}, "macro.dbt.get_empty_schema_sql": {"name": "get_empty_schema_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_empty_schema_sql", "macro_sql": "{% macro get_empty_schema_sql(columns) -%}\n {{ return(adapter.dispatch('get_empty_schema_sql', 'dbt')(columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_empty_schema_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.824285, "supported_languages": null}, "macro.dbt.default__get_empty_schema_sql": {"name": "default__get_empty_schema_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_empty_schema_sql", "macro_sql": "{% macro default__get_empty_schema_sql(columns) %}\n {%- set col_err = [] -%}\n {%- set col_naked_numeric = [] -%}\n select\n {% for i in columns %}\n {%- set col = columns[i] -%}\n {%- if col['data_type'] is not defined -%}\n {%- do col_err.append(col['name']) -%}\n {#-- If this column's type is just 'numeric' then it is missing precision/scale, raise a warning --#}\n {%- elif col['data_type'].strip().lower() in ('numeric', 'decimal', 'number') -%}\n {%- do col_naked_numeric.append(col['name']) -%}\n {%- endif -%}\n {% set col_name = adapter.quote(col['name']) if col.get('quote') else col['name'] %}\n cast(null as {{ col['data_type'] }}) as {{ col_name }}{{ \", \" if not loop.last }}\n {%- endfor -%}\n {%- if (col_err | length) > 0 -%}\n {{ exceptions.column_type_missing(column_names=col_err) }}\n {%- elif (col_naked_numeric | length) > 0 -%}\n {{ exceptions.warn(\"Detected columns with numeric type and unspecified precision/scale, this can lead to unintended rounding: \" ~ col_naked_numeric ~ \"`\") }}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.8263, "supported_languages": null}, "macro.dbt.get_column_schema_from_query": {"name": "get_column_schema_from_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_column_schema_from_query", "macro_sql": "{% macro get_column_schema_from_query(select_sql, select_sql_header=none) -%}\n {% set columns = [] %}\n {# -- Using an 'empty subquery' here to get the same schema as the given select_sql statement, without necessitating a data scan.#}\n {% set sql = get_empty_subquery_sql(select_sql, select_sql_header) %}\n {% set column_schema = adapter.get_column_schema_from_query(sql) %}\n {{ return(column_schema) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_empty_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.826881, "supported_languages": null}, "macro.dbt.get_columns_in_query": {"name": "get_columns_in_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_columns_in_query", "macro_sql": "{% macro get_columns_in_query(select_sql) -%}\n {{ return(adapter.dispatch('get_columns_in_query', 'dbt')(select_sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_columns_in_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.8271859, "supported_languages": null}, "macro.dbt.default__get_columns_in_query": {"name": "default__get_columns_in_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_columns_in_query", "macro_sql": "{% macro default__get_columns_in_query(select_sql) %}\n {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%}\n {{ get_empty_subquery_sql(select_sql) }}\n {% endcall %}\n {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.get_empty_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.827768, "supported_languages": null}, "macro.dbt.alter_column_type": {"name": "alter_column_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.alter_column_type", "macro_sql": "{% macro alter_column_type(relation, column_name, new_column_type) -%}\n {{ return(adapter.dispatch('alter_column_type', 'dbt')(relation, column_name, new_column_type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__alter_column_type"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.8281548, "supported_languages": null}, "macro.dbt.default__alter_column_type": {"name": "default__alter_column_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__alter_column_type", "macro_sql": "{% macro default__alter_column_type(relation, column_name, new_column_type) -%}\n {#\n 1. Create a new column (w/ temp name and correct type)\n 2. Copy data over to it\n 3. Drop the existing column (cascade!)\n 4. Rename the new column to existing column\n #}\n {%- set tmp_column = column_name + \"__dbt_alter\" -%}\n\n {% call statement('alter_column_type') %}\n alter table {{ relation }} add column {{ adapter.quote(tmp_column) }} {{ new_column_type }};\n update {{ relation }} set {{ adapter.quote(tmp_column) }} = {{ adapter.quote(column_name) }};\n alter table {{ relation }} drop column {{ adapter.quote(column_name) }} cascade;\n alter table {{ relation }} rename column {{ adapter.quote(tmp_column) }} to {{ adapter.quote(column_name) }}\n {% endcall %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.829144, "supported_languages": null}, "macro.dbt.alter_relation_add_remove_columns": {"name": "alter_relation_add_remove_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.alter_relation_add_remove_columns", "macro_sql": "{% macro alter_relation_add_remove_columns(relation, add_columns = none, remove_columns = none) -%}\n {{ return(adapter.dispatch('alter_relation_add_remove_columns', 'dbt')(relation, add_columns, remove_columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__alter_relation_add_remove_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.8295648, "supported_languages": null}, "macro.dbt.default__alter_relation_add_remove_columns": {"name": "default__alter_relation_add_remove_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__alter_relation_add_remove_columns", "macro_sql": "{% macro default__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}\n\n {% if add_columns is none %}\n {% set add_columns = [] %}\n {% endif %}\n {% if remove_columns is none %}\n {% set remove_columns = [] %}\n {% endif %}\n\n {% set sql -%}\n\n alter {{ relation.type }} {{ relation }}\n\n {% for column in add_columns %}\n add column {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }}\n {% endfor %}{{ ',' if add_columns and remove_columns }}\n\n {% for column in remove_columns %}\n drop column {{ column.name }}{{ ',' if not loop.last }}\n {% endfor %}\n\n {%- endset -%}\n\n {% do run_query(sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.8308558, "supported_languages": null}, "macro.dbt.resolve_model_name": {"name": "resolve_model_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.resolve_model_name", "macro_sql": "{% macro resolve_model_name(input_model_name) %}\n {{ return(adapter.dispatch('resolve_model_name', 'dbt')(input_model_name)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__resolve_model_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.833535, "supported_languages": null}, "macro.dbt.default__resolve_model_name": {"name": "default__resolve_model_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.default__resolve_model_name", "macro_sql": "\n\n{%- macro default__resolve_model_name(input_model_name) -%}\n {{ input_model_name | string | replace('\"', '\\\"') }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.833805, "supported_languages": null}, "macro.dbt.build_ref_function": {"name": "build_ref_function", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.build_ref_function", "macro_sql": "{% macro build_ref_function(model) %}\n\n {%- set ref_dict = {} -%}\n {%- for _ref in model.refs -%}\n {% set _ref_args = [_ref.get('package'), _ref['name']] if _ref.get('package') else [_ref['name'],] %}\n {%- set resolved = ref(*_ref_args, v=_ref.get('version')) -%}\n {%- if _ref.get('version') -%}\n {% do _ref_args.extend([\"v\" ~ _ref['version']]) %}\n {%- endif -%}\n {%- do ref_dict.update({_ref_args | join('.'): resolve_model_name(resolved)}) -%}\n {%- endfor -%}\n\ndef ref(*args, **kwargs):\n refs = {{ ref_dict | tojson }}\n key = '.'.join(args)\n version = kwargs.get(\"v\") or kwargs.get(\"version\")\n if version:\n key += f\".v{version}\"\n dbt_load_df_function = kwargs.get(\"dbt_load_df_function\")\n return dbt_load_df_function(refs[key])\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.resolve_model_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.835222, "supported_languages": null}, "macro.dbt.build_source_function": {"name": "build_source_function", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.build_source_function", "macro_sql": "{% macro build_source_function(model) %}\n\n {%- set source_dict = {} -%}\n {%- for _source in model.sources -%}\n {%- set resolved = source(*_source) -%}\n {%- do source_dict.update({_source | join('.'): resolve_model_name(resolved)}) -%}\n {%- endfor -%}\n\ndef source(*args, dbt_load_df_function):\n sources = {{ source_dict | tojson }}\n key = '.'.join(args)\n return dbt_load_df_function(sources[key])\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.resolve_model_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.835905, "supported_languages": null}, "macro.dbt.build_config_dict": {"name": "build_config_dict", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.build_config_dict", "macro_sql": "{% macro build_config_dict(model) %}\n {%- set config_dict = {} -%}\n {% set config_dbt_used = zip(model.config.config_keys_used, model.config.config_keys_defaults) | list %}\n {%- for key, default in config_dbt_used -%}\n {# weird type testing with enum, would be much easier to write this logic in Python! #}\n {%- if key == \"language\" -%}\n {%- set value = \"python\" -%}\n {%- endif -%}\n {%- set value = model.config.get(key, default) -%}\n {%- do config_dict.update({key: value}) -%}\n {%- endfor -%}\nconfig_dict = {{ config_dict }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.836863, "supported_languages": null}, "macro.dbt.py_script_postfix": {"name": "py_script_postfix", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.py_script_postfix", "macro_sql": "{% macro py_script_postfix(model) %}\n# This part is user provided model code\n# you will need to copy the next section to run the code\n# COMMAND ----------\n# this part is dbt logic for get ref work, do not modify\n\n{{ build_ref_function(model ) }}\n{{ build_source_function(model ) }}\n{{ build_config_dict(model) }}\n\nclass config:\n def __init__(self, *args, **kwargs):\n pass\n\n @staticmethod\n def get(key, default=None):\n return config_dict.get(key, default)\n\nclass this:\n \"\"\"dbt.this() or dbt.this.identifier\"\"\"\n database = \"{{ this.database }}\"\n schema = \"{{ this.schema }}\"\n identifier = \"{{ this.identifier }}\"\n {% set this_relation_name = resolve_model_name(this) %}\n def __repr__(self):\n return '{{ this_relation_name }}'\n\n\nclass dbtObj:\n def __init__(self, load_df_function) -> None:\n self.source = lambda *args: source(*args, dbt_load_df_function=load_df_function)\n self.ref = lambda *args, **kwargs: ref(*args, **kwargs, dbt_load_df_function=load_df_function)\n self.config = config\n self.this = this()\n self.is_incremental = {{ is_incremental() }}\n\n# COMMAND ----------\n{{py_script_comment()}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.build_ref_function", "macro.dbt.build_source_function", "macro.dbt.build_config_dict", "macro.dbt.resolve_model_name", "macro.dbt.is_incremental", "macro.dbt.py_script_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.8376498, "supported_languages": null}, "macro.dbt.py_script_comment": {"name": "py_script_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.py_script_comment", "macro_sql": "{%macro py_script_comment()%}\n{%endmacro%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.837773, "supported_languages": null}, "macro.dbt.test_unique": {"name": "test_unique", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_unique", "macro_sql": "{% test unique(model, column_name) %}\n {% set macro = adapter.dispatch('test_unique', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_unique"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.838603, "supported_languages": null}, "macro.dbt.test_not_null": {"name": "test_not_null", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_not_null", "macro_sql": "{% test not_null(model, column_name) %}\n {% set macro = adapter.dispatch('test_not_null', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_not_null"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.8389928, "supported_languages": null}, "macro.dbt.test_accepted_values": {"name": "test_accepted_values", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_accepted_values", "macro_sql": "{% test accepted_values(model, column_name, values, quote=True) %}\n {% set macro = adapter.dispatch('test_accepted_values', 'dbt') %}\n {{ macro(model, column_name, values, quote) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_accepted_values"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.839478, "supported_languages": null}, "macro.dbt.test_relationships": {"name": "test_relationships", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_relationships", "macro_sql": "{% test relationships(model, column_name, to, field) %}\n {% set macro = adapter.dispatch('test_relationships', 'dbt') %}\n {{ macro(model, column_name, to, field) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_relationships"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1706614034.839941, "supported_languages": null}}, "docs": {"doc.test.somedoc": {"name": "somedoc", "resource_type": "doc", "package_name": "test", "path": "somedoc.md", "original_file_path": "models/somedoc.md", "unique_id": "doc.test.somedoc", "block_contents": "Testing, testing"}, "doc.dbt.__overview__": {"name": "__overview__", "resource_type": "doc", "package_name": "dbt", "path": "overview.md", "original_file_path": "docs/overview.md", "unique_id": "doc.dbt.__overview__", "block_contents": "### Welcome!\n\nWelcome to the auto-generated documentation for your dbt project!\n\n### Navigation\n\nYou can use the `Project` and `Database` navigation tabs on the left side of the window to explore the models\nin your project.\n\n#### Project Tab\nThe `Project` tab mirrors the directory structure of your dbt project. In this tab, you can see all of the\nmodels defined in your dbt project, as well as models imported from dbt packages.\n\n#### Database Tab\nThe `Database` tab also exposes your models, but in a format that looks more like a database explorer. This view\nshows relations (tables and views) grouped into database schemas. Note that ephemeral models are _not_ shown\nin this interface, as they do not exist in the database.\n\n### Graph Exploration\nYou can click the blue icon on the bottom-right corner of the page to view the lineage graph of your models.\n\nOn model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the `Expand`\nbutton at the top-right of this lineage pane, you'll be able to see all of the models that are used to build,\nor are built from, the model you're exploring.\n\nOnce expanded, you'll be able to use the `--select` and `--exclude` model selection syntax to filter the\nmodels in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax).\n\nNote that you can also right-click on models to interactively filter and explore the graph.\n\n---\n\n### More information\n\n- [What is dbt](https://docs.getdbt.com/docs/introduction)?\n- Read the [dbt viewpoint](https://docs.getdbt.com/docs/viewpoint)\n- [Installation](https://docs.getdbt.com/docs/installation)\n- Join the [dbt Community](https://www.getdbt.com/community/) for questions and discussion"}}, "exposures": {"exposure.test.simple_exposure": {"name": "simple_exposure", "resource_type": "exposure", "package_name": "test", "path": "schema.yml", "original_file_path": "models/schema.yml", "unique_id": "exposure.test.simple_exposure", "fqn": ["test", "simple_exposure"], "type": "dashboard", "owner": {"email": "something@example.com", "name": null}, "description": "", "label": null, "maturity": null, "meta": {}, "tags": [], "config": {"enabled": true}, "unrendered_config": {}, "url": null, "depends_on": {"macros": [], "nodes": ["source.test.my_source.my_table", "model.test.my_model"]}, "refs": [{"name": "my_model", "package": null, "version": null}], "sources": [["my_source", "my_table"]], "metrics": [], "created_at": 1706614035.762317}}, "metrics": {"metric.test.blue_customers_post_2010": {"name": "blue_customers_post_2010", "resource_type": "metric", "package_name": "test", "path": "schema.yml", "original_file_path": "models/schema.yml", "unique_id": "metric.test.blue_customers_post_2010", "fqn": ["test", "blue_customers_post_2010"], "description": "", "label": "Blue Customers since 2010", "type": "simple", "type_params": {"measure": {"name": "customers", "filter": {"where_filters": [{"where_sql_template": "{{ Dimension('id__favorite_color') }} = 'blue'"}]}, "alias": null, "join_to_timespine": false, "fill_nulls_with": null}, "input_measures": [{"name": "customers", "filter": {"where_filters": [{"where_sql_template": "{{ Dimension('id__favorite_color') }} = 'blue'"}]}, "alias": null, "join_to_timespine": false, "fill_nulls_with": null}], "numerator": null, "denominator": null, "expr": null, "window": null, "grain_to_date": null, "metrics": [], "conversion_type_params": null}, "filter": {"where_filters": [{"where_sql_template": "{{ TimeDimension('id__created_at', 'day') }} > '2010-01-01'"}]}, "metadata": null, "meta": {}, "tags": [], "config": {"enabled": true, "group": null}, "unrendered_config": {}, "sources": [], "depends_on": {"macros": [], "nodes": ["semantic_model.test.semantic_people"]}, "refs": [], "metrics": [], "created_at": 1706614035.8904068, "group": null}, "metric.test.customers": {"name": "customers", "resource_type": "metric", "package_name": "test", "path": "schema.yml", "original_file_path": "models/schema.yml", "unique_id": "metric.test.customers", "fqn": ["test", "customers"], "description": "", "label": "Customers Metric", "type": "simple", "type_params": {"measure": {"name": "customers", "filter": null, "alias": null, "join_to_timespine": false, "fill_nulls_with": null}, "input_measures": [{"name": "customers", "filter": null, "alias": null, "join_to_timespine": false, "fill_nulls_with": null}], "numerator": null, "denominator": null, "expr": null, "window": null, "grain_to_date": null, "metrics": [], "conversion_type_params": null}, "filter": null, "metadata": null, "meta": {}, "tags": [], "config": {"enabled": true, "group": null}, "unrendered_config": {}, "sources": [], "depends_on": {"macros": [], "nodes": ["semantic_model.test.semantic_people"]}, "refs": [], "metrics": [], "created_at": 1706614035.891094, "group": null}, "metric.test.ratio_of_blue_customers_to_red_customers": {"name": "ratio_of_blue_customers_to_red_customers", "resource_type": "metric", "package_name": "test", "path": "schema.yml", "original_file_path": "models/schema.yml", "unique_id": "metric.test.ratio_of_blue_customers_to_red_customers", "fqn": ["test", "ratio_of_blue_customers_to_red_customers"], "description": "", "label": "Very Important Customer Color Ratio", "type": "ratio", "type_params": {"measure": null, "input_measures": [{"name": "customers", "filter": null, "alias": null, "join_to_timespine": false, "fill_nulls_with": null}, {"name": "customers", "filter": null, "alias": null, "join_to_timespine": false, "fill_nulls_with": null}], "numerator": {"name": "customers", "filter": {"where_filters": [{"where_sql_template": "{{ Dimension('id__favorite_color')}} = 'blue'"}]}, "alias": null, "offset_window": null, "offset_to_grain": null}, "denominator": {"name": "customers", "filter": {"where_filters": [{"where_sql_template": "{{ Dimension('id__favorite_color')}} = 'red'"}]}, "alias": null, "offset_window": null, "offset_to_grain": null}, "expr": null, "window": null, "grain_to_date": null, "metrics": [], "conversion_type_params": null}, "filter": null, "metadata": null, "meta": {}, "tags": [], "config": {"enabled": true, "group": null}, "unrendered_config": {}, "sources": [], "depends_on": {"macros": [], "nodes": ["metric.test.customers"]}, "refs": [], "metrics": [], "created_at": 1706614035.892986, "group": null}, "metric.test.doubled_blue_customers": {"name": "doubled_blue_customers", "resource_type": "metric", "package_name": "test", "path": "schema.yml", "original_file_path": "models/schema.yml", "unique_id": "metric.test.doubled_blue_customers", "fqn": ["test", "doubled_blue_customers"], "description": "", "label": "Inflated blue customer numbers", "type": "derived", "type_params": {"measure": null, "input_measures": [{"name": "customers", "filter": null, "alias": null, "join_to_timespine": false, "fill_nulls_with": null}], "numerator": null, "denominator": null, "expr": "customers * 2", "window": null, "grain_to_date": null, "metrics": [{"name": "customers", "filter": {"where_filters": [{"where_sql_template": "{{ Dimension('id__favorite_color')}} = 'blue'"}]}, "alias": null, "offset_window": null, "offset_to_grain": null}], "conversion_type_params": null}, "filter": null, "metadata": null, "meta": {}, "tags": [], "config": {"enabled": true, "group": null}, "unrendered_config": {}, "sources": [], "depends_on": {"macros": [], "nodes": ["metric.test.customers"]}, "refs": [], "metrics": [], "created_at": 1706614035.894361, "group": null}}, "groups": {}, "selectors": {}, "disabled": {"model.test.disabled_model": [{"database": "dbt", "schema": "test17066140326720908695_test_previous_version_state", "name": "disabled_model", "resource_type": "model", "package_name": "test", "path": "disabled_model.sql", "original_file_path": "models/disabled_model.sql", "unique_id": "model.test.disabled_model", "fqn": ["test", "disabled_model"], "alias": "disabled_model", "checksum": {"name": "sha256", "checksum": "597106d23ce34e3cd2430588e5c1cf474ebdd138fc47e09b925a4ab258a27acc"}, "config": {"enabled": false, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "access": "protected"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {"enabled": false}, "created_at": 1706614035.24477, "config_call_dict": {"enabled": false}, "relation_name": "\"dbt\".\"test17066140326720908695_test_previous_version_state\".\"disabled_model\"", "raw_code": "{{ config(enabled=False) }}\nselect 2 as id", "language": "sql", "refs": [], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null, "defer_relation": null}], "snapshot.test.disabled_snapshot_seed": [{"database": "dbt", "schema": "test17066140326720908695_test_previous_version_state", "name": "disabled_snapshot_seed", "resource_type": "snapshot", "package_name": "test", "path": "disabled_snapshot_seed.sql", "original_file_path": "snapshots/disabled_snapshot_seed.sql", "unique_id": "snapshot.test.disabled_snapshot_seed", "fqn": ["test", "disabled_snapshot_seed", "disabled_snapshot_seed"], "alias": "disabled_snapshot_seed", "checksum": {"name": "sha256", "checksum": "fe76c9dd437341c9e82a0f2a8baf3148f961b768eaa0a4410cd27d3c071bd617"}, "config": {"enabled": false, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "snapshot", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": "id", "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "strategy": "check", "target_schema": "test17066140326720908695_test_previous_version_state", "target_database": null, "updated_at": null, "check_cols": "all"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {"unique_key": "id", "strategy": "check", "check_cols": "all", "target_schema": "test17066140326720908695_test_previous_version_state", "enabled": false}, "created_at": 1706614035.3636532, "config_call_dict": {"unique_key": "id", "strategy": "check", "check_cols": "all", "target_schema": "test17066140326720908695_test_previous_version_state", "enabled": false}, "relation_name": "\"dbt\".\"test17066140326720908695_test_previous_version_state\".\"disabled_snapshot_seed\"", "raw_code": "\n{{\n config(\n unique_key='id',\n strategy='check',\n check_cols='all',\n target_schema=schema,\n enabled=False,\n )\n}}\nselect * from {{ ref('my_seed') }}\n", "language": "sql", "refs": [{"name": "my_seed", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "defer_relation": null}], "analysis.test.disabled_al": [{"database": "dbt", "schema": "test17066140326720908695_test_previous_version_state", "name": "disabled_al", "resource_type": "analysis", "package_name": "test", "path": "analysis/disabled_al.sql", "original_file_path": "analyses/disabled_al.sql", "unique_id": "analysis.test.disabled_al", "fqn": ["test", "analysis", "disabled_al"], "alias": "disabled_al", "checksum": {"name": "sha256", "checksum": "32d36ad6cff0786eb562440ba60ef6c9b9a7f4c282dfb7a52eaf19d36370f0e1"}, "config": {"enabled": false, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {"enabled": false}, "created_at": 1706614035.422348, "config_call_dict": {"enabled": false}, "relation_name": null, "raw_code": "{{ config(enabled=False) }}\nselect 9 as id", "language": "sql", "refs": [], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}}], "test.test.disabled_just_my": [{"database": "dbt", "schema": "test17066140326720908695_test_previous_version_state_dbt_test__audit", "name": "disabled_just_my", "resource_type": "test", "package_name": "test", "path": "disabled_just_my.sql", "original_file_path": "tests/disabled_just_my.sql", "unique_id": "test.test.disabled_just_my", "fqn": ["test", "disabled_just_my"], "alias": "disabled_just_my", "checksum": {"name": "sha256", "checksum": "4f2268fd89a3b4ef899264ada6d7aa33603671cbc5d5acead7dc2eadf1add985"}, "config": {"enabled": false, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {"enabled": false}, "created_at": 1706614035.489865, "config_call_dict": {"enabled": false}, "relation_name": null, "raw_code": "{{ config(enabled=False) }}\n\nselect * from {{ ref('my_model') }}\nwhere false", "language": "sql", "refs": [{"name": "my_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}}], "test.test.disabled_check_nothing_my_model_.f2c6a72d37": [{"test_metadata": {"name": "disabled_check_nothing", "kwargs": {"model": "{{ get_where_subquery(ref('my_model')) }}"}, "namespace": null}, "database": "dbt", "schema": "test17066140326720908695_test_previous_version_state_dbt_test__audit", "name": "disabled_check_nothing_my_model_", "resource_type": "test", "package_name": "test", "path": "disabled_check_nothing_my_model_.sql", "original_file_path": "models/schema.yml", "unique_id": "test.test.disabled_check_nothing_my_model_.f2c6a72d37", "fqn": ["test", "disabled_check_nothing_my_model_"], "alias": "disabled_check_nothing_my_model_", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": false, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {"enabled": false}, "created_at": 1706614035.6771069, "config_call_dict": {"enabled": false}, "relation_name": null, "raw_code": "{{ test_disabled_check_nothing(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "my_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.test.test_disabled_check_nothing", "macro.dbt.get_where_subquery"], "nodes": []}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": null, "file_key_name": "models.my_model", "attached_node": "model.test.my_model"}], "exposure.test.disabled_exposure": [{"name": "disabled_exposure", "resource_type": "exposure", "package_name": "test", "path": "schema.yml", "original_file_path": "models/schema.yml", "unique_id": "exposure.test.disabled_exposure", "fqn": ["test", "disabled_exposure"], "type": "dashboard", "owner": {"email": "something@example.com", "name": null}, "description": "", "label": null, "maturity": null, "meta": {}, "tags": [], "config": {"enabled": false}, "unrendered_config": {"enabled": false}, "url": null, "depends_on": {"macros": [], "nodes": []}, "refs": [{"name": "my_model", "package": null, "version": null}], "sources": [], "metrics": [], "created_at": 1706614035.763594}], "metric.test.disabled_metric": [{"name": "disabled_metric", "resource_type": "metric", "package_name": "test", "path": "schema.yml", "original_file_path": "models/schema.yml", "unique_id": "metric.test.disabled_metric", "fqn": ["test", "disabled_metric"], "description": "", "label": "Count records", "type": "simple", "type_params": {"measure": {"name": "customers", "filter": null, "alias": null, "join_to_timespine": false, "fill_nulls_with": null}, "input_measures": [], "numerator": null, "denominator": null, "expr": null, "window": null, "grain_to_date": null, "metrics": [], "conversion_type_params": null}, "filter": {"where_filters": [{"where_sql_template": "{{ Dimension('id__favorite_color') }} = 'blue'"}]}, "metadata": null, "meta": {}, "tags": [], "config": {"enabled": false, "group": null}, "unrendered_config": {"enabled": false}, "sources": [], "depends_on": {"macros": [], "nodes": []}, "refs": [], "metrics": [], "created_at": 1706614035.891953, "group": null}], "seed.test.disabled_seed": [{"database": "dbt", "schema": "test17066140326720908695_test_previous_version_state", "name": "disabled_seed", "resource_type": "seed", "package_name": "test", "path": "disabled_seed.csv", "original_file_path": "seeds/disabled_seed.csv", "unique_id": "seed.test.disabled_seed", "fqn": ["test", "disabled_seed"], "alias": "disabled_seed", "checksum": {"name": "sha256", "checksum": "31fddd8ec40c6aba6a3a8e7d83fedea2fd0a56c47b64ea3df1847ec1b018e2d1"}, "config": {"enabled": false, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "seed", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "delimiter": ",", "quote_columns": null}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": "test://models/schema.yml", "build_path": null, "deferred": false, "unrendered_config": {"enabled": false}, "created_at": 1706614035.695063, "config_call_dict": {}, "relation_name": "\"dbt\".\"test17066140326720908695_test_previous_version_state\".\"disabled_seed\"", "raw_code": "", "root_path": "/private/var/folders/7h/hj5_fw9j291c58hwfdvy5xbm0000gp/T/pytest-of-jerco/pytest-10/project0", "depends_on": {"macros": []}, "defer_relation": null}], "source.test.my_source.disabled_table": [{"database": "dbt", "schema": "my_source", "name": "disabled_table", "resource_type": "source", "package_name": "test", "path": "models/schema.yml", "original_file_path": "models/schema.yml", "unique_id": "source.test.my_source.disabled_table", "fqn": ["test", "my_source", "disabled_table"], "source_name": "my_source", "source_description": "My source", "loader": "a_loader", "identifier": "disabled_table", "quoting": {"database": null, "schema": null, "identifier": null, "column": null}, "loaded_at_field": null, "freshness": {"warn_after": {"count": null, "period": null}, "error_after": {"count": null, "period": null}, "filter": null}, "external": null, "description": "Disabled table", "columns": {}, "meta": {}, "source_meta": {}, "tags": [], "config": {"enabled": false}, "patch_path": null, "unrendered_config": {"enabled": false}, "relation_name": "\"dbt\".\"my_source\".\"disabled_table\"", "created_at": 1706614035.93874}]}, "parent_map": {"model.test.my_model": [], "model.test.metricflow_time_spine": [], "snapshot.test.snapshot_seed": ["seed.test.my_seed"], "analysis.test.a": [], "test.test.just_my": ["model.test.my_model"], "seed.test.my_seed": [], "test.test.not_null_my_model_id.43e0e9183a": ["model.test.my_model"], "test.test.check_nothing_my_model_.d5a5e66110": ["model.test.my_model"], "source.test.my_source.my_table": [], "exposure.test.simple_exposure": ["model.test.my_model", "source.test.my_source.my_table"], "metric.test.blue_customers_post_2010": ["semantic_model.test.semantic_people"], "metric.test.customers": ["semantic_model.test.semantic_people"], "metric.test.ratio_of_blue_customers_to_red_customers": ["metric.test.customers"], "metric.test.doubled_blue_customers": ["metric.test.customers"], "semantic_model.test.semantic_people": ["model.test.my_model"]}, "child_map": {"model.test.my_model": ["exposure.test.simple_exposure", "semantic_model.test.semantic_people", "test.test.check_nothing_my_model_.d5a5e66110", "test.test.just_my", "test.test.not_null_my_model_id.43e0e9183a"], "model.test.metricflow_time_spine": [], "snapshot.test.snapshot_seed": [], "analysis.test.a": [], "test.test.just_my": [], "seed.test.my_seed": ["snapshot.test.snapshot_seed"], "test.test.not_null_my_model_id.43e0e9183a": [], "test.test.check_nothing_my_model_.d5a5e66110": [], "source.test.my_source.my_table": ["exposure.test.simple_exposure"], "exposure.test.simple_exposure": [], "metric.test.blue_customers_post_2010": [], "metric.test.customers": ["metric.test.doubled_blue_customers", "metric.test.ratio_of_blue_customers_to_red_customers"], "metric.test.ratio_of_blue_customers_to_red_customers": [], "metric.test.doubled_blue_customers": [], "semantic_model.test.semantic_people": ["metric.test.blue_customers_post_2010", "metric.test.customers"]}, "group_map": {}, "saved_queries": {}, "semantic_models": {"semantic_model.test.semantic_people": {"name": "semantic_people", "resource_type": "semantic_model", "package_name": "test", "path": "schema.yml", "original_file_path": "models/schema.yml", "unique_id": "semantic_model.test.semantic_people", "fqn": ["test", "semantic_people"], "model": "ref('my_model')", "node_relation": {"alias": "my_model", "schema_name": "test17066140326720908695_test_previous_version_state", "database": "dbt", "relation_name": "\"dbt\".\"test17066140326720908695_test_previous_version_state\".\"my_model\""}, "description": null, "label": null, "defaults": {"agg_time_dimension": "created_at"}, "entities": [{"name": "id", "type": "primary", "description": null, "label": null, "role": null, "expr": null}], "measures": [{"name": "years_tenure", "agg": "sum", "description": null, "label": null, "create_metric": false, "expr": "tenure", "agg_params": null, "non_additive_dimension": null, "agg_time_dimension": null}, {"name": "people", "agg": "count", "description": null, "label": null, "create_metric": false, "expr": "id", "agg_params": null, "non_additive_dimension": null, "agg_time_dimension": null}, {"name": "customers", "agg": "count", "description": null, "label": null, "create_metric": false, "expr": "id", "agg_params": null, "non_additive_dimension": null, "agg_time_dimension": null}], "dimensions": [{"name": "favorite_color", "type": "categorical", "description": null, "label": null, "is_partition": false, "type_params": null, "expr": null, "metadata": null}, {"name": "created_at", "type": "time", "description": null, "label": null, "is_partition": false, "type_params": {"time_granularity": "day", "validity_params": null}, "expr": null, "metadata": null}], "metadata": null, "depends_on": {"macros": [], "nodes": ["model.test.my_model"]}, "refs": [{"name": "my_model", "package": null, "version": null}], "created_at": 1706614035.935194, "config": {"enabled": true, "group": null, "meta": {}}, "unrendered_config": {}, "primary_entity": null, "group": null}}} ================================================ FILE: tests/functional/artifacts/data/state/v12/manifest.json ================================================ {"metadata": {"dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v12.json", "dbt_version": "1.11.0a1", "generated_at": "2025-10-02T16:48:38.578317Z", "invocation_id": "44bddff0-d7eb-4be7-8e0c-f4e1709ae7fe", "invocation_started_at": "2025-10-02T16:48:35.875974+00:00", "env": {}, "project_name": "test", "project_id": "098f6bcd4621d373cade4e832627b4f6", "user_id": null, "send_anonymous_usage_stats": false, "adapter_type": "postgres", "quoting": {"database": true, "schema": true, "identifier": true, "column": null}, "run_started_at": "2025-10-02T16:48:35.876320+00:00"}, "nodes": {"model.test.my_model": {"database": "dbt", "schema": "test17594237163467600387_test_previous_version_state", "name": "my_model", "resource_type": "model", "package_name": "test", "path": "my_model.sql", "original_file_path": "models/my_model.sql", "unique_id": "model.test.my_model", "fqn": ["test", "my_model"], "alias": "my_model", "checksum": {"name": "sha256", "checksum": "3ea0f972fa1b56aa2dc2f56ee784b6a5796312f9a813d59ae70fd8855f10d16d"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "batch_size": null, "lookback": 1, "begin": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "event_time": null, "concurrent_batches": null, "access": "protected", "freshness": null}, "tags": [], "description": "Example model", "columns": {"id": {"name": "id", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "config": {"meta": {}, "tags": []}, "tags": [], "granularity": null, "doc_blocks": []}}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": "test://models/schema.yml", "build_path": null, "unrendered_config": {}, "created_at": 1759423716.418234, "relation_name": "\"dbt\".\"test17594237163467600387_test_previous_version_state\".\"my_model\"", "raw_code": "select 1 as id", "doc_blocks": [], "language": "sql", "refs": [], "sources": [], "metrics": [], "functions": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null, "primary_key": [], "time_spine": null}, "model.test.metricflow_time_spine": {"database": "dbt", "schema": "test17594237163467600387_test_previous_version_state", "name": "metricflow_time_spine", "resource_type": "model", "package_name": "test", "path": "metricflow_time_spine.sql", "original_file_path": "models/metricflow_time_spine.sql", "unique_id": "model.test.metricflow_time_spine", "fqn": ["test", "metricflow_time_spine"], "alias": "metricflow_time_spine", "checksum": {"name": "sha256", "checksum": "954d9b349821edb5558a373119a7d91eeac9e620aaa96cd112c0d14bab729fdb"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "batch_size": null, "lookback": 1, "begin": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "event_time": null, "concurrent_batches": null, "access": "protected", "freshness": null}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1759423716.101295, "relation_name": "\"dbt\".\"test17594237163467600387_test_previous_version_state\".\"metricflow_time_spine\"", "raw_code": "SELECT to_date('02/20/2023', 'mm/dd/yyyy') as date_day", "doc_blocks": [], "language": "sql", "refs": [], "sources": [], "metrics": [], "functions": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null, "primary_key": [], "time_spine": null}, "snapshot.test.snapshot_seed": {"database": "dbt", "schema": "test17594237163467600387_test_previous_version_state", "name": "snapshot_seed", "resource_type": "snapshot", "package_name": "test", "path": "snapshot_seed.sql", "original_file_path": "snapshots/snapshot_seed.sql", "unique_id": "snapshot.test.snapshot_seed", "fqn": ["test", "snapshot_seed", "snapshot_seed"], "alias": "snapshot_seed", "checksum": {"name": "sha256", "checksum": "5fc998f39655f8fe52443a919e749b6e23883ef90202b040412baac13c6bfe18"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "snapshot", "incremental_strategy": null, "batch_size": null, "lookback": 1, "begin": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": "id", "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "event_time": null, "concurrent_batches": null, "strategy": "check", "target_schema": "test17594237163467600387_test_previous_version_state", "target_database": null, "updated_at": null, "check_cols": "all", "snapshot_meta_column_names": {"dbt_valid_to": null, "dbt_valid_from": null, "dbt_scd_id": null, "dbt_updated_at": null, "dbt_is_deleted": null}, "dbt_valid_to_current": null}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {"unique_key": "id", "strategy": "check", "check_cols": "all", "target_schema": "test17594237163467600387_test_previous_version_state"}, "created_at": 1759423716.114626, "relation_name": "\"dbt\".\"test17594237163467600387_test_previous_version_state\".\"snapshot_seed\"", "raw_code": "\n{{\n config(\n unique_key='id',\n strategy='check',\n check_cols='all',\n target_schema=schema,\n )\n}}\nselect * from {{ ref('my_seed') }}\n", "doc_blocks": [], "language": "sql", "refs": [{"name": "my_seed", "package": null, "version": null}], "sources": [], "metrics": [], "functions": [], "depends_on": {"macros": [], "nodes": ["seed.test.my_seed"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}}, "analysis.test.a": {"database": "dbt", "schema": "test17594237163467600387_test_previous_version_state", "name": "a", "resource_type": "analysis", "package_name": "test", "path": "analysis/a.sql", "original_file_path": "analyses/a.sql", "unique_id": "analysis.test.a", "fqn": ["test", "analysis", "a"], "alias": "a", "checksum": {"name": "sha256", "checksum": "a389c282f569f0bbdc2a8a4f174dea746c28582fdaf2048d31d9226af9feab23"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "batch_size": null, "lookback": 1, "begin": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "event_time": null, "concurrent_batches": null}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1759423716.1562681, "relation_name": null, "raw_code": "select 4 as id", "doc_blocks": [], "language": "sql", "refs": [], "sources": [], "metrics": [], "functions": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}}, "test.test.just_my": {"database": "dbt", "schema": "test17594237163467600387_test_previous_version_state_dbt_test__audit", "name": "just_my", "resource_type": "test", "package_name": "test", "path": "just_my.sql", "original_file_path": "tests/just_my.sql", "unique_id": "test.test.just_my", "fqn": ["test", "just_my"], "alias": "just_my", "checksum": {"name": "sha256", "checksum": "744889a2e2d9ce380619265e1217d7ccf6e6ca896c048d42ebe0f9cfb74d7156"}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": ["data_test_tag"], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": ["data_test_tag"], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {"tags": ["data_test_tag"]}, "created_at": 1759423716.287831, "relation_name": null, "raw_code": "{{ config(tags = ['data_test_tag']) }}\n\nselect * from {{ ref('my_model') }}\nwhere false", "doc_blocks": [], "language": "sql", "refs": [{"name": "my_model", "package": null, "version": null}], "sources": [], "metrics": [], "functions": [], "depends_on": {"macros": [], "nodes": ["model.test.my_model"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}}, "seed.test.my_seed": {"database": "dbt", "schema": "test17594237163467600387_test_previous_version_state", "name": "my_seed", "resource_type": "seed", "package_name": "test", "path": "my_seed.csv", "original_file_path": "seeds/my_seed.csv", "unique_id": "seed.test.my_seed", "fqn": ["test", "my_seed"], "alias": "my_seed", "checksum": {"name": "sha256", "checksum": "f7ede83f36165ac6b7a047aa2c3f212dff385bfa9f35f395108cd06fc8e96943"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "seed", "incremental_strategy": null, "batch_size": null, "lookback": 1, "begin": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "event_time": null, "concurrent_batches": null, "delimiter": ",", "quote_columns": null}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1759423716.320315, "relation_name": "\"dbt\".\"test17594237163467600387_test_previous_version_state\".\"my_seed\"", "raw_code": "", "doc_blocks": [], "root_path": "/private/var/folders/79/5290gpvn3lx5jdryk4844rm80000gn/T/pytest-of-quigleymalcolm/pytest-154/popen-gw6/project18", "depends_on": {"macros": []}}, "test.test.not_null_my_model_id.43e0e9183a": {"database": "dbt", "schema": "test17594237163467600387_test_previous_version_state_dbt_test__audit", "name": "not_null_my_model_id", "resource_type": "test", "package_name": "test", "path": "not_null_my_model_id.sql", "original_file_path": "models/schema.yml", "unique_id": "test.test.not_null_my_model_id.43e0e9183a", "fqn": ["test", "not_null_my_model_id"], "alias": "not_null_my_model_id", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1759423716.41968, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "doc_blocks": [], "language": "sql", "refs": [{"name": "my_model", "package": null, "version": null}], "sources": [], "metrics": [], "functions": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.test.my_model"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "id", "file_key_name": "models.my_model", "attached_node": "model.test.my_model", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "id", "model": "{{ get_where_subquery(ref('my_model')) }}"}, "namespace": null}}, "test.test.check_nothing_my_model_.d5a5e66110": {"database": "dbt", "schema": "test17594237163467600387_test_previous_version_state_dbt_test__audit", "name": "check_nothing_my_model_", "resource_type": "test", "package_name": "test", "path": "check_nothing_my_model_.sql", "original_file_path": "models/schema.yml", "unique_id": "test.test.check_nothing_my_model_.d5a5e66110", "fqn": ["test", "check_nothing_my_model_"], "alias": "check_nothing_my_model_", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1759423716.421907, "relation_name": null, "raw_code": "{{ test_check_nothing(**_dbt_generic_test_kwargs) }}", "doc_blocks": [], "language": "sql", "refs": [{"name": "my_model", "package": null, "version": null}], "sources": [], "metrics": [], "functions": [], "depends_on": {"macros": ["macro.test.test_check_nothing", "macro.dbt.get_where_subquery"], "nodes": ["model.test.my_model"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": null, "file_key_name": "models.my_model", "attached_node": "model.test.my_model", "test_metadata": {"name": "check_nothing", "kwargs": {"model": "{{ get_where_subquery(ref('my_model')) }}"}, "namespace": null}}}, "sources": {"source.test.my_source.my_table": {"database": "dbt", "schema": "my_source", "name": "my_table", "resource_type": "source", "package_name": "test", "path": "models/schema.yml", "original_file_path": "models/schema.yml", "unique_id": "source.test.my_source.my_table", "fqn": ["test", "my_source", "my_table"], "source_name": "my_source", "source_description": "My source", "loader": "a_loader", "identifier": "my_seed", "quoting": {"database": null, "schema": null, "identifier": null, "column": null}, "loaded_at_field": null, "loaded_at_query": null, "freshness": {"warn_after": {"count": null, "period": null}, "error_after": {"count": null, "period": null}, "filter": null}, "external": null, "description": "My table", "columns": {}, "meta": {}, "source_meta": {}, "tags": [], "config": {"enabled": true, "event_time": null, "freshness": {"warn_after": {"count": null, "period": null}, "error_after": {"count": null, "period": null}, "filter": null}, "loaded_at_field": null, "loaded_at_query": null, "meta": {}, "tags": []}, "patch_path": null, "unrendered_config": {"loaded_at_field": null, "loaded_at_query": null, "meta": {}, "tags": []}, "relation_name": "\"dbt\".\"my_source\".\"my_seed\"", "created_at": 1759423716.976106, "unrendered_database": null, "unrendered_schema": null, "doc_blocks": []}}, "macros": {"macro.test.test_check_nothing": {"name": "test_check_nothing", "resource_type": "macro", "package_name": "test", "path": "macros/dummy_test.sql", "original_file_path": "macros/dummy_test.sql", "unique_id": "macro.test.test_check_nothing", "macro_sql": "{% test check_nothing(model) %}\n-- a silly test to make sure that table-level tests show up in the manifest\n-- without a column_name field\n\nselect 0\n\n{% endtest %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.008358, "supported_languages": null}, "macro.test.test_disabled_check_nothing": {"name": "test_disabled_check_nothing", "resource_type": "macro", "package_name": "test", "path": "macros/disabled_dummy_test.sql", "original_file_path": "macros/disabled_dummy_test.sql", "unique_id": "macro.test.test_disabled_check_nothing", "macro_sql": "{% test disabled_check_nothing(model) %}\n-- a silly test to make sure that table-level tests show up in the manifest\n-- without a column_name field\n\n{{ config(enabled=False) }}\nselect 0\n\n{% endtest %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0087419, "supported_languages": null}, "macro.test.do_nothing": {"name": "do_nothing", "resource_type": "macro", "package_name": "test", "path": "macros/do_nothing.sql", "original_file_path": "macros/do_nothing.sql", "unique_id": "macro.test.do_nothing", "macro_sql": "{% macro do_nothing(foo2, bar2) %}\n select\n '{{ foo2 }}' as foo2,\n '{{ bar2 }}' as bar2\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.009166, "supported_languages": null}, "macro.dbt_postgres.postgres__current_timestamp": {"name": "postgres__current_timestamp", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/timestamps.sql", "original_file_path": "macros/timestamps.sql", "unique_id": "macro.dbt_postgres.postgres__current_timestamp", "macro_sql": "{% macro postgres__current_timestamp() -%}\n now()\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0092008, "supported_languages": null}, "macro.dbt_postgres.postgres__snapshot_string_as_time": {"name": "postgres__snapshot_string_as_time", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/timestamps.sql", "original_file_path": "macros/timestamps.sql", "unique_id": "macro.dbt_postgres.postgres__snapshot_string_as_time", "macro_sql": "{% macro postgres__snapshot_string_as_time(timestamp) -%}\n {%- set result = \"'\" ~ timestamp ~ \"'::timestamp without time zone\" -%}\n {{ return(result) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.009213, "supported_languages": null}, "macro.dbt_postgres.postgres__snapshot_get_time": {"name": "postgres__snapshot_get_time", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/timestamps.sql", "original_file_path": "macros/timestamps.sql", "unique_id": "macro.dbt_postgres.postgres__snapshot_get_time", "macro_sql": "{% macro postgres__snapshot_get_time() -%}\n {{ current_timestamp() }}::timestamp without time zone\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.009224, "supported_languages": null}, "macro.dbt_postgres.postgres__current_timestamp_backcompat": {"name": "postgres__current_timestamp_backcompat", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/timestamps.sql", "original_file_path": "macros/timestamps.sql", "unique_id": "macro.dbt_postgres.postgres__current_timestamp_backcompat", "macro_sql": "{% macro postgres__current_timestamp_backcompat() %}\n current_timestamp::{{ type_timestamp() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.type_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.009235, "supported_languages": null}, "macro.dbt_postgres.postgres__current_timestamp_in_utc_backcompat": {"name": "postgres__current_timestamp_in_utc_backcompat", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/timestamps.sql", "original_file_path": "macros/timestamps.sql", "unique_id": "macro.dbt_postgres.postgres__current_timestamp_in_utc_backcompat", "macro_sql": "{% macro postgres__current_timestamp_in_utc_backcompat() %}\n (current_timestamp at time zone 'utc')::{{ type_timestamp() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.type_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0092459, "supported_languages": null}, "macro.dbt_postgres.postgres__get_catalog_relations": {"name": "postgres__get_catalog_relations", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/catalog.sql", "original_file_path": "macros/catalog.sql", "unique_id": "macro.dbt_postgres.postgres__get_catalog_relations", "macro_sql": "{% macro postgres__get_catalog_relations(information_schema, relations) -%}\n {%- call statement('catalog', fetch_result=True) -%}\n\n {#\n If the user has multiple databases set and the first one is wrong, this will fail.\n But we won't fail in the case where there are multiple quoting-difference-only dbs, which is better.\n #}\n {% set database = information_schema.database %}\n {{ adapter.verify_database(database) }}\n\n select\n '{{ database }}' as table_database,\n sch.nspname as table_schema,\n tbl.relname as table_name,\n case tbl.relkind\n when 'v' then 'VIEW'\n when 'm' then 'MATERIALIZED VIEW'\n else 'BASE TABLE'\n end as table_type,\n tbl_desc.description as table_comment,\n col.attname as column_name,\n col.attnum as column_index,\n pg_catalog.format_type(col.atttypid, col.atttypmod) as column_type,\n col_desc.description as column_comment,\n pg_get_userbyid(tbl.relowner) as table_owner\n\n from pg_catalog.pg_namespace sch\n join pg_catalog.pg_class tbl on tbl.relnamespace = sch.oid\n join pg_catalog.pg_attribute col on col.attrelid = tbl.oid\n left outer join pg_catalog.pg_description tbl_desc on (tbl_desc.objoid = tbl.oid and tbl_desc.objsubid = 0)\n left outer join pg_catalog.pg_description col_desc on (col_desc.objoid = tbl.oid and col_desc.objsubid = col.attnum)\n where (\n {%- for relation in relations -%}\n {%- if relation.identifier -%}\n (upper(sch.nspname) = upper('{{ relation.schema }}') and\n upper(tbl.relname) = upper('{{ relation.identifier }}'))\n {%- else-%}\n upper(sch.nspname) = upper('{{ relation.schema }}')\n {%- endif -%}\n {%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n )\n and not pg_is_other_temp_schema(sch.oid) -- not a temporary schema belonging to another session\n and tbl.relpersistence in ('p', 'u') -- [p]ermanent table or [u]nlogged table. Exclude [t]emporary tables\n and tbl.relkind in ('r', 'v', 'f', 'p', 'm') -- o[r]dinary table, [v]iew, [f]oreign table, [p]artitioned table, [m]aterialized view. Other values are [i]ndex, [S]equence, [c]omposite type, [t]OAST table\n and col.attnum > 0 -- negative numbers are used for system columns such as oid\n and not col.attisdropped -- column as not been dropped\n\n order by\n sch.nspname,\n tbl.relname,\n col.attnum\n\n {%- endcall -%}\n\n {{ return(load_result('catalog').table) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.009269, "supported_languages": null}, "macro.dbt_postgres.postgres__get_catalog": {"name": "postgres__get_catalog", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/catalog.sql", "original_file_path": "macros/catalog.sql", "unique_id": "macro.dbt_postgres.postgres__get_catalog", "macro_sql": "{% macro postgres__get_catalog(information_schema, schemas) -%}\n {%- set relations = [] -%}\n {%- for schema in schemas -%}\n {%- set dummy = relations.append({'schema': schema}) -%}\n {%- endfor -%}\n {{ return(postgres__get_catalog_relations(information_schema, relations)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_catalog_relations"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.009279, "supported_languages": null}, "macro.dbt_postgres.postgres__get_relations": {"name": "postgres__get_relations", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations.sql", "original_file_path": "macros/relations.sql", "unique_id": "macro.dbt_postgres.postgres__get_relations", "macro_sql": "{% macro postgres__get_relations() -%}\n\n {#\n -- in pg_depend, objid is the dependent, refobjid is the referenced object\n -- > a pg_depend entry indicates that the referenced object cannot be\n -- > dropped without also dropping the dependent object.\n #}\n\n {%- call statement('relations', fetch_result=True) -%}\n select distinct\n dependent_namespace.nspname as dependent_schema,\n dependent_class.relname as dependent_name,\n referenced_namespace.nspname as referenced_schema,\n referenced_class.relname as referenced_name\n\n -- Query for views: views are entries in pg_class with an entry in pg_rewrite, but we avoid\n -- a seq scan on pg_rewrite by leveraging the fact there is an \"internal\" row in pg_depend for\n -- the view...\n from pg_class as dependent_class\n join pg_namespace as dependent_namespace on dependent_namespace.oid = dependent_class.relnamespace\n join pg_depend as dependent_depend on dependent_depend.refobjid = dependent_class.oid\n and dependent_depend.classid = 'pg_rewrite'::regclass\n and dependent_depend.refclassid = 'pg_class'::regclass\n and dependent_depend.deptype = 'i'\n\n -- ... and via pg_depend (that has a row per column, hence the need for \"distinct\" above, and\n -- making sure to exclude the internal row to avoid a view appearing to depend on itself)...\n join pg_depend as joining_depend on joining_depend.objid = dependent_depend.objid\n and joining_depend.classid = 'pg_rewrite'::regclass\n and joining_depend.refclassid = 'pg_class'::regclass\n and joining_depend.refobjid != dependent_depend.refobjid\n\n -- ... we can find the tables they query from in pg_class, but excluding system tables. Note we\n -- don't need need to exclude _dependent_ system tables, because they only query from other\n -- system tables, and so are automatically excluded by excluding _referenced_ system tables\n join pg_class as referenced_class on referenced_class.oid = joining_depend.refobjid\n join pg_namespace as referenced_namespace on referenced_namespace.oid = referenced_class.relnamespace\n and referenced_namespace.nspname != 'information_schema'\n and referenced_namespace.nspname not like 'pg\\_%'\n\n order by\n dependent_schema, dependent_name, referenced_schema, referenced_name;\n\n {%- endcall -%}\n\n {{ return(load_result('relations').table) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.009302, "supported_languages": null}, "macro.dbt_postgres.postgres_get_relations": {"name": "postgres_get_relations", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations.sql", "original_file_path": "macros/relations.sql", "unique_id": "macro.dbt_postgres.postgres_get_relations", "macro_sql": "{% macro postgres_get_relations() %}\n {{ return(postgres__get_relations()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_relations"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.009314, "supported_languages": null}, "macro.dbt_postgres.postgres__create_table_as": {"name": "postgres__create_table_as", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__create_table_as", "macro_sql": "{% macro postgres__create_table_as(temporary, relation, sql) -%}\n {%- set unlogged = config.get('unlogged', default=false) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary -%}\n temporary\n {%- elif unlogged -%}\n unlogged\n {%- endif %} table {{ relation }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {% endif -%}\n {% if contract_config.enforced and (not temporary) -%}\n {{ get_table_columns_and_constraints() }} ;\n insert into {{ relation }} (\n {{ adapter.dispatch('get_column_names', 'dbt')() }}\n )\n {%- set sql = get_select_subquery(sql) %}\n {% else %}\n as\n {% endif %}\n (\n {{ sql }}\n );\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_assert_columns_equivalent", "macro.dbt.get_table_columns_and_constraints", "macro.dbt.default__get_column_names", "macro.dbt.get_select_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.009342, "supported_languages": null}, "macro.dbt_postgres.postgres__get_create_index_sql": {"name": "postgres__get_create_index_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__get_create_index_sql", "macro_sql": "{% macro postgres__get_create_index_sql(relation, index_dict) -%}\n {%- set index_config = adapter.parse_index(index_dict) -%}\n {%- set comma_separated_columns = \", \".join(index_config.columns) -%}\n {%- set index_name = index_config.render(relation) -%}\n\n create {% if index_config.unique -%}\n unique\n {%- endif %} index if not exists\n \"{{ index_name }}\"\n on {{ relation }} {% if index_config.type -%}\n using {{ index_config.type }}\n {%- endif %}\n ({{ comma_separated_columns }})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.009353, "supported_languages": null}, "macro.dbt_postgres.postgres__create_schema": {"name": "postgres__create_schema", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__create_schema", "macro_sql": "{% macro postgres__create_schema(relation) -%}\n {% if relation.database -%}\n {{ adapter.verify_database(relation.database) }}\n {%- endif -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier().include(database=False) }}\n {%- endcall -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.009364, "supported_languages": null}, "macro.dbt_postgres.postgres__drop_schema": {"name": "postgres__drop_schema", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__drop_schema", "macro_sql": "{% macro postgres__drop_schema(relation) -%}\n {% if relation.database -%}\n {{ adapter.verify_database(relation.database) }}\n {%- endif -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier().include(database=False) }} cascade\n {%- endcall -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.009374, "supported_languages": null}, "macro.dbt_postgres.postgres__get_columns_in_relation": {"name": "postgres__get_columns_in_relation", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__get_columns_in_relation", "macro_sql": "{% macro postgres__get_columns_in_relation(relation) -%}\n {% call statement('get_columns_in_relation', fetch_result=True) %}\n select\n column_name,\n data_type,\n character_maximum_length,\n numeric_precision,\n numeric_scale\n\n from {{ relation.information_schema('columns') }}\n where table_name = '{{ relation.identifier }}'\n {% if relation.schema %}\n and table_schema = '{{ relation.schema }}'\n {% endif %}\n order by ordinal_position\n\n {% endcall %}\n {% set table = load_result('get_columns_in_relation').table %}\n {{ return(sql_convert_columns_in_relation(table)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.sql_convert_columns_in_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.009384, "supported_languages": null}, "macro.dbt_postgres.postgres__list_relations_without_caching": {"name": "postgres__list_relations_without_caching", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__list_relations_without_caching", "macro_sql": "{% macro postgres__list_relations_without_caching(schema_relation) %}\n {% call statement('list_relations_without_caching', fetch_result=True) -%}\n select\n '{{ schema_relation.database }}' as database,\n tablename as name,\n schemaname as schema,\n 'table' as type\n from pg_tables\n where schemaname ilike '{{ schema_relation.schema }}'\n union all\n select\n '{{ schema_relation.database }}' as database,\n viewname as name,\n schemaname as schema,\n 'view' as type\n from pg_views\n where schemaname ilike '{{ schema_relation.schema }}'\n union all\n select\n '{{ schema_relation.database }}' as database,\n matviewname as name,\n schemaname as schema,\n 'materialized_view' as type\n from pg_matviews\n where schemaname ilike '{{ schema_relation.schema }}'\n {% endcall %}\n {{ return(load_result('list_relations_without_caching').table) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.009394, "supported_languages": null}, "macro.dbt_postgres.postgres__information_schema_name": {"name": "postgres__information_schema_name", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__information_schema_name", "macro_sql": "{% macro postgres__information_schema_name(database) -%}\n {% if database_name -%}\n {{ adapter.verify_database(database_name) }}\n {%- endif -%}\n information_schema\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.009404, "supported_languages": null}, "macro.dbt_postgres.postgres__list_schemas": {"name": "postgres__list_schemas", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__list_schemas", "macro_sql": "{% macro postgres__list_schemas(database) %}\n {% if database -%}\n {{ adapter.verify_database(database) }}\n {%- endif -%}\n {% call statement('list_schemas', fetch_result=True, auto_begin=False) %}\n select distinct nspname from pg_namespace\n {% endcall %}\n {{ return(load_result('list_schemas').table) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.009414, "supported_languages": null}, "macro.dbt_postgres.postgres__check_schema_exists": {"name": "postgres__check_schema_exists", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__check_schema_exists", "macro_sql": "{% macro postgres__check_schema_exists(information_schema, schema) -%}\n {% if information_schema.database -%}\n {{ adapter.verify_database(information_schema.database) }}\n {%- endif -%}\n {% call statement('check_schema_exists', fetch_result=True, auto_begin=False) %}\n select count(*) from pg_namespace where nspname = '{{ schema }}'\n {% endcall %}\n {{ return(load_result('check_schema_exists').table) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.009425, "supported_languages": null}, "macro.dbt_postgres.postgres__make_relation_with_suffix": {"name": "postgres__make_relation_with_suffix", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__make_relation_with_suffix", "macro_sql": "{% macro postgres__make_relation_with_suffix(base_relation, suffix, dstring) %}\n {% if dstring %}\n {% set dt = modules.datetime.datetime.now() %}\n {% set dtstring = dt.strftime(\"%H%M%S%f\") %}\n {% set suffix = suffix ~ dtstring %}\n {% endif %}\n {% set suffix_length = suffix|length %}\n {% set relation_max_name_length = base_relation.relation_max_name_length() %}\n {% if suffix_length > relation_max_name_length %}\n {% do exceptions.raise_compiler_error('Relation suffix is too long (' ~ suffix_length ~ ' characters). Maximum length is ' ~ relation_max_name_length ~ ' characters.') %}\n {% endif %}\n {% set identifier = base_relation.identifier[:relation_max_name_length - suffix_length] ~ suffix %}\n\n {{ return(base_relation.incorporate(path={\"identifier\": identifier })) }}\n\n {% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.009435, "supported_languages": null}, "macro.dbt_postgres.postgres__make_intermediate_relation": {"name": "postgres__make_intermediate_relation", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__make_intermediate_relation", "macro_sql": "{% macro postgres__make_intermediate_relation(base_relation, suffix) %}\n {{ return(postgres__make_relation_with_suffix(base_relation, suffix, dstring=False)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_relation_with_suffix"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0094469, "supported_languages": null}, "macro.dbt_postgres.postgres__make_temp_relation": {"name": "postgres__make_temp_relation", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__make_temp_relation", "macro_sql": "{% macro postgres__make_temp_relation(base_relation, suffix) %}\n {% set temp_relation = postgres__make_relation_with_suffix(base_relation, suffix, dstring=True) %}\n {{ return(temp_relation.incorporate(path={\"schema\": none,\n \"database\": none})) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_relation_with_suffix"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0094569, "supported_languages": null}, "macro.dbt_postgres.postgres__make_backup_relation": {"name": "postgres__make_backup_relation", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__make_backup_relation", "macro_sql": "{% macro postgres__make_backup_relation(base_relation, backup_relation_type, suffix) %}\n {% set backup_relation = postgres__make_relation_with_suffix(base_relation, suffix, dstring=False) %}\n {{ return(backup_relation.incorporate(type=backup_relation_type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_relation_with_suffix"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.009467, "supported_languages": null}, "macro.dbt_postgres.postgres_escape_comment": {"name": "postgres_escape_comment", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres_escape_comment", "macro_sql": "{% macro postgres_escape_comment(comment) -%}\n {% if comment is not string %}\n {% do exceptions.raise_compiler_error('cannot escape a non-string: ' ~ comment) %}\n {% endif %}\n {%- set magic = '$dbt_comment_literal_block$' -%}\n {%- if magic in comment -%}\n {%- do exceptions.raise_compiler_error('The string ' ~ magic ~ ' is not allowed in comments.') -%}\n {%- endif -%}\n {{ magic }}{{ comment }}{{ magic }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0094779, "supported_languages": null}, "macro.dbt_postgres.postgres__alter_relation_comment": {"name": "postgres__alter_relation_comment", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__alter_relation_comment", "macro_sql": "{% macro postgres__alter_relation_comment(relation, comment) %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n {% if relation.type == 'materialized_view' -%}\n {% set relation_type = \"materialized view\" %}\n {%- else -%}\n {%- set relation_type = relation.type -%}\n {%- endif -%}\n comment on {{ relation_type }} {{ relation }} is {{ escaped_comment }};\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres_escape_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0094879, "supported_languages": null}, "macro.dbt_postgres.postgres__alter_column_comment": {"name": "postgres__alter_column_comment", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__alter_column_comment", "macro_sql": "{% macro postgres__alter_column_comment(relation, column_dict) %}\n {% set existing_columns = adapter.get_columns_in_relation(relation) | map(attribute=\"name\") | list %}\n {% for column_name in column_dict if (column_name in existing_columns) %}\n {% set comment = column_dict[column_name]['description'] %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n comment on column {{ relation }}.{{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }} is {{ escaped_comment }};\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres_escape_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0094981, "supported_languages": null}, "macro.dbt_postgres.postgres__get_show_grant_sql": {"name": "postgres__get_show_grant_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__get_show_grant_sql", "macro_sql": "\n\n{%- macro postgres__get_show_grant_sql(relation) -%}\n select grantee, privilege_type\n from {{ relation.information_schema('role_table_grants') }}\n where grantor = current_role\n and grantee != current_role\n and table_schema = '{{ relation.schema }}'\n and table_name = '{{ relation.identifier }}'\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.009509, "supported_languages": null}, "macro.dbt_postgres.postgres__copy_grants": {"name": "postgres__copy_grants", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__copy_grants", "macro_sql": "{% macro postgres__copy_grants() %}\n {{ return(False) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.009519, "supported_languages": null}, "macro.dbt_postgres.postgres__get_show_indexes_sql": {"name": "postgres__get_show_indexes_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__get_show_indexes_sql", "macro_sql": "{% macro postgres__get_show_indexes_sql(relation) %}\n select\n i.relname as name,\n m.amname as method,\n ix.indisunique as \"unique\",\n array_to_string(array_agg(a.attname), ',') as column_names\n from pg_index ix\n join pg_class i\n on i.oid = ix.indexrelid\n join pg_am m\n on m.oid=i.relam\n join pg_class t\n on t.oid = ix.indrelid\n join pg_namespace n\n on n.oid = t.relnamespace\n join pg_attribute a\n on a.attrelid = t.oid\n and a.attnum = ANY(ix.indkey)\n where t.relname = '{{ relation.identifier }}'\n and n.nspname = '{{ relation.schema }}'\n and t.relkind in ('r', 'm')\n group by 1, 2, 3\n order by 1, 2, 3\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.009529, "supported_languages": null}, "macro.dbt_postgres.postgres__get_drop_index_sql": {"name": "postgres__get_drop_index_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__get_drop_index_sql", "macro_sql": "\n\n\n{%- macro postgres__get_drop_index_sql(relation, index_name) -%}\n drop index if exists \"{{ relation.schema }}\".\"{{ index_name }}\"\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0095391, "supported_languages": null}, "macro.dbt_postgres.postgres__get_incremental_default_sql": {"name": "postgres__get_incremental_default_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/materializations/incremental_strategies.sql", "original_file_path": "macros/materializations/incremental_strategies.sql", "unique_id": "macro.dbt_postgres.postgres__get_incremental_default_sql", "macro_sql": "{% macro postgres__get_incremental_default_sql(arg_dict) %}\n\n {% if arg_dict[\"unique_key\"] %}\n {% do return(get_incremental_delete_insert_sql(arg_dict)) %}\n {% else %}\n {% do return(get_incremental_append_sql(arg_dict)) %}\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_incremental_delete_insert_sql", "macro.dbt.get_incremental_append_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0095592, "supported_languages": null}, "macro.dbt_postgres.postgres__get_incremental_microbatch_sql": {"name": "postgres__get_incremental_microbatch_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/materializations/incremental_strategies.sql", "original_file_path": "macros/materializations/incremental_strategies.sql", "unique_id": "macro.dbt_postgres.postgres__get_incremental_microbatch_sql", "macro_sql": "{% macro postgres__get_incremental_microbatch_sql(arg_dict) %}\n\n {% if arg_dict[\"unique_key\"] %}\n {% do return(adapter.dispatch('get_incremental_merge_sql', 'dbt')(arg_dict)) %}\n {% else %}\n {{ exceptions.raise_compiler_error(\"dbt-postgres 'microbatch' requires a `unique_key` config\") }}\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_incremental_merge_sql", "macro.dbt.default__get_incremental_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0095701, "supported_languages": null}, "macro.dbt_postgres.postgres__snapshot_merge_sql": {"name": "postgres__snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/materializations/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshot_merge.sql", "unique_id": "macro.dbt_postgres.postgres__snapshot_merge_sql", "macro_sql": "{% macro postgres__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n {%- set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() -%}\n\n update {{ target }}\n set {{ columns.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ columns.dbt_valid_to }}\n from {{ source }} as DBT_INTERNAL_SOURCE\n where DBT_INTERNAL_SOURCE.{{ columns.dbt_scd_id }}::text = {{ target }}.{{ columns.dbt_scd_id }}::text\n and DBT_INTERNAL_SOURCE.dbt_change_type::text in ('update'::text, 'delete'::text)\n {% if config.get(\"dbt_valid_to_current\") %}\n and ({{ target }}.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }} or {{ target }}.{{ columns.dbt_valid_to }} is null);\n {% else %}\n and {{ target }}.{{ columns.dbt_valid_to }} is null;\n {% endif %}\n\n\n insert into {{ target }} ({{ insert_cols_csv }})\n select {% for column in insert_cols -%}\n DBT_INTERNAL_SOURCE.{{ column }} {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n from {{ source }} as DBT_INTERNAL_SOURCE\n where DBT_INTERNAL_SOURCE.dbt_change_type::text = 'insert'::text;\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.00959, "supported_languages": null}, "macro.dbt_postgres.postgres__drop_materialized_view": {"name": "postgres__drop_materialized_view", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/materialized_view/drop.sql", "original_file_path": "macros/relations/materialized_view/drop.sql", "unique_id": "macro.dbt_postgres.postgres__drop_materialized_view", "macro_sql": "{% macro postgres__drop_materialized_view(relation) -%}\n drop materialized view if exists {{ relation }} cascade\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.009609, "supported_languages": null}, "macro.dbt_postgres.postgres__describe_materialized_view": {"name": "postgres__describe_materialized_view", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/materialized_view/describe.sql", "original_file_path": "macros/relations/materialized_view/describe.sql", "unique_id": "macro.dbt_postgres.postgres__describe_materialized_view", "macro_sql": "{% macro postgres__describe_materialized_view(relation) %}\n -- for now just get the indexes, we don't need the name or the query yet\n {% set _indexes = run_query(get_show_indexes_sql(relation)) %}\n {% do return({'indexes': _indexes}) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query", "macro.dbt.get_show_indexes_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.009629, "supported_languages": null}, "macro.dbt_postgres.postgres__refresh_materialized_view": {"name": "postgres__refresh_materialized_view", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/materialized_view/refresh.sql", "original_file_path": "macros/relations/materialized_view/refresh.sql", "unique_id": "macro.dbt_postgres.postgres__refresh_materialized_view", "macro_sql": "{% macro postgres__refresh_materialized_view(relation) %}\n refresh materialized view {{ relation }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0096471, "supported_languages": null}, "macro.dbt_postgres.postgres__get_rename_materialized_view_sql": {"name": "postgres__get_rename_materialized_view_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/materialized_view/rename.sql", "original_file_path": "macros/relations/materialized_view/rename.sql", "unique_id": "macro.dbt_postgres.postgres__get_rename_materialized_view_sql", "macro_sql": "{% macro postgres__get_rename_materialized_view_sql(relation, new_name) %}\n alter materialized view {{ relation }} rename to {{ new_name }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.009665, "supported_languages": null}, "macro.dbt_postgres.postgres__get_alter_materialized_view_as_sql": {"name": "postgres__get_alter_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt_postgres.postgres__get_alter_materialized_view_as_sql", "macro_sql": "{% macro postgres__get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n\n -- apply a full refresh immediately if needed\n {% if configuration_changes.requires_full_refresh %}\n\n {{ get_replace_sql(existing_relation, relation, sql) }}\n\n -- otherwise apply individual changes as needed\n {% else %}\n\n {{ postgres__update_indexes_on_materialized_view(relation, configuration_changes.indexes) }}\n\n {%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_replace_sql", "macro.dbt_postgres.postgres__update_indexes_on_materialized_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.009685, "supported_languages": null}, "macro.dbt_postgres.postgres__update_indexes_on_materialized_view": {"name": "postgres__update_indexes_on_materialized_view", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt_postgres.postgres__update_indexes_on_materialized_view", "macro_sql": "\n\n\n{%- macro postgres__update_indexes_on_materialized_view(relation, index_changes) -%}\n {{- log(\"Applying UPDATE INDEXES to: \" ~ relation) -}}\n\n {%- for _index_change in index_changes -%}\n {%- set _index = _index_change.context -%}\n\n {%- if _index_change.action == \"drop\" -%}\n\n {{ postgres__get_drop_index_sql(relation, _index.name) }}\n\n {%- elif _index_change.action == \"create\" -%}\n\n {{ postgres__get_create_index_sql(relation, _index.as_node_config) }}\n\n {%- endif -%}\n\t{{ ';' if not loop.last else \"\" }}\n\n {%- endfor -%}\n\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_drop_index_sql", "macro.dbt_postgres.postgres__get_create_index_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.009695, "supported_languages": null}, "macro.dbt_postgres.postgres__get_materialized_view_configuration_changes": {"name": "postgres__get_materialized_view_configuration_changes", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt_postgres.postgres__get_materialized_view_configuration_changes", "macro_sql": "{% macro postgres__get_materialized_view_configuration_changes(existing_relation, new_config) %}\n {% set _existing_materialized_view = postgres__describe_materialized_view(existing_relation) %}\n {% set _configuration_changes = existing_relation.get_materialized_view_config_change_collection(_existing_materialized_view, new_config.model) %}\n {% do return(_configuration_changes) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__describe_materialized_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.009711, "supported_languages": null}, "macro.dbt_postgres.postgres__get_create_materialized_view_as_sql": {"name": "postgres__get_create_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/materialized_view/create.sql", "original_file_path": "macros/relations/materialized_view/create.sql", "unique_id": "macro.dbt_postgres.postgres__get_create_materialized_view_as_sql", "macro_sql": "{% macro postgres__get_create_materialized_view_as_sql(relation, sql) %}\n create materialized view if not exists {{ relation }} as {{ sql }};\n\n {% for _index_dict in config.get('indexes', []) -%}\n {{- get_create_index_sql(relation, _index_dict) -}}{{ ';' if not loop.last else \"\" }}\n {%- endfor -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_create_index_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.009731, "supported_languages": null}, "macro.dbt_postgres.postgres__drop_table": {"name": "postgres__drop_table", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/table/drop.sql", "original_file_path": "macros/relations/table/drop.sql", "unique_id": "macro.dbt_postgres.postgres__drop_table", "macro_sql": "{% macro postgres__drop_table(relation) -%}\n drop table if exists {{ relation }} cascade\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.009758, "supported_languages": null}, "macro.dbt_postgres.postgres__get_replace_table_sql": {"name": "postgres__get_replace_table_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/table/replace.sql", "original_file_path": "macros/relations/table/replace.sql", "unique_id": "macro.dbt_postgres.postgres__get_replace_table_sql", "macro_sql": "{% macro postgres__get_replace_table_sql(relation, sql) -%}\n\n {%- set sql_header = config.get('sql_header', none) -%}\n {{ sql_header if sql_header is not none }}\n\n create or replace table {{ relation }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {{ get_table_columns_and_constraints() }}\n {%- set sql = get_select_subquery(sql) %}\n {% endif %}\n as (\n {{ sql }}\n );\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_assert_columns_equivalent", "macro.dbt.get_table_columns_and_constraints", "macro.dbt.get_select_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.009779, "supported_languages": null}, "macro.dbt_postgres.postgres__get_rename_table_sql": {"name": "postgres__get_rename_table_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/table/rename.sql", "original_file_path": "macros/relations/table/rename.sql", "unique_id": "macro.dbt_postgres.postgres__get_rename_table_sql", "macro_sql": "{% macro postgres__get_rename_table_sql(relation, new_name) %}\n alter table {{ relation }} rename to {{ new_name }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0097978, "supported_languages": null}, "macro.dbt_postgres.postgres__drop_view": {"name": "postgres__drop_view", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/view/drop.sql", "original_file_path": "macros/relations/view/drop.sql", "unique_id": "macro.dbt_postgres.postgres__drop_view", "macro_sql": "{% macro postgres__drop_view(relation) -%}\n drop view if exists {{ relation }} cascade\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.009823, "supported_languages": null}, "macro.dbt_postgres.postgres__get_replace_view_sql": {"name": "postgres__get_replace_view_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt_postgres.postgres__get_replace_view_sql", "macro_sql": "{% macro postgres__get_replace_view_sql(relation, sql) -%}\n\n {%- set sql_header = config.get('sql_header', none) -%}\n {{ sql_header if sql_header is not none }}\n\n create or replace view {{ relation }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {%- endif %}\n as (\n {{ sql }}\n );\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.009842, "supported_languages": null}, "macro.dbt_postgres.postgres__get_rename_view_sql": {"name": "postgres__get_rename_view_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/view/rename.sql", "original_file_path": "macros/relations/view/rename.sql", "unique_id": "macro.dbt_postgres.postgres__get_rename_view_sql", "macro_sql": "{% macro postgres__get_rename_view_sql(relation, new_name) %}\n alter view {{ relation }} rename to {{ new_name }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.00986, "supported_languages": null}, "macro.dbt_postgres.postgres__dateadd": {"name": "postgres__dateadd", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt_postgres.postgres__dateadd", "macro_sql": "{% macro postgres__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n {{ from_date_or_timestamp }} + ((interval '1 {{ datepart }}') * ({{ interval }}))\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.009887, "supported_languages": null}, "macro.dbt_postgres.postgres__listagg": {"name": "postgres__listagg", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "unique_id": "macro.dbt_postgres.postgres__listagg", "macro_sql": "{% macro postgres__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n {% if limit_num -%}\n array_to_string(\n (array_agg(\n {{ measure }}\n {% if order_by_clause -%}\n {{ order_by_clause }}\n {%- endif %}\n ))[1:{{ limit_num }}],\n {{ delimiter_text }}\n )\n {%- else %}\n string_agg(\n {{ measure }},\n {{ delimiter_text }}\n {% if order_by_clause -%}\n {{ order_by_clause }}\n {%- endif %}\n )\n {%- endif %}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0099058, "supported_languages": null}, "macro.dbt_postgres.postgres__datediff": {"name": "postgres__datediff", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt_postgres.postgres__datediff", "macro_sql": "{% macro postgres__datediff(first_date, second_date, datepart) -%}\n\n {% if datepart == 'year' %}\n (date_part('year', ({{second_date}})::date) - date_part('year', ({{first_date}})::date))\n {% elif datepart == 'quarter' %}\n ({{ datediff(first_date, second_date, 'year') }} * 4 + date_part('quarter', ({{second_date}})::date) - date_part('quarter', ({{first_date}})::date))\n {% elif datepart == 'month' %}\n ({{ datediff(first_date, second_date, 'year') }} * 12 + date_part('month', ({{second_date}})::date) - date_part('month', ({{first_date}})::date))\n {% elif datepart == 'day' %}\n (({{second_date}})::date - ({{first_date}})::date)\n {% elif datepart == 'week' %}\n ({{ datediff(first_date, second_date, 'day') }} / 7 + case\n when date_part('dow', ({{first_date}})::timestamp) <= date_part('dow', ({{second_date}})::timestamp) then\n case when {{first_date}} <= {{second_date}} then 0 else -1 end\n else\n case when {{first_date}} <= {{second_date}} then 1 else 0 end\n end)\n {% elif datepart == 'hour' %}\n ({{ datediff(first_date, second_date, 'day') }} * 24 + date_part('hour', ({{second_date}})::timestamp) - date_part('hour', ({{first_date}})::timestamp))\n {% elif datepart == 'minute' %}\n ({{ datediff(first_date, second_date, 'hour') }} * 60 + date_part('minute', ({{second_date}})::timestamp) - date_part('minute', ({{first_date}})::timestamp))\n {% elif datepart == 'second' %}\n ({{ datediff(first_date, second_date, 'minute') }} * 60 + floor(date_part('second', ({{second_date}})::timestamp)) - floor(date_part('second', ({{first_date}})::timestamp)))\n {% elif datepart == 'millisecond' %}\n ({{ datediff(first_date, second_date, 'minute') }} * 60000 + floor(date_part('millisecond', ({{second_date}})::timestamp)) - floor(date_part('millisecond', ({{first_date}})::timestamp)))\n {% elif datepart == 'microsecond' %}\n ({{ datediff(first_date, second_date, 'minute') }} * 60000000 + floor(date_part('microsecond', ({{second_date}})::timestamp)) - floor(date_part('microsecond', ({{first_date}})::timestamp)))\n {% else %}\n {{ exceptions.raise_compiler_error(\"Unsupported datepart for macro datediff in postgres: {!r}\".format(datepart)) }}\n {% endif %}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.datediff"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0099258, "supported_languages": null}, "macro.dbt_postgres.postgres__any_value": {"name": "postgres__any_value", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "unique_id": "macro.dbt_postgres.postgres__any_value", "macro_sql": "{% macro postgres__any_value(expression) -%}\n\n min({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.009944, "supported_languages": null}, "macro.dbt_postgres.postgres__last_day": {"name": "postgres__last_day", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt_postgres.postgres__last_day", "macro_sql": "{% macro postgres__last_day(date, datepart) -%}\n\n {%- if datepart == 'quarter' -%}\n -- postgres dateadd does not support quarter interval.\n cast(\n {{dbt.dateadd('day', '-1',\n dbt.dateadd('month', '3', dbt.date_trunc(datepart, date))\n )}}\n as date)\n {%- else -%}\n {{dbt.default_last_day(date, datepart)}}\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.dateadd", "macro.dbt.date_trunc", "macro.dbt.default_last_day"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.009963, "supported_languages": null}, "macro.dbt_postgres.postgres__split_part": {"name": "postgres__split_part", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt_postgres.postgres__split_part", "macro_sql": "{% macro postgres__split_part(string_text, delimiter_text, part_number) %}\n\n {% if part_number >= 0 %}\n {{ dbt.default__split_part(string_text, delimiter_text, part_number) }}\n {% else %}\n {{ dbt._split_part_negative(string_text, delimiter_text, part_number) }}\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__split_part", "macro.dbt._split_part_negative"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.009981, "supported_languages": null}, "macro.dbt.run_hooks": {"name": "run_hooks", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.run_hooks", "macro_sql": "{% macro run_hooks(hooks, inside_transaction=True) %}\n {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %}\n {% if not inside_transaction and loop.first %}\n {% call statement(auto_begin=inside_transaction) %}\n commit;\n {% endcall %}\n {% endif %}\n {% set rendered = render(hook.get('sql')) | trim %}\n {% if (rendered | length) > 0 %}\n {% call statement(auto_begin=inside_transaction) %}\n {{ rendered }}\n {% endcall %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010005, "supported_languages": null}, "macro.dbt.make_hook_config": {"name": "make_hook_config", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.make_hook_config", "macro_sql": "{% macro make_hook_config(sql, inside_transaction) %}\n {{ tojson({\"sql\": sql, \"transaction\": inside_transaction}) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010015, "supported_languages": null}, "macro.dbt.before_begin": {"name": "before_begin", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.before_begin", "macro_sql": "{% macro before_begin(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010025, "supported_languages": null}, "macro.dbt.in_transaction": {"name": "in_transaction", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.in_transaction", "macro_sql": "{% macro in_transaction(sql) %}\n {{ make_hook_config(sql, inside_transaction=True) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010036, "supported_languages": null}, "macro.dbt.after_commit": {"name": "after_commit", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.after_commit", "macro_sql": "{% macro after_commit(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010046, "supported_languages": null}, "macro.dbt.set_sql_header": {"name": "set_sql_header", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "unique_id": "macro.dbt.set_sql_header", "macro_sql": "{% macro set_sql_header(config) -%}\n {{ config.set('sql_header', caller()) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0100648, "supported_languages": null}, "macro.dbt.should_full_refresh": {"name": "should_full_refresh", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "unique_id": "macro.dbt.should_full_refresh", "macro_sql": "{% macro should_full_refresh() %}\n {% set config_full_refresh = config.get('full_refresh') %}\n {% if config_full_refresh is none %}\n {% set config_full_refresh = flags.FULL_REFRESH %}\n {% endif %}\n {% do return(config_full_refresh) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0100758, "supported_languages": null}, "macro.dbt.should_store_failures": {"name": "should_store_failures", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "unique_id": "macro.dbt.should_store_failures", "macro_sql": "{% macro should_store_failures() %}\n {% set config_store_failures = config.get('store_failures') %}\n {% if config_store_failures is none %}\n {% set config_store_failures = flags.STORE_FAILURES %}\n {% endif %}\n {% do return(config_store_failures) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010085, "supported_languages": null}, "macro.dbt.snapshot_merge_sql": {"name": "snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", "unique_id": "macro.dbt.snapshot_merge_sql", "macro_sql": "{% macro snapshot_merge_sql(target, source, insert_cols) -%}\n {{ adapter.dispatch('snapshot_merge_sql', 'dbt')(target, source, insert_cols) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__snapshot_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0101051, "supported_languages": null}, "macro.dbt.default__snapshot_merge_sql": {"name": "default__snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", "unique_id": "macro.dbt.default__snapshot_merge_sql", "macro_sql": "{% macro default__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n {%- set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() -%}\n\n merge into {{ target.render() }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on DBT_INTERNAL_SOURCE.{{ columns.dbt_scd_id }} = DBT_INTERNAL_DEST.{{ columns.dbt_scd_id }}\n\n when matched\n {% if config.get(\"dbt_valid_to_current\") %}\n\t{% set source_unique_key = (\"DBT_INTERNAL_DEST.\" ~ columns.dbt_valid_to) | trim %}\n\t{% set target_unique_key = config.get('dbt_valid_to_current') | trim %}\n\tand ({{ equals(source_unique_key, target_unique_key) }} or {{ source_unique_key }} is null)\n\n {% else %}\n and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null\n {% endif %}\n and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')\n then update\n set {{ columns.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ columns.dbt_valid_to }}\n\n when not matched\n and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'\n then insert ({{ insert_cols_csv }})\n values ({{ insert_cols_csv }})\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names", "macro.dbt.equals"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0101151, "supported_languages": null}, "macro.dbt.strategy_dispatch": {"name": "strategy_dispatch", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.strategy_dispatch", "macro_sql": "{% macro strategy_dispatch(name) -%}\n{% set original_name = name %}\n {% if '.' in name %}\n {% set package_name, name = name.split(\".\", 1) %}\n {% else %}\n {% set package_name = none %}\n {% endif %}\n\n {% if package_name is none %}\n {% set package_context = context %}\n {% elif package_name in context %}\n {% set package_context = context[package_name] %}\n {% else %}\n {% set error_msg %}\n Could not find package '{{package_name}}', called with '{{original_name}}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n\n {%- set search_name = 'snapshot_' ~ name ~ '_strategy' -%}\n\n {% if search_name not in package_context %}\n {% set error_msg %}\n The specified strategy macro '{{name}}' was not found in package '{{ package_name }}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n {{ return(package_context[search_name]) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010141, "supported_languages": null}, "macro.dbt.snapshot_hash_arguments": {"name": "snapshot_hash_arguments", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_hash_arguments", "macro_sql": "{% macro snapshot_hash_arguments(args) -%}\n {{ adapter.dispatch('snapshot_hash_arguments', 'dbt')(args) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010151, "supported_languages": null}, "macro.dbt.default__snapshot_hash_arguments": {"name": "default__snapshot_hash_arguments", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.default__snapshot_hash_arguments", "macro_sql": "{% macro default__snapshot_hash_arguments(args) -%}\n md5({%- for arg in args -%}\n coalesce(cast({{ arg }} as varchar ), '')\n {% if not loop.last %} || '|' || {% endif %}\n {%- endfor -%})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010161, "supported_languages": null}, "macro.dbt.snapshot_timestamp_strategy": {"name": "snapshot_timestamp_strategy", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_timestamp_strategy", "macro_sql": "{% macro snapshot_timestamp_strategy(node, snapshotted_rel, current_rel, model_config, target_exists) %}\n {# The model_config parameter is no longer used, but is passed in anyway for compatibility. #}\n {% set primary_key = config.get('unique_key') %}\n {% set updated_at = config.get('updated_at') %}\n {% set hard_deletes = adapter.get_hard_deletes_behavior(config) %}\n {% set invalidate_hard_deletes = hard_deletes == 'invalidate' %}\n {% set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() %}\n\n {#/*\n The snapshot relation might not have an {{ updated_at }} value if the\n snapshot strategy is changed from `check` to `timestamp`. We\n should use a dbt-created column for the comparison in the snapshot\n table instead of assuming that the user-supplied {{ updated_at }}\n will be present in the historical data.\n\n See https://github.com/dbt-labs/dbt-core/issues/2350\n */ #}\n {% set row_changed_expr -%}\n ({{ snapshotted_rel }}.{{ columns.dbt_valid_from }} < {{ current_rel }}.{{ updated_at }})\n {%- endset %}\n\n {% set scd_args = api.Relation.scd_args(primary_key, updated_at) %}\n {% set scd_id_expr = snapshot_hash_arguments(scd_args) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes,\n \"hard_deletes\": hard_deletes\n }) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names", "macro.dbt.snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010171, "supported_languages": null}, "macro.dbt.snapshot_string_as_time": {"name": "snapshot_string_as_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_string_as_time", "macro_sql": "{% macro snapshot_string_as_time(timestamp) -%}\n {{ adapter.dispatch('snapshot_string_as_time', 'dbt')(timestamp) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__snapshot_string_as_time"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0101812, "supported_languages": null}, "macro.dbt.default__snapshot_string_as_time": {"name": "default__snapshot_string_as_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.default__snapshot_string_as_time", "macro_sql": "{% macro default__snapshot_string_as_time(timestamp) %}\n {% do exceptions.raise_not_implemented(\n 'snapshot_string_as_time macro not implemented for adapter '+adapter.type()\n ) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.01019, "supported_languages": null}, "macro.dbt.snapshot_check_all_get_existing_columns": {"name": "snapshot_check_all_get_existing_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_check_all_get_existing_columns", "macro_sql": "{% macro snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) -%}\n {%- if not target_exists -%}\n {#-- no table yet -> return whatever the query does --#}\n {{ return((false, query_columns)) }}\n {%- endif -%}\n\n {#-- handle any schema changes --#}\n {%- set target_relation = adapter.get_relation(database=node.database, schema=node.schema, identifier=node.alias) -%}\n\n {% if check_cols_config == 'all' %}\n {%- set query_columns = get_columns_in_query(node['compiled_code']) -%}\n\n {% elif check_cols_config is iterable and (check_cols_config | length) > 0 %}\n {#-- query for proper casing/quoting, to support comparison below --#}\n {%- set select_check_cols_from_target -%}\n {#-- N.B. The whitespace below is necessary to avoid edge case issue with comments --#}\n {#-- See: https://github.com/dbt-labs/dbt-core/issues/6781 --#}\n select {{ check_cols_config | join(', ') }} from (\n {{ node['compiled_code'] }}\n ) subq\n {%- endset -%}\n {% set query_columns = get_columns_in_query(select_check_cols_from_target) %}\n\n {% else %}\n {% do exceptions.raise_compiler_error(\"Invalid value for 'check_cols': \" ~ check_cols_config) %}\n {% endif %}\n\n {%- set existing_cols = adapter.get_columns_in_relation(target_relation) | map(attribute = 'name') | list -%}\n {%- set ns = namespace() -%} {#-- handle for-loop scoping with a namespace --#}\n {%- set ns.column_added = false -%}\n\n {%- set intersection = [] -%}\n {%- for col in query_columns -%}\n {%- if col in existing_cols -%}\n {%- do intersection.append(adapter.quote(col)) -%}\n {%- else -%}\n {% set ns.column_added = true %}\n {%- endif -%}\n {%- endfor -%}\n {{ return((ns.column_added, intersection)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_columns_in_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010201, "supported_languages": null}, "macro.dbt.snapshot_check_strategy": {"name": "snapshot_check_strategy", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_check_strategy", "macro_sql": "{% macro snapshot_check_strategy(node, snapshotted_rel, current_rel, model_config, target_exists) %}\n {# The model_config parameter is no longer used, but is passed in anyway for compatibility. #}\n {% set check_cols_config = config.get('check_cols') %}\n {% set primary_key = config.get('unique_key') %}\n {% set hard_deletes = adapter.get_hard_deletes_behavior(config) %}\n {% set invalidate_hard_deletes = hard_deletes == 'invalidate' %}\n {% set updated_at = config.get('updated_at') or snapshot_get_time() %}\n\n {% set column_added = false %}\n\n {% set column_added, check_cols = snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) %}\n\n {%- set row_changed_expr -%}\n (\n {%- if column_added -%}\n {{ get_true_sql() }}\n {%- else -%}\n {%- for col in check_cols -%}\n {{ snapshotted_rel }}.{{ col }} != {{ current_rel }}.{{ col }}\n or\n (\n (({{ snapshotted_rel }}.{{ col }} is null) and not ({{ current_rel }}.{{ col }} is null))\n or\n ((not {{ snapshotted_rel }}.{{ col }} is null) and ({{ current_rel }}.{{ col }} is null))\n )\n {%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n {%- endif -%}\n )\n {%- endset %}\n\n {% set scd_args = api.Relation.scd_args(primary_key, updated_at) %}\n {% set scd_id_expr = snapshot_hash_arguments(scd_args) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes,\n \"hard_deletes\": hard_deletes\n }) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.snapshot_get_time", "macro.dbt.snapshot_check_all_get_existing_columns", "macro.dbt.get_true_sql", "macro.dbt.snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010211, "supported_languages": null}, "macro.dbt.create_columns": {"name": "create_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.create_columns", "macro_sql": "{% macro create_columns(relation, columns) %}\n {{ adapter.dispatch('create_columns', 'dbt')(relation, columns) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__create_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.01024, "supported_languages": null}, "macro.dbt.default__create_columns": {"name": "default__create_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__create_columns", "macro_sql": "{% macro default__create_columns(relation, columns) %}\n {% for column in columns %}\n {% call statement() %}\n alter table {{ relation.render() }} add column {{ adapter.quote(column.name) }} {{ column.data_type }};\n {% endcall %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.01025, "supported_languages": null}, "macro.dbt.post_snapshot": {"name": "post_snapshot", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.post_snapshot", "macro_sql": "{% macro post_snapshot(staging_relation) %}\n {{ adapter.dispatch('post_snapshot', 'dbt')(staging_relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010286, "supported_languages": null}, "macro.dbt.default__post_snapshot": {"name": "default__post_snapshot", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__post_snapshot", "macro_sql": "{% macro default__post_snapshot(staging_relation) %}\n {# no-op #}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010296, "supported_languages": null}, "macro.dbt.get_true_sql": {"name": "get_true_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.get_true_sql", "macro_sql": "{% macro get_true_sql() %}\n {{ adapter.dispatch('get_true_sql', 'dbt')() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_true_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0103061, "supported_languages": null}, "macro.dbt.default__get_true_sql": {"name": "default__get_true_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__get_true_sql", "macro_sql": "{% macro default__get_true_sql() %}\n {{ return('TRUE') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0103161, "supported_languages": null}, "macro.dbt.snapshot_staging_table": {"name": "snapshot_staging_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.snapshot_staging_table", "macro_sql": "{% macro snapshot_staging_table(strategy, source_sql, target_relation) -%}\n {{ adapter.dispatch('snapshot_staging_table', 'dbt')(strategy, source_sql, target_relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__snapshot_staging_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010325, "supported_languages": null}, "macro.dbt.get_snapshot_table_column_names": {"name": "get_snapshot_table_column_names", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.get_snapshot_table_column_names", "macro_sql": "{% macro get_snapshot_table_column_names() %}\n {{ return({'dbt_valid_to': 'dbt_valid_to', 'dbt_valid_from': 'dbt_valid_from', 'dbt_scd_id': 'dbt_scd_id', 'dbt_updated_at': 'dbt_updated_at', 'dbt_is_deleted': 'dbt_is_deleted'}) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010335, "supported_languages": null}, "macro.dbt.default__snapshot_staging_table": {"name": "default__snapshot_staging_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__snapshot_staging_table", "macro_sql": "{% macro default__snapshot_staging_table(strategy, source_sql, target_relation) -%}\n {% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %}\n {% if strategy.hard_deletes == 'new_record' %}\n {% set new_scd_id = snapshot_hash_arguments([columns.dbt_scd_id, snapshot_get_time()]) %}\n {% endif %}\n with snapshot_query as (\n\n {{ source_sql }}\n\n ),\n\n snapshotted_data as (\n\n select *, {{ unique_key_fields(strategy.unique_key) }}\n from {{ target_relation }}\n where\n {% if config.get('dbt_valid_to_current') %}\n\t\t{% set source_unique_key = columns.dbt_valid_to | trim %}\n\t\t{% set target_unique_key = config.get('dbt_valid_to_current') | trim %}\n\n\t\t{# The exact equals semantics between NULL values depends on the current behavior flag set. Also, update records if the source field is null #}\n ( {{ equals(source_unique_key, target_unique_key) }} or {{ source_unique_key }} is null )\n {% else %}\n {{ columns.dbt_valid_to }} is null\n {% endif %}\n\n ),\n\n insertions_source_data as (\n\n select *, {{ unique_key_fields(strategy.unique_key) }},\n {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},\n {{ get_dbt_valid_to_current(strategy, columns) }},\n {{ strategy.scd_id }} as {{ columns.dbt_scd_id }}\n\n from snapshot_query\n ),\n\n updates_source_data as (\n\n select *, {{ unique_key_fields(strategy.unique_key) }},\n {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_to }}\n\n from snapshot_query\n ),\n\n {%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' %}\n\n deletes_source_data as (\n\n select *, {{ unique_key_fields(strategy.unique_key) }}\n from snapshot_query\n ),\n {% endif %}\n\n insertions as (\n\n select\n 'insert' as dbt_change_type,\n source_data.*\n {%- if strategy.hard_deletes == 'new_record' -%}\n ,'False' as {{ columns.dbt_is_deleted }}\n {%- endif %}\n\n from insertions_source_data as source_data\n left outer join snapshotted_data\n on {{ unique_key_join_on(strategy.unique_key, \"snapshotted_data\", \"source_data\") }}\n where {{ unique_key_is_null(strategy.unique_key, \"snapshotted_data\") }}\n or ({{ unique_key_is_not_null(strategy.unique_key, \"snapshotted_data\") }} and (\n {{ strategy.row_changed }} {%- if strategy.hard_deletes == 'new_record' -%} or snapshotted_data.{{ columns.dbt_is_deleted }} = 'True' {% endif %}\n )\n\n )\n\n ),\n\n updates as (\n\n select\n 'update' as dbt_change_type,\n source_data.*,\n snapshotted_data.{{ columns.dbt_scd_id }}\n {%- if strategy.hard_deletes == 'new_record' -%}\n , snapshotted_data.{{ columns.dbt_is_deleted }}\n {%- endif %}\n\n from updates_source_data as source_data\n join snapshotted_data\n on {{ unique_key_join_on(strategy.unique_key, \"snapshotted_data\", \"source_data\") }}\n where (\n {{ strategy.row_changed }} {%- if strategy.hard_deletes == 'new_record' -%} or snapshotted_data.{{ columns.dbt_is_deleted }} = 'True' {% endif %}\n )\n )\n\n {%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' %}\n ,\n deletes as (\n\n select\n 'delete' as dbt_change_type,\n source_data.*,\n {{ snapshot_get_time() }} as {{ columns.dbt_valid_from }},\n {{ snapshot_get_time() }} as {{ columns.dbt_updated_at }},\n {{ snapshot_get_time() }} as {{ columns.dbt_valid_to }},\n snapshotted_data.{{ columns.dbt_scd_id }}\n {%- if strategy.hard_deletes == 'new_record' -%}\n , snapshotted_data.{{ columns.dbt_is_deleted }}\n {%- endif %}\n from snapshotted_data\n left join deletes_source_data as source_data\n on {{ unique_key_join_on(strategy.unique_key, \"snapshotted_data\", \"source_data\") }}\n where {{ unique_key_is_null(strategy.unique_key, \"source_data\") }}\n\n {%- if strategy.hard_deletes == 'new_record' %}\n and not (\n --avoid updating the record's valid_to if the latest entry is marked as deleted\n snapshotted_data.{{ columns.dbt_is_deleted }} = 'True'\n and\n {% if config.get('dbt_valid_to_current') -%}\n snapshotted_data.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }}\n {%- else -%}\n snapshotted_data.{{ columns.dbt_valid_to }} is null\n {%- endif %}\n )\n {%- endif %}\n )\n {%- endif %}\n\n {%- if strategy.hard_deletes == 'new_record' %}\n {% set snapshotted_cols = get_list_of_column_names(get_columns_in_relation(target_relation)) %}\n {% set source_sql_cols = get_column_schema_from_query(source_sql) %}\n ,\n deletion_records as (\n\n select\n 'insert' as dbt_change_type,\n {#/*\n If a column has been added to the source it won't yet exist in the\n snapshotted table so we insert a null value as a placeholder for the column.\n */#}\n {%- for col in source_sql_cols -%}\n {%- if col.name in snapshotted_cols -%}\n snapshotted_data.{{ adapter.quote(col.column) }},\n {%- else -%}\n NULL as {{ adapter.quote(col.column) }},\n {%- endif -%}\n {% endfor -%}\n {%- if strategy.unique_key | is_list -%}\n {%- for key in strategy.unique_key -%}\n snapshotted_data.{{ key }} as dbt_unique_key_{{ loop.index }},\n {% endfor -%}\n {%- else -%}\n snapshotted_data.dbt_unique_key as dbt_unique_key,\n {% endif -%}\n {{ snapshot_get_time() }} as {{ columns.dbt_valid_from }},\n {{ snapshot_get_time() }} as {{ columns.dbt_updated_at }},\n snapshotted_data.{{ columns.dbt_valid_to }} as {{ columns.dbt_valid_to }},\n {{ new_scd_id }} as {{ columns.dbt_scd_id }},\n 'True' as {{ columns.dbt_is_deleted }}\n from snapshotted_data\n left join deletes_source_data as source_data\n on {{ unique_key_join_on(strategy.unique_key, \"snapshotted_data\", \"source_data\") }}\n where {{ unique_key_is_null(strategy.unique_key, \"source_data\") }}\n and not (\n --avoid inserting a new record if the latest one is marked as deleted\n snapshotted_data.{{ columns.dbt_is_deleted }} = 'True'\n and\n {% if config.get('dbt_valid_to_current') -%}\n snapshotted_data.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }}\n {%- else -%}\n snapshotted_data.{{ columns.dbt_valid_to }} is null\n {%- endif %}\n )\n\n )\n {%- endif %}\n\n select * from insertions\n union all\n select * from updates\n {%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' %}\n union all\n select * from deletes\n {%- endif %}\n {%- if strategy.hard_deletes == 'new_record' %}\n union all\n select * from deletion_records\n {%- endif %}\n\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names", "macro.dbt.snapshot_hash_arguments", "macro.dbt.snapshot_get_time", "macro.dbt.unique_key_fields", "macro.dbt.equals", "macro.dbt.get_dbt_valid_to_current", "macro.dbt.unique_key_join_on", "macro.dbt.unique_key_is_null", "macro.dbt.unique_key_is_not_null", "macro.dbt.get_list_of_column_names", "macro.dbt.get_columns_in_relation", "macro.dbt.get_column_schema_from_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010345, "supported_languages": null}, "macro.dbt.build_snapshot_table": {"name": "build_snapshot_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.build_snapshot_table", "macro_sql": "{% macro build_snapshot_table(strategy, sql) -%}\n {{ adapter.dispatch('build_snapshot_table', 'dbt')(strategy, sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__build_snapshot_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010355, "supported_languages": null}, "macro.dbt.default__build_snapshot_table": {"name": "default__build_snapshot_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__build_snapshot_table", "macro_sql": "{% macro default__build_snapshot_table(strategy, sql) %}\n {% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %}\n\n select *,\n {{ strategy.scd_id }} as {{ columns.dbt_scd_id }},\n {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},\n {{ get_dbt_valid_to_current(strategy, columns) }}\n {%- if strategy.hard_deletes == 'new_record' -%}\n , 'False' as {{ columns.dbt_is_deleted }}\n {% endif -%}\n from (\n {{ sql }}\n ) sbq\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names", "macro.dbt.get_dbt_valid_to_current"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010365, "supported_languages": null}, "macro.dbt.build_snapshot_staging_table": {"name": "build_snapshot_staging_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.build_snapshot_staging_table", "macro_sql": "{% macro build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% set temp_relation = make_temp_relation(target_relation) %}\n\n {% set select = snapshot_staging_table(strategy, sql, target_relation) %}\n\n {% call statement('build_snapshot_staging_relation') %}\n {{ create_table_as(True, temp_relation, select) }}\n {% endcall %}\n\n {% do return(temp_relation) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_temp_relation", "macro.dbt.snapshot_staging_table", "macro.dbt.statement", "macro.dbt.create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010375, "supported_languages": null}, "macro.dbt.get_updated_at_column_data_type": {"name": "get_updated_at_column_data_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.get_updated_at_column_data_type", "macro_sql": "{% macro get_updated_at_column_data_type(snapshot_sql) %}\n {% set snapshot_sql_column_schema = get_column_schema_from_query(snapshot_sql) %}\n {% set dbt_updated_at_data_type = null %}\n {% set ns = namespace() -%} {#-- handle for-loop scoping with a namespace --#}\n {% set ns.dbt_updated_at_data_type = null -%}\n {% for column in snapshot_sql_column_schema %}\n {% if ((column.column == 'dbt_updated_at') or (column.column == 'DBT_UPDATED_AT')) %}\n {% set ns.dbt_updated_at_data_type = column.dtype %}\n {% endif %}\n {% endfor %}\n {{ return(ns.dbt_updated_at_data_type or none) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_column_schema_from_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0103838, "supported_languages": null}, "macro.dbt.check_time_data_types": {"name": "check_time_data_types", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.check_time_data_types", "macro_sql": "{% macro check_time_data_types(sql) %}\n {% set dbt_updated_at_data_type = get_updated_at_column_data_type(sql) %}\n {% set snapshot_get_time_data_type = get_snapshot_get_time_data_type() %}\n {% if snapshot_get_time_data_type is not none and dbt_updated_at_data_type is not none and snapshot_get_time_data_type != dbt_updated_at_data_type %}\n {% if exceptions.warn_snapshot_timestamp_data_types %}\n {{ exceptions.warn_snapshot_timestamp_data_types(snapshot_get_time_data_type, dbt_updated_at_data_type) }}\n {% endif %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_updated_at_column_data_type", "macro.dbt.get_snapshot_get_time_data_type"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010397, "supported_languages": null}, "macro.dbt.get_dbt_valid_to_current": {"name": "get_dbt_valid_to_current", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.get_dbt_valid_to_current", "macro_sql": "{% macro get_dbt_valid_to_current(strategy, columns) %}\n {% set dbt_valid_to_current = config.get('dbt_valid_to_current') or \"null\" %}\n coalesce(nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}), {{dbt_valid_to_current}})\n as {{ columns.dbt_valid_to }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0104058, "supported_languages": null}, "macro.dbt.unique_key_fields": {"name": "unique_key_fields", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.unique_key_fields", "macro_sql": "{% macro unique_key_fields(unique_key) %}\n {% if unique_key | is_list %}\n {% for key in unique_key %}\n {{ key }} as dbt_unique_key_{{ loop.index }}\n {%- if not loop.last %} , {%- endif %}\n {% endfor %}\n {% else %}\n {{ unique_key }} as dbt_unique_key\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010416, "supported_languages": null}, "macro.dbt.unique_key_join_on": {"name": "unique_key_join_on", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.unique_key_join_on", "macro_sql": "{% macro unique_key_join_on(unique_key, identifier, from_identifier) %}\n {% if unique_key | is_list %}\n {% for key in unique_key %}\n\t {% set source_unique_key = (identifier ~ \".dbt_unique_key_\" ~ loop.index) | trim %}\n\t {% set target_unique_key = (from_identifier ~ \".dbt_unique_key_\" ~ loop.index) | trim %}\n\t {{ equals(source_unique_key, target_unique_key) }}\n {%- if not loop.last %} and {%- endif %}\n {% endfor %}\n {% else %}\n {{ identifier }}.dbt_unique_key = {{ from_identifier }}.dbt_unique_key\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.equals"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010426, "supported_languages": null}, "macro.dbt.unique_key_is_null": {"name": "unique_key_is_null", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.unique_key_is_null", "macro_sql": "{% macro unique_key_is_null(unique_key, identifier) %}\n {% if unique_key | is_list %}\n {{ identifier }}.dbt_unique_key_1 is null\n {% else %}\n {{ identifier }}.dbt_unique_key is null\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010435, "supported_languages": null}, "macro.dbt.unique_key_is_not_null": {"name": "unique_key_is_not_null", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.unique_key_is_not_null", "macro_sql": "{% macro unique_key_is_not_null(unique_key, identifier) %}\n {% if unique_key | is_list %}\n {{ identifier }}.dbt_unique_key_1 is not null\n {% else %}\n {{ identifier }}.dbt_unique_key is not null\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010445, "supported_languages": null}, "macro.dbt.materialization_snapshot_default": {"name": "materialization_snapshot_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/snapshot.sql", "original_file_path": "macros/materializations/snapshots/snapshot.sql", "unique_id": "macro.dbt.materialization_snapshot_default", "macro_sql": "{% materialization snapshot, default %}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n -- grab current tables grants config for comparision later on\n {%- set grant_config = config.get('grants') -%}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=model.database,\n schema=model.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {# The model['config'] parameter below is no longer used, but passing anyway for compatibility #}\n {# It was a dictionary of config, instead of the config object from the context #}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", model['config'], target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_code']) %}\n {% set build_or_select_sql = build_sql %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% else %}\n\n {% set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() %}\n\n {{ adapter.assert_valid_snapshot_target_given_strategy(target_relation, columns, strategy) }}\n\n {% set build_or_select_sql = snapshot_staging_table(strategy, sql, target_relation) %}\n {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set remove_columns = ['dbt_change_type', 'DBT_CHANGE_TYPE', 'dbt_unique_key', 'DBT_UNIQUE_KEY'] %}\n {% if unique_key | is_list %}\n {% for key in strategy.unique_key %}\n {{ remove_columns.append('dbt_unique_key_' + loop.index|string) }}\n {{ remove_columns.append('DBT_UNIQUE_KEY_' + loop.index|string) }}\n {% endfor %}\n {% endif %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'in', remove_columns)\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'in', remove_columns)\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% endif %}\n\n\n {{ check_time_data_types(build_or_select_sql) }}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode=False) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if not target_relation_exists %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if staging_table is defined %}\n {% do post_snapshot(staging_table) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.get_or_create_relation", "macro.dbt.run_hooks", "macro.dbt.strategy_dispatch", "macro.dbt.build_snapshot_table", "macro.dbt.create_table_as", "macro.dbt.get_snapshot_table_column_names", "macro.dbt.snapshot_staging_table", "macro.dbt.build_snapshot_staging_table", "macro.dbt.create_columns", "macro.dbt.snapshot_merge_sql", "macro.dbt.check_time_data_types", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes", "macro.dbt.post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010466, "supported_languages": ["sql"]}, "macro.dbt.materialization_test_default": {"name": "materialization_test_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/test.sql", "original_file_path": "macros/materializations/tests/test.sql", "unique_id": "macro.dbt.materialization_test_default", "macro_sql": "{%- materialization test, default -%}\n\n {% set relations = [] %}\n {% set limit = config.get('limit') %}\n\n {% set sql_with_limit %}\n {{ get_limit_subquery_sql(sql, limit) }}\n {% endset %}\n\n {% if should_store_failures() %}\n\n {% set identifier = model['alias'] %}\n {% set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n\n {% set store_failures_as = config.get('store_failures_as') %}\n -- if `--store-failures` is invoked via command line and `store_failures_as` is not set,\n -- config.get('store_failures_as', 'table') returns None, not 'table'\n {% if store_failures_as == none %}{% set store_failures_as = 'table' %}{% endif %}\n {% if store_failures_as not in ['table', 'view'] %}\n {{ exceptions.raise_compiler_error(\n \"'\" ~ store_failures_as ~ \"' is not a valid value for `store_failures_as`. \"\n \"Accepted values are: ['ephemeral', 'table', 'view']\"\n ) }}\n {% endif %}\n\n {% set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database, type=store_failures_as) -%} %}\n\n {% if old_relation %}\n {% do adapter.drop_relation(old_relation) %}\n {% endif %}\n\n {% call statement(auto_begin=True) %}\n {{ get_create_sql(target_relation, sql_with_limit) }}\n {% endcall %}\n\n {% do relations.append(target_relation) %}\n\n {# Since the test failures have already been saved to the database, reuse that result rather than querying again #}\n {% set main_sql %}\n select *\n from {{ target_relation }}\n {% endset %}\n\n {{ adapter.commit() }}\n\n {% else %}\n\n {% set main_sql = sql_with_limit %}\n\n {% endif %}\n\n {% set fail_calc = config.get('fail_calc') %}\n {% set warn_if = config.get('warn_if') %}\n {% set error_if = config.get('error_if') %}\n\n {% call statement('main', fetch_result=True) -%}\n\n {# The limit has already been included above, and we do not want to duplicate it again. We also want to be safe for macro overrides treating `limit` as a required parameter. #}\n {{ get_test_sql(main_sql, fail_calc, warn_if, error_if, limit=none)}}\n\n {%- endcall %}\n\n {{ return({'relations': relations}) }}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.get_limit_subquery_sql", "macro.dbt.should_store_failures", "macro.dbt.statement", "macro.dbt.get_create_sql", "macro.dbt.get_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0105011, "supported_languages": ["sql"]}, "macro.dbt.get_test_sql": {"name": "get_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.get_test_sql", "macro_sql": "{% macro get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n {{ adapter.dispatch('get_test_sql', 'dbt')(main_sql, fail_calc, warn_if, error_if, limit) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010528, "supported_languages": null}, "macro.dbt.default__get_test_sql": {"name": "default__get_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.default__get_test_sql", "macro_sql": "{% macro default__get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n select\n {{ fail_calc }} as failures,\n {{ fail_calc }} {{ warn_if }} as should_warn,\n {{ fail_calc }} {{ error_if }} as should_error\n from (\n {{ main_sql }}\n {{ \"limit \" ~ limit if limit != none }}\n ) dbt_internal_test\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010539, "supported_languages": null}, "macro.dbt.get_unit_test_sql": {"name": "get_unit_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.get_unit_test_sql", "macro_sql": "{% macro get_unit_test_sql(main_sql, expected_fixture_sql, expected_column_names) -%}\n {{ adapter.dispatch('get_unit_test_sql', 'dbt')(main_sql, expected_fixture_sql, expected_column_names) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_unit_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010549, "supported_languages": null}, "macro.dbt.default__get_unit_test_sql": {"name": "default__get_unit_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.default__get_unit_test_sql", "macro_sql": "{% macro default__get_unit_test_sql(main_sql, expected_fixture_sql, expected_column_names) -%}\n-- Build actual result given inputs\nwith dbt_internal_unit_test_actual as (\n select\n {% for expected_column_name in expected_column_names %}{{expected_column_name}}{% if not loop.last -%},{% endif %}{%- endfor -%}, {{ dbt.string_literal(\"actual\") }} as {{ adapter.quote(\"actual_or_expected\") }}\n from (\n {{ main_sql }}\n ) _dbt_internal_unit_test_actual\n),\n-- Build expected result\ndbt_internal_unit_test_expected as (\n select\n {% for expected_column_name in expected_column_names %}{{expected_column_name}}{% if not loop.last -%}, {% endif %}{%- endfor -%}, {{ dbt.string_literal(\"expected\") }} as {{ adapter.quote(\"actual_or_expected\") }}\n from (\n {{ expected_fixture_sql }}\n ) _dbt_internal_unit_test_expected\n)\n-- Union actual and expected results\nselect * from dbt_internal_unit_test_actual\nunion all\nselect * from dbt_internal_unit_test_expected\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.string_literal"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010559, "supported_languages": null}, "macro.dbt.get_where_subquery": {"name": "get_where_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/where_subquery.sql", "original_file_path": "macros/materializations/tests/where_subquery.sql", "unique_id": "macro.dbt.get_where_subquery", "macro_sql": "{% macro get_where_subquery(relation) -%}\n {% do return(adapter.dispatch('get_where_subquery', 'dbt')(relation)) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_where_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010578, "supported_languages": null}, "macro.dbt.default__get_where_subquery": {"name": "default__get_where_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/where_subquery.sql", "original_file_path": "macros/materializations/tests/where_subquery.sql", "unique_id": "macro.dbt.default__get_where_subquery", "macro_sql": "{% macro default__get_where_subquery(relation) -%}\n {% set where = config.get('where', '') %}\n {% if where %}\n {%- set filtered -%}\n (select * from {{ relation }} where {{ where }}) dbt_subquery\n {%- endset -%}\n {% do return(filtered) %}\n {%- else -%}\n {% do return(relation) %}\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010588, "supported_languages": null}, "macro.dbt.materialization_unit_default": {"name": "materialization_unit_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/unit.sql", "original_file_path": "macros/materializations/tests/unit.sql", "unique_id": "macro.dbt.materialization_unit_default", "macro_sql": "{%- materialization unit, default -%}\n\n {% set relations = [] %}\n\n {% set expected_rows = config.get('expected_rows') %}\n {% set expected_sql = config.get('expected_sql') %}\n {% set tested_expected_column_names = expected_rows[0].keys() if (expected_rows | length ) > 0 else get_columns_in_query(sql) %}\n\n {%- set target_relation = this.incorporate(type='table') -%}\n {%- set temp_relation = make_temp_relation(target_relation)-%}\n {% do run_query(get_create_table_as_sql(True, temp_relation, get_empty_subquery_sql(sql))) %}\n {%- set columns_in_relation = adapter.get_columns_in_relation(temp_relation) -%}\n {%- set column_name_to_data_types = {} -%}\n {%- set column_name_to_quoted = {} -%}\n {%- for column in columns_in_relation -%}\n {%- do column_name_to_data_types.update({column.name|lower: column.data_type}) -%}\n {%- do column_name_to_quoted.update({column.name|lower: column.quoted}) -%}\n {%- endfor -%}\n\n {%- set expected_column_names_quoted = [] -%}\n {%- for column_name in tested_expected_column_names -%}\n {%- do expected_column_names_quoted.append(column_name_to_quoted[column_name|lower]) -%}\n {%- endfor -%}\n\n {% if not expected_sql %}\n {% set expected_sql = get_expected_sql(expected_rows, column_name_to_data_types, column_name_to_quoted) %}\n {% endif %}\n {% set unit_test_sql = get_unit_test_sql(sql, expected_sql, expected_column_names_quoted) %}\n\n {% call statement('main', fetch_result=True) -%}\n\n {{ unit_test_sql }}\n\n {%- endcall %}\n\n {% do adapter.drop_relation(temp_relation) %}\n\n {{ return({'relations': relations}) }}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.get_columns_in_query", "macro.dbt.make_temp_relation", "macro.dbt.run_query", "macro.dbt.get_create_table_as_sql", "macro.dbt.get_empty_subquery_sql", "macro.dbt.get_expected_sql", "macro.dbt.get_unit_test_sql", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010607, "supported_languages": ["sql"]}, "macro.dbt.materialization_materialized_view_default": {"name": "materialization_materialized_view_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialization_materialized_view_default", "macro_sql": "{% materialization materialized_view, default %}\n {% set existing_relation = load_cached_relation(this) %}\n {% set target_relation = this.incorporate(type=this.MaterializedView) %}\n {% set intermediate_relation = make_intermediate_relation(target_relation) %}\n {% set backup_relation_type = target_relation.MaterializedView if existing_relation is none else existing_relation.type %}\n {% set backup_relation = make_backup_relation(target_relation, backup_relation_type) %}\n\n {{ materialized_view_setup(backup_relation, intermediate_relation, pre_hooks) }}\n\n {% set build_sql = materialized_view_get_build_sql(existing_relation, target_relation, backup_relation, intermediate_relation) %}\n\n {% if build_sql == '' %}\n {{ materialized_view_execute_no_op(target_relation) }}\n {% else %}\n {{ materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) }}\n {% endif %}\n\n {{ materialized_view_teardown(backup_relation, intermediate_relation, post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.materialized_view_setup", "macro.dbt.materialized_view_get_build_sql", "macro.dbt.materialized_view_execute_no_op", "macro.dbt.materialized_view_execute_build_sql", "macro.dbt.materialized_view_teardown"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010638, "supported_languages": ["sql"]}, "macro.dbt.materialized_view_setup": {"name": "materialized_view_setup", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_setup", "macro_sql": "{% macro materialized_view_setup(backup_relation, intermediate_relation, pre_hooks) %}\n\n -- backup_relation and intermediate_relation should not already exist in the database\n -- it's possible these exist because of a previous run that exited unexpectedly\n {% set preexisting_backup_relation = load_cached_relation(backup_relation) %}\n {% set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) %}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010655, "supported_languages": null}, "macro.dbt.materialized_view_teardown": {"name": "materialized_view_teardown", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_teardown", "macro_sql": "{% macro materialized_view_teardown(backup_relation, intermediate_relation, post_hooks) %}\n\n -- drop the temp relations if they exist to leave the database clean for the next run\n {{ drop_relation_if_exists(backup_relation) }}\n {{ drop_relation_if_exists(intermediate_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010665, "supported_languages": null}, "macro.dbt.materialized_view_get_build_sql": {"name": "materialized_view_get_build_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_get_build_sql", "macro_sql": "{% macro materialized_view_get_build_sql(existing_relation, target_relation, backup_relation, intermediate_relation) %}\n\n {% set full_refresh_mode = should_full_refresh() %}\n\n -- determine the scenario we're in: create, full_refresh, alter, refresh data\n {% if existing_relation is none %}\n {% set build_sql = get_create_materialized_view_as_sql(target_relation, sql) %}\n {% elif full_refresh_mode or not existing_relation.is_materialized_view %}\n {% set build_sql = get_replace_sql(existing_relation, target_relation, sql) %}\n {% else %}\n\n -- get config options\n {% set on_configuration_change = config.get('on_configuration_change') %}\n {% set configuration_changes = get_materialized_view_configuration_changes(existing_relation, config) %}\n\n {% if configuration_changes is none %}\n {% set build_sql = refresh_materialized_view(target_relation) %}\n\n {% elif on_configuration_change == 'apply' %}\n {% set build_sql = get_alter_materialized_view_as_sql(target_relation, configuration_changes, sql, existing_relation, backup_relation, intermediate_relation) %}\n {% elif on_configuration_change == 'continue' %}\n {% set build_sql = '' %}\n {{ exceptions.warn(\"Configuration changes were identified and `on_configuration_change` was set to `continue` for `\" ~ target_relation.render() ~ \"`\") }}\n {% elif on_configuration_change == 'fail' %}\n {{ exceptions.raise_fail_fast_error(\"Configuration changes were identified and `on_configuration_change` was set to `fail` for `\" ~ target_relation.render() ~ \"`\") }}\n\n {% else %}\n -- this only happens if the user provides a value other than `apply`, 'skip', 'fail'\n {{ exceptions.raise_compiler_error(\"Unexpected configuration scenario\") }}\n\n {% endif %}\n\n {% endif %}\n\n {% do return(build_sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.get_create_materialized_view_as_sql", "macro.dbt.get_replace_sql", "macro.dbt.get_materialized_view_configuration_changes", "macro.dbt.refresh_materialized_view", "macro.dbt.get_alter_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010675, "supported_languages": null}, "macro.dbt.materialized_view_execute_no_op": {"name": "materialized_view_execute_no_op", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_execute_no_op", "macro_sql": "{% macro materialized_view_execute_no_op(target_relation) %}\n {% do store_raw_result(\n name=\"main\",\n message=\"skip \" ~ target_relation,\n code=\"skip\",\n rows_affected=\"-1\"\n ) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010688, "supported_languages": null}, "macro.dbt.materialized_view_execute_build_sql": {"name": "materialized_view_execute_build_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_execute_build_sql", "macro_sql": "{% macro materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) %}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set grant_config = config.get('grants') %}\n\n {% call statement(name=\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010698, "supported_languages": null}, "macro.dbt.materialization_view_default": {"name": "materialization_view_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/view.sql", "original_file_path": "macros/materializations/models/view.sql", "unique_id": "macro.dbt.materialization_view_default", "macro_sql": "{%- materialization view, default -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='view') -%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}\n\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}\n /*\n This relation (probably) doesn't exist yet. If it does exist, it's a leftover from\n a previous run, and we're going to try to drop it immediately. At the end of this\n materialization, we're going to rename the \"existing_relation\" to this identifier,\n and then we're going to drop it. In order to make sure we run the correct one of:\n - drop view ...\n - drop table ...\n\n We need to set the type of this relation to be the type of the existing_relation, if it exists,\n or else \"view\" as a sane default if it does not. Note that if the existing_relation does not\n exist, then there is nothing to move out of the way and subsequentally drop. In that case,\n this relation will be effectively unused.\n */\n {%- set backup_relation_type = 'view' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n -- move the existing view out of the way\n {% if existing_relation is not none %}\n /* Do the equivalent of rename_if_exists. 'existing_relation' could have been dropped\n since the variable was first set. */\n {% set existing_relation = load_cached_relation(existing_relation) %}\n {% if existing_relation is not none %}\n {{ adapter.rename_relation(existing_relation, backup_relation) }}\n {% endif %}\n {% endif %}\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.run_hooks", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.get_create_view_as_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0107179, "supported_languages": ["sql"]}, "macro.dbt.materialization_table_default": {"name": "materialization_table_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table.sql", "original_file_path": "macros/materializations/models/table.sql", "unique_id": "macro.dbt.materialization_table_default", "macro_sql": "{% materialization table, default %}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') %}\n {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}\n /*\n See ../view/view.sql for more information about this relation.\n */\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_table_as_sql(False, intermediate_relation, sql) }}\n {%- endcall %}\n\n {% do create_indexes(intermediate_relation) %}\n\n -- cleanup\n {% if existing_relation is not none %}\n /* Do the equivalent of rename_if_exists. 'existing_relation' could have been dropped\n since the variable was first set. */\n {% set existing_relation = load_cached_relation(existing_relation) %}\n {% if existing_relation is not none %}\n {{ adapter.rename_relation(existing_relation, backup_relation) }}\n {% endif %}\n {% endif %}\n\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n -- finally, drop the existing/backup relation after the commit\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.get_create_table_as_sql", "macro.dbt.create_indexes", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010744, "supported_languages": ["sql"]}, "macro.dbt.get_quoted_csv": {"name": "get_quoted_csv", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.get_quoted_csv", "macro_sql": "{% macro get_quoted_csv(column_names) %}\n\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote(col)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010772, "supported_languages": null}, "macro.dbt.diff_columns": {"name": "diff_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.diff_columns", "macro_sql": "{% macro diff_columns(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% set source_names = source_columns | map(attribute = 'column') | list %}\n {% set target_names = target_columns | map(attribute = 'column') | list %}\n\n {# --check whether the name attribute exists in the target - this does not perform a data type check #}\n {% for sc in source_columns %}\n {% if sc.name not in target_names %}\n {{ result.append(sc) }}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0107882, "supported_languages": null}, "macro.dbt.diff_column_data_types": {"name": "diff_column_data_types", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.diff_column_data_types", "macro_sql": "{% macro diff_column_data_types(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% for sc in source_columns %}\n {% set tc = target_columns | selectattr(\"name\", \"equalto\", sc.name) | list | first %}\n {% if tc %}\n {% if sc.data_type != tc.data_type and not sc.can_expand_to(other_column=tc) %}\n {{ result.append( { 'column_name': tc.name, 'new_type': sc.data_type } ) }}\n {% endif %}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0107992, "supported_languages": null}, "macro.dbt.get_merge_update_columns": {"name": "get_merge_update_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.get_merge_update_columns", "macro_sql": "{% macro get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {{ return(adapter.dispatch('get_merge_update_columns', 'dbt')(merge_update_columns, merge_exclude_columns, dest_columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_merge_update_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0108259, "supported_languages": null}, "macro.dbt.default__get_merge_update_columns": {"name": "default__get_merge_update_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.default__get_merge_update_columns", "macro_sql": "{% macro default__get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {%- set default_cols = dest_columns | map(attribute=\"quoted\") | list -%}\n\n {%- if merge_update_columns and merge_exclude_columns -%}\n {{ exceptions.raise_compiler_error(\n 'Model cannot specify merge_update_columns and merge_exclude_columns. Please update model to use only one config'\n )}}\n {%- elif merge_update_columns -%}\n {%- set update_columns = merge_update_columns -%}\n {%- elif merge_exclude_columns -%}\n {%- set update_columns = [] -%}\n {%- for column in dest_columns -%}\n {% if column.column | lower not in merge_exclude_columns | map(\"lower\") | list %}\n {%- do update_columns.append(column.quoted) -%}\n {% endif %}\n {%- endfor -%}\n {%- else -%}\n {%- set update_columns = default_cols -%}\n {%- endif -%}\n\n {{ return(update_columns) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010836, "supported_languages": null}, "macro.dbt.get_merge_sql": {"name": "get_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.get_merge_sql", "macro_sql": "{% macro get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) -%}\n -- back compat for old kwarg name\n {% set incremental_predicates = kwargs.get('predicates', incremental_predicates) %}\n {{ adapter.dispatch('get_merge_sql', 'dbt')(target, source, unique_key, dest_columns, incremental_predicates) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.010859, "supported_languages": null}, "macro.dbt.default__get_merge_sql": {"name": "default__get_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.default__get_merge_sql", "macro_sql": "{% macro default__get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) -%}\n {%- set predicates = [] if incremental_predicates is none else [] + incremental_predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set merge_update_columns = config.get('merge_update_columns') -%}\n {%- set merge_exclude_columns = config.get('merge_exclude_columns') -%}\n {%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}\n {% for key in unique_key %}\n {% set this_key_match %}\n DBT_INTERNAL_SOURCE.{{ key }} = DBT_INTERNAL_DEST.{{ key }}\n {% endset %}\n {% do predicates.append(this_key_match) %}\n {% endfor %}\n {% else %}\n {% set source_unique_key = (\"DBT_INTERNAL_SOURCE.\" ~ unique_key) | trim %}\n\t {% set target_unique_key = (\"DBT_INTERNAL_DEST.\" ~ unique_key) | trim %}\n\t {% set unique_key_match = equals(source_unique_key, target_unique_key) | trim %}\n {% do predicates.append(unique_key_match) %}\n {% endif %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n {{ sql_header if sql_header is not none }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{\"(\" ~ predicates | join(\") and (\") ~ \")\"}}\n\n {% if unique_key %}\n when matched then update set\n {% for column_name in update_columns -%}\n {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}\n {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n {% endif %}\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv", "macro.dbt.get_merge_update_columns", "macro.dbt.equals"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0121658, "supported_languages": null}, "macro.dbt.get_delete_insert_merge_sql": {"name": "get_delete_insert_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.get_delete_insert_merge_sql", "macro_sql": "{% macro get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}\n {{ adapter.dispatch('get_delete_insert_merge_sql', 'dbt')(target, source, unique_key, dest_columns, incremental_predicates) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_delete_insert_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.012267, "supported_languages": null}, "macro.dbt.default__get_delete_insert_merge_sql": {"name": "default__get_delete_insert_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.default__get_delete_insert_merge_sql", "macro_sql": "{% macro default__get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n {% if unique_key %}\n {% if unique_key is string %}\n {% set unique_key = [unique_key] %}\n {% endif %}\n\n {%- set unique_key_str = unique_key|join(', ') -%}\n\n delete from {{ target }} as DBT_INTERNAL_DEST\n where ({{ unique_key_str }}) in (\n select distinct {{ unique_key_str }}\n from {{ source }} as DBT_INTERNAL_SOURCE\n )\n {%- if incremental_predicates %}\n {% for predicate in incremental_predicates %}\n and {{ predicate }}\n {% endfor %}\n {%- endif -%};\n\n {% endif %}\n\n insert into {{ target }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ source }}\n )\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.012331, "supported_languages": null}, "macro.dbt.get_insert_overwrite_merge_sql": {"name": "get_insert_overwrite_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.get_insert_overwrite_merge_sql", "macro_sql": "{% macro get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header=false) -%}\n {{ adapter.dispatch('get_insert_overwrite_merge_sql', 'dbt')(target, source, dest_columns, predicates, include_sql_header) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_insert_overwrite_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.012369, "supported_languages": null}, "macro.dbt.default__get_insert_overwrite_merge_sql": {"name": "default__get_insert_overwrite_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.default__get_insert_overwrite_merge_sql", "macro_sql": "{% macro default__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) -%}\n {#-- The only time include_sql_header is True: --#}\n {#-- BigQuery + insert_overwrite strategy + \"static\" partitions config --#}\n {#-- We should consider including the sql header at the materialization level instead --#}\n\n {%- set predicates = [] if predicates is none else [] + predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none and include_sql_header }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on FALSE\n\n when not matched by source\n {% if predicates %} and {{ predicates | join(' and ') }} {% endif %}\n then delete\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.012474, "supported_languages": null}, "macro.dbt.is_incremental": {"name": "is_incremental", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/is_incremental.sql", "original_file_path": "macros/materializations/models/incremental/is_incremental.sql", "unique_id": "macro.dbt.is_incremental", "macro_sql": "{% macro is_incremental() %}\n {#-- do not run introspective queries in parsing #}\n {% if not execute %}\n {{ return(False) }}\n {% else %}\n {% set relation = adapter.get_relation(this.database, this.schema, this.table) %}\n {{ return(relation is not none\n and relation.type == 'table'\n and model.config.materialized == 'incremental'\n and not should_full_refresh()) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.012625, "supported_languages": null}, "macro.dbt.get_incremental_append_sql": {"name": "get_incremental_append_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_append_sql", "macro_sql": "{% macro get_incremental_append_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_append_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_append_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.012676, "supported_languages": null}, "macro.dbt.default__get_incremental_append_sql": {"name": "default__get_incremental_append_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_append_sql", "macro_sql": "{% macro default__get_incremental_append_sql(arg_dict) %}\n\n {% do return(get_insert_into_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"dest_columns\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_insert_into_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0126889, "supported_languages": null}, "macro.dbt.get_incremental_delete_insert_sql": {"name": "get_incremental_delete_insert_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_delete_insert_sql", "macro_sql": "{% macro get_incremental_delete_insert_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_delete_insert_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_delete_insert_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0126998, "supported_languages": null}, "macro.dbt.default__get_incremental_delete_insert_sql": {"name": "default__get_incremental_delete_insert_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_delete_insert_sql", "macro_sql": "{% macro default__get_incremental_delete_insert_sql(arg_dict) %}\n\n {% do return(get_delete_insert_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"unique_key\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_delete_insert_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0127108, "supported_languages": null}, "macro.dbt.get_incremental_merge_sql": {"name": "get_incremental_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_merge_sql", "macro_sql": "{% macro get_incremental_merge_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_merge_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.012722, "supported_languages": null}, "macro.dbt.default__get_incremental_merge_sql": {"name": "default__get_incremental_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_merge_sql", "macro_sql": "{% macro default__get_incremental_merge_sql(arg_dict) %}\n\n {% do return(get_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"unique_key\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0127318, "supported_languages": null}, "macro.dbt.get_incremental_insert_overwrite_sql": {"name": "get_incremental_insert_overwrite_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_insert_overwrite_sql", "macro_sql": "{% macro get_incremental_insert_overwrite_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_insert_overwrite_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_insert_overwrite_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0127418, "supported_languages": null}, "macro.dbt.default__get_incremental_insert_overwrite_sql": {"name": "default__get_incremental_insert_overwrite_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_insert_overwrite_sql", "macro_sql": "{% macro default__get_incremental_insert_overwrite_sql(arg_dict) %}\n\n {% do return(get_insert_overwrite_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_insert_overwrite_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.012753, "supported_languages": null}, "macro.dbt.get_incremental_default_sql": {"name": "get_incremental_default_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_default_sql", "macro_sql": "{% macro get_incremental_default_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_default_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_incremental_default_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0127628, "supported_languages": null}, "macro.dbt.default__get_incremental_default_sql": {"name": "default__get_incremental_default_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_default_sql", "macro_sql": "{% macro default__get_incremental_default_sql(arg_dict) %}\n\n {% do return(get_incremental_append_sql(arg_dict)) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_incremental_append_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.012773, "supported_languages": null}, "macro.dbt.get_incremental_microbatch_sql": {"name": "get_incremental_microbatch_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_microbatch_sql", "macro_sql": "{% macro get_incremental_microbatch_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_microbatch_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_incremental_microbatch_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0127969, "supported_languages": null}, "macro.dbt.default__get_incremental_microbatch_sql": {"name": "default__get_incremental_microbatch_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_microbatch_sql", "macro_sql": "{% macro default__get_incremental_microbatch_sql(arg_dict) %}\n\n {{ exceptions.raise_not_implemented('microbatch materialization strategy not implemented for adapter ' + adapter.type()) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.012822, "supported_languages": null}, "macro.dbt.get_insert_into_sql": {"name": "get_insert_into_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_insert_into_sql", "macro_sql": "{% macro get_insert_into_sql(target_relation, temp_relation, dest_columns) %}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n insert into {{ target_relation }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ temp_relation }}\n )\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.012835, "supported_languages": null}, "macro.dbt.materialization_incremental_default": {"name": "materialization_incremental_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/incremental.sql", "original_file_path": "macros/materializations/models/incremental/incremental.sql", "unique_id": "macro.dbt.materialization_incremental_default", "macro_sql": "{% materialization incremental, default -%}\n\n -- relations\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') -%}\n {%- set temp_relation = make_temp_relation(target_relation)-%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation)-%}\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n\n -- configs\n {%- set unique_key = config.get('unique_key') -%}\n {%- set full_refresh_mode = (should_full_refresh() or existing_relation.is_view) -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n\n -- the temp_ and backup_ relations should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation. This has to happen before\n -- BEGIN, in a separate transaction\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation)-%}\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set to_drop = [] %}\n\n {% set incremental_strategy = config.get('incremental_strategy') or 'default' %}\n {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %}\n\n {% if existing_relation is none %}\n {% set build_sql = get_create_table_as_sql(False, target_relation, sql) %}\n {% set relation_for_indexes = target_relation %}\n {% elif full_refresh_mode %}\n {% set build_sql = get_create_table_as_sql(False, intermediate_relation, sql) %}\n {% set relation_for_indexes = intermediate_relation %}\n {% set need_swap = true %}\n {% else %}\n {% do run_query(get_create_table_as_sql(True, temp_relation, sql)) %}\n {% set relation_for_indexes = temp_relation %}\n {% set contract_config = config.get('contract') %}\n {% if not contract_config or not contract_config.enforced %}\n {% do adapter.expand_target_column_types(\n from_relation=temp_relation,\n to_relation=target_relation) %}\n {% endif %}\n {#-- Process schema changes. Returns dict of changes if successful. Use source columns for upserting/merging --#}\n {% set dest_columns = process_schema_changes(on_schema_change, temp_relation, existing_relation) %}\n {% if not dest_columns %}\n {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %}\n {% endif %}\n\n {#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#}\n {% set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) %}\n {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': temp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'incremental_predicates': incremental_predicates }) %}\n {% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %}\n\n {% endif %}\n\n {% call statement(\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% if existing_relation is none or existing_relation.is_view or should_full_refresh() %}\n {% do create_indexes(relation_for_indexes) %}\n {% endif %}\n\n {% if need_swap %}\n {% do adapter.rename_relation(target_relation, backup_relation) %}\n {% do adapter.rename_relation(intermediate_relation, target_relation) %}\n {% do to_drop.append(backup_relation) %}\n {% endif %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {% do adapter.commit() %}\n\n {% for rel in to_drop %}\n {% do adapter.drop_relation(rel) %}\n {% endfor %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_temp_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.should_full_refresh", "macro.dbt.incremental_validate_on_schema_change", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks", "macro.dbt.get_create_table_as_sql", "macro.dbt.run_query", "macro.dbt.process_schema_changes", "macro.dbt.statement", "macro.dbt.create_indexes", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.012867, "supported_languages": ["sql"]}, "macro.dbt.incremental_validate_on_schema_change": {"name": "incremental_validate_on_schema_change", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.incremental_validate_on_schema_change", "macro_sql": "{% macro incremental_validate_on_schema_change(on_schema_change, default='ignore') %}\n\n {% if on_schema_change not in ['sync_all_columns', 'append_new_columns', 'fail', 'ignore'] %}\n\n {% set log_message = 'Invalid value for on_schema_change (%s) specified. Setting default value of %s.' % (on_schema_change, default) %}\n {% do log(log_message) %}\n\n {{ return(default) }}\n\n {% else %}\n\n {{ return(on_schema_change) }}\n\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.012906, "supported_languages": null}, "macro.dbt.check_for_schema_changes": {"name": "check_for_schema_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.check_for_schema_changes", "macro_sql": "{% macro check_for_schema_changes(source_relation, target_relation) %}\n\n {% set schema_changed = False %}\n\n {%- set source_columns = adapter.get_columns_in_relation(source_relation) -%}\n {%- set target_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set source_not_in_target = diff_columns(source_columns, target_columns) -%}\n {%- set target_not_in_source = diff_columns(target_columns, source_columns) -%}\n\n {% set new_target_types = diff_column_data_types(source_columns, target_columns) %}\n\n {% if source_not_in_target != [] %}\n {% set schema_changed = True %}\n {% elif target_not_in_source != [] or new_target_types != [] %}\n {% set schema_changed = True %}\n {% elif new_target_types != [] %}\n {% set schema_changed = True %}\n {% endif %}\n\n {% set changes_dict = {\n 'schema_changed': schema_changed,\n 'source_not_in_target': source_not_in_target,\n 'target_not_in_source': target_not_in_source,\n 'source_columns': source_columns,\n 'target_columns': target_columns,\n 'new_target_types': new_target_types\n } %}\n\n {% set msg %}\n In {{ target_relation }}:\n Schema changed: {{ schema_changed }}\n Source columns not in target: {{ source_not_in_target }}\n Target columns not in source: {{ target_not_in_source }}\n New column types: {{ new_target_types }}\n {% endset %}\n\n {% do log(msg) %}\n\n {{ return(changes_dict) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.diff_columns", "macro.dbt.diff_column_data_types"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.012925, "supported_languages": null}, "macro.dbt.sync_column_schemas": {"name": "sync_column_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.sync_column_schemas", "macro_sql": "{% macro sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {%- set add_to_target_arr = schema_changes_dict['source_not_in_target'] -%}\n\n {%- if on_schema_change == 'append_new_columns'-%}\n {%- if add_to_target_arr | length > 0 -%}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, none) -%}\n {%- endif -%}\n\n {% elif on_schema_change == 'sync_all_columns' %}\n {%- set remove_from_target_arr = schema_changes_dict['target_not_in_source'] -%}\n {%- set new_target_types = schema_changes_dict['new_target_types'] -%}\n\n {% if add_to_target_arr | length > 0 or remove_from_target_arr | length > 0 %}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, remove_from_target_arr) -%}\n {% endif %}\n\n {% if new_target_types != [] %}\n {% for ntt in new_target_types %}\n {% set column_name = ntt['column_name'] %}\n {% set new_type = ntt['new_type'] %}\n {% do alter_column_type(target_relation, column_name, new_type) %}\n {% endfor %}\n {% endif %}\n\n {% endif %}\n\n {% set schema_change_message %}\n In {{ target_relation }}:\n Schema change approach: {{ on_schema_change }}\n Columns added: {{ add_to_target_arr }}\n Columns removed: {{ remove_from_target_arr }}\n Data types changed: {{ new_target_types }}\n {% endset %}\n\n {% do log(schema_change_message) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.alter_relation_add_remove_columns", "macro.dbt.alter_column_type"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0129359, "supported_languages": null}, "macro.dbt.process_schema_changes": {"name": "process_schema_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.process_schema_changes", "macro_sql": "{% macro process_schema_changes(on_schema_change, source_relation, target_relation) %}\n\n {% if on_schema_change == 'ignore' %}\n\n {{ return({}) }}\n\n {% else %}\n\n {% set schema_changes_dict = check_for_schema_changes(source_relation, target_relation) %}\n\n {% if schema_changes_dict['schema_changed'] %}\n\n {% if on_schema_change == 'fail' %}\n\n {% set fail_msg %}\n The source and target schemas on this incremental model are out of sync!\n They can be reconciled in several ways:\n - set the `on_schema_change` config to either append_new_columns or sync_all_columns, depending on your situation.\n - Re-run the incremental model with `full_refresh: True` to update the target schema.\n - update the schema manually and re-run the process.\n\n Additional troubleshooting context:\n Source columns not in target: {{ schema_changes_dict['source_not_in_target'] }}\n Target columns not in source: {{ schema_changes_dict['target_not_in_source'] }}\n New column types: {{ schema_changes_dict['new_target_types'] }}\n {% endset %}\n\n {% do exceptions.raise_compiler_error(fail_msg) %}\n\n {# -- unless we ignore, run the sync operation per the config #}\n {% else %}\n\n {% do sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {% endif %}\n\n {% endif %}\n\n {{ return(schema_changes_dict['source_columns']) }}\n\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.check_for_schema_changes", "macro.dbt.sync_column_schemas"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.012948, "supported_languages": null}, "macro.dbt.can_clone_table": {"name": "can_clone_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/can_clone_table.sql", "original_file_path": "macros/materializations/models/clone/can_clone_table.sql", "unique_id": "macro.dbt.can_clone_table", "macro_sql": "{% macro can_clone_table() %}\n {{ return(adapter.dispatch('can_clone_table', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__can_clone_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.012969, "supported_languages": null}, "macro.dbt.default__can_clone_table": {"name": "default__can_clone_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/can_clone_table.sql", "original_file_path": "macros/materializations/models/clone/can_clone_table.sql", "unique_id": "macro.dbt.default__can_clone_table", "macro_sql": "{% macro default__can_clone_table() %}\n {{ return(False) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.01298, "supported_languages": null}, "macro.dbt.create_or_replace_clone": {"name": "create_or_replace_clone", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/create_or_replace_clone.sql", "original_file_path": "macros/materializations/models/clone/create_or_replace_clone.sql", "unique_id": "macro.dbt.create_or_replace_clone", "macro_sql": "{% macro create_or_replace_clone(this_relation, defer_relation) %}\n {{ return(adapter.dispatch('create_or_replace_clone', 'dbt')(this_relation, defer_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__create_or_replace_clone"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.013037, "supported_languages": null}, "macro.dbt.default__create_or_replace_clone": {"name": "default__create_or_replace_clone", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/create_or_replace_clone.sql", "original_file_path": "macros/materializations/models/clone/create_or_replace_clone.sql", "unique_id": "macro.dbt.default__create_or_replace_clone", "macro_sql": "{% macro default__create_or_replace_clone(this_relation, defer_relation) %}\n create or replace table {{ this_relation.render() }} clone {{ defer_relation.render() }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0130482, "supported_languages": null}, "macro.dbt.materialization_clone_default": {"name": "materialization_clone_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/clone.sql", "original_file_path": "macros/materializations/models/clone/clone.sql", "unique_id": "macro.dbt.materialization_clone_default", "macro_sql": "{%- materialization clone, default -%}\n\n {%- set relations = {'relations': []} -%}\n\n {%- if not defer_relation -%}\n -- nothing to do\n {{ log(\"No relation found in state manifest for \" ~ model.unique_id, info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n\n {%- if existing_relation and not flags.FULL_REFRESH -%}\n -- noop!\n {{ log(\"Relation \" ~ existing_relation ~ \" already exists\", info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set other_existing_relation = load_cached_relation(defer_relation) -%}\n\n -- If this is a database that can do zero-copy cloning of tables, and the other relation is a table, then this will be a table\n -- Otherwise, this will be a view\n\n {% set can_clone_table = can_clone_table() %}\n\n {%- if other_existing_relation and other_existing_relation.type == 'table' and can_clone_table -%}\n\n {%- set target_relation = this.incorporate(type='table') -%}\n {% if existing_relation is not none and not existing_relation.is_table %}\n {{ log(\"Dropping relation \" ~ existing_relation.render() ~ \" because it is of type \" ~ existing_relation.type) }}\n {{ drop_relation_if_exists(existing_relation) }}\n {% endif %}\n\n -- as a general rule, data platforms that can clone tables can also do atomic 'create or replace'\n {% if target_relation.database == defer_relation.database and\n target_relation.schema == defer_relation.schema and\n target_relation.identifier == defer_relation.identifier %}\n {{ log(\"Target relation and defer relation are the same, skipping clone for relation: \" ~ target_relation.render()) }}\n {% else %}\n {% call statement('main') %}\n {{ create_or_replace_clone(target_relation, defer_relation) }}\n {% endcall %}\n {% endif %}\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n {% do persist_docs(target_relation, model) %}\n\n {{ return({'relations': [target_relation]}) }}\n\n {%- else -%}\n\n {%- set target_relation = this.incorporate(type='view') -%}\n\n -- reuse the view materialization\n -- TODO: support actual dispatch for materialization macros\n -- Tracking ticket: https://github.com/dbt-labs/dbt-core/issues/7799\n {% set search_name = \"materialization_view_\" ~ adapter.type() %}\n {% if not search_name in context %}\n {% set search_name = \"materialization_view_default\" %}\n {% endif %}\n {% set materialization_macro = context[search_name] %}\n {% set relations = materialization_macro() %}\n {{ return(relations) }}\n\n {%- endif -%}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.can_clone_table", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.create_or_replace_clone", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.013071, "supported_languages": ["sql"]}, "macro.dbt.materialization_seed_default": {"name": "materialization_seed_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/seed.sql", "original_file_path": "macros/materializations/seeds/seed.sql", "unique_id": "macro.dbt.materialization_seed_default", "macro_sql": "{% materialization seed, default %}\n\n {%- set identifier = model['alias'] -%}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set grant_config = config.get('grants') -%}\n {%- set agate_table = load_agate_table() -%}\n -- grab current tables grants config for comparison later on\n\n {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% set create_table_sql = \"\" %}\n {% if exists_as_view %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a view\".format(old_relation.render())) }}\n {% elif exists_as_table %}\n {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}\n {% else %}\n {% set create_table_sql = create_csv_table(model, agate_table) %}\n {% endif %}\n\n {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}\n {% set rows_affected = (agate_table.rows | length) %}\n {% set sql = load_csv_rows(model, agate_table) %}\n\n {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}\n {{ get_csv_sql(create_table_sql, sql) }};\n {% endcall %}\n\n {% set target_relation = this.incorporate(type='table') %}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if full_refresh_mode or not exists_as_table %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.run_hooks", "macro.dbt.reset_csv_table", "macro.dbt.create_csv_table", "macro.dbt.load_csv_rows", "macro.dbt.noop_statement", "macro.dbt.get_csv_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.013102, "supported_languages": ["sql"]}, "macro.dbt.create_csv_table": {"name": "create_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.create_csv_table", "macro_sql": "{% macro create_csv_table(model, agate_table) -%}\n {{ adapter.dispatch('create_csv_table', 'dbt')(model, agate_table) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0131352, "supported_languages": null}, "macro.dbt.default__create_csv_table": {"name": "default__create_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__create_csv_table", "macro_sql": "{% macro default__create_csv_table(model, agate_table) %}\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n\n {% set sql %}\n create table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0131462, "supported_languages": null}, "macro.dbt.reset_csv_table": {"name": "reset_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.reset_csv_table", "macro_sql": "{% macro reset_csv_table(model, full_refresh, old_relation, agate_table) -%}\n {{ adapter.dispatch('reset_csv_table', 'dbt')(model, full_refresh, old_relation, agate_table) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__reset_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.01316, "supported_languages": null}, "macro.dbt.default__reset_csv_table": {"name": "default__reset_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__reset_csv_table", "macro_sql": "{% macro default__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% set sql = \"\" %}\n {% if full_refresh %}\n {{ adapter.drop_relation(old_relation) }}\n {% set sql = create_csv_table(model, agate_table) %}\n {% else %}\n {{ adapter.truncate_relation(old_relation) }}\n {% set sql = \"truncate table \" ~ old_relation.render() %}\n {% endif %}\n\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.013171, "supported_languages": null}, "macro.dbt.get_csv_sql": {"name": "get_csv_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_csv_sql", "macro_sql": "{% macro get_csv_sql(create_or_truncate_sql, insert_sql) %}\n {{ adapter.dispatch('get_csv_sql', 'dbt')(create_or_truncate_sql, insert_sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_csv_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.013224, "supported_languages": null}, "macro.dbt.default__get_csv_sql": {"name": "default__get_csv_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__get_csv_sql", "macro_sql": "{% macro default__get_csv_sql(create_or_truncate_sql, insert_sql) %}\n {{ create_or_truncate_sql }};\n -- dbt seed --\n {{ insert_sql }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.013238, "supported_languages": null}, "macro.dbt.get_binding_char": {"name": "get_binding_char", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_binding_char", "macro_sql": "{% macro get_binding_char() -%}\n {{ adapter.dispatch('get_binding_char', 'dbt')() }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.013248, "supported_languages": null}, "macro.dbt.default__get_binding_char": {"name": "default__get_binding_char", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__get_binding_char", "macro_sql": "{% macro default__get_binding_char() %}\n {{ return('%s') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.013259, "supported_languages": null}, "macro.dbt.get_batch_size": {"name": "get_batch_size", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_batch_size", "macro_sql": "{% macro get_batch_size() -%}\n {{ return(adapter.dispatch('get_batch_size', 'dbt')()) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_batch_size"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.01327, "supported_languages": null}, "macro.dbt.default__get_batch_size": {"name": "default__get_batch_size", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__get_batch_size", "macro_sql": "{% macro default__get_batch_size() %}\n {{ return(10000) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.01328, "supported_languages": null}, "macro.dbt.get_seed_column_quoted_csv": {"name": "get_seed_column_quoted_csv", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_seed_column_quoted_csv", "macro_sql": "{% macro get_seed_column_quoted_csv(model, column_names) %}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote_seed_column(col, quote_seed_column)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.01329, "supported_languages": null}, "macro.dbt.load_csv_rows": {"name": "load_csv_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.load_csv_rows", "macro_sql": "{% macro load_csv_rows(model, agate_table) -%}\n {{ adapter.dispatch('load_csv_rows', 'dbt')(model, agate_table) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__load_csv_rows"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.013301, "supported_languages": null}, "macro.dbt.default__load_csv_rows": {"name": "default__load_csv_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__load_csv_rows", "macro_sql": "{% macro default__load_csv_rows(model, agate_table) %}\n\n {% set batch_size = get_batch_size() %}\n\n {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}\n {% set bindings = [] %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert into {{ this.render() }} ({{ cols_sql }}) values\n {% for row in chunk -%}\n ({%- for column in agate_table.column_names -%}\n {{ get_binding_char() }}\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_batch_size", "macro.dbt.get_seed_column_quoted_csv", "macro.dbt.get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.013311, "supported_languages": null}, "macro.dbt.scalar_function_sql": {"name": "scalar_function_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.scalar_function_sql", "macro_sql": "{% macro scalar_function_sql(target_relation) %}\n {{ return(adapter.dispatch('scalar_function_sql', 'dbt')(target_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__scalar_function_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.013334, "supported_languages": null}, "macro.dbt.default__scalar_function_sql": {"name": "default__scalar_function_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.default__scalar_function_sql", "macro_sql": "{% macro default__scalar_function_sql(target_relation) %}\n {{ scalar_function_create_replace_signature_sql(target_relation) }}\n {{ scalar_function_body_sql() }};\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.scalar_function_create_replace_signature_sql", "macro.dbt.scalar_function_body_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0133471, "supported_languages": null}, "macro.dbt.scalar_function_create_replace_signature_sql": {"name": "scalar_function_create_replace_signature_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.scalar_function_create_replace_signature_sql", "macro_sql": "{% macro scalar_function_create_replace_signature_sql(target_relation) %}\n {{ return(adapter.dispatch('scalar_function_create_replace_signature_sql', 'dbt')(target_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__scalar_function_create_replace_signature_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.013358, "supported_languages": null}, "macro.dbt.default__scalar_function_create_replace_signature_sql": {"name": "default__scalar_function_create_replace_signature_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.default__scalar_function_create_replace_signature_sql", "macro_sql": "{% macro default__scalar_function_create_replace_signature_sql(target_relation) %}\n CREATE OR REPLACE FUNCTION {{ target_relation.render() }} ({{ formatted_scalar_function_args_sql()}}) RETURNS {{ model.return_type.type }} AS\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.formatted_scalar_function_args_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0133681, "supported_languages": null}, "macro.dbt.formatted_scalar_function_args_sql": {"name": "formatted_scalar_function_args_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.formatted_scalar_function_args_sql", "macro_sql": "{% macro formatted_scalar_function_args_sql() %}\n {{ return(adapter.dispatch('formatted_scalar_function_args_sql', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__formatted_scalar_function_args_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0133839, "supported_languages": null}, "macro.dbt.default__formatted_scalar_function_args_sql": {"name": "default__formatted_scalar_function_args_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.default__formatted_scalar_function_args_sql", "macro_sql": "{% macro default__formatted_scalar_function_args_sql() %}\n {% set args = [] %}\n {% for arg in model.arguments -%}\n {%- do args.append(arg.name ~ ' ' ~ arg.type) -%}\n {%- endfor %}\n {{ args | join(', ') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0133948, "supported_languages": null}, "macro.dbt.scalar_function_body_sql": {"name": "scalar_function_body_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.scalar_function_body_sql", "macro_sql": "{% macro scalar_function_body_sql() %}\n {{ return(adapter.dispatch('scalar_function_body_sql', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__scalar_function_body_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0134048, "supported_languages": null}, "macro.dbt.default__scalar_function_body_sql": {"name": "default__scalar_function_body_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.default__scalar_function_body_sql", "macro_sql": "{% macro default__scalar_function_body_sql() %}\n $$\n {{ model.compiled_code }}\n $$ LANGUAGE SQL\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.013416, "supported_languages": null}, "macro.dbt.function_execute_build_sql": {"name": "function_execute_build_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/helpers.sql", "original_file_path": "macros/materializations/functions/helpers.sql", "unique_id": "macro.dbt.function_execute_build_sql", "macro_sql": "{% macro function_execute_build_sql(build_sql, existing_relation, target_relation) %}\n {{ return(adapter.dispatch('function_execute_build_sql', 'dbt')(build_sql, existing_relation, target_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__function_execute_build_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.013437, "supported_languages": null}, "macro.dbt.default__function_execute_build_sql": {"name": "default__function_execute_build_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/helpers.sql", "original_file_path": "macros/materializations/functions/helpers.sql", "unique_id": "macro.dbt.default__function_execute_build_sql", "macro_sql": "{% macro default__function_execute_build_sql(build_sql, existing_relation, target_relation) %}\n\n {% set grant_config = config.get('grants') %}\n\n {% call statement(name=\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ adapter.commit() }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0134468, "supported_languages": null}, "macro.dbt.get_function_macro": {"name": "get_function_macro", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/helpers.sql", "original_file_path": "macros/materializations/functions/helpers.sql", "unique_id": "macro.dbt.get_function_macro", "macro_sql": "{% macro get_function_macro(function_type, function_language) %}\n {{ return(adapter.dispatch('get_function_macro', 'dbt')(function_type, function_language)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_function_macro"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.013458, "supported_languages": null}, "macro.dbt.default__get_function_macro": {"name": "default__get_function_macro", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/helpers.sql", "original_file_path": "macros/materializations/functions/helpers.sql", "unique_id": "macro.dbt.default__get_function_macro", "macro_sql": "{% macro default__get_function_macro(function_type, function_language) %}\n {% set macro_name = function_type ~ \"_function_\" ~ function_language %}\n {% if not macro_name in context %}\n {{ exceptions.raise_not_implemented(function_language ~ ' ' ~ function_type ~ ' function not implemented for adapter ' ~adapter.type()) }}\n {% endif %}\n {% set macro = context[macro_name] %}\n {{ return(macro) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0134678, "supported_languages": null}, "macro.dbt.materialization_function_default": {"name": "materialization_function_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/function.sql", "original_file_path": "macros/materializations/functions/function.sql", "unique_id": "macro.dbt.materialization_function_default", "macro_sql": "{% materialization function, default %}\n {% set existing_relation = load_cached_relation(this) %}\n {% set target_relation = this.incorporate(type=this.Function) %}\n\n {{ run_hooks(pre_hooks) }}\n\n {% set function_type_macro = get_function_macro('scalar', 'sql') %}\n {% set build_sql = function_type_macro(target_relation) %}\n\n {{ function_execute_build_sql(build_sql, existing_relation, target_relation) }}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.run_hooks", "macro.dbt.get_function_macro", "macro.dbt.function_execute_build_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.01349, "supported_languages": ["sql"]}, "macro.dbt.generate_alias_name": {"name": "generate_alias_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_alias.sql", "original_file_path": "macros/get_custom_name/get_custom_alias.sql", "unique_id": "macro.dbt.generate_alias_name", "macro_sql": "{% macro generate_alias_name(custom_alias_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_alias_name', 'dbt')(custom_alias_name, node)) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__generate_alias_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.013665, "supported_languages": null}, "macro.dbt.default__generate_alias_name": {"name": "default__generate_alias_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_alias.sql", "original_file_path": "macros/get_custom_name/get_custom_alias.sql", "unique_id": "macro.dbt.default__generate_alias_name", "macro_sql": "{% macro default__generate_alias_name(custom_alias_name=none, node=none) -%}\n\n {%- if custom_alias_name -%}\n\n {{ custom_alias_name | trim }}\n\n {%- elif node.version -%}\n\n {{ return(node.name ~ \"_v\" ~ (node.version | replace(\".\", \"_\"))) }}\n\n {%- else -%}\n\n {{ node.name }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0136828, "supported_languages": null}, "macro.dbt.generate_schema_name": {"name": "generate_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "unique_id": "macro.dbt.generate_schema_name", "macro_sql": "{% macro generate_schema_name(custom_schema_name=none, node=none) -%}\n {{ return(adapter.dispatch('generate_schema_name', 'dbt')(custom_schema_name, node)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__generate_schema_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.013712, "supported_languages": null}, "macro.dbt.default__generate_schema_name": {"name": "default__generate_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "unique_id": "macro.dbt.default__generate_schema_name", "macro_sql": "{% macro default__generate_schema_name(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if custom_schema_name is none -%}\n\n {{ default_schema }}\n\n {%- else -%}\n\n {{ default_schema }}_{{ custom_schema_name | trim }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0137239, "supported_languages": null}, "macro.dbt.generate_schema_name_for_env": {"name": "generate_schema_name_for_env", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "unique_id": "macro.dbt.generate_schema_name_for_env", "macro_sql": "{% macro generate_schema_name_for_env(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if target.name == 'prod' and custom_schema_name is not none -%}\n\n {{ custom_schema_name | trim }}\n\n {%- else -%}\n\n {{ default_schema }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0137348, "supported_languages": null}, "macro.dbt.generate_database_name": {"name": "generate_database_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "unique_id": "macro.dbt.generate_database_name", "macro_sql": "{% macro generate_database_name(custom_database_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_database_name', 'dbt')(custom_database_name, node)) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__generate_database_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0137558, "supported_languages": null}, "macro.dbt.default__generate_database_name": {"name": "default__generate_database_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "unique_id": "macro.dbt.default__generate_database_name", "macro_sql": "{% macro default__generate_database_name(custom_database_name=none, node=none) -%}\n {%- set default_database = target.database -%}\n {%- if custom_database_name is none -%}\n\n {{ default_database }}\n\n {%- else -%}\n\n {{ custom_database_name }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.013767, "supported_languages": null}, "macro.dbt.get_drop_sql": {"name": "get_drop_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.get_drop_sql", "macro_sql": "{%- macro get_drop_sql(relation) -%}\n {{- log('Applying DROP to: ' ~ relation) -}}\n {{- adapter.dispatch('get_drop_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_drop_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.013788, "supported_languages": null}, "macro.dbt.default__get_drop_sql": {"name": "default__get_drop_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.default__get_drop_sql", "macro_sql": "{%- macro default__get_drop_sql(relation) -%}\n\n {%- if relation.is_view -%}\n {{ drop_view(relation) }}\n\n {%- elif relation.is_table -%}\n {{ drop_table(relation) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ drop_materialized_view(relation) }}\n\n {%- else -%}\n drop {{ relation.type }} if exists {{ relation.render() }} cascade\n\n {%- endif -%}\n\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.drop_view", "macro.dbt.drop_table", "macro.dbt.drop_materialized_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.013799, "supported_languages": null}, "macro.dbt.drop_relation": {"name": "drop_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.drop_relation", "macro_sql": "{% macro drop_relation(relation) -%}\n {{ return(adapter.dispatch('drop_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__drop_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.01381, "supported_languages": null}, "macro.dbt.default__drop_relation": {"name": "default__drop_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.default__drop_relation", "macro_sql": "{% macro default__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n {{ get_drop_sql(relation) }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.get_drop_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.01382, "supported_languages": null}, "macro.dbt.drop_relation_if_exists": {"name": "drop_relation_if_exists", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.drop_relation_if_exists", "macro_sql": "{% macro drop_relation_if_exists(relation) %}\n {% if relation is not none %}\n {{ adapter.drop_relation(relation) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.013831, "supported_languages": null}, "macro.dbt.get_replace_sql": {"name": "get_replace_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt.get_replace_sql", "macro_sql": "{% macro get_replace_sql(existing_relation, target_relation, sql) %}\n {{- log('Applying REPLACE to: ' ~ existing_relation) -}}\n {{- adapter.dispatch('get_replace_sql', 'dbt')(existing_relation, target_relation, sql) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_replace_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.013851, "supported_languages": null}, "macro.dbt.default__get_replace_sql": {"name": "default__get_replace_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt.default__get_replace_sql", "macro_sql": "{% macro default__get_replace_sql(existing_relation, target_relation, sql) %}\n\n {# /* use a create or replace statement if possible */ #}\n\n {% set is_replaceable = existing_relation.type == target_relation.type and existing_relation.can_be_replaced %}\n\n {% if is_replaceable and existing_relation.is_view %}\n {{ get_replace_view_sql(target_relation, sql) }}\n\n {% elif is_replaceable and existing_relation.is_table %}\n {{ get_replace_table_sql(target_relation, sql) }}\n\n {% elif is_replaceable and existing_relation.is_materialized_view %}\n {{ get_replace_materialized_view_sql(target_relation, sql) }}\n\n {# /* a create or replace statement is not possible, so try to stage and/or backup to be safe */ #}\n\n {# /* create target_relation as an intermediate relation, then swap it out with the existing one using a backup */ #}\n {%- elif target_relation.can_be_renamed and existing_relation.can_be_renamed -%}\n {{ get_create_intermediate_sql(target_relation, sql) }};\n {{ get_create_backup_sql(existing_relation) }};\n {{ get_rename_intermediate_sql(target_relation) }};\n {{ get_drop_backup_sql(existing_relation) }}\n\n {# /* create target_relation as an intermediate relation, then swap it out with the existing one without using a backup */ #}\n {%- elif target_relation.can_be_renamed -%}\n {{ get_create_intermediate_sql(target_relation, sql) }};\n {{ get_drop_sql(existing_relation) }};\n {{ get_rename_intermediate_sql(target_relation) }}\n\n {# /* create target_relation in place by first backing up the existing relation */ #}\n {%- elif existing_relation.can_be_renamed -%}\n {{ get_create_backup_sql(existing_relation) }};\n {{ get_create_sql(target_relation, sql) }};\n {{ get_drop_backup_sql(existing_relation) }}\n\n {# /* no renaming is allowed, so just drop and create */ #}\n {%- else -%}\n {{ get_drop_sql(existing_relation) }};\n {{ get_create_sql(target_relation, sql) }}\n\n {%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_replace_view_sql", "macro.dbt.get_replace_table_sql", "macro.dbt.get_replace_materialized_view_sql", "macro.dbt.get_create_intermediate_sql", "macro.dbt.get_create_backup_sql", "macro.dbt.get_rename_intermediate_sql", "macro.dbt.get_drop_backup_sql", "macro.dbt.get_drop_sql", "macro.dbt.get_create_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.013862, "supported_languages": null}, "macro.dbt.get_create_intermediate_sql": {"name": "get_create_intermediate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create_intermediate.sql", "original_file_path": "macros/relations/create_intermediate.sql", "unique_id": "macro.dbt.get_create_intermediate_sql", "macro_sql": "{%- macro get_create_intermediate_sql(relation, sql) -%}\n {{- log('Applying CREATE INTERMEDIATE to: ' ~ relation) -}}\n {{- adapter.dispatch('get_create_intermediate_sql', 'dbt')(relation, sql) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_create_intermediate_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0138822, "supported_languages": null}, "macro.dbt.default__get_create_intermediate_sql": {"name": "default__get_create_intermediate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create_intermediate.sql", "original_file_path": "macros/relations/create_intermediate.sql", "unique_id": "macro.dbt.default__get_create_intermediate_sql", "macro_sql": "{%- macro default__get_create_intermediate_sql(relation, sql) -%}\n\n -- get the standard intermediate name\n {% set intermediate_relation = make_intermediate_relation(relation) %}\n\n -- drop any pre-existing intermediate\n {{ get_drop_sql(intermediate_relation) }};\n\n {{ get_create_sql(intermediate_relation, sql) }}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.make_intermediate_relation", "macro.dbt.get_drop_sql", "macro.dbt.get_create_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0138922, "supported_languages": null}, "macro.dbt.drop_schema_named": {"name": "drop_schema_named", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/schema.sql", "original_file_path": "macros/relations/schema.sql", "unique_id": "macro.dbt.drop_schema_named", "macro_sql": "{% macro drop_schema_named(schema_name) %}\n {{ return(adapter.dispatch('drop_schema_named', 'dbt') (schema_name)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__drop_schema_named"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.013912, "supported_languages": null}, "macro.dbt.default__drop_schema_named": {"name": "default__drop_schema_named", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/schema.sql", "original_file_path": "macros/relations/schema.sql", "unique_id": "macro.dbt.default__drop_schema_named", "macro_sql": "{% macro default__drop_schema_named(schema_name) %}\n {% set schema_relation = api.Relation.create(schema=schema_name) %}\n {{ adapter.drop_schema(schema_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.013922, "supported_languages": null}, "macro.dbt.get_drop_backup_sql": {"name": "get_drop_backup_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop_backup.sql", "original_file_path": "macros/relations/drop_backup.sql", "unique_id": "macro.dbt.get_drop_backup_sql", "macro_sql": "{%- macro get_drop_backup_sql(relation) -%}\n {{- log('Applying DROP BACKUP to: ' ~ relation) -}}\n {{- adapter.dispatch('get_drop_backup_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_drop_backup_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.013942, "supported_languages": null}, "macro.dbt.default__get_drop_backup_sql": {"name": "default__get_drop_backup_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop_backup.sql", "original_file_path": "macros/relations/drop_backup.sql", "unique_id": "macro.dbt.default__get_drop_backup_sql", "macro_sql": "{%- macro default__get_drop_backup_sql(relation) -%}\n\n -- get the standard backup name\n {% set backup_relation = make_backup_relation(relation, relation.type) %}\n\n {{ get_drop_sql(backup_relation) }}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.make_backup_relation", "macro.dbt.get_drop_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.013952, "supported_languages": null}, "macro.dbt.get_rename_sql": {"name": "get_rename_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename.sql", "original_file_path": "macros/relations/rename.sql", "unique_id": "macro.dbt.get_rename_sql", "macro_sql": "{%- macro get_rename_sql(relation, new_name) -%}\n {{- log('Applying RENAME to: ' ~ relation) -}}\n {{- adapter.dispatch('get_rename_sql', 'dbt')(relation, new_name) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_rename_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.013972, "supported_languages": null}, "macro.dbt.default__get_rename_sql": {"name": "default__get_rename_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename.sql", "original_file_path": "macros/relations/rename.sql", "unique_id": "macro.dbt.default__get_rename_sql", "macro_sql": "{%- macro default__get_rename_sql(relation, new_name) -%}\n\n {%- if relation.is_view -%}\n {{ get_rename_view_sql(relation, new_name) }}\n\n {%- elif relation.is_table -%}\n {{ get_rename_table_sql(relation, new_name) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ get_rename_materialized_view_sql(relation, new_name) }}\n\n {%- else -%}\n {{- exceptions.raise_compiler_error(\"`get_rename_sql` has not been implemented for: \" ~ relation.type ) -}}\n\n {%- endif -%}\n\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.get_rename_view_sql", "macro.dbt.get_rename_table_sql", "macro.dbt.get_rename_materialized_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.013983, "supported_languages": null}, "macro.dbt.rename_relation": {"name": "rename_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename.sql", "original_file_path": "macros/relations/rename.sql", "unique_id": "macro.dbt.rename_relation", "macro_sql": "{% macro rename_relation(from_relation, to_relation) -%}\n {{ return(adapter.dispatch('rename_relation', 'dbt')(from_relation, to_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__rename_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.013993, "supported_languages": null}, "macro.dbt.default__rename_relation": {"name": "default__rename_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename.sql", "original_file_path": "macros/relations/rename.sql", "unique_id": "macro.dbt.default__rename_relation", "macro_sql": "{% macro default__rename_relation(from_relation, to_relation) -%}\n {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %}\n {% call statement('rename_relation') -%}\n alter table {{ from_relation.render() }} rename to {{ target_name }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.014003, "supported_languages": null}, "macro.dbt.get_create_backup_sql": {"name": "get_create_backup_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create_backup.sql", "original_file_path": "macros/relations/create_backup.sql", "unique_id": "macro.dbt.get_create_backup_sql", "macro_sql": "{%- macro get_create_backup_sql(relation) -%}\n {{- log('Applying CREATE BACKUP to: ' ~ relation) -}}\n {{- adapter.dispatch('get_create_backup_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_create_backup_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0140219, "supported_languages": null}, "macro.dbt.default__get_create_backup_sql": {"name": "default__get_create_backup_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create_backup.sql", "original_file_path": "macros/relations/create_backup.sql", "unique_id": "macro.dbt.default__get_create_backup_sql", "macro_sql": "{%- macro default__get_create_backup_sql(relation) -%}\n\n -- get the standard backup name\n {% set backup_relation = make_backup_relation(relation, relation.type) %}\n\n -- drop any pre-existing backup\n {{ get_drop_sql(backup_relation) }};\n\n {{ get_rename_sql(relation, backup_relation.identifier) }}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.make_backup_relation", "macro.dbt.get_drop_sql", "macro.dbt.get_rename_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0140328, "supported_languages": null}, "macro.dbt.get_create_sql": {"name": "get_create_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create.sql", "original_file_path": "macros/relations/create.sql", "unique_id": "macro.dbt.get_create_sql", "macro_sql": "{%- macro get_create_sql(relation, sql) -%}\n {{- log('Applying CREATE to: ' ~ relation) -}}\n {{- adapter.dispatch('get_create_sql', 'dbt')(relation, sql) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_create_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0140522, "supported_languages": null}, "macro.dbt.default__get_create_sql": {"name": "default__get_create_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create.sql", "original_file_path": "macros/relations/create.sql", "unique_id": "macro.dbt.default__get_create_sql", "macro_sql": "{%- macro default__get_create_sql(relation, sql) -%}\n\n {%- if relation.is_view -%}\n {{ get_create_view_as_sql(relation, sql) }}\n\n {%- elif relation.is_table -%}\n {{ get_create_table_as_sql(False, relation, sql) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ get_create_materialized_view_as_sql(relation, sql) }}\n\n {%- else -%}\n {{- exceptions.raise_compiler_error(\"`get_create_sql` has not been implemented for: \" ~ relation.type ) -}}\n\n {%- endif -%}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.get_create_view_as_sql", "macro.dbt.get_create_table_as_sql", "macro.dbt.get_create_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0140631, "supported_languages": null}, "macro.dbt.get_rename_intermediate_sql": {"name": "get_rename_intermediate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename_intermediate.sql", "original_file_path": "macros/relations/rename_intermediate.sql", "unique_id": "macro.dbt.get_rename_intermediate_sql", "macro_sql": "{%- macro get_rename_intermediate_sql(relation) -%}\n {{- log('Applying RENAME INTERMEDIATE to: ' ~ relation) -}}\n {{- adapter.dispatch('get_rename_intermediate_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_rename_intermediate_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0140831, "supported_languages": null}, "macro.dbt.default__get_rename_intermediate_sql": {"name": "default__get_rename_intermediate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename_intermediate.sql", "original_file_path": "macros/relations/rename_intermediate.sql", "unique_id": "macro.dbt.default__get_rename_intermediate_sql", "macro_sql": "{%- macro default__get_rename_intermediate_sql(relation) -%}\n\n -- get the standard intermediate name\n {% set intermediate_relation = make_intermediate_relation(relation) %}\n\n {{ get_rename_sql(intermediate_relation, relation.identifier) }}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.make_intermediate_relation", "macro.dbt.get_rename_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0140932, "supported_languages": null}, "macro.dbt.drop_materialized_view": {"name": "drop_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/drop.sql", "original_file_path": "macros/relations/materialized_view/drop.sql", "unique_id": "macro.dbt.drop_materialized_view", "macro_sql": "{% macro drop_materialized_view(relation) -%}\n {{- adapter.dispatch('drop_materialized_view', 'dbt')(relation) -}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__drop_materialized_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.014113, "supported_languages": null}, "macro.dbt.default__drop_materialized_view": {"name": "default__drop_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/drop.sql", "original_file_path": "macros/relations/materialized_view/drop.sql", "unique_id": "macro.dbt.default__drop_materialized_view", "macro_sql": "{% macro default__drop_materialized_view(relation) -%}\n drop materialized view if exists {{ relation.render() }} cascade\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.014123, "supported_languages": null}, "macro.dbt.get_replace_materialized_view_sql": {"name": "get_replace_materialized_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/replace.sql", "original_file_path": "macros/relations/materialized_view/replace.sql", "unique_id": "macro.dbt.get_replace_materialized_view_sql", "macro_sql": "{% macro get_replace_materialized_view_sql(relation, sql) %}\n {{- adapter.dispatch('get_replace_materialized_view_sql', 'dbt')(relation, sql) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_replace_materialized_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.014143, "supported_languages": null}, "macro.dbt.default__get_replace_materialized_view_sql": {"name": "default__get_replace_materialized_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/replace.sql", "original_file_path": "macros/relations/materialized_view/replace.sql", "unique_id": "macro.dbt.default__get_replace_materialized_view_sql", "macro_sql": "{% macro default__get_replace_materialized_view_sql(relation, sql) %}\n {{ exceptions.raise_compiler_error(\n \"`get_replace_materialized_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.014154, "supported_languages": null}, "macro.dbt.refresh_materialized_view": {"name": "refresh_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/refresh.sql", "original_file_path": "macros/relations/materialized_view/refresh.sql", "unique_id": "macro.dbt.refresh_materialized_view", "macro_sql": "{% macro refresh_materialized_view(relation) %}\n {{- log('Applying REFRESH to: ' ~ relation) -}}\n {{- adapter.dispatch('refresh_materialized_view', 'dbt')(relation) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__refresh_materialized_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.014177, "supported_languages": null}, "macro.dbt.default__refresh_materialized_view": {"name": "default__refresh_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/refresh.sql", "original_file_path": "macros/relations/materialized_view/refresh.sql", "unique_id": "macro.dbt.default__refresh_materialized_view", "macro_sql": "{% macro default__refresh_materialized_view(relation) %}\n {{ exceptions.raise_compiler_error(\"`refresh_materialized_view` has not been implemented for this adapter.\") }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.014192, "supported_languages": null}, "macro.dbt.get_rename_materialized_view_sql": {"name": "get_rename_materialized_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/rename.sql", "original_file_path": "macros/relations/materialized_view/rename.sql", "unique_id": "macro.dbt.get_rename_materialized_view_sql", "macro_sql": "{% macro get_rename_materialized_view_sql(relation, new_name) %}\n {{- adapter.dispatch('get_rename_materialized_view_sql', 'dbt')(relation, new_name) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_rename_materialized_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0142121, "supported_languages": null}, "macro.dbt.default__get_rename_materialized_view_sql": {"name": "default__get_rename_materialized_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/rename.sql", "original_file_path": "macros/relations/materialized_view/rename.sql", "unique_id": "macro.dbt.default__get_rename_materialized_view_sql", "macro_sql": "{% macro default__get_rename_materialized_view_sql(relation, new_name) %}\n {{ exceptions.raise_compiler_error(\n \"`get_rename_materialized_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.014223, "supported_languages": null}, "macro.dbt.get_alter_materialized_view_as_sql": {"name": "get_alter_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt.get_alter_materialized_view_as_sql", "macro_sql": "{% macro get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n {{- log('Applying ALTER to: ' ~ relation) -}}\n {{- adapter.dispatch('get_alter_materialized_view_as_sql', 'dbt')(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n ) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_alter_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0142431, "supported_languages": null}, "macro.dbt.default__get_alter_materialized_view_as_sql": {"name": "default__get_alter_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt.default__get_alter_materialized_view_as_sql", "macro_sql": "{% macro default__get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n {{ exceptions.raise_compiler_error(\"Materialized views have not been implemented for this adapter.\") }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0152268, "supported_languages": null}, "macro.dbt.get_materialized_view_configuration_changes": {"name": "get_materialized_view_configuration_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt.get_materialized_view_configuration_changes", "macro_sql": "{% macro get_materialized_view_configuration_changes(existing_relation, new_config) %}\n /* {#\n It's recommended that configuration changes be formatted as follows:\n {\"\": [{\"action\": \"\", \"context\": ...}]}\n\n For example:\n {\n \"indexes\": [\n {\"action\": \"drop\", \"context\": \"index_abc\"},\n {\"action\": \"create\", \"context\": {\"columns\": [\"column_1\", \"column_2\"], \"type\": \"hash\", \"unique\": True}},\n ],\n }\n\n Either way, `get_materialized_view_configuration_changes` needs to align with `get_alter_materialized_view_as_sql`.\n #} */\n {{- log('Determining configuration changes on: ' ~ existing_relation) -}}\n {%- do return(adapter.dispatch('get_materialized_view_configuration_changes', 'dbt')(existing_relation, new_config)) -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_materialized_view_configuration_changes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.015249, "supported_languages": null}, "macro.dbt.default__get_materialized_view_configuration_changes": {"name": "default__get_materialized_view_configuration_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt.default__get_materialized_view_configuration_changes", "macro_sql": "{% macro default__get_materialized_view_configuration_changes(existing_relation, new_config) %}\n {{ exceptions.raise_compiler_error(\"Materialized views have not been implemented for this adapter.\") }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.015261, "supported_languages": null}, "macro.dbt.get_create_materialized_view_as_sql": {"name": "get_create_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/create.sql", "original_file_path": "macros/relations/materialized_view/create.sql", "unique_id": "macro.dbt.get_create_materialized_view_as_sql", "macro_sql": "{% macro get_create_materialized_view_as_sql(relation, sql) -%}\n {{- adapter.dispatch('get_create_materialized_view_as_sql', 'dbt')(relation, sql) -}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_create_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.015289, "supported_languages": null}, "macro.dbt.default__get_create_materialized_view_as_sql": {"name": "default__get_create_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/create.sql", "original_file_path": "macros/relations/materialized_view/create.sql", "unique_id": "macro.dbt.default__get_create_materialized_view_as_sql", "macro_sql": "{% macro default__get_create_materialized_view_as_sql(relation, sql) -%}\n {{ exceptions.raise_compiler_error(\n \"`get_create_materialized_view_as_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0153, "supported_languages": null}, "macro.dbt.get_table_columns_and_constraints": {"name": "get_table_columns_and_constraints", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.get_table_columns_and_constraints", "macro_sql": "{%- macro get_table_columns_and_constraints() -%}\n {{ adapter.dispatch('get_table_columns_and_constraints', 'dbt')() }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__get_table_columns_and_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.015323, "supported_languages": null}, "macro.dbt.default__get_table_columns_and_constraints": {"name": "default__get_table_columns_and_constraints", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.default__get_table_columns_and_constraints", "macro_sql": "{% macro default__get_table_columns_and_constraints() -%}\n {{ return(table_columns_and_constraints()) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.table_columns_and_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0153332, "supported_languages": null}, "macro.dbt.table_columns_and_constraints": {"name": "table_columns_and_constraints", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.table_columns_and_constraints", "macro_sql": "{% macro table_columns_and_constraints() %}\n {# loop through user_provided_columns to create DDL with data types and constraints #}\n {%- set raw_column_constraints = adapter.render_raw_columns_constraints(raw_columns=model['columns']) -%}\n {%- set raw_model_constraints = adapter.render_raw_model_constraints(raw_constraints=model['constraints']) -%}\n (\n {% for c in raw_column_constraints -%}\n {{ c }}{{ \",\" if not loop.last or raw_model_constraints }}\n {% endfor %}\n {% for c in raw_model_constraints -%}\n {{ c }}{{ \",\" if not loop.last }}\n {% endfor -%}\n )\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0153432, "supported_languages": null}, "macro.dbt.get_assert_columns_equivalent": {"name": "get_assert_columns_equivalent", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.get_assert_columns_equivalent", "macro_sql": "\n\n{%- macro get_assert_columns_equivalent(sql) -%}\n {{ adapter.dispatch('get_assert_columns_equivalent', 'dbt')(sql) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__get_assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0153532, "supported_languages": null}, "macro.dbt.default__get_assert_columns_equivalent": {"name": "default__get_assert_columns_equivalent", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.default__get_assert_columns_equivalent", "macro_sql": "{% macro default__get_assert_columns_equivalent(sql) -%}\n {{ return(assert_columns_equivalent(sql)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.015363, "supported_languages": null}, "macro.dbt.assert_columns_equivalent": {"name": "assert_columns_equivalent", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.assert_columns_equivalent", "macro_sql": "{% macro assert_columns_equivalent(sql) %}\n\n {#-- First ensure the user has defined 'columns' in yaml specification --#}\n {%- set user_defined_columns = model['columns'] -%}\n {%- if not user_defined_columns -%}\n {{ exceptions.raise_contract_error([], []) }}\n {%- endif -%}\n\n {#-- Obtain the column schema provided by sql file. #}\n {%- set sql_file_provided_columns = get_column_schema_from_query(sql, config.get('sql_header', none)) -%}\n {#--Obtain the column schema provided by the schema file by generating an 'empty schema' query from the model's columns. #}\n {%- set schema_file_provided_columns = get_column_schema_from_query(get_empty_schema_sql(user_defined_columns)) -%}\n\n {#-- create dictionaries with name and formatted data type and strings for exception #}\n {%- set sql_columns = format_columns(sql_file_provided_columns) -%}\n {%- set yaml_columns = format_columns(schema_file_provided_columns) -%}\n\n {%- if sql_columns|length != yaml_columns|length -%}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n\n {%- for sql_col in sql_columns -%}\n {%- set yaml_col = [] -%}\n {%- for this_col in yaml_columns -%}\n {%- if this_col['name'] == sql_col['name'] -%}\n {%- do yaml_col.append(this_col) -%}\n {%- break -%}\n {%- endif -%}\n {%- endfor -%}\n {%- if not yaml_col -%}\n {#-- Column with name not found in yaml #}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n {%- if sql_col['formatted'] != yaml_col[0]['formatted'] -%}\n {#-- Column data types don't match #}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n {%- endfor -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_column_schema_from_query", "macro.dbt.get_empty_schema_sql", "macro.dbt.format_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0153742, "supported_languages": null}, "macro.dbt.format_columns": {"name": "format_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.format_columns", "macro_sql": "{% macro format_columns(columns) %}\n {% set formatted_columns = [] %}\n {% for column in columns %}\n {%- set formatted_column = adapter.dispatch('format_column', 'dbt')(column) -%}\n {%- do formatted_columns.append(formatted_column) -%}\n {% endfor %}\n {{ return(formatted_columns) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__format_column"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.015383, "supported_languages": null}, "macro.dbt.default__format_column": {"name": "default__format_column", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.default__format_column", "macro_sql": "{% macro default__format_column(column) -%}\n {% set data_type = column.dtype %}\n {% set formatted = column.column.lower() ~ \" \" ~ data_type %}\n {{ return({'name': column.name, 'data_type': data_type, 'formatted': formatted}) }}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.015393, "supported_languages": null}, "macro.dbt.drop_table": {"name": "drop_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/drop.sql", "original_file_path": "macros/relations/table/drop.sql", "unique_id": "macro.dbt.drop_table", "macro_sql": "{% macro drop_table(relation) -%}\n {{- adapter.dispatch('drop_table', 'dbt')(relation) -}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__drop_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.015413, "supported_languages": null}, "macro.dbt.default__drop_table": {"name": "default__drop_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/drop.sql", "original_file_path": "macros/relations/table/drop.sql", "unique_id": "macro.dbt.default__drop_table", "macro_sql": "{% macro default__drop_table(relation) -%}\n drop table if exists {{ relation.render() }} cascade\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.015423, "supported_languages": null}, "macro.dbt.get_replace_table_sql": {"name": "get_replace_table_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/replace.sql", "original_file_path": "macros/relations/table/replace.sql", "unique_id": "macro.dbt.get_replace_table_sql", "macro_sql": "{% macro get_replace_table_sql(relation, sql) %}\n {{- adapter.dispatch('get_replace_table_sql', 'dbt')(relation, sql) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_replace_table_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.015442, "supported_languages": null}, "macro.dbt.default__get_replace_table_sql": {"name": "default__get_replace_table_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/replace.sql", "original_file_path": "macros/relations/table/replace.sql", "unique_id": "macro.dbt.default__get_replace_table_sql", "macro_sql": "{% macro default__get_replace_table_sql(relation, sql) %}\n {{ exceptions.raise_compiler_error(\n \"`get_replace_table_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.015454, "supported_languages": null}, "macro.dbt.get_rename_table_sql": {"name": "get_rename_table_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/rename.sql", "original_file_path": "macros/relations/table/rename.sql", "unique_id": "macro.dbt.get_rename_table_sql", "macro_sql": "{% macro get_rename_table_sql(relation, new_name) %}\n {{- adapter.dispatch('get_rename_table_sql', 'dbt')(relation, new_name) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_rename_table_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.015473, "supported_languages": null}, "macro.dbt.default__get_rename_table_sql": {"name": "default__get_rename_table_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/rename.sql", "original_file_path": "macros/relations/table/rename.sql", "unique_id": "macro.dbt.default__get_rename_table_sql", "macro_sql": "{% macro default__get_rename_table_sql(relation, new_name) %}\n {{ exceptions.raise_compiler_error(\n \"`get_rename_table_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.015483, "supported_languages": null}, "macro.dbt.get_create_table_as_sql": {"name": "get_create_table_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.get_create_table_as_sql", "macro_sql": "{% macro get_create_table_as_sql(temporary, relation, sql) -%}\n {{ adapter.dispatch('get_create_table_as_sql', 'dbt')(temporary, relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_create_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.015504, "supported_languages": null}, "macro.dbt.default__get_create_table_as_sql": {"name": "default__get_create_table_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.default__get_create_table_as_sql", "macro_sql": "{% macro default__get_create_table_as_sql(temporary, relation, sql) -%}\n {{ return(create_table_as(temporary, relation, sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.015539, "supported_languages": null}, "macro.dbt.create_table_as": {"name": "create_table_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.create_table_as", "macro_sql": "{% macro create_table_as(temporary, relation, compiled_code, language='sql') -%}\n {# backward compatibility for create_table_as that does not support language #}\n {% if language == \"sql\" %}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, compiled_code)}}\n {% else %}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, compiled_code, language) }}\n {% endif %}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.015549, "supported_languages": null}, "macro.dbt.default__create_table_as": {"name": "default__create_table_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.default__create_table_as", "macro_sql": "{% macro default__create_table_as(temporary, relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary: -%}temporary{%- endif %} table\n {{ relation.include(database=(not temporary), schema=(not temporary)) }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced and (not temporary) %}\n {{ get_assert_columns_equivalent(sql) }}\n {{ get_table_columns_and_constraints() }}\n {%- set sql = get_select_subquery(sql) %}\n {% endif %}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_assert_columns_equivalent", "macro.dbt.get_table_columns_and_constraints", "macro.dbt.get_select_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.015559, "supported_languages": null}, "macro.dbt.default__get_column_names": {"name": "default__get_column_names", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.default__get_column_names", "macro_sql": "{% macro default__get_column_names() %}\n {#- loop through user_provided_columns to get column names -#}\n {%- set user_provided_columns = model['columns'] -%}\n {%- for i in user_provided_columns %}\n {%- set col = user_provided_columns[i] -%}\n {%- set col_name = adapter.quote(col['name']) if col.get('quote') else col['name'] -%}\n {{ col_name }}{{ \", \" if not loop.last }}\n {%- endfor -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.015569, "supported_languages": null}, "macro.dbt.get_select_subquery": {"name": "get_select_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.get_select_subquery", "macro_sql": "{% macro get_select_subquery(sql) %}\n {{ return(adapter.dispatch('get_select_subquery', 'dbt')(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_select_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.015578, "supported_languages": null}, "macro.dbt.default__get_select_subquery": {"name": "default__get_select_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.default__get_select_subquery", "macro_sql": "{% macro default__get_select_subquery(sql) %}\n select {{ adapter.dispatch('get_column_names', 'dbt')() }}\n from (\n {{ sql }}\n ) as model_subq\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_column_names"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0155892, "supported_languages": null}, "macro.dbt.drop_view": {"name": "drop_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/drop.sql", "original_file_path": "macros/relations/view/drop.sql", "unique_id": "macro.dbt.drop_view", "macro_sql": "{% macro drop_view(relation) -%}\n {{- adapter.dispatch('drop_view', 'dbt')(relation) -}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__drop_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.015608, "supported_languages": null}, "macro.dbt.default__drop_view": {"name": "default__drop_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/drop.sql", "original_file_path": "macros/relations/view/drop.sql", "unique_id": "macro.dbt.default__drop_view", "macro_sql": "{% macro default__drop_view(relation) -%}\n drop view if exists {{ relation.render() }} cascade\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.015618, "supported_languages": null}, "macro.dbt.get_replace_view_sql": {"name": "get_replace_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.get_replace_view_sql", "macro_sql": "{% macro get_replace_view_sql(relation, sql) %}\n {{- adapter.dispatch('get_replace_view_sql', 'dbt')(relation, sql) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_replace_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.015639, "supported_languages": null}, "macro.dbt.default__get_replace_view_sql": {"name": "default__get_replace_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.default__get_replace_view_sql", "macro_sql": "{% macro default__get_replace_view_sql(relation, sql) %}\n {{ exceptions.raise_compiler_error(\n \"`get_replace_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0156958, "supported_languages": null}, "macro.dbt.create_or_replace_view": {"name": "create_or_replace_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.create_or_replace_view", "macro_sql": "{% macro create_or_replace_view() %}\n {%- set identifier = model['alias'] -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database,\n type='view') -%}\n {% set grant_config = config.get('grants') %}\n\n {{ run_hooks(pre_hooks) }}\n\n -- If there's a table with the same name and we weren't told to full refresh,\n -- that's an error. If we were told to full refresh, drop it. This behavior differs\n -- for Snowflake and BigQuery, so multiple dispatch is used.\n {%- if old_relation is not none and old_relation.is_table -%}\n {{ handle_existing_table(should_full_refresh(), old_relation) }}\n {%- endif -%}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(target_relation, sql) }}\n {%- endcall %}\n\n {% set should_revoke = should_revoke(exists_as_view, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.handle_existing_table", "macro.dbt.should_full_refresh", "macro.dbt.statement", "macro.dbt.get_create_view_as_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.015811, "supported_languages": null}, "macro.dbt.handle_existing_table": {"name": "handle_existing_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.handle_existing_table", "macro_sql": "{% macro handle_existing_table(full_refresh, old_relation) %}\n {{ adapter.dispatch('handle_existing_table', 'dbt')(full_refresh, old_relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__handle_existing_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.015876, "supported_languages": null}, "macro.dbt.default__handle_existing_table": {"name": "default__handle_existing_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.default__handle_existing_table", "macro_sql": "{% macro default__handle_existing_table(full_refresh, old_relation) %}\n {{ log(\"Dropping relation \" ~ old_relation.render() ~ \" because it is of type \" ~ old_relation.type) }}\n {{ adapter.drop_relation(old_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.015913, "supported_languages": null}, "macro.dbt.get_rename_view_sql": {"name": "get_rename_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/rename.sql", "original_file_path": "macros/relations/view/rename.sql", "unique_id": "macro.dbt.get_rename_view_sql", "macro_sql": "{% macro get_rename_view_sql(relation, new_name) %}\n {{- adapter.dispatch('get_rename_view_sql', 'dbt')(relation, new_name) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_rename_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.016115, "supported_languages": null}, "macro.dbt.default__get_rename_view_sql": {"name": "default__get_rename_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/rename.sql", "original_file_path": "macros/relations/view/rename.sql", "unique_id": "macro.dbt.default__get_rename_view_sql", "macro_sql": "{% macro default__get_rename_view_sql(relation, new_name) %}\n {{ exceptions.raise_compiler_error(\n \"`get_rename_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.016132, "supported_languages": null}, "macro.dbt.get_create_view_as_sql": {"name": "get_create_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt.get_create_view_as_sql", "macro_sql": "{% macro get_create_view_as_sql(relation, sql) -%}\n {{ adapter.dispatch('get_create_view_as_sql', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_create_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0161622, "supported_languages": null}, "macro.dbt.default__get_create_view_as_sql": {"name": "default__get_create_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt.default__get_create_view_as_sql", "macro_sql": "{% macro default__get_create_view_as_sql(relation, sql) -%}\n {{ return(create_view_as(relation, sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.016174, "supported_languages": null}, "macro.dbt.create_view_as": {"name": "create_view_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt.create_view_as", "macro_sql": "{% macro create_view_as(relation, sql) -%}\n {{ adapter.dispatch('create_view_as', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.016185, "supported_languages": null}, "macro.dbt.default__create_view_as": {"name": "default__create_view_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt.default__create_view_as", "macro_sql": "{% macro default__create_view_as(relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n create view {{ relation.render() }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {%- endif %}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.016196, "supported_languages": null}, "macro.dbt.default__test_relationships": {"name": "default__test_relationships", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/relationships.sql", "original_file_path": "macros/generic_test_sql/relationships.sql", "unique_id": "macro.dbt.default__test_relationships", "macro_sql": "{% macro default__test_relationships(model, column_name, to, field) %}\n\nwith child as (\n select {{ column_name }} as from_field\n from {{ model }}\n where {{ column_name }} is not null\n),\n\nparent as (\n select {{ field }} as to_field\n from {{ to }}\n)\n\nselect\n from_field\n\nfrom child\nleft join parent\n on child.from_field = parent.to_field\n\nwhere parent.to_field is null\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.016305, "supported_languages": null}, "macro.dbt.default__test_not_null": {"name": "default__test_not_null", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/not_null.sql", "original_file_path": "macros/generic_test_sql/not_null.sql", "unique_id": "macro.dbt.default__test_not_null", "macro_sql": "{% macro default__test_not_null(model, column_name) %}\n\n{% set column_list = '*' if should_store_failures() else column_name %}\n\nselect {{ column_list }}\nfrom {{ model }}\nwhere {{ column_name }} is null\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_store_failures"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.016336, "supported_languages": null}, "macro.dbt.default__test_unique": {"name": "default__test_unique", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/unique.sql", "original_file_path": "macros/generic_test_sql/unique.sql", "unique_id": "macro.dbt.default__test_unique", "macro_sql": "{% macro default__test_unique(model, column_name) %}\n\nselect\n {{ column_name }} as unique_field,\n count(*) as n_records\n\nfrom {{ model }}\nwhere {{ column_name }} is not null\ngroup by {{ column_name }}\nhaving count(*) > 1\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0163572, "supported_languages": null}, "macro.dbt.default__test_accepted_values": {"name": "default__test_accepted_values", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/accepted_values.sql", "original_file_path": "macros/generic_test_sql/accepted_values.sql", "unique_id": "macro.dbt.default__test_accepted_values", "macro_sql": "{% macro default__test_accepted_values(model, column_name, values, quote=True) %}\n\nwith all_values as (\n\n select\n {{ column_name }} as value_field,\n count(*) as n_records\n\n from {{ model }}\n group by {{ column_name }}\n\n)\n\nselect *\nfrom all_values\nwhere value_field not in (\n {% for value in values -%}\n {% if quote -%}\n '{{ value }}'\n {%- else -%}\n {{ value }}\n {%- endif -%}\n {%- if not loop.last -%},{%- endif %}\n {%- endfor %}\n)\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.016377, "supported_languages": null}, "macro.dbt.statement": {"name": "statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt.statement", "macro_sql": "\n{%- macro statement(name=None, fetch_result=False, auto_begin=True, language='sql') -%}\n {%- if execute: -%}\n {%- set compiled_code = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime {} for node \"{}\"'.format(language, model['unique_id'])) }}\n {{ write(compiled_code) }}\n {%- endif -%}\n {%- if language == 'sql'-%}\n {%- set res, table = adapter.execute(compiled_code, auto_begin=auto_begin, fetch=fetch_result) -%}\n {%- elif language == 'python' -%}\n {%- set res = submit_python_job(model, compiled_code) -%}\n {#-- TODO: What should table be for python models? --#}\n {%- set table = None -%}\n {%- else -%}\n {% do exceptions.raise_compiler_error(\"statement macro didn't get supported language\") %}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_result(name, response=res, agate_table=table) }}\n {%- endif -%}\n\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0163991, "supported_languages": null}, "macro.dbt.noop_statement": {"name": "noop_statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt.noop_statement", "macro_sql": "{% macro noop_statement(name=None, message=None, code=None, rows_affected=None, res=None) -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_raw_result(name, message=message, code=code, rows_affected=rows_affected, agate_table=res) }}\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0164108, "supported_languages": null}, "macro.dbt.run_query": {"name": "run_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt.run_query", "macro_sql": "{% macro run_query(sql) %}\n {% call statement(\"run_query_statement\", fetch_result=true, auto_begin=false) %}\n {{ sql }}\n {% endcall %}\n\n {% do return(load_result(\"run_query_statement\").table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.016421, "supported_languages": null}, "macro.dbt.convert_datetime": {"name": "convert_datetime", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.convert_datetime", "macro_sql": "{% macro convert_datetime(date_str, date_fmt) %}\n\n {% set error_msg -%}\n The provided partition date '{{ date_str }}' does not match the expected format '{{ date_fmt }}'\n {%- endset %}\n\n {% set res = try_or_compiler_error(error_msg, modules.datetime.datetime.strptime, date_str.strip(), date_fmt) %}\n {{ return(res) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.016442, "supported_languages": null}, "macro.dbt.dates_in_range": {"name": "dates_in_range", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.dates_in_range", "macro_sql": "{% macro dates_in_range(start_date_str, end_date_str=none, in_fmt=\"%Y%m%d\", out_fmt=\"%Y%m%d\") %}\n {% set end_date_str = start_date_str if end_date_str is none else end_date_str %}\n\n {% set start_date = convert_datetime(start_date_str, in_fmt) %}\n {% set end_date = convert_datetime(end_date_str, in_fmt) %}\n\n {% set day_count = (end_date - start_date).days %}\n {% if day_count < 0 %}\n {% set msg -%}\n Partition start date is after the end date ({{ start_date }}, {{ end_date }})\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg, model) }}\n {% endif %}\n\n {% set date_list = [] %}\n {% for i in range(0, day_count + 1) %}\n {% set the_date = (modules.datetime.timedelta(days=i) + start_date) %}\n {% if not out_fmt %}\n {% set _ = date_list.append(the_date) %}\n {% else %}\n {% set _ = date_list.append(the_date.strftime(out_fmt)) %}\n {% endif %}\n {% endfor %}\n\n {{ return(date_list) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.convert_datetime"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.016452, "supported_languages": null}, "macro.dbt.partition_range": {"name": "partition_range", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.partition_range", "macro_sql": "{% macro partition_range(raw_partition_date, date_fmt='%Y%m%d') %}\n {% set partition_range = (raw_partition_date | string).split(\",\") %}\n\n {% if (partition_range | length) == 1 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = none %}\n {% elif (partition_range | length) == 2 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = partition_range[1] %}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Invalid partition time. Expected format: {Start Date}[,{End Date}]. Got: \" ~ raw_partition_date) }}\n {% endif %}\n\n {{ return(dates_in_range(start_date, end_date, in_fmt=date_fmt)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.dates_in_range"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.016462, "supported_languages": null}, "macro.dbt.py_current_timestring": {"name": "py_current_timestring", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.py_current_timestring", "macro_sql": "{% macro py_current_timestring() %}\n {% set dt = modules.datetime.datetime.now() %}\n {% do return(dt.strftime(\"%Y%m%d%H%M%S%f\")) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.016473, "supported_languages": null}, "macro.dbt.except": {"name": "except", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/except.sql", "original_file_path": "macros/utils/except.sql", "unique_id": "macro.dbt.except", "macro_sql": "{% macro except() %}\n {{ return(adapter.dispatch('except', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__except"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.016493, "supported_languages": null}, "macro.dbt.default__except": {"name": "default__except", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/except.sql", "original_file_path": "macros/utils/except.sql", "unique_id": "macro.dbt.default__except", "macro_sql": "{% macro default__except() %}\n\n except\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.016503, "supported_languages": null}, "macro.dbt.get_intervals_between": {"name": "get_intervals_between", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_spine.sql", "original_file_path": "macros/utils/date_spine.sql", "unique_id": "macro.dbt.get_intervals_between", "macro_sql": "{% macro get_intervals_between(start_date, end_date, datepart) -%}\n {{ return(adapter.dispatch('get_intervals_between', 'dbt')(start_date, end_date, datepart)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_intervals_between"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.016522, "supported_languages": null}, "macro.dbt.default__get_intervals_between": {"name": "default__get_intervals_between", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_spine.sql", "original_file_path": "macros/utils/date_spine.sql", "unique_id": "macro.dbt.default__get_intervals_between", "macro_sql": "{% macro default__get_intervals_between(start_date, end_date, datepart) -%}\n {%- call statement('get_intervals_between', fetch_result=True) %}\n\n select {{ dbt.datediff(start_date, end_date, datepart) }}\n\n {%- endcall -%}\n\n {%- set value_list = load_result('get_intervals_between') -%}\n\n {%- if value_list and value_list['data'] -%}\n {%- set values = value_list['data'] | map(attribute=0) | list %}\n {{ return(values[0]) }}\n {%- else -%}\n {{ return(1) }}\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.datediff"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.016533, "supported_languages": null}, "macro.dbt.date_spine": {"name": "date_spine", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_spine.sql", "original_file_path": "macros/utils/date_spine.sql", "unique_id": "macro.dbt.date_spine", "macro_sql": "{% macro date_spine(datepart, start_date, end_date) %}\n {{ return(adapter.dispatch('date_spine', 'dbt')(datepart, start_date, end_date)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__date_spine"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.016543, "supported_languages": null}, "macro.dbt.default__date_spine": {"name": "default__date_spine", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_spine.sql", "original_file_path": "macros/utils/date_spine.sql", "unique_id": "macro.dbt.default__date_spine", "macro_sql": "{% macro default__date_spine(datepart, start_date, end_date) %}\n\n\n {# call as follows:\n\n date_spine(\n \"day\",\n \"to_date('01/01/2016', 'mm/dd/yyyy')\",\n \"dbt.dateadd(week, 1, current_date)\"\n ) #}\n\n\n with rawdata as (\n\n {{dbt.generate_series(\n dbt.get_intervals_between(start_date, end_date, datepart)\n )}}\n\n ),\n\n all_periods as (\n\n select (\n {{\n dbt.dateadd(\n datepart,\n \"row_number() over (order by 1) - 1\",\n start_date\n )\n }}\n ) as date_{{datepart}}\n from rawdata\n\n ),\n\n filtered as (\n\n select *\n from all_periods\n where date_{{datepart}} <= {{ end_date }}\n\n )\n\n select * from filtered\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.generate_series", "macro.dbt.get_intervals_between", "macro.dbt.dateadd"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.016553, "supported_languages": null}, "macro.dbt.date": {"name": "date", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date.sql", "original_file_path": "macros/utils/date.sql", "unique_id": "macro.dbt.date", "macro_sql": "{% macro date(year, month, day) %}\n {{ return(adapter.dispatch('date', 'dbt') (year, month, day)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__date"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.016574, "supported_languages": null}, "macro.dbt.default__date": {"name": "default__date", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date.sql", "original_file_path": "macros/utils/date.sql", "unique_id": "macro.dbt.default__date", "macro_sql": "{% macro default__date(year, month, day) -%}\n {%- set dt = modules.datetime.date(year, month, day) -%}\n {%- set iso_8601_formatted_date = dt.strftime('%Y-%m-%d') -%}\n to_date('{{ iso_8601_formatted_date }}', 'YYYY-MM-DD')\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0165849, "supported_languages": null}, "macro.dbt.replace": {"name": "replace", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/replace.sql", "original_file_path": "macros/utils/replace.sql", "unique_id": "macro.dbt.replace", "macro_sql": "{% macro replace(field, old_chars, new_chars) -%}\n {{ return(adapter.dispatch('replace', 'dbt') (field, old_chars, new_chars)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__replace"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.016604, "supported_languages": null}, "macro.dbt.default__replace": {"name": "default__replace", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/replace.sql", "original_file_path": "macros/utils/replace.sql", "unique_id": "macro.dbt.default__replace", "macro_sql": "{% macro default__replace(field, old_chars, new_chars) %}\n\n replace(\n {{ field }},\n {{ old_chars }},\n {{ new_chars }}\n )\n\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.016615, "supported_languages": null}, "macro.dbt.concat": {"name": "concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/concat.sql", "original_file_path": "macros/utils/concat.sql", "unique_id": "macro.dbt.concat", "macro_sql": "{% macro concat(fields) -%}\n {{ return(adapter.dispatch('concat', 'dbt')(fields)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__concat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0166342, "supported_languages": null}, "macro.dbt.default__concat": {"name": "default__concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/concat.sql", "original_file_path": "macros/utils/concat.sql", "unique_id": "macro.dbt.default__concat", "macro_sql": "{% macro default__concat(fields) -%}\n {{ fields|join(' || ') }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.016644, "supported_languages": null}, "macro.dbt.get_powers_of_two": {"name": "get_powers_of_two", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/generate_series.sql", "original_file_path": "macros/utils/generate_series.sql", "unique_id": "macro.dbt.get_powers_of_two", "macro_sql": "{% macro get_powers_of_two(upper_bound) %}\n {{ return(adapter.dispatch('get_powers_of_two', 'dbt')(upper_bound)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_powers_of_two"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.016665, "supported_languages": null}, "macro.dbt.default__get_powers_of_two": {"name": "default__get_powers_of_two", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/generate_series.sql", "original_file_path": "macros/utils/generate_series.sql", "unique_id": "macro.dbt.default__get_powers_of_two", "macro_sql": "{% macro default__get_powers_of_two(upper_bound) %}\n\n {% if upper_bound <= 0 %}\n {{ exceptions.raise_compiler_error(\"upper bound must be positive\") }}\n {% endif %}\n\n {% for _ in range(1, 100) %}\n {% if upper_bound <= 2 ** loop.index %}{{ return(loop.index) }}{% endif %}\n {% endfor %}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.016675, "supported_languages": null}, "macro.dbt.generate_series": {"name": "generate_series", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/generate_series.sql", "original_file_path": "macros/utils/generate_series.sql", "unique_id": "macro.dbt.generate_series", "macro_sql": "{% macro generate_series(upper_bound) %}\n {{ return(adapter.dispatch('generate_series', 'dbt')(upper_bound)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__generate_series"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.016685, "supported_languages": null}, "macro.dbt.default__generate_series": {"name": "default__generate_series", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/generate_series.sql", "original_file_path": "macros/utils/generate_series.sql", "unique_id": "macro.dbt.default__generate_series", "macro_sql": "{% macro default__generate_series(upper_bound) %}\n\n {% set n = dbt.get_powers_of_two(upper_bound) %}\n\n with p as (\n select 0 as generated_number union all select 1\n ), unioned as (\n\n select\n\n {% for i in range(n) %}\n p{{i}}.generated_number * power(2, {{i}})\n {% if not loop.last %} + {% endif %}\n {% endfor %}\n + 1\n as generated_number\n\n from\n\n {% for i in range(n) %}\n p as p{{i}}\n {% if not loop.last %} cross join {% endif %}\n {% endfor %}\n\n )\n\n select *\n from unioned\n where generated_number <= {{upper_bound}}\n order by generated_number\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_powers_of_two"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.016695, "supported_languages": null}, "macro.dbt.length": {"name": "length", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/length.sql", "original_file_path": "macros/utils/length.sql", "unique_id": "macro.dbt.length", "macro_sql": "{% macro length(expression) -%}\n {{ return(adapter.dispatch('length', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__length"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.016714, "supported_languages": null}, "macro.dbt.default__length": {"name": "default__length", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/length.sql", "original_file_path": "macros/utils/length.sql", "unique_id": "macro.dbt.default__length", "macro_sql": "{% macro default__length(expression) %}\n\n length(\n {{ expression }}\n )\n\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.016725, "supported_languages": null}, "macro.dbt.dateadd": {"name": "dateadd", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt.dateadd", "macro_sql": "{% macro dateadd(datepart, interval, from_date_or_timestamp) %}\n {{ return(adapter.dispatch('dateadd', 'dbt')(datepart, interval, from_date_or_timestamp)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__dateadd"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.016744, "supported_languages": null}, "macro.dbt.default__dateadd": {"name": "default__dateadd", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt.default__dateadd", "macro_sql": "{% macro default__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n dateadd(\n {{ datepart }},\n {{ interval }},\n {{ from_date_or_timestamp }}\n )\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.016754, "supported_languages": null}, "macro.dbt.intersect": {"name": "intersect", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/intersect.sql", "original_file_path": "macros/utils/intersect.sql", "unique_id": "macro.dbt.intersect", "macro_sql": "{% macro intersect() %}\n {{ return(adapter.dispatch('intersect', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__intersect"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0167758, "supported_languages": null}, "macro.dbt.default__intersect": {"name": "default__intersect", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/intersect.sql", "original_file_path": "macros/utils/intersect.sql", "unique_id": "macro.dbt.default__intersect", "macro_sql": "{% macro default__intersect() %}\n\n intersect\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.016788, "supported_languages": null}, "macro.dbt.escape_single_quotes": {"name": "escape_single_quotes", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/escape_single_quotes.sql", "original_file_path": "macros/utils/escape_single_quotes.sql", "unique_id": "macro.dbt.escape_single_quotes", "macro_sql": "{% macro escape_single_quotes(expression) %}\n {{ return(adapter.dispatch('escape_single_quotes', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__escape_single_quotes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0169332, "supported_languages": null}, "macro.dbt.default__escape_single_quotes": {"name": "default__escape_single_quotes", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/escape_single_quotes.sql", "original_file_path": "macros/utils/escape_single_quotes.sql", "unique_id": "macro.dbt.default__escape_single_quotes", "macro_sql": "{% macro default__escape_single_quotes(expression) -%}\n{{ expression | replace(\"'\",\"''\") }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0169458, "supported_languages": null}, "macro.dbt.right": {"name": "right", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/right.sql", "original_file_path": "macros/utils/right.sql", "unique_id": "macro.dbt.right", "macro_sql": "{% macro right(string_text, length_expression) -%}\n {{ return(adapter.dispatch('right', 'dbt') (string_text, length_expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__right"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.016968, "supported_languages": null}, "macro.dbt.default__right": {"name": "default__right", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/right.sql", "original_file_path": "macros/utils/right.sql", "unique_id": "macro.dbt.default__right", "macro_sql": "{% macro default__right(string_text, length_expression) %}\n\n right(\n {{ string_text }},\n {{ length_expression }}\n )\n\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.016979, "supported_languages": null}, "macro.dbt.listagg": {"name": "listagg", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "unique_id": "macro.dbt.listagg", "macro_sql": "{% macro listagg(measure, delimiter_text=\"','\", order_by_clause=none, limit_num=none) -%}\n {{ return(adapter.dispatch('listagg', 'dbt') (measure, delimiter_text, order_by_clause, limit_num)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__listagg"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.017, "supported_languages": null}, "macro.dbt.default__listagg": {"name": "default__listagg", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "unique_id": "macro.dbt.default__listagg", "macro_sql": "{% macro default__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n {% if limit_num -%}\n array_to_string(\n array_slice(\n array_agg(\n {{ measure }}\n ){% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n ,0\n ,{{ limit_num }}\n ),\n {{ delimiter_text }}\n )\n {%- else %}\n listagg(\n {{ measure }},\n {{ delimiter_text }}\n )\n {% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n {%- endif %}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.01701, "supported_languages": null}, "macro.dbt.datediff": {"name": "datediff", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt.datediff", "macro_sql": "{% macro datediff(first_date, second_date, datepart) %}\n {{ return(adapter.dispatch('datediff', 'dbt')(first_date, second_date, datepart)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__datediff"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.01703, "supported_languages": null}, "macro.dbt.default__datediff": {"name": "default__datediff", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt.default__datediff", "macro_sql": "{% macro default__datediff(first_date, second_date, datepart) -%}\n\n datediff(\n {{ datepart }},\n {{ first_date }},\n {{ second_date }}\n )\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.017548, "supported_languages": null}, "macro.dbt.safe_cast": {"name": "safe_cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "unique_id": "macro.dbt.safe_cast", "macro_sql": "{% macro safe_cast(field, type) %}\n {{ return(adapter.dispatch('safe_cast', 'dbt') (field, type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__safe_cast"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.017575, "supported_languages": null}, "macro.dbt.default__safe_cast": {"name": "default__safe_cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "unique_id": "macro.dbt.default__safe_cast", "macro_sql": "{% macro default__safe_cast(field, type) %}\n {# most databases don't support this function yet\n so we just need to use cast #}\n cast({{field}} as {{type}})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.017586, "supported_languages": null}, "macro.dbt.equals": {"name": "equals", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/equals.sql", "original_file_path": "macros/utils/equals.sql", "unique_id": "macro.dbt.equals", "macro_sql": "{% macro equals(expr1, expr2) %}\n {{ return(adapter.dispatch('equals', 'dbt') (expr1, expr2)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__equals"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.017608, "supported_languages": null}, "macro.dbt.default__equals": {"name": "default__equals", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/equals.sql", "original_file_path": "macros/utils/equals.sql", "unique_id": "macro.dbt.default__equals", "macro_sql": "{% macro default__equals(expr1, expr2) -%}\n{%- if adapter.behavior.enable_truthy_nulls_equals_macro.no_warn %}\n case when (({{ expr1 }} = {{ expr2 }}) or ({{ expr1 }} is null and {{ expr2 }} is null))\n then 0\n else 1\n end = 0\n{%- else -%}\n ({{ expr1 }} = {{ expr2 }})\n{%- endif %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.017618, "supported_languages": null}, "macro.dbt.hash": {"name": "hash", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/hash.sql", "original_file_path": "macros/utils/hash.sql", "unique_id": "macro.dbt.hash", "macro_sql": "{% macro hash(field) -%}\n {{ return(adapter.dispatch('hash', 'dbt') (field)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__hash"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.017652, "supported_languages": null}, "macro.dbt.default__hash": {"name": "default__hash", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/hash.sql", "original_file_path": "macros/utils/hash.sql", "unique_id": "macro.dbt.default__hash", "macro_sql": "{% macro default__hash(field) -%}\n md5(cast({{ field }} as {{ api.Column.translate_type('string') }}))\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.017663, "supported_languages": null}, "macro.dbt.cast_bool_to_text": {"name": "cast_bool_to_text", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast_bool_to_text.sql", "original_file_path": "macros/utils/cast_bool_to_text.sql", "unique_id": "macro.dbt.cast_bool_to_text", "macro_sql": "{% macro cast_bool_to_text(field) %}\n {{ adapter.dispatch('cast_bool_to_text', 'dbt') (field) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__cast_bool_to_text"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0176818, "supported_languages": null}, "macro.dbt.default__cast_bool_to_text": {"name": "default__cast_bool_to_text", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast_bool_to_text.sql", "original_file_path": "macros/utils/cast_bool_to_text.sql", "unique_id": "macro.dbt.default__cast_bool_to_text", "macro_sql": "{% macro default__cast_bool_to_text(field) %}\n cast({{ field }} as {{ api.Column.translate_type('string') }})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0177002, "supported_languages": null}, "macro.dbt.cast": {"name": "cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast.sql", "original_file_path": "macros/utils/cast.sql", "unique_id": "macro.dbt.cast", "macro_sql": "{% macro cast(field, type) %}\n {{ return(adapter.dispatch('cast', 'dbt') (field, type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__cast"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.017718, "supported_languages": null}, "macro.dbt.default__cast": {"name": "default__cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast.sql", "original_file_path": "macros/utils/cast.sql", "unique_id": "macro.dbt.default__cast", "macro_sql": "{% macro default__cast(field, type) %}\n cast({{field}} as {{type}})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.017728, "supported_languages": null}, "macro.dbt.any_value": {"name": "any_value", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "unique_id": "macro.dbt.any_value", "macro_sql": "{% macro any_value(expression) -%}\n {{ return(adapter.dispatch('any_value', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__any_value"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0177479, "supported_languages": null}, "macro.dbt.default__any_value": {"name": "default__any_value", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "unique_id": "macro.dbt.default__any_value", "macro_sql": "{% macro default__any_value(expression) -%}\n\n any_value({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.017758, "supported_languages": null}, "macro.dbt.position": {"name": "position", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/position.sql", "original_file_path": "macros/utils/position.sql", "unique_id": "macro.dbt.position", "macro_sql": "{% macro position(substring_text, string_text) -%}\n {{ return(adapter.dispatch('position', 'dbt') (substring_text, string_text)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__position"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.017777, "supported_languages": null}, "macro.dbt.default__position": {"name": "default__position", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/position.sql", "original_file_path": "macros/utils/position.sql", "unique_id": "macro.dbt.default__position", "macro_sql": "{% macro default__position(substring_text, string_text) %}\n\n position(\n {{ substring_text }} in {{ string_text }}\n )\n\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.017787, "supported_languages": null}, "macro.dbt.string_literal": {"name": "string_literal", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/literal.sql", "original_file_path": "macros/utils/literal.sql", "unique_id": "macro.dbt.string_literal", "macro_sql": "{%- macro string_literal(value) -%}\n {{ return(adapter.dispatch('string_literal', 'dbt') (value)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__string_literal"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.017806, "supported_languages": null}, "macro.dbt.default__string_literal": {"name": "default__string_literal", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/literal.sql", "original_file_path": "macros/utils/literal.sql", "unique_id": "macro.dbt.default__string_literal", "macro_sql": "{% macro default__string_literal(value) -%}\n '{{ value }}'\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.017816, "supported_languages": null}, "macro.dbt.type_string": {"name": "type_string", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_string", "macro_sql": "\n\n{%- macro type_string() -%}\n {{ return(adapter.dispatch('type_string', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_string"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.017843, "supported_languages": null}, "macro.dbt.default__type_string": {"name": "default__type_string", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_string", "macro_sql": "{% macro default__type_string() %}\n {{ return(api.Column.translate_type(\"string\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.017853, "supported_languages": null}, "macro.dbt.type_timestamp": {"name": "type_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_timestamp", "macro_sql": "\n\n{%- macro type_timestamp() -%}\n {{ return(adapter.dispatch('type_timestamp', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0178628, "supported_languages": null}, "macro.dbt.default__type_timestamp": {"name": "default__type_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_timestamp", "macro_sql": "{% macro default__type_timestamp() %}\n {{ return(api.Column.translate_type(\"timestamp\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0178728, "supported_languages": null}, "macro.dbt.type_float": {"name": "type_float", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_float", "macro_sql": "\n\n{%- macro type_float() -%}\n {{ return(adapter.dispatch('type_float', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_float"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0178828, "supported_languages": null}, "macro.dbt.default__type_float": {"name": "default__type_float", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_float", "macro_sql": "{% macro default__type_float() %}\n {{ return(api.Column.translate_type(\"float\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.017893, "supported_languages": null}, "macro.dbt.type_numeric": {"name": "type_numeric", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_numeric", "macro_sql": "\n\n{%- macro type_numeric() -%}\n {{ return(adapter.dispatch('type_numeric', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_numeric"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.017905, "supported_languages": null}, "macro.dbt.default__type_numeric": {"name": "default__type_numeric", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_numeric", "macro_sql": "{% macro default__type_numeric() %}\n {{ return(api.Column.numeric_type(\"numeric\", 28, 6)) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.017916, "supported_languages": null}, "macro.dbt.type_bigint": {"name": "type_bigint", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_bigint", "macro_sql": "\n\n{%- macro type_bigint() -%}\n {{ return(adapter.dispatch('type_bigint', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_bigint"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.017929, "supported_languages": null}, "macro.dbt.default__type_bigint": {"name": "default__type_bigint", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_bigint", "macro_sql": "{% macro default__type_bigint() %}\n {{ return(api.Column.translate_type(\"bigint\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0179389, "supported_languages": null}, "macro.dbt.type_int": {"name": "type_int", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_int", "macro_sql": "\n\n{%- macro type_int() -%}\n {{ return(adapter.dispatch('type_int', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_int"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.017977, "supported_languages": null}, "macro.dbt.default__type_int": {"name": "default__type_int", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_int", "macro_sql": "{%- macro default__type_int() -%}\n {{ return(api.Column.translate_type(\"integer\")) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018013, "supported_languages": null}, "macro.dbt.type_boolean": {"name": "type_boolean", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_boolean", "macro_sql": "\n\n{%- macro type_boolean() -%}\n {{ return(adapter.dispatch('type_boolean', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_boolean"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0180259, "supported_languages": null}, "macro.dbt.default__type_boolean": {"name": "default__type_boolean", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_boolean", "macro_sql": "{%- macro default__type_boolean() -%}\n {{ return(api.Column.translate_type(\"boolean\")) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018037, "supported_languages": null}, "macro.dbt.array_concat": {"name": "array_concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_concat.sql", "original_file_path": "macros/utils/array_concat.sql", "unique_id": "macro.dbt.array_concat", "macro_sql": "{% macro array_concat(array_1, array_2) -%}\n {{ return(adapter.dispatch('array_concat', 'dbt')(array_1, array_2)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__array_concat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018074, "supported_languages": null}, "macro.dbt.default__array_concat": {"name": "default__array_concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_concat.sql", "original_file_path": "macros/utils/array_concat.sql", "unique_id": "macro.dbt.default__array_concat", "macro_sql": "{% macro default__array_concat(array_1, array_2) -%}\n array_cat({{ array_1 }}, {{ array_2 }})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018085, "supported_languages": null}, "macro.dbt.bool_or": {"name": "bool_or", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/bool_or.sql", "original_file_path": "macros/utils/bool_or.sql", "unique_id": "macro.dbt.bool_or", "macro_sql": "{% macro bool_or(expression) -%}\n {{ return(adapter.dispatch('bool_or', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__bool_or"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0181062, "supported_languages": null}, "macro.dbt.default__bool_or": {"name": "default__bool_or", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/bool_or.sql", "original_file_path": "macros/utils/bool_or.sql", "unique_id": "macro.dbt.default__bool_or", "macro_sql": "{% macro default__bool_or(expression) -%}\n\n bool_or({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018116, "supported_languages": null}, "macro.dbt.last_day": {"name": "last_day", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt.last_day", "macro_sql": "{% macro last_day(date, datepart) %}\n {{ return(adapter.dispatch('last_day', 'dbt') (date, datepart)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__last_day"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018135, "supported_languages": null}, "macro.dbt.default_last_day": {"name": "default_last_day", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt.default_last_day", "macro_sql": "\n\n{%- macro default_last_day(date, datepart) -%}\n cast(\n {{dbt.dateadd('day', '-1',\n dbt.dateadd(datepart, '1', dbt.date_trunc(datepart, date))\n )}}\n as date)\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.dateadd", "macro.dbt.date_trunc"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018145, "supported_languages": null}, "macro.dbt.default__last_day": {"name": "default__last_day", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt.default__last_day", "macro_sql": "{% macro default__last_day(date, datepart) -%}\n {{dbt.default_last_day(date, datepart)}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default_last_day"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018155, "supported_languages": null}, "macro.dbt.split_part": {"name": "split_part", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt.split_part", "macro_sql": "{% macro split_part(string_text, delimiter_text, part_number) %}\n {{ return(adapter.dispatch('split_part', 'dbt') (string_text, delimiter_text, part_number)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__split_part"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018175, "supported_languages": null}, "macro.dbt.default__split_part": {"name": "default__split_part", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt.default__split_part", "macro_sql": "{% macro default__split_part(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n {{ part_number }}\n )\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018193, "supported_languages": null}, "macro.dbt._split_part_negative": {"name": "_split_part_negative", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt._split_part_negative", "macro_sql": "{% macro _split_part_negative(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n length({{ string_text }})\n - length(\n replace({{ string_text }}, {{ delimiter_text }}, '')\n ) + 2 + {{ part_number }}\n )\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0182028, "supported_languages": null}, "macro.dbt.date_trunc": {"name": "date_trunc", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_trunc.sql", "original_file_path": "macros/utils/date_trunc.sql", "unique_id": "macro.dbt.date_trunc", "macro_sql": "{% macro date_trunc(datepart, date) -%}\n {{ return(adapter.dispatch('date_trunc', 'dbt') (datepart, date)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__date_trunc"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018223, "supported_languages": null}, "macro.dbt.default__date_trunc": {"name": "default__date_trunc", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_trunc.sql", "original_file_path": "macros/utils/date_trunc.sql", "unique_id": "macro.dbt.default__date_trunc", "macro_sql": "{% macro default__date_trunc(datepart, date) -%}\n date_trunc('{{datepart}}', {{date}})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018233, "supported_languages": null}, "macro.dbt.array_construct": {"name": "array_construct", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_construct.sql", "original_file_path": "macros/utils/array_construct.sql", "unique_id": "macro.dbt.array_construct", "macro_sql": "{% macro array_construct(inputs=[], data_type=api.Column.translate_type('integer')) -%}\n {{ return(adapter.dispatch('array_construct', 'dbt')(inputs, data_type)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__array_construct"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0182521, "supported_languages": null}, "macro.dbt.default__array_construct": {"name": "default__array_construct", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_construct.sql", "original_file_path": "macros/utils/array_construct.sql", "unique_id": "macro.dbt.default__array_construct", "macro_sql": "{% macro default__array_construct(inputs, data_type) -%}\n {% if inputs|length > 0 %}\n array[ {{ inputs|join(' , ') }} ]\n {% else %}\n array[]::{{data_type}}[]\n {% endif %}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018263, "supported_languages": null}, "macro.dbt.array_append": {"name": "array_append", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_append.sql", "original_file_path": "macros/utils/array_append.sql", "unique_id": "macro.dbt.array_append", "macro_sql": "{% macro array_append(array, new_element) -%}\n {{ return(adapter.dispatch('array_append', 'dbt')(array, new_element)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__array_append"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018283, "supported_languages": null}, "macro.dbt.default__array_append": {"name": "default__array_append", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_append.sql", "original_file_path": "macros/utils/array_append.sql", "unique_id": "macro.dbt.default__array_append", "macro_sql": "{% macro default__array_append(array, new_element) -%}\n array_append({{ array }}, {{ new_element }})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018293, "supported_languages": null}, "macro.dbt.create_schema": {"name": "create_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.create_schema", "macro_sql": "{% macro create_schema(relation) -%}\n {{ adapter.dispatch('create_schema', 'dbt')(relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__create_schema"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018314, "supported_languages": null}, "macro.dbt.default__create_schema": {"name": "default__create_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.default__create_schema", "macro_sql": "{% macro default__create_schema(relation) -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier() }}\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018324, "supported_languages": null}, "macro.dbt.drop_schema": {"name": "drop_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.drop_schema", "macro_sql": "{% macro drop_schema(relation) -%}\n {{ adapter.dispatch('drop_schema', 'dbt')(relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__drop_schema"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018334, "supported_languages": null}, "macro.dbt.default__drop_schema": {"name": "default__drop_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.default__drop_schema", "macro_sql": "{% macro default__drop_schema(relation) -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier() }} cascade\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018343, "supported_languages": null}, "macro.dbt.current_timestamp": {"name": "current_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.current_timestamp", "macro_sql": "{%- macro current_timestamp() -%}\n {{ adapter.dispatch('current_timestamp', 'dbt')() }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_postgres.postgres__current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018365, "supported_languages": null}, "macro.dbt.default__current_timestamp": {"name": "default__current_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__current_timestamp", "macro_sql": "{% macro default__current_timestamp() -%}\n {{ exceptions.raise_not_implemented(\n 'current_timestamp macro not implemented for adapter ' + adapter.type()) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0183759, "supported_languages": null}, "macro.dbt.snapshot_get_time": {"name": "snapshot_get_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.snapshot_get_time", "macro_sql": "\n\n{%- macro snapshot_get_time() -%}\n {{ adapter.dispatch('snapshot_get_time', 'dbt')() }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_postgres.postgres__snapshot_get_time"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018404, "supported_languages": null}, "macro.dbt.default__snapshot_get_time": {"name": "default__snapshot_get_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__snapshot_get_time", "macro_sql": "{% macro default__snapshot_get_time() %}\n {{ current_timestamp() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018414, "supported_languages": null}, "macro.dbt.get_snapshot_get_time_data_type": {"name": "get_snapshot_get_time_data_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.get_snapshot_get_time_data_type", "macro_sql": "{% macro get_snapshot_get_time_data_type() %}\n {% set snapshot_time = adapter.dispatch('snapshot_get_time', 'dbt')() %}\n {% set time_data_type_sql = 'select ' ~ snapshot_time ~ ' as dbt_snapshot_time' %}\n {% set snapshot_time_column_schema = get_column_schema_from_query(time_data_type_sql) %}\n {% set time_data_type = snapshot_time_column_schema[0].dtype %}\n {{ return(time_data_type or none) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.snapshot_get_time", "macro.dbt_postgres.postgres__snapshot_get_time", "macro.dbt.get_column_schema_from_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018424, "supported_languages": null}, "macro.dbt.current_timestamp_backcompat": {"name": "current_timestamp_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.current_timestamp_backcompat", "macro_sql": "{% macro current_timestamp_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_backcompat', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__current_timestamp_backcompat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018434, "supported_languages": null}, "macro.dbt.default__current_timestamp_backcompat": {"name": "default__current_timestamp_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__current_timestamp_backcompat", "macro_sql": "{% macro default__current_timestamp_backcompat() %}\n current_timestamp::timestamp\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018444, "supported_languages": null}, "macro.dbt.current_timestamp_in_utc_backcompat": {"name": "current_timestamp_in_utc_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.current_timestamp_in_utc_backcompat", "macro_sql": "{% macro current_timestamp_in_utc_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_in_utc_backcompat', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__current_timestamp_in_utc_backcompat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018453, "supported_languages": null}, "macro.dbt.default__current_timestamp_in_utc_backcompat": {"name": "default__current_timestamp_in_utc_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__current_timestamp_in_utc_backcompat", "macro_sql": "{% macro default__current_timestamp_in_utc_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_backcompat', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.current_timestamp_backcompat", "macro.dbt_postgres.postgres__current_timestamp_backcompat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018463, "supported_languages": null}, "macro.dbt.get_create_index_sql": {"name": "get_create_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.get_create_index_sql", "macro_sql": "{% macro get_create_index_sql(relation, index_dict) -%}\n {{ return(adapter.dispatch('get_create_index_sql', 'dbt')(relation, index_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_create_index_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018485, "supported_languages": null}, "macro.dbt.default__get_create_index_sql": {"name": "default__get_create_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__get_create_index_sql", "macro_sql": "{% macro default__get_create_index_sql(relation, index_dict) -%}\n {% do return(None) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018496, "supported_languages": null}, "macro.dbt.create_indexes": {"name": "create_indexes", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.create_indexes", "macro_sql": "{% macro create_indexes(relation) -%}\n {{ adapter.dispatch('create_indexes', 'dbt')(relation) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.01851, "supported_languages": null}, "macro.dbt.default__create_indexes": {"name": "default__create_indexes", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__create_indexes", "macro_sql": "{% macro default__create_indexes(relation) -%}\n {%- set _indexes = config.get('indexes', default=[]) -%}\n\n {% for _index_dict in _indexes %}\n {% set create_index_sql = get_create_index_sql(relation, _index_dict) %}\n {% if create_index_sql %}\n {% do run_query(create_index_sql) %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_create_index_sql", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.01852, "supported_languages": null}, "macro.dbt.get_drop_index_sql": {"name": "get_drop_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.get_drop_index_sql", "macro_sql": "{% macro get_drop_index_sql(relation, index_name) -%}\n {{ adapter.dispatch('get_drop_index_sql', 'dbt')(relation, index_name) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_drop_index_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018529, "supported_languages": null}, "macro.dbt.default__get_drop_index_sql": {"name": "default__get_drop_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__get_drop_index_sql", "macro_sql": "{% macro default__get_drop_index_sql(relation, index_name) -%}\n {{ exceptions.raise_compiler_error(\"`get_drop_index_sql has not been implemented for this adapter.\") }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018539, "supported_languages": null}, "macro.dbt.get_show_indexes_sql": {"name": "get_show_indexes_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.get_show_indexes_sql", "macro_sql": "{% macro get_show_indexes_sql(relation) -%}\n {{ adapter.dispatch('get_show_indexes_sql', 'dbt')(relation) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_show_indexes_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018549, "supported_languages": null}, "macro.dbt.default__get_show_indexes_sql": {"name": "default__get_show_indexes_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__get_show_indexes_sql", "macro_sql": "{% macro default__get_show_indexes_sql(relation) -%}\n {{ exceptions.raise_compiler_error(\"`get_show_indexes_sql has not been implemented for this adapter.\") }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018559, "supported_languages": null}, "macro.dbt.make_intermediate_relation": {"name": "make_intermediate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.make_intermediate_relation", "macro_sql": "{% macro make_intermediate_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_intermediate_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_intermediate_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018581, "supported_languages": null}, "macro.dbt.default__make_intermediate_relation": {"name": "default__make_intermediate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__make_intermediate_relation", "macro_sql": "{% macro default__make_intermediate_relation(base_relation, suffix) %}\n {{ return(default__make_temp_relation(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__make_temp_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018591, "supported_languages": null}, "macro.dbt.make_temp_relation": {"name": "make_temp_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.make_temp_relation", "macro_sql": "{% macro make_temp_relation(base_relation, suffix='__dbt_tmp') %}\n {#-- This ensures microbatch batches get unique temp relations to avoid clobbering --#}\n {% if suffix == '__dbt_tmp' and model.batch %}\n {% set suffix = suffix ~ '_' ~ model.batch.id %}\n {% endif %}\n\n {{ return(adapter.dispatch('make_temp_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_temp_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0186298, "supported_languages": null}, "macro.dbt.default__make_temp_relation": {"name": "default__make_temp_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__make_temp_relation", "macro_sql": "{% macro default__make_temp_relation(base_relation, suffix) %}\n {%- set temp_identifier = base_relation.identifier ~ suffix -%}\n {%- set temp_relation = base_relation.incorporate(\n path={\"identifier\": temp_identifier}) -%}\n\n {{ return(temp_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018641, "supported_languages": null}, "macro.dbt.make_backup_relation": {"name": "make_backup_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.make_backup_relation", "macro_sql": "{% macro make_backup_relation(base_relation, backup_relation_type, suffix='__dbt_backup') %}\n {{ return(adapter.dispatch('make_backup_relation', 'dbt')(base_relation, backup_relation_type, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_backup_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018652, "supported_languages": null}, "macro.dbt.default__make_backup_relation": {"name": "default__make_backup_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__make_backup_relation", "macro_sql": "{% macro default__make_backup_relation(base_relation, backup_relation_type, suffix) %}\n {%- set backup_identifier = base_relation.identifier ~ suffix -%}\n {%- set backup_relation = base_relation.incorporate(\n path={\"identifier\": backup_identifier},\n type=backup_relation_type\n ) -%}\n {{ return(backup_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018662, "supported_languages": null}, "macro.dbt.truncate_relation": {"name": "truncate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.truncate_relation", "macro_sql": "{% macro truncate_relation(relation) -%}\n {{ return(adapter.dispatch('truncate_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__truncate_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018671, "supported_languages": null}, "macro.dbt.default__truncate_relation": {"name": "default__truncate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__truncate_relation", "macro_sql": "{% macro default__truncate_relation(relation) -%}\n {% call statement('truncate_relation') -%}\n truncate table {{ relation.render() }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018682, "supported_languages": null}, "macro.dbt.get_or_create_relation": {"name": "get_or_create_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.get_or_create_relation", "macro_sql": "{% macro get_or_create_relation(database, schema, identifier, type) -%}\n {{ return(adapter.dispatch('get_or_create_relation', 'dbt')(database, schema, identifier, type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_or_create_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018691, "supported_languages": null}, "macro.dbt.default__get_or_create_relation": {"name": "default__get_or_create_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__get_or_create_relation", "macro_sql": "{% macro default__get_or_create_relation(database, schema, identifier, type) %}\n {%- set target_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n\n {% if target_relation %}\n {% do return([true, target_relation]) %}\n {% endif %}\n\n {%- set new_relation = api.Relation.create(\n database=database,\n schema=schema,\n identifier=identifier,\n type=type\n ) -%}\n {% do return([false, new_relation]) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0187001, "supported_languages": null}, "macro.dbt.load_cached_relation": {"name": "load_cached_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.load_cached_relation", "macro_sql": "{% macro load_cached_relation(relation) %}\n {% do return(adapter.get_relation(\n database=relation.database,\n schema=relation.schema,\n identifier=relation.identifier\n )) -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.01871, "supported_languages": null}, "macro.dbt.load_relation": {"name": "load_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.load_relation", "macro_sql": "{% macro load_relation(relation) %}\n {{ return(load_cached_relation(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.01872, "supported_languages": null}, "macro.dbt.collect_freshness": {"name": "collect_freshness", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "unique_id": "macro.dbt.collect_freshness", "macro_sql": "{% macro collect_freshness(source, loaded_at_field, filter) %}\n {{ return(adapter.dispatch('collect_freshness', 'dbt')(source, loaded_at_field, filter))}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__collect_freshness"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018741, "supported_languages": null}, "macro.dbt.default__collect_freshness": {"name": "default__collect_freshness", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "unique_id": "macro.dbt.default__collect_freshness", "macro_sql": "{% macro default__collect_freshness(source, loaded_at_field, filter) %}\n {% call statement('collect_freshness', fetch_result=True, auto_begin=False) -%}\n select\n max({{ loaded_at_field }}) as max_loaded_at,\n {{ current_timestamp() }} as snapshotted_at\n from {{ source }}\n {% if filter %}\n where {{ filter }}\n {% endif %}\n {% endcall %}\n {{ return(load_result('collect_freshness')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.01875, "supported_languages": null}, "macro.dbt.collect_freshness_custom_sql": {"name": "collect_freshness_custom_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "unique_id": "macro.dbt.collect_freshness_custom_sql", "macro_sql": "{% macro collect_freshness_custom_sql(source, loaded_at_query) %}\n {{ return(adapter.dispatch('collect_freshness_custom_sql', 'dbt')(source, loaded_at_query))}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__collect_freshness_custom_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.01876, "supported_languages": null}, "macro.dbt.default__collect_freshness_custom_sql": {"name": "default__collect_freshness_custom_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "unique_id": "macro.dbt.default__collect_freshness_custom_sql", "macro_sql": "{% macro default__collect_freshness_custom_sql(source, loaded_at_query) %}\n {% call statement('collect_freshness_custom_sql', fetch_result=True, auto_begin=False) -%}\n with source_query as (\n {{ loaded_at_query }}\n )\n select\n (select * from source_query) as max_loaded_at,\n {{ current_timestamp() }} as snapshotted_at\n {% endcall %}\n {{ return(load_result('collect_freshness_custom_sql')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.01877, "supported_languages": null}, "macro.dbt.validate_sql": {"name": "validate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/validate_sql.sql", "original_file_path": "macros/adapters/validate_sql.sql", "unique_id": "macro.dbt.validate_sql", "macro_sql": "{% macro validate_sql(sql) -%}\n {{ return(adapter.dispatch('validate_sql', 'dbt')(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__validate_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018788, "supported_languages": null}, "macro.dbt.default__validate_sql": {"name": "default__validate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/validate_sql.sql", "original_file_path": "macros/adapters/validate_sql.sql", "unique_id": "macro.dbt.default__validate_sql", "macro_sql": "{% macro default__validate_sql(sql) -%}\n {% call statement('validate_sql') -%}\n explain {{ sql }}\n {% endcall %}\n {{ return(load_result('validate_sql')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0187972, "supported_languages": null}, "macro.dbt.copy_grants": {"name": "copy_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.copy_grants", "macro_sql": "{% macro copy_grants() %}\n {{ return(adapter.dispatch('copy_grants', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__copy_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018823, "supported_languages": null}, "macro.dbt.default__copy_grants": {"name": "default__copy_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__copy_grants", "macro_sql": "{% macro default__copy_grants() %}\n {{ return(True) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018833, "supported_languages": null}, "macro.dbt.support_multiple_grantees_per_dcl_statement": {"name": "support_multiple_grantees_per_dcl_statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.support_multiple_grantees_per_dcl_statement", "macro_sql": "{% macro support_multiple_grantees_per_dcl_statement() %}\n {{ return(adapter.dispatch('support_multiple_grantees_per_dcl_statement', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__support_multiple_grantees_per_dcl_statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018843, "supported_languages": null}, "macro.dbt.default__support_multiple_grantees_per_dcl_statement": {"name": "default__support_multiple_grantees_per_dcl_statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__support_multiple_grantees_per_dcl_statement", "macro_sql": "\n\n{%- macro default__support_multiple_grantees_per_dcl_statement() -%}\n {{ return(True) }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018852, "supported_languages": null}, "macro.dbt.should_revoke": {"name": "should_revoke", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.should_revoke", "macro_sql": "{% macro should_revoke(existing_relation, full_refresh_mode=True) %}\n\n {% if not existing_relation %}\n {#-- The table doesn't already exist, so no grants to copy over --#}\n {{ return(False) }}\n {% elif full_refresh_mode %}\n {#-- The object is being REPLACED -- whether grants are copied over depends on the value of user config --#}\n {{ return(copy_grants()) }}\n {% else %}\n {#-- The table is being merged/upserted/inserted -- grants will be carried over --#}\n {{ return(True) }}\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.copy_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018862, "supported_languages": null}, "macro.dbt.get_show_grant_sql": {"name": "get_show_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_show_grant_sql", "macro_sql": "{% macro get_show_grant_sql(relation) %}\n {{ return(adapter.dispatch(\"get_show_grant_sql\", \"dbt\")(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_show_grant_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018871, "supported_languages": null}, "macro.dbt.default__get_show_grant_sql": {"name": "default__get_show_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_show_grant_sql", "macro_sql": "{% macro default__get_show_grant_sql(relation) %}\n show grants on {{ relation.render() }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018881, "supported_languages": null}, "macro.dbt.get_grant_sql": {"name": "get_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_grant_sql", "macro_sql": "{% macro get_grant_sql(relation, privilege, grantees) %}\n {{ return(adapter.dispatch('get_grant_sql', 'dbt')(relation, privilege, grantees)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_grant_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018891, "supported_languages": null}, "macro.dbt.default__get_grant_sql": {"name": "default__get_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_grant_sql", "macro_sql": "\n\n{%- macro default__get_grant_sql(relation, privilege, grantees) -%}\n grant {{ privilege }} on {{ relation.render() }} to {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018901, "supported_languages": null}, "macro.dbt.get_revoke_sql": {"name": "get_revoke_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_revoke_sql", "macro_sql": "{% macro get_revoke_sql(relation, privilege, grantees) %}\n {{ return(adapter.dispatch('get_revoke_sql', 'dbt')(relation, privilege, grantees)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_revoke_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018911, "supported_languages": null}, "macro.dbt.default__get_revoke_sql": {"name": "default__get_revoke_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_revoke_sql", "macro_sql": "\n\n{%- macro default__get_revoke_sql(relation, privilege, grantees) -%}\n revoke {{ privilege }} on {{ relation.render() }} from {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.01892, "supported_languages": null}, "macro.dbt.get_dcl_statement_list": {"name": "get_dcl_statement_list", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_dcl_statement_list", "macro_sql": "{% macro get_dcl_statement_list(relation, grant_config, get_dcl_macro) %}\n {{ return(adapter.dispatch('get_dcl_statement_list', 'dbt')(relation, grant_config, get_dcl_macro)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_dcl_statement_list"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.01893, "supported_languages": null}, "macro.dbt.default__get_dcl_statement_list": {"name": "default__get_dcl_statement_list", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_dcl_statement_list", "macro_sql": "\n\n{%- macro default__get_dcl_statement_list(relation, grant_config, get_dcl_macro) -%}\n {#\n -- Unpack grant_config into specific privileges and the set of users who need them granted/revoked.\n -- Depending on whether this database supports multiple grantees per statement, pass in the list of\n -- all grantees per privilege, or (if not) template one statement per privilege-grantee pair.\n -- `get_dcl_macro` will be either `get_grant_sql` or `get_revoke_sql`\n #}\n {%- set dcl_statements = [] -%}\n {%- for privilege, grantees in grant_config.items() %}\n {%- if support_multiple_grantees_per_dcl_statement() and grantees -%}\n {%- set dcl = get_dcl_macro(relation, privilege, grantees) -%}\n {%- do dcl_statements.append(dcl) -%}\n {%- else -%}\n {%- for grantee in grantees -%}\n {% set dcl = get_dcl_macro(relation, privilege, [grantee]) %}\n {%- do dcl_statements.append(dcl) -%}\n {% endfor -%}\n {%- endif -%}\n {%- endfor -%}\n {{ return(dcl_statements) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.support_multiple_grantees_per_dcl_statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0189471, "supported_languages": null}, "macro.dbt.call_dcl_statements": {"name": "call_dcl_statements", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.call_dcl_statements", "macro_sql": "{% macro call_dcl_statements(dcl_statement_list) %}\n {{ return(adapter.dispatch(\"call_dcl_statements\", \"dbt\")(dcl_statement_list)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__call_dcl_statements"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018959, "supported_languages": null}, "macro.dbt.default__call_dcl_statements": {"name": "default__call_dcl_statements", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__call_dcl_statements", "macro_sql": "{% macro default__call_dcl_statements(dcl_statement_list) %}\n {#\n -- By default, supply all grant + revoke statements in a single semicolon-separated block,\n -- so that they're all processed together.\n\n -- Some databases do not support this. Those adapters will need to override this macro\n -- to run each statement individually.\n #}\n {% call statement('grants') %}\n {% for dcl_statement in dcl_statement_list %}\n {{ dcl_statement }};\n {% endfor %}\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.018969, "supported_languages": null}, "macro.dbt.apply_grants": {"name": "apply_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.apply_grants", "macro_sql": "{% macro apply_grants(relation, grant_config, should_revoke) %}\n {{ return(adapter.dispatch(\"apply_grants\", \"dbt\")(relation, grant_config, should_revoke)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__apply_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.01898, "supported_languages": null}, "macro.dbt.default__apply_grants": {"name": "default__apply_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__apply_grants", "macro_sql": "{% macro default__apply_grants(relation, grant_config, should_revoke=True) %}\n {#-- If grant_config is {} or None, this is a no-op --#}\n {% if grant_config %}\n {% if should_revoke %}\n {#-- We think previous grants may have carried over --#}\n {#-- Show current grants and calculate diffs --#}\n {% set current_grants_table = run_query(get_show_grant_sql(relation)) %}\n {% set current_grants_dict = adapter.standardize_grants_dict(current_grants_table) %}\n {% set needs_granting = diff_of_two_dicts(grant_config, current_grants_dict) %}\n {% set needs_revoking = diff_of_two_dicts(current_grants_dict, grant_config) %}\n {% if not (needs_granting or needs_revoking) %}\n {{ log('On ' ~ relation.render() ~': All grants are in place, no revocation or granting needed.')}}\n {% endif %}\n {% else %}\n {#-- We don't think there's any chance of previous grants having carried over. --#}\n {#-- Jump straight to granting what the user has configured. --#}\n {% set needs_revoking = {} %}\n {% set needs_granting = grant_config %}\n {% endif %}\n {% if needs_granting or needs_revoking %}\n {% set revoke_statement_list = get_dcl_statement_list(relation, needs_revoking, get_revoke_sql) %}\n {% set grant_statement_list = get_dcl_statement_list(relation, needs_granting, get_grant_sql) %}\n {% set dcl_statement_list = revoke_statement_list + grant_statement_list %}\n {% if dcl_statement_list %}\n {{ call_dcl_statements(dcl_statement_list) }}\n {% endif %}\n {% endif %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query", "macro.dbt.get_show_grant_sql", "macro.dbt.get_dcl_statement_list", "macro.dbt.call_dcl_statements"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.01899, "supported_languages": null}, "macro.dbt.get_show_sql": {"name": "get_show_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/show.sql", "original_file_path": "macros/adapters/show.sql", "unique_id": "macro.dbt.get_show_sql", "macro_sql": "{% macro get_show_sql(compiled_code, sql_header, limit) -%}\n {%- if sql_header is not none -%}\n {{ sql_header }}\n {%- endif %}\n {{ get_limit_subquery_sql(compiled_code, limit) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_limit_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.019017, "supported_languages": null}, "macro.dbt.get_limit_subquery_sql": {"name": "get_limit_subquery_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/show.sql", "original_file_path": "macros/adapters/show.sql", "unique_id": "macro.dbt.get_limit_subquery_sql", "macro_sql": "\n{%- macro get_limit_subquery_sql(sql, limit) -%}\n {{ adapter.dispatch('get_limit_sql', 'dbt')(sql, limit) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__get_limit_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.019028, "supported_languages": null}, "macro.dbt.default__get_limit_sql": {"name": "default__get_limit_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/show.sql", "original_file_path": "macros/adapters/show.sql", "unique_id": "macro.dbt.default__get_limit_sql", "macro_sql": "{% macro default__get_limit_sql(sql, limit) %}\n {{ sql }}\n {% if limit is not none %}\n limit {{ limit }}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.019038, "supported_languages": null}, "macro.dbt.alter_column_comment": {"name": "alter_column_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.alter_column_comment", "macro_sql": "{% macro alter_column_comment(relation, column_dict) -%}\n {{ return(adapter.dispatch('alter_column_comment', 'dbt')(relation, column_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.019069, "supported_languages": null}, "macro.dbt.default__alter_column_comment": {"name": "default__alter_column_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.default__alter_column_comment", "macro_sql": "{% macro default__alter_column_comment(relation, column_dict) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_column_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.01908, "supported_languages": null}, "macro.dbt.alter_relation_comment": {"name": "alter_relation_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.alter_relation_comment", "macro_sql": "{% macro alter_relation_comment(relation, relation_comment) -%}\n {{ return(adapter.dispatch('alter_relation_comment', 'dbt')(relation, relation_comment)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__alter_relation_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.019098, "supported_languages": null}, "macro.dbt.default__alter_relation_comment": {"name": "default__alter_relation_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.default__alter_relation_comment", "macro_sql": "{% macro default__alter_relation_comment(relation, relation_comment) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_relation_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.019109, "supported_languages": null}, "macro.dbt.persist_docs": {"name": "persist_docs", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.persist_docs", "macro_sql": "{% macro persist_docs(relation, model, for_relation=true, for_columns=true) -%}\n {{ return(adapter.dispatch('persist_docs', 'dbt')(relation, model, for_relation, for_columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.019119, "supported_languages": null}, "macro.dbt.default__persist_docs": {"name": "default__persist_docs", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.default__persist_docs", "macro_sql": "{% macro default__persist_docs(relation, model, for_relation, for_columns) -%}\n {% if for_relation and config.persist_relation_docs() and model.description %}\n {% do run_query(alter_relation_comment(relation, model.description)) %}\n {% endif %}\n\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {% do run_query(alter_column_comment(relation, model.columns)) %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query", "macro.dbt.alter_relation_comment", "macro.dbt.alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.019129, "supported_languages": null}, "macro.dbt.get_catalog_relations": {"name": "get_catalog_relations", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_catalog_relations", "macro_sql": "{% macro get_catalog_relations(information_schema, relations) -%}\n {{ return(adapter.dispatch('get_catalog_relations', 'dbt')(information_schema, relations)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_catalog_relations"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0191538, "supported_languages": null}, "macro.dbt.default__get_catalog_relations": {"name": "default__get_catalog_relations", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_catalog_relations", "macro_sql": "{% macro default__get_catalog_relations(information_schema, relations) -%}\n {% set typename = adapter.type() %}\n {% set msg -%}\n get_catalog_relations not implemented for {{ typename }}\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0191638, "supported_languages": null}, "macro.dbt.get_catalog": {"name": "get_catalog", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_catalog", "macro_sql": "{% macro get_catalog(information_schema, schemas) -%}\n {{ return(adapter.dispatch('get_catalog', 'dbt')(information_schema, schemas)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_catalog"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0191739, "supported_languages": null}, "macro.dbt.default__get_catalog": {"name": "default__get_catalog", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_catalog", "macro_sql": "{% macro default__get_catalog(information_schema, schemas) -%}\n\n {% set typename = adapter.type() %}\n {% set msg -%}\n get_catalog not implemented for {{ typename }}\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.019184, "supported_languages": null}, "macro.dbt.information_schema_name": {"name": "information_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.information_schema_name", "macro_sql": "{% macro information_schema_name(database) %}\n {{ return(adapter.dispatch('information_schema_name', 'dbt')(database)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__information_schema_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0191948, "supported_languages": null}, "macro.dbt.default__information_schema_name": {"name": "default__information_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__information_schema_name", "macro_sql": "{% macro default__information_schema_name(database) -%}\n {%- if database -%}\n {{ database }}.INFORMATION_SCHEMA\n {%- else -%}\n INFORMATION_SCHEMA\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.019205, "supported_languages": null}, "macro.dbt.list_schemas": {"name": "list_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.list_schemas", "macro_sql": "{% macro list_schemas(database) -%}\n {{ return(adapter.dispatch('list_schemas', 'dbt')(database)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__list_schemas"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0192142, "supported_languages": null}, "macro.dbt.default__list_schemas": {"name": "default__list_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__list_schemas", "macro_sql": "{% macro default__list_schemas(database) -%}\n {% set sql %}\n select distinct schema_name\n from {{ information_schema_name(database) }}.SCHEMATA\n where catalog_name ilike '{{ database }}'\n {% endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.information_schema_name", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0192242, "supported_languages": null}, "macro.dbt.check_schema_exists": {"name": "check_schema_exists", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.check_schema_exists", "macro_sql": "{% macro check_schema_exists(information_schema, schema) -%}\n {{ return(adapter.dispatch('check_schema_exists', 'dbt')(information_schema, schema)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__check_schema_exists"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.019234, "supported_languages": null}, "macro.dbt.default__check_schema_exists": {"name": "default__check_schema_exists", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__check_schema_exists", "macro_sql": "{% macro default__check_schema_exists(information_schema, schema) -%}\n {% set sql -%}\n select count(*)\n from {{ information_schema.replace(information_schema_view='SCHEMATA') }}\n where catalog_name='{{ information_schema.database }}'\n and schema_name='{{ schema }}'\n {%- endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.replace", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.019244, "supported_languages": null}, "macro.dbt.list_relations_without_caching": {"name": "list_relations_without_caching", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.list_relations_without_caching", "macro_sql": "{% macro list_relations_without_caching(schema_relation) %}\n {{ return(adapter.dispatch('list_relations_without_caching', 'dbt')(schema_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__list_relations_without_caching"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.019254, "supported_languages": null}, "macro.dbt.default__list_relations_without_caching": {"name": "default__list_relations_without_caching", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__list_relations_without_caching", "macro_sql": "{% macro default__list_relations_without_caching(schema_relation) %}\n {{ exceptions.raise_not_implemented(\n 'list_relations_without_caching macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.019263, "supported_languages": null}, "macro.dbt.get_catalog_for_single_relation": {"name": "get_catalog_for_single_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_catalog_for_single_relation", "macro_sql": "{% macro get_catalog_for_single_relation(relation) %}\n {{ return(adapter.dispatch('get_catalog_for_single_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_catalog_for_single_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.019273, "supported_languages": null}, "macro.dbt.default__get_catalog_for_single_relation": {"name": "default__get_catalog_for_single_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_catalog_for_single_relation", "macro_sql": "{% macro default__get_catalog_for_single_relation(relation) %}\n {{ exceptions.raise_not_implemented(\n 'get_catalog_for_single_relation macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.019283, "supported_languages": null}, "macro.dbt.get_relations": {"name": "get_relations", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_relations", "macro_sql": "{% macro get_relations() %}\n {{ return(adapter.dispatch('get_relations', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_relations"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0192919, "supported_languages": null}, "macro.dbt.default__get_relations": {"name": "default__get_relations", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_relations", "macro_sql": "{% macro default__get_relations() %}\n {{ exceptions.raise_not_implemented(\n 'get_relations macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.019302, "supported_languages": null}, "macro.dbt.get_relation_last_modified": {"name": "get_relation_last_modified", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_relation_last_modified", "macro_sql": "{% macro get_relation_last_modified(information_schema, relations) %}\n {{ return(adapter.dispatch('get_relation_last_modified', 'dbt')(information_schema, relations)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_relation_last_modified"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0193121, "supported_languages": null}, "macro.dbt.default__get_relation_last_modified": {"name": "default__get_relation_last_modified", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_relation_last_modified", "macro_sql": "{% macro default__get_relation_last_modified(information_schema, relations) %}\n {{ exceptions.raise_not_implemented(\n 'get_relation_last_modified macro not implemented for adapter ' + adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0193212, "supported_languages": null}, "macro.dbt.get_columns_in_relation": {"name": "get_columns_in_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_columns_in_relation", "macro_sql": "{% macro get_columns_in_relation(relation) -%}\n {{ return(adapter.dispatch('get_columns_in_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_columns_in_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.019346, "supported_languages": null}, "macro.dbt.default__get_columns_in_relation": {"name": "default__get_columns_in_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_columns_in_relation", "macro_sql": "{% macro default__get_columns_in_relation(relation) -%}\n {{ exceptions.raise_not_implemented(\n 'get_columns_in_relation macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.019357, "supported_languages": null}, "macro.dbt.sql_convert_columns_in_relation": {"name": "sql_convert_columns_in_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.sql_convert_columns_in_relation", "macro_sql": "{% macro sql_convert_columns_in_relation(table) -%}\n {% set columns = [] %}\n {% for row in table %}\n {% do columns.append(api.Column(*row)) %}\n {% endfor %}\n {{ return(columns) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0193658, "supported_languages": null}, "macro.dbt.get_list_of_column_names": {"name": "get_list_of_column_names", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_list_of_column_names", "macro_sql": "\n\n{%- macro get_list_of_column_names(columns) -%}\n {% set col_names = [] %}\n {% for col in columns %}\n {% do col_names.append(col.name) %}\n {% endfor %}\n {{ return(col_names) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.019376, "supported_languages": null}, "macro.dbt.get_empty_subquery_sql": {"name": "get_empty_subquery_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_empty_subquery_sql", "macro_sql": "{% macro get_empty_subquery_sql(select_sql, select_sql_header=none) -%}\n {{ return(adapter.dispatch('get_empty_subquery_sql', 'dbt')(select_sql, select_sql_header)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_empty_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0193868, "supported_languages": null}, "macro.dbt.default__get_empty_subquery_sql": {"name": "default__get_empty_subquery_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_empty_subquery_sql", "macro_sql": "{% macro default__get_empty_subquery_sql(select_sql, select_sql_header=none) %}\n {%- if select_sql_header is not none -%}\n {{ select_sql_header }}\n {%- endif -%}\n select * from (\n {{ select_sql }}\n ) as __dbt_sbq\n where false\n limit 0\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.019396, "supported_languages": null}, "macro.dbt.get_empty_schema_sql": {"name": "get_empty_schema_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_empty_schema_sql", "macro_sql": "{% macro get_empty_schema_sql(columns) -%}\n {{ return(adapter.dispatch('get_empty_schema_sql', 'dbt')(columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_empty_schema_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.019406, "supported_languages": null}, "macro.dbt.default__get_empty_schema_sql": {"name": "default__get_empty_schema_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_empty_schema_sql", "macro_sql": "{% macro default__get_empty_schema_sql(columns) %}\n {%- set col_err = [] -%}\n {%- set col_naked_numeric = [] -%}\n select\n {% for i in columns %}\n {%- set col = columns[i] -%}\n {%- if col['data_type'] is not defined -%}\n {%- do col_err.append(col['name']) -%}\n {#-- If this column's type is just 'numeric' then it is missing precision/scale, raise a warning --#}\n {%- elif col['data_type'].strip().lower() in ('numeric', 'decimal', 'number') -%}\n {%- do col_naked_numeric.append(col['name']) -%}\n {%- endif -%}\n {% set col_name = adapter.quote(col['name']) if col.get('quote') else col['name'] %}\n {{ cast('null', col['data_type']) }} as {{ col_name }}{{ \", \" if not loop.last }}\n {%- endfor -%}\n {%- if (col_err | length) > 0 -%}\n {{ exceptions.column_type_missing(column_names=col_err) }}\n {%- elif (col_naked_numeric | length) > 0 -%}\n {{ exceptions.warn(\"Detected columns with numeric type and unspecified precision/scale, this can lead to unintended rounding: \" ~ col_naked_numeric ~ \"`\") }}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.cast"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0194151, "supported_languages": null}, "macro.dbt.get_column_schema_from_query": {"name": "get_column_schema_from_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_column_schema_from_query", "macro_sql": "{% macro get_column_schema_from_query(select_sql, select_sql_header=none) -%}\n {% set columns = [] %}\n {# -- Using an 'empty subquery' here to get the same schema as the given select_sql statement, without necessitating a data scan.#}\n {% set sql = get_empty_subquery_sql(select_sql, select_sql_header) %}\n {% set column_schema = adapter.get_column_schema_from_query(sql) %}\n {{ return(column_schema) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_empty_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.019425, "supported_languages": null}, "macro.dbt.get_columns_in_query": {"name": "get_columns_in_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_columns_in_query", "macro_sql": "{% macro get_columns_in_query(select_sql) -%}\n {{ return(adapter.dispatch('get_columns_in_query', 'dbt')(select_sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_columns_in_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.019435, "supported_languages": null}, "macro.dbt.default__get_columns_in_query": {"name": "default__get_columns_in_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_columns_in_query", "macro_sql": "{% macro default__get_columns_in_query(select_sql) %}\n {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%}\n {{ get_empty_subquery_sql(select_sql) }}\n {% endcall %}\n {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.get_empty_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.019465, "supported_languages": null}, "macro.dbt.alter_column_type": {"name": "alter_column_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.alter_column_type", "macro_sql": "{% macro alter_column_type(relation, column_name, new_column_type) -%}\n {{ return(adapter.dispatch('alter_column_type', 'dbt')(relation, column_name, new_column_type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__alter_column_type"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.019476, "supported_languages": null}, "macro.dbt.default__alter_column_type": {"name": "default__alter_column_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__alter_column_type", "macro_sql": "{% macro default__alter_column_type(relation, column_name, new_column_type) -%}\n {#\n 1. Create a new column (w/ temp name and correct type)\n 2. Copy data over to it\n 3. Drop the existing column (cascade!)\n 4. Rename the new column to existing column\n #}\n {%- set tmp_column = column_name + \"__dbt_alter\" -%}\n\n {% call statement('alter_column_type') %}\n alter table {{ relation.render() }} add column {{ adapter.quote(tmp_column) }} {{ new_column_type }};\n update {{ relation.render() }} set {{ adapter.quote(tmp_column) }} = {{ adapter.quote(column_name) }};\n alter table {{ relation.render() }} drop column {{ adapter.quote(column_name) }} cascade;\n alter table {{ relation.render() }} rename column {{ adapter.quote(tmp_column) }} to {{ adapter.quote(column_name) }}\n {% endcall %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.019485, "supported_languages": null}, "macro.dbt.alter_relation_add_remove_columns": {"name": "alter_relation_add_remove_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.alter_relation_add_remove_columns", "macro_sql": "{% macro alter_relation_add_remove_columns(relation, add_columns = none, remove_columns = none) -%}\n {{ return(adapter.dispatch('alter_relation_add_remove_columns', 'dbt')(relation, add_columns, remove_columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__alter_relation_add_remove_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.019495, "supported_languages": null}, "macro.dbt.default__alter_relation_add_remove_columns": {"name": "default__alter_relation_add_remove_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__alter_relation_add_remove_columns", "macro_sql": "{% macro default__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}\n\n {% if add_columns is none %}\n {% set add_columns = [] %}\n {% endif %}\n {% if remove_columns is none %}\n {% set remove_columns = [] %}\n {% endif %}\n\n {% set sql -%}\n\n alter {{ relation.type }} {{ relation.render() }}\n\n {% for column in add_columns %}\n add column {{ column.quoted }} {{ column.data_type }}{{ ',' if not loop.last }}\n {% endfor %}{{ ',' if add_columns and remove_columns }}\n\n {% for column in remove_columns %}\n drop column {{ column.quoted }}{{ ',' if not loop.last }}\n {% endfor %}\n\n {%- endset -%}\n\n {% do run_query(sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0195048, "supported_languages": null}, "macro.dbt.get_fixture_sql": {"name": "get_fixture_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.get_fixture_sql", "macro_sql": "{% macro get_fixture_sql(rows, column_name_to_data_types) %}\n-- Fixture for {{ model.name }}\n{% set default_row = {} %}\n\n{%- if not column_name_to_data_types -%}\n{#-- Use defer_relation IFF it is available in the manifest and 'this' is missing from the database --#}\n{%- set this_or_defer_relation = defer_relation if (defer_relation and not load_relation(this)) else this -%}\n{%- set columns_in_relation = adapter.get_columns_in_relation(this_or_defer_relation) -%}\n\n{%- set column_name_to_data_types = {} -%}\n{%- set column_name_to_quoted = {} -%}\n{%- for column in columns_in_relation -%}\n\n{#-- This needs to be a case-insensitive comparison --#}\n{%- do column_name_to_data_types.update({column.name|lower: column.data_type}) -%}\n{%- do column_name_to_quoted.update({column.name|lower: column.quoted}) -%}\n{%- endfor -%}\n{%- endif -%}\n\n{%- if not column_name_to_data_types -%}\n {{ exceptions.raise_compiler_error(\"Not able to get columns for unit test '\" ~ model.name ~ \"' from relation \" ~ this ~ \" because the relation doesn't exist\") }}\n{%- endif -%}\n\n{%- for column_name, column_type in column_name_to_data_types.items() -%}\n {%- do default_row.update({column_name: (safe_cast(\"null\", column_type) | trim )}) -%}\n{%- endfor -%}\n\n{{ validate_fixture_rows(rows, row_number) }}\n\n{%- for row in rows -%}\n{%- set formatted_row = format_row(row, column_name_to_data_types) -%}\n{%- set default_row_copy = default_row.copy() -%}\n{%- do default_row_copy.update(formatted_row) -%}\nselect\n{%- for column_name, column_value in default_row_copy.items() %} {{ column_value }} as {{ column_name_to_quoted[column_name] }}{% if not loop.last -%}, {%- endif %}\n{%- endfor %}\n{%- if not loop.last %}\nunion all\n{% endif %}\n{%- endfor -%}\n\n{%- if (rows | length) == 0 -%}\n select\n {%- for column_name, column_value in default_row.items() %} {{ column_value }} as {{ column_name_to_quoted[column_name] }}{% if not loop.last -%},{%- endif %}\n {%- endfor %}\n limit 0\n{%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.load_relation", "macro.dbt.safe_cast", "macro.dbt.validate_fixture_rows", "macro.dbt.format_row"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.019527, "supported_languages": null}, "macro.dbt.get_expected_sql": {"name": "get_expected_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.get_expected_sql", "macro_sql": "{% macro get_expected_sql(rows, column_name_to_data_types, column_name_to_quoted) %}\n\n{%- if (rows | length) == 0 -%}\n select * from dbt_internal_unit_test_actual\n limit 0\n{%- else -%}\n{%- for row in rows -%}\n{%- set formatted_row = format_row(row, column_name_to_data_types) -%}\nselect\n{%- for column_name, column_value in formatted_row.items() %} {{ column_value }} as {{ column_name_to_quoted[column_name] }}{% if not loop.last -%}, {%- endif %}\n{%- endfor %}\n{%- if not loop.last %}\nunion all\n{% endif %}\n{%- endfor -%}\n{%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.format_row"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.019538, "supported_languages": null}, "macro.dbt.format_row": {"name": "format_row", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.format_row", "macro_sql": "\n\n{%- macro format_row(row, column_name_to_data_types) -%}\n {#-- generate case-insensitive formatted row --#}\n {% set formatted_row = {} %}\n {%- for column_name, column_value in row.items() -%}\n {% set column_name = column_name|lower %}\n\n {%- if column_name not in column_name_to_data_types %}\n {#-- if user-provided row contains column name that relation does not contain, raise an error --#}\n {% set fixture_name = \"expected output\" if model.resource_type == 'unit_test' else (\"'\" ~ model.name ~ \"'\") %}\n {{ exceptions.raise_compiler_error(\n \"Invalid column name: '\" ~ column_name ~ \"' in unit test fixture for \" ~ fixture_name ~ \".\"\n \"\\nAccepted columns for \" ~ fixture_name ~ \" are: \" ~ (column_name_to_data_types.keys()|list)\n ) }}\n {%- endif -%}\n\n {%- set column_type = column_name_to_data_types[column_name] %}\n\n {#-- sanitize column_value: wrap yaml strings in quotes, apply cast --#}\n {%- set column_value_clean = column_value -%}\n {%- if column_value is string -%}\n {%- set column_value_clean = dbt.string_literal(dbt.escape_single_quotes(column_value)) -%}\n {%- elif column_value is none -%}\n {%- set column_value_clean = 'null' -%}\n {%- endif -%}\n\n {%- set row_update = {column_name: safe_cast(column_value_clean, column_type) } -%}\n {%- do formatted_row.update(row_update) -%}\n {%- endfor -%}\n {{ return(formatted_row) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.string_literal", "macro.dbt.escape_single_quotes", "macro.dbt.safe_cast"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.019548, "supported_languages": null}, "macro.dbt.validate_fixture_rows": {"name": "validate_fixture_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.validate_fixture_rows", "macro_sql": "{%- macro validate_fixture_rows(rows, row_number) -%}\n {{ return(adapter.dispatch('validate_fixture_rows', 'dbt')(rows, row_number)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__validate_fixture_rows"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.019558, "supported_languages": null}, "macro.dbt.default__validate_fixture_rows": {"name": "default__validate_fixture_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.default__validate_fixture_rows", "macro_sql": "{%- macro default__validate_fixture_rows(rows, row_number) -%}\n {# This is an abstract method for adapter overrides as needed #}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0195682, "supported_languages": null}, "macro.dbt.resolve_model_name": {"name": "resolve_model_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.resolve_model_name", "macro_sql": "{% macro resolve_model_name(input_model_name) %}\n {{ return(adapter.dispatch('resolve_model_name', 'dbt')(input_model_name)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__resolve_model_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.01959, "supported_languages": null}, "macro.dbt.default__resolve_model_name": {"name": "default__resolve_model_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.default__resolve_model_name", "macro_sql": "\n\n{%- macro default__resolve_model_name(input_model_name) -%}\n {{ input_model_name | string | replace('\"', '\\\"') }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0196, "supported_languages": null}, "macro.dbt.build_ref_function": {"name": "build_ref_function", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.build_ref_function", "macro_sql": "{% macro build_ref_function(model) %}\n\n {%- set ref_dict = {} -%}\n {%- for _ref in model.refs -%}\n {% set _ref_args = [_ref.get('package'), _ref['name']] if _ref.get('package') else [_ref['name'],] %}\n {%- set resolved = ref(*_ref_args, v=_ref.get('version')) -%}\n\n {#\n We want to get the string of the returned relation by calling .render() in order to skip sample/empty\n mode rendering logic. However, people override the default ref macro, and often return a string instead\n of a relation (like the ref macro does by default). Thus, to make sure we dont blow things up, we have\n to ensure the resolved relation has a .render() method.\n #}\n {%- if resolved.render is defined and resolved.render is callable -%}\n {%- set resolved = resolved.render() -%}\n {%- endif -%}\n\n {%- if _ref.get('version') -%}\n {% do _ref_args.extend([\"v\" ~ _ref['version']]) %}\n {%- endif -%}\n {%- do ref_dict.update({_ref_args | join('.'): resolve_model_name(resolved)}) -%}\n {%- endfor -%}\n\ndef ref(*args, **kwargs):\n refs = {{ ref_dict | tojson }}\n key = '.'.join(args)\n version = kwargs.get(\"v\") or kwargs.get(\"version\")\n if version:\n key += f\".v{version}\"\n dbt_load_df_function = kwargs.get(\"dbt_load_df_function\")\n return dbt_load_df_function(refs[key])\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.resolve_model_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.019609, "supported_languages": null}, "macro.dbt.build_source_function": {"name": "build_source_function", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.build_source_function", "macro_sql": "{% macro build_source_function(model) %}\n\n {%- set source_dict = {} -%}\n {%- for _source in model.sources -%}\n {%- set resolved = source(*_source) -%}\n {%- do source_dict.update({_source | join('.'): resolve_model_name(resolved)}) -%}\n {%- endfor -%}\n\ndef source(*args, dbt_load_df_function):\n sources = {{ source_dict | tojson }}\n key = '.'.join(args)\n return dbt_load_df_function(sources[key])\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.resolve_model_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.019619, "supported_languages": null}, "macro.dbt.build_config_dict": {"name": "build_config_dict", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.build_config_dict", "macro_sql": "{% macro build_config_dict(model) %}\n {%- set config_dict = {} -%}\n {% set config_dbt_used = zip(model.config.config_keys_used, model.config.config_keys_defaults) | list %}\n {%- for key, default in config_dbt_used -%}\n {# weird type testing with enum, would be much easier to write this logic in Python! #}\n {%- if key == \"language\" -%}\n {%- set value = \"python\" -%}\n {%- endif -%}\n {%- set value = model.config.get(key, default) -%}\n {%- do config_dict.update({key: value}) -%}\n {%- endfor -%}\nconfig_dict = {{ config_dict }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.019629, "supported_languages": null}, "macro.dbt.py_script_postfix": {"name": "py_script_postfix", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.py_script_postfix", "macro_sql": "{% macro py_script_postfix(model) %}\n# This part is user provided model code\n# you will need to copy the next section to run the code\n# COMMAND ----------\n# this part is dbt logic for get ref work, do not modify\n\n{{ build_ref_function(model ) }}\n{{ build_source_function(model ) }}\n{{ build_config_dict(model) }}\n\nclass config:\n def __init__(self, *args, **kwargs):\n pass\n\n @staticmethod\n def get(key, default=None):\n return config_dict.get(key, default)\n\nclass this:\n \"\"\"dbt.this() or dbt.this.identifier\"\"\"\n database = \"{{ this.database }}\"\n schema = \"{{ this.schema }}\"\n identifier = \"{{ this.identifier }}\"\n {% set this_relation_name = resolve_model_name(this) %}\n def __repr__(self):\n return '{{ this_relation_name }}'\n\n\nclass dbtObj:\n def __init__(self, load_df_function) -> None:\n self.source = lambda *args: source(*args, dbt_load_df_function=load_df_function)\n self.ref = lambda *args, **kwargs: ref(*args, **kwargs, dbt_load_df_function=load_df_function)\n self.config = config\n self.this = this()\n self.is_incremental = {{ is_incremental() }}\n\n# COMMAND ----------\n{{py_script_comment()}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.build_ref_function", "macro.dbt.build_source_function", "macro.dbt.build_config_dict", "macro.dbt.resolve_model_name", "macro.dbt.is_incremental", "macro.dbt.py_script_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.019638, "supported_languages": null}, "macro.dbt.py_script_comment": {"name": "py_script_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.py_script_comment", "macro_sql": "{%macro py_script_comment()%}\n{%endmacro%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.019648, "supported_languages": null}, "macro.dbt.test_unique": {"name": "test_unique", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_unique", "macro_sql": "{% test unique(model, column_name) %}\n {% set macro = adapter.dispatch('test_unique', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_unique"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0197601, "supported_languages": null}, "macro.dbt.test_not_null": {"name": "test_not_null", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_not_null", "macro_sql": "{% test not_null(model, column_name) %}\n {% set macro = adapter.dispatch('test_not_null', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_not_null"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.019809, "supported_languages": null}, "macro.dbt.test_accepted_values": {"name": "test_accepted_values", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_accepted_values", "macro_sql": "{% test accepted_values(model, column_name, values, quote=True) %}\n {% set macro = adapter.dispatch('test_accepted_values', 'dbt') %}\n {{ macro(model, column_name, values, quote) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_accepted_values"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.019872, "supported_languages": null}, "macro.dbt.test_relationships": {"name": "test_relationships", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_relationships", "macro_sql": "{% test relationships(model, column_name, to, field) %}\n {% set macro = adapter.dispatch('test_relationships', 'dbt') %}\n {{ macro(model, column_name, to, field) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_relationships"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1759423716.0199392, "supported_languages": null}}, "docs": {"doc.test.somedoc": {"name": "somedoc", "resource_type": "doc", "package_name": "test", "path": "somedoc.md", "original_file_path": "models/somedoc.md", "unique_id": "doc.test.somedoc", "block_contents": "Testing, testing"}, "doc.dbt.__overview__": {"name": "__overview__", "resource_type": "doc", "package_name": "dbt", "path": "overview.md", "original_file_path": "docs/overview.md", "unique_id": "doc.dbt.__overview__", "block_contents": "### Welcome!\n\nWelcome to the auto-generated documentation for your dbt project!\n\n### Navigation\n\nYou can use the `Project` and `Database` navigation tabs on the left side of the window to explore the models\nin your project.\n\n#### Project Tab\nThe `Project` tab mirrors the directory structure of your dbt project. In this tab, you can see all of the\nmodels defined in your dbt project, as well as models imported from dbt packages.\n\n#### Database Tab\nThe `Database` tab also exposes your models, but in a format that looks more like a database explorer. This view\nshows relations (tables and views) grouped into database schemas. Note that ephemeral models are _not_ shown\nin this interface, as they do not exist in the database.\n\n### Graph Exploration\nYou can click the blue icon on the bottom-right corner of the page to view the lineage graph of your models.\n\nOn model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the `Expand`\nbutton at the top-right of this lineage pane, you'll be able to see all of the models that are used to build,\nor are built from, the model you're exploring.\n\nOnce expanded, you'll be able to use the `--select` and `--exclude` model selection syntax to filter the\nmodels in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax).\n\nNote that you can also right-click on models to interactively filter and explore the graph.\n\n---\n\n### More information\n\n- [What is dbt](https://docs.getdbt.com/docs/introduction)?\n- Read the [dbt viewpoint](https://docs.getdbt.com/docs/viewpoint)\n- [Installation](https://docs.getdbt.com/docs/installation)\n- Join the [dbt Community](https://www.getdbt.com/community/) for questions and discussion"}}, "exposures": {"exposure.test.simple_exposure": {"name": "simple_exposure", "resource_type": "exposure", "package_name": "test", "path": "schema.yml", "original_file_path": "models/schema.yml", "unique_id": "exposure.test.simple_exposure", "fqn": ["test", "simple_exposure"], "type": "dashboard", "owner": {"email": "something@example.com", "name": null}, "description": "", "label": null, "maturity": null, "meta": {}, "tags": [], "config": {"enabled": true, "tags": [], "meta": {}}, "unrendered_config": {}, "url": null, "depends_on": {"macros": [], "nodes": ["source.test.my_source.my_table", "model.test.my_model"]}, "refs": [{"name": "my_model", "package": null, "version": null}], "sources": [["my_source", "my_table"]], "metrics": [], "created_at": 1759423716.4544902}}, "metrics": {"metric.test.blue_customers_post_2010": {"name": "blue_customers_post_2010", "resource_type": "metric", "package_name": "test", "path": "schema.yml", "original_file_path": "models/schema.yml", "unique_id": "metric.test.blue_customers_post_2010", "fqn": ["test", "blue_customers_post_2010"], "description": "", "label": "Blue Customers since 2010", "type": "simple", "type_params": {"measure": {"name": "customers", "filter": {"where_filters": [{"where_sql_template": "{{ Dimension('id__favorite_color') }} = 'blue'"}]}, "alias": null, "join_to_timespine": false, "fill_nulls_with": null}, "input_measures": [{"name": "customers", "filter": {"where_filters": [{"where_sql_template": "{{ Dimension('id__favorite_color') }} = 'blue'"}]}, "alias": null, "join_to_timespine": false, "fill_nulls_with": null}], "numerator": null, "denominator": null, "expr": null, "window": null, "grain_to_date": null, "metrics": [], "conversion_type_params": null, "cumulative_type_params": null, "metric_aggregation_params": null}, "filter": {"where_filters": [{"where_sql_template": "{{ TimeDimension('id__created_at', 'day') }} > '2010-01-01'"}]}, "metadata": null, "time_granularity": null, "meta": {}, "tags": [], "config": {"enabled": true, "group": null, "meta": {}}, "unrendered_config": {}, "sources": [], "depends_on": {"macros": [], "nodes": ["semantic_model.test.semantic_people"]}, "refs": [], "metrics": [], "created_at": 1759423716.7306838, "group": null}, "metric.test.customers": {"name": "customers", "resource_type": "metric", "package_name": "test", "path": "schema.yml", "original_file_path": "models/schema.yml", "unique_id": "metric.test.customers", "fqn": ["test", "customers"], "description": "", "label": "Customers Metric", "type": "simple", "type_params": {"measure": {"name": "customers", "filter": null, "alias": null, "join_to_timespine": false, "fill_nulls_with": null}, "input_measures": [{"name": "customers", "filter": null, "alias": null, "join_to_timespine": false, "fill_nulls_with": null}], "numerator": null, "denominator": null, "expr": null, "window": null, "grain_to_date": null, "metrics": [], "conversion_type_params": null, "cumulative_type_params": null, "metric_aggregation_params": null}, "filter": null, "metadata": null, "time_granularity": null, "meta": {}, "tags": [], "config": {"enabled": true, "group": null, "meta": {}}, "unrendered_config": {}, "sources": [], "depends_on": {"macros": [], "nodes": ["semantic_model.test.semantic_people"]}, "refs": [], "metrics": [], "created_at": 1759423716.734397, "group": null}, "metric.test.ratio_of_blue_customers_to_red_customers": {"name": "ratio_of_blue_customers_to_red_customers", "resource_type": "metric", "package_name": "test", "path": "schema.yml", "original_file_path": "models/schema.yml", "unique_id": "metric.test.ratio_of_blue_customers_to_red_customers", "fqn": ["test", "ratio_of_blue_customers_to_red_customers"], "description": "", "label": "Very Important Customer Color Ratio", "type": "ratio", "type_params": {"measure": null, "input_measures": [{"name": "customers", "filter": null, "alias": null, "join_to_timespine": false, "fill_nulls_with": null}], "numerator": {"name": "customers", "filter": {"where_filters": [{"where_sql_template": "{{ Dimension('id__favorite_color')}} = 'blue'"}]}, "alias": null, "offset_window": null, "offset_to_grain": null}, "denominator": {"name": "customers", "filter": {"where_filters": [{"where_sql_template": "{{ Dimension('id__favorite_color')}} = 'red'"}]}, "alias": null, "offset_window": null, "offset_to_grain": null}, "expr": null, "window": null, "grain_to_date": null, "metrics": [], "conversion_type_params": null, "cumulative_type_params": null, "metric_aggregation_params": null}, "filter": null, "metadata": null, "time_granularity": null, "meta": {}, "tags": [], "config": {"enabled": true, "group": null, "meta": {}}, "unrendered_config": {}, "sources": [], "depends_on": {"macros": [], "nodes": ["metric.test.customers"]}, "refs": [], "metrics": [], "created_at": 1759423716.748488, "group": null}, "metric.test.doubled_blue_customers": {"name": "doubled_blue_customers", "resource_type": "metric", "package_name": "test", "path": "schema.yml", "original_file_path": "models/schema.yml", "unique_id": "metric.test.doubled_blue_customers", "fqn": ["test", "doubled_blue_customers"], "description": "", "label": "Inflated blue customer numbers", "type": "derived", "type_params": {"measure": null, "input_measures": [{"name": "customers", "filter": null, "alias": null, "join_to_timespine": false, "fill_nulls_with": null}], "numerator": null, "denominator": null, "expr": "customers * 2", "window": null, "grain_to_date": null, "metrics": [{"name": "customers", "filter": {"where_filters": [{"where_sql_template": "{{ Dimension('id__favorite_color')}} = 'blue'"}]}, "alias": null, "offset_window": null, "offset_to_grain": null}], "conversion_type_params": null, "cumulative_type_params": null, "metric_aggregation_params": null}, "filter": null, "metadata": null, "time_granularity": null, "meta": {}, "tags": [], "config": {"enabled": true, "group": null, "meta": {}}, "unrendered_config": {}, "sources": [], "depends_on": {"macros": [], "nodes": ["metric.test.customers"]}, "refs": [], "metrics": [], "created_at": 1759423716.753501, "group": null}}, "groups": {}, "selectors": {}, "disabled": {"model.test.disabled_model": [{"database": "dbt", "schema": "test17594237163467600387_test_previous_version_state", "name": "disabled_model", "resource_type": "model", "package_name": "test", "path": "disabled_model.sql", "original_file_path": "models/disabled_model.sql", "unique_id": "model.test.disabled_model", "fqn": ["test", "disabled_model"], "alias": "disabled_model", "checksum": {"name": "sha256", "checksum": "597106d23ce34e3cd2430588e5c1cf474ebdd138fc47e09b925a4ab258a27acc"}, "config": {"enabled": false, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "batch_size": null, "lookback": 1, "begin": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "event_time": null, "concurrent_batches": null, "access": "protected", "freshness": null}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {"enabled": false}, "created_at": 1759423716.096579, "relation_name": "\"dbt\".\"test17594237163467600387_test_previous_version_state\".\"disabled_model\"", "raw_code": "{{ config(enabled=False) }}\nselect 2 as id", "doc_blocks": [], "language": "sql", "refs": [], "sources": [], "metrics": [], "functions": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null, "primary_key": [], "time_spine": null}], "snapshot.test.disabled_snapshot_seed": [{"database": "dbt", "schema": "test17594237163467600387_test_previous_version_state", "name": "disabled_snapshot_seed", "resource_type": "snapshot", "package_name": "test", "path": "disabled_snapshot_seed.sql", "original_file_path": "snapshots/disabled_snapshot_seed.sql", "unique_id": "snapshot.test.disabled_snapshot_seed", "fqn": ["test", "disabled_snapshot_seed", "disabled_snapshot_seed"], "alias": "disabled_snapshot_seed", "checksum": {"name": "sha256", "checksum": "fe76c9dd437341c9e82a0f2a8baf3148f961b768eaa0a4410cd27d3c071bd617"}, "config": {"enabled": false, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "snapshot", "incremental_strategy": null, "batch_size": null, "lookback": 1, "begin": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": "id", "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "event_time": null, "concurrent_batches": null, "strategy": "check", "target_schema": "test17594237163467600387_test_previous_version_state", "target_database": null, "updated_at": null, "check_cols": "all", "snapshot_meta_column_names": {"dbt_valid_to": null, "dbt_valid_from": null, "dbt_scd_id": null, "dbt_updated_at": null, "dbt_is_deleted": null}, "dbt_valid_to_current": null}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {"unique_key": "id", "strategy": "check", "check_cols": "all", "target_schema": "test17594237163467600387_test_previous_version_state", "enabled": false}, "created_at": 1759423716.123988, "relation_name": "\"dbt\".\"test17594237163467600387_test_previous_version_state\".\"disabled_snapshot_seed\"", "raw_code": "\n{{\n config(\n unique_key='id',\n strategy='check',\n check_cols='all',\n target_schema=schema,\n enabled=False,\n )\n}}\nselect * from {{ ref('my_seed') }}\n", "doc_blocks": [], "language": "sql", "refs": [{"name": "my_seed", "package": null, "version": null}], "sources": [], "metrics": [], "functions": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}}], "analysis.test.disabled_al": [{"database": "dbt", "schema": "test17594237163467600387_test_previous_version_state", "name": "disabled_al", "resource_type": "analysis", "package_name": "test", "path": "analysis/disabled_al.sql", "original_file_path": "analyses/disabled_al.sql", "unique_id": "analysis.test.disabled_al", "fqn": ["test", "analysis", "disabled_al"], "alias": "disabled_al", "checksum": {"name": "sha256", "checksum": "32d36ad6cff0786eb562440ba60ef6c9b9a7f4c282dfb7a52eaf19d36370f0e1"}, "config": {"enabled": false, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "batch_size": null, "lookback": 1, "begin": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "event_time": null, "concurrent_batches": null}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {"enabled": false}, "created_at": 1759423716.143224, "relation_name": null, "raw_code": "{{ config(enabled=False) }}\nselect 9 as id", "doc_blocks": [], "language": "sql", "refs": [], "sources": [], "metrics": [], "functions": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}}], "test.test.disabled_just_my": [{"database": "dbt", "schema": "test17594237163467600387_test_previous_version_state_dbt_test__audit", "name": "disabled_just_my", "resource_type": "test", "package_name": "test", "path": "disabled_just_my.sql", "original_file_path": "tests/disabled_just_my.sql", "unique_id": "test.test.disabled_just_my", "fqn": ["test", "disabled_just_my"], "alias": "disabled_just_my", "checksum": {"name": "sha256", "checksum": "4f2268fd89a3b4ef899264ada6d7aa33603671cbc5d5acead7dc2eadf1add985"}, "config": {"enabled": false, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {"enabled": false}, "created_at": 1759423716.270129, "relation_name": null, "raw_code": "{{ config(enabled=False) }}\n\nselect * from {{ ref('my_model') }}\nwhere false", "doc_blocks": [], "language": "sql", "refs": [{"name": "my_model", "package": null, "version": null}], "sources": [], "metrics": [], "functions": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}}], "test.test.disabled_check_nothing_my_model_.f2c6a72d37": [{"database": "dbt", "schema": "test17594237163467600387_test_previous_version_state_dbt_test__audit", "name": "disabled_check_nothing_my_model_", "resource_type": "test", "package_name": "test", "path": "disabled_check_nothing_my_model_.sql", "original_file_path": "models/schema.yml", "unique_id": "test.test.disabled_check_nothing_my_model_.f2c6a72d37", "fqn": ["test", "disabled_check_nothing_my_model_"], "alias": "disabled_check_nothing_my_model_", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": false, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {"enabled": false}, "created_at": 1759423716.4264681, "relation_name": null, "raw_code": "{{ test_disabled_check_nothing(**_dbt_generic_test_kwargs) }}", "doc_blocks": [], "language": "sql", "refs": [{"name": "my_model", "package": null, "version": null}], "sources": [], "metrics": [], "functions": [], "depends_on": {"macros": ["macro.test.test_disabled_check_nothing", "macro.dbt.get_where_subquery"], "nodes": []}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": null, "file_key_name": "models.my_model", "attached_node": "model.test.my_model", "test_metadata": {"name": "disabled_check_nothing", "kwargs": {"model": "{{ get_where_subquery(ref('my_model')) }}"}, "namespace": null}}], "exposure.test.disabled_exposure": [{"name": "disabled_exposure", "resource_type": "exposure", "package_name": "test", "path": "schema.yml", "original_file_path": "models/schema.yml", "unique_id": "exposure.test.disabled_exposure", "fqn": ["test", "disabled_exposure"], "type": "dashboard", "owner": {"email": "something@example.com", "name": null}, "description": "", "label": null, "maturity": null, "meta": {}, "tags": [], "config": {"enabled": false, "tags": [], "meta": {}}, "unrendered_config": {"enabled": false}, "url": null, "depends_on": {"macros": [], "nodes": []}, "refs": [{"name": "my_model", "package": null, "version": null}], "sources": [], "metrics": [], "created_at": 1759423716.459041}], "metric.test.disabled_metric": [{"name": "disabled_metric", "resource_type": "metric", "package_name": "test", "path": "schema.yml", "original_file_path": "models/schema.yml", "unique_id": "metric.test.disabled_metric", "fqn": ["test", "disabled_metric"], "description": "", "label": "Count records", "type": "simple", "type_params": {"measure": {"name": "customers", "filter": null, "alias": null, "join_to_timespine": false, "fill_nulls_with": null}, "input_measures": [], "numerator": null, "denominator": null, "expr": null, "window": null, "grain_to_date": null, "metrics": [], "conversion_type_params": null, "cumulative_type_params": null, "metric_aggregation_params": null}, "filter": {"where_filters": [{"where_sql_template": "{{ Dimension('id__favorite_color') }} = 'blue'"}]}, "metadata": null, "time_granularity": null, "meta": {}, "tags": [], "config": {"enabled": false, "group": null, "meta": {}}, "unrendered_config": {"enabled": false}, "sources": [], "depends_on": {"macros": [], "nodes": []}, "refs": [], "metrics": [], "created_at": 1759423716.738817, "group": null}], "seed.test.disabled_seed": [{"database": "dbt", "schema": "test17594237163467600387_test_previous_version_state", "name": "disabled_seed", "resource_type": "seed", "package_name": "test", "path": "disabled_seed.csv", "original_file_path": "seeds/disabled_seed.csv", "unique_id": "seed.test.disabled_seed", "fqn": ["test", "disabled_seed"], "alias": "disabled_seed", "checksum": {"name": "sha256", "checksum": "31fddd8ec40c6aba6a3a8e7d83fedea2fd0a56c47b64ea3df1847ec1b018e2d1"}, "config": {"enabled": false, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "seed", "incremental_strategy": null, "batch_size": null, "lookback": 1, "begin": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "event_time": null, "concurrent_batches": null, "delimiter": ",", "quote_columns": null}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": "test://models/schema.yml", "build_path": null, "unrendered_config": {"enabled": false}, "created_at": 1759423716.450551, "relation_name": "\"dbt\".\"test17594237163467600387_test_previous_version_state\".\"disabled_seed\"", "raw_code": "", "doc_blocks": [], "root_path": "/private/var/folders/79/5290gpvn3lx5jdryk4844rm80000gn/T/pytest-of-quigleymalcolm/pytest-154/popen-gw6/project18", "depends_on": {"macros": []}}], "source.test.my_source.disabled_table": [{"database": "dbt", "schema": "my_source", "name": "disabled_table", "resource_type": "source", "package_name": "test", "path": "models/schema.yml", "original_file_path": "models/schema.yml", "unique_id": "source.test.my_source.disabled_table", "fqn": ["test", "my_source", "disabled_table"], "source_name": "my_source", "source_description": "My source", "loader": "a_loader", "identifier": "disabled_table", "quoting": {"database": null, "schema": null, "identifier": null, "column": null}, "loaded_at_field": null, "loaded_at_query": null, "freshness": {"warn_after": {"count": null, "period": null}, "error_after": {"count": null, "period": null}, "filter": null}, "external": null, "description": "Disabled table", "columns": {}, "meta": {}, "source_meta": {}, "tags": [], "config": {"enabled": false, "event_time": null, "freshness": {"warn_after": {"count": null, "period": null}, "error_after": {"count": null, "period": null}, "filter": null}, "loaded_at_field": null, "loaded_at_query": null, "meta": {}, "tags": []}, "patch_path": null, "unrendered_config": {"enabled": false, "loaded_at_field": null, "loaded_at_query": null, "meta": {}, "tags": []}, "relation_name": "\"dbt\".\"my_source\".\"disabled_table\"", "created_at": 1759423716.977084, "unrendered_database": null, "unrendered_schema": null, "doc_blocks": []}]}, "parent_map": {"model.test.my_model": [], "model.test.metricflow_time_spine": [], "snapshot.test.snapshot_seed": ["seed.test.my_seed"], "analysis.test.a": [], "test.test.just_my": ["model.test.my_model"], "seed.test.my_seed": [], "test.test.not_null_my_model_id.43e0e9183a": ["model.test.my_model"], "test.test.check_nothing_my_model_.d5a5e66110": ["model.test.my_model"], "source.test.my_source.my_table": [], "exposure.test.simple_exposure": ["model.test.my_model", "source.test.my_source.my_table"], "metric.test.blue_customers_post_2010": ["semantic_model.test.semantic_people"], "metric.test.customers": ["semantic_model.test.semantic_people"], "metric.test.ratio_of_blue_customers_to_red_customers": ["metric.test.customers"], "metric.test.doubled_blue_customers": ["metric.test.customers"], "semantic_model.test.semantic_people": ["model.test.my_model"]}, "child_map": {"model.test.my_model": ["exposure.test.simple_exposure", "semantic_model.test.semantic_people", "test.test.check_nothing_my_model_.d5a5e66110", "test.test.just_my", "test.test.not_null_my_model_id.43e0e9183a"], "model.test.metricflow_time_spine": [], "snapshot.test.snapshot_seed": [], "analysis.test.a": [], "test.test.just_my": [], "seed.test.my_seed": ["snapshot.test.snapshot_seed"], "test.test.not_null_my_model_id.43e0e9183a": [], "test.test.check_nothing_my_model_.d5a5e66110": [], "source.test.my_source.my_table": ["exposure.test.simple_exposure"], "exposure.test.simple_exposure": [], "metric.test.blue_customers_post_2010": [], "metric.test.customers": ["metric.test.doubled_blue_customers", "metric.test.ratio_of_blue_customers_to_red_customers"], "metric.test.ratio_of_blue_customers_to_red_customers": [], "metric.test.doubled_blue_customers": [], "semantic_model.test.semantic_people": ["metric.test.blue_customers_post_2010", "metric.test.customers"]}, "group_map": {}, "saved_queries": {}, "semantic_models": {"semantic_model.test.semantic_people": {"name": "semantic_people", "resource_type": "semantic_model", "package_name": "test", "path": "schema.yml", "original_file_path": "models/schema.yml", "unique_id": "semantic_model.test.semantic_people", "fqn": ["test", "semantic_people"], "model": "ref('my_model')", "node_relation": {"alias": "my_model", "schema_name": "test17594237163467600387_test_previous_version_state", "database": "dbt", "relation_name": "\"dbt\".\"test17594237163467600387_test_previous_version_state\".\"my_model\""}, "description": null, "label": null, "defaults": {"agg_time_dimension": "created_at"}, "entities": [{"name": "id", "type": "primary", "description": null, "label": null, "role": null, "expr": null, "config": {"meta": {}}}], "measures": [{"name": "years_tenure", "agg": "sum", "description": null, "label": null, "create_metric": false, "expr": "tenure", "agg_params": null, "non_additive_dimension": null, "agg_time_dimension": null, "config": {"meta": {}}}, {"name": "people", "agg": "count", "description": null, "label": null, "create_metric": false, "expr": "id", "agg_params": null, "non_additive_dimension": null, "agg_time_dimension": null, "config": {"meta": {}}}, {"name": "customers", "agg": "count", "description": null, "label": null, "create_metric": false, "expr": "id", "agg_params": null, "non_additive_dimension": null, "agg_time_dimension": null, "config": {"meta": {}}}], "dimensions": [{"name": "favorite_color", "type": "categorical", "description": null, "label": null, "is_partition": false, "type_params": null, "expr": null, "metadata": null, "config": {"meta": {}}}, {"name": "created_at", "type": "time", "description": null, "label": null, "is_partition": false, "type_params": {"time_granularity": "day", "validity_params": null}, "expr": null, "metadata": null, "config": {"meta": {}}}], "metadata": null, "depends_on": {"macros": [], "nodes": ["model.test.my_model"]}, "refs": [{"name": "my_model", "package": null, "version": null}], "created_at": 1759423716.958543, "config": {"enabled": true, "group": null, "meta": {}}, "unrendered_config": {}, "primary_entity": null, "group": null}}, "unit_tests": {}, "functions": {}} ================================================ FILE: tests/functional/artifacts/data/state/v2/manifest.json ================================================ {"metadata": {"dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v2.json", "dbt_version": "0.20.2", "generated_at": "2022-06-08T05:12:43.870174Z", "invocation_id": "b9b21a26-1804-47f9-866b-620501fe5540", "env": {}, "project_id": "098f6bcd4621d373cade4e832627b4f6", "user_id": null, "send_anonymous_usage_stats": false, "adapter_type": "postgres"}, "nodes": {"model.test.my_model": {"raw_sql": "select 1 as id", "resource_type": "model", "depends_on": {"macros": [], "nodes": []}, "config": {"enabled": true, "materialized": "view", "persist_docs": {}, "vars": {}, "quoting": {}, "column_types": {}, "alias": null, "schema": null, "database": null, "tags": [], "full_refresh": null, "post-hook": [], "pre-hook": []}, "database": "jerco", "schema": "dbt_jcohen", "fqn": ["test", "my_model"], "unique_id": "model.test.my_model", "package_name": "test", "root_path": "/Users/jerco/dev/scratch/testy", "path": "my_model.sql", "original_file_path": "models/my_model.sql", "name": "my_model", "alias": "my_model", "checksum": {"name": "sha256", "checksum": "479636cb85ce8d3b0f8db5ff13cf338b61254ad98d905630eac61f963e719e9d"}, "tags": [], "refs": [], "sources": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true}, "patch_path": null, "compiled_path": null, "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1654665164}}, "sources": {}, "macros": {"macro.test.drop_relation": {"unique_id": "macro.test.drop_relation", "package_name": "test", "root_path": "/Users/jerco/dev/scratch/testy", "path": "macros/whatever.sql", "original_file_path": "macros/whatever.sql", "name": "drop_relation", "macro_sql": "{% macro drop_relation(relation) -%}\n {{ return(dbt_labs_materialized_views.drop_relation(relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.test.drop_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.test.postgres__list_relations_without_caching": {"unique_id": "macro.test.postgres__list_relations_without_caching", "package_name": "test", "root_path": "/Users/jerco/dev/scratch/testy", "path": "macros/whatever.sql", "original_file_path": "macros/whatever.sql", "name": "postgres__list_relations_without_caching", "macro_sql": "{% macro postgres__list_relations_without_caching(schema_relation) %}\n {{ return(dbt_labs_materialized_views.postgres__list_relations_without_caching(schema_relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.test.postgres__list_relations_without_caching"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.test.postgres_get_relations": {"unique_id": "macro.test.postgres_get_relations", "package_name": "test", "root_path": "/Users/jerco/dev/scratch/testy", "path": "macros/whatever.sql", "original_file_path": "macros/whatever.sql", "name": "postgres_get_relations", "macro_sql": "{% macro postgres_get_relations() %}\n {{ return(dbt_labs_materialized_views.postgres_get_relations()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.test.postgres_get_relations"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.test.redshift__list_relations_without_caching": {"unique_id": "macro.test.redshift__list_relations_without_caching", "package_name": "test", "root_path": "/Users/jerco/dev/scratch/testy", "path": "macros/whatever.sql", "original_file_path": "macros/whatever.sql", "name": "redshift__list_relations_without_caching", "macro_sql": "{% macro redshift__list_relations_without_caching(schema_relation) %}\n {{ return(dbt_labs_materialized_views.redshift__list_relations_without_caching(schema_relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.test.redshift__list_relations_without_caching"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.test.load_relation": {"unique_id": "macro.test.load_relation", "package_name": "test", "root_path": "/Users/jerco/dev/scratch/testy", "path": "macros/whatever.sql", "original_file_path": "macros/whatever.sql", "name": "load_relation", "macro_sql": "{% macro load_relation(relation) %}\n {{ return(dbt_labs_materialized_views.redshift_load_relation_or_mv(relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt_postgres.postgres__get_catalog": {"unique_id": "macro.dbt_postgres.postgres__get_catalog", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/catalog.sql", "original_file_path": "macros/catalog.sql", "name": "postgres__get_catalog", "macro_sql": "{% macro postgres__get_catalog(information_schema, schemas) -%}\n\n {%- call statement('catalog', fetch_result=True) -%}\n {#\n If the user has multiple databases set and the first one is wrong, this will fail.\n But we won't fail in the case where there are multiple quoting-difference-only dbs, which is better.\n #}\n {% set database = information_schema.database %}\n {{ adapter.verify_database(database) }}\n\n select\n '{{ database }}' as table_database,\n sch.nspname as table_schema,\n tbl.relname as table_name,\n case tbl.relkind\n when 'v' then 'VIEW'\n else 'BASE TABLE'\n end as table_type,\n tbl_desc.description as table_comment,\n col.attname as column_name,\n col.attnum as column_index,\n pg_catalog.format_type(col.atttypid, col.atttypmod) as column_type,\n col_desc.description as column_comment,\n pg_get_userbyid(tbl.relowner) as table_owner\n\n from pg_catalog.pg_namespace sch\n join pg_catalog.pg_class tbl on tbl.relnamespace = sch.oid\n join pg_catalog.pg_attribute col on col.attrelid = tbl.oid\n left outer join pg_catalog.pg_description tbl_desc on (tbl_desc.objoid = tbl.oid and tbl_desc.objsubid = 0)\n left outer join pg_catalog.pg_description col_desc on (col_desc.objoid = tbl.oid and col_desc.objsubid = col.attnum)\n\n where (\n {%- for schema in schemas -%}\n upper(sch.nspname) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n )\n and not pg_is_other_temp_schema(sch.oid) -- not a temporary schema belonging to another session\n and tbl.relpersistence = 'p' -- [p]ermanent table. Other values are [u]nlogged table, [t]emporary table\n and tbl.relkind in ('r', 'v', 'f', 'p') -- o[r]dinary table, [v]iew, [f]oreign table, [p]artitioned table. Other values are [i]ndex, [S]equence, [c]omposite type, [t]OAST table, [m]aterialized view\n and col.attnum > 0 -- negative numbers are used for system columns such as oid\n and not col.attisdropped -- column as not been dropped\n\n order by\n sch.nspname,\n tbl.relname,\n col.attnum\n\n {%- endcall -%}\n\n {{ return(load_result('catalog').table) }}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt_postgres.postgres_get_relations": {"unique_id": "macro.dbt_postgres.postgres_get_relations", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/relations.sql", "original_file_path": "macros/relations.sql", "name": "postgres_get_relations", "macro_sql": "{% macro postgres_get_relations () -%}\n\n {#\n -- in pg_depend, objid is the dependent, refobjid is the referenced object\n -- > a pg_depend entry indicates that the referenced object cannot be\n -- > dropped without also dropping the dependent object.\n #}\n\n {%- call statement('relations', fetch_result=True) -%}\n with relation as (\n select\n pg_rewrite.ev_class as class,\n pg_rewrite.oid as id\n from pg_rewrite\n ),\n class as (\n select\n oid as id,\n relname as name,\n relnamespace as schema,\n relkind as kind\n from pg_class\n ),\n dependency as (\n select\n pg_depend.objid as id,\n pg_depend.refobjid as ref\n from pg_depend\n ),\n schema as (\n select\n pg_namespace.oid as id,\n pg_namespace.nspname as name\n from pg_namespace\n where nspname != 'information_schema' and nspname not like 'pg\\_%'\n ),\n referenced as (\n select\n relation.id AS id,\n referenced_class.name ,\n referenced_class.schema ,\n referenced_class.kind\n from relation\n join class as referenced_class on relation.class=referenced_class.id\n where referenced_class.kind in ('r', 'v')\n ),\n relationships as (\n select\n referenced.name as referenced_name,\n referenced.schema as referenced_schema_id,\n dependent_class.name as dependent_name,\n dependent_class.schema as dependent_schema_id,\n referenced.kind as kind\n from referenced\n join dependency on referenced.id=dependency.id\n join class as dependent_class on dependency.ref=dependent_class.id\n where\n (referenced.name != dependent_class.name or\n referenced.schema != dependent_class.schema)\n )\n\n select\n referenced_schema.name as referenced_schema,\n relationships.referenced_name as referenced_name,\n dependent_schema.name as dependent_schema,\n relationships.dependent_name as dependent_name\n from relationships\n join schema as dependent_schema on relationships.dependent_schema_id=dependent_schema.id\n join schema as referenced_schema on relationships.referenced_schema_id=referenced_schema.id\n group by referenced_schema, referenced_name, dependent_schema, dependent_name\n order by referenced_schema, referenced_name, dependent_schema, dependent_name;\n\n {%- endcall -%}\n\n {{ return(load_result('relations').table) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt_postgres.postgres__create_table_as": {"unique_id": "macro.dbt_postgres.postgres__create_table_as", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__create_table_as", "macro_sql": "{% macro postgres__create_table_as(temporary, relation, sql) -%}\n {%- set unlogged = config.get('unlogged', default=false) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary -%}\n temporary\n {%- elif unlogged -%}\n unlogged\n {%- endif %} table {{ relation }}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt_postgres.postgres__get_create_index_sql": {"unique_id": "macro.dbt_postgres.postgres__get_create_index_sql", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__get_create_index_sql", "macro_sql": "{% macro postgres__get_create_index_sql(relation, index_dict) -%}\n {%- set index_config = adapter.parse_index(index_dict) -%}\n {%- set comma_separated_columns = \", \".join(index_config.columns) -%}\n {%- set index_name = index_config.render(relation) -%}\n\n create {% if index_config.unique -%}\n unique\n {%- endif %} index if not exists\n \"{{ index_name }}\"\n on {{ relation }} {% if index_config.type -%}\n using {{ index_config.type }}\n {%- endif %}\n ({{ comma_separated_columns }});\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt_postgres.postgres__create_schema": {"unique_id": "macro.dbt_postgres.postgres__create_schema", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__create_schema", "macro_sql": "{% macro postgres__create_schema(relation) -%}\n {% if relation.database -%}\n {{ adapter.verify_database(relation.database) }}\n {%- endif -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier().include(database=False) }}\n {%- endcall -%}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt_postgres.postgres__drop_schema": {"unique_id": "macro.dbt_postgres.postgres__drop_schema", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__drop_schema", "macro_sql": "{% macro postgres__drop_schema(relation) -%}\n {% if relation.database -%}\n {{ adapter.verify_database(relation.database) }}\n {%- endif -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier().include(database=False) }} cascade\n {%- endcall -%}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt_postgres.postgres__get_columns_in_relation": {"unique_id": "macro.dbt_postgres.postgres__get_columns_in_relation", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__get_columns_in_relation", "macro_sql": "{% macro postgres__get_columns_in_relation(relation) -%}\n {% call statement('get_columns_in_relation', fetch_result=True) %}\n select\n column_name,\n data_type,\n character_maximum_length,\n numeric_precision,\n numeric_scale\n\n from {{ relation.information_schema('columns') }}\n where table_name = '{{ relation.identifier }}'\n {% if relation.schema %}\n and table_schema = '{{ relation.schema }}'\n {% endif %}\n order by ordinal_position\n\n {% endcall %}\n {% set table = load_result('get_columns_in_relation').table %}\n {{ return(sql_convert_columns_in_relation(table)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.sql_convert_columns_in_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt_postgres.postgres__list_relations_without_caching": {"unique_id": "macro.dbt_postgres.postgres__list_relations_without_caching", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__list_relations_without_caching", "macro_sql": "{% macro postgres__list_relations_without_caching(schema_relation) %}\n {% call statement('list_relations_without_caching', fetch_result=True) -%}\n select\n '{{ schema_relation.database }}' as database,\n tablename as name,\n schemaname as schema,\n 'table' as type\n from pg_tables\n where schemaname ilike '{{ schema_relation.schema }}'\n union all\n select\n '{{ schema_relation.database }}' as database,\n viewname as name,\n schemaname as schema,\n 'view' as type\n from pg_views\n where schemaname ilike '{{ schema_relation.schema }}'\n {% endcall %}\n {{ return(load_result('list_relations_without_caching').table) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt_postgres.postgres__information_schema_name": {"unique_id": "macro.dbt_postgres.postgres__information_schema_name", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__information_schema_name", "macro_sql": "{% macro postgres__information_schema_name(database) -%}\n {% if database_name -%}\n {{ adapter.verify_database(database_name) }}\n {%- endif -%}\n information_schema\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt_postgres.postgres__list_schemas": {"unique_id": "macro.dbt_postgres.postgres__list_schemas", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__list_schemas", "macro_sql": "{% macro postgres__list_schemas(database) %}\n {% if database -%}\n {{ adapter.verify_database(database) }}\n {%- endif -%}\n {% call statement('list_schemas', fetch_result=True, auto_begin=False) %}\n select distinct nspname from pg_namespace\n {% endcall %}\n {{ return(load_result('list_schemas').table) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt_postgres.postgres__check_schema_exists": {"unique_id": "macro.dbt_postgres.postgres__check_schema_exists", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__check_schema_exists", "macro_sql": "{% macro postgres__check_schema_exists(information_schema, schema) -%}\n {% if information_schema.database -%}\n {{ adapter.verify_database(information_schema.database) }}\n {%- endif -%}\n {% call statement('check_schema_exists', fetch_result=True, auto_begin=False) %}\n select count(*) from pg_namespace where nspname = '{{ schema }}'\n {% endcall %}\n {{ return(load_result('check_schema_exists').table) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt_postgres.postgres__current_timestamp": {"unique_id": "macro.dbt_postgres.postgres__current_timestamp", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__current_timestamp", "macro_sql": "{% macro postgres__current_timestamp() -%}\n now()\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt_postgres.postgres__snapshot_string_as_time": {"unique_id": "macro.dbt_postgres.postgres__snapshot_string_as_time", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__snapshot_string_as_time", "macro_sql": "{% macro postgres__snapshot_string_as_time(timestamp) -%}\n {%- set result = \"'\" ~ timestamp ~ \"'::timestamp without time zone\" -%}\n {{ return(result) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt_postgres.postgres__snapshot_get_time": {"unique_id": "macro.dbt_postgres.postgres__snapshot_get_time", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__snapshot_get_time", "macro_sql": "{% macro postgres__snapshot_get_time() -%}\n {{ current_timestamp() }}::timestamp without time zone\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt_postgres.postgres__make_temp_relation": {"unique_id": "macro.dbt_postgres.postgres__make_temp_relation", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__make_temp_relation", "macro_sql": "{% macro postgres__make_temp_relation(base_relation, suffix) %}\n {% set dt = modules.datetime.datetime.now() %}\n {% set dtstring = dt.strftime(\"%H%M%S%f\") %}\n {% set suffix_length = suffix|length + dtstring|length %}\n {% set relation_max_name_length = 63 %}\n {% if suffix_length > relation_max_name_length %}\n {% do exceptions.raise_compiler_error('Temp relation suffix is too long (' ~ suffix|length ~ ' characters). Maximum length is ' ~ (relation_max_name_length - dtstring|length) ~ ' characters.') %}\n {% endif %}\n {% set tmp_identifier = base_relation.identifier[:relation_max_name_length - suffix_length] ~ suffix ~ dtstring %}\n {% do return(base_relation.incorporate(\n path={\n \"identifier\": tmp_identifier,\n \"schema\": none,\n \"database\": none\n })) -%}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt_postgres.postgres_escape_comment": {"unique_id": "macro.dbt_postgres.postgres_escape_comment", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres_escape_comment", "macro_sql": "{% macro postgres_escape_comment(comment) -%}\n {% if comment is not string %}\n {% do exceptions.raise_compiler_error('cannot escape a non-string: ' ~ comment) %}\n {% endif %}\n {%- set magic = '$dbt_comment_literal_block$' -%}\n {%- if magic in comment -%}\n {%- do exceptions.raise_compiler_error('The string ' ~ magic ~ ' is not allowed in comments.') -%}\n {%- endif -%}\n {{ magic }}{{ comment }}{{ magic }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt_postgres.postgres__alter_relation_comment": {"unique_id": "macro.dbt_postgres.postgres__alter_relation_comment", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__alter_relation_comment", "macro_sql": "{% macro postgres__alter_relation_comment(relation, comment) %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n comment on {{ relation.type }} {{ relation }} is {{ escaped_comment }};\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres_escape_comment"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt_postgres.postgres__alter_column_comment": {"unique_id": "macro.dbt_postgres.postgres__alter_column_comment", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__alter_column_comment", "macro_sql": "{% macro postgres__alter_column_comment(relation, column_dict) %}\n {% for column_name in column_dict %}\n {% set comment = column_dict[column_name]['description'] %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n comment on column {{ relation }}.{{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }} is {{ escaped_comment }};\n {% endfor %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres_escape_comment"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt_postgres.postgres__snapshot_merge_sql": {"unique_id": "macro.dbt_postgres.postgres__snapshot_merge_sql", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/materializations/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshot_merge.sql", "name": "postgres__snapshot_merge_sql", "macro_sql": "{% macro postgres__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n update {{ target }}\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n from {{ source }} as DBT_INTERNAL_SOURCE\n where DBT_INTERNAL_SOURCE.dbt_scd_id::text = {{ target }}.dbt_scd_id::text\n and DBT_INTERNAL_SOURCE.dbt_change_type::text in ('update'::text, 'delete'::text)\n and {{ target }}.dbt_valid_to is null;\n\n insert into {{ target }} ({{ insert_cols_csv }})\n select {% for column in insert_cols -%}\n DBT_INTERNAL_SOURCE.{{ column }} {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n from {{ source }} as DBT_INTERNAL_SOURCE\n where DBT_INTERNAL_SOURCE.dbt_change_type::text = 'insert'::text;\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.statement": {"unique_id": "macro.dbt.statement", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/core.sql", "original_file_path": "macros/core.sql", "name": "statement", "macro_sql": "{% macro statement(name=None, fetch_result=False, auto_begin=True) -%}\n {%- if execute: -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- set res, table = adapter.execute(sql, auto_begin=auto_begin, fetch=fetch_result) -%}\n {%- if name is not none -%}\n {{ store_result(name, response=res, agate_table=table) }}\n {%- endif -%}\n\n {%- endif -%}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.noop_statement": {"unique_id": "macro.dbt.noop_statement", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/core.sql", "original_file_path": "macros/core.sql", "name": "noop_statement", "macro_sql": "{% macro noop_statement(name=None, message=None, code=None, rows_affected=None, res=None) -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_raw_result(name, message=message, code=code, rows_affected=rows_affected, agate_table=res) }}\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.get_test_sql": {"unique_id": "macro.dbt.get_test_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/test.sql", "original_file_path": "macros/materializations/test.sql", "name": "get_test_sql", "macro_sql": "{% macro get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n {{ adapter.dispatch('get_test_sql')(main_sql, fail_calc, warn_if, error_if, limit) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.default__get_test_sql": {"unique_id": "macro.dbt.default__get_test_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/test.sql", "original_file_path": "macros/materializations/test.sql", "name": "default__get_test_sql", "macro_sql": "{% macro default__get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n select\n {{ fail_calc }} as failures,\n {{ fail_calc }} {{ warn_if }} as should_warn,\n {{ fail_calc }} {{ error_if }} as should_error\n from (\n {{ main_sql }}\n {{ \"limit \" ~ limit if limit != none }}\n ) dbt_internal_test\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.materialization_test_default": {"unique_id": "macro.dbt.materialization_test_default", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/test.sql", "original_file_path": "macros/materializations/test.sql", "name": "materialization_test_default", "macro_sql": "\n\n{%- materialization test, default -%}\n\n {% set relations = [] %}\n\n {% if should_store_failures() %}\n\n {% set identifier = model['alias'] %}\n {% set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n {% set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database, type='table') -%} %}\n \n {% if old_relation %}\n {% do adapter.drop_relation(old_relation) %}\n {% endif %}\n \n {% call statement(auto_begin=True) %}\n {{ create_table_as(False, target_relation, sql) }}\n {% endcall %}\n \n {% do relations.append(target_relation) %}\n \n {% set main_sql %}\n select *\n from {{ target_relation }}\n {% endset %}\n \n {{ adapter.commit() }}\n \n {% else %}\n\n {% set main_sql = sql %}\n \n {% endif %}\n\n {% set limit = config.get('limit') %}\n {% set fail_calc = config.get('fail_calc') %}\n {% set warn_if = config.get('warn_if') %}\n {% set error_if = config.get('error_if') %}\n\n {% call statement('main', fetch_result=True) -%}\n\n {{ get_test_sql(main_sql, fail_calc, warn_if, error_if, limit)}}\n\n {%- endcall %}\n \n {{ return({'relations': relations}) }}\n\n{%- endmaterialization -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.should_store_failures", "macro.dbt.statement", "macro.dbt.create_table_as", "macro.dbt.get_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.run_hooks": {"unique_id": "macro.dbt.run_hooks", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/helpers.sql", "original_file_path": "macros/materializations/helpers.sql", "name": "run_hooks", "macro_sql": "{% macro run_hooks(hooks, inside_transaction=True) %}\n {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %}\n {% if not inside_transaction and loop.first %}\n {% call statement(auto_begin=inside_transaction) %}\n commit;\n {% endcall %}\n {% endif %}\n {% set rendered = render(hook.get('sql')) | trim %}\n {% if (rendered | length) > 0 %}\n {% call statement(auto_begin=inside_transaction) %}\n {{ rendered }}\n {% endcall %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.column_list": {"unique_id": "macro.dbt.column_list", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/helpers.sql", "original_file_path": "macros/materializations/helpers.sql", "name": "column_list", "macro_sql": "{% macro column_list(columns) %}\n {%- for col in columns %}\n {{ col.name }} {% if not loop.last %},{% endif %}\n {% endfor -%}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.column_list_for_create_table": {"unique_id": "macro.dbt.column_list_for_create_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/helpers.sql", "original_file_path": "macros/materializations/helpers.sql", "name": "column_list_for_create_table", "macro_sql": "{% macro column_list_for_create_table(columns) %}\n {%- for col in columns %}\n {{ col.name }} {{ col.data_type }} {%- if not loop.last %},{% endif %}\n {% endfor -%}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.make_hook_config": {"unique_id": "macro.dbt.make_hook_config", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/helpers.sql", "original_file_path": "macros/materializations/helpers.sql", "name": "make_hook_config", "macro_sql": "{% macro make_hook_config(sql, inside_transaction) %}\n {{ tojson({\"sql\": sql, \"transaction\": inside_transaction}) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.before_begin": {"unique_id": "macro.dbt.before_begin", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/helpers.sql", "original_file_path": "macros/materializations/helpers.sql", "name": "before_begin", "macro_sql": "{% macro before_begin(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.in_transaction": {"unique_id": "macro.dbt.in_transaction", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/helpers.sql", "original_file_path": "macros/materializations/helpers.sql", "name": "in_transaction", "macro_sql": "{% macro in_transaction(sql) %}\n {{ make_hook_config(sql, inside_transaction=True) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.after_commit": {"unique_id": "macro.dbt.after_commit", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/helpers.sql", "original_file_path": "macros/materializations/helpers.sql", "name": "after_commit", "macro_sql": "{% macro after_commit(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.drop_relation_if_exists": {"unique_id": "macro.dbt.drop_relation_if_exists", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/helpers.sql", "original_file_path": "macros/materializations/helpers.sql", "name": "drop_relation_if_exists", "macro_sql": "{% macro drop_relation_if_exists(relation) %}\n {% if relation is not none %}\n {{ adapter.drop_relation(relation) }}\n {% endif %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.load_relation": {"unique_id": "macro.dbt.load_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/helpers.sql", "original_file_path": "macros/materializations/helpers.sql", "name": "load_relation", "macro_sql": "{% macro load_relation(relation) %}\n {% do return(adapter.get_relation(\n database=relation.database,\n schema=relation.schema,\n identifier=relation.identifier\n )) -%}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.should_full_refresh": {"unique_id": "macro.dbt.should_full_refresh", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/helpers.sql", "original_file_path": "macros/materializations/helpers.sql", "name": "should_full_refresh", "macro_sql": "{% macro should_full_refresh() %}\n {% set config_full_refresh = config.get('full_refresh') %}\n {% if config_full_refresh is none %}\n {% set config_full_refresh = flags.FULL_REFRESH %}\n {% endif %}\n {% do return(config_full_refresh) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.should_store_failures": {"unique_id": "macro.dbt.should_store_failures", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/helpers.sql", "original_file_path": "macros/materializations/helpers.sql", "name": "should_store_failures", "macro_sql": "{% macro should_store_failures() %}\n {% set config_store_failures = config.get('store_failures') %}\n {% if config_store_failures is none %}\n {% set config_store_failures = flags.STORE_FAILURES %}\n {% endif %}\n {% do return(config_store_failures) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.snapshot_merge_sql": {"unique_id": "macro.dbt.snapshot_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshot/snapshot_merge.sql", "name": "snapshot_merge_sql", "macro_sql": "{% macro snapshot_merge_sql(target, source, insert_cols) -%}\n {{ adapter.dispatch('snapshot_merge_sql')(target, source, insert_cols) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__snapshot_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.default__snapshot_merge_sql": {"unique_id": "macro.dbt.default__snapshot_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshot/snapshot_merge.sql", "name": "default__snapshot_merge_sql", "macro_sql": "{% macro default__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on DBT_INTERNAL_SOURCE.dbt_scd_id = DBT_INTERNAL_DEST.dbt_scd_id\n\n when matched\n and DBT_INTERNAL_DEST.dbt_valid_to is null\n and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')\n then update\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n\n when not matched\n and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'\n then insert ({{ insert_cols_csv }})\n values ({{ insert_cols_csv }})\n ;\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.strategy_dispatch": {"unique_id": "macro.dbt.strategy_dispatch", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/strategies.sql", "original_file_path": "macros/materializations/snapshot/strategies.sql", "name": "strategy_dispatch", "macro_sql": "{% macro strategy_dispatch(name) -%}\n{% set original_name = name %}\n {% if '.' in name %}\n {% set package_name, name = name.split(\".\", 1) %}\n {% else %}\n {% set package_name = none %}\n {% endif %}\n\n {% if package_name is none %}\n {% set package_context = context %}\n {% elif package_name in context %}\n {% set package_context = context[package_name] %}\n {% else %}\n {% set error_msg %}\n Could not find package '{{package_name}}', called with '{{original_name}}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n\n {%- set search_name = 'snapshot_' ~ name ~ '_strategy' -%}\n\n {% if search_name not in package_context %}\n {% set error_msg %}\n The specified strategy macro '{{name}}' was not found in package '{{ package_name }}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n {{ return(package_context[search_name]) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.snapshot_hash_arguments": {"unique_id": "macro.dbt.snapshot_hash_arguments", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/strategies.sql", "original_file_path": "macros/materializations/snapshot/strategies.sql", "name": "snapshot_hash_arguments", "macro_sql": "{% macro snapshot_hash_arguments(args) -%}\n {{ adapter.dispatch('snapshot_hash_arguments')(args) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.default__snapshot_hash_arguments": {"unique_id": "macro.dbt.default__snapshot_hash_arguments", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/strategies.sql", "original_file_path": "macros/materializations/snapshot/strategies.sql", "name": "default__snapshot_hash_arguments", "macro_sql": "{% macro default__snapshot_hash_arguments(args) -%}\n md5({%- for arg in args -%}\n coalesce(cast({{ arg }} as varchar ), '')\n {% if not loop.last %} || '|' || {% endif %}\n {%- endfor -%})\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.snapshot_get_time": {"unique_id": "macro.dbt.snapshot_get_time", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/strategies.sql", "original_file_path": "macros/materializations/snapshot/strategies.sql", "name": "snapshot_get_time", "macro_sql": "{% macro snapshot_get_time() -%}\n {{ adapter.dispatch('snapshot_get_time')() }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__snapshot_get_time"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.default__snapshot_get_time": {"unique_id": "macro.dbt.default__snapshot_get_time", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/strategies.sql", "original_file_path": "macros/materializations/snapshot/strategies.sql", "name": "default__snapshot_get_time", "macro_sql": "{% macro default__snapshot_get_time() -%}\n {{ current_timestamp() }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.snapshot_timestamp_strategy": {"unique_id": "macro.dbt.snapshot_timestamp_strategy", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/strategies.sql", "original_file_path": "macros/materializations/snapshot/strategies.sql", "name": "snapshot_timestamp_strategy", "macro_sql": "{% macro snapshot_timestamp_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}\n {% set primary_key = config['unique_key'] %}\n {% set updated_at = config['updated_at'] %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %}\n\n {#/*\n The snapshot relation might not have an {{ updated_at }} value if the\n snapshot strategy is changed from `check` to `timestamp`. We\n should use a dbt-created column for the comparison in the snapshot\n table instead of assuming that the user-supplied {{ updated_at }}\n will be present in the historical data.\n\n See https://github.com/fishtown-analytics/dbt/issues/2350\n */ #}\n {% set row_changed_expr -%}\n ({{ snapshotted_rel }}.dbt_valid_from < {{ current_rel }}.{{ updated_at }})\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.snapshot_string_as_time": {"unique_id": "macro.dbt.snapshot_string_as_time", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/strategies.sql", "original_file_path": "macros/materializations/snapshot/strategies.sql", "name": "snapshot_string_as_time", "macro_sql": "{% macro snapshot_string_as_time(timestamp) -%}\n {{ adapter.dispatch('snapshot_string_as_time')(timestamp) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__snapshot_string_as_time"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.default__snapshot_string_as_time": {"unique_id": "macro.dbt.default__snapshot_string_as_time", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/strategies.sql", "original_file_path": "macros/materializations/snapshot/strategies.sql", "name": "default__snapshot_string_as_time", "macro_sql": "{% macro default__snapshot_string_as_time(timestamp) %}\n {% do exceptions.raise_not_implemented(\n 'snapshot_string_as_time macro not implemented for adapter '+adapter.type()\n ) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.snapshot_check_all_get_existing_columns": {"unique_id": "macro.dbt.snapshot_check_all_get_existing_columns", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/strategies.sql", "original_file_path": "macros/materializations/snapshot/strategies.sql", "name": "snapshot_check_all_get_existing_columns", "macro_sql": "{% macro snapshot_check_all_get_existing_columns(node, target_exists) -%}\n {%- set query_columns = get_columns_in_query(node['compiled_sql']) -%}\n {%- if not target_exists -%}\n {# no table yet -> return whatever the query does #}\n {{ return([false, query_columns]) }}\n {%- endif -%}\n {# handle any schema changes #}\n {%- set target_table = node.get('alias', node.get('name')) -%}\n {%- set target_relation = adapter.get_relation(database=node.database, schema=node.schema, identifier=target_table) -%}\n {%- set existing_cols = get_columns_in_query('select * from ' ~ target_relation) -%}\n {%- set ns = namespace() -%} {# handle for-loop scoping with a namespace #}\n {%- set ns.column_added = false -%}\n\n {%- set intersection = [] -%}\n {%- for col in query_columns -%}\n {%- if col in existing_cols -%}\n {%- do intersection.append(col) -%}\n {%- else -%}\n {% set ns.column_added = true %}\n {%- endif -%}\n {%- endfor -%}\n {{ return([ns.column_added, intersection]) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_columns_in_query"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.snapshot_check_strategy": {"unique_id": "macro.dbt.snapshot_check_strategy", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/strategies.sql", "original_file_path": "macros/materializations/snapshot/strategies.sql", "name": "snapshot_check_strategy", "macro_sql": "{% macro snapshot_check_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}\n {% set check_cols_config = config['check_cols'] %}\n {% set primary_key = config['unique_key'] %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %}\n \n {% set select_current_time -%}\n select {{ snapshot_get_time() }} as snapshot_start\n {%- endset %}\n\n {#-- don't access the column by name, to avoid dealing with casing issues on snowflake #}\n {%- set now = run_query(select_current_time)[0][0] -%}\n {% if now is none or now is undefined -%}\n {%- do exceptions.raise_compiler_error('Could not get a snapshot start time from the database') -%}\n {%- endif %}\n {% set updated_at = config.get('updated_at', snapshot_string_as_time(now)) %}\n\n {% set column_added = false %}\n\n {% if check_cols_config == 'all' %}\n {% set column_added, check_cols = snapshot_check_all_get_existing_columns(node, target_exists) %}\n {% elif check_cols_config is iterable and (check_cols_config | length) > 0 %}\n {% set check_cols = check_cols_config %}\n {% else %}\n {% do exceptions.raise_compiler_error(\"Invalid value for 'check_cols': \" ~ check_cols_config) %}\n {% endif %}\n\n {%- set row_changed_expr -%}\n (\n {%- if column_added -%}\n TRUE\n {%- else -%}\n {%- for col in check_cols -%}\n {{ snapshotted_rel }}.{{ col }} != {{ current_rel }}.{{ col }}\n or\n (\n (({{ snapshotted_rel }}.{{ col }} is null) and not ({{ current_rel }}.{{ col }} is null))\n or\n ((not {{ snapshotted_rel }}.{{ col }} is null) and ({{ current_rel }}.{{ col }} is null))\n )\n {%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n {%- endif -%}\n )\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.snapshot_get_time", "macro.dbt.run_query", "macro.dbt.snapshot_string_as_time", "macro.dbt.snapshot_check_all_get_existing_columns", "macro.dbt.snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.create_columns": {"unique_id": "macro.dbt.create_columns", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/snapshot.sql", "original_file_path": "macros/materializations/snapshot/snapshot.sql", "name": "create_columns", "macro_sql": "{% macro create_columns(relation, columns) %}\n {{ adapter.dispatch('create_columns')(relation, columns) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__create_columns"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.default__create_columns": {"unique_id": "macro.dbt.default__create_columns", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/snapshot.sql", "original_file_path": "macros/materializations/snapshot/snapshot.sql", "name": "default__create_columns", "macro_sql": "{% macro default__create_columns(relation, columns) %}\n {% for column in columns %}\n {% call statement() %}\n alter table {{ relation }} add column \"{{ column.name }}\" {{ column.data_type }};\n {% endcall %}\n {% endfor %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.post_snapshot": {"unique_id": "macro.dbt.post_snapshot", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/snapshot.sql", "original_file_path": "macros/materializations/snapshot/snapshot.sql", "name": "post_snapshot", "macro_sql": "{% macro post_snapshot(staging_relation) %}\n {{ adapter.dispatch('post_snapshot')(staging_relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.default__post_snapshot": {"unique_id": "macro.dbt.default__post_snapshot", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/snapshot.sql", "original_file_path": "macros/materializations/snapshot/snapshot.sql", "name": "default__post_snapshot", "macro_sql": "{% macro default__post_snapshot(staging_relation) %}\n {# no-op #}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.snapshot_staging_table": {"unique_id": "macro.dbt.snapshot_staging_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/snapshot.sql", "original_file_path": "macros/materializations/snapshot/snapshot.sql", "name": "snapshot_staging_table", "macro_sql": "{% macro snapshot_staging_table(strategy, source_sql, target_relation) -%}\n\n with snapshot_query as (\n\n {{ source_sql }}\n\n ),\n\n snapshotted_data as (\n\n select *,\n {{ strategy.unique_key }} as dbt_unique_key\n\n from {{ target_relation }}\n where dbt_valid_to is null\n\n ),\n\n insertions_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to,\n {{ strategy.scd_id }} as dbt_scd_id\n\n from snapshot_query\n ),\n\n updates_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n {{ strategy.updated_at }} as dbt_valid_to\n\n from snapshot_query\n ),\n\n {%- if strategy.invalidate_hard_deletes %}\n\n deletes_source_data as (\n\n select \n *,\n {{ strategy.unique_key }} as dbt_unique_key\n from snapshot_query\n ),\n {% endif %}\n\n insertions as (\n\n select\n 'insert' as dbt_change_type,\n source_data.*\n\n from insertions_source_data as source_data\n left outer join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where snapshotted_data.dbt_unique_key is null\n or (\n snapshotted_data.dbt_unique_key is not null\n and (\n {{ strategy.row_changed }}\n )\n )\n\n ),\n\n updates as (\n\n select\n 'update' as dbt_change_type,\n source_data.*,\n snapshotted_data.dbt_scd_id\n\n from updates_source_data as source_data\n join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where (\n {{ strategy.row_changed }}\n )\n )\n\n {%- if strategy.invalidate_hard_deletes -%}\n ,\n\n deletes as (\n \n select\n 'delete' as dbt_change_type,\n source_data.*,\n {{ snapshot_get_time() }} as dbt_valid_from,\n {{ snapshot_get_time() }} as dbt_updated_at,\n {{ snapshot_get_time() }} as dbt_valid_to,\n snapshotted_data.dbt_scd_id\n \n from snapshotted_data\n left join deletes_source_data as source_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where source_data.dbt_unique_key is null\n )\n {%- endif %}\n\n select * from insertions\n union all\n select * from updates\n {%- if strategy.invalidate_hard_deletes %}\n union all\n select * from deletes\n {%- endif %}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.snapshot_get_time"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.build_snapshot_table": {"unique_id": "macro.dbt.build_snapshot_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/snapshot.sql", "original_file_path": "macros/materializations/snapshot/snapshot.sql", "name": "build_snapshot_table", "macro_sql": "{% macro build_snapshot_table(strategy, sql) %}\n\n select *,\n {{ strategy.scd_id }} as dbt_scd_id,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to\n from (\n {{ sql }}\n ) sbq\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.get_or_create_relation": {"unique_id": "macro.dbt.get_or_create_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/snapshot.sql", "original_file_path": "macros/materializations/snapshot/snapshot.sql", "name": "get_or_create_relation", "macro_sql": "{% macro get_or_create_relation(database, schema, identifier, type) %}\n {%- set target_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n\n {% if target_relation %}\n {% do return([true, target_relation]) %}\n {% endif %}\n\n {%- set new_relation = api.Relation.create(\n database=database,\n schema=schema,\n identifier=identifier,\n type=type\n ) -%}\n {% do return([false, new_relation]) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.build_snapshot_staging_table": {"unique_id": "macro.dbt.build_snapshot_staging_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/snapshot.sql", "original_file_path": "macros/materializations/snapshot/snapshot.sql", "name": "build_snapshot_staging_table", "macro_sql": "{% macro build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% set tmp_relation = make_temp_relation(target_relation) %}\n\n {% set select = snapshot_staging_table(strategy, sql, target_relation) %}\n\n {% call statement('build_snapshot_staging_relation') %}\n {{ create_table_as(True, tmp_relation, select) }}\n {% endcall %}\n\n {% do return(tmp_relation) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.make_temp_relation", "macro.dbt.snapshot_staging_table", "macro.dbt.statement", "macro.dbt.create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.materialization_snapshot_default": {"unique_id": "macro.dbt.materialization_snapshot_default", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/snapshot.sql", "original_file_path": "macros/materializations/snapshot/snapshot.sql", "name": "materialization_snapshot_default", "macro_sql": "{% materialization snapshot, default %}\n {%- set config = model['config'] -%}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n\n {% if not adapter.check_schema_exists(model.database, model.schema) %}\n {% do create_schema(model.database, model.schema) %}\n {% endif %}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=model.database,\n schema=model.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", config, target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_sql']) %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% else %}\n\n {{ adapter.valid_snapshot_target(target_relation) }}\n\n {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% endif %}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if not target_relation_exists %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if staging_table is defined %}\n {% do post_snapshot(staging_table) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.create_schema", "macro.dbt.get_or_create_relation", "macro.dbt.run_hooks", "macro.dbt.strategy_dispatch", "macro.dbt.build_snapshot_table", "macro.dbt.create_table_as", "macro.dbt.build_snapshot_staging_table", "macro.dbt.create_columns", "macro.dbt.snapshot_merge_sql", "macro.dbt.statement", "macro.dbt.persist_docs", "macro.dbt.create_indexes", "macro.dbt.post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.create_csv_table": {"unique_id": "macro.dbt.create_csv_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/seed/seed.sql", "original_file_path": "macros/materializations/seed/seed.sql", "name": "create_csv_table", "macro_sql": "{% macro create_csv_table(model, agate_table) -%}\n {{ adapter.dispatch('create_csv_table')(model, agate_table) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.reset_csv_table": {"unique_id": "macro.dbt.reset_csv_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/seed/seed.sql", "original_file_path": "macros/materializations/seed/seed.sql", "name": "reset_csv_table", "macro_sql": "{% macro reset_csv_table(model, full_refresh, old_relation, agate_table) -%}\n {{ adapter.dispatch('reset_csv_table')(model, full_refresh, old_relation, agate_table) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__reset_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.load_csv_rows": {"unique_id": "macro.dbt.load_csv_rows", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/seed/seed.sql", "original_file_path": "macros/materializations/seed/seed.sql", "name": "load_csv_rows", "macro_sql": "{% macro load_csv_rows(model, agate_table) -%}\n {{ adapter.dispatch('load_csv_rows')(model, agate_table) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__load_csv_rows"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.default__create_csv_table": {"unique_id": "macro.dbt.default__create_csv_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/seed/seed.sql", "original_file_path": "macros/materializations/seed/seed.sql", "name": "default__create_csv_table", "macro_sql": "{% macro default__create_csv_table(model, agate_table) %}\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n\n {% set sql %}\n create table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.default__reset_csv_table": {"unique_id": "macro.dbt.default__reset_csv_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/seed/seed.sql", "original_file_path": "macros/materializations/seed/seed.sql", "name": "default__reset_csv_table", "macro_sql": "{% macro default__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% set sql = \"\" %}\n {% if full_refresh %}\n {{ adapter.drop_relation(old_relation) }}\n {% set sql = create_csv_table(model, agate_table) %}\n {% else %}\n {{ adapter.truncate_relation(old_relation) }}\n {% set sql = \"truncate table \" ~ old_relation %}\n {% endif %}\n\n {{ return(sql) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.get_seed_column_quoted_csv": {"unique_id": "macro.dbt.get_seed_column_quoted_csv", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/seed/seed.sql", "original_file_path": "macros/materializations/seed/seed.sql", "name": "get_seed_column_quoted_csv", "macro_sql": "{% macro get_seed_column_quoted_csv(model, column_names) %}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote_seed_column(col, quote_seed_column)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665163}, "macro.dbt.basic_load_csv_rows": {"unique_id": "macro.dbt.basic_load_csv_rows", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/seed/seed.sql", "original_file_path": "macros/materializations/seed/seed.sql", "name": "basic_load_csv_rows", "macro_sql": "{% macro basic_load_csv_rows(model, batch_size, agate_table) %}\n {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}\n {% set bindings = [] %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert into {{ this.render() }} ({{ cols_sql }}) values\n {% for row in chunk -%}\n ({%- for column in agate_table.column_names -%}\n %s\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_seed_column_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.default__load_csv_rows": {"unique_id": "macro.dbt.default__load_csv_rows", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/seed/seed.sql", "original_file_path": "macros/materializations/seed/seed.sql", "name": "default__load_csv_rows", "macro_sql": "{% macro default__load_csv_rows(model, agate_table) %}\n {{ return(basic_load_csv_rows(model, 10000, agate_table) )}}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.basic_load_csv_rows"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.materialization_seed_default": {"unique_id": "macro.dbt.materialization_seed_default", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/seed/seed.sql", "original_file_path": "macros/materializations/seed/seed.sql", "name": "materialization_seed_default", "macro_sql": "{% materialization seed, default %}\n\n {%- set identifier = model['alias'] -%}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set agate_table = load_agate_table() -%}\n {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% set create_table_sql = \"\" %}\n {% if exists_as_view %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a view\".format(old_relation)) }}\n {% elif exists_as_table %}\n {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}\n {% else %}\n {% set create_table_sql = create_csv_table(model, agate_table) %}\n {% endif %}\n\n {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}\n {% set rows_affected = (agate_table.rows | length) %}\n {% set sql = load_csv_rows(model, agate_table) %}\n\n {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}\n {{ create_table_sql }};\n -- dbt seed --\n {{ sql }}\n {% endcall %}\n\n {% set target_relation = this.incorporate(type='table') %}\n {% do persist_docs(target_relation, model) %}\n\n {% if full_refresh_mode or not exists_as_table %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.run_hooks", "macro.dbt.reset_csv_table", "macro.dbt.create_csv_table", "macro.dbt.load_csv_rows", "macro.dbt.noop_statement", "macro.dbt.persist_docs", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.incremental_upsert": {"unique_id": "macro.dbt.incremental_upsert", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/incremental/helpers.sql", "original_file_path": "macros/materializations/incremental/helpers.sql", "name": "incremental_upsert", "macro_sql": "{% macro incremental_upsert(tmp_relation, target_relation, unique_key=none, statement_name=\"main\") %}\n {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}\n\n {%- if unique_key is not none -%}\n delete\n from {{ target_relation }}\n where ({{ unique_key }}) in (\n select ({{ unique_key }})\n from {{ tmp_relation }}\n );\n {%- endif %}\n\n insert into {{ target_relation }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ tmp_relation }}\n );\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.materialization_incremental_default": {"unique_id": "macro.dbt.materialization_incremental_default", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/incremental/incremental.sql", "original_file_path": "macros/materializations/incremental/incremental.sql", "name": "materialization_incremental_default", "macro_sql": "{% materialization incremental, default -%}\n\n {% set unique_key = config.get('unique_key') %}\n\n {% set target_relation = this.incorporate(type='table') %}\n {% set existing_relation = load_relation(this) %}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set to_drop = [] %}\n {% if existing_relation is none %}\n {% set build_sql = create_table_as(False, target_relation, sql) %}\n {% elif existing_relation.is_view or should_full_refresh() %}\n {#-- Make sure the backup doesn't exist so we don't encounter issues with the rename below #}\n {% set tmp_identifier = model['name'] + '__dbt_tmp' %}\n {% set backup_identifier = model['name'] + \"__dbt_backup\" %}\n\n {% set intermediate_relation = existing_relation.incorporate(path={\"identifier\": tmp_identifier}) %}\n {% set backup_relation = existing_relation.incorporate(path={\"identifier\": backup_identifier}) %}\n\n {% do adapter.drop_relation(intermediate_relation) %}\n {% do adapter.drop_relation(backup_relation) %}\n\n {% set build_sql = create_table_as(False, intermediate_relation, sql) %}\n {% set need_swap = true %}\n {% do to_drop.append(backup_relation) %}\n {% else %}\n {% set tmp_relation = make_temp_relation(target_relation) %}\n {% do run_query(create_table_as(True, tmp_relation, sql)) %}\n {% do adapter.expand_target_column_types(\n from_relation=tmp_relation,\n to_relation=target_relation) %}\n {% set build_sql = incremental_upsert(tmp_relation, target_relation, unique_key=unique_key) %}\n {% endif %}\n\n {% call statement(\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% if need_swap %} \n {% do adapter.rename_relation(target_relation, backup_relation) %} \n {% do adapter.rename_relation(intermediate_relation, target_relation) %} \n {% endif %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if existing_relation is none or existing_relation.is_view or should_full_refresh() %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {% do adapter.commit() %}\n\n {% for rel in to_drop %}\n {% do adapter.drop_relation(rel) %}\n {% endfor %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.test.load_relation", "macro.dbt.run_hooks", "macro.dbt.create_table_as", "macro.dbt.should_full_refresh", "macro.dbt.make_temp_relation", "macro.dbt.run_query", "macro.dbt.incremental_upsert", "macro.dbt.statement", "macro.dbt.persist_docs", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.get_merge_sql": {"unique_id": "macro.dbt.get_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/common/merge.sql", "original_file_path": "macros/materializations/common/merge.sql", "name": "get_merge_sql", "macro_sql": "{% macro get_merge_sql(target, source, unique_key, dest_columns, predicates=none) -%}\n {{ adapter.dispatch('get_merge_sql')(target, source, unique_key, dest_columns, predicates) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.get_delete_insert_merge_sql": {"unique_id": "macro.dbt.get_delete_insert_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/common/merge.sql", "original_file_path": "macros/materializations/common/merge.sql", "name": "get_delete_insert_merge_sql", "macro_sql": "{% macro get_delete_insert_merge_sql(target, source, unique_key, dest_columns) -%}\n {{ adapter.dispatch('get_delete_insert_merge_sql')(target, source, unique_key, dest_columns) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_delete_insert_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.get_insert_overwrite_merge_sql": {"unique_id": "macro.dbt.get_insert_overwrite_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/common/merge.sql", "original_file_path": "macros/materializations/common/merge.sql", "name": "get_insert_overwrite_merge_sql", "macro_sql": "{% macro get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header=false) -%}\n {{ adapter.dispatch('get_insert_overwrite_merge_sql')(target, source, dest_columns, predicates, include_sql_header) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_insert_overwrite_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.default__get_merge_sql": {"unique_id": "macro.dbt.default__get_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/common/merge.sql", "original_file_path": "macros/materializations/common/merge.sql", "name": "default__get_merge_sql", "macro_sql": "{% macro default__get_merge_sql(target, source, unique_key, dest_columns, predicates) -%}\n {%- set predicates = [] if predicates is none else [] + predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set update_columns = config.get('merge_update_columns', default = dest_columns | map(attribute=\"quoted\") | list) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {% if unique_key %}\n {% set unique_key_match %}\n DBT_INTERNAL_SOURCE.{{ unique_key }} = DBT_INTERNAL_DEST.{{ unique_key }}\n {% endset %}\n {% do predicates.append(unique_key_match) %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n {{ sql_header if sql_header is not none }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{ predicates | join(' and ') }}\n\n {% if unique_key %}\n when matched then update set\n {% for column_name in update_columns -%}\n {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}\n {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n {% endif %}\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.get_quoted_csv": {"unique_id": "macro.dbt.get_quoted_csv", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/common/merge.sql", "original_file_path": "macros/materializations/common/merge.sql", "name": "get_quoted_csv", "macro_sql": "{% macro get_quoted_csv(column_names) %}\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote(col)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.common_get_delete_insert_merge_sql": {"unique_id": "macro.dbt.common_get_delete_insert_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/common/merge.sql", "original_file_path": "macros/materializations/common/merge.sql", "name": "common_get_delete_insert_merge_sql", "macro_sql": "{% macro common_get_delete_insert_merge_sql(target, source, unique_key, dest_columns) -%}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n {% if unique_key is not none %}\n delete from {{ target }}\n where ({{ unique_key }}) in (\n select ({{ unique_key }})\n from {{ source }}\n );\n {% endif %}\n\n insert into {{ target }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ source }}\n );\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.default__get_delete_insert_merge_sql": {"unique_id": "macro.dbt.default__get_delete_insert_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/common/merge.sql", "original_file_path": "macros/materializations/common/merge.sql", "name": "default__get_delete_insert_merge_sql", "macro_sql": "{% macro default__get_delete_insert_merge_sql(target, source, unique_key, dest_columns) -%}\n {{ common_get_delete_insert_merge_sql(target, source, unique_key, dest_columns) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.common_get_delete_insert_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.default__get_insert_overwrite_merge_sql": {"unique_id": "macro.dbt.default__get_insert_overwrite_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/common/merge.sql", "original_file_path": "macros/materializations/common/merge.sql", "name": "default__get_insert_overwrite_merge_sql", "macro_sql": "{% macro default__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) -%}\n {%- set predicates = [] if predicates is none else [] + predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none and include_sql_header }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on FALSE\n\n when not matched by source\n {% if predicates %} and {{ predicates | join(' and ') }} {% endif %}\n then delete\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.materialization_table_default": {"unique_id": "macro.dbt.materialization_table_default", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/table/table.sql", "original_file_path": "macros/materializations/table/table.sql", "name": "materialization_table_default", "macro_sql": "{% materialization table, default %}\n {%- set identifier = model['alias'] -%}\n {%- set tmp_identifier = model['name'] + '__dbt_tmp' -%}\n {%- set backup_identifier = model['name'] + '__dbt_backup' -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set target_relation = api.Relation.create(identifier=identifier,\n schema=schema,\n database=database,\n type='table') -%}\n {%- set intermediate_relation = api.Relation.create(identifier=tmp_identifier,\n schema=schema,\n database=database,\n type='table') -%}\n\n /*\n See ../view/view.sql for more information about this relation.\n */\n {%- set backup_relation_type = 'table' if old_relation is none else old_relation.type -%}\n {%- set backup_relation = api.Relation.create(identifier=backup_identifier,\n schema=schema,\n database=database,\n type=backup_relation_type) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n\n -- drop the temp relations if they exists for some reason\n {{ adapter.drop_relation(intermediate_relation) }}\n {{ adapter.drop_relation(backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ create_table_as(False, intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n {% if old_relation is not none %}\n {{ adapter.rename_relation(target_relation, backup_relation) }}\n {% endif %}\n\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% do create_indexes(target_relation) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {% do persist_docs(target_relation, model) %}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n -- finally, drop the existing/backup relation after the commit\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n{% endmaterialization %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.create_table_as", "macro.dbt.create_indexes", "macro.dbt.persist_docs", "macro.dbt.drop_relation_if_exists"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.materialization_view_default": {"unique_id": "macro.dbt.materialization_view_default", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/view/view.sql", "original_file_path": "macros/materializations/view/view.sql", "name": "materialization_view_default", "macro_sql": "{%- materialization view, default -%}\n\n {%- set identifier = model['alias'] -%}\n {%- set tmp_identifier = model['name'] + '__dbt_tmp' -%}\n {%- set backup_identifier = model['name'] + '__dbt_backup' -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set target_relation = api.Relation.create(identifier=identifier, schema=schema, database=database,\n type='view') -%}\n {%- set intermediate_relation = api.Relation.create(identifier=tmp_identifier,\n schema=schema, database=database, type='view') -%}\n\n /*\n This relation (probably) doesn't exist yet. If it does exist, it's a leftover from\n a previous run, and we're going to try to drop it immediately. At the end of this\n materialization, we're going to rename the \"old_relation\" to this identifier,\n and then we're going to drop it. In order to make sure we run the correct one of:\n - drop view ...\n - drop table ...\n\n We need to set the type of this relation to be the type of the old_relation, if it exists,\n or else \"view\" as a sane default if it does not. Note that if the old_relation does not\n exist, then there is nothing to move out of the way and subsequentally drop. In that case,\n this relation will be effectively unused.\n */\n {%- set backup_relation_type = 'view' if old_relation is none else old_relation.type -%}\n {%- set backup_relation = api.Relation.create(identifier=backup_identifier,\n schema=schema, database=database,\n type=backup_relation_type) -%}\n\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- drop the temp relations if they exists for some reason\n {{ adapter.drop_relation(intermediate_relation) }}\n {{ adapter.drop_relation(backup_relation) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ create_view_as(intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n -- move the existing view out of the way\n {% if old_relation is not none %}\n {{ adapter.rename_relation(target_relation, backup_relation) }}\n {% endif %}\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.create_view_as", "macro.dbt.persist_docs", "macro.dbt.drop_relation_if_exists"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.handle_existing_table": {"unique_id": "macro.dbt.handle_existing_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/view/create_or_replace_view.sql", "original_file_path": "macros/materializations/view/create_or_replace_view.sql", "name": "handle_existing_table", "macro_sql": "{% macro handle_existing_table(full_refresh, old_relation) %}\n {{ adapter.dispatch('handle_existing_table', macro_namespace = 'dbt')(full_refresh, old_relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__handle_existing_table"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.default__handle_existing_table": {"unique_id": "macro.dbt.default__handle_existing_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/view/create_or_replace_view.sql", "original_file_path": "macros/materializations/view/create_or_replace_view.sql", "name": "default__handle_existing_table", "macro_sql": "{% macro default__handle_existing_table(full_refresh, old_relation) %}\n {{ adapter.drop_relation(old_relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.create_or_replace_view": {"unique_id": "macro.dbt.create_or_replace_view", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/view/create_or_replace_view.sql", "original_file_path": "macros/materializations/view/create_or_replace_view.sql", "name": "create_or_replace_view", "macro_sql": "{% macro create_or_replace_view(run_outside_transaction_hooks=True) %}\n {%- set identifier = model['alias'] -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database,\n type='view') -%}\n\n {% if run_outside_transaction_hooks %}\n -- no transactions on BigQuery\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n {% endif %}\n\n -- `BEGIN` happens here on Snowflake\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- If there's a table with the same name and we weren't told to full refresh,\n -- that's an error. If we were told to full refresh, drop it. This behavior differs\n -- for Snowflake and BigQuery, so multiple dispatch is used.\n {%- if old_relation is not none and old_relation.is_table -%}\n {{ handle_existing_table(should_full_refresh(), old_relation) }}\n {%- endif -%}\n\n -- build model\n {% call statement('main') -%}\n {{ create_view_as(target_relation, sql) }}\n {%- endcall %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if run_outside_transaction_hooks %}\n -- No transactions on BigQuery\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n {% endif %}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.handle_existing_table", "macro.dbt.should_full_refresh", "macro.dbt.statement", "macro.dbt.create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.generate_alias_name": {"unique_id": "macro.dbt.generate_alias_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/etc/get_custom_alias.sql", "original_file_path": "macros/etc/get_custom_alias.sql", "name": "generate_alias_name", "macro_sql": "{% macro generate_alias_name(custom_alias_name=none, node=none) -%}\n\n {%- if custom_alias_name is none -%}\n\n {{ node.name }}\n\n {%- else -%}\n\n {{ custom_alias_name | trim }}\n\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.run_query": {"unique_id": "macro.dbt.run_query", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/etc/query.sql", "original_file_path": "macros/etc/query.sql", "name": "run_query", "macro_sql": "{% macro run_query(sql) %}\n {% call statement(\"run_query_statement\", fetch_result=true, auto_begin=false) %}\n {{ sql }}\n {% endcall %}\n\n {% do return(load_result(\"run_query_statement\").table) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.is_incremental": {"unique_id": "macro.dbt.is_incremental", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/etc/is_incremental.sql", "original_file_path": "macros/etc/is_incremental.sql", "name": "is_incremental", "macro_sql": "{% macro is_incremental() %}\n {#-- do not run introspective queries in parsing #}\n {% if not execute %}\n {{ return(False) }}\n {% else %}\n {% set relation = adapter.get_relation(this.database, this.schema, this.table) %}\n {{ return(relation is not none\n and relation.type == 'table'\n and model.config.materialized == 'incremental'\n and not should_full_refresh()) }}\n {% endif %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.should_full_refresh"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.convert_datetime": {"unique_id": "macro.dbt.convert_datetime", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "name": "convert_datetime", "macro_sql": "{% macro convert_datetime(date_str, date_fmt) %}\n\n {% set error_msg -%}\n The provided partition date '{{ date_str }}' does not match the expected format '{{ date_fmt }}'\n {%- endset %}\n\n {% set res = try_or_compiler_error(error_msg, modules.datetime.datetime.strptime, date_str.strip(), date_fmt) %}\n {{ return(res) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.dates_in_range": {"unique_id": "macro.dbt.dates_in_range", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "name": "dates_in_range", "macro_sql": "{% macro dates_in_range(start_date_str, end_date_str=none, in_fmt=\"%Y%m%d\", out_fmt=\"%Y%m%d\") %}\n {% set end_date_str = start_date_str if end_date_str is none else end_date_str %}\n\n {% set start_date = convert_datetime(start_date_str, in_fmt) %}\n {% set end_date = convert_datetime(end_date_str, in_fmt) %}\n\n {% set day_count = (end_date - start_date).days %}\n {% if day_count < 0 %}\n {% set msg -%}\n Partition start date is after the end date ({{ start_date }}, {{ end_date }})\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg, model) }}\n {% endif %}\n\n {% set date_list = [] %}\n {% for i in range(0, day_count + 1) %}\n {% set the_date = (modules.datetime.timedelta(days=i) + start_date) %}\n {% if not out_fmt %}\n {% set _ = date_list.append(the_date) %}\n {% else %}\n {% set _ = date_list.append(the_date.strftime(out_fmt)) %}\n {% endif %}\n {% endfor %}\n\n {{ return(date_list) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.convert_datetime"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.partition_range": {"unique_id": "macro.dbt.partition_range", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "name": "partition_range", "macro_sql": "{% macro partition_range(raw_partition_date, date_fmt='%Y%m%d') %}\n {% set partition_range = (raw_partition_date | string).split(\",\") %}\n\n {% if (partition_range | length) == 1 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = none %}\n {% elif (partition_range | length) == 2 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = partition_range[1] %}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Invalid partition time. Expected format: {Start Date}[,{End Date}]. Got: \" ~ raw_partition_date) }}\n {% endif %}\n\n {{ return(dates_in_range(start_date, end_date, in_fmt=date_fmt)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.dates_in_range"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.py_current_timestring": {"unique_id": "macro.dbt.py_current_timestring", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "name": "py_current_timestring", "macro_sql": "{% macro py_current_timestring() %}\n {% set dt = modules.datetime.datetime.now() %}\n {% do return(dt.strftime(\"%Y%m%d%H%M%S%f\")) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.generate_schema_name": {"unique_id": "macro.dbt.generate_schema_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/etc/get_custom_schema.sql", "original_file_path": "macros/etc/get_custom_schema.sql", "name": "generate_schema_name", "macro_sql": "{% macro generate_schema_name(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if custom_schema_name is none -%}\n\n {{ default_schema }}\n\n {%- else -%}\n\n {{ default_schema }}_{{ custom_schema_name | trim }}\n\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.generate_schema_name_for_env": {"unique_id": "macro.dbt.generate_schema_name_for_env", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/etc/get_custom_schema.sql", "original_file_path": "macros/etc/get_custom_schema.sql", "name": "generate_schema_name_for_env", "macro_sql": "{% macro generate_schema_name_for_env(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if target.name == 'prod' and custom_schema_name is not none -%}\n\n {{ custom_schema_name | trim }}\n\n {%- else -%}\n\n {{ default_schema }}\n\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.generate_database_name": {"unique_id": "macro.dbt.generate_database_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/etc/get_custom_database.sql", "original_file_path": "macros/etc/get_custom_database.sql", "name": "generate_database_name", "macro_sql": "{% macro generate_database_name(custom_database_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_database_name')(custom_database_name, node)) %}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__generate_database_name"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.default__generate_database_name": {"unique_id": "macro.dbt.default__generate_database_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/etc/get_custom_database.sql", "original_file_path": "macros/etc/get_custom_database.sql", "name": "default__generate_database_name", "macro_sql": "{% macro default__generate_database_name(custom_database_name=none, node=none) -%}\n {%- set default_database = target.database -%}\n {%- if custom_database_name is none -%}\n\n {{ default_database }}\n\n {%- else -%}\n\n {{ custom_database_name }}\n\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.get_columns_in_query": {"unique_id": "macro.dbt.get_columns_in_query", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "get_columns_in_query", "macro_sql": "{% macro get_columns_in_query(select_sql) -%}\n {{ return(adapter.dispatch('get_columns_in_query')(select_sql)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_columns_in_query"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.default__get_columns_in_query": {"unique_id": "macro.dbt.default__get_columns_in_query", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__get_columns_in_query", "macro_sql": "{% macro default__get_columns_in_query(select_sql) %}\n {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%}\n select * from (\n {{ select_sql }}\n ) as __dbt_sbq\n where false\n limit 0\n {% endcall %}\n\n {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.create_schema": {"unique_id": "macro.dbt.create_schema", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "create_schema", "macro_sql": "{% macro create_schema(relation) -%}\n {{ adapter.dispatch('create_schema')(relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__create_schema"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.default__create_schema": {"unique_id": "macro.dbt.default__create_schema", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__create_schema", "macro_sql": "{% macro default__create_schema(relation) -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier() }}\n {% endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.drop_schema": {"unique_id": "macro.dbt.drop_schema", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "drop_schema", "macro_sql": "{% macro drop_schema(relation) -%}\n {{ adapter.dispatch('drop_schema')(relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__drop_schema"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.default__drop_schema": {"unique_id": "macro.dbt.default__drop_schema", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__drop_schema", "macro_sql": "{% macro default__drop_schema(relation) -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier() }} cascade\n {% endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.create_table_as": {"unique_id": "macro.dbt.create_table_as", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "create_table_as", "macro_sql": "{% macro create_table_as(temporary, relation, sql) -%}\n {{ adapter.dispatch('create_table_as')(temporary, relation, sql) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.default__create_table_as": {"unique_id": "macro.dbt.default__create_table_as", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__create_table_as", "macro_sql": "{% macro default__create_table_as(temporary, relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary: -%}temporary{%- endif %} table\n {{ relation.include(database=(not temporary), schema=(not temporary)) }}\n as (\n {{ sql }}\n );\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.get_create_index_sql": {"unique_id": "macro.dbt.get_create_index_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "get_create_index_sql", "macro_sql": "{% macro get_create_index_sql(relation, index_dict) -%}\n {{ return(adapter.dispatch('get_create_index_sql')(relation, index_dict)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_create_index_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.default__get_create_index_sql": {"unique_id": "macro.dbt.default__get_create_index_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__get_create_index_sql", "macro_sql": "{% macro default__get_create_index_sql(relation, index_dict) -%}\n {% do return(None) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.create_indexes": {"unique_id": "macro.dbt.create_indexes", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "create_indexes", "macro_sql": "{% macro create_indexes(relation) -%}\n {{ adapter.dispatch('create_indexes')(relation) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.default__create_indexes": {"unique_id": "macro.dbt.default__create_indexes", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__create_indexes", "macro_sql": "{% macro default__create_indexes(relation) -%}\n {%- set _indexes = config.get('indexes', default=[]) -%}\n\n {% for _index_dict in _indexes %}\n {% set create_index_sql = get_create_index_sql(relation, _index_dict) %}\n {% if create_index_sql %}\n {% do run_query(create_index_sql) %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_create_index_sql", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.create_view_as": {"unique_id": "macro.dbt.create_view_as", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "create_view_as", "macro_sql": "{% macro create_view_as(relation, sql) -%}\n {{ adapter.dispatch('create_view_as')(relation, sql) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.default__create_view_as": {"unique_id": "macro.dbt.default__create_view_as", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__create_view_as", "macro_sql": "{% macro default__create_view_as(relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n create view {{ relation }} as (\n {{ sql }}\n );\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.get_catalog": {"unique_id": "macro.dbt.get_catalog", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "get_catalog", "macro_sql": "{% macro get_catalog(information_schema, schemas) -%}\n {{ return(adapter.dispatch('get_catalog')(information_schema, schemas)) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_catalog"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.default__get_catalog": {"unique_id": "macro.dbt.default__get_catalog", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__get_catalog", "macro_sql": "{% macro default__get_catalog(information_schema, schemas) -%}\n\n {% set typename = adapter.type() %}\n {% set msg -%}\n get_catalog not implemented for {{ typename }}\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.get_columns_in_relation": {"unique_id": "macro.dbt.get_columns_in_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "get_columns_in_relation", "macro_sql": "{% macro get_columns_in_relation(relation) -%}\n {{ return(adapter.dispatch('get_columns_in_relation')(relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_columns_in_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.sql_convert_columns_in_relation": {"unique_id": "macro.dbt.sql_convert_columns_in_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "sql_convert_columns_in_relation", "macro_sql": "{% macro sql_convert_columns_in_relation(table) -%}\n {% set columns = [] %}\n {% for row in table %}\n {% do columns.append(api.Column(*row)) %}\n {% endfor %}\n {{ return(columns) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.default__get_columns_in_relation": {"unique_id": "macro.dbt.default__get_columns_in_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__get_columns_in_relation", "macro_sql": "{% macro default__get_columns_in_relation(relation) -%}\n {{ exceptions.raise_not_implemented(\n 'get_columns_in_relation macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.alter_column_type": {"unique_id": "macro.dbt.alter_column_type", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "alter_column_type", "macro_sql": "{% macro alter_column_type(relation, column_name, new_column_type) -%}\n {{ return(adapter.dispatch('alter_column_type')(relation, column_name, new_column_type)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__alter_column_type"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.alter_column_comment": {"unique_id": "macro.dbt.alter_column_comment", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "alter_column_comment", "macro_sql": "{% macro alter_column_comment(relation, column_dict) -%}\n {{ return(adapter.dispatch('alter_column_comment')(relation, column_dict)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.default__alter_column_comment": {"unique_id": "macro.dbt.default__alter_column_comment", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__alter_column_comment", "macro_sql": "{% macro default__alter_column_comment(relation, column_dict) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_column_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.alter_relation_comment": {"unique_id": "macro.dbt.alter_relation_comment", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "alter_relation_comment", "macro_sql": "{% macro alter_relation_comment(relation, relation_comment) -%}\n {{ return(adapter.dispatch('alter_relation_comment')(relation, relation_comment)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__alter_relation_comment"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.default__alter_relation_comment": {"unique_id": "macro.dbt.default__alter_relation_comment", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__alter_relation_comment", "macro_sql": "{% macro default__alter_relation_comment(relation, relation_comment) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_relation_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.persist_docs": {"unique_id": "macro.dbt.persist_docs", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "persist_docs", "macro_sql": "{% macro persist_docs(relation, model, for_relation=true, for_columns=true) -%}\n {{ return(adapter.dispatch('persist_docs')(relation, model, for_relation, for_columns)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.default__persist_docs": {"unique_id": "macro.dbt.default__persist_docs", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__persist_docs", "macro_sql": "{% macro default__persist_docs(relation, model, for_relation, for_columns) -%}\n {% if for_relation and config.persist_relation_docs() and model.description %}\n {% do run_query(alter_relation_comment(relation, model.description)) %}\n {% endif %}\n\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {% do run_query(alter_column_comment(relation, model.columns)) %}\n {% endif %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.run_query", "macro.dbt.alter_relation_comment", "macro.dbt.alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.default__alter_column_type": {"unique_id": "macro.dbt.default__alter_column_type", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__alter_column_type", "macro_sql": "{% macro default__alter_column_type(relation, column_name, new_column_type) -%}\n {#\n 1. Create a new column (w/ temp name and correct type)\n 2. Copy data over to it\n 3. Drop the existing column (cascade!)\n 4. Rename the new column to existing column\n #}\n {%- set tmp_column = column_name + \"__dbt_alter\" -%}\n\n {% call statement('alter_column_type') %}\n alter table {{ relation }} add column {{ adapter.quote(tmp_column) }} {{ new_column_type }};\n update {{ relation }} set {{ adapter.quote(tmp_column) }} = {{ adapter.quote(column_name) }};\n alter table {{ relation }} drop column {{ adapter.quote(column_name) }} cascade;\n alter table {{ relation }} rename column {{ adapter.quote(tmp_column) }} to {{ adapter.quote(column_name) }}\n {% endcall %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.drop_relation": {"unique_id": "macro.dbt.drop_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "drop_relation", "macro_sql": "{% macro drop_relation(relation) -%}\n {{ return(adapter.dispatch('drop_relation')(relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__drop_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.default__drop_relation": {"unique_id": "macro.dbt.default__drop_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__drop_relation", "macro_sql": "{% macro default__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n drop {{ relation.type }} if exists {{ relation }} cascade\n {%- endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.truncate_relation": {"unique_id": "macro.dbt.truncate_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "truncate_relation", "macro_sql": "{% macro truncate_relation(relation) -%}\n {{ return(adapter.dispatch('truncate_relation')(relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__truncate_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.default__truncate_relation": {"unique_id": "macro.dbt.default__truncate_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__truncate_relation", "macro_sql": "{% macro default__truncate_relation(relation) -%}\n {% call statement('truncate_relation') -%}\n truncate table {{ relation }}\n {%- endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.rename_relation": {"unique_id": "macro.dbt.rename_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "rename_relation", "macro_sql": "{% macro rename_relation(from_relation, to_relation) -%}\n {{ return(adapter.dispatch('rename_relation')(from_relation, to_relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__rename_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.default__rename_relation": {"unique_id": "macro.dbt.default__rename_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__rename_relation", "macro_sql": "{% macro default__rename_relation(from_relation, to_relation) -%}\n {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %}\n {% call statement('rename_relation') -%}\n alter table {{ from_relation }} rename to {{ target_name }}\n {%- endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.information_schema_name": {"unique_id": "macro.dbt.information_schema_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "information_schema_name", "macro_sql": "{% macro information_schema_name(database) %}\n {{ return(adapter.dispatch('information_schema_name')(database)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__information_schema_name"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.default__information_schema_name": {"unique_id": "macro.dbt.default__information_schema_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__information_schema_name", "macro_sql": "{% macro default__information_schema_name(database) -%}\n {%- if database -%}\n {{ database }}.INFORMATION_SCHEMA\n {%- else -%}\n INFORMATION_SCHEMA\n {%- endif -%}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.list_schemas": {"unique_id": "macro.dbt.list_schemas", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "list_schemas", "macro_sql": "{% macro list_schemas(database) -%}\n {{ return(adapter.dispatch('list_schemas')(database)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__list_schemas"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.default__list_schemas": {"unique_id": "macro.dbt.default__list_schemas", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__list_schemas", "macro_sql": "{% macro default__list_schemas(database) -%}\n {% set sql %}\n select distinct schema_name\n from {{ information_schema_name(database) }}.SCHEMATA\n where catalog_name ilike '{{ database }}'\n {% endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.information_schema_name", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.check_schema_exists": {"unique_id": "macro.dbt.check_schema_exists", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "check_schema_exists", "macro_sql": "{% macro check_schema_exists(information_schema, schema) -%}\n {{ return(adapter.dispatch('check_schema_exists')(information_schema, schema)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__check_schema_exists"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.default__check_schema_exists": {"unique_id": "macro.dbt.default__check_schema_exists", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__check_schema_exists", "macro_sql": "{% macro default__check_schema_exists(information_schema, schema) -%}\n {% set sql -%}\n select count(*)\n from {{ information_schema.replace(information_schema_view='SCHEMATA') }}\n where catalog_name='{{ information_schema.database }}'\n and schema_name='{{ schema }}'\n {%- endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.list_relations_without_caching": {"unique_id": "macro.dbt.list_relations_without_caching", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "list_relations_without_caching", "macro_sql": "{% macro list_relations_without_caching(schema_relation) %}\n {{ return(adapter.dispatch('list_relations_without_caching')(schema_relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.test.postgres__list_relations_without_caching"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.default__list_relations_without_caching": {"unique_id": "macro.dbt.default__list_relations_without_caching", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__list_relations_without_caching", "macro_sql": "{% macro default__list_relations_without_caching(schema_relation) %}\n {{ exceptions.raise_not_implemented(\n 'list_relations_without_caching macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.current_timestamp": {"unique_id": "macro.dbt.current_timestamp", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "current_timestamp", "macro_sql": "{% macro current_timestamp() -%}\n {{ adapter.dispatch('current_timestamp')() }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.default__current_timestamp": {"unique_id": "macro.dbt.default__current_timestamp", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__current_timestamp", "macro_sql": "{% macro default__current_timestamp() -%}\n {{ exceptions.raise_not_implemented(\n 'current_timestamp macro not implemented for adapter '+adapter.type()) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.collect_freshness": {"unique_id": "macro.dbt.collect_freshness", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "collect_freshness", "macro_sql": "{% macro collect_freshness(source, loaded_at_field, filter) %}\n {{ return(adapter.dispatch('collect_freshness')(source, loaded_at_field, filter))}}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__collect_freshness"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.default__collect_freshness": {"unique_id": "macro.dbt.default__collect_freshness", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__collect_freshness", "macro_sql": "{% macro default__collect_freshness(source, loaded_at_field, filter) %}\n {% call statement('collect_freshness', fetch_result=True, auto_begin=False) -%}\n select\n max({{ loaded_at_field }}) as max_loaded_at,\n {{ current_timestamp() }} as snapshotted_at\n from {{ source }}\n {% if filter %}\n where {{ filter }}\n {% endif %}\n {% endcall %}\n {{ return(load_result('collect_freshness').table) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.make_temp_relation": {"unique_id": "macro.dbt.make_temp_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "make_temp_relation", "macro_sql": "{% macro make_temp_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_temp_relation')(base_relation, suffix))}}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_temp_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.default__make_temp_relation": {"unique_id": "macro.dbt.default__make_temp_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__make_temp_relation", "macro_sql": "{% macro default__make_temp_relation(base_relation, suffix) %}\n {% set tmp_identifier = base_relation.identifier ~ suffix %}\n {% set tmp_relation = base_relation.incorporate(\n path={\"identifier\": tmp_identifier}) -%}\n\n {% do return(tmp_relation) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.set_sql_header": {"unique_id": "macro.dbt.set_sql_header", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "set_sql_header", "macro_sql": "{% macro set_sql_header(config) -%}\n {{ config.set('sql_header', caller()) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.default__test_relationships": {"unique_id": "macro.dbt.default__test_relationships", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/schema_tests/relationships.sql", "original_file_path": "macros/schema_tests/relationships.sql", "name": "default__test_relationships", "macro_sql": "{% macro default__test_relationships(model, column_name, to, field) %}\n\nwith child as (\n select {{ column_name }} as from_field\n from {{ model }}\n where {{ column_name }} is not null\n),\n\nparent as (\n select {{ field }} as to_field\n from {{ to }}\n)\n\nselect\n from_field\n\nfrom child\nleft join parent\n on child.from_field = parent.to_field\n\nwhere parent.to_field is null\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.test_relationships": {"unique_id": "macro.dbt.test_relationships", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/schema_tests/relationships.sql", "original_file_path": "macros/schema_tests/relationships.sql", "name": "test_relationships", "macro_sql": "{% test relationships(model, column_name, to, field) %}\n {% set macro = adapter.dispatch('test_relationships') %}\n {{ macro(model, column_name, to, field) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__test_relationships"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.default__test_not_null": {"unique_id": "macro.dbt.default__test_not_null", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/schema_tests/not_null.sql", "original_file_path": "macros/schema_tests/not_null.sql", "name": "default__test_not_null", "macro_sql": "{% macro default__test_not_null(model, column_name) %}\n\nselect *\nfrom {{ model }}\nwhere {{ column_name }} is null\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.test_not_null": {"unique_id": "macro.dbt.test_not_null", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/schema_tests/not_null.sql", "original_file_path": "macros/schema_tests/not_null.sql", "name": "test_not_null", "macro_sql": "{% test not_null(model, column_name) %}\n {% set macro = adapter.dispatch('test_not_null') %}\n {{ macro(model, column_name) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__test_not_null"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.default__test_unique": {"unique_id": "macro.dbt.default__test_unique", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/schema_tests/unique.sql", "original_file_path": "macros/schema_tests/unique.sql", "name": "default__test_unique", "macro_sql": "{% macro default__test_unique(model, column_name) %}\n\nselect\n {{ column_name }} as unique_field,\n count(*) as n_records\n\nfrom {{ model }}\nwhere {{ column_name }} is not null\ngroup by {{ column_name }}\nhaving count(*) > 1\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.test_unique": {"unique_id": "macro.dbt.test_unique", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/schema_tests/unique.sql", "original_file_path": "macros/schema_tests/unique.sql", "name": "test_unique", "macro_sql": "{% test unique(model, column_name) %}\n {% set macro = adapter.dispatch('test_unique') %}\n {{ macro(model, column_name) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__test_unique"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.default__test_accepted_values": {"unique_id": "macro.dbt.default__test_accepted_values", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/schema_tests/accepted_values.sql", "original_file_path": "macros/schema_tests/accepted_values.sql", "name": "default__test_accepted_values", "macro_sql": "{% macro default__test_accepted_values(model, column_name, values, quote=True) %}\n\nwith all_values as (\n\n select\n {{ column_name }} as value_field,\n count(*) as n_records\n\n from {{ model }}\n group by 1\n\n)\n\nselect *\nfrom all_values\nwhere value_field not in (\n {% for value in values -%}\n {% if quote -%}\n '{{ value }}'\n {%- else -%}\n {{ value }}\n {%- endif -%}\n {%- if not loop.last -%},{%- endif %}\n {%- endfor %}\n)\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}, "macro.dbt.test_accepted_values": {"unique_id": "macro.dbt.test_accepted_values", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/schema_tests/accepted_values.sql", "original_file_path": "macros/schema_tests/accepted_values.sql", "name": "test_accepted_values", "macro_sql": "{% test accepted_values(model, column_name, values, quote=True) %}\n {% set macro = adapter.dispatch('test_accepted_values') %}\n {{ macro(model, column_name, values, quote) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__test_accepted_values"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665164}}, "docs": {"dbt.__overview__": {"unique_id": "dbt.__overview__", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "overview.md", "original_file_path": "docs/overview.md", "name": "__overview__", "block_contents": "### Welcome!\n\nWelcome to the auto-generated documentation for your dbt project!\n\n### Navigation\n\nYou can use the `Project` and `Database` navigation tabs on the left side of the window to explore the models\nin your project.\n\n#### Project Tab\nThe `Project` tab mirrors the directory structure of your dbt project. In this tab, you can see all of the\nmodels defined in your dbt project, as well as models imported from dbt packages.\n\n#### Database Tab\nThe `Database` tab also exposes your models, but in a format that looks more like a database explorer. This view\nshows relations (tables and views) grouped into database schemas. Note that ephemeral models are _not_ shown\nin this interface, as they do not exist in the database.\n\n### Graph Exploration\nYou can click the blue icon on the bottom-right corner of the page to view the lineage graph of your models.\n\nOn model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the `Expand`\nbutton at the top-right of this lineage pane, you'll be able to see all of the models that are used to build,\nor are built from, the model you're exploring.\n\nOnce expanded, you'll be able to use the `--models` and `--exclude` model selection syntax to filter the\nmodels in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax).\n\nNote that you can also right-click on models to interactively filter and explore the graph.\n\n---\n\n### More information\n\n- [What is dbt](https://docs.getdbt.com/docs/overview)?\n- Read the [dbt viewpoint](https://docs.getdbt.com/docs/viewpoint)\n- [Installation](https://docs.getdbt.com/docs/installation)\n- Join the [chat](https://community.getdbt.com/) on Slack for live questions and support."}}, "exposures": {}, "selectors": {}, "disabled": [], "parent_map": {"model.test.my_model": []}, "child_map": {"model.test.my_model": []}} ================================================ FILE: tests/functional/artifacts/data/state/v3/manifest.json ================================================ {"metadata": {"dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v3.json", "dbt_version": "0.21.1", "generated_at": "2022-06-08T05:12:26.978818Z", "invocation_id": "a2594229-14b7-46fe-864f-37cabb5f5f65", "env": {}, "project_id": "098f6bcd4621d373cade4e832627b4f6", "user_id": null, "send_anonymous_usage_stats": false, "adapter_type": "postgres"}, "nodes": {"model.test.my_model": {"raw_sql": "select 1 as id", "resource_type": "model", "depends_on": {"macros": [], "nodes": []}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "on_schema_change": "ignore", "post-hook": [], "pre-hook": []}, "database": "jerco", "schema": "dbt_jcohen", "fqn": ["test", "my_model"], "unique_id": "model.test.my_model", "package_name": "test", "root_path": "/Users/jerco/dev/scratch/testy", "path": "my_model.sql", "original_file_path": "models/my_model.sql", "name": "my_model", "alias": "my_model", "checksum": {"name": "sha256", "checksum": "479636cb85ce8d3b0f8db5ff13cf338b61254ad98d905630eac61f963e719e9d"}, "tags": [], "refs": [], "sources": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true}, "patch_path": null, "compiled_path": null, "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1654665147}}, "sources": {}, "macros": {"macro.test.drop_relation": {"unique_id": "macro.test.drop_relation", "package_name": "test", "root_path": "/Users/jerco/dev/scratch/testy", "path": "macros/whatever.sql", "original_file_path": "macros/whatever.sql", "name": "drop_relation", "macro_sql": "{% macro drop_relation(relation) -%}\n {{ return(dbt_labs_materialized_views.drop_relation(relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.test.drop_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665146}, "macro.test.postgres__list_relations_without_caching": {"unique_id": "macro.test.postgres__list_relations_without_caching", "package_name": "test", "root_path": "/Users/jerco/dev/scratch/testy", "path": "macros/whatever.sql", "original_file_path": "macros/whatever.sql", "name": "postgres__list_relations_without_caching", "macro_sql": "{% macro postgres__list_relations_without_caching(schema_relation) %}\n {{ return(dbt_labs_materialized_views.postgres__list_relations_without_caching(schema_relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.test.postgres__list_relations_without_caching"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665146}, "macro.test.postgres_get_relations": {"unique_id": "macro.test.postgres_get_relations", "package_name": "test", "root_path": "/Users/jerco/dev/scratch/testy", "path": "macros/whatever.sql", "original_file_path": "macros/whatever.sql", "name": "postgres_get_relations", "macro_sql": "{% macro postgres_get_relations() %}\n {{ return(dbt_labs_materialized_views.postgres_get_relations()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.test.postgres_get_relations"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665146}, "macro.test.redshift__list_relations_without_caching": {"unique_id": "macro.test.redshift__list_relations_without_caching", "package_name": "test", "root_path": "/Users/jerco/dev/scratch/testy", "path": "macros/whatever.sql", "original_file_path": "macros/whatever.sql", "name": "redshift__list_relations_without_caching", "macro_sql": "{% macro redshift__list_relations_without_caching(schema_relation) %}\n {{ return(dbt_labs_materialized_views.redshift__list_relations_without_caching(schema_relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.test.redshift__list_relations_without_caching"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665146}, "macro.test.load_relation": {"unique_id": "macro.test.load_relation", "package_name": "test", "root_path": "/Users/jerco/dev/scratch/testy", "path": "macros/whatever.sql", "original_file_path": "macros/whatever.sql", "name": "load_relation", "macro_sql": "{% macro load_relation(relation) %}\n {{ return(dbt_labs_materialized_views.redshift_load_relation_or_mv(relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665146}, "macro.dbt_postgres.postgres__get_catalog": {"unique_id": "macro.dbt_postgres.postgres__get_catalog", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/catalog.sql", "original_file_path": "macros/catalog.sql", "name": "postgres__get_catalog", "macro_sql": "{% macro postgres__get_catalog(information_schema, schemas) -%}\n\n {%- call statement('catalog', fetch_result=True) -%}\n {#\n If the user has multiple databases set and the first one is wrong, this will fail.\n But we won't fail in the case where there are multiple quoting-difference-only dbs, which is better.\n #}\n {% set database = information_schema.database %}\n {{ adapter.verify_database(database) }}\n\n select\n '{{ database }}' as table_database,\n sch.nspname as table_schema,\n tbl.relname as table_name,\n case tbl.relkind\n when 'v' then 'VIEW'\n else 'BASE TABLE'\n end as table_type,\n tbl_desc.description as table_comment,\n col.attname as column_name,\n col.attnum as column_index,\n pg_catalog.format_type(col.atttypid, col.atttypmod) as column_type,\n col_desc.description as column_comment,\n pg_get_userbyid(tbl.relowner) as table_owner\n\n from pg_catalog.pg_namespace sch\n join pg_catalog.pg_class tbl on tbl.relnamespace = sch.oid\n join pg_catalog.pg_attribute col on col.attrelid = tbl.oid\n left outer join pg_catalog.pg_description tbl_desc on (tbl_desc.objoid = tbl.oid and tbl_desc.objsubid = 0)\n left outer join pg_catalog.pg_description col_desc on (col_desc.objoid = tbl.oid and col_desc.objsubid = col.attnum)\n\n where (\n {%- for schema in schemas -%}\n upper(sch.nspname) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n )\n and not pg_is_other_temp_schema(sch.oid) -- not a temporary schema belonging to another session\n and tbl.relpersistence = 'p' -- [p]ermanent table. Other values are [u]nlogged table, [t]emporary table\n and tbl.relkind in ('r', 'v', 'f', 'p') -- o[r]dinary table, [v]iew, [f]oreign table, [p]artitioned table. Other values are [i]ndex, [S]equence, [c]omposite type, [t]OAST table, [m]aterialized view\n and col.attnum > 0 -- negative numbers are used for system columns such as oid\n and not col.attisdropped -- column as not been dropped\n\n order by\n sch.nspname,\n tbl.relname,\n col.attnum\n\n {%- endcall -%}\n\n {{ return(load_result('catalog').table) }}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665146}, "macro.dbt_postgres.postgres_get_relations": {"unique_id": "macro.dbt_postgres.postgres_get_relations", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/relations.sql", "original_file_path": "macros/relations.sql", "name": "postgres_get_relations", "macro_sql": "{% macro postgres_get_relations () -%}\n\n {#\n -- in pg_depend, objid is the dependent, refobjid is the referenced object\n -- > a pg_depend entry indicates that the referenced object cannot be\n -- > dropped without also dropping the dependent object.\n #}\n\n {%- call statement('relations', fetch_result=True) -%}\n with relation as (\n select\n pg_rewrite.ev_class as class,\n pg_rewrite.oid as id\n from pg_rewrite\n ),\n class as (\n select\n oid as id,\n relname as name,\n relnamespace as schema,\n relkind as kind\n from pg_class\n ),\n dependency as (\n select\n pg_depend.objid as id,\n pg_depend.refobjid as ref\n from pg_depend\n ),\n schema as (\n select\n pg_namespace.oid as id,\n pg_namespace.nspname as name\n from pg_namespace\n where nspname != 'information_schema' and nspname not like 'pg\\_%'\n ),\n referenced as (\n select\n relation.id AS id,\n referenced_class.name ,\n referenced_class.schema ,\n referenced_class.kind\n from relation\n join class as referenced_class on relation.class=referenced_class.id\n where referenced_class.kind in ('r', 'v')\n ),\n relationships as (\n select\n referenced.name as referenced_name,\n referenced.schema as referenced_schema_id,\n dependent_class.name as dependent_name,\n dependent_class.schema as dependent_schema_id,\n referenced.kind as kind\n from referenced\n join dependency on referenced.id=dependency.id\n join class as dependent_class on dependency.ref=dependent_class.id\n where\n (referenced.name != dependent_class.name or\n referenced.schema != dependent_class.schema)\n )\n\n select\n referenced_schema.name as referenced_schema,\n relationships.referenced_name as referenced_name,\n dependent_schema.name as dependent_schema,\n relationships.dependent_name as dependent_name\n from relationships\n join schema as dependent_schema on relationships.dependent_schema_id=dependent_schema.id\n join schema as referenced_schema on relationships.referenced_schema_id=referenced_schema.id\n group by referenced_schema, referenced_name, dependent_schema, dependent_name\n order by referenced_schema, referenced_name, dependent_schema, dependent_name;\n\n {%- endcall -%}\n\n {{ return(load_result('relations').table) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665146}, "macro.dbt_postgres.postgres__create_table_as": {"unique_id": "macro.dbt_postgres.postgres__create_table_as", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__create_table_as", "macro_sql": "{% macro postgres__create_table_as(temporary, relation, sql) -%}\n {%- set unlogged = config.get('unlogged', default=false) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary -%}\n temporary\n {%- elif unlogged -%}\n unlogged\n {%- endif %} table {{ relation }}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt_postgres.postgres__get_create_index_sql": {"unique_id": "macro.dbt_postgres.postgres__get_create_index_sql", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__get_create_index_sql", "macro_sql": "{% macro postgres__get_create_index_sql(relation, index_dict) -%}\n {%- set index_config = adapter.parse_index(index_dict) -%}\n {%- set comma_separated_columns = \", \".join(index_config.columns) -%}\n {%- set index_name = index_config.render(relation) -%}\n\n create {% if index_config.unique -%}\n unique\n {%- endif %} index if not exists\n \"{{ index_name }}\"\n on {{ relation }} {% if index_config.type -%}\n using {{ index_config.type }}\n {%- endif %}\n ({{ comma_separated_columns }});\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt_postgres.postgres__create_schema": {"unique_id": "macro.dbt_postgres.postgres__create_schema", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__create_schema", "macro_sql": "{% macro postgres__create_schema(relation) -%}\n {% if relation.database -%}\n {{ adapter.verify_database(relation.database) }}\n {%- endif -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier().include(database=False) }}\n {%- endcall -%}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt_postgres.postgres__drop_schema": {"unique_id": "macro.dbt_postgres.postgres__drop_schema", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__drop_schema", "macro_sql": "{% macro postgres__drop_schema(relation) -%}\n {% if relation.database -%}\n {{ adapter.verify_database(relation.database) }}\n {%- endif -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier().include(database=False) }} cascade\n {%- endcall -%}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt_postgres.postgres__get_columns_in_relation": {"unique_id": "macro.dbt_postgres.postgres__get_columns_in_relation", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__get_columns_in_relation", "macro_sql": "{% macro postgres__get_columns_in_relation(relation) -%}\n {% call statement('get_columns_in_relation', fetch_result=True) %}\n select\n column_name,\n data_type,\n character_maximum_length,\n numeric_precision,\n numeric_scale\n\n from {{ relation.information_schema('columns') }}\n where table_name = '{{ relation.identifier }}'\n {% if relation.schema %}\n and table_schema = '{{ relation.schema }}'\n {% endif %}\n order by ordinal_position\n\n {% endcall %}\n {% set table = load_result('get_columns_in_relation').table %}\n {{ return(sql_convert_columns_in_relation(table)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.sql_convert_columns_in_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt_postgres.postgres__list_relations_without_caching": {"unique_id": "macro.dbt_postgres.postgres__list_relations_without_caching", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__list_relations_without_caching", "macro_sql": "{% macro postgres__list_relations_without_caching(schema_relation) %}\n {% call statement('list_relations_without_caching', fetch_result=True) -%}\n select\n '{{ schema_relation.database }}' as database,\n tablename as name,\n schemaname as schema,\n 'table' as type\n from pg_tables\n where schemaname ilike '{{ schema_relation.schema }}'\n union all\n select\n '{{ schema_relation.database }}' as database,\n viewname as name,\n schemaname as schema,\n 'view' as type\n from pg_views\n where schemaname ilike '{{ schema_relation.schema }}'\n {% endcall %}\n {{ return(load_result('list_relations_without_caching').table) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt_postgres.postgres__information_schema_name": {"unique_id": "macro.dbt_postgres.postgres__information_schema_name", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__information_schema_name", "macro_sql": "{% macro postgres__information_schema_name(database) -%}\n {% if database_name -%}\n {{ adapter.verify_database(database_name) }}\n {%- endif -%}\n information_schema\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt_postgres.postgres__list_schemas": {"unique_id": "macro.dbt_postgres.postgres__list_schemas", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__list_schemas", "macro_sql": "{% macro postgres__list_schemas(database) %}\n {% if database -%}\n {{ adapter.verify_database(database) }}\n {%- endif -%}\n {% call statement('list_schemas', fetch_result=True, auto_begin=False) %}\n select distinct nspname from pg_namespace\n {% endcall %}\n {{ return(load_result('list_schemas').table) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt_postgres.postgres__check_schema_exists": {"unique_id": "macro.dbt_postgres.postgres__check_schema_exists", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__check_schema_exists", "macro_sql": "{% macro postgres__check_schema_exists(information_schema, schema) -%}\n {% if information_schema.database -%}\n {{ adapter.verify_database(information_schema.database) }}\n {%- endif -%}\n {% call statement('check_schema_exists', fetch_result=True, auto_begin=False) %}\n select count(*) from pg_namespace where nspname = '{{ schema }}'\n {% endcall %}\n {{ return(load_result('check_schema_exists').table) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt_postgres.postgres__current_timestamp": {"unique_id": "macro.dbt_postgres.postgres__current_timestamp", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__current_timestamp", "macro_sql": "{% macro postgres__current_timestamp() -%}\n now()\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt_postgres.postgres__snapshot_string_as_time": {"unique_id": "macro.dbt_postgres.postgres__snapshot_string_as_time", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__snapshot_string_as_time", "macro_sql": "{% macro postgres__snapshot_string_as_time(timestamp) -%}\n {%- set result = \"'\" ~ timestamp ~ \"'::timestamp without time zone\" -%}\n {{ return(result) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt_postgres.postgres__snapshot_get_time": {"unique_id": "macro.dbt_postgres.postgres__snapshot_get_time", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__snapshot_get_time", "macro_sql": "{% macro postgres__snapshot_get_time() -%}\n {{ current_timestamp() }}::timestamp without time zone\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt_postgres.postgres__make_temp_relation": {"unique_id": "macro.dbt_postgres.postgres__make_temp_relation", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__make_temp_relation", "macro_sql": "{% macro postgres__make_temp_relation(base_relation, suffix) %}\n {% set dt = modules.datetime.datetime.now() %}\n {% set dtstring = dt.strftime(\"%H%M%S%f\") %}\n {% set suffix_length = suffix|length + dtstring|length %}\n {% set relation_max_name_length = 63 %}\n {% if suffix_length > relation_max_name_length %}\n {% do exceptions.raise_compiler_error('Temp relation suffix is too long (' ~ suffix|length ~ ' characters). Maximum length is ' ~ (relation_max_name_length - dtstring|length) ~ ' characters.') %}\n {% endif %}\n {% set tmp_identifier = base_relation.identifier[:relation_max_name_length - suffix_length] ~ suffix ~ dtstring %}\n {% do return(base_relation.incorporate(\n path={\n \"identifier\": tmp_identifier,\n \"schema\": none,\n \"database\": none\n })) -%}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt_postgres.postgres_escape_comment": {"unique_id": "macro.dbt_postgres.postgres_escape_comment", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres_escape_comment", "macro_sql": "{% macro postgres_escape_comment(comment) -%}\n {% if comment is not string %}\n {% do exceptions.raise_compiler_error('cannot escape a non-string: ' ~ comment) %}\n {% endif %}\n {%- set magic = '$dbt_comment_literal_block$' -%}\n {%- if magic in comment -%}\n {%- do exceptions.raise_compiler_error('The string ' ~ magic ~ ' is not allowed in comments.') -%}\n {%- endif -%}\n {{ magic }}{{ comment }}{{ magic }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt_postgres.postgres__alter_relation_comment": {"unique_id": "macro.dbt_postgres.postgres__alter_relation_comment", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__alter_relation_comment", "macro_sql": "{% macro postgres__alter_relation_comment(relation, comment) %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n comment on {{ relation.type }} {{ relation }} is {{ escaped_comment }};\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres_escape_comment"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt_postgres.postgres__alter_column_comment": {"unique_id": "macro.dbt_postgres.postgres__alter_column_comment", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__alter_column_comment", "macro_sql": "{% macro postgres__alter_column_comment(relation, column_dict) %}\n {% for column_name in column_dict %}\n {% set comment = column_dict[column_name]['description'] %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n comment on column {{ relation }}.{{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }} is {{ escaped_comment }};\n {% endfor %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres_escape_comment"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt_postgres.postgres__snapshot_merge_sql": {"unique_id": "macro.dbt_postgres.postgres__snapshot_merge_sql", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/product/dbt-core/plugins/postgres/dbt/include/postgres", "path": "macros/materializations/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshot_merge.sql", "name": "postgres__snapshot_merge_sql", "macro_sql": "{% macro postgres__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n update {{ target }}\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n from {{ source }} as DBT_INTERNAL_SOURCE\n where DBT_INTERNAL_SOURCE.dbt_scd_id::text = {{ target }}.dbt_scd_id::text\n and DBT_INTERNAL_SOURCE.dbt_change_type::text in ('update'::text, 'delete'::text)\n and {{ target }}.dbt_valid_to is null;\n\n insert into {{ target }} ({{ insert_cols_csv }})\n select {% for column in insert_cols -%}\n DBT_INTERNAL_SOURCE.{{ column }} {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n from {{ source }} as DBT_INTERNAL_SOURCE\n where DBT_INTERNAL_SOURCE.dbt_change_type::text = 'insert'::text;\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.statement": {"unique_id": "macro.dbt.statement", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/core.sql", "original_file_path": "macros/core.sql", "name": "statement", "macro_sql": "{% macro statement(name=None, fetch_result=False, auto_begin=True) -%}\n {%- if execute: -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- set res, table = adapter.execute(sql, auto_begin=auto_begin, fetch=fetch_result) -%}\n {%- if name is not none -%}\n {{ store_result(name, response=res, agate_table=table) }}\n {%- endif -%}\n\n {%- endif -%}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.noop_statement": {"unique_id": "macro.dbt.noop_statement", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/core.sql", "original_file_path": "macros/core.sql", "name": "noop_statement", "macro_sql": "{% macro noop_statement(name=None, message=None, code=None, rows_affected=None, res=None) -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_raw_result(name, message=message, code=code, rows_affected=rows_affected, agate_table=res) }}\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.get_test_sql": {"unique_id": "macro.dbt.get_test_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/test.sql", "original_file_path": "macros/materializations/test.sql", "name": "get_test_sql", "macro_sql": "{% macro get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n {{ adapter.dispatch('get_test_sql', 'dbt')(main_sql, fail_calc, warn_if, error_if, limit) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__get_test_sql": {"unique_id": "macro.dbt.default__get_test_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/test.sql", "original_file_path": "macros/materializations/test.sql", "name": "default__get_test_sql", "macro_sql": "{% macro default__get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n select\n {{ fail_calc }} as failures,\n {{ fail_calc }} {{ warn_if }} as should_warn,\n {{ fail_calc }} {{ error_if }} as should_error\n from (\n {{ main_sql }}\n {{ \"limit \" ~ limit if limit != none }}\n ) dbt_internal_test\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.materialization_test_default": {"unique_id": "macro.dbt.materialization_test_default", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/test.sql", "original_file_path": "macros/materializations/test.sql", "name": "materialization_test_default", "macro_sql": "\n\n{%- materialization test, default -%}\n\n {% set relations = [] %}\n\n {% if should_store_failures() %}\n\n {% set identifier = model['alias'] %}\n {% set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n {% set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database, type='table') -%} %}\n \n {% if old_relation %}\n {% do adapter.drop_relation(old_relation) %}\n {% endif %}\n \n {% call statement(auto_begin=True) %}\n {{ create_table_as(False, target_relation, sql) }}\n {% endcall %}\n \n {% do relations.append(target_relation) %}\n \n {% set main_sql %}\n select *\n from {{ target_relation }}\n {% endset %}\n \n {{ adapter.commit() }}\n \n {% else %}\n\n {% set main_sql = sql %}\n \n {% endif %}\n\n {% set limit = config.get('limit') %}\n {% set fail_calc = config.get('fail_calc') %}\n {% set warn_if = config.get('warn_if') %}\n {% set error_if = config.get('error_if') %}\n\n {% call statement('main', fetch_result=True) -%}\n\n {{ get_test_sql(main_sql, fail_calc, warn_if, error_if, limit)}}\n\n {%- endcall %}\n \n {{ return({'relations': relations}) }}\n\n{%- endmaterialization -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.should_store_failures", "macro.dbt.statement", "macro.dbt.create_table_as", "macro.dbt.get_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.run_hooks": {"unique_id": "macro.dbt.run_hooks", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/helpers.sql", "original_file_path": "macros/materializations/helpers.sql", "name": "run_hooks", "macro_sql": "{% macro run_hooks(hooks, inside_transaction=True) %}\n {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %}\n {% if not inside_transaction and loop.first %}\n {% call statement(auto_begin=inside_transaction) %}\n commit;\n {% endcall %}\n {% endif %}\n {% set rendered = render(hook.get('sql')) | trim %}\n {% if (rendered | length) > 0 %}\n {% call statement(auto_begin=inside_transaction) %}\n {{ rendered }}\n {% endcall %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.column_list": {"unique_id": "macro.dbt.column_list", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/helpers.sql", "original_file_path": "macros/materializations/helpers.sql", "name": "column_list", "macro_sql": "{% macro column_list(columns) %}\n {%- for col in columns %}\n {{ col.name }} {% if not loop.last %},{% endif %}\n {% endfor -%}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.column_list_for_create_table": {"unique_id": "macro.dbt.column_list_for_create_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/helpers.sql", "original_file_path": "macros/materializations/helpers.sql", "name": "column_list_for_create_table", "macro_sql": "{% macro column_list_for_create_table(columns) %}\n {%- for col in columns %}\n {{ col.name }} {{ col.data_type }} {%- if not loop.last %},{% endif %}\n {% endfor -%}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.make_hook_config": {"unique_id": "macro.dbt.make_hook_config", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/helpers.sql", "original_file_path": "macros/materializations/helpers.sql", "name": "make_hook_config", "macro_sql": "{% macro make_hook_config(sql, inside_transaction) %}\n {{ tojson({\"sql\": sql, \"transaction\": inside_transaction}) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.before_begin": {"unique_id": "macro.dbt.before_begin", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/helpers.sql", "original_file_path": "macros/materializations/helpers.sql", "name": "before_begin", "macro_sql": "{% macro before_begin(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.in_transaction": {"unique_id": "macro.dbt.in_transaction", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/helpers.sql", "original_file_path": "macros/materializations/helpers.sql", "name": "in_transaction", "macro_sql": "{% macro in_transaction(sql) %}\n {{ make_hook_config(sql, inside_transaction=True) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.after_commit": {"unique_id": "macro.dbt.after_commit", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/helpers.sql", "original_file_path": "macros/materializations/helpers.sql", "name": "after_commit", "macro_sql": "{% macro after_commit(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.drop_relation_if_exists": {"unique_id": "macro.dbt.drop_relation_if_exists", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/helpers.sql", "original_file_path": "macros/materializations/helpers.sql", "name": "drop_relation_if_exists", "macro_sql": "{% macro drop_relation_if_exists(relation) %}\n {% if relation is not none %}\n {{ adapter.drop_relation(relation) }}\n {% endif %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.load_relation": {"unique_id": "macro.dbt.load_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/helpers.sql", "original_file_path": "macros/materializations/helpers.sql", "name": "load_relation", "macro_sql": "{% macro load_relation(relation) %}\n {% do return(adapter.get_relation(\n database=relation.database,\n schema=relation.schema,\n identifier=relation.identifier\n )) -%}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.should_full_refresh": {"unique_id": "macro.dbt.should_full_refresh", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/helpers.sql", "original_file_path": "macros/materializations/helpers.sql", "name": "should_full_refresh", "macro_sql": "{% macro should_full_refresh() %}\n {% set config_full_refresh = config.get('full_refresh') %}\n {% if config_full_refresh is none %}\n {% set config_full_refresh = flags.FULL_REFRESH %}\n {% endif %}\n {% do return(config_full_refresh) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.should_store_failures": {"unique_id": "macro.dbt.should_store_failures", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/helpers.sql", "original_file_path": "macros/materializations/helpers.sql", "name": "should_store_failures", "macro_sql": "{% macro should_store_failures() %}\n {% set config_store_failures = config.get('store_failures') %}\n {% if config_store_failures is none %}\n {% set config_store_failures = flags.STORE_FAILURES %}\n {% endif %}\n {% do return(config_store_failures) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.snapshot_merge_sql": {"unique_id": "macro.dbt.snapshot_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshot/snapshot_merge.sql", "name": "snapshot_merge_sql", "macro_sql": "{% macro snapshot_merge_sql(target, source, insert_cols) -%}\n {{ adapter.dispatch('snapshot_merge_sql', 'dbt')(target, source, insert_cols) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__snapshot_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__snapshot_merge_sql": {"unique_id": "macro.dbt.default__snapshot_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshot/snapshot_merge.sql", "name": "default__snapshot_merge_sql", "macro_sql": "{% macro default__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on DBT_INTERNAL_SOURCE.dbt_scd_id = DBT_INTERNAL_DEST.dbt_scd_id\n\n when matched\n and DBT_INTERNAL_DEST.dbt_valid_to is null\n and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')\n then update\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n\n when not matched\n and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'\n then insert ({{ insert_cols_csv }})\n values ({{ insert_cols_csv }})\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.strategy_dispatch": {"unique_id": "macro.dbt.strategy_dispatch", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/strategies.sql", "original_file_path": "macros/materializations/snapshot/strategies.sql", "name": "strategy_dispatch", "macro_sql": "{% macro strategy_dispatch(name) -%}\n{% set original_name = name %}\n {% if '.' in name %}\n {% set package_name, name = name.split(\".\", 1) %}\n {% else %}\n {% set package_name = none %}\n {% endif %}\n\n {% if package_name is none %}\n {% set package_context = context %}\n {% elif package_name in context %}\n {% set package_context = context[package_name] %}\n {% else %}\n {% set error_msg %}\n Could not find package '{{package_name}}', called with '{{original_name}}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n\n {%- set search_name = 'snapshot_' ~ name ~ '_strategy' -%}\n\n {% if search_name not in package_context %}\n {% set error_msg %}\n The specified strategy macro '{{name}}' was not found in package '{{ package_name }}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n {{ return(package_context[search_name]) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.snapshot_hash_arguments": {"unique_id": "macro.dbt.snapshot_hash_arguments", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/strategies.sql", "original_file_path": "macros/materializations/snapshot/strategies.sql", "name": "snapshot_hash_arguments", "macro_sql": "{% macro snapshot_hash_arguments(args) -%}\n {{ adapter.dispatch('snapshot_hash_arguments', 'dbt')(args) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__snapshot_hash_arguments": {"unique_id": "macro.dbt.default__snapshot_hash_arguments", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/strategies.sql", "original_file_path": "macros/materializations/snapshot/strategies.sql", "name": "default__snapshot_hash_arguments", "macro_sql": "{% macro default__snapshot_hash_arguments(args) -%}\n md5({%- for arg in args -%}\n coalesce(cast({{ arg }} as varchar ), '')\n {% if not loop.last %} || '|' || {% endif %}\n {%- endfor -%})\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.snapshot_get_time": {"unique_id": "macro.dbt.snapshot_get_time", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/strategies.sql", "original_file_path": "macros/materializations/snapshot/strategies.sql", "name": "snapshot_get_time", "macro_sql": "{% macro snapshot_get_time() -%}\n {{ adapter.dispatch('snapshot_get_time', 'dbt')() }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__snapshot_get_time"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__snapshot_get_time": {"unique_id": "macro.dbt.default__snapshot_get_time", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/strategies.sql", "original_file_path": "macros/materializations/snapshot/strategies.sql", "name": "default__snapshot_get_time", "macro_sql": "{% macro default__snapshot_get_time() -%}\n {{ current_timestamp() }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.snapshot_timestamp_strategy": {"unique_id": "macro.dbt.snapshot_timestamp_strategy", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/strategies.sql", "original_file_path": "macros/materializations/snapshot/strategies.sql", "name": "snapshot_timestamp_strategy", "macro_sql": "{% macro snapshot_timestamp_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}\n {% set primary_key = config['unique_key'] %}\n {% set updated_at = config['updated_at'] %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %}\n\n {#/*\n The snapshot relation might not have an {{ updated_at }} value if the\n snapshot strategy is changed from `check` to `timestamp`. We\n should use a dbt-created column for the comparison in the snapshot\n table instead of assuming that the user-supplied {{ updated_at }}\n will be present in the historical data.\n\n See https://github.com/dbt-labs/dbt/issues/2350\n */ #}\n {% set row_changed_expr -%}\n ({{ snapshotted_rel }}.dbt_valid_from < {{ current_rel }}.{{ updated_at }})\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.snapshot_string_as_time": {"unique_id": "macro.dbt.snapshot_string_as_time", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/strategies.sql", "original_file_path": "macros/materializations/snapshot/strategies.sql", "name": "snapshot_string_as_time", "macro_sql": "{% macro snapshot_string_as_time(timestamp) -%}\n {{ adapter.dispatch('snapshot_string_as_time', 'dbt')(timestamp) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__snapshot_string_as_time"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__snapshot_string_as_time": {"unique_id": "macro.dbt.default__snapshot_string_as_time", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/strategies.sql", "original_file_path": "macros/materializations/snapshot/strategies.sql", "name": "default__snapshot_string_as_time", "macro_sql": "{% macro default__snapshot_string_as_time(timestamp) %}\n {% do exceptions.raise_not_implemented(\n 'snapshot_string_as_time macro not implemented for adapter '+adapter.type()\n ) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.snapshot_check_all_get_existing_columns": {"unique_id": "macro.dbt.snapshot_check_all_get_existing_columns", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/strategies.sql", "original_file_path": "macros/materializations/snapshot/strategies.sql", "name": "snapshot_check_all_get_existing_columns", "macro_sql": "{% macro snapshot_check_all_get_existing_columns(node, target_exists) -%}\n {%- set query_columns = get_columns_in_query(node['compiled_sql']) -%}\n {%- if not target_exists -%}\n {# no table yet -> return whatever the query does #}\n {{ return([false, query_columns]) }}\n {%- endif -%}\n {# handle any schema changes #}\n {%- set target_table = node.get('alias', node.get('name')) -%}\n {%- set target_relation = adapter.get_relation(database=node.database, schema=node.schema, identifier=target_table) -%}\n {%- set existing_cols = get_columns_in_query('select * from ' ~ target_relation) -%}\n {%- set ns = namespace() -%} {# handle for-loop scoping with a namespace #}\n {%- set ns.column_added = false -%}\n\n {%- set intersection = [] -%}\n {%- for col in query_columns -%}\n {%- if col in existing_cols -%}\n {%- do intersection.append(col) -%}\n {%- else -%}\n {% set ns.column_added = true %}\n {%- endif -%}\n {%- endfor -%}\n {{ return([ns.column_added, intersection]) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_columns_in_query"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.snapshot_check_strategy": {"unique_id": "macro.dbt.snapshot_check_strategy", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/strategies.sql", "original_file_path": "macros/materializations/snapshot/strategies.sql", "name": "snapshot_check_strategy", "macro_sql": "{% macro snapshot_check_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}\n {% set check_cols_config = config['check_cols'] %}\n {% set primary_key = config['unique_key'] %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %}\n \n {% set select_current_time -%}\n select {{ snapshot_get_time() }} as snapshot_start\n {%- endset %}\n\n {#-- don't access the column by name, to avoid dealing with casing issues on snowflake #}\n {%- set now = run_query(select_current_time)[0][0] -%}\n {% if now is none or now is undefined -%}\n {%- do exceptions.raise_compiler_error('Could not get a snapshot start time from the database') -%}\n {%- endif %}\n {% set updated_at = config.get('updated_at', snapshot_string_as_time(now)) %}\n\n {% set column_added = false %}\n\n {% if check_cols_config == 'all' %}\n {% set column_added, check_cols = snapshot_check_all_get_existing_columns(node, target_exists) %}\n {% elif check_cols_config is iterable and (check_cols_config | length) > 0 %}\n {% set check_cols = check_cols_config %}\n {% else %}\n {% do exceptions.raise_compiler_error(\"Invalid value for 'check_cols': \" ~ check_cols_config) %}\n {% endif %}\n\n {%- set row_changed_expr -%}\n (\n {%- if column_added -%}\n TRUE\n {%- else -%}\n {%- for col in check_cols -%}\n {{ snapshotted_rel }}.{{ col }} != {{ current_rel }}.{{ col }}\n or\n (\n (({{ snapshotted_rel }}.{{ col }} is null) and not ({{ current_rel }}.{{ col }} is null))\n or\n ((not {{ snapshotted_rel }}.{{ col }} is null) and ({{ current_rel }}.{{ col }} is null))\n )\n {%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n {%- endif -%}\n )\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.snapshot_get_time", "macro.dbt.run_query", "macro.dbt.snapshot_string_as_time", "macro.dbt.snapshot_check_all_get_existing_columns", "macro.dbt.snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.create_columns": {"unique_id": "macro.dbt.create_columns", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/snapshot.sql", "original_file_path": "macros/materializations/snapshot/snapshot.sql", "name": "create_columns", "macro_sql": "{% macro create_columns(relation, columns) %}\n {{ adapter.dispatch('create_columns', 'dbt')(relation, columns) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__create_columns"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__create_columns": {"unique_id": "macro.dbt.default__create_columns", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/snapshot.sql", "original_file_path": "macros/materializations/snapshot/snapshot.sql", "name": "default__create_columns", "macro_sql": "{% macro default__create_columns(relation, columns) %}\n {% for column in columns %}\n {% call statement() %}\n alter table {{ relation }} add column \"{{ column.name }}\" {{ column.data_type }};\n {% endcall %}\n {% endfor %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.post_snapshot": {"unique_id": "macro.dbt.post_snapshot", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/snapshot.sql", "original_file_path": "macros/materializations/snapshot/snapshot.sql", "name": "post_snapshot", "macro_sql": "{% macro post_snapshot(staging_relation) %}\n {{ adapter.dispatch('post_snapshot', 'dbt')(staging_relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__post_snapshot": {"unique_id": "macro.dbt.default__post_snapshot", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/snapshot.sql", "original_file_path": "macros/materializations/snapshot/snapshot.sql", "name": "default__post_snapshot", "macro_sql": "{% macro default__post_snapshot(staging_relation) %}\n {# no-op #}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.snapshot_staging_table": {"unique_id": "macro.dbt.snapshot_staging_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/snapshot.sql", "original_file_path": "macros/materializations/snapshot/snapshot.sql", "name": "snapshot_staging_table", "macro_sql": "{% macro snapshot_staging_table(strategy, source_sql, target_relation) -%}\n\n with snapshot_query as (\n\n {{ source_sql }}\n\n ),\n\n snapshotted_data as (\n\n select *,\n {{ strategy.unique_key }} as dbt_unique_key\n\n from {{ target_relation }}\n where dbt_valid_to is null\n\n ),\n\n insertions_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to,\n {{ strategy.scd_id }} as dbt_scd_id\n\n from snapshot_query\n ),\n\n updates_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n {{ strategy.updated_at }} as dbt_valid_to\n\n from snapshot_query\n ),\n\n {%- if strategy.invalidate_hard_deletes %}\n\n deletes_source_data as (\n\n select \n *,\n {{ strategy.unique_key }} as dbt_unique_key\n from snapshot_query\n ),\n {% endif %}\n\n insertions as (\n\n select\n 'insert' as dbt_change_type,\n source_data.*\n\n from insertions_source_data as source_data\n left outer join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where snapshotted_data.dbt_unique_key is null\n or (\n snapshotted_data.dbt_unique_key is not null\n and (\n {{ strategy.row_changed }}\n )\n )\n\n ),\n\n updates as (\n\n select\n 'update' as dbt_change_type,\n source_data.*,\n snapshotted_data.dbt_scd_id\n\n from updates_source_data as source_data\n join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where (\n {{ strategy.row_changed }}\n )\n )\n\n {%- if strategy.invalidate_hard_deletes -%}\n ,\n\n deletes as (\n \n select\n 'delete' as dbt_change_type,\n source_data.*,\n {{ snapshot_get_time() }} as dbt_valid_from,\n {{ snapshot_get_time() }} as dbt_updated_at,\n {{ snapshot_get_time() }} as dbt_valid_to,\n snapshotted_data.dbt_scd_id\n \n from snapshotted_data\n left join deletes_source_data as source_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where source_data.dbt_unique_key is null\n )\n {%- endif %}\n\n select * from insertions\n union all\n select * from updates\n {%- if strategy.invalidate_hard_deletes %}\n union all\n select * from deletes\n {%- endif %}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.snapshot_get_time"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.build_snapshot_table": {"unique_id": "macro.dbt.build_snapshot_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/snapshot.sql", "original_file_path": "macros/materializations/snapshot/snapshot.sql", "name": "build_snapshot_table", "macro_sql": "{% macro build_snapshot_table(strategy, sql) %}\n\n select *,\n {{ strategy.scd_id }} as dbt_scd_id,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to\n from (\n {{ sql }}\n ) sbq\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.get_or_create_relation": {"unique_id": "macro.dbt.get_or_create_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/snapshot.sql", "original_file_path": "macros/materializations/snapshot/snapshot.sql", "name": "get_or_create_relation", "macro_sql": "{% macro get_or_create_relation(database, schema, identifier, type) %}\n {%- set target_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n\n {% if target_relation %}\n {% do return([true, target_relation]) %}\n {% endif %}\n\n {%- set new_relation = api.Relation.create(\n database=database,\n schema=schema,\n identifier=identifier,\n type=type\n ) -%}\n {% do return([false, new_relation]) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.build_snapshot_staging_table": {"unique_id": "macro.dbt.build_snapshot_staging_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/snapshot.sql", "original_file_path": "macros/materializations/snapshot/snapshot.sql", "name": "build_snapshot_staging_table", "macro_sql": "{% macro build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% set tmp_relation = make_temp_relation(target_relation) %}\n\n {% set select = snapshot_staging_table(strategy, sql, target_relation) %}\n\n {% call statement('build_snapshot_staging_relation') %}\n {{ create_table_as(True, tmp_relation, select) }}\n {% endcall %}\n\n {% do return(tmp_relation) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.make_temp_relation", "macro.dbt.snapshot_staging_table", "macro.dbt.statement", "macro.dbt.create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.materialization_snapshot_default": {"unique_id": "macro.dbt.materialization_snapshot_default", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/snapshot/snapshot.sql", "original_file_path": "macros/materializations/snapshot/snapshot.sql", "name": "materialization_snapshot_default", "macro_sql": "{% materialization snapshot, default %}\n {%- set config = model['config'] -%}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n\n {% if not adapter.check_schema_exists(model.database, model.schema) %}\n {% do create_schema(model.database, model.schema) %}\n {% endif %}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=model.database,\n schema=model.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", config, target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_sql']) %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% else %}\n\n {{ adapter.valid_snapshot_target(target_relation) }}\n\n {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% endif %}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if not target_relation_exists %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if staging_table is defined %}\n {% do post_snapshot(staging_table) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.create_schema", "macro.dbt.get_or_create_relation", "macro.dbt.run_hooks", "macro.dbt.strategy_dispatch", "macro.dbt.build_snapshot_table", "macro.dbt.create_table_as", "macro.dbt.build_snapshot_staging_table", "macro.dbt.create_columns", "macro.dbt.snapshot_merge_sql", "macro.dbt.statement", "macro.dbt.persist_docs", "macro.dbt.create_indexes", "macro.dbt.post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.create_csv_table": {"unique_id": "macro.dbt.create_csv_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/seed/seed.sql", "original_file_path": "macros/materializations/seed/seed.sql", "name": "create_csv_table", "macro_sql": "{% macro create_csv_table(model, agate_table) -%}\n {{ adapter.dispatch('create_csv_table', 'dbt')(model, agate_table) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__create_csv_table": {"unique_id": "macro.dbt.default__create_csv_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/seed/seed.sql", "original_file_path": "macros/materializations/seed/seed.sql", "name": "default__create_csv_table", "macro_sql": "{% macro default__create_csv_table(model, agate_table) %}\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n\n {% set sql %}\n create table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.reset_csv_table": {"unique_id": "macro.dbt.reset_csv_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/seed/seed.sql", "original_file_path": "macros/materializations/seed/seed.sql", "name": "reset_csv_table", "macro_sql": "{% macro reset_csv_table(model, full_refresh, old_relation, agate_table) -%}\n {{ adapter.dispatch('reset_csv_table', 'dbt')(model, full_refresh, old_relation, agate_table) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__reset_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__reset_csv_table": {"unique_id": "macro.dbt.default__reset_csv_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/seed/seed.sql", "original_file_path": "macros/materializations/seed/seed.sql", "name": "default__reset_csv_table", "macro_sql": "{% macro default__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% set sql = \"\" %}\n {% if full_refresh %}\n {{ adapter.drop_relation(old_relation) }}\n {% set sql = create_csv_table(model, agate_table) %}\n {% else %}\n {{ adapter.truncate_relation(old_relation) }}\n {% set sql = \"truncate table \" ~ old_relation %}\n {% endif %}\n\n {{ return(sql) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.get_binding_char": {"unique_id": "macro.dbt.get_binding_char", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/seed/seed.sql", "original_file_path": "macros/materializations/seed/seed.sql", "name": "get_binding_char", "macro_sql": "{% macro get_binding_char() -%}\n {{ adapter.dispatch('get_binding_char', 'dbt')() }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__get_binding_char": {"unique_id": "macro.dbt.default__get_binding_char", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/seed/seed.sql", "original_file_path": "macros/materializations/seed/seed.sql", "name": "default__get_binding_char", "macro_sql": "{% macro default__get_binding_char() %}\n {{ return('%s') }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.get_batch_size": {"unique_id": "macro.dbt.get_batch_size", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/seed/seed.sql", "original_file_path": "macros/materializations/seed/seed.sql", "name": "get_batch_size", "macro_sql": "{% macro get_batch_size() -%}\n {{ return(adapter.dispatch('get_batch_size', 'dbt')()) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_batch_size"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__get_batch_size": {"unique_id": "macro.dbt.default__get_batch_size", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/seed/seed.sql", "original_file_path": "macros/materializations/seed/seed.sql", "name": "default__get_batch_size", "macro_sql": "{% macro default__get_batch_size() %}\n {{ return(10000) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.get_seed_column_quoted_csv": {"unique_id": "macro.dbt.get_seed_column_quoted_csv", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/seed/seed.sql", "original_file_path": "macros/materializations/seed/seed.sql", "name": "get_seed_column_quoted_csv", "macro_sql": "{% macro get_seed_column_quoted_csv(model, column_names) %}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote_seed_column(col, quote_seed_column)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.load_csv_rows": {"unique_id": "macro.dbt.load_csv_rows", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/seed/seed.sql", "original_file_path": "macros/materializations/seed/seed.sql", "name": "load_csv_rows", "macro_sql": "{% macro load_csv_rows(model, agate_table) -%}\n {{ adapter.dispatch('load_csv_rows', 'dbt')(model, agate_table) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__load_csv_rows"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__load_csv_rows": {"unique_id": "macro.dbt.default__load_csv_rows", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/seed/seed.sql", "original_file_path": "macros/materializations/seed/seed.sql", "name": "default__load_csv_rows", "macro_sql": "{% macro default__load_csv_rows(model, agate_table) %}\n\n {% set batch_size = get_batch_size() %}\n\n {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}\n {% set bindings = [] %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert into {{ this.render() }} ({{ cols_sql }}) values\n {% for row in chunk -%}\n ({%- for column in agate_table.column_names -%}\n {{ get_binding_char() }}\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_batch_size", "macro.dbt.get_seed_column_quoted_csv", "macro.dbt.get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.materialization_seed_default": {"unique_id": "macro.dbt.materialization_seed_default", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/seed/seed.sql", "original_file_path": "macros/materializations/seed/seed.sql", "name": "materialization_seed_default", "macro_sql": "{% materialization seed, default %}\n\n {%- set identifier = model['alias'] -%}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set agate_table = load_agate_table() -%}\n {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% set create_table_sql = \"\" %}\n {% if exists_as_view %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a view\".format(old_relation)) }}\n {% elif exists_as_table %}\n {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}\n {% else %}\n {% set create_table_sql = create_csv_table(model, agate_table) %}\n {% endif %}\n\n {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}\n {% set rows_affected = (agate_table.rows | length) %}\n {% set sql = load_csv_rows(model, agate_table) %}\n\n {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}\n {{ create_table_sql }};\n -- dbt seed --\n {{ sql }}\n {% endcall %}\n\n {% set target_relation = this.incorporate(type='table') %}\n {% do persist_docs(target_relation, model) %}\n\n {% if full_refresh_mode or not exists_as_table %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.run_hooks", "macro.dbt.reset_csv_table", "macro.dbt.create_csv_table", "macro.dbt.load_csv_rows", "macro.dbt.noop_statement", "macro.dbt.persist_docs", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.incremental_upsert": {"unique_id": "macro.dbt.incremental_upsert", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/incremental/helpers.sql", "original_file_path": "macros/materializations/incremental/helpers.sql", "name": "incremental_upsert", "macro_sql": "{% macro incremental_upsert(tmp_relation, target_relation, unique_key=none, statement_name=\"main\") %}\n \n {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}\n\n {%- if unique_key is not none -%}\n delete\n from {{ target_relation }}\n where ({{ unique_key }}) in (\n select ({{ unique_key }})\n from {{ tmp_relation }}\n );\n {%- endif %}\n\n insert into {{ target_relation }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ tmp_relation }}\n );\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.materialization_incremental_default": {"unique_id": "macro.dbt.materialization_incremental_default", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/incremental/incremental.sql", "original_file_path": "macros/materializations/incremental/incremental.sql", "name": "materialization_incremental_default", "macro_sql": "{% materialization incremental, default -%}\n\n {% set unique_key = config.get('unique_key') %}\n\n {% set target_relation = this.incorporate(type='table') %}\n {% set existing_relation = load_relation(this) %}\n {% set tmp_relation = make_temp_relation(target_relation) %}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {% set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') %}\n\n {% set tmp_identifier = model['name'] + '__dbt_tmp' %}\n {% set backup_identifier = model['name'] + \"__dbt_backup\" %}\n\n -- the intermediate_ and backup_ relations should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation. This has to happen before\n -- BEGIN, in a separate transaction\n {% set preexisting_intermediate_relation = adapter.get_relation(identifier=tmp_identifier, \n schema=schema,\n database=database) %} \n {% set preexisting_backup_relation = adapter.get_relation(identifier=backup_identifier,\n schema=schema,\n database=database) %}\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set to_drop = [] %}\n\n {# -- first check whether we want to full refresh for source view or config reasons #}\n {% set trigger_full_refresh = (full_refresh_mode or existing_relation.is_view) %}\n\n {% if existing_relation is none %}\n {% set build_sql = create_table_as(False, target_relation, sql) %}\n{% elif trigger_full_refresh %}\n {#-- Make sure the backup doesn't exist so we don't encounter issues with the rename below #}\n {% set tmp_identifier = model['name'] + '__dbt_tmp' %}\n {% set backup_identifier = model['name'] + '__dbt_backup' %}\n {% set intermediate_relation = existing_relation.incorporate(path={\"identifier\": tmp_identifier}) %}\n {% set backup_relation = existing_relation.incorporate(path={\"identifier\": backup_identifier}) %}\n\n {% set build_sql = create_table_as(False, intermediate_relation, sql) %}\n {% set need_swap = true %}\n {% do to_drop.append(backup_relation) %}\n {% else %}\n {% do run_query(create_table_as(True, tmp_relation, sql)) %}\n {% do adapter.expand_target_column_types(\n from_relation=tmp_relation,\n to_relation=target_relation) %}\n {% do process_schema_changes(on_schema_change, tmp_relation, existing_relation) %}\n {% set build_sql = incremental_upsert(tmp_relation, target_relation, unique_key=unique_key) %}\n \n {% endif %}\n\n {% call statement(\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% if need_swap %} \n {% do adapter.rename_relation(target_relation, backup_relation) %} \n {% do adapter.rename_relation(intermediate_relation, target_relation) %} \n {% endif %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if existing_relation is none or existing_relation.is_view or should_full_refresh() %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {% do adapter.commit() %}\n\n {% for rel in to_drop %}\n {% do adapter.drop_relation(rel) %}\n {% endfor %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.test.load_relation", "macro.dbt.make_temp_relation", "macro.dbt.should_full_refresh", "macro.dbt.incremental_validate_on_schema_change", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks", "macro.dbt.create_table_as", "macro.dbt.run_query", "macro.dbt.process_schema_changes", "macro.dbt.incremental_upsert", "macro.dbt.statement", "macro.dbt.persist_docs", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.incremental_validate_on_schema_change": {"unique_id": "macro.dbt.incremental_validate_on_schema_change", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/incremental/on_schema_change.sql", "name": "incremental_validate_on_schema_change", "macro_sql": "{% macro incremental_validate_on_schema_change(on_schema_change, default='ignore') %}\n \n {% if on_schema_change not in ['sync_all_columns', 'append_new_columns', 'fail', 'ignore'] %}\n \n {% set log_message = 'Invalid value for on_schema_change (%s) specified. Setting default value of %s.' % (on_schema_change, default) %}\n {% do log(log_message) %}\n \n {{ return(default) }}\n\n {% else %}\n\n {{ return(on_schema_change) }}\n \n {% endif %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.diff_columns": {"unique_id": "macro.dbt.diff_columns", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/incremental/on_schema_change.sql", "name": "diff_columns", "macro_sql": "{% macro diff_columns(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% set source_names = source_columns | map(attribute = 'column') | list %}\n {% set target_names = target_columns | map(attribute = 'column') | list %}\n \n {# --check whether the name attribute exists in the target - this does not perform a data type check #}\n {% for sc in source_columns %}\n {% if sc.name not in target_names %}\n {{ result.append(sc) }}\n {% endif %}\n {% endfor %}\n \n {{ return(result) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.diff_column_data_types": {"unique_id": "macro.dbt.diff_column_data_types", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/incremental/on_schema_change.sql", "name": "diff_column_data_types", "macro_sql": "{% macro diff_column_data_types(source_columns, target_columns) %}\n \n {% set result = [] %}\n {% for sc in source_columns %}\n {% set tc = target_columns | selectattr(\"name\", \"equalto\", sc.name) | list | first %}\n {% if tc %}\n {% if sc.data_type != tc.data_type %}\n {{ result.append( { 'column_name': tc.name, 'new_type': sc.data_type } ) }} \n {% endif %}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.check_for_schema_changes": {"unique_id": "macro.dbt.check_for_schema_changes", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/incremental/on_schema_change.sql", "name": "check_for_schema_changes", "macro_sql": "{% macro check_for_schema_changes(source_relation, target_relation) %}\n \n {% set schema_changed = False %}\n \n {%- set source_columns = adapter.get_columns_in_relation(source_relation) -%}\n {%- set target_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set source_not_in_target = diff_columns(source_columns, target_columns) -%}\n {%- set target_not_in_source = diff_columns(target_columns, source_columns) -%}\n \n {% set new_target_types = diff_column_data_types(source_columns, target_columns) %}\n\n {% if source_not_in_target != [] %}\n {% set schema_changed = True %}\n {% elif target_not_in_source != [] or new_target_types != [] %}\n {% set schema_changed = True %}\n {% elif new_target_types != [] %}\n {% set schema_changed = True %}\n {% endif %}\n \n {% set changes_dict = {\n 'schema_changed': schema_changed,\n 'source_not_in_target': source_not_in_target,\n 'target_not_in_source': target_not_in_source,\n 'new_target_types': new_target_types\n } %}\n\n {% set msg %}\n In {{ target_relation }}:\n Schema changed: {{ schema_changed }}\n Source columns not in target: {{ source_not_in_target }}\n Target columns not in source: {{ target_not_in_source }}\n New column types: {{ new_target_types }}\n {% endset %}\n \n {% do log(msg) %}\n\n {{ return(changes_dict) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.diff_columns", "macro.dbt.diff_column_data_types"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.sync_column_schemas": {"unique_id": "macro.dbt.sync_column_schemas", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/incremental/on_schema_change.sql", "name": "sync_column_schemas", "macro_sql": "{% macro sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n \n {%- set add_to_target_arr = schema_changes_dict['source_not_in_target'] -%}\n\n {%- if on_schema_change == 'append_new_columns'-%}\n {%- if add_to_target_arr | length > 0 -%}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, none) -%}\n {%- endif -%}\n \n {% elif on_schema_change == 'sync_all_columns' %}\n {%- set remove_from_target_arr = schema_changes_dict['target_not_in_source'] -%}\n {%- set new_target_types = schema_changes_dict['new_target_types'] -%}\n \n {% if add_to_target_arr | length > 0 or remove_from_target_arr | length > 0 %} \n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, remove_from_target_arr) -%}\n {% endif %}\n\n {% if new_target_types != [] %}\n {% for ntt in new_target_types %}\n {% set column_name = ntt['column_name'] %}\n {% set new_type = ntt['new_type'] %}\n {% do alter_column_type(target_relation, column_name, new_type) %}\n {% endfor %}\n {% endif %}\n \n {% endif %}\n\n {% set schema_change_message %}\n In {{ target_relation }}:\n Schema change approach: {{ on_schema_change }}\n Columns added: {{ add_to_target_arr }}\n Columns removed: {{ remove_from_target_arr }}\n Data types changed: {{ new_target_types }}\n {% endset %}\n \n {% do log(schema_change_message) %}\n \n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.alter_relation_add_remove_columns", "macro.dbt.alter_column_type"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.process_schema_changes": {"unique_id": "macro.dbt.process_schema_changes", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/incremental/on_schema_change.sql", "name": "process_schema_changes", "macro_sql": "{% macro process_schema_changes(on_schema_change, source_relation, target_relation) %}\n \n {% if on_schema_change != 'ignore' %}\n \n {% set schema_changes_dict = check_for_schema_changes(source_relation, target_relation) %}\n \n {% if schema_changes_dict['schema_changed'] %}\n \n {% if on_schema_change == 'fail' %}\n \n {% set fail_msg %}\n The source and target schemas on this incremental model are out of sync!\n They can be reconciled in several ways: \n - set the `on_schema_change` config to either append_new_columns or sync_all_columns, depending on your situation.\n - Re-run the incremental model with `full_refresh: True` to update the target schema.\n - update the schema manually and re-run the process.\n {% endset %}\n \n {% do exceptions.raise_compiler_error(fail_msg) %}\n \n {# -- unless we ignore, run the sync operation per the config #}\n {% else %}\n \n {% do sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n \n {% endif %}\n \n {% endif %}\n \n {% endif %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.check_for_schema_changes", "macro.dbt.sync_column_schemas"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.get_merge_sql": {"unique_id": "macro.dbt.get_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/common/merge.sql", "original_file_path": "macros/materializations/common/merge.sql", "name": "get_merge_sql", "macro_sql": "{% macro get_merge_sql(target, source, unique_key, dest_columns, predicates=none) -%}\n {{ adapter.dispatch('get_merge_sql', 'dbt')(target, source, unique_key, dest_columns, predicates) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.get_delete_insert_merge_sql": {"unique_id": "macro.dbt.get_delete_insert_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/common/merge.sql", "original_file_path": "macros/materializations/common/merge.sql", "name": "get_delete_insert_merge_sql", "macro_sql": "{% macro get_delete_insert_merge_sql(target, source, unique_key, dest_columns) -%}\n {{ adapter.dispatch('get_delete_insert_merge_sql', 'dbt')(target, source, unique_key, dest_columns) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_delete_insert_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.get_insert_overwrite_merge_sql": {"unique_id": "macro.dbt.get_insert_overwrite_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/common/merge.sql", "original_file_path": "macros/materializations/common/merge.sql", "name": "get_insert_overwrite_merge_sql", "macro_sql": "{% macro get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header=false) -%}\n {{ adapter.dispatch('get_insert_overwrite_merge_sql', 'dbt')(target, source, dest_columns, predicates, include_sql_header) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_insert_overwrite_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__get_merge_sql": {"unique_id": "macro.dbt.default__get_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/common/merge.sql", "original_file_path": "macros/materializations/common/merge.sql", "name": "default__get_merge_sql", "macro_sql": "{% macro default__get_merge_sql(target, source, unique_key, dest_columns, predicates) -%}\n {%- set predicates = [] if predicates is none else [] + predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set update_columns = config.get('merge_update_columns', default = dest_columns | map(attribute=\"quoted\") | list) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {% if unique_key %}\n {% set unique_key_match %}\n DBT_INTERNAL_SOURCE.{{ unique_key }} = DBT_INTERNAL_DEST.{{ unique_key }}\n {% endset %}\n {% do predicates.append(unique_key_match) %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n {{ sql_header if sql_header is not none }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{ predicates | join(' and ') }}\n\n {% if unique_key %}\n when matched then update set\n {% for column_name in update_columns -%}\n {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}\n {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n {% endif %}\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.get_quoted_csv": {"unique_id": "macro.dbt.get_quoted_csv", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/common/merge.sql", "original_file_path": "macros/materializations/common/merge.sql", "name": "get_quoted_csv", "macro_sql": "{% macro get_quoted_csv(column_names) %}\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote(col)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.common_get_delete_insert_merge_sql": {"unique_id": "macro.dbt.common_get_delete_insert_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/common/merge.sql", "original_file_path": "macros/materializations/common/merge.sql", "name": "common_get_delete_insert_merge_sql", "macro_sql": "{% macro common_get_delete_insert_merge_sql(target, source, unique_key, dest_columns) -%}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n {% if unique_key is not none %}\n delete from {{ target }}\n where ({{ unique_key }}) in (\n select ({{ unique_key }})\n from {{ source }}\n );\n {% endif %}\n\n insert into {{ target }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ source }}\n )\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__get_delete_insert_merge_sql": {"unique_id": "macro.dbt.default__get_delete_insert_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/common/merge.sql", "original_file_path": "macros/materializations/common/merge.sql", "name": "default__get_delete_insert_merge_sql", "macro_sql": "{% macro default__get_delete_insert_merge_sql(target, source, unique_key, dest_columns) -%}\n {{ common_get_delete_insert_merge_sql(target, source, unique_key, dest_columns) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.common_get_delete_insert_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__get_insert_overwrite_merge_sql": {"unique_id": "macro.dbt.default__get_insert_overwrite_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/common/merge.sql", "original_file_path": "macros/materializations/common/merge.sql", "name": "default__get_insert_overwrite_merge_sql", "macro_sql": "{% macro default__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) -%}\n {%- set predicates = [] if predicates is none else [] + predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none and include_sql_header }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on FALSE\n\n when not matched by source\n {% if predicates %} and {{ predicates | join(' and ') }} {% endif %}\n then delete\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.materialization_table_default": {"unique_id": "macro.dbt.materialization_table_default", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/table/table.sql", "original_file_path": "macros/materializations/table/table.sql", "name": "materialization_table_default", "macro_sql": "{% materialization table, default %}\n {%- set identifier = model['alias'] -%}\n {%- set tmp_identifier = model['name'] + '__dbt_tmp' -%}\n {%- set backup_identifier = model['name'] + '__dbt_backup' -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set target_relation = api.Relation.create(identifier=identifier,\n schema=schema,\n database=database,\n type='table') -%}\n {%- set intermediate_relation = api.Relation.create(identifier=tmp_identifier,\n schema=schema,\n database=database,\n type='table') -%}\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = adapter.get_relation(identifier=tmp_identifier, \n schema=schema,\n database=database) -%}\n /*\n See ../view/view.sql for more information about this relation.\n */\n {%- set backup_relation_type = 'table' if old_relation is none else old_relation.type -%}\n {%- set backup_relation = api.Relation.create(identifier=backup_identifier,\n schema=schema,\n database=database,\n type=backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = adapter.get_relation(identifier=backup_identifier,\n schema=schema,\n database=database) -%}\n\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ create_table_as(False, intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n {% if old_relation is not none %}\n {{ adapter.rename_relation(old_relation, backup_relation) }}\n {% endif %}\n\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% do create_indexes(target_relation) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {% do persist_docs(target_relation, model) %}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n -- finally, drop the existing/backup relation after the commit\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n{% endmaterialization %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.create_table_as", "macro.dbt.create_indexes", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.materialization_view_default": {"unique_id": "macro.dbt.materialization_view_default", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/view/view.sql", "original_file_path": "macros/materializations/view/view.sql", "name": "materialization_view_default", "macro_sql": "{%- materialization view, default -%}\n\n {%- set identifier = model['alias'] -%}\n {%- set tmp_identifier = model['name'] + '__dbt_tmp' -%}\n {%- set backup_identifier = model['name'] + '__dbt_backup' -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set target_relation = api.Relation.create(identifier=identifier, schema=schema, database=database,\n type='view') -%}\n {%- set intermediate_relation = api.Relation.create(identifier=tmp_identifier,\n schema=schema, database=database, type='view') -%}\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = adapter.get_relation(identifier=tmp_identifier, \n schema=schema,\n database=database) -%}\n /*\n This relation (probably) doesn't exist yet. If it does exist, it's a leftover from\n a previous run, and we're going to try to drop it immediately. At the end of this\n materialization, we're going to rename the \"old_relation\" to this identifier,\n and then we're going to drop it. In order to make sure we run the correct one of:\n - drop view ...\n - drop table ...\n\n We need to set the type of this relation to be the type of the old_relation, if it exists,\n or else \"view\" as a sane default if it does not. Note that if the old_relation does not\n exist, then there is nothing to move out of the way and subsequentally drop. In that case,\n this relation will be effectively unused.\n */\n {%- set backup_relation_type = 'view' if old_relation is none else old_relation.type -%}\n {%- set backup_relation = api.Relation.create(identifier=backup_identifier,\n schema=schema, database=database,\n type=backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = adapter.get_relation(identifier=backup_identifier,\n schema=schema,\n database=database) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ create_view_as(intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n -- move the existing view out of the way\n {% if old_relation is not none %}\n {{ adapter.rename_relation(old_relation, backup_relation) }}\n {% endif %}\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.create_view_as", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.handle_existing_table": {"unique_id": "macro.dbt.handle_existing_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/view/create_or_replace_view.sql", "original_file_path": "macros/materializations/view/create_or_replace_view.sql", "name": "handle_existing_table", "macro_sql": "{% macro handle_existing_table(full_refresh, old_relation) %}\n {{ adapter.dispatch('handle_existing_table', 'dbt')(full_refresh, old_relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__handle_existing_table"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__handle_existing_table": {"unique_id": "macro.dbt.default__handle_existing_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/view/create_or_replace_view.sql", "original_file_path": "macros/materializations/view/create_or_replace_view.sql", "name": "default__handle_existing_table", "macro_sql": "{% macro default__handle_existing_table(full_refresh, old_relation) %}\n {{ log(\"Dropping relation \" ~ old_relation ~ \" because it is of type \" ~ old_relation.type) }}\n {{ adapter.drop_relation(old_relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.create_or_replace_view": {"unique_id": "macro.dbt.create_or_replace_view", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/materializations/view/create_or_replace_view.sql", "original_file_path": "macros/materializations/view/create_or_replace_view.sql", "name": "create_or_replace_view", "macro_sql": "{% macro create_or_replace_view() %}\n {%- set identifier = model['alias'] -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database,\n type='view') -%}\n\n {{ run_hooks(pre_hooks) }}\n\n -- If there's a table with the same name and we weren't told to full refresh,\n -- that's an error. If we were told to full refresh, drop it. This behavior differs\n -- for Snowflake and BigQuery, so multiple dispatch is used.\n {%- if old_relation is not none and old_relation.is_table -%}\n {{ handle_existing_table(should_full_refresh(), old_relation) }}\n {%- endif -%}\n\n -- build model\n {% call statement('main') -%}\n {{ create_view_as(target_relation, sql) }}\n {%- endcall %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.handle_existing_table", "macro.dbt.should_full_refresh", "macro.dbt.statement", "macro.dbt.create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.generate_alias_name": {"unique_id": "macro.dbt.generate_alias_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/etc/get_custom_alias.sql", "original_file_path": "macros/etc/get_custom_alias.sql", "name": "generate_alias_name", "macro_sql": "{% macro generate_alias_name(custom_alias_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_alias_name', 'dbt')(custom_alias_name, node)) %}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__generate_alias_name"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__generate_alias_name": {"unique_id": "macro.dbt.default__generate_alias_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/etc/get_custom_alias.sql", "original_file_path": "macros/etc/get_custom_alias.sql", "name": "default__generate_alias_name", "macro_sql": "{% macro default__generate_alias_name(custom_alias_name=none, node=none) -%}\n\n {%- if custom_alias_name is none -%}\n\n {{ node.name }}\n\n {%- else -%}\n\n {{ custom_alias_name | trim }}\n\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.run_query": {"unique_id": "macro.dbt.run_query", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/etc/query.sql", "original_file_path": "macros/etc/query.sql", "name": "run_query", "macro_sql": "{% macro run_query(sql) %}\n {% call statement(\"run_query_statement\", fetch_result=true, auto_begin=false) %}\n {{ sql }}\n {% endcall %}\n\n {% do return(load_result(\"run_query_statement\").table) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.is_incremental": {"unique_id": "macro.dbt.is_incremental", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/etc/is_incremental.sql", "original_file_path": "macros/etc/is_incremental.sql", "name": "is_incremental", "macro_sql": "{% macro is_incremental() %}\n {#-- do not run introspective queries in parsing #}\n {% if not execute %}\n {{ return(False) }}\n {% else %}\n {% set relation = adapter.get_relation(this.database, this.schema, this.table) %}\n {{ return(relation is not none\n and relation.type == 'table'\n and model.config.materialized == 'incremental'\n and not should_full_refresh()) }}\n {% endif %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.should_full_refresh"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.convert_datetime": {"unique_id": "macro.dbt.convert_datetime", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "name": "convert_datetime", "macro_sql": "{% macro convert_datetime(date_str, date_fmt) %}\n\n {% set error_msg -%}\n The provided partition date '{{ date_str }}' does not match the expected format '{{ date_fmt }}'\n {%- endset %}\n\n {% set res = try_or_compiler_error(error_msg, modules.datetime.datetime.strptime, date_str.strip(), date_fmt) %}\n {{ return(res) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.dates_in_range": {"unique_id": "macro.dbt.dates_in_range", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "name": "dates_in_range", "macro_sql": "{% macro dates_in_range(start_date_str, end_date_str=none, in_fmt=\"%Y%m%d\", out_fmt=\"%Y%m%d\") %}\n {% set end_date_str = start_date_str if end_date_str is none else end_date_str %}\n\n {% set start_date = convert_datetime(start_date_str, in_fmt) %}\n {% set end_date = convert_datetime(end_date_str, in_fmt) %}\n\n {% set day_count = (end_date - start_date).days %}\n {% if day_count < 0 %}\n {% set msg -%}\n Partition start date is after the end date ({{ start_date }}, {{ end_date }})\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg, model) }}\n {% endif %}\n\n {% set date_list = [] %}\n {% for i in range(0, day_count + 1) %}\n {% set the_date = (modules.datetime.timedelta(days=i) + start_date) %}\n {% if not out_fmt %}\n {% set _ = date_list.append(the_date) %}\n {% else %}\n {% set _ = date_list.append(the_date.strftime(out_fmt)) %}\n {% endif %}\n {% endfor %}\n\n {{ return(date_list) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.convert_datetime"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.partition_range": {"unique_id": "macro.dbt.partition_range", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "name": "partition_range", "macro_sql": "{% macro partition_range(raw_partition_date, date_fmt='%Y%m%d') %}\n {% set partition_range = (raw_partition_date | string).split(\",\") %}\n\n {% if (partition_range | length) == 1 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = none %}\n {% elif (partition_range | length) == 2 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = partition_range[1] %}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Invalid partition time. Expected format: {Start Date}[,{End Date}]. Got: \" ~ raw_partition_date) }}\n {% endif %}\n\n {{ return(dates_in_range(start_date, end_date, in_fmt=date_fmt)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.dates_in_range"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.py_current_timestring": {"unique_id": "macro.dbt.py_current_timestring", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "name": "py_current_timestring", "macro_sql": "{% macro py_current_timestring() %}\n {% set dt = modules.datetime.datetime.now() %}\n {% do return(dt.strftime(\"%Y%m%d%H%M%S%f\")) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.get_where_subquery": {"unique_id": "macro.dbt.get_where_subquery", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/etc/where_subquery.sql", "original_file_path": "macros/etc/where_subquery.sql", "name": "get_where_subquery", "macro_sql": "{% macro get_where_subquery(relation) -%}\n {% do return(adapter.dispatch('get_where_subquery', 'dbt')(relation)) %}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_where_subquery"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__get_where_subquery": {"unique_id": "macro.dbt.default__get_where_subquery", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/etc/where_subquery.sql", "original_file_path": "macros/etc/where_subquery.sql", "name": "default__get_where_subquery", "macro_sql": "{% macro default__get_where_subquery(relation) -%}\n {% set where = config.get('where', '') %}\n {% if where %}\n {%- set filtered -%}\n (select * from {{ relation }} where {{ where }}) dbt_subquery\n {%- endset -%}\n {% do return(filtered) %}\n {%- else -%}\n {% do return(relation) %}\n {%- endif -%}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.generate_schema_name": {"unique_id": "macro.dbt.generate_schema_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/etc/get_custom_schema.sql", "original_file_path": "macros/etc/get_custom_schema.sql", "name": "generate_schema_name", "macro_sql": "{% macro generate_schema_name(custom_schema_name, node) -%}\n {{ return(adapter.dispatch('generate_schema_name', 'dbt')(custom_schema_name, node)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__generate_schema_name"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__generate_schema_name": {"unique_id": "macro.dbt.default__generate_schema_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/etc/get_custom_schema.sql", "original_file_path": "macros/etc/get_custom_schema.sql", "name": "default__generate_schema_name", "macro_sql": "{% macro default__generate_schema_name(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if custom_schema_name is none -%}\n\n {{ default_schema }}\n\n {%- else -%}\n\n {{ default_schema }}_{{ custom_schema_name | trim }}\n\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.generate_schema_name_for_env": {"unique_id": "macro.dbt.generate_schema_name_for_env", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/etc/get_custom_schema.sql", "original_file_path": "macros/etc/get_custom_schema.sql", "name": "generate_schema_name_for_env", "macro_sql": "{% macro generate_schema_name_for_env(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if target.name == 'prod' and custom_schema_name is not none -%}\n\n {{ custom_schema_name | trim }}\n\n {%- else -%}\n\n {{ default_schema }}\n\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.generate_database_name": {"unique_id": "macro.dbt.generate_database_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/etc/get_custom_database.sql", "original_file_path": "macros/etc/get_custom_database.sql", "name": "generate_database_name", "macro_sql": "{% macro generate_database_name(custom_database_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_database_name', 'dbt')(custom_database_name, node)) %}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__generate_database_name"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__generate_database_name": {"unique_id": "macro.dbt.default__generate_database_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/etc/get_custom_database.sql", "original_file_path": "macros/etc/get_custom_database.sql", "name": "default__generate_database_name", "macro_sql": "{% macro default__generate_database_name(custom_database_name=none, node=none) -%}\n {%- set default_database = target.database -%}\n {%- if custom_database_name is none -%}\n\n {{ default_database }}\n\n {%- else -%}\n\n {{ custom_database_name }}\n\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.get_columns_in_query": {"unique_id": "macro.dbt.get_columns_in_query", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "get_columns_in_query", "macro_sql": "{% macro get_columns_in_query(select_sql) -%}\n {{ return(adapter.dispatch('get_columns_in_query', 'dbt')(select_sql)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_columns_in_query"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__get_columns_in_query": {"unique_id": "macro.dbt.default__get_columns_in_query", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__get_columns_in_query", "macro_sql": "{% macro default__get_columns_in_query(select_sql) %}\n {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%}\n select * from (\n {{ select_sql }}\n ) as __dbt_sbq\n where false\n limit 0\n {% endcall %}\n\n {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.create_schema": {"unique_id": "macro.dbt.create_schema", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "create_schema", "macro_sql": "{% macro create_schema(relation) -%}\n {{ adapter.dispatch('create_schema', 'dbt')(relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__create_schema"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__create_schema": {"unique_id": "macro.dbt.default__create_schema", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__create_schema", "macro_sql": "{% macro default__create_schema(relation) -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier() }}\n {% endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.drop_schema": {"unique_id": "macro.dbt.drop_schema", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "drop_schema", "macro_sql": "{% macro drop_schema(relation) -%}\n {{ adapter.dispatch('drop_schema', 'dbt')(relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__drop_schema"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__drop_schema": {"unique_id": "macro.dbt.default__drop_schema", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__drop_schema", "macro_sql": "{% macro default__drop_schema(relation) -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier() }} cascade\n {% endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.create_table_as": {"unique_id": "macro.dbt.create_table_as", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "create_table_as", "macro_sql": "{% macro create_table_as(temporary, relation, sql) -%}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, sql) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__create_table_as": {"unique_id": "macro.dbt.default__create_table_as", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__create_table_as", "macro_sql": "{% macro default__create_table_as(temporary, relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary: -%}temporary{%- endif %} table\n {{ relation.include(database=(not temporary), schema=(not temporary)) }}\n as (\n {{ sql }}\n );\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.get_create_index_sql": {"unique_id": "macro.dbt.get_create_index_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "get_create_index_sql", "macro_sql": "{% macro get_create_index_sql(relation, index_dict) -%}\n {{ return(adapter.dispatch('get_create_index_sql', 'dbt')(relation, index_dict)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_create_index_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__get_create_index_sql": {"unique_id": "macro.dbt.default__get_create_index_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__get_create_index_sql", "macro_sql": "{% macro default__get_create_index_sql(relation, index_dict) -%}\n {% do return(None) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.create_indexes": {"unique_id": "macro.dbt.create_indexes", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "create_indexes", "macro_sql": "{% macro create_indexes(relation) -%}\n {{ adapter.dispatch('create_indexes', 'dbt')(relation) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__create_indexes": {"unique_id": "macro.dbt.default__create_indexes", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__create_indexes", "macro_sql": "{% macro default__create_indexes(relation) -%}\n {%- set _indexes = config.get('indexes', default=[]) -%}\n\n {% for _index_dict in _indexes %}\n {% set create_index_sql = get_create_index_sql(relation, _index_dict) %}\n {% if create_index_sql %}\n {% do run_query(create_index_sql) %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_create_index_sql", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.create_view_as": {"unique_id": "macro.dbt.create_view_as", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "create_view_as", "macro_sql": "{% macro create_view_as(relation, sql) -%}\n {{ adapter.dispatch('create_view_as', 'dbt')(relation, sql) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__create_view_as": {"unique_id": "macro.dbt.default__create_view_as", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__create_view_as", "macro_sql": "{% macro default__create_view_as(relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n create view {{ relation }} as (\n {{ sql }}\n );\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.get_catalog": {"unique_id": "macro.dbt.get_catalog", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "get_catalog", "macro_sql": "{% macro get_catalog(information_schema, schemas) -%}\n {{ return(adapter.dispatch('get_catalog', 'dbt')(information_schema, schemas)) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_catalog"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__get_catalog": {"unique_id": "macro.dbt.default__get_catalog", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__get_catalog", "macro_sql": "{% macro default__get_catalog(information_schema, schemas) -%}\n\n {% set typename = adapter.type() %}\n {% set msg -%}\n get_catalog not implemented for {{ typename }}\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.get_columns_in_relation": {"unique_id": "macro.dbt.get_columns_in_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "get_columns_in_relation", "macro_sql": "{% macro get_columns_in_relation(relation) -%}\n {{ return(adapter.dispatch('get_columns_in_relation', 'dbt')(relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_columns_in_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.sql_convert_columns_in_relation": {"unique_id": "macro.dbt.sql_convert_columns_in_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "sql_convert_columns_in_relation", "macro_sql": "{% macro sql_convert_columns_in_relation(table) -%}\n {% set columns = [] %}\n {% for row in table %}\n {% do columns.append(api.Column(*row)) %}\n {% endfor %}\n {{ return(columns) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__get_columns_in_relation": {"unique_id": "macro.dbt.default__get_columns_in_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__get_columns_in_relation", "macro_sql": "{% macro default__get_columns_in_relation(relation) -%}\n {{ exceptions.raise_not_implemented(\n 'get_columns_in_relation macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.alter_column_type": {"unique_id": "macro.dbt.alter_column_type", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "alter_column_type", "macro_sql": "{% macro alter_column_type(relation, column_name, new_column_type) -%}\n {{ return(adapter.dispatch('alter_column_type', 'dbt')(relation, column_name, new_column_type)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__alter_column_type"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.alter_column_comment": {"unique_id": "macro.dbt.alter_column_comment", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "alter_column_comment", "macro_sql": "{% macro alter_column_comment(relation, column_dict) -%}\n {{ return(adapter.dispatch('alter_column_comment', 'dbt')(relation, column_dict)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__alter_column_comment": {"unique_id": "macro.dbt.default__alter_column_comment", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__alter_column_comment", "macro_sql": "{% macro default__alter_column_comment(relation, column_dict) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_column_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.alter_relation_comment": {"unique_id": "macro.dbt.alter_relation_comment", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "alter_relation_comment", "macro_sql": "{% macro alter_relation_comment(relation, relation_comment) -%}\n {{ return(adapter.dispatch('alter_relation_comment', 'dbt')(relation, relation_comment)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__alter_relation_comment"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__alter_relation_comment": {"unique_id": "macro.dbt.default__alter_relation_comment", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__alter_relation_comment", "macro_sql": "{% macro default__alter_relation_comment(relation, relation_comment) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_relation_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.persist_docs": {"unique_id": "macro.dbt.persist_docs", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "persist_docs", "macro_sql": "{% macro persist_docs(relation, model, for_relation=true, for_columns=true) -%}\n {{ return(adapter.dispatch('persist_docs', 'dbt')(relation, model, for_relation, for_columns)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__persist_docs": {"unique_id": "macro.dbt.default__persist_docs", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__persist_docs", "macro_sql": "{% macro default__persist_docs(relation, model, for_relation, for_columns) -%}\n {% if for_relation and config.persist_relation_docs() and model.description %}\n {% do run_query(alter_relation_comment(relation, model.description)) %}\n {% endif %}\n\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {% do run_query(alter_column_comment(relation, model.columns)) %}\n {% endif %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.run_query", "macro.dbt.alter_relation_comment", "macro.dbt.alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__alter_column_type": {"unique_id": "macro.dbt.default__alter_column_type", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__alter_column_type", "macro_sql": "{% macro default__alter_column_type(relation, column_name, new_column_type) -%}\n {#\n 1. Create a new column (w/ temp name and correct type)\n 2. Copy data over to it\n 3. Drop the existing column (cascade!)\n 4. Rename the new column to existing column\n #}\n {%- set tmp_column = column_name + \"__dbt_alter\" -%}\n\n {% call statement('alter_column_type') %}\n alter table {{ relation }} add column {{ adapter.quote(tmp_column) }} {{ new_column_type }};\n update {{ relation }} set {{ adapter.quote(tmp_column) }} = {{ adapter.quote(column_name) }};\n alter table {{ relation }} drop column {{ adapter.quote(column_name) }} cascade;\n alter table {{ relation }} rename column {{ adapter.quote(tmp_column) }} to {{ adapter.quote(column_name) }}\n {% endcall %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.drop_relation": {"unique_id": "macro.dbt.drop_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "drop_relation", "macro_sql": "{% macro drop_relation(relation) -%}\n {{ return(adapter.dispatch('drop_relation', 'dbt')(relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__drop_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__drop_relation": {"unique_id": "macro.dbt.default__drop_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__drop_relation", "macro_sql": "{% macro default__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n drop {{ relation.type }} if exists {{ relation }} cascade\n {%- endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.truncate_relation": {"unique_id": "macro.dbt.truncate_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "truncate_relation", "macro_sql": "{% macro truncate_relation(relation) -%}\n {{ return(adapter.dispatch('truncate_relation', 'dbt')(relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__truncate_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__truncate_relation": {"unique_id": "macro.dbt.default__truncate_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__truncate_relation", "macro_sql": "{% macro default__truncate_relation(relation) -%}\n {% call statement('truncate_relation') -%}\n truncate table {{ relation }}\n {%- endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.rename_relation": {"unique_id": "macro.dbt.rename_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "rename_relation", "macro_sql": "{% macro rename_relation(from_relation, to_relation) -%}\n {{ return(adapter.dispatch('rename_relation', 'dbt')(from_relation, to_relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__rename_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__rename_relation": {"unique_id": "macro.dbt.default__rename_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__rename_relation", "macro_sql": "{% macro default__rename_relation(from_relation, to_relation) -%}\n {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %}\n {% call statement('rename_relation') -%}\n alter table {{ from_relation }} rename to {{ target_name }}\n {%- endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.information_schema_name": {"unique_id": "macro.dbt.information_schema_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "information_schema_name", "macro_sql": "{% macro information_schema_name(database) %}\n {{ return(adapter.dispatch('information_schema_name', 'dbt')(database)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__information_schema_name"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__information_schema_name": {"unique_id": "macro.dbt.default__information_schema_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__information_schema_name", "macro_sql": "{% macro default__information_schema_name(database) -%}\n {%- if database -%}\n {{ database }}.INFORMATION_SCHEMA\n {%- else -%}\n INFORMATION_SCHEMA\n {%- endif -%}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.list_schemas": {"unique_id": "macro.dbt.list_schemas", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "list_schemas", "macro_sql": "{% macro list_schemas(database) -%}\n {{ return(adapter.dispatch('list_schemas', 'dbt')(database)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__list_schemas"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__list_schemas": {"unique_id": "macro.dbt.default__list_schemas", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__list_schemas", "macro_sql": "{% macro default__list_schemas(database) -%}\n {% set sql %}\n select distinct schema_name\n from {{ information_schema_name(database) }}.SCHEMATA\n where catalog_name ilike '{{ database }}'\n {% endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.information_schema_name", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.check_schema_exists": {"unique_id": "macro.dbt.check_schema_exists", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "check_schema_exists", "macro_sql": "{% macro check_schema_exists(information_schema, schema) -%}\n {{ return(adapter.dispatch('check_schema_exists', 'dbt')(information_schema, schema)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__check_schema_exists"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__check_schema_exists": {"unique_id": "macro.dbt.default__check_schema_exists", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__check_schema_exists", "macro_sql": "{% macro default__check_schema_exists(information_schema, schema) -%}\n {% set sql -%}\n select count(*)\n from {{ information_schema.replace(information_schema_view='SCHEMATA') }}\n where catalog_name='{{ information_schema.database }}'\n and schema_name='{{ schema }}'\n {%- endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.list_relations_without_caching": {"unique_id": "macro.dbt.list_relations_without_caching", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "list_relations_without_caching", "macro_sql": "{% macro list_relations_without_caching(schema_relation) %}\n {{ return(adapter.dispatch('list_relations_without_caching', 'dbt')(schema_relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.test.postgres__list_relations_without_caching"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__list_relations_without_caching": {"unique_id": "macro.dbt.default__list_relations_without_caching", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__list_relations_without_caching", "macro_sql": "{% macro default__list_relations_without_caching(schema_relation) %}\n {{ exceptions.raise_not_implemented(\n 'list_relations_without_caching macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.current_timestamp": {"unique_id": "macro.dbt.current_timestamp", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "current_timestamp", "macro_sql": "{% macro current_timestamp() -%}\n {{ adapter.dispatch('current_timestamp', 'dbt')() }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__current_timestamp": {"unique_id": "macro.dbt.default__current_timestamp", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__current_timestamp", "macro_sql": "{% macro default__current_timestamp() -%}\n {{ exceptions.raise_not_implemented(\n 'current_timestamp macro not implemented for adapter '+adapter.type()) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.collect_freshness": {"unique_id": "macro.dbt.collect_freshness", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "collect_freshness", "macro_sql": "{% macro collect_freshness(source, loaded_at_field, filter) %}\n {{ return(adapter.dispatch('collect_freshness', 'dbt')(source, loaded_at_field, filter))}}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__collect_freshness"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__collect_freshness": {"unique_id": "macro.dbt.default__collect_freshness", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__collect_freshness", "macro_sql": "{% macro default__collect_freshness(source, loaded_at_field, filter) %}\n {% call statement('collect_freshness', fetch_result=True, auto_begin=False) -%}\n select\n max({{ loaded_at_field }}) as max_loaded_at,\n {{ current_timestamp() }} as snapshotted_at\n from {{ source }}\n {% if filter %}\n where {{ filter }}\n {% endif %}\n {% endcall %}\n {{ return(load_result('collect_freshness').table) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.make_temp_relation": {"unique_id": "macro.dbt.make_temp_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "make_temp_relation", "macro_sql": "{% macro make_temp_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_temp_relation', 'dbt')(base_relation, suffix))}}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_temp_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__make_temp_relation": {"unique_id": "macro.dbt.default__make_temp_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__make_temp_relation", "macro_sql": "{% macro default__make_temp_relation(base_relation, suffix) %}\n {% set tmp_identifier = base_relation.identifier ~ suffix %}\n {% set tmp_relation = base_relation.incorporate(\n path={\"identifier\": tmp_identifier}) -%}\n\n {% do return(tmp_relation) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.set_sql_header": {"unique_id": "macro.dbt.set_sql_header", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "set_sql_header", "macro_sql": "{% macro set_sql_header(config) -%}\n {{ config.set('sql_header', caller()) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.alter_relation_add_remove_columns": {"unique_id": "macro.dbt.alter_relation_add_remove_columns", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "alter_relation_add_remove_columns", "macro_sql": "{% macro alter_relation_add_remove_columns(relation, add_columns = none, remove_columns = none) -%}\n {{ return(adapter.dispatch('alter_relation_add_remove_columns', 'dbt')(relation, add_columns, remove_columns)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__alter_relation_add_remove_columns"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__alter_relation_add_remove_columns": {"unique_id": "macro.dbt.default__alter_relation_add_remove_columns", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/adapters/common.sql", "original_file_path": "macros/adapters/common.sql", "name": "default__alter_relation_add_remove_columns", "macro_sql": "{% macro default__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}\n \n {% if add_columns is none %}\n {% set add_columns = [] %}\n {% endif %}\n {% if remove_columns is none %}\n {% set remove_columns = [] %}\n {% endif %}\n \n {% set sql -%}\n \n alter {{ relation.type }} {{ relation }}\n \n {% for column in add_columns %}\n add column {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }}\n {% endfor %}{{ ',' if add_columns and remove_columns }}\n \n {% for column in remove_columns %}\n drop column {{ column.name }}{{ ',' if not loop.last }}\n {% endfor %}\n \n {%- endset -%}\n\n {% do run_query(sql) %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__test_relationships": {"unique_id": "macro.dbt.default__test_relationships", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/schema_tests/relationships.sql", "original_file_path": "macros/schema_tests/relationships.sql", "name": "default__test_relationships", "macro_sql": "{% macro default__test_relationships(model, column_name, to, field) %}\n\nwith child as (\n select {{ column_name }} as from_field\n from {{ model }}\n where {{ column_name }} is not null\n),\n\nparent as (\n select {{ field }} as to_field\n from {{ to }}\n)\n\nselect\n from_field\n\nfrom child\nleft join parent\n on child.from_field = parent.to_field\n\nwhere parent.to_field is null\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.test_relationships": {"unique_id": "macro.dbt.test_relationships", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/schema_tests/relationships.sql", "original_file_path": "macros/schema_tests/relationships.sql", "name": "test_relationships", "macro_sql": "{% test relationships(model, column_name, to, field) %}\n {% set macro = adapter.dispatch('test_relationships', 'dbt') %}\n {{ macro(model, column_name, to, field) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__test_relationships"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__test_not_null": {"unique_id": "macro.dbt.default__test_not_null", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/schema_tests/not_null.sql", "original_file_path": "macros/schema_tests/not_null.sql", "name": "default__test_not_null", "macro_sql": "{% macro default__test_not_null(model, column_name) %}\n\nselect *\nfrom {{ model }}\nwhere {{ column_name }} is null\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.test_not_null": {"unique_id": "macro.dbt.test_not_null", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/schema_tests/not_null.sql", "original_file_path": "macros/schema_tests/not_null.sql", "name": "test_not_null", "macro_sql": "{% test not_null(model, column_name) %}\n {% set macro = adapter.dispatch('test_not_null', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__test_not_null"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__test_unique": {"unique_id": "macro.dbt.default__test_unique", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/schema_tests/unique.sql", "original_file_path": "macros/schema_tests/unique.sql", "name": "default__test_unique", "macro_sql": "{% macro default__test_unique(model, column_name) %}\n\nselect\n {{ column_name }} as unique_field,\n count(*) as n_records\n\nfrom {{ model }}\nwhere {{ column_name }} is not null\ngroup by {{ column_name }}\nhaving count(*) > 1\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.test_unique": {"unique_id": "macro.dbt.test_unique", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/schema_tests/unique.sql", "original_file_path": "macros/schema_tests/unique.sql", "name": "test_unique", "macro_sql": "{% test unique(model, column_name) %}\n {% set macro = adapter.dispatch('test_unique', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__test_unique"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.default__test_accepted_values": {"unique_id": "macro.dbt.default__test_accepted_values", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/schema_tests/accepted_values.sql", "original_file_path": "macros/schema_tests/accepted_values.sql", "name": "default__test_accepted_values", "macro_sql": "{% macro default__test_accepted_values(model, column_name, values, quote=True) %}\n\nwith all_values as (\n\n select\n {{ column_name }} as value_field,\n count(*) as n_records\n\n from {{ model }}\n group by {{ column_name }}\n\n)\n\nselect *\nfrom all_values\nwhere value_field not in (\n {% for value in values -%}\n {% if quote -%}\n '{{ value }}'\n {%- else -%}\n {{ value }}\n {%- endif -%}\n {%- if not loop.last -%},{%- endif %}\n {%- endfor %}\n)\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}, "macro.dbt.test_accepted_values": {"unique_id": "macro.dbt.test_accepted_values", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "macros/schema_tests/accepted_values.sql", "original_file_path": "macros/schema_tests/accepted_values.sql", "name": "test_accepted_values", "macro_sql": "{% test accepted_values(model, column_name, values, quote=True) %}\n {% set macro = adapter.dispatch('test_accepted_values', 'dbt') %}\n {{ macro(model, column_name, values, quote) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__test_accepted_values"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654665147}}, "docs": {"dbt.__overview__": {"unique_id": "dbt.__overview__", "package_name": "dbt", "root_path": "/Users/jerco/dev/product/dbt-core/core/dbt/include/global_project", "path": "overview.md", "original_file_path": "docs/overview.md", "name": "__overview__", "block_contents": "### Welcome!\n\nWelcome to the auto-generated documentation for your dbt project!\n\n### Navigation\n\nYou can use the `Project` and `Database` navigation tabs on the left side of the window to explore the models\nin your project.\n\n#### Project Tab\nThe `Project` tab mirrors the directory structure of your dbt project. In this tab, you can see all of the\nmodels defined in your dbt project, as well as models imported from dbt packages.\n\n#### Database Tab\nThe `Database` tab also exposes your models, but in a format that looks more like a database explorer. This view\nshows relations (tables and views) grouped into database schemas. Note that ephemeral models are _not_ shown\nin this interface, as they do not exist in the database.\n\n### Graph Exploration\nYou can click the blue icon on the bottom-right corner of the page to view the lineage graph of your models.\n\nOn model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the `Expand`\nbutton at the top-right of this lineage pane, you'll be able to see all of the models that are used to build,\nor are built from, the model you're exploring.\n\nOnce expanded, you'll be able to use the `--models` and `--exclude` model selection syntax to filter the\nmodels in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax).\n\nNote that you can also right-click on models to interactively filter and explore the graph.\n\n---\n\n### More information\n\n- [What is dbt](https://docs.getdbt.com/docs/overview)?\n- Read the [dbt viewpoint](https://docs.getdbt.com/docs/viewpoint)\n- [Installation](https://docs.getdbt.com/docs/installation)\n- Join the [chat](https://community.getdbt.com/) on Slack for live questions and support."}}, "exposures": {}, "selectors": {}, "disabled": [], "parent_map": {"model.test.my_model": []}, "child_map": {"model.test.my_model": []}} ================================================ FILE: tests/functional/artifacts/data/state/v4/manifest.json ================================================ {"metadata": {"dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v4.json", "dbt_version": "1.0.8", "generated_at": "2022-09-13T08:43:20.641750Z", "invocation_id": "5da6faab-41cb-4180-ab19-8375c0e1f1a5", "env": {}, "project_id": "098f6bcd4621d373cade4e832627b4f6", "user_id": null, "send_anonymous_usage_stats": false, "adapter_type": "postgres"}, "nodes": {"model.test.my_model": {"raw_sql": "select 1 as id", "resource_type": "model", "depends_on": {"macros": [], "nodes": []}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "on_schema_change": "ignore", "post-hook": [], "pre-hook": []}, "database": "jerco", "schema": "dbt_jcohen", "fqn": ["test", "my_model"], "unique_id": "model.test.my_model", "package_name": "test", "root_path": "/Users/jerco/dev/scratch/testy", "path": "my_model.sql", "original_file_path": "models/my_model.sql", "name": "my_model", "alias": "my_model", "checksum": {"name": "sha256", "checksum": "479636cb85ce8d3b0f8db5ff13cf338b61254ad98d905630eac61f963e719e9d"}, "tags": [], "refs": [], "sources": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true}, "patch_path": null, "compiled_path": null, "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1663058601.2387}}, "sources": {}, "macros": {"macro.dbt_postgres.postgres__get_catalog": {"unique_id": "macro.dbt_postgres.postgres__get_catalog", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/catalog.sql", "original_file_path": "macros/catalog.sql", "name": "postgres__get_catalog", "macro_sql": "{% macro postgres__get_catalog(information_schema, schemas) -%}\n\n {%- call statement('catalog', fetch_result=True) -%}\n {#\n If the user has multiple databases set and the first one is wrong, this will fail.\n But we won't fail in the case where there are multiple quoting-difference-only dbs, which is better.\n #}\n {% set database = information_schema.database %}\n {{ adapter.verify_database(database) }}\n\n select\n '{{ database }}' as table_database,\n sch.nspname as table_schema,\n tbl.relname as table_name,\n case tbl.relkind\n when 'v' then 'VIEW'\n else 'BASE TABLE'\n end as table_type,\n tbl_desc.description as table_comment,\n col.attname as column_name,\n col.attnum as column_index,\n pg_catalog.format_type(col.atttypid, col.atttypmod) as column_type,\n col_desc.description as column_comment,\n pg_get_userbyid(tbl.relowner) as table_owner\n\n from pg_catalog.pg_namespace sch\n join pg_catalog.pg_class tbl on tbl.relnamespace = sch.oid\n join pg_catalog.pg_attribute col on col.attrelid = tbl.oid\n left outer join pg_catalog.pg_description tbl_desc on (tbl_desc.objoid = tbl.oid and tbl_desc.objsubid = 0)\n left outer join pg_catalog.pg_description col_desc on (col_desc.objoid = tbl.oid and col_desc.objsubid = col.attnum)\n\n where (\n {%- for schema in schemas -%}\n upper(sch.nspname) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n )\n and not pg_is_other_temp_schema(sch.oid) -- not a temporary schema belonging to another session\n and tbl.relpersistence in ('p', 'u') -- [p]ermanent table or [u]nlogged table. Exclude [t]emporary tables\n and tbl.relkind in ('r', 'v', 'f', 'p') -- o[r]dinary table, [v]iew, [f]oreign table, [p]artitioned table. Other values are [i]ndex, [S]equence, [c]omposite type, [t]OAST table, [m]aterialized view\n and col.attnum > 0 -- negative numbers are used for system columns such as oid\n and not col.attisdropped -- column as not been dropped\n\n order by\n sch.nspname,\n tbl.relname,\n col.attnum\n\n {%- endcall -%}\n\n {{ return(load_result('catalog').table) }}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.6944451}, "macro.dbt_postgres.postgres_get_relations": {"unique_id": "macro.dbt_postgres.postgres_get_relations", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/relations.sql", "original_file_path": "macros/relations.sql", "name": "postgres_get_relations", "macro_sql": "{% macro postgres_get_relations () -%}\n\n {#\n -- in pg_depend, objid is the dependent, refobjid is the referenced object\n -- > a pg_depend entry indicates that the referenced object cannot be\n -- > dropped without also dropping the dependent object.\n #}\n\n {%- call statement('relations', fetch_result=True) -%}\n with relation as (\n select\n pg_rewrite.ev_class as class,\n pg_rewrite.oid as id\n from pg_rewrite\n ),\n class as (\n select\n oid as id,\n relname as name,\n relnamespace as schema,\n relkind as kind\n from pg_class\n ),\n dependency as (\n select\n pg_depend.objid as id,\n pg_depend.refobjid as ref\n from pg_depend\n ),\n schema as (\n select\n pg_namespace.oid as id,\n pg_namespace.nspname as name\n from pg_namespace\n where nspname != 'information_schema' and nspname not like 'pg\\_%'\n ),\n referenced as (\n select\n relation.id AS id,\n referenced_class.name ,\n referenced_class.schema ,\n referenced_class.kind\n from relation\n join class as referenced_class on relation.class=referenced_class.id\n where referenced_class.kind in ('r', 'v')\n ),\n relationships as (\n select\n referenced.name as referenced_name,\n referenced.schema as referenced_schema_id,\n dependent_class.name as dependent_name,\n dependent_class.schema as dependent_schema_id,\n referenced.kind as kind\n from referenced\n join dependency on referenced.id=dependency.id\n join class as dependent_class on dependency.ref=dependent_class.id\n where\n (referenced.name != dependent_class.name or\n referenced.schema != dependent_class.schema)\n )\n\n select\n referenced_schema.name as referenced_schema,\n relationships.referenced_name as referenced_name,\n dependent_schema.name as dependent_schema,\n relationships.dependent_name as dependent_name\n from relationships\n join schema as dependent_schema on relationships.dependent_schema_id=dependent_schema.id\n join schema as referenced_schema on relationships.referenced_schema_id=referenced_schema.id\n group by referenced_schema, referenced_name, dependent_schema, dependent_name\n order by referenced_schema, referenced_name, dependent_schema, dependent_name;\n\n {%- endcall -%}\n\n {{ return(load_result('relations').table) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.696331}, "macro.dbt_postgres.postgres__create_table_as": {"unique_id": "macro.dbt_postgres.postgres__create_table_as", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__create_table_as", "macro_sql": "{% macro postgres__create_table_as(temporary, relation, sql) -%}\n {%- set unlogged = config.get('unlogged', default=false) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary -%}\n temporary\n {%- elif unlogged -%}\n unlogged\n {%- endif %} table {{ relation }}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.712787}, "macro.dbt_postgres.postgres__get_create_index_sql": {"unique_id": "macro.dbt_postgres.postgres__get_create_index_sql", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__get_create_index_sql", "macro_sql": "{% macro postgres__get_create_index_sql(relation, index_dict) -%}\n {%- set index_config = adapter.parse_index(index_dict) -%}\n {%- set comma_separated_columns = \", \".join(index_config.columns) -%}\n {%- set index_name = index_config.render(relation) -%}\n\n create {% if index_config.unique -%}\n unique\n {%- endif %} index if not exists\n \"{{ index_name }}\"\n on {{ relation }} {% if index_config.type -%}\n using {{ index_config.type }}\n {%- endif %}\n ({{ comma_separated_columns }});\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.714135}, "macro.dbt_postgres.postgres__create_schema": {"unique_id": "macro.dbt_postgres.postgres__create_schema", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__create_schema", "macro_sql": "{% macro postgres__create_schema(relation) -%}\n {% if relation.database -%}\n {{ adapter.verify_database(relation.database) }}\n {%- endif -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier().include(database=False) }}\n {%- endcall -%}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.714985}, "macro.dbt_postgres.postgres__drop_schema": {"unique_id": "macro.dbt_postgres.postgres__drop_schema", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__drop_schema", "macro_sql": "{% macro postgres__drop_schema(relation) -%}\n {% if relation.database -%}\n {{ adapter.verify_database(relation.database) }}\n {%- endif -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier().include(database=False) }} cascade\n {%- endcall -%}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.7158241}, "macro.dbt_postgres.postgres__get_columns_in_relation": {"unique_id": "macro.dbt_postgres.postgres__get_columns_in_relation", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__get_columns_in_relation", "macro_sql": "{% macro postgres__get_columns_in_relation(relation) -%}\n {% call statement('get_columns_in_relation', fetch_result=True) %}\n select\n column_name,\n data_type,\n character_maximum_length,\n numeric_precision,\n numeric_scale\n\n from {{ relation.information_schema('columns') }}\n where table_name = '{{ relation.identifier }}'\n {% if relation.schema %}\n and table_schema = '{{ relation.schema }}'\n {% endif %}\n order by ordinal_position\n\n {% endcall %}\n {% set table = load_result('get_columns_in_relation').table %}\n {{ return(sql_convert_columns_in_relation(table)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.sql_convert_columns_in_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.717091}, "macro.dbt_postgres.postgres__list_relations_without_caching": {"unique_id": "macro.dbt_postgres.postgres__list_relations_without_caching", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__list_relations_without_caching", "macro_sql": "{% macro postgres__list_relations_without_caching(schema_relation) %}\n {% call statement('list_relations_without_caching', fetch_result=True) -%}\n select\n '{{ schema_relation.database }}' as database,\n tablename as name,\n schemaname as schema,\n 'table' as type\n from pg_tables\n where schemaname ilike '{{ schema_relation.schema }}'\n union all\n select\n '{{ schema_relation.database }}' as database,\n viewname as name,\n schemaname as schema,\n 'view' as type\n from pg_views\n where schemaname ilike '{{ schema_relation.schema }}'\n {% endcall %}\n {{ return(load_result('list_relations_without_caching').table) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.718103}, "macro.dbt_postgres.postgres__information_schema_name": {"unique_id": "macro.dbt_postgres.postgres__information_schema_name", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__information_schema_name", "macro_sql": "{% macro postgres__information_schema_name(database) -%}\n {% if database_name -%}\n {{ adapter.verify_database(database_name) }}\n {%- endif -%}\n information_schema\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.7185578}, "macro.dbt_postgres.postgres__list_schemas": {"unique_id": "macro.dbt_postgres.postgres__list_schemas", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__list_schemas", "macro_sql": "{% macro postgres__list_schemas(database) %}\n {% if database -%}\n {{ adapter.verify_database(database) }}\n {%- endif -%}\n {% call statement('list_schemas', fetch_result=True, auto_begin=False) %}\n select distinct nspname from pg_namespace\n {% endcall %}\n {{ return(load_result('list_schemas').table) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.7194948}, "macro.dbt_postgres.postgres__check_schema_exists": {"unique_id": "macro.dbt_postgres.postgres__check_schema_exists", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__check_schema_exists", "macro_sql": "{% macro postgres__check_schema_exists(information_schema, schema) -%}\n {% if information_schema.database -%}\n {{ adapter.verify_database(information_schema.database) }}\n {%- endif -%}\n {% call statement('check_schema_exists', fetch_result=True, auto_begin=False) %}\n select count(*) from pg_namespace where nspname = '{{ schema }}'\n {% endcall %}\n {{ return(load_result('check_schema_exists').table) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.720533}, "macro.dbt_postgres.postgres__current_timestamp": {"unique_id": "macro.dbt_postgres.postgres__current_timestamp", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__current_timestamp", "macro_sql": "{% macro postgres__current_timestamp() -%}\n now()\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.720746}, "macro.dbt_postgres.postgres__snapshot_string_as_time": {"unique_id": "macro.dbt_postgres.postgres__snapshot_string_as_time", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__snapshot_string_as_time", "macro_sql": "{% macro postgres__snapshot_string_as_time(timestamp) -%}\n {%- set result = \"'\" ~ timestamp ~ \"'::timestamp without time zone\" -%}\n {{ return(result) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.72122}, "macro.dbt_postgres.postgres__snapshot_get_time": {"unique_id": "macro.dbt_postgres.postgres__snapshot_get_time", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__snapshot_get_time", "macro_sql": "{% macro postgres__snapshot_get_time() -%}\n {{ current_timestamp() }}::timestamp without time zone\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.721492}, "macro.dbt_postgres.postgres__make_temp_relation": {"unique_id": "macro.dbt_postgres.postgres__make_temp_relation", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__make_temp_relation", "macro_sql": "{% macro postgres__make_temp_relation(base_relation, suffix) %}\n {% set dt = modules.datetime.datetime.now() %}\n {% set dtstring = dt.strftime(\"%H%M%S%f\") %}\n {% set suffix_length = suffix|length + dtstring|length %}\n {% set relation_max_name_length = 63 %}\n {% if suffix_length > relation_max_name_length %}\n {% do exceptions.raise_compiler_error('Temp relation suffix is too long (' ~ suffix|length ~ ' characters). Maximum length is ' ~ (relation_max_name_length - dtstring|length) ~ ' characters.') %}\n {% endif %}\n {% set tmp_identifier = base_relation.identifier[:relation_max_name_length - suffix_length] ~ suffix ~ dtstring %}\n {% do return(base_relation.incorporate(\n path={\n \"identifier\": tmp_identifier,\n \"schema\": none,\n \"database\": none\n })) -%}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.7240572}, "macro.dbt_postgres.postgres_escape_comment": {"unique_id": "macro.dbt_postgres.postgres_escape_comment", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres_escape_comment", "macro_sql": "{% macro postgres_escape_comment(comment) -%}\n {% if comment is not string %}\n {% do exceptions.raise_compiler_error('cannot escape a non-string: ' ~ comment) %}\n {% endif %}\n {%- set magic = '$dbt_comment_literal_block$' -%}\n {%- if magic in comment -%}\n {%- do exceptions.raise_compiler_error('The string ' ~ magic ~ ' is not allowed in comments.') -%}\n {%- endif -%}\n {{ magic }}{{ comment }}{{ magic }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.7252119}, "macro.dbt_postgres.postgres__alter_relation_comment": {"unique_id": "macro.dbt_postgres.postgres__alter_relation_comment", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__alter_relation_comment", "macro_sql": "{% macro postgres__alter_relation_comment(relation, comment) %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n comment on {{ relation.type }} {{ relation }} is {{ escaped_comment }};\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres_escape_comment"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.7258098}, "macro.dbt_postgres.postgres__alter_column_comment": {"unique_id": "macro.dbt_postgres.postgres__alter_column_comment", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__alter_column_comment", "macro_sql": "{% macro postgres__alter_column_comment(relation, column_dict) %}\n {% set existing_columns = adapter.get_columns_in_relation(relation) | map(attribute=\"name\") | list %}\n {% for column_name in column_dict if (column_name in existing_columns) %}\n {% set comment = column_dict[column_name]['description'] %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n comment on column {{ relation }}.{{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }} is {{ escaped_comment }};\n {% endfor %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres_escape_comment"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.727401}, "macro.dbt_postgres.postgres__snapshot_merge_sql": {"unique_id": "macro.dbt_postgres.postgres__snapshot_merge_sql", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/materializations/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshot_merge.sql", "name": "postgres__snapshot_merge_sql", "macro_sql": "{% macro postgres__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n update {{ target }}\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n from {{ source }} as DBT_INTERNAL_SOURCE\n where DBT_INTERNAL_SOURCE.dbt_scd_id::text = {{ target }}.dbt_scd_id::text\n and DBT_INTERNAL_SOURCE.dbt_change_type::text in ('update'::text, 'delete'::text)\n and {{ target }}.dbt_valid_to is null;\n\n insert into {{ target }} ({{ insert_cols_csv }})\n select {% for column in insert_cols -%}\n DBT_INTERNAL_SOURCE.{{ column }} {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n from {{ source }} as DBT_INTERNAL_SOURCE\n where DBT_INTERNAL_SOURCE.dbt_change_type::text = 'insert'::text;\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.729517}, "macro.dbt.run_hooks": {"unique_id": "macro.dbt.run_hooks", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "name": "run_hooks", "macro_sql": "{% macro run_hooks(hooks, inside_transaction=True) %}\n {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %}\n {% if not inside_transaction and loop.first %}\n {% call statement(auto_begin=inside_transaction) %}\n commit;\n {% endcall %}\n {% endif %}\n {% set rendered = render(hook.get('sql')) | trim %}\n {% if (rendered | length) > 0 %}\n {% call statement(auto_begin=inside_transaction) %}\n {{ rendered }}\n {% endcall %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.732625}, "macro.dbt.make_hook_config": {"unique_id": "macro.dbt.make_hook_config", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "name": "make_hook_config", "macro_sql": "{% macro make_hook_config(sql, inside_transaction) %}\n {{ tojson({\"sql\": sql, \"transaction\": inside_transaction}) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.7331321}, "macro.dbt.before_begin": {"unique_id": "macro.dbt.before_begin", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "name": "before_begin", "macro_sql": "{% macro before_begin(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.7335098}, "macro.dbt.in_transaction": {"unique_id": "macro.dbt.in_transaction", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "name": "in_transaction", "macro_sql": "{% macro in_transaction(sql) %}\n {{ make_hook_config(sql, inside_transaction=True) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.733889}, "macro.dbt.after_commit": {"unique_id": "macro.dbt.after_commit", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "name": "after_commit", "macro_sql": "{% macro after_commit(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.7342541}, "macro.dbt.set_sql_header": {"unique_id": "macro.dbt.set_sql_header", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "name": "set_sql_header", "macro_sql": "{% macro set_sql_header(config) -%}\n {{ config.set('sql_header', caller()) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.735413}, "macro.dbt.should_full_refresh": {"unique_id": "macro.dbt.should_full_refresh", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "name": "should_full_refresh", "macro_sql": "{% macro should_full_refresh() %}\n {% set config_full_refresh = config.get('full_refresh') %}\n {% if config_full_refresh is none %}\n {% set config_full_refresh = flags.FULL_REFRESH %}\n {% endif %}\n {% do return(config_full_refresh) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.736206}, "macro.dbt.should_store_failures": {"unique_id": "macro.dbt.should_store_failures", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "name": "should_store_failures", "macro_sql": "{% macro should_store_failures() %}\n {% set config_store_failures = config.get('store_failures') %}\n {% if config_store_failures is none %}\n {% set config_store_failures = flags.STORE_FAILURES %}\n {% endif %}\n {% do return(config_store_failures) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.737014}, "macro.dbt.snapshot_merge_sql": {"unique_id": "macro.dbt.snapshot_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", "name": "snapshot_merge_sql", "macro_sql": "{% macro snapshot_merge_sql(target, source, insert_cols) -%}\n {{ adapter.dispatch('snapshot_merge_sql', 'dbt')(target, source, insert_cols) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__snapshot_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.738338}, "macro.dbt.default__snapshot_merge_sql": {"unique_id": "macro.dbt.default__snapshot_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", "name": "default__snapshot_merge_sql", "macro_sql": "{% macro default__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on DBT_INTERNAL_SOURCE.dbt_scd_id = DBT_INTERNAL_DEST.dbt_scd_id\n\n when matched\n and DBT_INTERNAL_DEST.dbt_valid_to is null\n and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')\n then update\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n\n when not matched\n and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'\n then insert ({{ insert_cols_csv }})\n values ({{ insert_cols_csv }})\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.7390552}, "macro.dbt.strategy_dispatch": {"unique_id": "macro.dbt.strategy_dispatch", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "strategy_dispatch", "macro_sql": "{% macro strategy_dispatch(name) -%}\n{% set original_name = name %}\n {% if '.' in name %}\n {% set package_name, name = name.split(\".\", 1) %}\n {% else %}\n {% set package_name = none %}\n {% endif %}\n\n {% if package_name is none %}\n {% set package_context = context %}\n {% elif package_name in context %}\n {% set package_context = context[package_name] %}\n {% else %}\n {% set error_msg %}\n Could not find package '{{package_name}}', called with '{{original_name}}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n\n {%- set search_name = 'snapshot_' ~ name ~ '_strategy' -%}\n\n {% if search_name not in package_context %}\n {% set error_msg %}\n The specified strategy macro '{{name}}' was not found in package '{{ package_name }}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n {{ return(package_context[search_name]) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.7487428}, "macro.dbt.snapshot_hash_arguments": {"unique_id": "macro.dbt.snapshot_hash_arguments", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "snapshot_hash_arguments", "macro_sql": "{% macro snapshot_hash_arguments(args) -%}\n {{ adapter.dispatch('snapshot_hash_arguments', 'dbt')(args) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.7491798}, "macro.dbt.default__snapshot_hash_arguments": {"unique_id": "macro.dbt.default__snapshot_hash_arguments", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "default__snapshot_hash_arguments", "macro_sql": "{% macro default__snapshot_hash_arguments(args) -%}\n md5({%- for arg in args -%}\n coalesce(cast({{ arg }} as varchar ), '')\n {% if not loop.last %} || '|' || {% endif %}\n {%- endfor -%})\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.74975}, "macro.dbt.snapshot_get_time": {"unique_id": "macro.dbt.snapshot_get_time", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "snapshot_get_time", "macro_sql": "{% macro snapshot_get_time() -%}\n {{ adapter.dispatch('snapshot_get_time', 'dbt')() }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__snapshot_get_time"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.7501109}, "macro.dbt.default__snapshot_get_time": {"unique_id": "macro.dbt.default__snapshot_get_time", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "default__snapshot_get_time", "macro_sql": "{% macro default__snapshot_get_time() -%}\n {{ current_timestamp() }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.7503612}, "macro.dbt.snapshot_timestamp_strategy": {"unique_id": "macro.dbt.snapshot_timestamp_strategy", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "snapshot_timestamp_strategy", "macro_sql": "{% macro snapshot_timestamp_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}\n {% set primary_key = config['unique_key'] %}\n {% set updated_at = config['updated_at'] %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %}\n\n {#/*\n The snapshot relation might not have an {{ updated_at }} value if the\n snapshot strategy is changed from `check` to `timestamp`. We\n should use a dbt-created column for the comparison in the snapshot\n table instead of assuming that the user-supplied {{ updated_at }}\n will be present in the historical data.\n\n See https://github.com/dbt-labs/dbt-core/issues/2350\n */ #}\n {% set row_changed_expr -%}\n ({{ snapshotted_rel }}.dbt_valid_from < {{ current_rel }}.{{ updated_at }})\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.7524228}, "macro.dbt.snapshot_string_as_time": {"unique_id": "macro.dbt.snapshot_string_as_time", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "snapshot_string_as_time", "macro_sql": "{% macro snapshot_string_as_time(timestamp) -%}\n {{ adapter.dispatch('snapshot_string_as_time', 'dbt')(timestamp) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__snapshot_string_as_time"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.752844}, "macro.dbt.default__snapshot_string_as_time": {"unique_id": "macro.dbt.default__snapshot_string_as_time", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "default__snapshot_string_as_time", "macro_sql": "{% macro default__snapshot_string_as_time(timestamp) %}\n {% do exceptions.raise_not_implemented(\n 'snapshot_string_as_time macro not implemented for adapter '+adapter.type()\n ) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.753275}, "macro.dbt.snapshot_check_all_get_existing_columns": {"unique_id": "macro.dbt.snapshot_check_all_get_existing_columns", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "snapshot_check_all_get_existing_columns", "macro_sql": "{% macro snapshot_check_all_get_existing_columns(node, target_exists) -%}\n {%- set query_columns = get_columns_in_query(node['compiled_sql']) -%}\n {%- if not target_exists -%}\n {# no table yet -> return whatever the query does #}\n {{ return([false, query_columns]) }}\n {%- endif -%}\n {# handle any schema changes #}\n {%- set target_table = node.get('alias', node.get('name')) -%}\n {%- set target_relation = adapter.get_relation(database=node.database, schema=node.schema, identifier=target_table) -%}\n {%- set existing_cols = get_columns_in_query('select * from ' ~ target_relation) -%}\n {%- set ns = namespace() -%} {# handle for-loop scoping with a namespace #}\n {%- set ns.column_added = false -%}\n\n {%- set intersection = [] -%}\n {%- for col in query_columns -%}\n {%- if col in existing_cols -%}\n {%- do intersection.append(col) -%}\n {%- else -%}\n {% set ns.column_added = true %}\n {%- endif -%}\n {%- endfor -%}\n {{ return([ns.column_added, intersection]) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_columns_in_query"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.755879}, "macro.dbt.snapshot_check_strategy": {"unique_id": "macro.dbt.snapshot_check_strategy", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "snapshot_check_strategy", "macro_sql": "{% macro snapshot_check_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}\n {% set check_cols_config = config['check_cols'] %}\n {% set primary_key = config['unique_key'] %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %}\n \n {% set select_current_time -%}\n select {{ snapshot_get_time() }} as snapshot_start\n {%- endset %}\n\n {#-- don't access the column by name, to avoid dealing with casing issues on snowflake #}\n {%- set now = run_query(select_current_time)[0][0] -%}\n {% if now is none or now is undefined -%}\n {%- do exceptions.raise_compiler_error('Could not get a snapshot start time from the database') -%}\n {%- endif %}\n {% set updated_at = config.get('updated_at', snapshot_string_as_time(now)) %}\n\n {% set column_added = false %}\n\n {% if check_cols_config == 'all' %}\n {% set column_added, check_cols = snapshot_check_all_get_existing_columns(node, target_exists) %}\n {% elif check_cols_config is iterable and (check_cols_config | length) > 0 %}\n {% set check_cols = check_cols_config %}\n {% else %}\n {% do exceptions.raise_compiler_error(\"Invalid value for 'check_cols': \" ~ check_cols_config) %}\n {% endif %}\n\n {%- set row_changed_expr -%}\n (\n {%- if column_added -%}\n TRUE\n {%- else -%}\n {%- for col in check_cols -%}\n {{ snapshotted_rel }}.{{ col }} != {{ current_rel }}.{{ col }}\n or\n (\n (({{ snapshotted_rel }}.{{ col }} is null) and not ({{ current_rel }}.{{ col }} is null))\n or\n ((not {{ snapshotted_rel }}.{{ col }} is null) and ({{ current_rel }}.{{ col }} is null))\n )\n {%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n {%- endif -%}\n )\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.snapshot_get_time", "macro.dbt.run_query", "macro.dbt.snapshot_string_as_time", "macro.dbt.snapshot_check_all_get_existing_columns", "macro.dbt.snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.760828}, "macro.dbt.create_columns": {"unique_id": "macro.dbt.create_columns", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "create_columns", "macro_sql": "{% macro create_columns(relation, columns) %}\n {{ adapter.dispatch('create_columns', 'dbt')(relation, columns) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__create_columns"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.7705312}, "macro.dbt.default__create_columns": {"unique_id": "macro.dbt.default__create_columns", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "default__create_columns", "macro_sql": "{% macro default__create_columns(relation, columns) %}\n {% for column in columns %}\n {% call statement() %}\n alter table {{ relation }} add column \"{{ column.name }}\" {{ column.data_type }};\n {% endcall %}\n {% endfor %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.7712681}, "macro.dbt.post_snapshot": {"unique_id": "macro.dbt.post_snapshot", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "post_snapshot", "macro_sql": "{% macro post_snapshot(staging_relation) %}\n {{ adapter.dispatch('post_snapshot', 'dbt')(staging_relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.771702}, "macro.dbt.default__post_snapshot": {"unique_id": "macro.dbt.default__post_snapshot", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "default__post_snapshot", "macro_sql": "{% macro default__post_snapshot(staging_relation) %}\n {# no-op #}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.77192}, "macro.dbt.snapshot_staging_table": {"unique_id": "macro.dbt.snapshot_staging_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "snapshot_staging_table", "macro_sql": "{% macro snapshot_staging_table(strategy, source_sql, target_relation) -%}\n {{ adapter.dispatch('snapshot_staging_table', 'dbt')(strategy, source_sql, target_relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__snapshot_staging_table"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.772458}, "macro.dbt.default__snapshot_staging_table": {"unique_id": "macro.dbt.default__snapshot_staging_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "default__snapshot_staging_table", "macro_sql": "{% macro default__snapshot_staging_table(strategy, source_sql, target_relation) -%}\n\n with snapshot_query as (\n\n {{ source_sql }}\n\n ),\n\n snapshotted_data as (\n\n select *,\n {{ strategy.unique_key }} as dbt_unique_key\n\n from {{ target_relation }}\n where dbt_valid_to is null\n\n ),\n\n insertions_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to,\n {{ strategy.scd_id }} as dbt_scd_id\n\n from snapshot_query\n ),\n\n updates_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n {{ strategy.updated_at }} as dbt_valid_to\n\n from snapshot_query\n ),\n\n {%- if strategy.invalidate_hard_deletes %}\n\n deletes_source_data as (\n\n select \n *,\n {{ strategy.unique_key }} as dbt_unique_key\n from snapshot_query\n ),\n {% endif %}\n\n insertions as (\n\n select\n 'insert' as dbt_change_type,\n source_data.*\n\n from insertions_source_data as source_data\n left outer join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where snapshotted_data.dbt_unique_key is null\n or (\n snapshotted_data.dbt_unique_key is not null\n and (\n {{ strategy.row_changed }}\n )\n )\n\n ),\n\n updates as (\n\n select\n 'update' as dbt_change_type,\n source_data.*,\n snapshotted_data.dbt_scd_id\n\n from updates_source_data as source_data\n join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where (\n {{ strategy.row_changed }}\n )\n )\n\n {%- if strategy.invalidate_hard_deletes -%}\n ,\n\n deletes as (\n \n select\n 'delete' as dbt_change_type,\n source_data.*,\n {{ snapshot_get_time() }} as dbt_valid_from,\n {{ snapshot_get_time() }} as dbt_updated_at,\n {{ snapshot_get_time() }} as dbt_valid_to,\n snapshotted_data.dbt_scd_id\n \n from snapshotted_data\n left join deletes_source_data as source_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where source_data.dbt_unique_key is null\n )\n {%- endif %}\n\n select * from insertions\n union all\n select * from updates\n {%- if strategy.invalidate_hard_deletes %}\n union all\n select * from deletes\n {%- endif %}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.snapshot_get_time"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.7748082}, "macro.dbt.build_snapshot_table": {"unique_id": "macro.dbt.build_snapshot_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "build_snapshot_table", "macro_sql": "{% macro build_snapshot_table(strategy, sql) -%}\n {{ adapter.dispatch('build_snapshot_table', 'dbt')(strategy, sql) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__build_snapshot_table"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.775297}, "macro.dbt.default__build_snapshot_table": {"unique_id": "macro.dbt.default__build_snapshot_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "default__build_snapshot_table", "macro_sql": "{% macro default__build_snapshot_table(strategy, sql) %}\n\n select *,\n {{ strategy.scd_id }} as dbt_scd_id,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to\n from (\n {{ sql }}\n ) sbq\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.775959}, "macro.dbt.build_snapshot_staging_table": {"unique_id": "macro.dbt.build_snapshot_staging_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "build_snapshot_staging_table", "macro_sql": "{% macro build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% set tmp_relation = make_temp_relation(target_relation) %}\n\n {% set select = snapshot_staging_table(strategy, sql, target_relation) %}\n\n {% call statement('build_snapshot_staging_relation') %}\n {{ create_table_as(True, tmp_relation, select) }}\n {% endcall %}\n\n {% do return(tmp_relation) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.make_temp_relation", "macro.dbt.snapshot_staging_table", "macro.dbt.statement", "macro.dbt.create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.777101}, "macro.dbt.materialization_snapshot_default": {"unique_id": "macro.dbt.materialization_snapshot_default", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/snapshot.sql", "original_file_path": "macros/materializations/snapshots/snapshot.sql", "name": "materialization_snapshot_default", "macro_sql": "{% materialization snapshot, default %}\n {%- set config = model['config'] -%}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n\n {% if not adapter.check_schema_exists(model.database, model.schema) %}\n {% do create_schema(model.database, model.schema) %}\n {% endif %}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=model.database,\n schema=model.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", config, target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_sql']) %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% else %}\n\n {{ adapter.valid_snapshot_target(target_relation) }}\n\n {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% endif %}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if not target_relation_exists %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if staging_table is defined %}\n {% do post_snapshot(staging_table) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.create_schema", "macro.dbt.get_or_create_relation", "macro.dbt.run_hooks", "macro.dbt.strategy_dispatch", "macro.dbt.build_snapshot_table", "macro.dbt.create_table_as", "macro.dbt.build_snapshot_staging_table", "macro.dbt.create_columns", "macro.dbt.snapshot_merge_sql", "macro.dbt.statement", "macro.dbt.persist_docs", "macro.dbt.create_indexes", "macro.dbt.post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.7941692}, "macro.dbt.materialization_test_default": {"unique_id": "macro.dbt.materialization_test_default", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/tests/test.sql", "original_file_path": "macros/materializations/tests/test.sql", "name": "materialization_test_default", "macro_sql": "{%- materialization test, default -%}\n\n {% set relations = [] %}\n\n {% if should_store_failures() %}\n\n {% set identifier = model['alias'] %}\n {% set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n {% set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database, type='table') -%} %}\n \n {% if old_relation %}\n {% do adapter.drop_relation(old_relation) %}\n {% endif %}\n \n {% call statement(auto_begin=True) %}\n {{ create_table_as(False, target_relation, sql) }}\n {% endcall %}\n \n {% do relations.append(target_relation) %}\n \n {% set main_sql %}\n select *\n from {{ target_relation }}\n {% endset %}\n \n {{ adapter.commit() }}\n \n {% else %}\n\n {% set main_sql = sql %}\n \n {% endif %}\n\n {% set limit = config.get('limit') %}\n {% set fail_calc = config.get('fail_calc') %}\n {% set warn_if = config.get('warn_if') %}\n {% set error_if = config.get('error_if') %}\n\n {% call statement('main', fetch_result=True) -%}\n\n {{ get_test_sql(main_sql, fail_calc, warn_if, error_if, limit)}}\n\n {%- endcall %}\n \n {{ return({'relations': relations}) }}\n\n{%- endmaterialization -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.should_store_failures", "macro.dbt.statement", "macro.dbt.create_table_as", "macro.dbt.get_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.7998898}, "macro.dbt.get_test_sql": {"unique_id": "macro.dbt.get_test_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "name": "get_test_sql", "macro_sql": "{% macro get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n {{ adapter.dispatch('get_test_sql', 'dbt')(main_sql, fail_calc, warn_if, error_if, limit) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.801226}, "macro.dbt.default__get_test_sql": {"unique_id": "macro.dbt.default__get_test_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "name": "default__get_test_sql", "macro_sql": "{% macro default__get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n select\n {{ fail_calc }} as failures,\n {{ fail_calc }} {{ warn_if }} as should_warn,\n {{ fail_calc }} {{ error_if }} as should_error\n from (\n {{ main_sql }}\n {{ \"limit \" ~ limit if limit != none }}\n ) dbt_internal_test\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.8020391}, "macro.dbt.get_where_subquery": {"unique_id": "macro.dbt.get_where_subquery", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/tests/where_subquery.sql", "original_file_path": "macros/materializations/tests/where_subquery.sql", "name": "get_where_subquery", "macro_sql": "{% macro get_where_subquery(relation) -%}\n {% do return(adapter.dispatch('get_where_subquery', 'dbt')(relation)) %}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_where_subquery"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.803186}, "macro.dbt.default__get_where_subquery": {"unique_id": "macro.dbt.default__get_where_subquery", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/tests/where_subquery.sql", "original_file_path": "macros/materializations/tests/where_subquery.sql", "name": "default__get_where_subquery", "macro_sql": "{% macro default__get_where_subquery(relation) -%}\n {% set where = config.get('where', '') %}\n {% if where %}\n {%- set filtered -%}\n (select * from {{ relation }} where {{ where }}) dbt_subquery\n {%- endset -%}\n {% do return(filtered) %}\n {%- else -%}\n {% do return(relation) %}\n {%- endif -%}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.804178}, "macro.dbt.get_quoted_csv": {"unique_id": "macro.dbt.get_quoted_csv", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "name": "get_quoted_csv", "macro_sql": "{% macro get_quoted_csv(column_names) %}\n \n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote(col)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.806505}, "macro.dbt.diff_columns": {"unique_id": "macro.dbt.diff_columns", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "name": "diff_columns", "macro_sql": "{% macro diff_columns(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% set source_names = source_columns | map(attribute = 'column') | list %}\n {% set target_names = target_columns | map(attribute = 'column') | list %}\n \n {# --check whether the name attribute exists in the target - this does not perform a data type check #}\n {% for sc in source_columns %}\n {% if sc.name not in target_names %}\n {{ result.append(sc) }}\n {% endif %}\n {% endfor %}\n \n {{ return(result) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.8079581}, "macro.dbt.diff_column_data_types": {"unique_id": "macro.dbt.diff_column_data_types", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "name": "diff_column_data_types", "macro_sql": "{% macro diff_column_data_types(source_columns, target_columns) %}\n \n {% set result = [] %}\n {% for sc in source_columns %}\n {% set tc = target_columns | selectattr(\"name\", \"equalto\", sc.name) | list | first %}\n {% if tc %}\n {% if sc.data_type != tc.data_type %}\n {{ result.append( { 'column_name': tc.name, 'new_type': sc.data_type } ) }} \n {% endif %}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.809567}, "macro.dbt.get_merge_sql": {"unique_id": "macro.dbt.get_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "name": "get_merge_sql", "macro_sql": "{% macro get_merge_sql(target, source, unique_key, dest_columns, predicates=none) -%}\n {{ adapter.dispatch('get_merge_sql', 'dbt')(target, source, unique_key, dest_columns, predicates) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.8179982}, "macro.dbt.default__get_merge_sql": {"unique_id": "macro.dbt.default__get_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "name": "default__get_merge_sql", "macro_sql": "{% macro default__get_merge_sql(target, source, unique_key, dest_columns, predicates) -%}\n {%- set predicates = [] if predicates is none else [] + predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set update_columns = config.get('merge_update_columns', default = dest_columns | map(attribute=\"quoted\") | list) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {% if unique_key %}\n {% set unique_key_match %}\n DBT_INTERNAL_SOURCE.{{ unique_key }} = DBT_INTERNAL_DEST.{{ unique_key }}\n {% endset %}\n {% do predicates.append(unique_key_match) %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n {{ sql_header if sql_header is not none }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{ predicates | join(' and ') }}\n\n {% if unique_key %}\n when matched then update set\n {% for column_name in update_columns -%}\n {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}\n {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n {% endif %}\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.8210711}, "macro.dbt.get_delete_insert_merge_sql": {"unique_id": "macro.dbt.get_delete_insert_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "name": "get_delete_insert_merge_sql", "macro_sql": "{% macro get_delete_insert_merge_sql(target, source, unique_key, dest_columns) -%}\n {{ adapter.dispatch('get_delete_insert_merge_sql', 'dbt')(target, source, unique_key, dest_columns) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_delete_insert_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.821682}, "macro.dbt.default__get_delete_insert_merge_sql": {"unique_id": "macro.dbt.default__get_delete_insert_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "name": "default__get_delete_insert_merge_sql", "macro_sql": "{% macro default__get_delete_insert_merge_sql(target, source, unique_key, dest_columns) -%}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n {% if unique_key is not none %}\n delete from {{ target }}\n where ({{ unique_key }}) in (\n select ({{ unique_key }})\n from {{ source }}\n );\n {% endif %}\n\n insert into {{ target }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ source }}\n )\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.8228061}, "macro.dbt.get_insert_overwrite_merge_sql": {"unique_id": "macro.dbt.get_insert_overwrite_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "name": "get_insert_overwrite_merge_sql", "macro_sql": "{% macro get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header=false) -%}\n {{ adapter.dispatch('get_insert_overwrite_merge_sql', 'dbt')(target, source, dest_columns, predicates, include_sql_header) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_insert_overwrite_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.823494}, "macro.dbt.default__get_insert_overwrite_merge_sql": {"unique_id": "macro.dbt.default__get_insert_overwrite_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "name": "default__get_insert_overwrite_merge_sql", "macro_sql": "{% macro default__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) -%}\n {%- set predicates = [] if predicates is none else [] + predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none and include_sql_header }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on FALSE\n\n when not matched by source\n {% if predicates %} and {{ predicates | join(' and ') }} {% endif %}\n then delete\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.825151}, "macro.dbt.is_incremental": {"unique_id": "macro.dbt.is_incremental", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/is_incremental.sql", "original_file_path": "macros/materializations/models/incremental/is_incremental.sql", "name": "is_incremental", "macro_sql": "{% macro is_incremental() %}\n {#-- do not run introspective queries in parsing #}\n {% if not execute %}\n {{ return(False) }}\n {% else %}\n {% set relation = adapter.get_relation(this.database, this.schema, this.table) %}\n {{ return(relation is not none\n and relation.type == 'table'\n and model.config.materialized == 'incremental'\n and not should_full_refresh()) }}\n {% endif %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.should_full_refresh"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.8269792}, "macro.dbt.materialization_incremental_default": {"unique_id": "macro.dbt.materialization_incremental_default", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/incremental.sql", "original_file_path": "macros/materializations/models/incremental/incremental.sql", "name": "materialization_incremental_default", "macro_sql": "{% materialization incremental, default -%}\n\n {% set unique_key = config.get('unique_key') %}\n\n {% set target_relation = this.incorporate(type='table') %}\n {% set existing_relation = load_relation(this) %}\n {% set tmp_relation = make_temp_relation(target_relation) %}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {% set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') %}\n\n {% set tmp_identifier = model['name'] + '__dbt_tmp' %}\n {% set backup_identifier = model['name'] + \"__dbt_backup\" %}\n\n -- the intermediate_ and backup_ relations should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation. This has to happen before\n -- BEGIN, in a separate transaction\n {% set preexisting_intermediate_relation = adapter.get_relation(identifier=tmp_identifier, \n schema=schema,\n database=database) %} \n {% set preexisting_backup_relation = adapter.get_relation(identifier=backup_identifier,\n schema=schema,\n database=database) %}\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set to_drop = [] %}\n\n {# -- first check whether we want to full refresh for source view or config reasons #}\n {% set trigger_full_refresh = (full_refresh_mode or existing_relation.is_view) %}\n\n {% if existing_relation is none %}\n {% set build_sql = create_table_as(False, target_relation, sql) %}\n{% elif trigger_full_refresh %}\n {#-- Make sure the backup doesn't exist so we don't encounter issues with the rename below #}\n {% set tmp_identifier = model['name'] + '__dbt_tmp' %}\n {% set backup_identifier = model['name'] + '__dbt_backup' %}\n {% set intermediate_relation = existing_relation.incorporate(path={\"identifier\": tmp_identifier}) %}\n {% set backup_relation = existing_relation.incorporate(path={\"identifier\": backup_identifier}) %}\n\n {% set build_sql = create_table_as(False, intermediate_relation, sql) %}\n {% set need_swap = true %}\n {% do to_drop.append(backup_relation) %}\n {% else %}\n {% do run_query(create_table_as(True, tmp_relation, sql)) %}\n {% do adapter.expand_target_column_types(\n from_relation=tmp_relation,\n to_relation=target_relation) %}\n {#-- Process schema changes. Returns dict of changes if successful. Use source columns for upserting/merging --#}\n {% set dest_columns = process_schema_changes(on_schema_change, tmp_relation, existing_relation) %}\n {% if not dest_columns %}\n {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %}\n {% endif %}\n {% set build_sql = get_delete_insert_merge_sql(target_relation, tmp_relation, unique_key, dest_columns) %}\n \n {% endif %}\n\n {% call statement(\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% if need_swap %} \n {% do adapter.rename_relation(target_relation, backup_relation) %} \n {% do adapter.rename_relation(intermediate_relation, target_relation) %} \n {% endif %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if existing_relation is none or existing_relation.is_view or should_full_refresh() %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {% do adapter.commit() %}\n\n {% for rel in to_drop %}\n {% do adapter.drop_relation(rel) %}\n {% endfor %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.load_relation", "macro.dbt.make_temp_relation", "macro.dbt.should_full_refresh", "macro.dbt.incremental_validate_on_schema_change", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks", "macro.dbt.create_table_as", "macro.dbt.run_query", "macro.dbt.process_schema_changes", "macro.dbt.get_delete_insert_merge_sql", "macro.dbt.statement", "macro.dbt.persist_docs", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.840609}, "macro.dbt.incremental_validate_on_schema_change": {"unique_id": "macro.dbt.incremental_validate_on_schema_change", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "name": "incremental_validate_on_schema_change", "macro_sql": "{% macro incremental_validate_on_schema_change(on_schema_change, default='ignore') %}\n \n {% if on_schema_change not in ['sync_all_columns', 'append_new_columns', 'fail', 'ignore'] %}\n \n {% set log_message = 'Invalid value for on_schema_change (%s) specified. Setting default value of %s.' % (on_schema_change, default) %}\n {% do log(log_message) %}\n \n {{ return(default) }}\n\n {% else %}\n\n {{ return(on_schema_change) }}\n \n {% endif %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.8552}, "macro.dbt.check_for_schema_changes": {"unique_id": "macro.dbt.check_for_schema_changes", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "name": "check_for_schema_changes", "macro_sql": "{% macro check_for_schema_changes(source_relation, target_relation) %}\n \n {% set schema_changed = False %}\n \n {%- set source_columns = adapter.get_columns_in_relation(source_relation) -%}\n {%- set target_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set source_not_in_target = diff_columns(source_columns, target_columns) -%}\n {%- set target_not_in_source = diff_columns(target_columns, source_columns) -%}\n\n {% set new_target_types = diff_column_data_types(source_columns, target_columns) %}\n\n {% if source_not_in_target != [] %}\n {% set schema_changed = True %}\n {% elif target_not_in_source != [] or new_target_types != [] %}\n {% set schema_changed = True %}\n {% elif new_target_types != [] %}\n {% set schema_changed = True %}\n {% endif %}\n \n {% set changes_dict = {\n 'schema_changed': schema_changed,\n 'source_not_in_target': source_not_in_target,\n 'target_not_in_source': target_not_in_source,\n 'source_columns': source_columns,\n 'target_columns': target_columns,\n 'new_target_types': new_target_types\n } %}\n\n {% set msg %}\n In {{ target_relation }}:\n Schema changed: {{ schema_changed }}\n Source columns not in target: {{ source_not_in_target }}\n Target columns not in source: {{ target_not_in_source }}\n New column types: {{ new_target_types }}\n {% endset %}\n \n {% do log(msg) %}\n\n {{ return(changes_dict) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.diff_columns", "macro.dbt.diff_column_data_types"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.858468}, "macro.dbt.sync_column_schemas": {"unique_id": "macro.dbt.sync_column_schemas", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "name": "sync_column_schemas", "macro_sql": "{% macro sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n \n {%- set add_to_target_arr = schema_changes_dict['source_not_in_target'] -%}\n\n {%- if on_schema_change == 'append_new_columns'-%}\n {%- if add_to_target_arr | length > 0 -%}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, none) -%}\n {%- endif -%}\n \n {% elif on_schema_change == 'sync_all_columns' %}\n {%- set remove_from_target_arr = schema_changes_dict['target_not_in_source'] -%}\n {%- set new_target_types = schema_changes_dict['new_target_types'] -%}\n \n {% if add_to_target_arr | length > 0 or remove_from_target_arr | length > 0 %} \n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, remove_from_target_arr) -%}\n {% endif %}\n\n {% if new_target_types != [] %}\n {% for ntt in new_target_types %}\n {% set column_name = ntt['column_name'] %}\n {% set new_type = ntt['new_type'] %}\n {% do alter_column_type(target_relation, column_name, new_type) %}\n {% endfor %}\n {% endif %}\n \n {% endif %}\n\n {% set schema_change_message %}\n In {{ target_relation }}:\n Schema change approach: {{ on_schema_change }}\n Columns added: {{ add_to_target_arr }}\n Columns removed: {{ remove_from_target_arr }}\n Data types changed: {{ new_target_types }}\n {% endset %}\n \n {% do log(schema_change_message) %}\n \n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.alter_relation_add_remove_columns", "macro.dbt.alter_column_type"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.8616462}, "macro.dbt.process_schema_changes": {"unique_id": "macro.dbt.process_schema_changes", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "name": "process_schema_changes", "macro_sql": "{% macro process_schema_changes(on_schema_change, source_relation, target_relation) %}\n \n {% if on_schema_change == 'ignore' %}\n\n {{ return({}) }}\n\n {% else %}\n \n {% set schema_changes_dict = check_for_schema_changes(source_relation, target_relation) %}\n \n {% if schema_changes_dict['schema_changed'] %}\n \n {% if on_schema_change == 'fail' %}\n \n {% set fail_msg %}\n The source and target schemas on this incremental model are out of sync!\n They can be reconciled in several ways: \n - set the `on_schema_change` config to either append_new_columns or sync_all_columns, depending on your situation.\n - Re-run the incremental model with `full_refresh: True` to update the target schema.\n - update the schema manually and re-run the process.\n {% endset %}\n \n {% do exceptions.raise_compiler_error(fail_msg) %}\n \n {# -- unless we ignore, run the sync operation per the config #}\n {% else %}\n \n {% do sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n \n {% endif %}\n \n {% endif %}\n\n {{ return(schema_changes_dict['source_columns']) }}\n \n {% endif %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.check_for_schema_changes", "macro.dbt.sync_column_schemas"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.8635662}, "macro.dbt.materialization_table_default": {"unique_id": "macro.dbt.materialization_table_default", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/table/table.sql", "original_file_path": "macros/materializations/models/table/table.sql", "name": "materialization_table_default", "macro_sql": "{% materialization table, default %}\n {%- set identifier = model['alias'] -%}\n {%- set tmp_identifier = model['name'] + '__dbt_tmp' -%}\n {%- set backup_identifier = model['name'] + '__dbt_backup' -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set target_relation = api.Relation.create(identifier=identifier,\n schema=schema,\n database=database,\n type='table') -%}\n {%- set intermediate_relation = api.Relation.create(identifier=tmp_identifier,\n schema=schema,\n database=database,\n type='table') -%}\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = adapter.get_relation(identifier=tmp_identifier, \n schema=schema,\n database=database) -%}\n /*\n See ../view/view.sql for more information about this relation.\n */\n {%- set backup_relation_type = 'table' if old_relation is none else old_relation.type -%}\n {%- set backup_relation = api.Relation.create(identifier=backup_identifier,\n schema=schema,\n database=database,\n type=backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = adapter.get_relation(identifier=backup_identifier,\n schema=schema,\n database=database) -%}\n\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_table_as_sql(False, intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n {% if old_relation is not none %}\n {{ adapter.rename_relation(old_relation, backup_relation) }}\n {% endif %}\n\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% do create_indexes(target_relation) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {% do persist_docs(target_relation, model) %}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n -- finally, drop the existing/backup relation after the commit\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n{% endmaterialization %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.get_create_table_as_sql", "macro.dbt.create_indexes", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.874618}, "macro.dbt.get_create_table_as_sql": {"unique_id": "macro.dbt.get_create_table_as_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/table/create_table_as.sql", "original_file_path": "macros/materializations/models/table/create_table_as.sql", "name": "get_create_table_as_sql", "macro_sql": "{% macro get_create_table_as_sql(temporary, relation, sql) -%}\n {{ adapter.dispatch('get_create_table_as_sql', 'dbt')(temporary, relation, sql) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_create_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.8760588}, "macro.dbt.default__get_create_table_as_sql": {"unique_id": "macro.dbt.default__get_create_table_as_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/table/create_table_as.sql", "original_file_path": "macros/materializations/models/table/create_table_as.sql", "name": "default__get_create_table_as_sql", "macro_sql": "{% macro default__get_create_table_as_sql(temporary, relation, sql) -%}\n {{ return(create_table_as(temporary, relation, sql)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.876541}, "macro.dbt.create_table_as": {"unique_id": "macro.dbt.create_table_as", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/table/create_table_as.sql", "original_file_path": "macros/materializations/models/table/create_table_as.sql", "name": "create_table_as", "macro_sql": "{% macro create_table_as(temporary, relation, sql) -%}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, sql) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.877064}, "macro.dbt.default__create_table_as": {"unique_id": "macro.dbt.default__create_table_as", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/table/create_table_as.sql", "original_file_path": "macros/materializations/models/table/create_table_as.sql", "name": "default__create_table_as", "macro_sql": "{% macro default__create_table_as(temporary, relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n \n {{ sql_header if sql_header is not none }}\n \n create {% if temporary: -%}temporary{%- endif %} table\n {{ relation.include(database=(not temporary), schema=(not temporary)) }}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.878151}, "macro.dbt.materialization_view_default": {"unique_id": "macro.dbt.materialization_view_default", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/view/view.sql", "original_file_path": "macros/materializations/models/view/view.sql", "name": "materialization_view_default", "macro_sql": "{%- materialization view, default -%}\n\n {%- set identifier = model['alias'] -%}\n {%- set tmp_identifier = model['name'] + '__dbt_tmp' -%}\n {%- set backup_identifier = model['name'] + '__dbt_backup' -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set target_relation = api.Relation.create(identifier=identifier, schema=schema, database=database,\n type='view') -%}\n {%- set intermediate_relation = api.Relation.create(identifier=tmp_identifier,\n schema=schema, database=database, type='view') -%}\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = adapter.get_relation(identifier=tmp_identifier, \n schema=schema,\n database=database) -%}\n /*\n This relation (probably) doesn't exist yet. If it does exist, it's a leftover from\n a previous run, and we're going to try to drop it immediately. At the end of this\n materialization, we're going to rename the \"old_relation\" to this identifier,\n and then we're going to drop it. In order to make sure we run the correct one of:\n - drop view ...\n - drop table ...\n\n We need to set the type of this relation to be the type of the old_relation, if it exists,\n or else \"view\" as a sane default if it does not. Note that if the old_relation does not\n exist, then there is nothing to move out of the way and subsequentally drop. In that case,\n this relation will be effectively unused.\n */\n {%- set backup_relation_type = 'view' if old_relation is none else old_relation.type -%}\n {%- set backup_relation = api.Relation.create(identifier=backup_identifier,\n schema=schema, database=database,\n type=backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = adapter.get_relation(identifier=backup_identifier,\n schema=schema,\n database=database) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ create_view_as(intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n -- move the existing view out of the way\n {% if old_relation is not none %}\n {{ adapter.rename_relation(old_relation, backup_relation) }}\n {% endif %}\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.create_view_as", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.888351}, "macro.dbt.handle_existing_table": {"unique_id": "macro.dbt.handle_existing_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/view/helpers.sql", "original_file_path": "macros/materializations/models/view/helpers.sql", "name": "handle_existing_table", "macro_sql": "{% macro handle_existing_table(full_refresh, old_relation) %}\n {{ adapter.dispatch('handle_existing_table', 'dbt')(full_refresh, old_relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__handle_existing_table"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.889299}, "macro.dbt.default__handle_existing_table": {"unique_id": "macro.dbt.default__handle_existing_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/view/helpers.sql", "original_file_path": "macros/materializations/models/view/helpers.sql", "name": "default__handle_existing_table", "macro_sql": "{% macro default__handle_existing_table(full_refresh, old_relation) %}\n {{ log(\"Dropping relation \" ~ old_relation ~ \" because it is of type \" ~ old_relation.type) }}\n {{ adapter.drop_relation(old_relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.889876}, "macro.dbt.create_or_replace_view": {"unique_id": "macro.dbt.create_or_replace_view", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/view/create_or_replace_view.sql", "original_file_path": "macros/materializations/models/view/create_or_replace_view.sql", "name": "create_or_replace_view", "macro_sql": "{% macro create_or_replace_view() %}\n {%- set identifier = model['alias'] -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database,\n type='view') -%}\n\n {{ run_hooks(pre_hooks) }}\n\n -- If there's a table with the same name and we weren't told to full refresh,\n -- that's an error. If we were told to full refresh, drop it. This behavior differs\n -- for Snowflake and BigQuery, so multiple dispatch is used.\n {%- if old_relation is not none and old_relation.is_table -%}\n {{ handle_existing_table(should_full_refresh(), old_relation) }}\n {%- endif -%}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(target_relation, sql) }}\n {%- endcall %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.handle_existing_table", "macro.dbt.should_full_refresh", "macro.dbt.statement", "macro.dbt.get_create_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.895395}, "macro.dbt.get_create_view_as_sql": {"unique_id": "macro.dbt.get_create_view_as_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/view/create_view_as.sql", "original_file_path": "macros/materializations/models/view/create_view_as.sql", "name": "get_create_view_as_sql", "macro_sql": "{% macro get_create_view_as_sql(relation, sql) -%}\n {{ adapter.dispatch('get_create_view_as_sql', 'dbt')(relation, sql) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_create_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.8964942}, "macro.dbt.default__get_create_view_as_sql": {"unique_id": "macro.dbt.default__get_create_view_as_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/view/create_view_as.sql", "original_file_path": "macros/materializations/models/view/create_view_as.sql", "name": "default__get_create_view_as_sql", "macro_sql": "{% macro default__get_create_view_as_sql(relation, sql) -%}\n {{ return(create_view_as(relation, sql)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.8969111}, "macro.dbt.create_view_as": {"unique_id": "macro.dbt.create_view_as", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/view/create_view_as.sql", "original_file_path": "macros/materializations/models/view/create_view_as.sql", "name": "create_view_as", "macro_sql": "{% macro create_view_as(relation, sql) -%}\n {{ adapter.dispatch('create_view_as', 'dbt')(relation, sql) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.897373}, "macro.dbt.default__create_view_as": {"unique_id": "macro.dbt.default__create_view_as", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/view/create_view_as.sql", "original_file_path": "macros/materializations/models/view/create_view_as.sql", "name": "default__create_view_as", "macro_sql": "{% macro default__create_view_as(relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n create view {{ relation }} as (\n {{ sql }}\n );\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.898056}, "macro.dbt.materialization_seed_default": {"unique_id": "macro.dbt.materialization_seed_default", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/seed.sql", "original_file_path": "macros/materializations/seeds/seed.sql", "name": "materialization_seed_default", "macro_sql": "{% materialization seed, default %}\n\n {%- set identifier = model['alias'] -%}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set agate_table = load_agate_table() -%}\n {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% set create_table_sql = \"\" %}\n {% if exists_as_view %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a view\".format(old_relation)) }}\n {% elif exists_as_table %}\n {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}\n {% else %}\n {% set create_table_sql = create_csv_table(model, agate_table) %}\n {% endif %}\n\n {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}\n {% set rows_affected = (agate_table.rows | length) %}\n {% set sql = load_csv_rows(model, agate_table) %}\n\n {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}\n {{ create_table_sql }};\n -- dbt seed --\n {{ sql }}\n {% endcall %}\n\n {% set target_relation = this.incorporate(type='table') %}\n {% do persist_docs(target_relation, model) %}\n\n {% if full_refresh_mode or not exists_as_table %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.run_hooks", "macro.dbt.reset_csv_table", "macro.dbt.create_csv_table", "macro.dbt.load_csv_rows", "macro.dbt.noop_statement", "macro.dbt.persist_docs", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.905844}, "macro.dbt.create_csv_table": {"unique_id": "macro.dbt.create_csv_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "create_csv_table", "macro_sql": "{% macro create_csv_table(model, agate_table) -%}\n {{ adapter.dispatch('create_csv_table', 'dbt')(model, agate_table) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.9181492}, "macro.dbt.default__create_csv_table": {"unique_id": "macro.dbt.default__create_csv_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "default__create_csv_table", "macro_sql": "{% macro default__create_csv_table(model, agate_table) %}\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n\n {% set sql %}\n create table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.920773}, "macro.dbt.reset_csv_table": {"unique_id": "macro.dbt.reset_csv_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "reset_csv_table", "macro_sql": "{% macro reset_csv_table(model, full_refresh, old_relation, agate_table) -%}\n {{ adapter.dispatch('reset_csv_table', 'dbt')(model, full_refresh, old_relation, agate_table) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__reset_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.921389}, "macro.dbt.default__reset_csv_table": {"unique_id": "macro.dbt.default__reset_csv_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "default__reset_csv_table", "macro_sql": "{% macro default__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% set sql = \"\" %}\n {% if full_refresh %}\n {{ adapter.drop_relation(old_relation) }}\n {% set sql = create_csv_table(model, agate_table) %}\n {% else %}\n {{ adapter.truncate_relation(old_relation) }}\n {% set sql = \"truncate table \" ~ old_relation %}\n {% endif %}\n\n {{ return(sql) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.922673}, "macro.dbt.get_binding_char": {"unique_id": "macro.dbt.get_binding_char", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "get_binding_char", "macro_sql": "{% macro get_binding_char() -%}\n {{ adapter.dispatch('get_binding_char', 'dbt')() }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.9230442}, "macro.dbt.default__get_binding_char": {"unique_id": "macro.dbt.default__get_binding_char", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "default__get_binding_char", "macro_sql": "{% macro default__get_binding_char() %}\n {{ return('%s') }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.923341}, "macro.dbt.get_batch_size": {"unique_id": "macro.dbt.get_batch_size", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "get_batch_size", "macro_sql": "{% macro get_batch_size() -%}\n {{ return(adapter.dispatch('get_batch_size', 'dbt')()) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_batch_size"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.923764}, "macro.dbt.default__get_batch_size": {"unique_id": "macro.dbt.default__get_batch_size", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "default__get_batch_size", "macro_sql": "{% macro default__get_batch_size() %}\n {{ return(10000) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.924055}, "macro.dbt.get_seed_column_quoted_csv": {"unique_id": "macro.dbt.get_seed_column_quoted_csv", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "get_seed_column_quoted_csv", "macro_sql": "{% macro get_seed_column_quoted_csv(model, column_names) %}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote_seed_column(col, quote_seed_column)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.925326}, "macro.dbt.load_csv_rows": {"unique_id": "macro.dbt.load_csv_rows", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "load_csv_rows", "macro_sql": "{% macro load_csv_rows(model, agate_table) -%}\n {{ adapter.dispatch('load_csv_rows', 'dbt')(model, agate_table) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__load_csv_rows"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.9259362}, "macro.dbt.default__load_csv_rows": {"unique_id": "macro.dbt.default__load_csv_rows", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "default__load_csv_rows", "macro_sql": "{% macro default__load_csv_rows(model, agate_table) %}\n\n {% set batch_size = get_batch_size() %}\n\n {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}\n {% set bindings = [] %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert into {{ this.render() }} ({{ cols_sql }}) values\n {% for row in chunk -%}\n ({%- for column in agate_table.column_names -%}\n {{ get_binding_char() }}\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_batch_size", "macro.dbt.get_seed_column_quoted_csv", "macro.dbt.get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.9291992}, "macro.dbt.generate_alias_name": {"unique_id": "macro.dbt.generate_alias_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/get_custom_name/get_custom_alias.sql", "original_file_path": "macros/get_custom_name/get_custom_alias.sql", "name": "generate_alias_name", "macro_sql": "{% macro generate_alias_name(custom_alias_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_alias_name', 'dbt')(custom_alias_name, node)) %}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__generate_alias_name"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.9304068}, "macro.dbt.default__generate_alias_name": {"unique_id": "macro.dbt.default__generate_alias_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/get_custom_name/get_custom_alias.sql", "original_file_path": "macros/get_custom_name/get_custom_alias.sql", "name": "default__generate_alias_name", "macro_sql": "{% macro default__generate_alias_name(custom_alias_name=none, node=none) -%}\n\n {%- if custom_alias_name is none -%}\n\n {{ node.name }}\n\n {%- else -%}\n\n {{ custom_alias_name | trim }}\n\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.930997}, "macro.dbt.generate_schema_name": {"unique_id": "macro.dbt.generate_schema_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "name": "generate_schema_name", "macro_sql": "{% macro generate_schema_name(custom_schema_name=none, node=none) -%}\n {{ return(adapter.dispatch('generate_schema_name', 'dbt')(custom_schema_name, node)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__generate_schema_name"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.93258}, "macro.dbt.default__generate_schema_name": {"unique_id": "macro.dbt.default__generate_schema_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "name": "default__generate_schema_name", "macro_sql": "{% macro default__generate_schema_name(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if custom_schema_name is none -%}\n\n {{ default_schema }}\n\n {%- else -%}\n\n {{ default_schema }}_{{ custom_schema_name | trim }}\n\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.933256}, "macro.dbt.generate_schema_name_for_env": {"unique_id": "macro.dbt.generate_schema_name_for_env", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "name": "generate_schema_name_for_env", "macro_sql": "{% macro generate_schema_name_for_env(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if target.name == 'prod' and custom_schema_name is not none -%}\n\n {{ custom_schema_name | trim }}\n\n {%- else -%}\n\n {{ default_schema }}\n\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.933998}, "macro.dbt.generate_database_name": {"unique_id": "macro.dbt.generate_database_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "name": "generate_database_name", "macro_sql": "{% macro generate_database_name(custom_database_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_database_name', 'dbt')(custom_database_name, node)) %}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__generate_database_name"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.93523}, "macro.dbt.default__generate_database_name": {"unique_id": "macro.dbt.default__generate_database_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "name": "default__generate_database_name", "macro_sql": "{% macro default__generate_database_name(custom_database_name=none, node=none) -%}\n {%- set default_database = target.database -%}\n {%- if custom_database_name is none -%}\n\n {{ default_database }}\n\n {%- else -%}\n\n {{ custom_database_name }}\n\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.93588}, "macro.dbt.default__test_relationships": {"unique_id": "macro.dbt.default__test_relationships", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/generic_test_sql/relationships.sql", "original_file_path": "macros/generic_test_sql/relationships.sql", "name": "default__test_relationships", "macro_sql": "{% macro default__test_relationships(model, column_name, to, field) %}\n\nwith child as (\n select {{ column_name }} as from_field\n from {{ model }}\n where {{ column_name }} is not null\n),\n\nparent as (\n select {{ field }} as to_field\n from {{ to }}\n)\n\nselect\n from_field\n\nfrom child\nleft join parent\n on child.from_field = parent.to_field\n\nwhere parent.to_field is null\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.9369469}, "macro.dbt.default__test_not_null": {"unique_id": "macro.dbt.default__test_not_null", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/generic_test_sql/not_null.sql", "original_file_path": "macros/generic_test_sql/not_null.sql", "name": "default__test_not_null", "macro_sql": "{% macro default__test_not_null(model, column_name) %}\n\nselect *\nfrom {{ model }}\nwhere {{ column_name }} is null\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.9375691}, "macro.dbt.default__test_unique": {"unique_id": "macro.dbt.default__test_unique", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/generic_test_sql/unique.sql", "original_file_path": "macros/generic_test_sql/unique.sql", "name": "default__test_unique", "macro_sql": "{% macro default__test_unique(model, column_name) %}\n\nselect\n {{ column_name }} as unique_field,\n count(*) as n_records\n\nfrom {{ model }}\nwhere {{ column_name }} is not null\ngroup by {{ column_name }}\nhaving count(*) > 1\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.9383721}, "macro.dbt.default__test_accepted_values": {"unique_id": "macro.dbt.default__test_accepted_values", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/generic_test_sql/accepted_values.sql", "original_file_path": "macros/generic_test_sql/accepted_values.sql", "name": "default__test_accepted_values", "macro_sql": "{% macro default__test_accepted_values(model, column_name, values, quote=True) %}\n\nwith all_values as (\n\n select\n {{ column_name }} as value_field,\n count(*) as n_records\n\n from {{ model }}\n group by {{ column_name }}\n\n)\n\nselect *\nfrom all_values\nwhere value_field not in (\n {% for value in values -%}\n {% if quote -%}\n '{{ value }}'\n {%- else -%}\n {{ value }}\n {%- endif -%}\n {%- if not loop.last -%},{%- endif %}\n {%- endfor %}\n)\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.940022}, "macro.dbt.statement": {"unique_id": "macro.dbt.statement", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "name": "statement", "macro_sql": "{% macro statement(name=None, fetch_result=False, auto_begin=True) -%}\n {%- if execute: -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- set res, table = adapter.execute(sql, auto_begin=auto_begin, fetch=fetch_result) -%}\n {%- if name is not none -%}\n {{ store_result(name, response=res, agate_table=table) }}\n {%- endif -%}\n\n {%- endif -%}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.943184}, "macro.dbt.noop_statement": {"unique_id": "macro.dbt.noop_statement", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "name": "noop_statement", "macro_sql": "{% macro noop_statement(name=None, message=None, code=None, rows_affected=None, res=None) -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_raw_result(name, message=message, code=code, rows_affected=rows_affected, agate_table=res) }}\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.944719}, "macro.dbt.run_query": {"unique_id": "macro.dbt.run_query", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "name": "run_query", "macro_sql": "{% macro run_query(sql) %}\n {% call statement(\"run_query_statement\", fetch_result=true, auto_begin=false) %}\n {{ sql }}\n {% endcall %}\n\n {% do return(load_result(\"run_query_statement\").table) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.94549}, "macro.dbt.convert_datetime": {"unique_id": "macro.dbt.convert_datetime", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "name": "convert_datetime", "macro_sql": "{% macro convert_datetime(date_str, date_fmt) %}\n\n {% set error_msg -%}\n The provided partition date '{{ date_str }}' does not match the expected format '{{ date_fmt }}'\n {%- endset %}\n\n {% set res = try_or_compiler_error(error_msg, modules.datetime.datetime.strptime, date_str.strip(), date_fmt) %}\n {{ return(res) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.950737}, "macro.dbt.dates_in_range": {"unique_id": "macro.dbt.dates_in_range", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "name": "dates_in_range", "macro_sql": "{% macro dates_in_range(start_date_str, end_date_str=none, in_fmt=\"%Y%m%d\", out_fmt=\"%Y%m%d\") %}\n {% set end_date_str = start_date_str if end_date_str is none else end_date_str %}\n\n {% set start_date = convert_datetime(start_date_str, in_fmt) %}\n {% set end_date = convert_datetime(end_date_str, in_fmt) %}\n\n {% set day_count = (end_date - start_date).days %}\n {% if day_count < 0 %}\n {% set msg -%}\n Partition start date is after the end date ({{ start_date }}, {{ end_date }})\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg, model) }}\n {% endif %}\n\n {% set date_list = [] %}\n {% for i in range(0, day_count + 1) %}\n {% set the_date = (modules.datetime.timedelta(days=i) + start_date) %}\n {% if not out_fmt %}\n {% set _ = date_list.append(the_date) %}\n {% else %}\n {% set _ = date_list.append(the_date.strftime(out_fmt)) %}\n {% endif %}\n {% endfor %}\n\n {{ return(date_list) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.convert_datetime"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.954061}, "macro.dbt.partition_range": {"unique_id": "macro.dbt.partition_range", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "name": "partition_range", "macro_sql": "{% macro partition_range(raw_partition_date, date_fmt='%Y%m%d') %}\n {% set partition_range = (raw_partition_date | string).split(\",\") %}\n\n {% if (partition_range | length) == 1 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = none %}\n {% elif (partition_range | length) == 2 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = partition_range[1] %}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Invalid partition time. Expected format: {Start Date}[,{End Date}]. Got: \" ~ raw_partition_date) }}\n {% endif %}\n\n {{ return(dates_in_range(start_date, end_date, in_fmt=date_fmt)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.dates_in_range"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.956046}, "macro.dbt.py_current_timestring": {"unique_id": "macro.dbt.py_current_timestring", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "name": "py_current_timestring", "macro_sql": "{% macro py_current_timestring() %}\n {% set dt = modules.datetime.datetime.now() %}\n {% do return(dt.strftime(\"%Y%m%d%H%M%S%f\")) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.956654}, "macro.dbt.create_schema": {"unique_id": "macro.dbt.create_schema", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "name": "create_schema", "macro_sql": "{% macro create_schema(relation) -%}\n {{ adapter.dispatch('create_schema', 'dbt')(relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__create_schema"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.9578362}, "macro.dbt.default__create_schema": {"unique_id": "macro.dbt.default__create_schema", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "name": "default__create_schema", "macro_sql": "{% macro default__create_schema(relation) -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier() }}\n {% endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.9583108}, "macro.dbt.drop_schema": {"unique_id": "macro.dbt.drop_schema", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "name": "drop_schema", "macro_sql": "{% macro drop_schema(relation) -%}\n {{ adapter.dispatch('drop_schema', 'dbt')(relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__drop_schema"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.9587321}, "macro.dbt.default__drop_schema": {"unique_id": "macro.dbt.default__drop_schema", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "name": "default__drop_schema", "macro_sql": "{% macro default__drop_schema(relation) -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier() }} cascade\n {% endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.9592059}, "macro.dbt.get_create_index_sql": {"unique_id": "macro.dbt.get_create_index_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "name": "get_create_index_sql", "macro_sql": "{% macro get_create_index_sql(relation, index_dict) -%}\n {{ return(adapter.dispatch('get_create_index_sql', 'dbt')(relation, index_dict)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_create_index_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.9605691}, "macro.dbt.default__get_create_index_sql": {"unique_id": "macro.dbt.default__get_create_index_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "name": "default__get_create_index_sql", "macro_sql": "{% macro default__get_create_index_sql(relation, index_dict) -%}\n {% do return(None) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.9609149}, "macro.dbt.create_indexes": {"unique_id": "macro.dbt.create_indexes", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "name": "create_indexes", "macro_sql": "{% macro create_indexes(relation) -%}\n {{ adapter.dispatch('create_indexes', 'dbt')(relation) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.9613152}, "macro.dbt.default__create_indexes": {"unique_id": "macro.dbt.default__create_indexes", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "name": "default__create_indexes", "macro_sql": "{% macro default__create_indexes(relation) -%}\n {%- set _indexes = config.get('indexes', default=[]) -%}\n\n {% for _index_dict in _indexes %}\n {% set create_index_sql = get_create_index_sql(relation, _index_dict) %}\n {% if create_index_sql %}\n {% do run_query(create_index_sql) %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_create_index_sql", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.962351}, "macro.dbt.make_temp_relation": {"unique_id": "macro.dbt.make_temp_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "make_temp_relation", "macro_sql": "{% macro make_temp_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_temp_relation', 'dbt')(base_relation, suffix))}}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_temp_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.9670699}, "macro.dbt.default__make_temp_relation": {"unique_id": "macro.dbt.default__make_temp_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "default__make_temp_relation", "macro_sql": "{% macro default__make_temp_relation(base_relation, suffix) %}\n {% set tmp_identifier = base_relation.identifier ~ suffix %}\n {% set tmp_relation = base_relation.incorporate(\n path={\"identifier\": tmp_identifier}) -%}\n\n {% do return(tmp_relation) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.967857}, "macro.dbt.drop_relation": {"unique_id": "macro.dbt.drop_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "drop_relation", "macro_sql": "{% macro drop_relation(relation) -%}\n {{ return(adapter.dispatch('drop_relation', 'dbt')(relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__drop_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.9683268}, "macro.dbt.default__drop_relation": {"unique_id": "macro.dbt.default__drop_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "default__drop_relation", "macro_sql": "{% macro default__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n drop {{ relation.type }} if exists {{ relation }} cascade\n {%- endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.968875}, "macro.dbt.truncate_relation": {"unique_id": "macro.dbt.truncate_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "truncate_relation", "macro_sql": "{% macro truncate_relation(relation) -%}\n {{ return(adapter.dispatch('truncate_relation', 'dbt')(relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__truncate_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.969353}, "macro.dbt.default__truncate_relation": {"unique_id": "macro.dbt.default__truncate_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "default__truncate_relation", "macro_sql": "{% macro default__truncate_relation(relation) -%}\n {% call statement('truncate_relation') -%}\n truncate table {{ relation }}\n {%- endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.9697769}, "macro.dbt.rename_relation": {"unique_id": "macro.dbt.rename_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "rename_relation", "macro_sql": "{% macro rename_relation(from_relation, to_relation) -%}\n {{ return(adapter.dispatch('rename_relation', 'dbt')(from_relation, to_relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__rename_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.9703019}, "macro.dbt.default__rename_relation": {"unique_id": "macro.dbt.default__rename_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "default__rename_relation", "macro_sql": "{% macro default__rename_relation(from_relation, to_relation) -%}\n {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %}\n {% call statement('rename_relation') -%}\n alter table {{ from_relation }} rename to {{ target_name }}\n {%- endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.971044}, "macro.dbt.get_or_create_relation": {"unique_id": "macro.dbt.get_or_create_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "get_or_create_relation", "macro_sql": "{% macro get_or_create_relation(database, schema, identifier, type) -%}\n {{ return(adapter.dispatch('get_or_create_relation', 'dbt')(database, schema, identifier, type)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_or_create_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.9716868}, "macro.dbt.default__get_or_create_relation": {"unique_id": "macro.dbt.default__get_or_create_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "default__get_or_create_relation", "macro_sql": "{% macro default__get_or_create_relation(database, schema, identifier, type) %}\n {%- set target_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n\n {% if target_relation %}\n {% do return([true, target_relation]) %}\n {% endif %}\n\n {%- set new_relation = api.Relation.create(\n database=database,\n schema=schema,\n identifier=identifier,\n type=type\n ) -%}\n {% do return([false, new_relation]) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.973294}, "macro.dbt.load_relation": {"unique_id": "macro.dbt.load_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "load_relation", "macro_sql": "{% macro load_relation(relation) %}\n {% do return(adapter.get_relation(\n database=relation.database,\n schema=relation.schema,\n identifier=relation.identifier\n )) -%}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.973898}, "macro.dbt.drop_relation_if_exists": {"unique_id": "macro.dbt.drop_relation_if_exists", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "drop_relation_if_exists", "macro_sql": "{% macro drop_relation_if_exists(relation) %}\n {% if relation is not none %}\n {{ adapter.drop_relation(relation) }}\n {% endif %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.974398}, "macro.dbt.current_timestamp": {"unique_id": "macro.dbt.current_timestamp", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "name": "current_timestamp", "macro_sql": "{% macro current_timestamp() -%}\n {{ adapter.dispatch('current_timestamp', 'dbt')() }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.9758022}, "macro.dbt.default__current_timestamp": {"unique_id": "macro.dbt.default__current_timestamp", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "name": "default__current_timestamp", "macro_sql": "{% macro default__current_timestamp() -%}\n {{ exceptions.raise_not_implemented(\n 'current_timestamp macro not implemented for adapter '+adapter.type()) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.9761899}, "macro.dbt.collect_freshness": {"unique_id": "macro.dbt.collect_freshness", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "name": "collect_freshness", "macro_sql": "{% macro collect_freshness(source, loaded_at_field, filter) %}\n {{ return(adapter.dispatch('collect_freshness', 'dbt')(source, loaded_at_field, filter))}}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__collect_freshness"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.9767869}, "macro.dbt.default__collect_freshness": {"unique_id": "macro.dbt.default__collect_freshness", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "name": "default__collect_freshness", "macro_sql": "{% macro default__collect_freshness(source, loaded_at_field, filter) %}\n {% call statement('collect_freshness', fetch_result=True, auto_begin=False) -%}\n select\n max({{ loaded_at_field }}) as max_loaded_at,\n {{ current_timestamp() }} as snapshotted_at\n from {{ source }}\n {% if filter %}\n where {{ filter }}\n {% endif %}\n {% endcall %}\n {{ return(load_result('collect_freshness').table) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.977885}, "macro.dbt.alter_column_comment": {"unique_id": "macro.dbt.alter_column_comment", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "name": "alter_column_comment", "macro_sql": "{% macro alter_column_comment(relation, column_dict) -%}\n {{ return(adapter.dispatch('alter_column_comment', 'dbt')(relation, column_dict)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.979867}, "macro.dbt.default__alter_column_comment": {"unique_id": "macro.dbt.default__alter_column_comment", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "name": "default__alter_column_comment", "macro_sql": "{% macro default__alter_column_comment(relation, column_dict) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_column_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.98031}, "macro.dbt.alter_relation_comment": {"unique_id": "macro.dbt.alter_relation_comment", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "name": "alter_relation_comment", "macro_sql": "{% macro alter_relation_comment(relation, relation_comment) -%}\n {{ return(adapter.dispatch('alter_relation_comment', 'dbt')(relation, relation_comment)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__alter_relation_comment"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.980829}, "macro.dbt.default__alter_relation_comment": {"unique_id": "macro.dbt.default__alter_relation_comment", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "name": "default__alter_relation_comment", "macro_sql": "{% macro default__alter_relation_comment(relation, relation_comment) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_relation_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.9812522}, "macro.dbt.persist_docs": {"unique_id": "macro.dbt.persist_docs", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "name": "persist_docs", "macro_sql": "{% macro persist_docs(relation, model, for_relation=true, for_columns=true) -%}\n {{ return(adapter.dispatch('persist_docs', 'dbt')(relation, model, for_relation, for_columns)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.981956}, "macro.dbt.default__persist_docs": {"unique_id": "macro.dbt.default__persist_docs", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "name": "default__persist_docs", "macro_sql": "{% macro default__persist_docs(relation, model, for_relation, for_columns) -%}\n {% if for_relation and config.persist_relation_docs() and model.description %}\n {% do run_query(alter_relation_comment(relation, model.description)) %}\n {% endif %}\n\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {% do run_query(alter_column_comment(relation, model.columns)) %}\n {% endif %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.run_query", "macro.dbt.alter_relation_comment", "macro.dbt.alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.9832299}, "macro.dbt.get_catalog": {"unique_id": "macro.dbt.get_catalog", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "get_catalog", "macro_sql": "{% macro get_catalog(information_schema, schemas) -%}\n {{ return(adapter.dispatch('get_catalog', 'dbt')(information_schema, schemas)) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_catalog"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.9874718}, "macro.dbt.default__get_catalog": {"unique_id": "macro.dbt.default__get_catalog", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "default__get_catalog", "macro_sql": "{% macro default__get_catalog(information_schema, schemas) -%}\n\n {% set typename = adapter.type() %}\n {% set msg -%}\n get_catalog not implemented for {{ typename }}\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.9881449}, "macro.dbt.information_schema_name": {"unique_id": "macro.dbt.information_schema_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "information_schema_name", "macro_sql": "{% macro information_schema_name(database) %}\n {{ return(adapter.dispatch('information_schema_name', 'dbt')(database)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__information_schema_name"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.98862}, "macro.dbt.default__information_schema_name": {"unique_id": "macro.dbt.default__information_schema_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "default__information_schema_name", "macro_sql": "{% macro default__information_schema_name(database) -%}\n {%- if database -%}\n {{ database }}.INFORMATION_SCHEMA\n {%- else -%}\n INFORMATION_SCHEMA\n {%- endif -%}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.989036}, "macro.dbt.list_schemas": {"unique_id": "macro.dbt.list_schemas", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "list_schemas", "macro_sql": "{% macro list_schemas(database) -%}\n {{ return(adapter.dispatch('list_schemas', 'dbt')(database)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__list_schemas"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.989502}, "macro.dbt.default__list_schemas": {"unique_id": "macro.dbt.default__list_schemas", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "default__list_schemas", "macro_sql": "{% macro default__list_schemas(database) -%}\n {% set sql %}\n select distinct schema_name\n from {{ information_schema_name(database) }}.SCHEMATA\n where catalog_name ilike '{{ database }}'\n {% endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.information_schema_name", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.990141}, "macro.dbt.check_schema_exists": {"unique_id": "macro.dbt.check_schema_exists", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "check_schema_exists", "macro_sql": "{% macro check_schema_exists(information_schema, schema) -%}\n {{ return(adapter.dispatch('check_schema_exists', 'dbt')(information_schema, schema)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__check_schema_exists"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.9906712}, "macro.dbt.default__check_schema_exists": {"unique_id": "macro.dbt.default__check_schema_exists", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "default__check_schema_exists", "macro_sql": "{% macro default__check_schema_exists(information_schema, schema) -%}\n {% set sql -%}\n select count(*)\n from {{ information_schema.replace(information_schema_view='SCHEMATA') }}\n where catalog_name='{{ information_schema.database }}'\n and schema_name='{{ schema }}'\n {%- endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.991599}, "macro.dbt.list_relations_without_caching": {"unique_id": "macro.dbt.list_relations_without_caching", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "list_relations_without_caching", "macro_sql": "{% macro list_relations_without_caching(schema_relation) %}\n {{ return(adapter.dispatch('list_relations_without_caching', 'dbt')(schema_relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__list_relations_without_caching"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.992081}, "macro.dbt.default__list_relations_without_caching": {"unique_id": "macro.dbt.default__list_relations_without_caching", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "default__list_relations_without_caching", "macro_sql": "{% macro default__list_relations_without_caching(schema_relation) %}\n {{ exceptions.raise_not_implemented(\n 'list_relations_without_caching macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.9924932}, "macro.dbt.get_columns_in_relation": {"unique_id": "macro.dbt.get_columns_in_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "get_columns_in_relation", "macro_sql": "{% macro get_columns_in_relation(relation) -%}\n {{ return(adapter.dispatch('get_columns_in_relation', 'dbt')(relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_columns_in_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.9975512}, "macro.dbt.default__get_columns_in_relation": {"unique_id": "macro.dbt.default__get_columns_in_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "default__get_columns_in_relation", "macro_sql": "{% macro default__get_columns_in_relation(relation) -%}\n {{ exceptions.raise_not_implemented(\n 'get_columns_in_relation macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.997951}, "macro.dbt.sql_convert_columns_in_relation": {"unique_id": "macro.dbt.sql_convert_columns_in_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "sql_convert_columns_in_relation", "macro_sql": "{% macro sql_convert_columns_in_relation(table) -%}\n {% set columns = [] %}\n {% for row in table %}\n {% do columns.append(api.Column(*row)) %}\n {% endfor %}\n {{ return(columns) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.9987469}, "macro.dbt.get_columns_in_query": {"unique_id": "macro.dbt.get_columns_in_query", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "get_columns_in_query", "macro_sql": "{% macro get_columns_in_query(select_sql) -%}\n {{ return(adapter.dispatch('get_columns_in_query', 'dbt')(select_sql)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_columns_in_query"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058600.9992452}, "macro.dbt.default__get_columns_in_query": {"unique_id": "macro.dbt.default__get_columns_in_query", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "default__get_columns_in_query", "macro_sql": "{% macro default__get_columns_in_query(select_sql) %}\n {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%}\n select * from (\n {{ select_sql }}\n ) as __dbt_sbq\n where false\n limit 0\n {% endcall %}\n\n {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058601.000154}, "macro.dbt.alter_column_type": {"unique_id": "macro.dbt.alter_column_type", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "alter_column_type", "macro_sql": "{% macro alter_column_type(relation, column_name, new_column_type) -%}\n {{ return(adapter.dispatch('alter_column_type', 'dbt')(relation, column_name, new_column_type)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__alter_column_type"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058601.0007622}, "macro.dbt.default__alter_column_type": {"unique_id": "macro.dbt.default__alter_column_type", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "default__alter_column_type", "macro_sql": "{% macro default__alter_column_type(relation, column_name, new_column_type) -%}\n {#\n 1. Create a new column (w/ temp name and correct type)\n 2. Copy data over to it\n 3. Drop the existing column (cascade!)\n 4. Rename the new column to existing column\n #}\n {%- set tmp_column = column_name + \"__dbt_alter\" -%}\n\n {% call statement('alter_column_type') %}\n alter table {{ relation }} add column {{ adapter.quote(tmp_column) }} {{ new_column_type }};\n update {{ relation }} set {{ adapter.quote(tmp_column) }} = {{ adapter.quote(column_name) }};\n alter table {{ relation }} drop column {{ adapter.quote(column_name) }} cascade;\n alter table {{ relation }} rename column {{ adapter.quote(tmp_column) }} to {{ adapter.quote(column_name) }}\n {% endcall %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058601.0023592}, "macro.dbt.alter_relation_add_remove_columns": {"unique_id": "macro.dbt.alter_relation_add_remove_columns", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "alter_relation_add_remove_columns", "macro_sql": "{% macro alter_relation_add_remove_columns(relation, add_columns = none, remove_columns = none) -%}\n {{ return(adapter.dispatch('alter_relation_add_remove_columns', 'dbt')(relation, add_columns, remove_columns)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__alter_relation_add_remove_columns"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058601.003041}, "macro.dbt.default__alter_relation_add_remove_columns": {"unique_id": "macro.dbt.default__alter_relation_add_remove_columns", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "default__alter_relation_add_remove_columns", "macro_sql": "{% macro default__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}\n \n {% if add_columns is none %}\n {% set add_columns = [] %}\n {% endif %}\n {% if remove_columns is none %}\n {% set remove_columns = [] %}\n {% endif %}\n \n {% set sql -%}\n \n alter {{ relation.type }} {{ relation }}\n \n {% for column in add_columns %}\n add column {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }}\n {% endfor %}{{ ',' if add_columns and remove_columns }}\n \n {% for column in remove_columns %}\n drop column {{ column.name }}{{ ',' if not loop.last }}\n {% endfor %}\n \n {%- endset -%}\n\n {% do run_query(sql) %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058601.005121}, "macro.dbt.test_unique": {"unique_id": "macro.dbt.test_unique", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "name": "test_unique", "macro_sql": "{% test unique(model, column_name) %}\n {% set macro = adapter.dispatch('test_unique', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__test_unique"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058601.006654}, "macro.dbt.test_not_null": {"unique_id": "macro.dbt.test_not_null", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "name": "test_not_null", "macro_sql": "{% test not_null(model, column_name) %}\n {% set macro = adapter.dispatch('test_not_null', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__test_not_null"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058601.007256}, "macro.dbt.test_accepted_values": {"unique_id": "macro.dbt.test_accepted_values", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "name": "test_accepted_values", "macro_sql": "{% test accepted_values(model, column_name, values, quote=True) %}\n {% set macro = adapter.dispatch('test_accepted_values', 'dbt') %}\n {{ macro(model, column_name, values, quote) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__test_accepted_values"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058601.007999}, "macro.dbt.test_relationships": {"unique_id": "macro.dbt.test_relationships", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "name": "test_relationships", "macro_sql": "{% test relationships(model, column_name, to, field) %}\n {% set macro = adapter.dispatch('test_relationships', 'dbt') %}\n {{ macro(model, column_name, to, field) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__test_relationships"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058601.008718}}, "docs": {"dbt.__overview__": {"unique_id": "dbt.__overview__", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "overview.md", "original_file_path": "docs/overview.md", "name": "__overview__", "block_contents": "### Welcome!\n\nWelcome to the auto-generated documentation for your dbt project!\n\n### Navigation\n\nYou can use the `Project` and `Database` navigation tabs on the left side of the window to explore the models\nin your project.\n\n#### Project Tab\nThe `Project` tab mirrors the directory structure of your dbt project. In this tab, you can see all of the\nmodels defined in your dbt project, as well as models imported from dbt packages.\n\n#### Database Tab\nThe `Database` tab also exposes your models, but in a format that looks more like a database explorer. This view\nshows relations (tables and views) grouped into database schemas. Note that ephemeral models are _not_ shown\nin this interface, as they do not exist in the database.\n\n### Graph Exploration\nYou can click the blue icon on the bottom-right corner of the page to view the lineage graph of your models.\n\nOn model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the `Expand`\nbutton at the top-right of this lineage pane, you'll be able to see all of the models that are used to build,\nor are built from, the model you're exploring.\n\nOnce expanded, you'll be able to use the `--select` and `--exclude` model selection syntax to filter the\nmodels in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax).\n\nNote that you can also right-click on models to interactively filter and explore the graph.\n\n---\n\n### More information\n\n- [What is dbt](https://docs.getdbt.com/docs/introduction)?\n- Read the [dbt viewpoint](https://docs.getdbt.com/docs/viewpoint)\n- [Installation](https://docs.getdbt.com/docs/installation)\n- Join the [dbt Community](https://www.getdbt.com/community/) for questions and discussion"}}, "exposures": {}, "metrics": {"metric.test.my_metric": {"fqn": ["test", "my_metric"], "unique_id": "metric.test.my_metric", "package_name": "test", "root_path": "/Users/jerco/dev/scratch/testy", "path": "metric.yml", "original_file_path": "models/metric.yml", "model": "ref('my_model')", "name": "my_metric", "description": "", "label": "Count records", "type": "count", "sql": "*", "timestamp": "updated_at", "filters": [], "time_grains": ["day"], "dimensions": [], "resource_type": "metric", "meta": {}, "tags": [], "sources": [], "depends_on": {"macros": [], "nodes": ["model.test.my_model"]}, "refs": [["my_model"]], "created_at": 1663058601.2723079}}, "selectors": {}, "disabled": {}, "parent_map": {"model.test.my_model": [], "metric.test.my_metric": ["model.test.my_model"]}, "child_map": {"model.test.my_model": ["metric.test.my_metric"], "metric.test.my_metric": []}} ================================================ FILE: tests/functional/artifacts/data/state/v5/manifest.json ================================================ {"metadata": {"dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v5.json", "dbt_version": "1.1.2", "generated_at": "2022-09-13T08:43:05.173401Z", "invocation_id": "46690f0c-35b6-44f7-95bc-3a91cbf87484", "env": {}, "project_id": "098f6bcd4621d373cade4e832627b4f6", "user_id": null, "send_anonymous_usage_stats": false, "adapter_type": "postgres"}, "nodes": {"model.test.my_model": {"raw_sql": "select 1 as id", "resource_type": "model", "depends_on": {"macros": [], "nodes": []}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "post-hook": [], "pre-hook": []}, "database": "jerco", "schema": "dbt_jcohen", "fqn": ["test", "my_model"], "unique_id": "model.test.my_model", "package_name": "test", "root_path": "/Users/jerco/dev/scratch/testy", "path": "my_model.sql", "original_file_path": "models/my_model.sql", "name": "my_model", "alias": "my_model", "checksum": {"name": "sha256", "checksum": "479636cb85ce8d3b0f8db5ff13cf338b61254ad98d905630eac61f963e719e9d"}, "tags": [], "refs": [], "sources": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true}, "patch_path": null, "compiled_path": null, "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1663058585.790391}}, "sources": {}, "macros": {"macro.dbt_postgres.postgres__get_catalog": {"unique_id": "macro.dbt_postgres.postgres__get_catalog", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/catalog.sql", "original_file_path": "macros/catalog.sql", "name": "postgres__get_catalog", "macro_sql": "{% macro postgres__get_catalog(information_schema, schemas) -%}\n\n {%- call statement('catalog', fetch_result=True) -%}\n {#\n If the user has multiple databases set and the first one is wrong, this will fail.\n But we won't fail in the case where there are multiple quoting-difference-only dbs, which is better.\n #}\n {% set database = information_schema.database %}\n {{ adapter.verify_database(database) }}\n\n select\n '{{ database }}' as table_database,\n sch.nspname as table_schema,\n tbl.relname as table_name,\n case tbl.relkind\n when 'v' then 'VIEW'\n else 'BASE TABLE'\n end as table_type,\n tbl_desc.description as table_comment,\n col.attname as column_name,\n col.attnum as column_index,\n pg_catalog.format_type(col.atttypid, col.atttypmod) as column_type,\n col_desc.description as column_comment,\n pg_get_userbyid(tbl.relowner) as table_owner\n\n from pg_catalog.pg_namespace sch\n join pg_catalog.pg_class tbl on tbl.relnamespace = sch.oid\n join pg_catalog.pg_attribute col on col.attrelid = tbl.oid\n left outer join pg_catalog.pg_description tbl_desc on (tbl_desc.objoid = tbl.oid and tbl_desc.objsubid = 0)\n left outer join pg_catalog.pg_description col_desc on (col_desc.objoid = tbl.oid and col_desc.objsubid = col.attnum)\n\n where (\n {%- for schema in schemas -%}\n upper(sch.nspname) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n )\n and not pg_is_other_temp_schema(sch.oid) -- not a temporary schema belonging to another session\n and tbl.relpersistence in ('p', 'u') -- [p]ermanent table or [u]nlogged table. Exclude [t]emporary tables\n and tbl.relkind in ('r', 'v', 'f', 'p') -- o[r]dinary table, [v]iew, [f]oreign table, [p]artitioned table. Other values are [i]ndex, [S]equence, [c]omposite type, [t]OAST table, [m]aterialized view\n and col.attnum > 0 -- negative numbers are used for system columns such as oid\n and not col.attisdropped -- column as not been dropped\n\n order by\n sch.nspname,\n tbl.relname,\n col.attnum\n\n {%- endcall -%}\n\n {{ return(load_result('catalog').table) }}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.2411761}, "macro.dbt_postgres.postgres_get_relations": {"unique_id": "macro.dbt_postgres.postgres_get_relations", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/relations.sql", "original_file_path": "macros/relations.sql", "name": "postgres_get_relations", "macro_sql": "{% macro postgres_get_relations () -%}\n\n {#\n -- in pg_depend, objid is the dependent, refobjid is the referenced object\n -- > a pg_depend entry indicates that the referenced object cannot be\n -- > dropped without also dropping the dependent object.\n #}\n\n {%- call statement('relations', fetch_result=True) -%}\n with relation as (\n select\n pg_rewrite.ev_class as class,\n pg_rewrite.oid as id\n from pg_rewrite\n ),\n class as (\n select\n oid as id,\n relname as name,\n relnamespace as schema,\n relkind as kind\n from pg_class\n ),\n dependency as (\n select distinct\n pg_depend.objid as id,\n pg_depend.refobjid as ref\n from pg_depend\n ),\n schema as (\n select\n pg_namespace.oid as id,\n pg_namespace.nspname as name\n from pg_namespace\n where nspname != 'information_schema' and nspname not like 'pg\\_%'\n ),\n referenced as (\n select\n relation.id AS id,\n referenced_class.name ,\n referenced_class.schema ,\n referenced_class.kind\n from relation\n join class as referenced_class on relation.class=referenced_class.id\n where referenced_class.kind in ('r', 'v')\n ),\n relationships as (\n select\n referenced.name as referenced_name,\n referenced.schema as referenced_schema_id,\n dependent_class.name as dependent_name,\n dependent_class.schema as dependent_schema_id,\n referenced.kind as kind\n from referenced\n join dependency on referenced.id=dependency.id\n join class as dependent_class on dependency.ref=dependent_class.id\n where\n (referenced.name != dependent_class.name or\n referenced.schema != dependent_class.schema)\n )\n\n select\n referenced_schema.name as referenced_schema,\n relationships.referenced_name as referenced_name,\n dependent_schema.name as dependent_schema,\n relationships.dependent_name as dependent_name\n from relationships\n join schema as dependent_schema on relationships.dependent_schema_id=dependent_schema.id\n join schema as referenced_schema on relationships.referenced_schema_id=referenced_schema.id\n group by referenced_schema, referenced_name, dependent_schema, dependent_name\n order by referenced_schema, referenced_name, dependent_schema, dependent_name;\n\n {%- endcall -%}\n\n {{ return(load_result('relations').table) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.2429922}, "macro.dbt_postgres.postgres__create_table_as": {"unique_id": "macro.dbt_postgres.postgres__create_table_as", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__create_table_as", "macro_sql": "{% macro postgres__create_table_as(temporary, relation, sql) -%}\n {%- set unlogged = config.get('unlogged', default=false) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary -%}\n temporary\n {%- elif unlogged -%}\n unlogged\n {%- endif %} table {{ relation }}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.258873}, "macro.dbt_postgres.postgres__get_create_index_sql": {"unique_id": "macro.dbt_postgres.postgres__get_create_index_sql", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__get_create_index_sql", "macro_sql": "{% macro postgres__get_create_index_sql(relation, index_dict) -%}\n {%- set index_config = adapter.parse_index(index_dict) -%}\n {%- set comma_separated_columns = \", \".join(index_config.columns) -%}\n {%- set index_name = index_config.render(relation) -%}\n\n create {% if index_config.unique -%}\n unique\n {%- endif %} index if not exists\n \"{{ index_name }}\"\n on {{ relation }} {% if index_config.type -%}\n using {{ index_config.type }}\n {%- endif %}\n ({{ comma_separated_columns }});\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.260246}, "macro.dbt_postgres.postgres__create_schema": {"unique_id": "macro.dbt_postgres.postgres__create_schema", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__create_schema", "macro_sql": "{% macro postgres__create_schema(relation) -%}\n {% if relation.database -%}\n {{ adapter.verify_database(relation.database) }}\n {%- endif -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier().include(database=False) }}\n {%- endcall -%}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.261101}, "macro.dbt_postgres.postgres__drop_schema": {"unique_id": "macro.dbt_postgres.postgres__drop_schema", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__drop_schema", "macro_sql": "{% macro postgres__drop_schema(relation) -%}\n {% if relation.database -%}\n {{ adapter.verify_database(relation.database) }}\n {%- endif -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier().include(database=False) }} cascade\n {%- endcall -%}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.2619379}, "macro.dbt_postgres.postgres__get_columns_in_relation": {"unique_id": "macro.dbt_postgres.postgres__get_columns_in_relation", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__get_columns_in_relation", "macro_sql": "{% macro postgres__get_columns_in_relation(relation) -%}\n {% call statement('get_columns_in_relation', fetch_result=True) %}\n select\n column_name,\n data_type,\n character_maximum_length,\n numeric_precision,\n numeric_scale\n\n from {{ relation.information_schema('columns') }}\n where table_name = '{{ relation.identifier }}'\n {% if relation.schema %}\n and table_schema = '{{ relation.schema }}'\n {% endif %}\n order by ordinal_position\n\n {% endcall %}\n {% set table = load_result('get_columns_in_relation').table %}\n {{ return(sql_convert_columns_in_relation(table)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.sql_convert_columns_in_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.263221}, "macro.dbt_postgres.postgres__list_relations_without_caching": {"unique_id": "macro.dbt_postgres.postgres__list_relations_without_caching", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__list_relations_without_caching", "macro_sql": "{% macro postgres__list_relations_without_caching(schema_relation) %}\n {% call statement('list_relations_without_caching', fetch_result=True) -%}\n select\n '{{ schema_relation.database }}' as database,\n tablename as name,\n schemaname as schema,\n 'table' as type\n from pg_tables\n where schemaname ilike '{{ schema_relation.schema }}'\n union all\n select\n '{{ schema_relation.database }}' as database,\n viewname as name,\n schemaname as schema,\n 'view' as type\n from pg_views\n where schemaname ilike '{{ schema_relation.schema }}'\n {% endcall %}\n {{ return(load_result('list_relations_without_caching').table) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.264239}, "macro.dbt_postgres.postgres__information_schema_name": {"unique_id": "macro.dbt_postgres.postgres__information_schema_name", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__information_schema_name", "macro_sql": "{% macro postgres__information_schema_name(database) -%}\n {% if database_name -%}\n {{ adapter.verify_database(database_name) }}\n {%- endif -%}\n information_schema\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.264697}, "macro.dbt_postgres.postgres__list_schemas": {"unique_id": "macro.dbt_postgres.postgres__list_schemas", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__list_schemas", "macro_sql": "{% macro postgres__list_schemas(database) %}\n {% if database -%}\n {{ adapter.verify_database(database) }}\n {%- endif -%}\n {% call statement('list_schemas', fetch_result=True, auto_begin=False) %}\n select distinct nspname from pg_namespace\n {% endcall %}\n {{ return(load_result('list_schemas').table) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.265624}, "macro.dbt_postgres.postgres__check_schema_exists": {"unique_id": "macro.dbt_postgres.postgres__check_schema_exists", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__check_schema_exists", "macro_sql": "{% macro postgres__check_schema_exists(information_schema, schema) -%}\n {% if information_schema.database -%}\n {{ adapter.verify_database(information_schema.database) }}\n {%- endif -%}\n {% call statement('check_schema_exists', fetch_result=True, auto_begin=False) %}\n select count(*) from pg_namespace where nspname = '{{ schema }}'\n {% endcall %}\n {{ return(load_result('check_schema_exists').table) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.2666838}, "macro.dbt_postgres.postgres__current_timestamp": {"unique_id": "macro.dbt_postgres.postgres__current_timestamp", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__current_timestamp", "macro_sql": "{% macro postgres__current_timestamp() -%}\n now()\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.266898}, "macro.dbt_postgres.postgres__snapshot_string_as_time": {"unique_id": "macro.dbt_postgres.postgres__snapshot_string_as_time", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__snapshot_string_as_time", "macro_sql": "{% macro postgres__snapshot_string_as_time(timestamp) -%}\n {%- set result = \"'\" ~ timestamp ~ \"'::timestamp without time zone\" -%}\n {{ return(result) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.2673979}, "macro.dbt_postgres.postgres__snapshot_get_time": {"unique_id": "macro.dbt_postgres.postgres__snapshot_get_time", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__snapshot_get_time", "macro_sql": "{% macro postgres__snapshot_get_time() -%}\n {{ current_timestamp() }}::timestamp without time zone\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.267664}, "macro.dbt_postgres.postgres__make_temp_relation": {"unique_id": "macro.dbt_postgres.postgres__make_temp_relation", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__make_temp_relation", "macro_sql": "{% macro postgres__make_temp_relation(base_relation, suffix) %}\n {% set dt = modules.datetime.datetime.now() %}\n {% set dtstring = dt.strftime(\"%H%M%S%f\") %}\n {% set suffix_length = suffix|length + dtstring|length %}\n {% set relation_max_name_length = 63 %}\n {% if suffix_length > relation_max_name_length %}\n {% do exceptions.raise_compiler_error('Temp relation suffix is too long (' ~ suffix|length ~ ' characters). Maximum length is ' ~ (relation_max_name_length - dtstring|length) ~ ' characters.') %}\n {% endif %}\n {% set tmp_identifier = base_relation.identifier[:relation_max_name_length - suffix_length] ~ suffix ~ dtstring %}\n {% do return(base_relation.incorporate(\n path={\n \"identifier\": tmp_identifier,\n \"schema\": none,\n \"database\": none\n })) -%}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.270232}, "macro.dbt_postgres.postgres_escape_comment": {"unique_id": "macro.dbt_postgres.postgres_escape_comment", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres_escape_comment", "macro_sql": "{% macro postgres_escape_comment(comment) -%}\n {% if comment is not string %}\n {% do exceptions.raise_compiler_error('cannot escape a non-string: ' ~ comment) %}\n {% endif %}\n {%- set magic = '$dbt_comment_literal_block$' -%}\n {%- if magic in comment -%}\n {%- do exceptions.raise_compiler_error('The string ' ~ magic ~ ' is not allowed in comments.') -%}\n {%- endif -%}\n {{ magic }}{{ comment }}{{ magic }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.271394}, "macro.dbt_postgres.postgres__alter_relation_comment": {"unique_id": "macro.dbt_postgres.postgres__alter_relation_comment", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__alter_relation_comment", "macro_sql": "{% macro postgres__alter_relation_comment(relation, comment) %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n comment on {{ relation.type }} {{ relation }} is {{ escaped_comment }};\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres_escape_comment"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.271986}, "macro.dbt_postgres.postgres__alter_column_comment": {"unique_id": "macro.dbt_postgres.postgres__alter_column_comment", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__alter_column_comment", "macro_sql": "{% macro postgres__alter_column_comment(relation, column_dict) %}\n {% set existing_columns = adapter.get_columns_in_relation(relation) | map(attribute=\"name\") | list %}\n {% for column_name in column_dict if (column_name in existing_columns) %}\n {% set comment = column_dict[column_name]['description'] %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n comment on column {{ relation }}.{{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }} is {{ escaped_comment }};\n {% endfor %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres_escape_comment"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.273597}, "macro.dbt_postgres.postgres__snapshot_merge_sql": {"unique_id": "macro.dbt_postgres.postgres__snapshot_merge_sql", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/materializations/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshot_merge.sql", "name": "postgres__snapshot_merge_sql", "macro_sql": "{% macro postgres__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n update {{ target }}\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n from {{ source }} as DBT_INTERNAL_SOURCE\n where DBT_INTERNAL_SOURCE.dbt_scd_id::text = {{ target }}.dbt_scd_id::text\n and DBT_INTERNAL_SOURCE.dbt_change_type::text in ('update'::text, 'delete'::text)\n and {{ target }}.dbt_valid_to is null;\n\n insert into {{ target }} ({{ insert_cols_csv }})\n select {% for column in insert_cols -%}\n DBT_INTERNAL_SOURCE.{{ column }} {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n from {{ source }} as DBT_INTERNAL_SOURCE\n where DBT_INTERNAL_SOURCE.dbt_change_type::text = 'insert'::text;\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.275718}, "macro.dbt.run_hooks": {"unique_id": "macro.dbt.run_hooks", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "name": "run_hooks", "macro_sql": "{% macro run_hooks(hooks, inside_transaction=True) %}\n {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %}\n {% if not inside_transaction and loop.first %}\n {% call statement(auto_begin=inside_transaction) %}\n commit;\n {% endcall %}\n {% endif %}\n {% set rendered = render(hook.get('sql')) | trim %}\n {% if (rendered | length) > 0 %}\n {% call statement(auto_begin=inside_transaction) %}\n {{ rendered }}\n {% endcall %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.278846}, "macro.dbt.make_hook_config": {"unique_id": "macro.dbt.make_hook_config", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "name": "make_hook_config", "macro_sql": "{% macro make_hook_config(sql, inside_transaction) %}\n {{ tojson({\"sql\": sql, \"transaction\": inside_transaction}) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.2793732}, "macro.dbt.before_begin": {"unique_id": "macro.dbt.before_begin", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "name": "before_begin", "macro_sql": "{% macro before_begin(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.2797408}, "macro.dbt.in_transaction": {"unique_id": "macro.dbt.in_transaction", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "name": "in_transaction", "macro_sql": "{% macro in_transaction(sql) %}\n {{ make_hook_config(sql, inside_transaction=True) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.280107}, "macro.dbt.after_commit": {"unique_id": "macro.dbt.after_commit", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "name": "after_commit", "macro_sql": "{% macro after_commit(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.280468}, "macro.dbt.set_sql_header": {"unique_id": "macro.dbt.set_sql_header", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "name": "set_sql_header", "macro_sql": "{% macro set_sql_header(config) -%}\n {{ config.set('sql_header', caller()) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.28161}, "macro.dbt.should_full_refresh": {"unique_id": "macro.dbt.should_full_refresh", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "name": "should_full_refresh", "macro_sql": "{% macro should_full_refresh() %}\n {% set config_full_refresh = config.get('full_refresh') %}\n {% if config_full_refresh is none %}\n {% set config_full_refresh = flags.FULL_REFRESH %}\n {% endif %}\n {% do return(config_full_refresh) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.28241}, "macro.dbt.should_store_failures": {"unique_id": "macro.dbt.should_store_failures", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "name": "should_store_failures", "macro_sql": "{% macro should_store_failures() %}\n {% set config_store_failures = config.get('store_failures') %}\n {% if config_store_failures is none %}\n {% set config_store_failures = flags.STORE_FAILURES %}\n {% endif %}\n {% do return(config_store_failures) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.283215}, "macro.dbt.snapshot_merge_sql": {"unique_id": "macro.dbt.snapshot_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", "name": "snapshot_merge_sql", "macro_sql": "{% macro snapshot_merge_sql(target, source, insert_cols) -%}\n {{ adapter.dispatch('snapshot_merge_sql', 'dbt')(target, source, insert_cols) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__snapshot_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.284517}, "macro.dbt.default__snapshot_merge_sql": {"unique_id": "macro.dbt.default__snapshot_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", "name": "default__snapshot_merge_sql", "macro_sql": "{% macro default__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on DBT_INTERNAL_SOURCE.dbt_scd_id = DBT_INTERNAL_DEST.dbt_scd_id\n\n when matched\n and DBT_INTERNAL_DEST.dbt_valid_to is null\n and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')\n then update\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n\n when not matched\n and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'\n then insert ({{ insert_cols_csv }})\n values ({{ insert_cols_csv }})\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.285232}, "macro.dbt.strategy_dispatch": {"unique_id": "macro.dbt.strategy_dispatch", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "strategy_dispatch", "macro_sql": "{% macro strategy_dispatch(name) -%}\n{% set original_name = name %}\n {% if '.' in name %}\n {% set package_name, name = name.split(\".\", 1) %}\n {% else %}\n {% set package_name = none %}\n {% endif %}\n\n {% if package_name is none %}\n {% set package_context = context %}\n {% elif package_name in context %}\n {% set package_context = context[package_name] %}\n {% else %}\n {% set error_msg %}\n Could not find package '{{package_name}}', called with '{{original_name}}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n\n {%- set search_name = 'snapshot_' ~ name ~ '_strategy' -%}\n\n {% if search_name not in package_context %}\n {% set error_msg %}\n The specified strategy macro '{{name}}' was not found in package '{{ package_name }}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n {{ return(package_context[search_name]) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.2955132}, "macro.dbt.snapshot_hash_arguments": {"unique_id": "macro.dbt.snapshot_hash_arguments", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "snapshot_hash_arguments", "macro_sql": "{% macro snapshot_hash_arguments(args) -%}\n {{ adapter.dispatch('snapshot_hash_arguments', 'dbt')(args) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.295966}, "macro.dbt.default__snapshot_hash_arguments": {"unique_id": "macro.dbt.default__snapshot_hash_arguments", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "default__snapshot_hash_arguments", "macro_sql": "{% macro default__snapshot_hash_arguments(args) -%}\n md5({%- for arg in args -%}\n coalesce(cast({{ arg }} as varchar ), '')\n {% if not loop.last %} || '|' || {% endif %}\n {%- endfor -%})\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.296581}, "macro.dbt.snapshot_get_time": {"unique_id": "macro.dbt.snapshot_get_time", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "snapshot_get_time", "macro_sql": "{% macro snapshot_get_time() -%}\n {{ adapter.dispatch('snapshot_get_time', 'dbt')() }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__snapshot_get_time"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.2969642}, "macro.dbt.default__snapshot_get_time": {"unique_id": "macro.dbt.default__snapshot_get_time", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "default__snapshot_get_time", "macro_sql": "{% macro default__snapshot_get_time() -%}\n {{ current_timestamp() }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.297218}, "macro.dbt.snapshot_timestamp_strategy": {"unique_id": "macro.dbt.snapshot_timestamp_strategy", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "snapshot_timestamp_strategy", "macro_sql": "{% macro snapshot_timestamp_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}\n {% set primary_key = config['unique_key'] %}\n {% set updated_at = config['updated_at'] %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %}\n\n {#/*\n The snapshot relation might not have an {{ updated_at }} value if the\n snapshot strategy is changed from `check` to `timestamp`. We\n should use a dbt-created column for the comparison in the snapshot\n table instead of assuming that the user-supplied {{ updated_at }}\n will be present in the historical data.\n\n See https://github.com/dbt-labs/dbt-core/issues/2350\n */ #}\n {% set row_changed_expr -%}\n ({{ snapshotted_rel }}.dbt_valid_from < {{ current_rel }}.{{ updated_at }})\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.29933}, "macro.dbt.snapshot_string_as_time": {"unique_id": "macro.dbt.snapshot_string_as_time", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "snapshot_string_as_time", "macro_sql": "{% macro snapshot_string_as_time(timestamp) -%}\n {{ adapter.dispatch('snapshot_string_as_time', 'dbt')(timestamp) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__snapshot_string_as_time"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.2997508}, "macro.dbt.default__snapshot_string_as_time": {"unique_id": "macro.dbt.default__snapshot_string_as_time", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "default__snapshot_string_as_time", "macro_sql": "{% macro default__snapshot_string_as_time(timestamp) %}\n {% do exceptions.raise_not_implemented(\n 'snapshot_string_as_time macro not implemented for adapter '+adapter.type()\n ) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.3001878}, "macro.dbt.snapshot_check_all_get_existing_columns": {"unique_id": "macro.dbt.snapshot_check_all_get_existing_columns", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "snapshot_check_all_get_existing_columns", "macro_sql": "{% macro snapshot_check_all_get_existing_columns(node, target_exists) -%}\n {%- set query_columns = get_columns_in_query(node['compiled_sql']) -%}\n {%- if not target_exists -%}\n {# no table yet -> return whatever the query does #}\n {{ return([false, query_columns]) }}\n {%- endif -%}\n {# handle any schema changes #}\n {%- set target_table = node.get('alias', node.get('name')) -%}\n {%- set target_relation = adapter.get_relation(database=node.database, schema=node.schema, identifier=target_table) -%}\n {%- set existing_cols = get_columns_in_query('select * from ' ~ target_relation) -%}\n {%- set ns = namespace() -%} {# handle for-loop scoping with a namespace #}\n {%- set ns.column_added = false -%}\n\n {%- set intersection = [] -%}\n {%- for col in query_columns -%}\n {%- if col in existing_cols -%}\n {%- do intersection.append(col) -%}\n {%- else -%}\n {% set ns.column_added = true %}\n {%- endif -%}\n {%- endfor -%}\n {{ return([ns.column_added, intersection]) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_columns_in_query"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.3028228}, "macro.dbt.snapshot_check_strategy": {"unique_id": "macro.dbt.snapshot_check_strategy", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "snapshot_check_strategy", "macro_sql": "{% macro snapshot_check_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}\n {% set check_cols_config = config['check_cols'] %}\n {% set primary_key = config['unique_key'] %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %}\n {% set updated_at = config.get('updated_at', snapshot_get_time()) %}\n\n {% set column_added = false %}\n\n {% if check_cols_config == 'all' %}\n {% set column_added, check_cols = snapshot_check_all_get_existing_columns(node, target_exists) %}\n {% elif check_cols_config is iterable and (check_cols_config | length) > 0 %}\n {% set check_cols = check_cols_config %}\n {% else %}\n {% do exceptions.raise_compiler_error(\"Invalid value for 'check_cols': \" ~ check_cols_config) %}\n {% endif %}\n\n {%- set row_changed_expr -%}\n (\n {%- if column_added -%}\n {{ get_true_sql() }}\n {%- else -%}\n {%- for col in check_cols -%}\n {{ snapshotted_rel }}.{{ col }} != {{ current_rel }}.{{ col }}\n or\n (\n (({{ snapshotted_rel }}.{{ col }} is null) and not ({{ current_rel }}.{{ col }} is null))\n or\n ((not {{ snapshotted_rel }}.{{ col }} is null) and ({{ current_rel }}.{{ col }} is null))\n )\n {%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n {%- endif -%}\n )\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.snapshot_get_time", "macro.dbt.snapshot_check_all_get_existing_columns", "macro.dbt.get_true_sql", "macro.dbt.snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.3070579}, "macro.dbt.create_columns": {"unique_id": "macro.dbt.create_columns", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "create_columns", "macro_sql": "{% macro create_columns(relation, columns) %}\n {{ adapter.dispatch('create_columns', 'dbt')(relation, columns) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__create_columns"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.3179069}, "macro.dbt.default__create_columns": {"unique_id": "macro.dbt.default__create_columns", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "default__create_columns", "macro_sql": "{% macro default__create_columns(relation, columns) %}\n {% for column in columns %}\n {% call statement() %}\n alter table {{ relation }} add column \"{{ column.name }}\" {{ column.data_type }};\n {% endcall %}\n {% endfor %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.318669}, "macro.dbt.post_snapshot": {"unique_id": "macro.dbt.post_snapshot", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "post_snapshot", "macro_sql": "{% macro post_snapshot(staging_relation) %}\n {{ adapter.dispatch('post_snapshot', 'dbt')(staging_relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.3191}, "macro.dbt.default__post_snapshot": {"unique_id": "macro.dbt.default__post_snapshot", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "default__post_snapshot", "macro_sql": "{% macro default__post_snapshot(staging_relation) %}\n {# no-op #}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.3193212}, "macro.dbt.get_true_sql": {"unique_id": "macro.dbt.get_true_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "get_true_sql", "macro_sql": "{% macro get_true_sql() %}\n {{ adapter.dispatch('get_true_sql', 'dbt')() }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_true_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.319695}, "macro.dbt.default__get_true_sql": {"unique_id": "macro.dbt.default__get_true_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "default__get_true_sql", "macro_sql": "{% macro default__get_true_sql() %}\n {{ return('TRUE') }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.319982}, "macro.dbt.snapshot_staging_table": {"unique_id": "macro.dbt.snapshot_staging_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "snapshot_staging_table", "macro_sql": "{% macro snapshot_staging_table(strategy, source_sql, target_relation) -%}\n {{ adapter.dispatch('snapshot_staging_table', 'dbt')(strategy, source_sql, target_relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__snapshot_staging_table"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.320519}, "macro.dbt.default__snapshot_staging_table": {"unique_id": "macro.dbt.default__snapshot_staging_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "default__snapshot_staging_table", "macro_sql": "{% macro default__snapshot_staging_table(strategy, source_sql, target_relation) -%}\n\n with snapshot_query as (\n\n {{ source_sql }}\n\n ),\n\n snapshotted_data as (\n\n select *,\n {{ strategy.unique_key }} as dbt_unique_key\n\n from {{ target_relation }}\n where dbt_valid_to is null\n\n ),\n\n insertions_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to,\n {{ strategy.scd_id }} as dbt_scd_id\n\n from snapshot_query\n ),\n\n updates_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n {{ strategy.updated_at }} as dbt_valid_to\n\n from snapshot_query\n ),\n\n {%- if strategy.invalidate_hard_deletes %}\n\n deletes_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key\n from snapshot_query\n ),\n {% endif %}\n\n insertions as (\n\n select\n 'insert' as dbt_change_type,\n source_data.*\n\n from insertions_source_data as source_data\n left outer join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where snapshotted_data.dbt_unique_key is null\n or (\n snapshotted_data.dbt_unique_key is not null\n and (\n {{ strategy.row_changed }}\n )\n )\n\n ),\n\n updates as (\n\n select\n 'update' as dbt_change_type,\n source_data.*,\n snapshotted_data.dbt_scd_id\n\n from updates_source_data as source_data\n join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where (\n {{ strategy.row_changed }}\n )\n )\n\n {%- if strategy.invalidate_hard_deletes -%}\n ,\n\n deletes as (\n\n select\n 'delete' as dbt_change_type,\n source_data.*,\n {{ snapshot_get_time() }} as dbt_valid_from,\n {{ snapshot_get_time() }} as dbt_updated_at,\n {{ snapshot_get_time() }} as dbt_valid_to,\n snapshotted_data.dbt_scd_id\n\n from snapshotted_data\n left join deletes_source_data as source_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where source_data.dbt_unique_key is null\n )\n {%- endif %}\n\n select * from insertions\n union all\n select * from updates\n {%- if strategy.invalidate_hard_deletes %}\n union all\n select * from deletes\n {%- endif %}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.snapshot_get_time"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.322867}, "macro.dbt.build_snapshot_table": {"unique_id": "macro.dbt.build_snapshot_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "build_snapshot_table", "macro_sql": "{% macro build_snapshot_table(strategy, sql) -%}\n {{ adapter.dispatch('build_snapshot_table', 'dbt')(strategy, sql) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__build_snapshot_table"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.323358}, "macro.dbt.default__build_snapshot_table": {"unique_id": "macro.dbt.default__build_snapshot_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "default__build_snapshot_table", "macro_sql": "{% macro default__build_snapshot_table(strategy, sql) %}\n\n select *,\n {{ strategy.scd_id }} as dbt_scd_id,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to\n from (\n {{ sql }}\n ) sbq\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.324033}, "macro.dbt.build_snapshot_staging_table": {"unique_id": "macro.dbt.build_snapshot_staging_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "build_snapshot_staging_table", "macro_sql": "{% macro build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% set tmp_relation = make_temp_relation(target_relation) %}\n\n {% set select = snapshot_staging_table(strategy, sql, target_relation) %}\n\n {% call statement('build_snapshot_staging_relation') %}\n {{ create_table_as(True, tmp_relation, select) }}\n {% endcall %}\n\n {% do return(tmp_relation) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.make_temp_relation", "macro.dbt.snapshot_staging_table", "macro.dbt.statement", "macro.dbt.create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.325182}, "macro.dbt.materialization_snapshot_default": {"unique_id": "macro.dbt.materialization_snapshot_default", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/snapshot.sql", "original_file_path": "macros/materializations/snapshots/snapshot.sql", "name": "materialization_snapshot_default", "macro_sql": "{% materialization snapshot, default %}\n {%- set config = model['config'] -%}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=model.database,\n schema=model.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", config, target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_sql']) %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% else %}\n\n {{ adapter.valid_snapshot_target(target_relation) }}\n\n {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% endif %}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if not target_relation_exists %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if staging_table is defined %}\n {% do post_snapshot(staging_table) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_or_create_relation", "macro.dbt.run_hooks", "macro.dbt.strategy_dispatch", "macro.dbt.build_snapshot_table", "macro.dbt.create_table_as", "macro.dbt.build_snapshot_staging_table", "macro.dbt.create_columns", "macro.dbt.snapshot_merge_sql", "macro.dbt.statement", "macro.dbt.persist_docs", "macro.dbt.create_indexes", "macro.dbt.post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.34037}, "macro.dbt.materialization_test_default": {"unique_id": "macro.dbt.materialization_test_default", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/tests/test.sql", "original_file_path": "macros/materializations/tests/test.sql", "name": "materialization_test_default", "macro_sql": "{%- materialization test, default -%}\n\n {% set relations = [] %}\n\n {% if should_store_failures() %}\n\n {% set identifier = model['alias'] %}\n {% set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n {% set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database, type='table') -%} %}\n\n {% if old_relation %}\n {% do adapter.drop_relation(old_relation) %}\n {% endif %}\n\n {% call statement(auto_begin=True) %}\n {{ create_table_as(False, target_relation, sql) }}\n {% endcall %}\n\n {% do relations.append(target_relation) %}\n\n {% set main_sql %}\n select *\n from {{ target_relation }}\n {% endset %}\n\n {{ adapter.commit() }}\n\n {% else %}\n\n {% set main_sql = sql %}\n\n {% endif %}\n\n {% set limit = config.get('limit') %}\n {% set fail_calc = config.get('fail_calc') %}\n {% set warn_if = config.get('warn_if') %}\n {% set error_if = config.get('error_if') %}\n\n {% call statement('main', fetch_result=True) -%}\n\n {{ get_test_sql(main_sql, fail_calc, warn_if, error_if, limit)}}\n\n {%- endcall %}\n\n {{ return({'relations': relations}) }}\n\n{%- endmaterialization -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.should_store_failures", "macro.dbt.statement", "macro.dbt.create_table_as", "macro.dbt.get_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.346029}, "macro.dbt.get_test_sql": {"unique_id": "macro.dbt.get_test_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "name": "get_test_sql", "macro_sql": "{% macro get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n {{ adapter.dispatch('get_test_sql', 'dbt')(main_sql, fail_calc, warn_if, error_if, limit) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.3473558}, "macro.dbt.default__get_test_sql": {"unique_id": "macro.dbt.default__get_test_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "name": "default__get_test_sql", "macro_sql": "{% macro default__get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n select\n {{ fail_calc }} as failures,\n {{ fail_calc }} {{ warn_if }} as should_warn,\n {{ fail_calc }} {{ error_if }} as should_error\n from (\n {{ main_sql }}\n {{ \"limit \" ~ limit if limit != none }}\n ) dbt_internal_test\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.348177}, "macro.dbt.get_where_subquery": {"unique_id": "macro.dbt.get_where_subquery", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/tests/where_subquery.sql", "original_file_path": "macros/materializations/tests/where_subquery.sql", "name": "get_where_subquery", "macro_sql": "{% macro get_where_subquery(relation) -%}\n {% do return(adapter.dispatch('get_where_subquery', 'dbt')(relation)) %}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_where_subquery"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.349312}, "macro.dbt.default__get_where_subquery": {"unique_id": "macro.dbt.default__get_where_subquery", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/tests/where_subquery.sql", "original_file_path": "macros/materializations/tests/where_subquery.sql", "name": "default__get_where_subquery", "macro_sql": "{% macro default__get_where_subquery(relation) -%}\n {% set where = config.get('where', '') %}\n {% if where %}\n {%- set filtered -%}\n (select * from {{ relation }} where {{ where }}) dbt_subquery\n {%- endset -%}\n {% do return(filtered) %}\n {%- else -%}\n {% do return(relation) %}\n {%- endif -%}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.350326}, "macro.dbt.get_quoted_csv": {"unique_id": "macro.dbt.get_quoted_csv", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "name": "get_quoted_csv", "macro_sql": "{% macro get_quoted_csv(column_names) %}\n\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote(col)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.352607}, "macro.dbt.diff_columns": {"unique_id": "macro.dbt.diff_columns", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "name": "diff_columns", "macro_sql": "{% macro diff_columns(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% set source_names = source_columns | map(attribute = 'column') | list %}\n {% set target_names = target_columns | map(attribute = 'column') | list %}\n\n {# --check whether the name attribute exists in the target - this does not perform a data type check #}\n {% for sc in source_columns %}\n {% if sc.name not in target_names %}\n {{ result.append(sc) }}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.35407}, "macro.dbt.diff_column_data_types": {"unique_id": "macro.dbt.diff_column_data_types", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "name": "diff_column_data_types", "macro_sql": "{% macro diff_column_data_types(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% for sc in source_columns %}\n {% set tc = target_columns | selectattr(\"name\", \"equalto\", sc.name) | list | first %}\n {% if tc %}\n {% if sc.data_type != tc.data_type %}\n {{ result.append( { 'column_name': tc.name, 'new_type': sc.data_type } ) }}\n {% endif %}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.3557}, "macro.dbt.get_merge_sql": {"unique_id": "macro.dbt.get_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "name": "get_merge_sql", "macro_sql": "{% macro get_merge_sql(target, source, unique_key, dest_columns, predicates=none) -%}\n {{ adapter.dispatch('get_merge_sql', 'dbt')(target, source, unique_key, dest_columns, predicates) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.370045}, "macro.dbt.default__get_merge_sql": {"unique_id": "macro.dbt.default__get_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "name": "default__get_merge_sql", "macro_sql": "{% macro default__get_merge_sql(target, source, unique_key, dest_columns, predicates) -%}\n {%- set predicates = [] if predicates is none else [] + predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set update_columns = config.get('merge_update_columns', default = dest_columns | map(attribute=\"quoted\") | list) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}\n {% for key in unique_key %}\n {% set this_key_match %}\n DBT_INTERNAL_SOURCE.{{ key }} = DBT_INTERNAL_DEST.{{ key }}\n {% endset %}\n {% do predicates.append(this_key_match) %}\n {% endfor %}\n {% else %}\n {% set unique_key_match %}\n DBT_INTERNAL_SOURCE.{{ unique_key }} = DBT_INTERNAL_DEST.{{ unique_key }}\n {% endset %}\n {% do predicates.append(unique_key_match) %}\n {% endif %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n {{ sql_header if sql_header is not none }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{ predicates | join(' and ') }}\n\n {% if unique_key %}\n when matched then update set\n {% for column_name in update_columns -%}\n {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}\n {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n {% endif %}\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.374055}, "macro.dbt.get_delete_insert_merge_sql": {"unique_id": "macro.dbt.get_delete_insert_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "name": "get_delete_insert_merge_sql", "macro_sql": "{% macro get_delete_insert_merge_sql(target, source, unique_key, dest_columns) -%}\n {{ adapter.dispatch('get_delete_insert_merge_sql', 'dbt')(target, source, unique_key, dest_columns) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_delete_insert_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.401227}, "macro.dbt.default__get_delete_insert_merge_sql": {"unique_id": "macro.dbt.default__get_delete_insert_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "name": "default__get_delete_insert_merge_sql", "macro_sql": "{% macro default__get_delete_insert_merge_sql(target, source, unique_key, dest_columns) -%}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not string %}\n delete from {{target }}\n using {{ source }}\n where (\n {% for key in unique_key %}\n {{ source }}.{{ key }} = {{ target }}.{{ key }}\n {{ \"and \" if not loop.last }}\n {% endfor %}\n );\n {% else %}\n delete from {{ target }}\n where (\n {{ unique_key }}) in (\n select ({{ unique_key }})\n from {{ source }}\n );\n\n {% endif %}\n {% endif %}\n\n insert into {{ target }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ source }}\n )\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.4033751}, "macro.dbt.get_insert_overwrite_merge_sql": {"unique_id": "macro.dbt.get_insert_overwrite_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "name": "get_insert_overwrite_merge_sql", "macro_sql": "{% macro get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header=false) -%}\n {{ adapter.dispatch('get_insert_overwrite_merge_sql', 'dbt')(target, source, dest_columns, predicates, include_sql_header) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_insert_overwrite_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.4041102}, "macro.dbt.default__get_insert_overwrite_merge_sql": {"unique_id": "macro.dbt.default__get_insert_overwrite_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "name": "default__get_insert_overwrite_merge_sql", "macro_sql": "{% macro default__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) -%}\n {%- set predicates = [] if predicates is none else [] + predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none and include_sql_header }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on FALSE\n\n when not matched by source\n {% if predicates %} and {{ predicates | join(' and ') }} {% endif %}\n then delete\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.405853}, "macro.dbt.is_incremental": {"unique_id": "macro.dbt.is_incremental", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/is_incremental.sql", "original_file_path": "macros/materializations/models/incremental/is_incremental.sql", "name": "is_incremental", "macro_sql": "{% macro is_incremental() %}\n {#-- do not run introspective queries in parsing #}\n {% if not execute %}\n {{ return(False) }}\n {% else %}\n {% set relation = adapter.get_relation(this.database, this.schema, this.table) %}\n {{ return(relation is not none\n and relation.type == 'table'\n and model.config.materialized == 'incremental'\n and not should_full_refresh()) }}\n {% endif %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.should_full_refresh"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.4078321}, "macro.dbt.materialization_incremental_default": {"unique_id": "macro.dbt.materialization_incremental_default", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/incremental.sql", "original_file_path": "macros/materializations/models/incremental/incremental.sql", "name": "materialization_incremental_default", "macro_sql": "{% materialization incremental, default -%}\n\n {% set unique_key = config.get('unique_key') %}\n\n {% set target_relation = this.incorporate(type='table') %}\n {% set existing_relation = load_relation(this) %}\n {% set tmp_relation = make_temp_relation(target_relation) %}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {% set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') %}\n\n {% set tmp_identifier = model['name'] + '__dbt_tmp' %}\n {% set backup_identifier = model['name'] + \"__dbt_backup\" %}\n\n -- the intermediate_ and backup_ relations should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation. This has to happen before\n -- BEGIN, in a separate transaction\n {% set preexisting_intermediate_relation = adapter.get_relation(identifier=tmp_identifier,\n schema=schema,\n database=database) %}\n {% set preexisting_backup_relation = adapter.get_relation(identifier=backup_identifier,\n schema=schema,\n database=database) %}\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set to_drop = [] %}\n\n {# -- first check whether we want to full refresh for source view or config reasons #}\n {% set trigger_full_refresh = (full_refresh_mode or existing_relation.is_view) %}\n\n {% if existing_relation is none %}\n {% set build_sql = create_table_as(False, target_relation, sql) %}\n{% elif trigger_full_refresh %}\n {#-- Make sure the backup doesn't exist so we don't encounter issues with the rename below #}\n {% set tmp_identifier = model['name'] + '__dbt_tmp' %}\n {% set backup_identifier = model['name'] + '__dbt_backup' %}\n {% set intermediate_relation = existing_relation.incorporate(path={\"identifier\": tmp_identifier}) %}\n {% set backup_relation = existing_relation.incorporate(path={\"identifier\": backup_identifier}) %}\n\n {% set build_sql = create_table_as(False, intermediate_relation, sql) %}\n {% set need_swap = true %}\n {% do to_drop.append(backup_relation) %}\n {% else %}\n {% do run_query(create_table_as(True, tmp_relation, sql)) %}\n {% do adapter.expand_target_column_types(\n from_relation=tmp_relation,\n to_relation=target_relation) %}\n {#-- Process schema changes. Returns dict of changes if successful. Use source columns for upserting/merging --#}\n {% set dest_columns = process_schema_changes(on_schema_change, tmp_relation, existing_relation) %}\n {% if not dest_columns %}\n {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %}\n {% endif %}\n {% set build_sql = get_delete_insert_merge_sql(target_relation, tmp_relation, unique_key, dest_columns) %}\n\n {% endif %}\n\n {% call statement(\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% if need_swap %}\n {% do adapter.rename_relation(target_relation, backup_relation) %}\n {% do adapter.rename_relation(intermediate_relation, target_relation) %}\n {% endif %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if existing_relation is none or existing_relation.is_view or should_full_refresh() %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {% do adapter.commit() %}\n\n {% for rel in to_drop %}\n {% do adapter.drop_relation(rel) %}\n {% endfor %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.load_relation", "macro.dbt.make_temp_relation", "macro.dbt.should_full_refresh", "macro.dbt.incremental_validate_on_schema_change", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks", "macro.dbt.create_table_as", "macro.dbt.run_query", "macro.dbt.process_schema_changes", "macro.dbt.get_delete_insert_merge_sql", "macro.dbt.statement", "macro.dbt.persist_docs", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.421077}, "macro.dbt.incremental_validate_on_schema_change": {"unique_id": "macro.dbt.incremental_validate_on_schema_change", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "name": "incremental_validate_on_schema_change", "macro_sql": "{% macro incremental_validate_on_schema_change(on_schema_change, default='ignore') %}\n\n {% if on_schema_change not in ['sync_all_columns', 'append_new_columns', 'fail', 'ignore'] %}\n\n {% set log_message = 'Invalid value for on_schema_change (%s) specified. Setting default value of %s.' % (on_schema_change, default) %}\n {% do log(log_message) %}\n\n {{ return(default) }}\n\n {% else %}\n\n {{ return(on_schema_change) }}\n\n {% endif %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.434217}, "macro.dbt.check_for_schema_changes": {"unique_id": "macro.dbt.check_for_schema_changes", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "name": "check_for_schema_changes", "macro_sql": "{% macro check_for_schema_changes(source_relation, target_relation) %}\n\n {% set schema_changed = False %}\n\n {%- set source_columns = adapter.get_columns_in_relation(source_relation) -%}\n {%- set target_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set source_not_in_target = diff_columns(source_columns, target_columns) -%}\n {%- set target_not_in_source = diff_columns(target_columns, source_columns) -%}\n\n {% set new_target_types = diff_column_data_types(source_columns, target_columns) %}\n\n {% if source_not_in_target != [] %}\n {% set schema_changed = True %}\n {% elif target_not_in_source != [] or new_target_types != [] %}\n {% set schema_changed = True %}\n {% elif new_target_types != [] %}\n {% set schema_changed = True %}\n {% endif %}\n\n {% set changes_dict = {\n 'schema_changed': schema_changed,\n 'source_not_in_target': source_not_in_target,\n 'target_not_in_source': target_not_in_source,\n 'source_columns': source_columns,\n 'target_columns': target_columns,\n 'new_target_types': new_target_types\n } %}\n\n {% set msg %}\n In {{ target_relation }}:\n Schema changed: {{ schema_changed }}\n Source columns not in target: {{ source_not_in_target }}\n Target columns not in source: {{ target_not_in_source }}\n New column types: {{ new_target_types }}\n {% endset %}\n\n {% do log(msg) %}\n\n {{ return(changes_dict) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.diff_columns", "macro.dbt.diff_column_data_types"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.43755}, "macro.dbt.sync_column_schemas": {"unique_id": "macro.dbt.sync_column_schemas", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "name": "sync_column_schemas", "macro_sql": "{% macro sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {%- set add_to_target_arr = schema_changes_dict['source_not_in_target'] -%}\n\n {%- if on_schema_change == 'append_new_columns'-%}\n {%- if add_to_target_arr | length > 0 -%}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, none) -%}\n {%- endif -%}\n\n {% elif on_schema_change == 'sync_all_columns' %}\n {%- set remove_from_target_arr = schema_changes_dict['target_not_in_source'] -%}\n {%- set new_target_types = schema_changes_dict['new_target_types'] -%}\n\n {% if add_to_target_arr | length > 0 or remove_from_target_arr | length > 0 %}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, remove_from_target_arr) -%}\n {% endif %}\n\n {% if new_target_types != [] %}\n {% for ntt in new_target_types %}\n {% set column_name = ntt['column_name'] %}\n {% set new_type = ntt['new_type'] %}\n {% do alter_column_type(target_relation, column_name, new_type) %}\n {% endfor %}\n {% endif %}\n\n {% endif %}\n\n {% set schema_change_message %}\n In {{ target_relation }}:\n Schema change approach: {{ on_schema_change }}\n Columns added: {{ add_to_target_arr }}\n Columns removed: {{ remove_from_target_arr }}\n Data types changed: {{ new_target_types }}\n {% endset %}\n\n {% do log(schema_change_message) %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.alter_relation_add_remove_columns", "macro.dbt.alter_column_type"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.440778}, "macro.dbt.process_schema_changes": {"unique_id": "macro.dbt.process_schema_changes", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "name": "process_schema_changes", "macro_sql": "{% macro process_schema_changes(on_schema_change, source_relation, target_relation) %}\n\n {% if on_schema_change == 'ignore' %}\n\n {{ return({}) }}\n\n {% else %}\n\n {% set schema_changes_dict = check_for_schema_changes(source_relation, target_relation) %}\n\n {% if schema_changes_dict['schema_changed'] %}\n\n {% if on_schema_change == 'fail' %}\n\n {% set fail_msg %}\n The source and target schemas on this incremental model are out of sync!\n They can be reconciled in several ways:\n - set the `on_schema_change` config to either append_new_columns or sync_all_columns, depending on your situation.\n - Re-run the incremental model with `full_refresh: True` to update the target schema.\n - update the schema manually and re-run the process.\n {% endset %}\n\n {% do exceptions.raise_compiler_error(fail_msg) %}\n\n {# -- unless we ignore, run the sync operation per the config #}\n {% else %}\n\n {% do sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {% endif %}\n\n {% endif %}\n\n {{ return(schema_changes_dict['source_columns']) }}\n\n {% endif %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.check_for_schema_changes", "macro.dbt.sync_column_schemas"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.442719}, "macro.dbt.materialization_table_default": {"unique_id": "macro.dbt.materialization_table_default", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/table/table.sql", "original_file_path": "macros/materializations/models/table/table.sql", "name": "materialization_table_default", "macro_sql": "{% materialization table, default %}\n {%- set identifier = model['alias'] -%}\n {%- set tmp_identifier = model['name'] + '__dbt_tmp' -%}\n {%- set backup_identifier = model['name'] + '__dbt_backup' -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set target_relation = api.Relation.create(identifier=identifier,\n schema=schema,\n database=database,\n type='table') -%}\n {%- set intermediate_relation = api.Relation.create(identifier=tmp_identifier,\n schema=schema,\n database=database,\n type='table') -%}\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = adapter.get_relation(identifier=tmp_identifier,\n schema=schema,\n database=database) -%}\n /*\n See ../view/view.sql for more information about this relation.\n */\n {%- set backup_relation_type = 'table' if old_relation is none else old_relation.type -%}\n {%- set backup_relation = api.Relation.create(identifier=backup_identifier,\n schema=schema,\n database=database,\n type=backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = adapter.get_relation(identifier=backup_identifier,\n schema=schema,\n database=database) -%}\n\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_table_as_sql(False, intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n {% if old_relation is not none %}\n {{ adapter.rename_relation(old_relation, backup_relation) }}\n {% endif %}\n\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% do create_indexes(target_relation) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {% do persist_docs(target_relation, model) %}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n -- finally, drop the existing/backup relation after the commit\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n{% endmaterialization %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.get_create_table_as_sql", "macro.dbt.create_indexes", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.452991}, "macro.dbt.get_create_table_as_sql": {"unique_id": "macro.dbt.get_create_table_as_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/table/create_table_as.sql", "original_file_path": "macros/materializations/models/table/create_table_as.sql", "name": "get_create_table_as_sql", "macro_sql": "{% macro get_create_table_as_sql(temporary, relation, sql) -%}\n {{ adapter.dispatch('get_create_table_as_sql', 'dbt')(temporary, relation, sql) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_create_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.454284}, "macro.dbt.default__get_create_table_as_sql": {"unique_id": "macro.dbt.default__get_create_table_as_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/table/create_table_as.sql", "original_file_path": "macros/materializations/models/table/create_table_as.sql", "name": "default__get_create_table_as_sql", "macro_sql": "{% macro default__get_create_table_as_sql(temporary, relation, sql) -%}\n {{ return(create_table_as(temporary, relation, sql)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.454771}, "macro.dbt.create_table_as": {"unique_id": "macro.dbt.create_table_as", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/table/create_table_as.sql", "original_file_path": "macros/materializations/models/table/create_table_as.sql", "name": "create_table_as", "macro_sql": "{% macro create_table_as(temporary, relation, sql) -%}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, sql) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.455303}, "macro.dbt.default__create_table_as": {"unique_id": "macro.dbt.default__create_table_as", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/table/create_table_as.sql", "original_file_path": "macros/materializations/models/table/create_table_as.sql", "name": "default__create_table_as", "macro_sql": "{% macro default__create_table_as(temporary, relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary: -%}temporary{%- endif %} table\n {{ relation.include(database=(not temporary), schema=(not temporary)) }}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.456414}, "macro.dbt.materialization_view_default": {"unique_id": "macro.dbt.materialization_view_default", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/view/view.sql", "original_file_path": "macros/materializations/models/view/view.sql", "name": "materialization_view_default", "macro_sql": "{%- materialization view, default -%}\n\n {%- set identifier = model['alias'] -%}\n {%- set tmp_identifier = model['name'] + '__dbt_tmp' -%}\n {%- set backup_identifier = model['name'] + '__dbt_backup' -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set target_relation = api.Relation.create(identifier=identifier, schema=schema, database=database,\n type='view') -%}\n {%- set intermediate_relation = api.Relation.create(identifier=tmp_identifier,\n schema=schema, database=database, type='view') -%}\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = adapter.get_relation(identifier=tmp_identifier,\n schema=schema,\n database=database) -%}\n /*\n This relation (probably) doesn't exist yet. If it does exist, it's a leftover from\n a previous run, and we're going to try to drop it immediately. At the end of this\n materialization, we're going to rename the \"old_relation\" to this identifier,\n and then we're going to drop it. In order to make sure we run the correct one of:\n - drop view ...\n - drop table ...\n\n We need to set the type of this relation to be the type of the old_relation, if it exists,\n or else \"view\" as a sane default if it does not. Note that if the old_relation does not\n exist, then there is nothing to move out of the way and subsequentally drop. In that case,\n this relation will be effectively unused.\n */\n {%- set backup_relation_type = 'view' if old_relation is none else old_relation.type -%}\n {%- set backup_relation = api.Relation.create(identifier=backup_identifier,\n schema=schema, database=database,\n type=backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = adapter.get_relation(identifier=backup_identifier,\n schema=schema,\n database=database) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ create_view_as(intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n -- move the existing view out of the way\n {% if old_relation is not none %}\n {{ adapter.rename_relation(old_relation, backup_relation) }}\n {% endif %}\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.create_view_as", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.4662242}, "macro.dbt.handle_existing_table": {"unique_id": "macro.dbt.handle_existing_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/view/helpers.sql", "original_file_path": "macros/materializations/models/view/helpers.sql", "name": "handle_existing_table", "macro_sql": "{% macro handle_existing_table(full_refresh, old_relation) %}\n {{ adapter.dispatch('handle_existing_table', 'dbt')(full_refresh, old_relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__handle_existing_table"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.467168}, "macro.dbt.default__handle_existing_table": {"unique_id": "macro.dbt.default__handle_existing_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/view/helpers.sql", "original_file_path": "macros/materializations/models/view/helpers.sql", "name": "default__handle_existing_table", "macro_sql": "{% macro default__handle_existing_table(full_refresh, old_relation) %}\n {{ log(\"Dropping relation \" ~ old_relation ~ \" because it is of type \" ~ old_relation.type) }}\n {{ adapter.drop_relation(old_relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.46778}, "macro.dbt.create_or_replace_view": {"unique_id": "macro.dbt.create_or_replace_view", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/view/create_or_replace_view.sql", "original_file_path": "macros/materializations/models/view/create_or_replace_view.sql", "name": "create_or_replace_view", "macro_sql": "{% macro create_or_replace_view() %}\n {%- set identifier = model['alias'] -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database,\n type='view') -%}\n\n {{ run_hooks(pre_hooks) }}\n\n -- If there's a table with the same name and we weren't told to full refresh,\n -- that's an error. If we were told to full refresh, drop it. This behavior differs\n -- for Snowflake and BigQuery, so multiple dispatch is used.\n {%- if old_relation is not none and old_relation.is_table -%}\n {{ handle_existing_table(should_full_refresh(), old_relation) }}\n {%- endif -%}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(target_relation, sql) }}\n {%- endcall %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.handle_existing_table", "macro.dbt.should_full_refresh", "macro.dbt.statement", "macro.dbt.get_create_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.4711108}, "macro.dbt.get_create_view_as_sql": {"unique_id": "macro.dbt.get_create_view_as_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/view/create_view_as.sql", "original_file_path": "macros/materializations/models/view/create_view_as.sql", "name": "get_create_view_as_sql", "macro_sql": "{% macro get_create_view_as_sql(relation, sql) -%}\n {{ adapter.dispatch('get_create_view_as_sql', 'dbt')(relation, sql) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_create_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.4722078}, "macro.dbt.default__get_create_view_as_sql": {"unique_id": "macro.dbt.default__get_create_view_as_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/view/create_view_as.sql", "original_file_path": "macros/materializations/models/view/create_view_as.sql", "name": "default__get_create_view_as_sql", "macro_sql": "{% macro default__get_create_view_as_sql(relation, sql) -%}\n {{ return(create_view_as(relation, sql)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.472634}, "macro.dbt.create_view_as": {"unique_id": "macro.dbt.create_view_as", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/view/create_view_as.sql", "original_file_path": "macros/materializations/models/view/create_view_as.sql", "name": "create_view_as", "macro_sql": "{% macro create_view_as(relation, sql) -%}\n {{ adapter.dispatch('create_view_as', 'dbt')(relation, sql) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.473103}, "macro.dbt.default__create_view_as": {"unique_id": "macro.dbt.default__create_view_as", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/view/create_view_as.sql", "original_file_path": "macros/materializations/models/view/create_view_as.sql", "name": "default__create_view_as", "macro_sql": "{% macro default__create_view_as(relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n create view {{ relation }} as (\n {{ sql }}\n );\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.473792}, "macro.dbt.materialization_seed_default": {"unique_id": "macro.dbt.materialization_seed_default", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/seed.sql", "original_file_path": "macros/materializations/seeds/seed.sql", "name": "materialization_seed_default", "macro_sql": "{% materialization seed, default %}\n\n {%- set identifier = model['alias'] -%}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set agate_table = load_agate_table() -%}\n {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% set create_table_sql = \"\" %}\n {% if exists_as_view %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a view\".format(old_relation)) }}\n {% elif exists_as_table %}\n {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}\n {% else %}\n {% set create_table_sql = create_csv_table(model, agate_table) %}\n {% endif %}\n\n {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}\n {% set rows_affected = (agate_table.rows | length) %}\n {% set sql = load_csv_rows(model, agate_table) %}\n\n {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}\n {{ create_table_sql }};\n -- dbt seed --\n {{ sql }}\n {% endcall %}\n\n {% set target_relation = this.incorporate(type='table') %}\n {% do persist_docs(target_relation, model) %}\n\n {% if full_refresh_mode or not exists_as_table %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.run_hooks", "macro.dbt.reset_csv_table", "macro.dbt.create_csv_table", "macro.dbt.load_csv_rows", "macro.dbt.noop_statement", "macro.dbt.persist_docs", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.4816241}, "macro.dbt.create_csv_table": {"unique_id": "macro.dbt.create_csv_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "create_csv_table", "macro_sql": "{% macro create_csv_table(model, agate_table) -%}\n {{ adapter.dispatch('create_csv_table', 'dbt')(model, agate_table) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.493353}, "macro.dbt.default__create_csv_table": {"unique_id": "macro.dbt.default__create_csv_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "default__create_csv_table", "macro_sql": "{% macro default__create_csv_table(model, agate_table) %}\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n\n {% set sql %}\n create table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.495795}, "macro.dbt.reset_csv_table": {"unique_id": "macro.dbt.reset_csv_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "reset_csv_table", "macro_sql": "{% macro reset_csv_table(model, full_refresh, old_relation, agate_table) -%}\n {{ adapter.dispatch('reset_csv_table', 'dbt')(model, full_refresh, old_relation, agate_table) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__reset_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.4964218}, "macro.dbt.default__reset_csv_table": {"unique_id": "macro.dbt.default__reset_csv_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "default__reset_csv_table", "macro_sql": "{% macro default__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% set sql = \"\" %}\n {% if full_refresh %}\n {{ adapter.drop_relation(old_relation) }}\n {% set sql = create_csv_table(model, agate_table) %}\n {% else %}\n {{ adapter.truncate_relation(old_relation) }}\n {% set sql = \"truncate table \" ~ old_relation %}\n {% endif %}\n\n {{ return(sql) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.497696}, "macro.dbt.get_binding_char": {"unique_id": "macro.dbt.get_binding_char", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "get_binding_char", "macro_sql": "{% macro get_binding_char() -%}\n {{ adapter.dispatch('get_binding_char', 'dbt')() }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.498073}, "macro.dbt.default__get_binding_char": {"unique_id": "macro.dbt.default__get_binding_char", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "default__get_binding_char", "macro_sql": "{% macro default__get_binding_char() %}\n {{ return('%s') }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.498365}, "macro.dbt.get_batch_size": {"unique_id": "macro.dbt.get_batch_size", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "get_batch_size", "macro_sql": "{% macro get_batch_size() -%}\n {{ return(adapter.dispatch('get_batch_size', 'dbt')()) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_batch_size"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.498788}, "macro.dbt.default__get_batch_size": {"unique_id": "macro.dbt.default__get_batch_size", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "default__get_batch_size", "macro_sql": "{% macro default__get_batch_size() %}\n {{ return(10000) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.499098}, "macro.dbt.get_seed_column_quoted_csv": {"unique_id": "macro.dbt.get_seed_column_quoted_csv", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "get_seed_column_quoted_csv", "macro_sql": "{% macro get_seed_column_quoted_csv(model, column_names) %}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote_seed_column(col, quote_seed_column)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.500374}, "macro.dbt.load_csv_rows": {"unique_id": "macro.dbt.load_csv_rows", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "load_csv_rows", "macro_sql": "{% macro load_csv_rows(model, agate_table) -%}\n {{ adapter.dispatch('load_csv_rows', 'dbt')(model, agate_table) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__load_csv_rows"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.500983}, "macro.dbt.default__load_csv_rows": {"unique_id": "macro.dbt.default__load_csv_rows", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "default__load_csv_rows", "macro_sql": "{% macro default__load_csv_rows(model, agate_table) %}\n\n {% set batch_size = get_batch_size() %}\n\n {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}\n {% set bindings = [] %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert into {{ this.render() }} ({{ cols_sql }}) values\n {% for row in chunk -%}\n ({%- for column in agate_table.column_names -%}\n {{ get_binding_char() }}\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_batch_size", "macro.dbt.get_seed_column_quoted_csv", "macro.dbt.get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.5042381}, "macro.dbt.generate_alias_name": {"unique_id": "macro.dbt.generate_alias_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/get_custom_name/get_custom_alias.sql", "original_file_path": "macros/get_custom_name/get_custom_alias.sql", "name": "generate_alias_name", "macro_sql": "{% macro generate_alias_name(custom_alias_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_alias_name', 'dbt')(custom_alias_name, node)) %}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__generate_alias_name"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.5054028}, "macro.dbt.default__generate_alias_name": {"unique_id": "macro.dbt.default__generate_alias_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/get_custom_name/get_custom_alias.sql", "original_file_path": "macros/get_custom_name/get_custom_alias.sql", "name": "default__generate_alias_name", "macro_sql": "{% macro default__generate_alias_name(custom_alias_name=none, node=none) -%}\n\n {%- if custom_alias_name is none -%}\n\n {{ node.name }}\n\n {%- else -%}\n\n {{ custom_alias_name | trim }}\n\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.505999}, "macro.dbt.generate_schema_name": {"unique_id": "macro.dbt.generate_schema_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "name": "generate_schema_name", "macro_sql": "{% macro generate_schema_name(custom_schema_name=none, node=none) -%}\n {{ return(adapter.dispatch('generate_schema_name', 'dbt')(custom_schema_name, node)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__generate_schema_name"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.507547}, "macro.dbt.default__generate_schema_name": {"unique_id": "macro.dbt.default__generate_schema_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "name": "default__generate_schema_name", "macro_sql": "{% macro default__generate_schema_name(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if custom_schema_name is none -%}\n\n {{ default_schema }}\n\n {%- else -%}\n\n {{ default_schema }}_{{ custom_schema_name | trim }}\n\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.508227}, "macro.dbt.generate_schema_name_for_env": {"unique_id": "macro.dbt.generate_schema_name_for_env", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "name": "generate_schema_name_for_env", "macro_sql": "{% macro generate_schema_name_for_env(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if target.name == 'prod' and custom_schema_name is not none -%}\n\n {{ custom_schema_name | trim }}\n\n {%- else -%}\n\n {{ default_schema }}\n\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.508971}, "macro.dbt.generate_database_name": {"unique_id": "macro.dbt.generate_database_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "name": "generate_database_name", "macro_sql": "{% macro generate_database_name(custom_database_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_database_name', 'dbt')(custom_database_name, node)) %}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__generate_database_name"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.5101619}, "macro.dbt.default__generate_database_name": {"unique_id": "macro.dbt.default__generate_database_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "name": "default__generate_database_name", "macro_sql": "{% macro default__generate_database_name(custom_database_name=none, node=none) -%}\n {%- set default_database = target.database -%}\n {%- if custom_database_name is none -%}\n\n {{ default_database }}\n\n {%- else -%}\n\n {{ custom_database_name }}\n\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.510827}, "macro.dbt.default__test_relationships": {"unique_id": "macro.dbt.default__test_relationships", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/generic_test_sql/relationships.sql", "original_file_path": "macros/generic_test_sql/relationships.sql", "name": "default__test_relationships", "macro_sql": "{% macro default__test_relationships(model, column_name, to, field) %}\n\nwith child as (\n select {{ column_name }} as from_field\n from {{ model }}\n where {{ column_name }} is not null\n),\n\nparent as (\n select {{ field }} as to_field\n from {{ to }}\n)\n\nselect\n from_field\n\nfrom child\nleft join parent\n on child.from_field = parent.to_field\n\nwhere parent.to_field is null\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.511889}, "macro.dbt.default__test_not_null": {"unique_id": "macro.dbt.default__test_not_null", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/generic_test_sql/not_null.sql", "original_file_path": "macros/generic_test_sql/not_null.sql", "name": "default__test_not_null", "macro_sql": "{% macro default__test_not_null(model, column_name) %}\n\n{% set column_list = '*' if should_store_failures() else column_name %}\n\nselect {{ column_list }}\nfrom {{ model }}\nwhere {{ column_name }} is null\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.should_store_failures"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.5128162}, "macro.dbt.default__test_unique": {"unique_id": "macro.dbt.default__test_unique", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/generic_test_sql/unique.sql", "original_file_path": "macros/generic_test_sql/unique.sql", "name": "default__test_unique", "macro_sql": "{% macro default__test_unique(model, column_name) %}\n\nselect\n {{ column_name }} as unique_field,\n count(*) as n_records\n\nfrom {{ model }}\nwhere {{ column_name }} is not null\ngroup by {{ column_name }}\nhaving count(*) > 1\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.5136151}, "macro.dbt.default__test_accepted_values": {"unique_id": "macro.dbt.default__test_accepted_values", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/generic_test_sql/accepted_values.sql", "original_file_path": "macros/generic_test_sql/accepted_values.sql", "name": "default__test_accepted_values", "macro_sql": "{% macro default__test_accepted_values(model, column_name, values, quote=True) %}\n\nwith all_values as (\n\n select\n {{ column_name }} as value_field,\n count(*) as n_records\n\n from {{ model }}\n group by {{ column_name }}\n\n)\n\nselect *\nfrom all_values\nwhere value_field not in (\n {% for value in values -%}\n {% if quote -%}\n '{{ value }}'\n {%- else -%}\n {{ value }}\n {%- endif -%}\n {%- if not loop.last -%},{%- endif %}\n {%- endfor %}\n)\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.515251}, "macro.dbt.statement": {"unique_id": "macro.dbt.statement", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "name": "statement", "macro_sql": "{% macro statement(name=None, fetch_result=False, auto_begin=True) -%}\n {%- if execute: -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- set res, table = adapter.execute(sql, auto_begin=auto_begin, fetch=fetch_result) -%}\n {%- if name is not none -%}\n {{ store_result(name, response=res, agate_table=table) }}\n {%- endif -%}\n\n {%- endif -%}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.518385}, "macro.dbt.noop_statement": {"unique_id": "macro.dbt.noop_statement", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "name": "noop_statement", "macro_sql": "{% macro noop_statement(name=None, message=None, code=None, rows_affected=None, res=None) -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_raw_result(name, message=message, code=code, rows_affected=rows_affected, agate_table=res) }}\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.519926}, "macro.dbt.run_query": {"unique_id": "macro.dbt.run_query", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "name": "run_query", "macro_sql": "{% macro run_query(sql) %}\n {% call statement(\"run_query_statement\", fetch_result=true, auto_begin=false) %}\n {{ sql }}\n {% endcall %}\n\n {% do return(load_result(\"run_query_statement\").table) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.5207071}, "macro.dbt.convert_datetime": {"unique_id": "macro.dbt.convert_datetime", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "name": "convert_datetime", "macro_sql": "{% macro convert_datetime(date_str, date_fmt) %}\n\n {% set error_msg -%}\n The provided partition date '{{ date_str }}' does not match the expected format '{{ date_fmt }}'\n {%- endset %}\n\n {% set res = try_or_compiler_error(error_msg, modules.datetime.datetime.strptime, date_str.strip(), date_fmt) %}\n {{ return(res) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.525802}, "macro.dbt.dates_in_range": {"unique_id": "macro.dbt.dates_in_range", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "name": "dates_in_range", "macro_sql": "{% macro dates_in_range(start_date_str, end_date_str=none, in_fmt=\"%Y%m%d\", out_fmt=\"%Y%m%d\") %}\n {% set end_date_str = start_date_str if end_date_str is none else end_date_str %}\n\n {% set start_date = convert_datetime(start_date_str, in_fmt) %}\n {% set end_date = convert_datetime(end_date_str, in_fmt) %}\n\n {% set day_count = (end_date - start_date).days %}\n {% if day_count < 0 %}\n {% set msg -%}\n Partition start date is after the end date ({{ start_date }}, {{ end_date }})\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg, model) }}\n {% endif %}\n\n {% set date_list = [] %}\n {% for i in range(0, day_count + 1) %}\n {% set the_date = (modules.datetime.timedelta(days=i) + start_date) %}\n {% if not out_fmt %}\n {% set _ = date_list.append(the_date) %}\n {% else %}\n {% set _ = date_list.append(the_date.strftime(out_fmt)) %}\n {% endif %}\n {% endfor %}\n\n {{ return(date_list) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.convert_datetime"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.52915}, "macro.dbt.partition_range": {"unique_id": "macro.dbt.partition_range", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "name": "partition_range", "macro_sql": "{% macro partition_range(raw_partition_date, date_fmt='%Y%m%d') %}\n {% set partition_range = (raw_partition_date | string).split(\",\") %}\n\n {% if (partition_range | length) == 1 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = none %}\n {% elif (partition_range | length) == 2 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = partition_range[1] %}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Invalid partition time. Expected format: {Start Date}[,{End Date}]. Got: \" ~ raw_partition_date) }}\n {% endif %}\n\n {{ return(dates_in_range(start_date, end_date, in_fmt=date_fmt)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.dates_in_range"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.53116}, "macro.dbt.py_current_timestring": {"unique_id": "macro.dbt.py_current_timestring", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "name": "py_current_timestring", "macro_sql": "{% macro py_current_timestring() %}\n {% set dt = modules.datetime.datetime.now() %}\n {% do return(dt.strftime(\"%Y%m%d%H%M%S%f\")) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.5317729}, "macro.dbt.create_schema": {"unique_id": "macro.dbt.create_schema", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "name": "create_schema", "macro_sql": "{% macro create_schema(relation) -%}\n {{ adapter.dispatch('create_schema', 'dbt')(relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__create_schema"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.532931}, "macro.dbt.default__create_schema": {"unique_id": "macro.dbt.default__create_schema", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "name": "default__create_schema", "macro_sql": "{% macro default__create_schema(relation) -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier() }}\n {% endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.533409}, "macro.dbt.drop_schema": {"unique_id": "macro.dbt.drop_schema", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "name": "drop_schema", "macro_sql": "{% macro drop_schema(relation) -%}\n {{ adapter.dispatch('drop_schema', 'dbt')(relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__drop_schema"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.533822}, "macro.dbt.default__drop_schema": {"unique_id": "macro.dbt.default__drop_schema", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "name": "default__drop_schema", "macro_sql": "{% macro default__drop_schema(relation) -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier() }} cascade\n {% endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.5342898}, "macro.dbt.get_create_index_sql": {"unique_id": "macro.dbt.get_create_index_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "name": "get_create_index_sql", "macro_sql": "{% macro get_create_index_sql(relation, index_dict) -%}\n {{ return(adapter.dispatch('get_create_index_sql', 'dbt')(relation, index_dict)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_create_index_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.5356438}, "macro.dbt.default__get_create_index_sql": {"unique_id": "macro.dbt.default__get_create_index_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "name": "default__get_create_index_sql", "macro_sql": "{% macro default__get_create_index_sql(relation, index_dict) -%}\n {% do return(None) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.536011}, "macro.dbt.create_indexes": {"unique_id": "macro.dbt.create_indexes", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "name": "create_indexes", "macro_sql": "{% macro create_indexes(relation) -%}\n {{ adapter.dispatch('create_indexes', 'dbt')(relation) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.536417}, "macro.dbt.default__create_indexes": {"unique_id": "macro.dbt.default__create_indexes", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "name": "default__create_indexes", "macro_sql": "{% macro default__create_indexes(relation) -%}\n {%- set _indexes = config.get('indexes', default=[]) -%}\n\n {% for _index_dict in _indexes %}\n {% set create_index_sql = get_create_index_sql(relation, _index_dict) %}\n {% if create_index_sql %}\n {% do run_query(create_index_sql) %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_create_index_sql", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.5374599}, "macro.dbt.make_temp_relation": {"unique_id": "macro.dbt.make_temp_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "make_temp_relation", "macro_sql": "{% macro make_temp_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_temp_relation', 'dbt')(base_relation, suffix))}}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_temp_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.542093}, "macro.dbt.default__make_temp_relation": {"unique_id": "macro.dbt.default__make_temp_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "default__make_temp_relation", "macro_sql": "{% macro default__make_temp_relation(base_relation, suffix) %}\n {% set tmp_identifier = base_relation.identifier ~ suffix %}\n {% set tmp_relation = base_relation.incorporate(\n path={\"identifier\": tmp_identifier}) -%}\n\n {% do return(tmp_relation) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.5428941}, "macro.dbt.drop_relation": {"unique_id": "macro.dbt.drop_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "drop_relation", "macro_sql": "{% macro drop_relation(relation) -%}\n {{ return(adapter.dispatch('drop_relation', 'dbt')(relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__drop_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.543366}, "macro.dbt.default__drop_relation": {"unique_id": "macro.dbt.default__drop_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "default__drop_relation", "macro_sql": "{% macro default__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n drop {{ relation.type }} if exists {{ relation }} cascade\n {%- endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.543921}, "macro.dbt.truncate_relation": {"unique_id": "macro.dbt.truncate_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "truncate_relation", "macro_sql": "{% macro truncate_relation(relation) -%}\n {{ return(adapter.dispatch('truncate_relation', 'dbt')(relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__truncate_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.544389}, "macro.dbt.default__truncate_relation": {"unique_id": "macro.dbt.default__truncate_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "default__truncate_relation", "macro_sql": "{% macro default__truncate_relation(relation) -%}\n {% call statement('truncate_relation') -%}\n truncate table {{ relation }}\n {%- endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.544798}, "macro.dbt.rename_relation": {"unique_id": "macro.dbt.rename_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "rename_relation", "macro_sql": "{% macro rename_relation(from_relation, to_relation) -%}\n {{ return(adapter.dispatch('rename_relation', 'dbt')(from_relation, to_relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__rename_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.5453591}, "macro.dbt.default__rename_relation": {"unique_id": "macro.dbt.default__rename_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "default__rename_relation", "macro_sql": "{% macro default__rename_relation(from_relation, to_relation) -%}\n {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %}\n {% call statement('rename_relation') -%}\n alter table {{ from_relation }} rename to {{ target_name }}\n {%- endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.546091}, "macro.dbt.get_or_create_relation": {"unique_id": "macro.dbt.get_or_create_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "get_or_create_relation", "macro_sql": "{% macro get_or_create_relation(database, schema, identifier, type) -%}\n {{ return(adapter.dispatch('get_or_create_relation', 'dbt')(database, schema, identifier, type)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_or_create_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.546772}, "macro.dbt.default__get_or_create_relation": {"unique_id": "macro.dbt.default__get_or_create_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "default__get_or_create_relation", "macro_sql": "{% macro default__get_or_create_relation(database, schema, identifier, type) %}\n {%- set target_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n\n {% if target_relation %}\n {% do return([true, target_relation]) %}\n {% endif %}\n\n {%- set new_relation = api.Relation.create(\n database=database,\n schema=schema,\n identifier=identifier,\n type=type\n ) -%}\n {% do return([false, new_relation]) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.548408}, "macro.dbt.load_relation": {"unique_id": "macro.dbt.load_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "load_relation", "macro_sql": "{% macro load_relation(relation) %}\n {% do return(adapter.get_relation(\n database=relation.database,\n schema=relation.schema,\n identifier=relation.identifier\n )) -%}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.54901}, "macro.dbt.drop_relation_if_exists": {"unique_id": "macro.dbt.drop_relation_if_exists", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "drop_relation_if_exists", "macro_sql": "{% macro drop_relation_if_exists(relation) %}\n {% if relation is not none %}\n {{ adapter.drop_relation(relation) }}\n {% endif %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.549522}, "macro.dbt.current_timestamp": {"unique_id": "macro.dbt.current_timestamp", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "name": "current_timestamp", "macro_sql": "{% macro current_timestamp() -%}\n {{ adapter.dispatch('current_timestamp', 'dbt')() }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.550913}, "macro.dbt.default__current_timestamp": {"unique_id": "macro.dbt.default__current_timestamp", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "name": "default__current_timestamp", "macro_sql": "{% macro default__current_timestamp() -%}\n {{ exceptions.raise_not_implemented(\n 'current_timestamp macro not implemented for adapter '+adapter.type()) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.551292}, "macro.dbt.collect_freshness": {"unique_id": "macro.dbt.collect_freshness", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "name": "collect_freshness", "macro_sql": "{% macro collect_freshness(source, loaded_at_field, filter) %}\n {{ return(adapter.dispatch('collect_freshness', 'dbt')(source, loaded_at_field, filter))}}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__collect_freshness"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.5518851}, "macro.dbt.default__collect_freshness": {"unique_id": "macro.dbt.default__collect_freshness", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "name": "default__collect_freshness", "macro_sql": "{% macro default__collect_freshness(source, loaded_at_field, filter) %}\n {% call statement('collect_freshness', fetch_result=True, auto_begin=False) -%}\n select\n max({{ loaded_at_field }}) as max_loaded_at,\n {{ current_timestamp() }} as snapshotted_at\n from {{ source }}\n {% if filter %}\n where {{ filter }}\n {% endif %}\n {% endcall %}\n {{ return(load_result('collect_freshness').table) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.552993}, "macro.dbt.alter_column_comment": {"unique_id": "macro.dbt.alter_column_comment", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "name": "alter_column_comment", "macro_sql": "{% macro alter_column_comment(relation, column_dict) -%}\n {{ return(adapter.dispatch('alter_column_comment', 'dbt')(relation, column_dict)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.554944}, "macro.dbt.default__alter_column_comment": {"unique_id": "macro.dbt.default__alter_column_comment", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "name": "default__alter_column_comment", "macro_sql": "{% macro default__alter_column_comment(relation, column_dict) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_column_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.5553741}, "macro.dbt.alter_relation_comment": {"unique_id": "macro.dbt.alter_relation_comment", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "name": "alter_relation_comment", "macro_sql": "{% macro alter_relation_comment(relation, relation_comment) -%}\n {{ return(adapter.dispatch('alter_relation_comment', 'dbt')(relation, relation_comment)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__alter_relation_comment"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.5559008}, "macro.dbt.default__alter_relation_comment": {"unique_id": "macro.dbt.default__alter_relation_comment", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "name": "default__alter_relation_comment", "macro_sql": "{% macro default__alter_relation_comment(relation, relation_comment) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_relation_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.556353}, "macro.dbt.persist_docs": {"unique_id": "macro.dbt.persist_docs", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "name": "persist_docs", "macro_sql": "{% macro persist_docs(relation, model, for_relation=true, for_columns=true) -%}\n {{ return(adapter.dispatch('persist_docs', 'dbt')(relation, model, for_relation, for_columns)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.557066}, "macro.dbt.default__persist_docs": {"unique_id": "macro.dbt.default__persist_docs", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "name": "default__persist_docs", "macro_sql": "{% macro default__persist_docs(relation, model, for_relation, for_columns) -%}\n {% if for_relation and config.persist_relation_docs() and model.description %}\n {% do run_query(alter_relation_comment(relation, model.description)) %}\n {% endif %}\n\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {% do run_query(alter_column_comment(relation, model.columns)) %}\n {% endif %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.run_query", "macro.dbt.alter_relation_comment", "macro.dbt.alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.55835}, "macro.dbt.get_catalog": {"unique_id": "macro.dbt.get_catalog", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "get_catalog", "macro_sql": "{% macro get_catalog(information_schema, schemas) -%}\n {{ return(adapter.dispatch('get_catalog', 'dbt')(information_schema, schemas)) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_catalog"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.5624719}, "macro.dbt.default__get_catalog": {"unique_id": "macro.dbt.default__get_catalog", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "default__get_catalog", "macro_sql": "{% macro default__get_catalog(information_schema, schemas) -%}\n\n {% set typename = adapter.type() %}\n {% set msg -%}\n get_catalog not implemented for {{ typename }}\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.563156}, "macro.dbt.information_schema_name": {"unique_id": "macro.dbt.information_schema_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "information_schema_name", "macro_sql": "{% macro information_schema_name(database) %}\n {{ return(adapter.dispatch('information_schema_name', 'dbt')(database)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__information_schema_name"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.563638}, "macro.dbt.default__information_schema_name": {"unique_id": "macro.dbt.default__information_schema_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "default__information_schema_name", "macro_sql": "{% macro default__information_schema_name(database) -%}\n {%- if database -%}\n {{ database }}.INFORMATION_SCHEMA\n {%- else -%}\n INFORMATION_SCHEMA\n {%- endif -%}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.564032}, "macro.dbt.list_schemas": {"unique_id": "macro.dbt.list_schemas", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "list_schemas", "macro_sql": "{% macro list_schemas(database) -%}\n {{ return(adapter.dispatch('list_schemas', 'dbt')(database)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__list_schemas"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.564497}, "macro.dbt.default__list_schemas": {"unique_id": "macro.dbt.default__list_schemas", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "default__list_schemas", "macro_sql": "{% macro default__list_schemas(database) -%}\n {% set sql %}\n select distinct schema_name\n from {{ information_schema_name(database) }}.SCHEMATA\n where catalog_name ilike '{{ database }}'\n {% endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.information_schema_name", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.565144}, "macro.dbt.check_schema_exists": {"unique_id": "macro.dbt.check_schema_exists", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "check_schema_exists", "macro_sql": "{% macro check_schema_exists(information_schema, schema) -%}\n {{ return(adapter.dispatch('check_schema_exists', 'dbt')(information_schema, schema)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__check_schema_exists"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.565687}, "macro.dbt.default__check_schema_exists": {"unique_id": "macro.dbt.default__check_schema_exists", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "default__check_schema_exists", "macro_sql": "{% macro default__check_schema_exists(information_schema, schema) -%}\n {% set sql -%}\n select count(*)\n from {{ information_schema.replace(information_schema_view='SCHEMATA') }}\n where catalog_name='{{ information_schema.database }}'\n and schema_name='{{ schema }}'\n {%- endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.566637}, "macro.dbt.list_relations_without_caching": {"unique_id": "macro.dbt.list_relations_without_caching", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "list_relations_without_caching", "macro_sql": "{% macro list_relations_without_caching(schema_relation) %}\n {{ return(adapter.dispatch('list_relations_without_caching', 'dbt')(schema_relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__list_relations_without_caching"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.5671191}, "macro.dbt.default__list_relations_without_caching": {"unique_id": "macro.dbt.default__list_relations_without_caching", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "default__list_relations_without_caching", "macro_sql": "{% macro default__list_relations_without_caching(schema_relation) %}\n {{ exceptions.raise_not_implemented(\n 'list_relations_without_caching macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.567532}, "macro.dbt.get_columns_in_relation": {"unique_id": "macro.dbt.get_columns_in_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "get_columns_in_relation", "macro_sql": "{% macro get_columns_in_relation(relation) -%}\n {{ return(adapter.dispatch('get_columns_in_relation', 'dbt')(relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_columns_in_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.572185}, "macro.dbt.default__get_columns_in_relation": {"unique_id": "macro.dbt.default__get_columns_in_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "default__get_columns_in_relation", "macro_sql": "{% macro default__get_columns_in_relation(relation) -%}\n {{ exceptions.raise_not_implemented(\n 'get_columns_in_relation macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.572603}, "macro.dbt.sql_convert_columns_in_relation": {"unique_id": "macro.dbt.sql_convert_columns_in_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "sql_convert_columns_in_relation", "macro_sql": "{% macro sql_convert_columns_in_relation(table) -%}\n {% set columns = [] %}\n {% for row in table %}\n {% do columns.append(api.Column(*row)) %}\n {% endfor %}\n {{ return(columns) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.5734131}, "macro.dbt.get_columns_in_query": {"unique_id": "macro.dbt.get_columns_in_query", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "get_columns_in_query", "macro_sql": "{% macro get_columns_in_query(select_sql) -%}\n {{ return(adapter.dispatch('get_columns_in_query', 'dbt')(select_sql)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_columns_in_query"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.573897}, "macro.dbt.default__get_columns_in_query": {"unique_id": "macro.dbt.default__get_columns_in_query", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "default__get_columns_in_query", "macro_sql": "{% macro default__get_columns_in_query(select_sql) %}\n {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%}\n select * from (\n {{ select_sql }}\n ) as __dbt_sbq\n where false\n limit 0\n {% endcall %}\n\n {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.5747972}, "macro.dbt.alter_column_type": {"unique_id": "macro.dbt.alter_column_type", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "alter_column_type", "macro_sql": "{% macro alter_column_type(relation, column_name, new_column_type) -%}\n {{ return(adapter.dispatch('alter_column_type', 'dbt')(relation, column_name, new_column_type)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__alter_column_type"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.5754082}, "macro.dbt.default__alter_column_type": {"unique_id": "macro.dbt.default__alter_column_type", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "default__alter_column_type", "macro_sql": "{% macro default__alter_column_type(relation, column_name, new_column_type) -%}\n {#\n 1. Create a new column (w/ temp name and correct type)\n 2. Copy data over to it\n 3. Drop the existing column (cascade!)\n 4. Rename the new column to existing column\n #}\n {%- set tmp_column = column_name + \"__dbt_alter\" -%}\n\n {% call statement('alter_column_type') %}\n alter table {{ relation }} add column {{ adapter.quote(tmp_column) }} {{ new_column_type }};\n update {{ relation }} set {{ adapter.quote(tmp_column) }} = {{ adapter.quote(column_name) }};\n alter table {{ relation }} drop column {{ adapter.quote(column_name) }} cascade;\n alter table {{ relation }} rename column {{ adapter.quote(tmp_column) }} to {{ adapter.quote(column_name) }}\n {% endcall %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.577007}, "macro.dbt.alter_relation_add_remove_columns": {"unique_id": "macro.dbt.alter_relation_add_remove_columns", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "alter_relation_add_remove_columns", "macro_sql": "{% macro alter_relation_add_remove_columns(relation, add_columns = none, remove_columns = none) -%}\n {{ return(adapter.dispatch('alter_relation_add_remove_columns', 'dbt')(relation, add_columns, remove_columns)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__alter_relation_add_remove_columns"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.577695}, "macro.dbt.default__alter_relation_add_remove_columns": {"unique_id": "macro.dbt.default__alter_relation_add_remove_columns", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "default__alter_relation_add_remove_columns", "macro_sql": "{% macro default__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}\n\n {% if add_columns is none %}\n {% set add_columns = [] %}\n {% endif %}\n {% if remove_columns is none %}\n {% set remove_columns = [] %}\n {% endif %}\n\n {% set sql -%}\n\n alter {{ relation.type }} {{ relation }}\n\n {% for column in add_columns %}\n add column {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }}\n {% endfor %}{{ ',' if add_columns and remove_columns }}\n\n {% for column in remove_columns %}\n drop column {{ column.name }}{{ ',' if not loop.last }}\n {% endfor %}\n\n {%- endset -%}\n\n {% do run_query(sql) %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.579798}, "macro.dbt.test_unique": {"unique_id": "macro.dbt.test_unique", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "name": "test_unique", "macro_sql": "{% test unique(model, column_name) %}\n {% set macro = adapter.dispatch('test_unique', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__test_unique"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.581336}, "macro.dbt.test_not_null": {"unique_id": "macro.dbt.test_not_null", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "name": "test_not_null", "macro_sql": "{% test not_null(model, column_name) %}\n {% set macro = adapter.dispatch('test_not_null', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__test_not_null"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.581941}, "macro.dbt.test_accepted_values": {"unique_id": "macro.dbt.test_accepted_values", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "name": "test_accepted_values", "macro_sql": "{% test accepted_values(model, column_name, values, quote=True) %}\n {% set macro = adapter.dispatch('test_accepted_values', 'dbt') %}\n {{ macro(model, column_name, values, quote) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__test_accepted_values"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.582701}, "macro.dbt.test_relationships": {"unique_id": "macro.dbt.test_relationships", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "name": "test_relationships", "macro_sql": "{% test relationships(model, column_name, to, field) %}\n {% set macro = adapter.dispatch('test_relationships', 'dbt') %}\n {{ macro(model, column_name, to, field) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__test_relationships"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058585.583425}}, "docs": {"dbt.__overview__": {"unique_id": "dbt.__overview__", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "overview.md", "original_file_path": "docs/overview.md", "name": "__overview__", "block_contents": "### Welcome!\n\nWelcome to the auto-generated documentation for your dbt project!\n\n### Navigation\n\nYou can use the `Project` and `Database` navigation tabs on the left side of the window to explore the models\nin your project.\n\n#### Project Tab\nThe `Project` tab mirrors the directory structure of your dbt project. In this tab, you can see all of the\nmodels defined in your dbt project, as well as models imported from dbt packages.\n\n#### Database Tab\nThe `Database` tab also exposes your models, but in a format that looks more like a database explorer. This view\nshows relations (tables and views) grouped into database schemas. Note that ephemeral models are _not_ shown\nin this interface, as they do not exist in the database.\n\n### Graph Exploration\nYou can click the blue icon on the bottom-right corner of the page to view the lineage graph of your models.\n\nOn model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the `Expand`\nbutton at the top-right of this lineage pane, you'll be able to see all of the models that are used to build,\nor are built from, the model you're exploring.\n\nOnce expanded, you'll be able to use the `--select` and `--exclude` model selection syntax to filter the\nmodels in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax).\n\nNote that you can also right-click on models to interactively filter and explore the graph.\n\n---\n\n### More information\n\n- [What is dbt](https://docs.getdbt.com/docs/introduction)?\n- Read the [dbt viewpoint](https://docs.getdbt.com/docs/viewpoint)\n- [Installation](https://docs.getdbt.com/docs/installation)\n- Join the [dbt Community](https://www.getdbt.com/community/) for questions and discussion"}}, "exposures": {}, "metrics": {"metric.test.my_metric": {"fqn": ["test", "my_metric"], "unique_id": "metric.test.my_metric", "package_name": "test", "root_path": "/Users/jerco/dev/scratch/testy", "path": "metric.yml", "original_file_path": "models/metric.yml", "model": "ref('my_model')", "name": "my_metric", "description": "", "label": "Count records", "type": "count", "sql": "*", "timestamp": "updated_at", "filters": [], "time_grains": ["day"], "dimensions": [], "resource_type": "metric", "meta": {}, "tags": [], "sources": [], "depends_on": {"macros": [], "nodes": ["model.test.my_model"]}, "refs": [["my_model"]], "created_at": 1663058585.822956}}, "selectors": {}, "disabled": {}, "parent_map": {"model.test.my_model": [], "metric.test.my_metric": ["model.test.my_model"]}, "child_map": {"model.test.my_model": ["metric.test.my_metric"], "metric.test.my_metric": []}} ================================================ FILE: tests/functional/artifacts/data/state/v6/manifest.json ================================================ {"metadata": {"dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v6.json", "dbt_version": "1.2.1", "generated_at": "2022-09-13T08:42:50.298210Z", "invocation_id": "aa834731-46c3-49aa-8ec8-956dae621b58", "env": {}, "project_id": "098f6bcd4621d373cade4e832627b4f6", "user_id": null, "send_anonymous_usage_stats": false, "adapter_type": "postgres"}, "nodes": {"model.test.my_model": {"raw_sql": "select 1 as id", "resource_type": "model", "depends_on": {"macros": [], "nodes": []}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "post-hook": [], "pre-hook": []}, "database": "jerco", "schema": "dbt_jcohen", "fqn": ["test", "my_model"], "unique_id": "model.test.my_model", "package_name": "test", "root_path": "/Users/jerco/dev/scratch/testy", "path": "my_model.sql", "original_file_path": "models/my_model.sql", "name": "my_model", "alias": "my_model", "checksum": {"name": "sha256", "checksum": "479636cb85ce8d3b0f8db5ff13cf338b61254ad98d905630eac61f963e719e9d"}, "tags": [], "refs": [], "sources": [], "metrics": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true}, "patch_path": null, "compiled_path": null, "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1663058263.145605}}, "sources": {}, "macros": {"macro.dbt_postgres.postgres__get_catalog": {"unique_id": "macro.dbt_postgres.postgres__get_catalog", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/catalog.sql", "original_file_path": "macros/catalog.sql", "name": "postgres__get_catalog", "macro_sql": "{% macro postgres__get_catalog(information_schema, schemas) -%}\n\n {%- call statement('catalog', fetch_result=True) -%}\n {#\n If the user has multiple databases set and the first one is wrong, this will fail.\n But we won't fail in the case where there are multiple quoting-difference-only dbs, which is better.\n #}\n {% set database = information_schema.database %}\n {{ adapter.verify_database(database) }}\n\n select\n '{{ database }}' as table_database,\n sch.nspname as table_schema,\n tbl.relname as table_name,\n case tbl.relkind\n when 'v' then 'VIEW'\n else 'BASE TABLE'\n end as table_type,\n tbl_desc.description as table_comment,\n col.attname as column_name,\n col.attnum as column_index,\n pg_catalog.format_type(col.atttypid, col.atttypmod) as column_type,\n col_desc.description as column_comment,\n pg_get_userbyid(tbl.relowner) as table_owner\n\n from pg_catalog.pg_namespace sch\n join pg_catalog.pg_class tbl on tbl.relnamespace = sch.oid\n join pg_catalog.pg_attribute col on col.attrelid = tbl.oid\n left outer join pg_catalog.pg_description tbl_desc on (tbl_desc.objoid = tbl.oid and tbl_desc.objsubid = 0)\n left outer join pg_catalog.pg_description col_desc on (col_desc.objoid = tbl.oid and col_desc.objsubid = col.attnum)\n\n where (\n {%- for schema in schemas -%}\n upper(sch.nspname) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n )\n and not pg_is_other_temp_schema(sch.oid) -- not a temporary schema belonging to another session\n and tbl.relpersistence in ('p', 'u') -- [p]ermanent table or [u]nlogged table. Exclude [t]emporary tables\n and tbl.relkind in ('r', 'v', 'f', 'p') -- o[r]dinary table, [v]iew, [f]oreign table, [p]artitioned table. Other values are [i]ndex, [S]equence, [c]omposite type, [t]OAST table, [m]aterialized view\n and col.attnum > 0 -- negative numbers are used for system columns such as oid\n and not col.attisdropped -- column as not been dropped\n\n order by\n sch.nspname,\n tbl.relname,\n col.attnum\n\n {%- endcall -%}\n\n {{ return(load_result('catalog').table) }}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.441694}, "macro.dbt_postgres.postgres_get_relations": {"unique_id": "macro.dbt_postgres.postgres_get_relations", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/relations.sql", "original_file_path": "macros/relations.sql", "name": "postgres_get_relations", "macro_sql": "{% macro postgres_get_relations () -%}\n\n {#\n -- in pg_depend, objid is the dependent, refobjid is the referenced object\n -- > a pg_depend entry indicates that the referenced object cannot be\n -- > dropped without also dropping the dependent object.\n #}\n\n {%- call statement('relations', fetch_result=True) -%}\n with relation as (\n select\n pg_rewrite.ev_class as class,\n pg_rewrite.oid as id\n from pg_rewrite\n ),\n class as (\n select\n oid as id,\n relname as name,\n relnamespace as schema,\n relkind as kind\n from pg_class\n ),\n dependency as (\n select distinct\n pg_depend.objid as id,\n pg_depend.refobjid as ref\n from pg_depend\n ),\n schema as (\n select\n pg_namespace.oid as id,\n pg_namespace.nspname as name\n from pg_namespace\n where nspname != 'information_schema' and nspname not like 'pg\\_%'\n ),\n referenced as (\n select\n relation.id AS id,\n referenced_class.name ,\n referenced_class.schema ,\n referenced_class.kind\n from relation\n join class as referenced_class on relation.class=referenced_class.id\n where referenced_class.kind in ('r', 'v')\n ),\n relationships as (\n select\n referenced.name as referenced_name,\n referenced.schema as referenced_schema_id,\n dependent_class.name as dependent_name,\n dependent_class.schema as dependent_schema_id,\n referenced.kind as kind\n from referenced\n join dependency on referenced.id=dependency.id\n join class as dependent_class on dependency.ref=dependent_class.id\n where\n (referenced.name != dependent_class.name or\n referenced.schema != dependent_class.schema)\n )\n\n select\n referenced_schema.name as referenced_schema,\n relationships.referenced_name as referenced_name,\n dependent_schema.name as dependent_schema,\n relationships.dependent_name as dependent_name\n from relationships\n join schema as dependent_schema on relationships.dependent_schema_id=dependent_schema.id\n join schema as referenced_schema on relationships.referenced_schema_id=referenced_schema.id\n group by referenced_schema, referenced_name, dependent_schema, dependent_name\n order by referenced_schema, referenced_name, dependent_schema, dependent_name;\n\n {%- endcall -%}\n\n {{ return(load_result('relations').table) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.443508}, "macro.dbt_postgres.postgres__create_table_as": {"unique_id": "macro.dbt_postgres.postgres__create_table_as", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__create_table_as", "macro_sql": "{% macro postgres__create_table_as(temporary, relation, sql) -%}\n {%- set unlogged = config.get('unlogged', default=false) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary -%}\n temporary\n {%- elif unlogged -%}\n unlogged\n {%- endif %} table {{ relation }}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.4609761}, "macro.dbt_postgres.postgres__get_create_index_sql": {"unique_id": "macro.dbt_postgres.postgres__get_create_index_sql", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__get_create_index_sql", "macro_sql": "{% macro postgres__get_create_index_sql(relation, index_dict) -%}\n {%- set index_config = adapter.parse_index(index_dict) -%}\n {%- set comma_separated_columns = \", \".join(index_config.columns) -%}\n {%- set index_name = index_config.render(relation) -%}\n\n create {% if index_config.unique -%}\n unique\n {%- endif %} index if not exists\n \"{{ index_name }}\"\n on {{ relation }} {% if index_config.type -%}\n using {{ index_config.type }}\n {%- endif %}\n ({{ comma_separated_columns }});\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.462327}, "macro.dbt_postgres.postgres__create_schema": {"unique_id": "macro.dbt_postgres.postgres__create_schema", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__create_schema", "macro_sql": "{% macro postgres__create_schema(relation) -%}\n {% if relation.database -%}\n {{ adapter.verify_database(relation.database) }}\n {%- endif -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier().include(database=False) }}\n {%- endcall -%}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.4631748}, "macro.dbt_postgres.postgres__drop_schema": {"unique_id": "macro.dbt_postgres.postgres__drop_schema", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__drop_schema", "macro_sql": "{% macro postgres__drop_schema(relation) -%}\n {% if relation.database -%}\n {{ adapter.verify_database(relation.database) }}\n {%- endif -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier().include(database=False) }} cascade\n {%- endcall -%}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.46401}, "macro.dbt_postgres.postgres__get_columns_in_relation": {"unique_id": "macro.dbt_postgres.postgres__get_columns_in_relation", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__get_columns_in_relation", "macro_sql": "{% macro postgres__get_columns_in_relation(relation) -%}\n {% call statement('get_columns_in_relation', fetch_result=True) %}\n select\n column_name,\n data_type,\n character_maximum_length,\n numeric_precision,\n numeric_scale\n\n from {{ relation.information_schema('columns') }}\n where table_name = '{{ relation.identifier }}'\n {% if relation.schema %}\n and table_schema = '{{ relation.schema }}'\n {% endif %}\n order by ordinal_position\n\n {% endcall %}\n {% set table = load_result('get_columns_in_relation').table %}\n {{ return(sql_convert_columns_in_relation(table)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.sql_convert_columns_in_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.465282}, "macro.dbt_postgres.postgres__list_relations_without_caching": {"unique_id": "macro.dbt_postgres.postgres__list_relations_without_caching", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__list_relations_without_caching", "macro_sql": "{% macro postgres__list_relations_without_caching(schema_relation) %}\n {% call statement('list_relations_without_caching', fetch_result=True) -%}\n select\n '{{ schema_relation.database }}' as database,\n tablename as name,\n schemaname as schema,\n 'table' as type\n from pg_tables\n where schemaname ilike '{{ schema_relation.schema }}'\n union all\n select\n '{{ schema_relation.database }}' as database,\n viewname as name,\n schemaname as schema,\n 'view' as type\n from pg_views\n where schemaname ilike '{{ schema_relation.schema }}'\n {% endcall %}\n {{ return(load_result('list_relations_without_caching').table) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.46641}, "macro.dbt_postgres.postgres__information_schema_name": {"unique_id": "macro.dbt_postgres.postgres__information_schema_name", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__information_schema_name", "macro_sql": "{% macro postgres__information_schema_name(database) -%}\n {% if database_name -%}\n {{ adapter.verify_database(database_name) }}\n {%- endif -%}\n information_schema\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.4668732}, "macro.dbt_postgres.postgres__list_schemas": {"unique_id": "macro.dbt_postgres.postgres__list_schemas", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__list_schemas", "macro_sql": "{% macro postgres__list_schemas(database) %}\n {% if database -%}\n {{ adapter.verify_database(database) }}\n {%- endif -%}\n {% call statement('list_schemas', fetch_result=True, auto_begin=False) %}\n select distinct nspname from pg_namespace\n {% endcall %}\n {{ return(load_result('list_schemas').table) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.4678211}, "macro.dbt_postgres.postgres__check_schema_exists": {"unique_id": "macro.dbt_postgres.postgres__check_schema_exists", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__check_schema_exists", "macro_sql": "{% macro postgres__check_schema_exists(information_schema, schema) -%}\n {% if information_schema.database -%}\n {{ adapter.verify_database(information_schema.database) }}\n {%- endif -%}\n {% call statement('check_schema_exists', fetch_result=True, auto_begin=False) %}\n select count(*) from pg_namespace where nspname = '{{ schema }}'\n {% endcall %}\n {{ return(load_result('check_schema_exists').table) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.4688902}, "macro.dbt_postgres.postgres__current_timestamp": {"unique_id": "macro.dbt_postgres.postgres__current_timestamp", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__current_timestamp", "macro_sql": "{% macro postgres__current_timestamp() -%}\n now()\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.46909}, "macro.dbt_postgres.postgres__snapshot_string_as_time": {"unique_id": "macro.dbt_postgres.postgres__snapshot_string_as_time", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__snapshot_string_as_time", "macro_sql": "{% macro postgres__snapshot_string_as_time(timestamp) -%}\n {%- set result = \"'\" ~ timestamp ~ \"'::timestamp without time zone\" -%}\n {{ return(result) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.469568}, "macro.dbt_postgres.postgres__snapshot_get_time": {"unique_id": "macro.dbt_postgres.postgres__snapshot_get_time", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__snapshot_get_time", "macro_sql": "{% macro postgres__snapshot_get_time() -%}\n {{ current_timestamp() }}::timestamp without time zone\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.469843}, "macro.dbt_postgres.postgres__make_relation_with_suffix": {"unique_id": "macro.dbt_postgres.postgres__make_relation_with_suffix", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__make_relation_with_suffix", "macro_sql": "{% macro postgres__make_relation_with_suffix(base_relation, suffix, dstring) %}\n {% if dstring %}\n {% set dt = modules.datetime.datetime.now() %}\n {% set dtstring = dt.strftime(\"%H%M%S%f\") %}\n {% set suffix = suffix ~ dtstring %}\n {% endif %}\n {% set suffix_length = suffix|length %}\n {% set relation_max_name_length = base_relation.relation_max_name_length() %}\n {% if suffix_length > relation_max_name_length %}\n {% do exceptions.raise_compiler_error('Relation suffix is too long (' ~ suffix_length ~ ' characters). Maximum length is ' ~ relation_max_name_length ~ ' characters.') %}\n {% endif %}\n {% set identifier = base_relation.identifier[:relation_max_name_length - suffix_length] ~ suffix %}\n\n {{ return(base_relation.incorporate(path={\"identifier\": identifier })) }}\n\n {% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.472267}, "macro.dbt_postgres.postgres__make_intermediate_relation": {"unique_id": "macro.dbt_postgres.postgres__make_intermediate_relation", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__make_intermediate_relation", "macro_sql": "{% macro postgres__make_intermediate_relation(base_relation, suffix) %}\n {{ return(postgres__make_relation_with_suffix(base_relation, suffix, dstring=False)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_relation_with_suffix"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.472806}, "macro.dbt_postgres.postgres__make_temp_relation": {"unique_id": "macro.dbt_postgres.postgres__make_temp_relation", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__make_temp_relation", "macro_sql": "{% macro postgres__make_temp_relation(base_relation, suffix) %}\n {% set temp_relation = postgres__make_relation_with_suffix(base_relation, suffix, dstring=True) %}\n {{ return(temp_relation.incorporate(path={\"schema\": none,\n \"database\": none})) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_relation_with_suffix"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.473689}, "macro.dbt_postgres.postgres__make_backup_relation": {"unique_id": "macro.dbt_postgres.postgres__make_backup_relation", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__make_backup_relation", "macro_sql": "{% macro postgres__make_backup_relation(base_relation, backup_relation_type, suffix) %}\n {% set backup_relation = postgres__make_relation_with_suffix(base_relation, suffix, dstring=False) %}\n {{ return(backup_relation.incorporate(type=backup_relation_type)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_relation_with_suffix"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.4744241}, "macro.dbt_postgres.postgres_escape_comment": {"unique_id": "macro.dbt_postgres.postgres_escape_comment", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres_escape_comment", "macro_sql": "{% macro postgres_escape_comment(comment) -%}\n {% if comment is not string %}\n {% do exceptions.raise_compiler_error('cannot escape a non-string: ' ~ comment) %}\n {% endif %}\n {%- set magic = '$dbt_comment_literal_block$' -%}\n {%- if magic in comment -%}\n {%- do exceptions.raise_compiler_error('The string ' ~ magic ~ ' is not allowed in comments.') -%}\n {%- endif -%}\n {{ magic }}{{ comment }}{{ magic }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.475544}, "macro.dbt_postgres.postgres__alter_relation_comment": {"unique_id": "macro.dbt_postgres.postgres__alter_relation_comment", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__alter_relation_comment", "macro_sql": "{% macro postgres__alter_relation_comment(relation, comment) %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n comment on {{ relation.type }} {{ relation }} is {{ escaped_comment }};\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres_escape_comment"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.4761422}, "macro.dbt_postgres.postgres__alter_column_comment": {"unique_id": "macro.dbt_postgres.postgres__alter_column_comment", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__alter_column_comment", "macro_sql": "{% macro postgres__alter_column_comment(relation, column_dict) %}\n {% set existing_columns = adapter.get_columns_in_relation(relation) | map(attribute=\"name\") | list %}\n {% for column_name in column_dict if (column_name in existing_columns) %}\n {% set comment = column_dict[column_name]['description'] %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n comment on column {{ relation }}.{{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }} is {{ escaped_comment }};\n {% endfor %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres_escape_comment"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.477773}, "macro.dbt_postgres.postgres__get_show_grant_sql": {"unique_id": "macro.dbt_postgres.postgres__get_show_grant_sql", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__get_show_grant_sql", "macro_sql": "\n\n{%- macro postgres__get_show_grant_sql(relation) -%}\n select grantee, privilege_type\n from {{ relation.information_schema('role_table_grants') }}\n where grantor = current_role\n and grantee != current_role\n and table_schema = '{{ relation.schema }}'\n and table_name = '{{ relation.identifier }}'\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.478305}, "macro.dbt_postgres.postgres__copy_grants": {"unique_id": "macro.dbt_postgres.postgres__copy_grants", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__copy_grants", "macro_sql": "{% macro postgres__copy_grants() %}\n {{ return(False) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.478604}, "macro.dbt_postgres.postgres__snapshot_merge_sql": {"unique_id": "macro.dbt_postgres.postgres__snapshot_merge_sql", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/materializations/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshot_merge.sql", "name": "postgres__snapshot_merge_sql", "macro_sql": "{% macro postgres__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n update {{ target }}\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n from {{ source }} as DBT_INTERNAL_SOURCE\n where DBT_INTERNAL_SOURCE.dbt_scd_id::text = {{ target }}.dbt_scd_id::text\n and DBT_INTERNAL_SOURCE.dbt_change_type::text in ('update'::text, 'delete'::text)\n and {{ target }}.dbt_valid_to is null;\n\n insert into {{ target }} ({{ insert_cols_csv }})\n select {% for column in insert_cols -%}\n DBT_INTERNAL_SOURCE.{{ column }} {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n from {{ source }} as DBT_INTERNAL_SOURCE\n where DBT_INTERNAL_SOURCE.dbt_change_type::text = 'insert'::text;\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.480701}, "macro.dbt_postgres.postgres__dateadd": {"unique_id": "macro.dbt_postgres.postgres__dateadd", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "name": "postgres__dateadd", "macro_sql": "{% macro postgres__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n {{ from_date_or_timestamp }} + ((interval '1 {{ datepart }}') * ({{ interval }}))\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.481434}, "macro.dbt_postgres.postgres__listagg": {"unique_id": "macro.dbt_postgres.postgres__listagg", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "name": "postgres__listagg", "macro_sql": "{% macro postgres__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n {% if limit_num -%}\n array_to_string(\n (array_agg(\n {{ measure }}\n {% if order_by_clause -%}\n {{ order_by_clause }}\n {%- endif %}\n ))[1:{{ limit_num }}],\n {{ delimiter_text }}\n )\n {%- else %}\n string_agg(\n {{ measure }},\n {{ delimiter_text }}\n {% if order_by_clause -%}\n {{ order_by_clause }}\n {%- endif %}\n )\n {%- endif %}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.483254}, "macro.dbt_postgres.postgres__datediff": {"unique_id": "macro.dbt_postgres.postgres__datediff", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "name": "postgres__datediff", "macro_sql": "{% macro postgres__datediff(first_date, second_date, datepart) -%}\n\n {% if datepart == 'year' %}\n (date_part('year', ({{second_date}})::date) - date_part('year', ({{first_date}})::date))\n {% elif datepart == 'quarter' %}\n ({{ datediff(first_date, second_date, 'year') }} * 4 + date_part('quarter', ({{second_date}})::date) - date_part('quarter', ({{first_date}})::date))\n {% elif datepart == 'month' %}\n ({{ datediff(first_date, second_date, 'year') }} * 12 + date_part('month', ({{second_date}})::date) - date_part('month', ({{first_date}})::date))\n {% elif datepart == 'day' %}\n (({{second_date}})::date - ({{first_date}})::date)\n {% elif datepart == 'week' %}\n ({{ datediff(first_date, second_date, 'day') }} / 7 + case\n when date_part('dow', ({{first_date}})::timestamp) <= date_part('dow', ({{second_date}})::timestamp) then\n case when {{first_date}} <= {{second_date}} then 0 else -1 end\n else\n case when {{first_date}} <= {{second_date}} then 1 else 0 end\n end)\n {% elif datepart == 'hour' %}\n ({{ datediff(first_date, second_date, 'day') }} * 24 + date_part('hour', ({{second_date}})::timestamp) - date_part('hour', ({{first_date}})::timestamp))\n {% elif datepart == 'minute' %}\n ({{ datediff(first_date, second_date, 'hour') }} * 60 + date_part('minute', ({{second_date}})::timestamp) - date_part('minute', ({{first_date}})::timestamp))\n {% elif datepart == 'second' %}\n ({{ datediff(first_date, second_date, 'minute') }} * 60 + floor(date_part('second', ({{second_date}})::timestamp)) - floor(date_part('second', ({{first_date}})::timestamp)))\n {% elif datepart == 'millisecond' %}\n ({{ datediff(first_date, second_date, 'minute') }} * 60000 + floor(date_part('millisecond', ({{second_date}})::timestamp)) - floor(date_part('millisecond', ({{first_date}})::timestamp)))\n {% elif datepart == 'microsecond' %}\n ({{ datediff(first_date, second_date, 'minute') }} * 60000000 + floor(date_part('microsecond', ({{second_date}})::timestamp)) - floor(date_part('microsecond', ({{first_date}})::timestamp)))\n {% else %}\n {{ exceptions.raise_compiler_error(\"Unsupported datepart for macro datediff in postgres: {!r}\".format(datepart)) }}\n {% endif %}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.datediff"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.492169}, "macro.dbt_postgres.postgres__any_value": {"unique_id": "macro.dbt_postgres.postgres__any_value", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "name": "postgres__any_value", "macro_sql": "{% macro postgres__any_value(expression) -%}\n\n min({{ expression }})\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.4927092}, "macro.dbt_postgres.postgres__last_day": {"unique_id": "macro.dbt_postgres.postgres__last_day", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "name": "postgres__last_day", "macro_sql": "{% macro postgres__last_day(date, datepart) -%}\n\n {%- if datepart == 'quarter' -%}\n -- postgres dateadd does not support quarter interval.\n cast(\n {{dbt.dateadd('day', '-1',\n dbt.dateadd('month', '3', dbt.date_trunc(datepart, date))\n )}}\n as date)\n {%- else -%}\n {{dbt.default_last_day(date, datepart)}}\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.dateadd", "macro.dbt.date_trunc", "macro.dbt.default_last_day"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.494153}, "macro.dbt_postgres.postgres__split_part": {"unique_id": "macro.dbt_postgres.postgres__split_part", "package_name": "dbt_postgres", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/postgres", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "name": "postgres__split_part", "macro_sql": "{% macro postgres__split_part(string_text, delimiter_text, part_number) %}\n\n {% if part_number >= 0 %}\n {{ dbt.default__split_part(string_text, delimiter_text, part_number) }}\n {% else %}\n {{ dbt._split_part_negative(string_text, delimiter_text, part_number) }}\n {% endif %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__split_part", "macro.dbt._split_part_negative"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.495394}, "macro.dbt.run_hooks": {"unique_id": "macro.dbt.run_hooks", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "name": "run_hooks", "macro_sql": "{% macro run_hooks(hooks, inside_transaction=True) %}\n {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %}\n {% if not inside_transaction and loop.first %}\n {% call statement(auto_begin=inside_transaction) %}\n commit;\n {% endcall %}\n {% endif %}\n {% set rendered = render(hook.get('sql')) | trim %}\n {% if (rendered | length) > 0 %}\n {% call statement(auto_begin=inside_transaction) %}\n {{ rendered }}\n {% endcall %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.498407}, "macro.dbt.make_hook_config": {"unique_id": "macro.dbt.make_hook_config", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "name": "make_hook_config", "macro_sql": "{% macro make_hook_config(sql, inside_transaction) %}\n {{ tojson({\"sql\": sql, \"transaction\": inside_transaction}) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.4989262}, "macro.dbt.before_begin": {"unique_id": "macro.dbt.before_begin", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "name": "before_begin", "macro_sql": "{% macro before_begin(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.499295}, "macro.dbt.in_transaction": {"unique_id": "macro.dbt.in_transaction", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "name": "in_transaction", "macro_sql": "{% macro in_transaction(sql) %}\n {{ make_hook_config(sql, inside_transaction=True) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.4996538}, "macro.dbt.after_commit": {"unique_id": "macro.dbt.after_commit", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "name": "after_commit", "macro_sql": "{% macro after_commit(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.5000162}, "macro.dbt.set_sql_header": {"unique_id": "macro.dbt.set_sql_header", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "name": "set_sql_header", "macro_sql": "{% macro set_sql_header(config) -%}\n {{ config.set('sql_header', caller()) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.5011232}, "macro.dbt.should_full_refresh": {"unique_id": "macro.dbt.should_full_refresh", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "name": "should_full_refresh", "macro_sql": "{% macro should_full_refresh() %}\n {% set config_full_refresh = config.get('full_refresh') %}\n {% if config_full_refresh is none %}\n {% set config_full_refresh = flags.FULL_REFRESH %}\n {% endif %}\n {% do return(config_full_refresh) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.50192}, "macro.dbt.should_store_failures": {"unique_id": "macro.dbt.should_store_failures", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "name": "should_store_failures", "macro_sql": "{% macro should_store_failures() %}\n {% set config_store_failures = config.get('store_failures') %}\n {% if config_store_failures is none %}\n {% set config_store_failures = flags.STORE_FAILURES %}\n {% endif %}\n {% do return(config_store_failures) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.502719}, "macro.dbt.snapshot_merge_sql": {"unique_id": "macro.dbt.snapshot_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", "name": "snapshot_merge_sql", "macro_sql": "{% macro snapshot_merge_sql(target, source, insert_cols) -%}\n {{ adapter.dispatch('snapshot_merge_sql', 'dbt')(target, source, insert_cols) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__snapshot_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.50401}, "macro.dbt.default__snapshot_merge_sql": {"unique_id": "macro.dbt.default__snapshot_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", "name": "default__snapshot_merge_sql", "macro_sql": "{% macro default__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on DBT_INTERNAL_SOURCE.dbt_scd_id = DBT_INTERNAL_DEST.dbt_scd_id\n\n when matched\n and DBT_INTERNAL_DEST.dbt_valid_to is null\n and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')\n then update\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n\n when not matched\n and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'\n then insert ({{ insert_cols_csv }})\n values ({{ insert_cols_csv }})\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.504728}, "macro.dbt.strategy_dispatch": {"unique_id": "macro.dbt.strategy_dispatch", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "strategy_dispatch", "macro_sql": "{% macro strategy_dispatch(name) -%}\n{% set original_name = name %}\n {% if '.' in name %}\n {% set package_name, name = name.split(\".\", 1) %}\n {% else %}\n {% set package_name = none %}\n {% endif %}\n\n {% if package_name is none %}\n {% set package_context = context %}\n {% elif package_name in context %}\n {% set package_context = context[package_name] %}\n {% else %}\n {% set error_msg %}\n Could not find package '{{package_name}}', called with '{{original_name}}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n\n {%- set search_name = 'snapshot_' ~ name ~ '_strategy' -%}\n\n {% if search_name not in package_context %}\n {% set error_msg %}\n The specified strategy macro '{{name}}' was not found in package '{{ package_name }}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n {{ return(package_context[search_name]) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.51448}, "macro.dbt.snapshot_hash_arguments": {"unique_id": "macro.dbt.snapshot_hash_arguments", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "snapshot_hash_arguments", "macro_sql": "{% macro snapshot_hash_arguments(args) -%}\n {{ adapter.dispatch('snapshot_hash_arguments', 'dbt')(args) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.51492}, "macro.dbt.default__snapshot_hash_arguments": {"unique_id": "macro.dbt.default__snapshot_hash_arguments", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "default__snapshot_hash_arguments", "macro_sql": "{% macro default__snapshot_hash_arguments(args) -%}\n md5({%- for arg in args -%}\n coalesce(cast({{ arg }} as varchar ), '')\n {% if not loop.last %} || '|' || {% endif %}\n {%- endfor -%})\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.5155}, "macro.dbt.snapshot_get_time": {"unique_id": "macro.dbt.snapshot_get_time", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "snapshot_get_time", "macro_sql": "{% macro snapshot_get_time() -%}\n {{ adapter.dispatch('snapshot_get_time', 'dbt')() }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__snapshot_get_time"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.515872}, "macro.dbt.default__snapshot_get_time": {"unique_id": "macro.dbt.default__snapshot_get_time", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "default__snapshot_get_time", "macro_sql": "{% macro default__snapshot_get_time() -%}\n {{ current_timestamp() }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.516114}, "macro.dbt.snapshot_timestamp_strategy": {"unique_id": "macro.dbt.snapshot_timestamp_strategy", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "snapshot_timestamp_strategy", "macro_sql": "{% macro snapshot_timestamp_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}\n {% set primary_key = config['unique_key'] %}\n {% set updated_at = config['updated_at'] %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %}\n\n {#/*\n The snapshot relation might not have an {{ updated_at }} value if the\n snapshot strategy is changed from `check` to `timestamp`. We\n should use a dbt-created column for the comparison in the snapshot\n table instead of assuming that the user-supplied {{ updated_at }}\n will be present in the historical data.\n\n See https://github.com/dbt-labs/dbt-core/issues/2350\n */ #}\n {% set row_changed_expr -%}\n ({{ snapshotted_rel }}.dbt_valid_from < {{ current_rel }}.{{ updated_at }})\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.518018}, "macro.dbt.snapshot_string_as_time": {"unique_id": "macro.dbt.snapshot_string_as_time", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "snapshot_string_as_time", "macro_sql": "{% macro snapshot_string_as_time(timestamp) -%}\n {{ adapter.dispatch('snapshot_string_as_time', 'dbt')(timestamp) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__snapshot_string_as_time"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.5184388}, "macro.dbt.default__snapshot_string_as_time": {"unique_id": "macro.dbt.default__snapshot_string_as_time", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "default__snapshot_string_as_time", "macro_sql": "{% macro default__snapshot_string_as_time(timestamp) %}\n {% do exceptions.raise_not_implemented(\n 'snapshot_string_as_time macro not implemented for adapter '+adapter.type()\n ) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.5188859}, "macro.dbt.snapshot_check_all_get_existing_columns": {"unique_id": "macro.dbt.snapshot_check_all_get_existing_columns", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "snapshot_check_all_get_existing_columns", "macro_sql": "{% macro snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) -%}\n {%- if not target_exists -%}\n {#-- no table yet -> return whatever the query does --#}\n {{ return((false, query_columns)) }}\n {%- endif -%}\n\n {#-- handle any schema changes --#}\n {%- set target_relation = adapter.get_relation(database=node.database, schema=node.schema, identifier=node.alias) -%}\n\n {% if check_cols_config == 'all' %}\n {%- set query_columns = get_columns_in_query(node['compiled_sql']) -%}\n\n {% elif check_cols_config is iterable and (check_cols_config | length) > 0 %}\n {#-- query for proper casing/quoting, to support comparison below --#}\n {%- set select_check_cols_from_target -%}\n select {{ check_cols_config | join(', ') }} from ({{ node['compiled_sql'] }}) subq\n {%- endset -%}\n {% set query_columns = get_columns_in_query(select_check_cols_from_target) %}\n\n {% else %}\n {% do exceptions.raise_compiler_error(\"Invalid value for 'check_cols': \" ~ check_cols_config) %}\n {% endif %}\n\n {%- set existing_cols = adapter.get_columns_in_relation(target_relation) | map(attribute = 'name') | list -%}\n {%- set ns = namespace() -%} {#-- handle for-loop scoping with a namespace --#}\n {%- set ns.column_added = false -%}\n\n {%- set intersection = [] -%}\n {%- for col in query_columns -%}\n {%- if col in existing_cols -%}\n {%- do intersection.append(adapter.quote(col)) -%}\n {%- else -%}\n {% set ns.column_added = true %}\n {%- endif -%}\n {%- endfor -%}\n {{ return((ns.column_added, intersection)) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_columns_in_query"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.522562}, "macro.dbt.snapshot_check_strategy": {"unique_id": "macro.dbt.snapshot_check_strategy", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "snapshot_check_strategy", "macro_sql": "{% macro snapshot_check_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}\n {% set check_cols_config = config['check_cols'] %}\n {% set primary_key = config['unique_key'] %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %}\n {% set updated_at = config.get('updated_at', snapshot_get_time()) %}\n\n {% set column_added = false %}\n\n {% set column_added, check_cols = snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) %}\n\n {%- set row_changed_expr -%}\n (\n {%- if column_added -%}\n {{ get_true_sql() }}\n {%- else -%}\n {%- for col in check_cols -%}\n {{ snapshotted_rel }}.{{ col }} != {{ current_rel }}.{{ col }}\n or\n (\n (({{ snapshotted_rel }}.{{ col }} is null) and not ({{ current_rel }}.{{ col }} is null))\n or\n ((not {{ snapshotted_rel }}.{{ col }} is null) and ({{ current_rel }}.{{ col }} is null))\n )\n {%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n {%- endif -%}\n )\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.snapshot_get_time", "macro.dbt.snapshot_check_all_get_existing_columns", "macro.dbt.get_true_sql", "macro.dbt.snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.526156}, "macro.dbt.create_columns": {"unique_id": "macro.dbt.create_columns", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "create_columns", "macro_sql": "{% macro create_columns(relation, columns) %}\n {{ adapter.dispatch('create_columns', 'dbt')(relation, columns) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__create_columns"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.53722}, "macro.dbt.default__create_columns": {"unique_id": "macro.dbt.default__create_columns", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "default__create_columns", "macro_sql": "{% macro default__create_columns(relation, columns) %}\n {% for column in columns %}\n {% call statement() %}\n alter table {{ relation }} add column \"{{ column.name }}\" {{ column.data_type }};\n {% endcall %}\n {% endfor %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.537965}, "macro.dbt.post_snapshot": {"unique_id": "macro.dbt.post_snapshot", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "post_snapshot", "macro_sql": "{% macro post_snapshot(staging_relation) %}\n {{ adapter.dispatch('post_snapshot', 'dbt')(staging_relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.538402}, "macro.dbt.default__post_snapshot": {"unique_id": "macro.dbt.default__post_snapshot", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "default__post_snapshot", "macro_sql": "{% macro default__post_snapshot(staging_relation) %}\n {# no-op #}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.538624}, "macro.dbt.get_true_sql": {"unique_id": "macro.dbt.get_true_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "get_true_sql", "macro_sql": "{% macro get_true_sql() %}\n {{ adapter.dispatch('get_true_sql', 'dbt')() }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_true_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.538997}, "macro.dbt.default__get_true_sql": {"unique_id": "macro.dbt.default__get_true_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "default__get_true_sql", "macro_sql": "{% macro default__get_true_sql() %}\n {{ return('TRUE') }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.539289}, "macro.dbt.snapshot_staging_table": {"unique_id": "macro.dbt.snapshot_staging_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "snapshot_staging_table", "macro_sql": "{% macro snapshot_staging_table(strategy, source_sql, target_relation) -%}\n {{ adapter.dispatch('snapshot_staging_table', 'dbt')(strategy, source_sql, target_relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__snapshot_staging_table"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.539823}, "macro.dbt.default__snapshot_staging_table": {"unique_id": "macro.dbt.default__snapshot_staging_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "default__snapshot_staging_table", "macro_sql": "{% macro default__snapshot_staging_table(strategy, source_sql, target_relation) -%}\n\n with snapshot_query as (\n\n {{ source_sql }}\n\n ),\n\n snapshotted_data as (\n\n select *,\n {{ strategy.unique_key }} as dbt_unique_key\n\n from {{ target_relation }}\n where dbt_valid_to is null\n\n ),\n\n insertions_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to,\n {{ strategy.scd_id }} as dbt_scd_id\n\n from snapshot_query\n ),\n\n updates_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n {{ strategy.updated_at }} as dbt_valid_to\n\n from snapshot_query\n ),\n\n {%- if strategy.invalidate_hard_deletes %}\n\n deletes_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key\n from snapshot_query\n ),\n {% endif %}\n\n insertions as (\n\n select\n 'insert' as dbt_change_type,\n source_data.*\n\n from insertions_source_data as source_data\n left outer join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where snapshotted_data.dbt_unique_key is null\n or (\n snapshotted_data.dbt_unique_key is not null\n and (\n {{ strategy.row_changed }}\n )\n )\n\n ),\n\n updates as (\n\n select\n 'update' as dbt_change_type,\n source_data.*,\n snapshotted_data.dbt_scd_id\n\n from updates_source_data as source_data\n join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where (\n {{ strategy.row_changed }}\n )\n )\n\n {%- if strategy.invalidate_hard_deletes -%}\n ,\n\n deletes as (\n\n select\n 'delete' as dbt_change_type,\n source_data.*,\n {{ snapshot_get_time() }} as dbt_valid_from,\n {{ snapshot_get_time() }} as dbt_updated_at,\n {{ snapshot_get_time() }} as dbt_valid_to,\n snapshotted_data.dbt_scd_id\n\n from snapshotted_data\n left join deletes_source_data as source_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where source_data.dbt_unique_key is null\n )\n {%- endif %}\n\n select * from insertions\n union all\n select * from updates\n {%- if strategy.invalidate_hard_deletes %}\n union all\n select * from deletes\n {%- endif %}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.snapshot_get_time"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.542159}, "macro.dbt.build_snapshot_table": {"unique_id": "macro.dbt.build_snapshot_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "build_snapshot_table", "macro_sql": "{% macro build_snapshot_table(strategy, sql) -%}\n {{ adapter.dispatch('build_snapshot_table', 'dbt')(strategy, sql) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__build_snapshot_table"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.5426402}, "macro.dbt.default__build_snapshot_table": {"unique_id": "macro.dbt.default__build_snapshot_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "default__build_snapshot_table", "macro_sql": "{% macro default__build_snapshot_table(strategy, sql) %}\n\n select *,\n {{ strategy.scd_id }} as dbt_scd_id,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to\n from (\n {{ sql }}\n ) sbq\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.543298}, "macro.dbt.build_snapshot_staging_table": {"unique_id": "macro.dbt.build_snapshot_staging_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "build_snapshot_staging_table", "macro_sql": "{% macro build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% set temp_relation = make_temp_relation(target_relation) %}\n\n {% set select = snapshot_staging_table(strategy, sql, target_relation) %}\n\n {% call statement('build_snapshot_staging_relation') %}\n {{ create_table_as(True, temp_relation, select) }}\n {% endcall %}\n\n {% do return(temp_relation) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.make_temp_relation", "macro.dbt.snapshot_staging_table", "macro.dbt.statement", "macro.dbt.create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.5444548}, "macro.dbt.materialization_snapshot_default": {"unique_id": "macro.dbt.materialization_snapshot_default", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/snapshot.sql", "original_file_path": "macros/materializations/snapshots/snapshot.sql", "name": "materialization_snapshot_default", "macro_sql": "{% materialization snapshot, default %}\n {%- set config = model['config'] -%}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n -- grab current tables grants config for comparision later on\n {%- set grant_config = config.get('grants') -%}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=model.database,\n schema=model.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", config, target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_sql']) %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% else %}\n\n {{ adapter.valid_snapshot_target(target_relation) }}\n\n {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% endif %}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode=False) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if not target_relation_exists %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if staging_table is defined %}\n {% do post_snapshot(staging_table) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_or_create_relation", "macro.dbt.run_hooks", "macro.dbt.strategy_dispatch", "macro.dbt.build_snapshot_table", "macro.dbt.create_table_as", "macro.dbt.build_snapshot_staging_table", "macro.dbt.create_columns", "macro.dbt.snapshot_merge_sql", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes", "macro.dbt.post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.561558}, "macro.dbt.materialization_test_default": {"unique_id": "macro.dbt.materialization_test_default", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/tests/test.sql", "original_file_path": "macros/materializations/tests/test.sql", "name": "materialization_test_default", "macro_sql": "{%- materialization test, default -%}\n\n {% set relations = [] %}\n\n {% if should_store_failures() %}\n\n {% set identifier = model['alias'] %}\n {% set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n {% set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database, type='table') -%} %}\n\n {% if old_relation %}\n {% do adapter.drop_relation(old_relation) %}\n {% endif %}\n\n {% call statement(auto_begin=True) %}\n {{ create_table_as(False, target_relation, sql) }}\n {% endcall %}\n\n {% do relations.append(target_relation) %}\n\n {% set main_sql %}\n select *\n from {{ target_relation }}\n {% endset %}\n\n {{ adapter.commit() }}\n\n {% else %}\n\n {% set main_sql = sql %}\n\n {% endif %}\n\n {% set limit = config.get('limit') %}\n {% set fail_calc = config.get('fail_calc') %}\n {% set warn_if = config.get('warn_if') %}\n {% set error_if = config.get('error_if') %}\n\n {% call statement('main', fetch_result=True) -%}\n\n {{ get_test_sql(main_sql, fail_calc, warn_if, error_if, limit)}}\n\n {%- endcall %}\n\n {{ return({'relations': relations}) }}\n\n{%- endmaterialization -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.should_store_failures", "macro.dbt.statement", "macro.dbt.create_table_as", "macro.dbt.get_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.5671492}, "macro.dbt.get_test_sql": {"unique_id": "macro.dbt.get_test_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "name": "get_test_sql", "macro_sql": "{% macro get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n {{ adapter.dispatch('get_test_sql', 'dbt')(main_sql, fail_calc, warn_if, error_if, limit) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.5684512}, "macro.dbt.default__get_test_sql": {"unique_id": "macro.dbt.default__get_test_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "name": "default__get_test_sql", "macro_sql": "{% macro default__get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n select\n {{ fail_calc }} as failures,\n {{ fail_calc }} {{ warn_if }} as should_warn,\n {{ fail_calc }} {{ error_if }} as should_error\n from (\n {{ main_sql }}\n {{ \"limit \" ~ limit if limit != none }}\n ) dbt_internal_test\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.569262}, "macro.dbt.get_where_subquery": {"unique_id": "macro.dbt.get_where_subquery", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/tests/where_subquery.sql", "original_file_path": "macros/materializations/tests/where_subquery.sql", "name": "get_where_subquery", "macro_sql": "{% macro get_where_subquery(relation) -%}\n {% do return(adapter.dispatch('get_where_subquery', 'dbt')(relation)) %}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_where_subquery"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.570368}, "macro.dbt.default__get_where_subquery": {"unique_id": "macro.dbt.default__get_where_subquery", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/tests/where_subquery.sql", "original_file_path": "macros/materializations/tests/where_subquery.sql", "name": "default__get_where_subquery", "macro_sql": "{% macro default__get_where_subquery(relation) -%}\n {% set where = config.get('where', '') %}\n {% if where %}\n {%- set filtered -%}\n (select * from {{ relation }} where {{ where }}) dbt_subquery\n {%- endset -%}\n {% do return(filtered) %}\n {%- else -%}\n {% do return(relation) %}\n {%- endif -%}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.5713542}, "macro.dbt.get_quoted_csv": {"unique_id": "macro.dbt.get_quoted_csv", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "name": "get_quoted_csv", "macro_sql": "{% macro get_quoted_csv(column_names) %}\n\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote(col)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.5735822}, "macro.dbt.diff_columns": {"unique_id": "macro.dbt.diff_columns", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "name": "diff_columns", "macro_sql": "{% macro diff_columns(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% set source_names = source_columns | map(attribute = 'column') | list %}\n {% set target_names = target_columns | map(attribute = 'column') | list %}\n\n {# --check whether the name attribute exists in the target - this does not perform a data type check #}\n {% for sc in source_columns %}\n {% if sc.name not in target_names %}\n {{ result.append(sc) }}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.5750248}, "macro.dbt.diff_column_data_types": {"unique_id": "macro.dbt.diff_column_data_types", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "name": "diff_column_data_types", "macro_sql": "{% macro diff_column_data_types(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% for sc in source_columns %}\n {% set tc = target_columns | selectattr(\"name\", \"equalto\", sc.name) | list | first %}\n {% if tc %}\n {% if sc.data_type != tc.data_type %}\n {{ result.append( { 'column_name': tc.name, 'new_type': sc.data_type } ) }}\n {% endif %}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.576639}, "macro.dbt.get_merge_sql": {"unique_id": "macro.dbt.get_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "name": "get_merge_sql", "macro_sql": "{% macro get_merge_sql(target, source, unique_key, dest_columns, predicates=none) -%}\n {{ adapter.dispatch('get_merge_sql', 'dbt')(target, source, unique_key, dest_columns, predicates) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.591182}, "macro.dbt.default__get_merge_sql": {"unique_id": "macro.dbt.default__get_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "name": "default__get_merge_sql", "macro_sql": "{% macro default__get_merge_sql(target, source, unique_key, dest_columns, predicates) -%}\n {%- set predicates = [] if predicates is none else [] + predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set update_columns = config.get('merge_update_columns', default = dest_columns | map(attribute=\"quoted\") | list) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}\n {% for key in unique_key %}\n {% set this_key_match %}\n DBT_INTERNAL_SOURCE.{{ key }} = DBT_INTERNAL_DEST.{{ key }}\n {% endset %}\n {% do predicates.append(this_key_match) %}\n {% endfor %}\n {% else %}\n {% set unique_key_match %}\n DBT_INTERNAL_SOURCE.{{ unique_key }} = DBT_INTERNAL_DEST.{{ unique_key }}\n {% endset %}\n {% do predicates.append(unique_key_match) %}\n {% endif %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n {{ sql_header if sql_header is not none }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{ predicates | join(' and ') }}\n\n {% if unique_key %}\n when matched then update set\n {% for column_name in update_columns -%}\n {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}\n {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n {% endif %}\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.595134}, "macro.dbt.get_delete_insert_merge_sql": {"unique_id": "macro.dbt.get_delete_insert_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "name": "get_delete_insert_merge_sql", "macro_sql": "{% macro get_delete_insert_merge_sql(target, source, unique_key, dest_columns) -%}\n {{ adapter.dispatch('get_delete_insert_merge_sql', 'dbt')(target, source, unique_key, dest_columns) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_delete_insert_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.622782}, "macro.dbt.default__get_delete_insert_merge_sql": {"unique_id": "macro.dbt.default__get_delete_insert_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "name": "default__get_delete_insert_merge_sql", "macro_sql": "{% macro default__get_delete_insert_merge_sql(target, source, unique_key, dest_columns) -%}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not string %}\n delete from {{target }}\n using {{ source }}\n where (\n {% for key in unique_key %}\n {{ source }}.{{ key }} = {{ target }}.{{ key }}\n {{ \"and \" if not loop.last }}\n {% endfor %}\n );\n {% else %}\n delete from {{ target }}\n where (\n {{ unique_key }}) in (\n select ({{ unique_key }})\n from {{ source }}\n );\n\n {% endif %}\n {% endif %}\n\n insert into {{ target }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ source }}\n )\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.624852}, "macro.dbt.get_insert_overwrite_merge_sql": {"unique_id": "macro.dbt.get_insert_overwrite_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "name": "get_insert_overwrite_merge_sql", "macro_sql": "{% macro get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header=false) -%}\n {{ adapter.dispatch('get_insert_overwrite_merge_sql', 'dbt')(target, source, dest_columns, predicates, include_sql_header) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_insert_overwrite_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.625576}, "macro.dbt.default__get_insert_overwrite_merge_sql": {"unique_id": "macro.dbt.default__get_insert_overwrite_merge_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "name": "default__get_insert_overwrite_merge_sql", "macro_sql": "{% macro default__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) -%}\n {%- set predicates = [] if predicates is none else [] + predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none and include_sql_header }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on FALSE\n\n when not matched by source\n {% if predicates %} and {{ predicates | join(' and ') }} {% endif %}\n then delete\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.627269}, "macro.dbt.is_incremental": {"unique_id": "macro.dbt.is_incremental", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/is_incremental.sql", "original_file_path": "macros/materializations/models/incremental/is_incremental.sql", "name": "is_incremental", "macro_sql": "{% macro is_incremental() %}\n {#-- do not run introspective queries in parsing #}\n {% if not execute %}\n {{ return(False) }}\n {% else %}\n {% set relation = adapter.get_relation(this.database, this.schema, this.table) %}\n {{ return(relation is not none\n and relation.type == 'table'\n and model.config.materialized == 'incremental'\n and not should_full_refresh()) }}\n {% endif %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.should_full_refresh"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.6292028}, "macro.dbt.materialization_incremental_default": {"unique_id": "macro.dbt.materialization_incremental_default", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/incremental.sql", "original_file_path": "macros/materializations/models/incremental/incremental.sql", "name": "materialization_incremental_default", "macro_sql": "{% materialization incremental, default -%}\n\n -- relations\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') -%}\n {%- set temp_relation = make_temp_relation(target_relation)-%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation)-%}\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n\n -- configs\n {%- set unique_key = config.get('unique_key') -%}\n {%- set full_refresh_mode = (should_full_refresh() or existing_relation.is_view) -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n\n -- the temp_ and backup_ relations should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation. This has to happen before\n -- BEGIN, in a separate transaction\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation)-%}\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set to_drop = [] %}\n\n {% if existing_relation is none %}\n {% set build_sql = get_create_table_as_sql(False, target_relation, sql) %}\n {% elif full_refresh_mode %}\n {% set build_sql = get_create_table_as_sql(False, intermediate_relation, sql) %}\n {% set need_swap = true %}\n {% else %}\n {% do run_query(get_create_table_as_sql(True, temp_relation, sql)) %}\n {% do adapter.expand_target_column_types(\n from_relation=temp_relation,\n to_relation=target_relation) %}\n {#-- Process schema changes. Returns dict of changes if successful. Use source columns for upserting/merging --#}\n {% set dest_columns = process_schema_changes(on_schema_change, temp_relation, existing_relation) %}\n {% if not dest_columns %}\n {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %}\n {% endif %}\n {% set build_sql = get_delete_insert_merge_sql(target_relation, temp_relation, unique_key, dest_columns) %}\n\n {% endif %}\n\n {% call statement(\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% if need_swap %}\n {% do adapter.rename_relation(target_relation, backup_relation) %}\n {% do adapter.rename_relation(intermediate_relation, target_relation) %}\n {% do to_drop.append(backup_relation) %}\n {% endif %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if existing_relation is none or existing_relation.is_view or should_full_refresh() %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {% do adapter.commit() %}\n\n {% for rel in to_drop %}\n {% do adapter.drop_relation(rel) %}\n {% endfor %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_temp_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.should_full_refresh", "macro.dbt.incremental_validate_on_schema_change", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks", "macro.dbt.get_create_table_as_sql", "macro.dbt.run_query", "macro.dbt.process_schema_changes", "macro.dbt.get_delete_insert_merge_sql", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.6413598}, "macro.dbt.incremental_validate_on_schema_change": {"unique_id": "macro.dbt.incremental_validate_on_schema_change", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "name": "incremental_validate_on_schema_change", "macro_sql": "{% macro incremental_validate_on_schema_change(on_schema_change, default='ignore') %}\n\n {% if on_schema_change not in ['sync_all_columns', 'append_new_columns', 'fail', 'ignore'] %}\n\n {% set log_message = 'Invalid value for on_schema_change (%s) specified. Setting default value of %s.' % (on_schema_change, default) %}\n {% do log(log_message) %}\n\n {{ return(default) }}\n\n {% else %}\n\n {{ return(on_schema_change) }}\n\n {% endif %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.654586}, "macro.dbt.check_for_schema_changes": {"unique_id": "macro.dbt.check_for_schema_changes", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "name": "check_for_schema_changes", "macro_sql": "{% macro check_for_schema_changes(source_relation, target_relation) %}\n\n {% set schema_changed = False %}\n\n {%- set source_columns = adapter.get_columns_in_relation(source_relation) -%}\n {%- set target_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set source_not_in_target = diff_columns(source_columns, target_columns) -%}\n {%- set target_not_in_source = diff_columns(target_columns, source_columns) -%}\n\n {% set new_target_types = diff_column_data_types(source_columns, target_columns) %}\n\n {% if source_not_in_target != [] %}\n {% set schema_changed = True %}\n {% elif target_not_in_source != [] or new_target_types != [] %}\n {% set schema_changed = True %}\n {% elif new_target_types != [] %}\n {% set schema_changed = True %}\n {% endif %}\n\n {% set changes_dict = {\n 'schema_changed': schema_changed,\n 'source_not_in_target': source_not_in_target,\n 'target_not_in_source': target_not_in_source,\n 'source_columns': source_columns,\n 'target_columns': target_columns,\n 'new_target_types': new_target_types\n } %}\n\n {% set msg %}\n In {{ target_relation }}:\n Schema changed: {{ schema_changed }}\n Source columns not in target: {{ source_not_in_target }}\n Target columns not in source: {{ target_not_in_source }}\n New column types: {{ new_target_types }}\n {% endset %}\n\n {% do log(msg) %}\n\n {{ return(changes_dict) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.diff_columns", "macro.dbt.diff_column_data_types"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.6579268}, "macro.dbt.sync_column_schemas": {"unique_id": "macro.dbt.sync_column_schemas", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "name": "sync_column_schemas", "macro_sql": "{% macro sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {%- set add_to_target_arr = schema_changes_dict['source_not_in_target'] -%}\n\n {%- if on_schema_change == 'append_new_columns'-%}\n {%- if add_to_target_arr | length > 0 -%}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, none) -%}\n {%- endif -%}\n\n {% elif on_schema_change == 'sync_all_columns' %}\n {%- set remove_from_target_arr = schema_changes_dict['target_not_in_source'] -%}\n {%- set new_target_types = schema_changes_dict['new_target_types'] -%}\n\n {% if add_to_target_arr | length > 0 or remove_from_target_arr | length > 0 %}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, remove_from_target_arr) -%}\n {% endif %}\n\n {% if new_target_types != [] %}\n {% for ntt in new_target_types %}\n {% set column_name = ntt['column_name'] %}\n {% set new_type = ntt['new_type'] %}\n {% do alter_column_type(target_relation, column_name, new_type) %}\n {% endfor %}\n {% endif %}\n\n {% endif %}\n\n {% set schema_change_message %}\n In {{ target_relation }}:\n Schema change approach: {{ on_schema_change }}\n Columns added: {{ add_to_target_arr }}\n Columns removed: {{ remove_from_target_arr }}\n Data types changed: {{ new_target_types }}\n {% endset %}\n\n {% do log(schema_change_message) %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.alter_relation_add_remove_columns", "macro.dbt.alter_column_type"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.6611688}, "macro.dbt.process_schema_changes": {"unique_id": "macro.dbt.process_schema_changes", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "name": "process_schema_changes", "macro_sql": "{% macro process_schema_changes(on_schema_change, source_relation, target_relation) %}\n\n {% if on_schema_change == 'ignore' %}\n\n {{ return({}) }}\n\n {% else %}\n\n {% set schema_changes_dict = check_for_schema_changes(source_relation, target_relation) %}\n\n {% if schema_changes_dict['schema_changed'] %}\n\n {% if on_schema_change == 'fail' %}\n\n {% set fail_msg %}\n The source and target schemas on this incremental model are out of sync!\n They can be reconciled in several ways:\n - set the `on_schema_change` config to either append_new_columns or sync_all_columns, depending on your situation.\n - Re-run the incremental model with `full_refresh: True` to update the target schema.\n - update the schema manually and re-run the process.\n {% endset %}\n\n {% do exceptions.raise_compiler_error(fail_msg) %}\n\n {# -- unless we ignore, run the sync operation per the config #}\n {% else %}\n\n {% do sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {% endif %}\n\n {% endif %}\n\n {{ return(schema_changes_dict['source_columns']) }}\n\n {% endif %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.check_for_schema_changes", "macro.dbt.sync_column_schemas"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.663094}, "macro.dbt.materialization_table_default": {"unique_id": "macro.dbt.materialization_table_default", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/table/table.sql", "original_file_path": "macros/materializations/models/table/table.sql", "name": "materialization_table_default", "macro_sql": "{% materialization table, default %}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') %}\n {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}\n /*\n See ../view/view.sql for more information about this relation.\n */\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_table_as_sql(False, intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n {% if existing_relation is not none %}\n {{ adapter.rename_relation(existing_relation, backup_relation) }}\n {% endif %}\n\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% do create_indexes(target_relation) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n -- finally, drop the existing/backup relation after the commit\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n{% endmaterialization %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.get_create_table_as_sql", "macro.dbt.create_indexes", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.670167}, "macro.dbt.get_create_table_as_sql": {"unique_id": "macro.dbt.get_create_table_as_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/table/create_table_as.sql", "original_file_path": "macros/materializations/models/table/create_table_as.sql", "name": "get_create_table_as_sql", "macro_sql": "{% macro get_create_table_as_sql(temporary, relation, sql) -%}\n {{ adapter.dispatch('get_create_table_as_sql', 'dbt')(temporary, relation, sql) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_create_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.6714208}, "macro.dbt.default__get_create_table_as_sql": {"unique_id": "macro.dbt.default__get_create_table_as_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/table/create_table_as.sql", "original_file_path": "macros/materializations/models/table/create_table_as.sql", "name": "default__get_create_table_as_sql", "macro_sql": "{% macro default__get_create_table_as_sql(temporary, relation, sql) -%}\n {{ return(create_table_as(temporary, relation, sql)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.671903}, "macro.dbt.create_table_as": {"unique_id": "macro.dbt.create_table_as", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/table/create_table_as.sql", "original_file_path": "macros/materializations/models/table/create_table_as.sql", "name": "create_table_as", "macro_sql": "{% macro create_table_as(temporary, relation, sql) -%}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, sql) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.67243}, "macro.dbt.default__create_table_as": {"unique_id": "macro.dbt.default__create_table_as", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/table/create_table_as.sql", "original_file_path": "macros/materializations/models/table/create_table_as.sql", "name": "default__create_table_as", "macro_sql": "{% macro default__create_table_as(temporary, relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary: -%}temporary{%- endif %} table\n {{ relation.include(database=(not temporary), schema=(not temporary)) }}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.673532}, "macro.dbt.materialization_view_default": {"unique_id": "macro.dbt.materialization_view_default", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/view/view.sql", "original_file_path": "macros/materializations/models/view/view.sql", "name": "materialization_view_default", "macro_sql": "{%- materialization view, default -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='view') -%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}\n\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}\n /*\n This relation (probably) doesn't exist yet. If it does exist, it's a leftover from\n a previous run, and we're going to try to drop it immediately. At the end of this\n materialization, we're going to rename the \"existing_relation\" to this identifier,\n and then we're going to drop it. In order to make sure we run the correct one of:\n - drop view ...\n - drop table ...\n\n We need to set the type of this relation to be the type of the existing_relation, if it exists,\n or else \"view\" as a sane default if it does not. Note that if the existing_relation does not\n exist, then there is nothing to move out of the way and subsequentally drop. In that case,\n this relation will be effectively unused.\n */\n {%- set backup_relation_type = 'view' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n -- move the existing view out of the way\n {% if existing_relation is not none %}\n {{ adapter.rename_relation(existing_relation, backup_relation) }}\n {% endif %}\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.run_hooks", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.get_create_view_as_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.680615}, "macro.dbt.handle_existing_table": {"unique_id": "macro.dbt.handle_existing_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/view/helpers.sql", "original_file_path": "macros/materializations/models/view/helpers.sql", "name": "handle_existing_table", "macro_sql": "{% macro handle_existing_table(full_refresh, old_relation) %}\n {{ adapter.dispatch('handle_existing_table', 'dbt')(full_refresh, old_relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__handle_existing_table"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.6815622}, "macro.dbt.default__handle_existing_table": {"unique_id": "macro.dbt.default__handle_existing_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/view/helpers.sql", "original_file_path": "macros/materializations/models/view/helpers.sql", "name": "default__handle_existing_table", "macro_sql": "{% macro default__handle_existing_table(full_refresh, old_relation) %}\n {{ log(\"Dropping relation \" ~ old_relation ~ \" because it is of type \" ~ old_relation.type) }}\n {{ adapter.drop_relation(old_relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.682152}, "macro.dbt.create_or_replace_view": {"unique_id": "macro.dbt.create_or_replace_view", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/view/create_or_replace_view.sql", "original_file_path": "macros/materializations/models/view/create_or_replace_view.sql", "name": "create_or_replace_view", "macro_sql": "{% macro create_or_replace_view() %}\n {%- set identifier = model['alias'] -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database,\n type='view') -%}\n {% set grant_config = config.get('grants') %}\n\n {{ run_hooks(pre_hooks) }}\n\n -- If there's a table with the same name and we weren't told to full refresh,\n -- that's an error. If we were told to full refresh, drop it. This behavior differs\n -- for Snowflake and BigQuery, so multiple dispatch is used.\n {%- if old_relation is not none and old_relation.is_table -%}\n {{ handle_existing_table(should_full_refresh(), old_relation) }}\n {%- endif -%}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(target_relation, sql) }}\n {%- endcall %}\n\n {% set should_revoke = should_revoke(exists_as_view, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=True) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.handle_existing_table", "macro.dbt.should_full_refresh", "macro.dbt.statement", "macro.dbt.get_create_view_as_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.686346}, "macro.dbt.get_create_view_as_sql": {"unique_id": "macro.dbt.get_create_view_as_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/view/create_view_as.sql", "original_file_path": "macros/materializations/models/view/create_view_as.sql", "name": "get_create_view_as_sql", "macro_sql": "{% macro get_create_view_as_sql(relation, sql) -%}\n {{ adapter.dispatch('get_create_view_as_sql', 'dbt')(relation, sql) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_create_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.687447}, "macro.dbt.default__get_create_view_as_sql": {"unique_id": "macro.dbt.default__get_create_view_as_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/view/create_view_as.sql", "original_file_path": "macros/materializations/models/view/create_view_as.sql", "name": "default__get_create_view_as_sql", "macro_sql": "{% macro default__get_create_view_as_sql(relation, sql) -%}\n {{ return(create_view_as(relation, sql)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.6878788}, "macro.dbt.create_view_as": {"unique_id": "macro.dbt.create_view_as", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/view/create_view_as.sql", "original_file_path": "macros/materializations/models/view/create_view_as.sql", "name": "create_view_as", "macro_sql": "{% macro create_view_as(relation, sql) -%}\n {{ adapter.dispatch('create_view_as', 'dbt')(relation, sql) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.688359}, "macro.dbt.default__create_view_as": {"unique_id": "macro.dbt.default__create_view_as", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/view/create_view_as.sql", "original_file_path": "macros/materializations/models/view/create_view_as.sql", "name": "default__create_view_as", "macro_sql": "{% macro default__create_view_as(relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n create view {{ relation }} as (\n {{ sql }}\n );\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.6890602}, "macro.dbt.materialization_seed_default": {"unique_id": "macro.dbt.materialization_seed_default", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/seed.sql", "original_file_path": "macros/materializations/seeds/seed.sql", "name": "materialization_seed_default", "macro_sql": "{% materialization seed, default %}\n\n {%- set identifier = model['alias'] -%}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set grant_config = config.get('grants') -%}\n {%- set agate_table = load_agate_table() -%}\n -- grab current tables grants config for comparision later on\n\n {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% set create_table_sql = \"\" %}\n {% if exists_as_view %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a view\".format(old_relation)) }}\n {% elif exists_as_table %}\n {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}\n {% else %}\n {% set create_table_sql = create_csv_table(model, agate_table) %}\n {% endif %}\n\n {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}\n {% set rows_affected = (agate_table.rows | length) %}\n {% set sql = load_csv_rows(model, agate_table) %}\n\n {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}\n {{ get_csv_sql(create_table_sql, sql) }};\n {% endcall %}\n\n {% set target_relation = this.incorporate(type='table') %}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if full_refresh_mode or not exists_as_table %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.run_hooks", "macro.dbt.reset_csv_table", "macro.dbt.create_csv_table", "macro.dbt.load_csv_rows", "macro.dbt.noop_statement", "macro.dbt.get_csv_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.698056}, "macro.dbt.create_csv_table": {"unique_id": "macro.dbt.create_csv_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "create_csv_table", "macro_sql": "{% macro create_csv_table(model, agate_table) -%}\n {{ adapter.dispatch('create_csv_table', 'dbt')(model, agate_table) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.7117178}, "macro.dbt.default__create_csv_table": {"unique_id": "macro.dbt.default__create_csv_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "default__create_csv_table", "macro_sql": "{% macro default__create_csv_table(model, agate_table) %}\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n\n {% set sql %}\n create table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.7141619}, "macro.dbt.reset_csv_table": {"unique_id": "macro.dbt.reset_csv_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "reset_csv_table", "macro_sql": "{% macro reset_csv_table(model, full_refresh, old_relation, agate_table) -%}\n {{ adapter.dispatch('reset_csv_table', 'dbt')(model, full_refresh, old_relation, agate_table) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__reset_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.714784}, "macro.dbt.default__reset_csv_table": {"unique_id": "macro.dbt.default__reset_csv_table", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "default__reset_csv_table", "macro_sql": "{% macro default__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% set sql = \"\" %}\n {% if full_refresh %}\n {{ adapter.drop_relation(old_relation) }}\n {% set sql = create_csv_table(model, agate_table) %}\n {% else %}\n {{ adapter.truncate_relation(old_relation) }}\n {% set sql = \"truncate table \" ~ old_relation %}\n {% endif %}\n\n {{ return(sql) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.716077}, "macro.dbt.get_csv_sql": {"unique_id": "macro.dbt.get_csv_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "get_csv_sql", "macro_sql": "{% macro get_csv_sql(create_or_truncate_sql, insert_sql) %}\n {{ adapter.dispatch('get_csv_sql', 'dbt')(create_or_truncate_sql, insert_sql) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_csv_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.716581}, "macro.dbt.default__get_csv_sql": {"unique_id": "macro.dbt.default__get_csv_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "default__get_csv_sql", "macro_sql": "{% macro default__get_csv_sql(create_or_truncate_sql, insert_sql) %}\n {{ create_or_truncate_sql }};\n -- dbt seed --\n {{ insert_sql }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.716925}, "macro.dbt.get_binding_char": {"unique_id": "macro.dbt.get_binding_char", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "get_binding_char", "macro_sql": "{% macro get_binding_char() -%}\n {{ adapter.dispatch('get_binding_char', 'dbt')() }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.717283}, "macro.dbt.default__get_binding_char": {"unique_id": "macro.dbt.default__get_binding_char", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "default__get_binding_char", "macro_sql": "{% macro default__get_binding_char() %}\n {{ return('%s') }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.717576}, "macro.dbt.get_batch_size": {"unique_id": "macro.dbt.get_batch_size", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "get_batch_size", "macro_sql": "{% macro get_batch_size() -%}\n {{ return(adapter.dispatch('get_batch_size', 'dbt')()) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_batch_size"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.717988}, "macro.dbt.default__get_batch_size": {"unique_id": "macro.dbt.default__get_batch_size", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "default__get_batch_size", "macro_sql": "{% macro default__get_batch_size() %}\n {{ return(10000) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.718283}, "macro.dbt.get_seed_column_quoted_csv": {"unique_id": "macro.dbt.get_seed_column_quoted_csv", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "get_seed_column_quoted_csv", "macro_sql": "{% macro get_seed_column_quoted_csv(model, column_names) %}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote_seed_column(col, quote_seed_column)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.719548}, "macro.dbt.load_csv_rows": {"unique_id": "macro.dbt.load_csv_rows", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "load_csv_rows", "macro_sql": "{% macro load_csv_rows(model, agate_table) -%}\n {{ adapter.dispatch('load_csv_rows', 'dbt')(model, agate_table) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__load_csv_rows"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.720028}, "macro.dbt.default__load_csv_rows": {"unique_id": "macro.dbt.default__load_csv_rows", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "default__load_csv_rows", "macro_sql": "{% macro default__load_csv_rows(model, agate_table) %}\n\n {% set batch_size = get_batch_size() %}\n\n {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}\n {% set bindings = [] %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert into {{ this.render() }} ({{ cols_sql }}) values\n {% for row in chunk -%}\n ({%- for column in agate_table.column_names -%}\n {{ get_binding_char() }}\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_batch_size", "macro.dbt.get_seed_column_quoted_csv", "macro.dbt.get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.723425}, "macro.dbt.generate_alias_name": {"unique_id": "macro.dbt.generate_alias_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/get_custom_name/get_custom_alias.sql", "original_file_path": "macros/get_custom_name/get_custom_alias.sql", "name": "generate_alias_name", "macro_sql": "{% macro generate_alias_name(custom_alias_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_alias_name', 'dbt')(custom_alias_name, node)) %}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__generate_alias_name"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.724572}, "macro.dbt.default__generate_alias_name": {"unique_id": "macro.dbt.default__generate_alias_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/get_custom_name/get_custom_alias.sql", "original_file_path": "macros/get_custom_name/get_custom_alias.sql", "name": "default__generate_alias_name", "macro_sql": "{% macro default__generate_alias_name(custom_alias_name=none, node=none) -%}\n\n {%- if custom_alias_name is none -%}\n\n {{ node.name }}\n\n {%- else -%}\n\n {{ custom_alias_name | trim }}\n\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.725169}, "macro.dbt.generate_schema_name": {"unique_id": "macro.dbt.generate_schema_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "name": "generate_schema_name", "macro_sql": "{% macro generate_schema_name(custom_schema_name=none, node=none) -%}\n {{ return(adapter.dispatch('generate_schema_name', 'dbt')(custom_schema_name, node)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__generate_schema_name"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.72671}, "macro.dbt.default__generate_schema_name": {"unique_id": "macro.dbt.default__generate_schema_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "name": "default__generate_schema_name", "macro_sql": "{% macro default__generate_schema_name(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if custom_schema_name is none -%}\n\n {{ default_schema }}\n\n {%- else -%}\n\n {{ default_schema }}_{{ custom_schema_name | trim }}\n\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.727396}, "macro.dbt.generate_schema_name_for_env": {"unique_id": "macro.dbt.generate_schema_name_for_env", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "name": "generate_schema_name_for_env", "macro_sql": "{% macro generate_schema_name_for_env(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if target.name == 'prod' and custom_schema_name is not none -%}\n\n {{ custom_schema_name | trim }}\n\n {%- else -%}\n\n {{ default_schema }}\n\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.728158}, "macro.dbt.generate_database_name": {"unique_id": "macro.dbt.generate_database_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "name": "generate_database_name", "macro_sql": "{% macro generate_database_name(custom_database_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_database_name', 'dbt')(custom_database_name, node)) %}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__generate_database_name"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.729355}, "macro.dbt.default__generate_database_name": {"unique_id": "macro.dbt.default__generate_database_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "name": "default__generate_database_name", "macro_sql": "{% macro default__generate_database_name(custom_database_name=none, node=none) -%}\n {%- set default_database = target.database -%}\n {%- if custom_database_name is none -%}\n\n {{ default_database }}\n\n {%- else -%}\n\n {{ custom_database_name }}\n\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.73002}, "macro.dbt.default__test_relationships": {"unique_id": "macro.dbt.default__test_relationships", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/generic_test_sql/relationships.sql", "original_file_path": "macros/generic_test_sql/relationships.sql", "name": "default__test_relationships", "macro_sql": "{% macro default__test_relationships(model, column_name, to, field) %}\n\nwith child as (\n select {{ column_name }} as from_field\n from {{ model }}\n where {{ column_name }} is not null\n),\n\nparent as (\n select {{ field }} as to_field\n from {{ to }}\n)\n\nselect\n from_field\n\nfrom child\nleft join parent\n on child.from_field = parent.to_field\n\nwhere parent.to_field is null\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.7310588}, "macro.dbt.default__test_not_null": {"unique_id": "macro.dbt.default__test_not_null", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/generic_test_sql/not_null.sql", "original_file_path": "macros/generic_test_sql/not_null.sql", "name": "default__test_not_null", "macro_sql": "{% macro default__test_not_null(model, column_name) %}\n\n{% set column_list = '*' if should_store_failures() else column_name %}\n\nselect {{ column_list }}\nfrom {{ model }}\nwhere {{ column_name }} is null\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.should_store_failures"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.731988}, "macro.dbt.default__test_unique": {"unique_id": "macro.dbt.default__test_unique", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/generic_test_sql/unique.sql", "original_file_path": "macros/generic_test_sql/unique.sql", "name": "default__test_unique", "macro_sql": "{% macro default__test_unique(model, column_name) %}\n\nselect\n {{ column_name }} as unique_field,\n count(*) as n_records\n\nfrom {{ model }}\nwhere {{ column_name }} is not null\ngroup by {{ column_name }}\nhaving count(*) > 1\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.732775}, "macro.dbt.default__test_accepted_values": {"unique_id": "macro.dbt.default__test_accepted_values", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/generic_test_sql/accepted_values.sql", "original_file_path": "macros/generic_test_sql/accepted_values.sql", "name": "default__test_accepted_values", "macro_sql": "{% macro default__test_accepted_values(model, column_name, values, quote=True) %}\n\nwith all_values as (\n\n select\n {{ column_name }} as value_field,\n count(*) as n_records\n\n from {{ model }}\n group by {{ column_name }}\n\n)\n\nselect *\nfrom all_values\nwhere value_field not in (\n {% for value in values -%}\n {% if quote -%}\n '{{ value }}'\n {%- else -%}\n {{ value }}\n {%- endif -%}\n {%- if not loop.last -%},{%- endif %}\n {%- endfor %}\n)\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.734423}, "macro.dbt.statement": {"unique_id": "macro.dbt.statement", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "name": "statement", "macro_sql": "{% macro statement(name=None, fetch_result=False, auto_begin=True) -%}\n {%- if execute: -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- set res, table = adapter.execute(sql, auto_begin=auto_begin, fetch=fetch_result) -%}\n {%- if name is not none -%}\n {{ store_result(name, response=res, agate_table=table) }}\n {%- endif -%}\n\n {%- endif -%}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.737539}, "macro.dbt.noop_statement": {"unique_id": "macro.dbt.noop_statement", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "name": "noop_statement", "macro_sql": "{% macro noop_statement(name=None, message=None, code=None, rows_affected=None, res=None) -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_raw_result(name, message=message, code=code, rows_affected=rows_affected, agate_table=res) }}\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.739098}, "macro.dbt.run_query": {"unique_id": "macro.dbt.run_query", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "name": "run_query", "macro_sql": "{% macro run_query(sql) %}\n {% call statement(\"run_query_statement\", fetch_result=true, auto_begin=false) %}\n {{ sql }}\n {% endcall %}\n\n {% do return(load_result(\"run_query_statement\").table) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.739882}, "macro.dbt.convert_datetime": {"unique_id": "macro.dbt.convert_datetime", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "name": "convert_datetime", "macro_sql": "{% macro convert_datetime(date_str, date_fmt) %}\n\n {% set error_msg -%}\n The provided partition date '{{ date_str }}' does not match the expected format '{{ date_fmt }}'\n {%- endset %}\n\n {% set res = try_or_compiler_error(error_msg, modules.datetime.datetime.strptime, date_str.strip(), date_fmt) %}\n {{ return(res) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.744986}, "macro.dbt.dates_in_range": {"unique_id": "macro.dbt.dates_in_range", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "name": "dates_in_range", "macro_sql": "{% macro dates_in_range(start_date_str, end_date_str=none, in_fmt=\"%Y%m%d\", out_fmt=\"%Y%m%d\") %}\n {% set end_date_str = start_date_str if end_date_str is none else end_date_str %}\n\n {% set start_date = convert_datetime(start_date_str, in_fmt) %}\n {% set end_date = convert_datetime(end_date_str, in_fmt) %}\n\n {% set day_count = (end_date - start_date).days %}\n {% if day_count < 0 %}\n {% set msg -%}\n Partition start date is after the end date ({{ start_date }}, {{ end_date }})\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg, model) }}\n {% endif %}\n\n {% set date_list = [] %}\n {% for i in range(0, day_count + 1) %}\n {% set the_date = (modules.datetime.timedelta(days=i) + start_date) %}\n {% if not out_fmt %}\n {% set _ = date_list.append(the_date) %}\n {% else %}\n {% set _ = date_list.append(the_date.strftime(out_fmt)) %}\n {% endif %}\n {% endfor %}\n\n {{ return(date_list) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.convert_datetime"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.7483132}, "macro.dbt.partition_range": {"unique_id": "macro.dbt.partition_range", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "name": "partition_range", "macro_sql": "{% macro partition_range(raw_partition_date, date_fmt='%Y%m%d') %}\n {% set partition_range = (raw_partition_date | string).split(\",\") %}\n\n {% if (partition_range | length) == 1 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = none %}\n {% elif (partition_range | length) == 2 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = partition_range[1] %}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Invalid partition time. Expected format: {Start Date}[,{End Date}]. Got: \" ~ raw_partition_date) }}\n {% endif %}\n\n {{ return(dates_in_range(start_date, end_date, in_fmt=date_fmt)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.dates_in_range"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.750348}, "macro.dbt.py_current_timestring": {"unique_id": "macro.dbt.py_current_timestring", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "name": "py_current_timestring", "macro_sql": "{% macro py_current_timestring() %}\n {% set dt = modules.datetime.datetime.now() %}\n {% do return(dt.strftime(\"%Y%m%d%H%M%S%f\")) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.750963}, "macro.dbt.except": {"unique_id": "macro.dbt.except", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/except.sql", "original_file_path": "macros/utils/except.sql", "name": "except", "macro_sql": "{% macro except() %}\n {{ return(adapter.dispatch('except', 'dbt')()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__except"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.7517078}, "macro.dbt.default__except": {"unique_id": "macro.dbt.default__except", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/except.sql", "original_file_path": "macros/utils/except.sql", "name": "default__except", "macro_sql": "{% macro default__except() %}\n\n except\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.751895}, "macro.dbt.replace": {"unique_id": "macro.dbt.replace", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/replace.sql", "original_file_path": "macros/utils/replace.sql", "name": "replace", "macro_sql": "{% macro replace(field, old_chars, new_chars) -%}\n {{ return(adapter.dispatch('replace', 'dbt') (field, old_chars, new_chars)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__replace"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.752911}, "macro.dbt.default__replace": {"unique_id": "macro.dbt.default__replace", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/replace.sql", "original_file_path": "macros/utils/replace.sql", "name": "default__replace", "macro_sql": "{% macro default__replace(field, old_chars, new_chars) %}\n\n replace(\n {{ field }},\n {{ old_chars }},\n {{ new_chars }}\n )\n\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.753338}, "macro.dbt.concat": {"unique_id": "macro.dbt.concat", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/concat.sql", "original_file_path": "macros/utils/concat.sql", "name": "concat", "macro_sql": "{% macro concat(fields) -%}\n {{ return(adapter.dispatch('concat', 'dbt')(fields)) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__concat"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.7541351}, "macro.dbt.default__concat": {"unique_id": "macro.dbt.default__concat", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/concat.sql", "original_file_path": "macros/utils/concat.sql", "name": "default__concat", "macro_sql": "{% macro default__concat(fields) -%}\n {{ fields|join(' || ') }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.754451}, "macro.dbt.length": {"unique_id": "macro.dbt.length", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/length.sql", "original_file_path": "macros/utils/length.sql", "name": "length", "macro_sql": "{% macro length(expression) -%}\n {{ return(adapter.dispatch('length', 'dbt') (expression)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__length"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.75524}, "macro.dbt.default__length": {"unique_id": "macro.dbt.default__length", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/length.sql", "original_file_path": "macros/utils/length.sql", "name": "default__length", "macro_sql": "{% macro default__length(expression) %}\n\n length(\n {{ expression }}\n )\n\n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.755511}, "macro.dbt.dateadd": {"unique_id": "macro.dbt.dateadd", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "name": "dateadd", "macro_sql": "{% macro dateadd(datepart, interval, from_date_or_timestamp) %}\n {{ return(adapter.dispatch('dateadd', 'dbt')(datepart, interval, from_date_or_timestamp)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__dateadd"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.7565598}, "macro.dbt.default__dateadd": {"unique_id": "macro.dbt.default__dateadd", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "name": "default__dateadd", "macro_sql": "{% macro default__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n dateadd(\n {{ datepart }},\n {{ interval }},\n {{ from_date_or_timestamp }}\n )\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.756995}, "macro.dbt.intersect": {"unique_id": "macro.dbt.intersect", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/intersect.sql", "original_file_path": "macros/utils/intersect.sql", "name": "intersect", "macro_sql": "{% macro intersect() %}\n {{ return(adapter.dispatch('intersect', 'dbt')()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__intersect"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.7577422}, "macro.dbt.default__intersect": {"unique_id": "macro.dbt.default__intersect", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/intersect.sql", "original_file_path": "macros/utils/intersect.sql", "name": "default__intersect", "macro_sql": "{% macro default__intersect() %}\n\n intersect\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.7579389}, "macro.dbt.escape_single_quotes": {"unique_id": "macro.dbt.escape_single_quotes", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/escape_single_quotes.sql", "original_file_path": "macros/utils/escape_single_quotes.sql", "name": "escape_single_quotes", "macro_sql": "{% macro escape_single_quotes(expression) %}\n {{ return(adapter.dispatch('escape_single_quotes', 'dbt') (expression)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__escape_single_quotes"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.7587788}, "macro.dbt.default__escape_single_quotes": {"unique_id": "macro.dbt.default__escape_single_quotes", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/escape_single_quotes.sql", "original_file_path": "macros/utils/escape_single_quotes.sql", "name": "default__escape_single_quotes", "macro_sql": "{% macro default__escape_single_quotes(expression) -%}\n{{ expression | replace(\"'\",\"''\") }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.75913}, "macro.dbt.right": {"unique_id": "macro.dbt.right", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/right.sql", "original_file_path": "macros/utils/right.sql", "name": "right", "macro_sql": "{% macro right(string_text, length_expression) -%}\n {{ return(adapter.dispatch('right', 'dbt') (string_text, length_expression)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__right"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.760025}, "macro.dbt.default__right": {"unique_id": "macro.dbt.default__right", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/right.sql", "original_file_path": "macros/utils/right.sql", "name": "default__right", "macro_sql": "{% macro default__right(string_text, length_expression) %}\n\n right(\n {{ string_text }},\n {{ length_expression }}\n )\n\n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.760372}, "macro.dbt.listagg": {"unique_id": "macro.dbt.listagg", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "name": "listagg", "macro_sql": "{% macro listagg(measure, delimiter_text=\"','\", order_by_clause=none, limit_num=none) -%}\n {{ return(adapter.dispatch('listagg', 'dbt') (measure, delimiter_text, order_by_clause, limit_num)) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__listagg"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.762194}, "macro.dbt.default__listagg": {"unique_id": "macro.dbt.default__listagg", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "name": "default__listagg", "macro_sql": "{% macro default__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n {% if limit_num -%}\n array_to_string(\n array_slice(\n array_agg(\n {{ measure }}\n ){% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n ,0\n ,{{ limit_num }}\n ),\n {{ delimiter_text }}\n )\n {%- else %}\n listagg(\n {{ measure }},\n {{ delimiter_text }}\n )\n {% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n {%- endif %}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.7634282}, "macro.dbt.datediff": {"unique_id": "macro.dbt.datediff", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "name": "datediff", "macro_sql": "{% macro datediff(first_date, second_date, datepart) %}\n {{ return(adapter.dispatch('datediff', 'dbt')(first_date, second_date, datepart)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__datediff"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.764456}, "macro.dbt.default__datediff": {"unique_id": "macro.dbt.default__datediff", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "name": "default__datediff", "macro_sql": "{% macro default__datediff(first_date, second_date, datepart) -%}\n\n datediff(\n {{ datepart }},\n {{ first_date }},\n {{ second_date }}\n )\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.764895}, "macro.dbt.safe_cast": {"unique_id": "macro.dbt.safe_cast", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "name": "safe_cast", "macro_sql": "{% macro safe_cast(field, type) %}\n {{ return(adapter.dispatch('safe_cast', 'dbt') (field, type)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__safe_cast"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.76579}, "macro.dbt.default__safe_cast": {"unique_id": "macro.dbt.default__safe_cast", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "name": "default__safe_cast", "macro_sql": "{% macro default__safe_cast(field, type) %}\n {# most databases don't support this function yet\n so we just need to use cast #}\n cast({{field}} as {{type}})\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.7661529}, "macro.dbt.hash": {"unique_id": "macro.dbt.hash", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/hash.sql", "original_file_path": "macros/utils/hash.sql", "name": "hash", "macro_sql": "{% macro hash(field) -%}\n {{ return(adapter.dispatch('hash', 'dbt') (field)) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__hash"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.766952}, "macro.dbt.default__hash": {"unique_id": "macro.dbt.default__hash", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/hash.sql", "original_file_path": "macros/utils/hash.sql", "name": "default__hash", "macro_sql": "{% macro default__hash(field) -%}\n md5(cast({{ field }} as {{ api.Column.translate_type('string') }}))\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.767371}, "macro.dbt.cast_bool_to_text": {"unique_id": "macro.dbt.cast_bool_to_text", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/cast_bool_to_text.sql", "original_file_path": "macros/utils/cast_bool_to_text.sql", "name": "cast_bool_to_text", "macro_sql": "{% macro cast_bool_to_text(field) %}\n {{ adapter.dispatch('cast_bool_to_text', 'dbt') (field) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__cast_bool_to_text"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.7681432}, "macro.dbt.default__cast_bool_to_text": {"unique_id": "macro.dbt.default__cast_bool_to_text", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/cast_bool_to_text.sql", "original_file_path": "macros/utils/cast_bool_to_text.sql", "name": "default__cast_bool_to_text", "macro_sql": "{% macro default__cast_bool_to_text(field) %}\n cast({{ field }} as {{ api.Column.translate_type('string') }})\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.768564}, "macro.dbt.any_value": {"unique_id": "macro.dbt.any_value", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "name": "any_value", "macro_sql": "{% macro any_value(expression) -%}\n {{ return(adapter.dispatch('any_value', 'dbt') (expression)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__any_value"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.769363}, "macro.dbt.default__any_value": {"unique_id": "macro.dbt.default__any_value", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "name": "default__any_value", "macro_sql": "{% macro default__any_value(expression) -%}\n\n any_value({{ expression }})\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.769631}, "macro.dbt.position": {"unique_id": "macro.dbt.position", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/position.sql", "original_file_path": "macros/utils/position.sql", "name": "position", "macro_sql": "{% macro position(substring_text, string_text) -%}\n {{ return(adapter.dispatch('position', 'dbt') (substring_text, string_text)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__position"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.770546}, "macro.dbt.default__position": {"unique_id": "macro.dbt.default__position", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/position.sql", "original_file_path": "macros/utils/position.sql", "name": "default__position", "macro_sql": "{% macro default__position(substring_text, string_text) %}\n\n position(\n {{ substring_text }} in {{ string_text }}\n )\n\n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.7708979}, "macro.dbt.string_literal": {"unique_id": "macro.dbt.string_literal", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/literal.sql", "original_file_path": "macros/utils/literal.sql", "name": "string_literal", "macro_sql": "{%- macro string_literal(value) -%}\n {{ return(adapter.dispatch('string_literal', 'dbt') (value)) }}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__string_literal"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.77168}, "macro.dbt.default__string_literal": {"unique_id": "macro.dbt.default__string_literal", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/literal.sql", "original_file_path": "macros/utils/literal.sql", "name": "default__string_literal", "macro_sql": "{% macro default__string_literal(value) -%}\n '{{ value }}'\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.771949}, "macro.dbt.type_string": {"unique_id": "macro.dbt.type_string", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "name": "type_string", "macro_sql": "\n\n{%- macro type_string() -%}\n {{ return(adapter.dispatch('type_string', 'dbt')()) }}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__type_string"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.775103}, "macro.dbt.default__type_string": {"unique_id": "macro.dbt.default__type_string", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "name": "default__type_string", "macro_sql": "{% macro default__type_string() %}\n {{ return(api.Column.translate_type(\"string\")) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.7755132}, "macro.dbt.type_timestamp": {"unique_id": "macro.dbt.type_timestamp", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "name": "type_timestamp", "macro_sql": "\n\n{%- macro type_timestamp() -%}\n {{ return(adapter.dispatch('type_timestamp', 'dbt')()) }}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__type_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.7759519}, "macro.dbt.default__type_timestamp": {"unique_id": "macro.dbt.default__type_timestamp", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "name": "default__type_timestamp", "macro_sql": "{% macro default__type_timestamp() %}\n {{ return(api.Column.translate_type(\"timestamp\")) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.776351}, "macro.dbt.type_float": {"unique_id": "macro.dbt.type_float", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "name": "type_float", "macro_sql": "\n\n{%- macro type_float() -%}\n {{ return(adapter.dispatch('type_float', 'dbt')()) }}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__type_float"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.7769148}, "macro.dbt.default__type_float": {"unique_id": "macro.dbt.default__type_float", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "name": "default__type_float", "macro_sql": "{% macro default__type_float() %}\n {{ return(api.Column.translate_type(\"float\")) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.777309}, "macro.dbt.type_numeric": {"unique_id": "macro.dbt.type_numeric", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "name": "type_numeric", "macro_sql": "\n\n{%- macro type_numeric() -%}\n {{ return(adapter.dispatch('type_numeric', 'dbt')()) }}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__type_numeric"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.7777238}, "macro.dbt.default__type_numeric": {"unique_id": "macro.dbt.default__type_numeric", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "name": "default__type_numeric", "macro_sql": "{% macro default__type_numeric() %}\n {{ return(api.Column.numeric_type(\"numeric\", 28, 6)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.778186}, "macro.dbt.type_bigint": {"unique_id": "macro.dbt.type_bigint", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "name": "type_bigint", "macro_sql": "\n\n{%- macro type_bigint() -%}\n {{ return(adapter.dispatch('type_bigint', 'dbt')()) }}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__type_bigint"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.7786}, "macro.dbt.default__type_bigint": {"unique_id": "macro.dbt.default__type_bigint", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "name": "default__type_bigint", "macro_sql": "{% macro default__type_bigint() %}\n {{ return(api.Column.translate_type(\"bigint\")) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.778995}, "macro.dbt.type_int": {"unique_id": "macro.dbt.type_int", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "name": "type_int", "macro_sql": "\n\n{%- macro type_int() -%}\n {{ return(adapter.dispatch('type_int', 'dbt')()) }}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__type_int"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.7794151}, "macro.dbt.default__type_int": {"unique_id": "macro.dbt.default__type_int", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "name": "default__type_int", "macro_sql": "{%- macro default__type_int() -%}\n {{ return(api.Column.translate_type(\"integer\")) }}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.779795}, "macro.dbt.bool_or": {"unique_id": "macro.dbt.bool_or", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/bool_or.sql", "original_file_path": "macros/utils/bool_or.sql", "name": "bool_or", "macro_sql": "{% macro bool_or(expression) -%}\n {{ return(adapter.dispatch('bool_or', 'dbt') (expression)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__bool_or"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.780659}, "macro.dbt.default__bool_or": {"unique_id": "macro.dbt.default__bool_or", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/bool_or.sql", "original_file_path": "macros/utils/bool_or.sql", "name": "default__bool_or", "macro_sql": "{% macro default__bool_or(expression) -%}\n\n bool_or({{ expression }})\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.780927}, "macro.dbt.last_day": {"unique_id": "macro.dbt.last_day", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "name": "last_day", "macro_sql": "{% macro last_day(date, datepart) %}\n {{ return(adapter.dispatch('last_day', 'dbt') (date, datepart)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__last_day"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.782002}, "macro.dbt.default_last_day": {"unique_id": "macro.dbt.default_last_day", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "name": "default_last_day", "macro_sql": "\n\n{%- macro default_last_day(date, datepart) -%}\n cast(\n {{dbt.dateadd('day', '-1',\n dbt.dateadd(datepart, '1', dbt.date_trunc(datepart, date))\n )}}\n as date)\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.dateadd", "macro.dbt.date_trunc"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.782704}, "macro.dbt.default__last_day": {"unique_id": "macro.dbt.default__last_day", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "name": "default__last_day", "macro_sql": "{% macro default__last_day(date, datepart) -%}\n {{dbt.default_last_day(date, datepart)}}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default_last_day"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.7830899}, "macro.dbt.split_part": {"unique_id": "macro.dbt.split_part", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "name": "split_part", "macro_sql": "{% macro split_part(string_text, delimiter_text, part_number) %}\n {{ return(adapter.dispatch('split_part', 'dbt') (string_text, delimiter_text, part_number)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__split_part"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.7846441}, "macro.dbt.default__split_part": {"unique_id": "macro.dbt.default__split_part", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "name": "default__split_part", "macro_sql": "{% macro default__split_part(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n {{ part_number }}\n )\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.7850811}, "macro.dbt._split_part_negative": {"unique_id": "macro.dbt._split_part_negative", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "name": "_split_part_negative", "macro_sql": "{% macro _split_part_negative(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n length({{ string_text }})\n - length(\n replace({{ string_text }}, {{ delimiter_text }}, '')\n ) + 2 {{ part_number }}\n )\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.785673}, "macro.dbt.date_trunc": {"unique_id": "macro.dbt.date_trunc", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/date_trunc.sql", "original_file_path": "macros/utils/date_trunc.sql", "name": "date_trunc", "macro_sql": "{% macro date_trunc(datepart, date) -%}\n {{ return(adapter.dispatch('date_trunc', 'dbt') (datepart, date)) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__date_trunc"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.7865448}, "macro.dbt.default__date_trunc": {"unique_id": "macro.dbt.default__date_trunc", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/utils/date_trunc.sql", "original_file_path": "macros/utils/date_trunc.sql", "name": "default__date_trunc", "macro_sql": "{% macro default__date_trunc(datepart, date) -%}\n date_trunc('{{datepart}}', {{date}})\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.786887}, "macro.dbt.create_schema": {"unique_id": "macro.dbt.create_schema", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "name": "create_schema", "macro_sql": "{% macro create_schema(relation) -%}\n {{ adapter.dispatch('create_schema', 'dbt')(relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__create_schema"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.788033}, "macro.dbt.default__create_schema": {"unique_id": "macro.dbt.default__create_schema", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "name": "default__create_schema", "macro_sql": "{% macro default__create_schema(relation) -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier() }}\n {% endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.7884989}, "macro.dbt.drop_schema": {"unique_id": "macro.dbt.drop_schema", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "name": "drop_schema", "macro_sql": "{% macro drop_schema(relation) -%}\n {{ adapter.dispatch('drop_schema', 'dbt')(relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__drop_schema"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.788918}, "macro.dbt.default__drop_schema": {"unique_id": "macro.dbt.default__drop_schema", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "name": "default__drop_schema", "macro_sql": "{% macro default__drop_schema(relation) -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier() }} cascade\n {% endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.7895281}, "macro.dbt.get_create_index_sql": {"unique_id": "macro.dbt.get_create_index_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "name": "get_create_index_sql", "macro_sql": "{% macro get_create_index_sql(relation, index_dict) -%}\n {{ return(adapter.dispatch('get_create_index_sql', 'dbt')(relation, index_dict)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_create_index_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.79088}, "macro.dbt.default__get_create_index_sql": {"unique_id": "macro.dbt.default__get_create_index_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "name": "default__get_create_index_sql", "macro_sql": "{% macro default__get_create_index_sql(relation, index_dict) -%}\n {% do return(None) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.791239}, "macro.dbt.create_indexes": {"unique_id": "macro.dbt.create_indexes", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "name": "create_indexes", "macro_sql": "{% macro create_indexes(relation) -%}\n {{ adapter.dispatch('create_indexes', 'dbt')(relation) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.791649}, "macro.dbt.default__create_indexes": {"unique_id": "macro.dbt.default__create_indexes", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "name": "default__create_indexes", "macro_sql": "{% macro default__create_indexes(relation) -%}\n {%- set _indexes = config.get('indexes', default=[]) -%}\n\n {% for _index_dict in _indexes %}\n {% set create_index_sql = get_create_index_sql(relation, _index_dict) %}\n {% if create_index_sql %}\n {% do run_query(create_index_sql) %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_create_index_sql", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.792712}, "macro.dbt.make_intermediate_relation": {"unique_id": "macro.dbt.make_intermediate_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "make_intermediate_relation", "macro_sql": "{% macro make_intermediate_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_intermediate_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_intermediate_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.80177}, "macro.dbt.default__make_intermediate_relation": {"unique_id": "macro.dbt.default__make_intermediate_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "default__make_intermediate_relation", "macro_sql": "{% macro default__make_intermediate_relation(base_relation, suffix) %}\n {{ return(default__make_temp_relation(base_relation, suffix)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__make_temp_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.802206}, "macro.dbt.make_temp_relation": {"unique_id": "macro.dbt.make_temp_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "make_temp_relation", "macro_sql": "{% macro make_temp_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_temp_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_temp_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.802784}, "macro.dbt.default__make_temp_relation": {"unique_id": "macro.dbt.default__make_temp_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "default__make_temp_relation", "macro_sql": "{% macro default__make_temp_relation(base_relation, suffix) %}\n {%- set temp_identifier = base_relation.identifier ~ suffix -%}\n {%- set temp_relation = base_relation.incorporate(\n path={\"identifier\": temp_identifier}) -%}\n\n {{ return(temp_relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.8035462}, "macro.dbt.make_backup_relation": {"unique_id": "macro.dbt.make_backup_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "make_backup_relation", "macro_sql": "{% macro make_backup_relation(base_relation, backup_relation_type, suffix='__dbt_backup') %}\n {{ return(adapter.dispatch('make_backup_relation', 'dbt')(base_relation, backup_relation_type, suffix)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_backup_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.804203}, "macro.dbt.default__make_backup_relation": {"unique_id": "macro.dbt.default__make_backup_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "default__make_backup_relation", "macro_sql": "{% macro default__make_backup_relation(base_relation, backup_relation_type, suffix) %}\n {%- set backup_identifier = base_relation.identifier ~ suffix -%}\n {%- set backup_relation = base_relation.incorporate(\n path={\"identifier\": backup_identifier},\n type=backup_relation_type\n ) -%}\n {{ return(backup_relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.805045}, "macro.dbt.drop_relation": {"unique_id": "macro.dbt.drop_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "drop_relation", "macro_sql": "{% macro drop_relation(relation) -%}\n {{ return(adapter.dispatch('drop_relation', 'dbt')(relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__drop_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.805522}, "macro.dbt.default__drop_relation": {"unique_id": "macro.dbt.default__drop_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "default__drop_relation", "macro_sql": "{% macro default__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n drop {{ relation.type }} if exists {{ relation }} cascade\n {%- endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.8060842}, "macro.dbt.truncate_relation": {"unique_id": "macro.dbt.truncate_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "truncate_relation", "macro_sql": "{% macro truncate_relation(relation) -%}\n {{ return(adapter.dispatch('truncate_relation', 'dbt')(relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__truncate_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.806554}, "macro.dbt.default__truncate_relation": {"unique_id": "macro.dbt.default__truncate_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "default__truncate_relation", "macro_sql": "{% macro default__truncate_relation(relation) -%}\n {% call statement('truncate_relation') -%}\n truncate table {{ relation }}\n {%- endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.806978}, "macro.dbt.rename_relation": {"unique_id": "macro.dbt.rename_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "rename_relation", "macro_sql": "{% macro rename_relation(from_relation, to_relation) -%}\n {{ return(adapter.dispatch('rename_relation', 'dbt')(from_relation, to_relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__rename_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.8075058}, "macro.dbt.default__rename_relation": {"unique_id": "macro.dbt.default__rename_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "default__rename_relation", "macro_sql": "{% macro default__rename_relation(from_relation, to_relation) -%}\n {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %}\n {% call statement('rename_relation') -%}\n alter table {{ from_relation }} rename to {{ target_name }}\n {%- endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.808251}, "macro.dbt.get_or_create_relation": {"unique_id": "macro.dbt.get_or_create_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "get_or_create_relation", "macro_sql": "{% macro get_or_create_relation(database, schema, identifier, type) -%}\n {{ return(adapter.dispatch('get_or_create_relation', 'dbt')(database, schema, identifier, type)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_or_create_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.808909}, "macro.dbt.default__get_or_create_relation": {"unique_id": "macro.dbt.default__get_or_create_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "default__get_or_create_relation", "macro_sql": "{% macro default__get_or_create_relation(database, schema, identifier, type) %}\n {%- set target_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n\n {% if target_relation %}\n {% do return([true, target_relation]) %}\n {% endif %}\n\n {%- set new_relation = api.Relation.create(\n database=database,\n schema=schema,\n identifier=identifier,\n type=type\n ) -%}\n {% do return([false, new_relation]) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.810538}, "macro.dbt.load_cached_relation": {"unique_id": "macro.dbt.load_cached_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "load_cached_relation", "macro_sql": "{% macro load_cached_relation(relation) %}\n {% do return(adapter.get_relation(\n database=relation.database,\n schema=relation.schema,\n identifier=relation.identifier\n )) -%}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.8111548}, "macro.dbt.load_relation": {"unique_id": "macro.dbt.load_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "load_relation", "macro_sql": "{% macro load_relation(relation) %}\n {{ return(load_cached_relation(relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.load_cached_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.811518}, "macro.dbt.drop_relation_if_exists": {"unique_id": "macro.dbt.drop_relation_if_exists", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "drop_relation_if_exists", "macro_sql": "{% macro drop_relation_if_exists(relation) %}\n {% if relation is not none %}\n {{ adapter.drop_relation(relation) }}\n {% endif %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.812018}, "macro.dbt.current_timestamp": {"unique_id": "macro.dbt.current_timestamp", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "name": "current_timestamp", "macro_sql": "{% macro current_timestamp() -%}\n {{ adapter.dispatch('current_timestamp', 'dbt')() }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.813409}, "macro.dbt.default__current_timestamp": {"unique_id": "macro.dbt.default__current_timestamp", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "name": "default__current_timestamp", "macro_sql": "{% macro default__current_timestamp() -%}\n {{ exceptions.raise_not_implemented(\n 'current_timestamp macro not implemented for adapter '+adapter.type()) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.813792}, "macro.dbt.collect_freshness": {"unique_id": "macro.dbt.collect_freshness", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "name": "collect_freshness", "macro_sql": "{% macro collect_freshness(source, loaded_at_field, filter) %}\n {{ return(adapter.dispatch('collect_freshness', 'dbt')(source, loaded_at_field, filter))}}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__collect_freshness"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.8143969}, "macro.dbt.default__collect_freshness": {"unique_id": "macro.dbt.default__collect_freshness", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "name": "default__collect_freshness", "macro_sql": "{% macro default__collect_freshness(source, loaded_at_field, filter) %}\n {% call statement('collect_freshness', fetch_result=True, auto_begin=False) -%}\n select\n max({{ loaded_at_field }}) as max_loaded_at,\n {{ current_timestamp() }} as snapshotted_at\n from {{ source }}\n {% if filter %}\n where {{ filter }}\n {% endif %}\n {% endcall %}\n {{ return(load_result('collect_freshness').table) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.815532}, "macro.dbt.copy_grants": {"unique_id": "macro.dbt.copy_grants", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "name": "copy_grants", "macro_sql": "{% macro copy_grants() %}\n {{ return(adapter.dispatch('copy_grants', 'dbt')()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__copy_grants"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.820182}, "macro.dbt.default__copy_grants": {"unique_id": "macro.dbt.default__copy_grants", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "name": "default__copy_grants", "macro_sql": "{% macro default__copy_grants() %}\n {{ return(True) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.820483}, "macro.dbt.support_multiple_grantees_per_dcl_statement": {"unique_id": "macro.dbt.support_multiple_grantees_per_dcl_statement", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "name": "support_multiple_grantees_per_dcl_statement", "macro_sql": "{% macro support_multiple_grantees_per_dcl_statement() %}\n {{ return(adapter.dispatch('support_multiple_grantees_per_dcl_statement', 'dbt')()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__support_multiple_grantees_per_dcl_statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.820917}, "macro.dbt.default__support_multiple_grantees_per_dcl_statement": {"unique_id": "macro.dbt.default__support_multiple_grantees_per_dcl_statement", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "name": "default__support_multiple_grantees_per_dcl_statement", "macro_sql": "\n\n{%- macro default__support_multiple_grantees_per_dcl_statement() -%}\n {{ return(True) }}\n{%- endmacro -%}\n\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.8212}, "macro.dbt.should_revoke": {"unique_id": "macro.dbt.should_revoke", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "name": "should_revoke", "macro_sql": "{% macro should_revoke(existing_relation, full_refresh_mode=True) %}\n\n {% if not existing_relation %}\n {#-- The table doesn't already exist, so no grants to copy over --#}\n {{ return(False) }}\n {% elif full_refresh_mode %}\n {#-- The object is being REPLACED -- whether grants are copied over depends on the value of user config --#}\n {{ return(copy_grants()) }}\n {% else %}\n {#-- The table is being merged/upserted/inserted -- grants will be carried over --#}\n {{ return(True) }}\n {% endif %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.copy_grants"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.8221052}, "macro.dbt.get_show_grant_sql": {"unique_id": "macro.dbt.get_show_grant_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "name": "get_show_grant_sql", "macro_sql": "{% macro get_show_grant_sql(relation) %}\n {{ return(adapter.dispatch(\"get_show_grant_sql\", \"dbt\")(relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_show_grant_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.8225951}, "macro.dbt.default__get_show_grant_sql": {"unique_id": "macro.dbt.default__get_show_grant_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "name": "default__get_show_grant_sql", "macro_sql": "{% macro default__get_show_grant_sql(relation) %}\n show grants on {{ relation }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.822855}, "macro.dbt.get_grant_sql": {"unique_id": "macro.dbt.get_grant_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "name": "get_grant_sql", "macro_sql": "{% macro get_grant_sql(relation, privilege, grantees) %}\n {{ return(adapter.dispatch('get_grant_sql', 'dbt')(relation, privilege, grantees)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_grant_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.823464}, "macro.dbt.default__get_grant_sql": {"unique_id": "macro.dbt.default__get_grant_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "name": "default__get_grant_sql", "macro_sql": "\n\n{%- macro default__get_grant_sql(relation, privilege, grantees) -%}\n grant {{ privilege }} on {{ relation }} to {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.823967}, "macro.dbt.get_revoke_sql": {"unique_id": "macro.dbt.get_revoke_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "name": "get_revoke_sql", "macro_sql": "{% macro get_revoke_sql(relation, privilege, grantees) %}\n {{ return(adapter.dispatch('get_revoke_sql', 'dbt')(relation, privilege, grantees)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_revoke_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.8245819}, "macro.dbt.default__get_revoke_sql": {"unique_id": "macro.dbt.default__get_revoke_sql", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "name": "default__get_revoke_sql", "macro_sql": "\n\n{%- macro default__get_revoke_sql(relation, privilege, grantees) -%}\n revoke {{ privilege }} on {{ relation }} from {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.825077}, "macro.dbt.get_dcl_statement_list": {"unique_id": "macro.dbt.get_dcl_statement_list", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "name": "get_dcl_statement_list", "macro_sql": "{% macro get_dcl_statement_list(relation, grant_config, get_dcl_macro) %}\n {{ return(adapter.dispatch('get_dcl_statement_list', 'dbt')(relation, grant_config, get_dcl_macro)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_dcl_statement_list"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.825684}, "macro.dbt.default__get_dcl_statement_list": {"unique_id": "macro.dbt.default__get_dcl_statement_list", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "name": "default__get_dcl_statement_list", "macro_sql": "\n\n{%- macro default__get_dcl_statement_list(relation, grant_config, get_dcl_macro) -%}\n {#\n -- Unpack grant_config into specific privileges and the set of users who need them granted/revoked.\n -- Depending on whether this database supports multiple grantees per statement, pass in the list of\n -- all grantees per privilege, or (if not) template one statement per privilege-grantee pair.\n -- `get_dcl_macro` will be either `get_grant_sql` or `get_revoke_sql`\n #}\n {%- set dcl_statements = [] -%}\n {%- for privilege, grantees in grant_config.items() %}\n {%- if support_multiple_grantees_per_dcl_statement() and grantees -%}\n {%- set dcl = get_dcl_macro(relation, privilege, grantees) -%}\n {%- do dcl_statements.append(dcl) -%}\n {%- else -%}\n {%- for grantee in grantees -%}\n {% set dcl = get_dcl_macro(relation, privilege, [grantee]) %}\n {%- do dcl_statements.append(dcl) -%}\n {% endfor -%}\n {%- endif -%}\n {%- endfor -%}\n {{ return(dcl_statements) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.support_multiple_grantees_per_dcl_statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.827622}, "macro.dbt.call_dcl_statements": {"unique_id": "macro.dbt.call_dcl_statements", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "name": "call_dcl_statements", "macro_sql": "{% macro call_dcl_statements(dcl_statement_list) %}\n {{ return(adapter.dispatch(\"call_dcl_statements\", \"dbt\")(dcl_statement_list)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__call_dcl_statements"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.828118}, "macro.dbt.default__call_dcl_statements": {"unique_id": "macro.dbt.default__call_dcl_statements", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "name": "default__call_dcl_statements", "macro_sql": "{% macro default__call_dcl_statements(dcl_statement_list) %}\n {#\n -- By default, supply all grant + revoke statements in a single semicolon-separated block,\n -- so that they're all processed together.\n\n -- Some databases do not support this. Those adapters will need to override this macro\n -- to run each statement individually.\n #}\n {% call statement('grants') %}\n {% for dcl_statement in dcl_statement_list %}\n {{ dcl_statement }};\n {% endfor %}\n {% endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.828738}, "macro.dbt.apply_grants": {"unique_id": "macro.dbt.apply_grants", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "name": "apply_grants", "macro_sql": "{% macro apply_grants(relation, grant_config, should_revoke) %}\n {{ return(adapter.dispatch(\"apply_grants\", \"dbt\")(relation, grant_config, should_revoke)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__apply_grants"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.8293319}, "macro.dbt.default__apply_grants": {"unique_id": "macro.dbt.default__apply_grants", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "name": "default__apply_grants", "macro_sql": "{% macro default__apply_grants(relation, grant_config, should_revoke=True) %}\n {#-- If grant_config is {} or None, this is a no-op --#}\n {% if grant_config %}\n {% if should_revoke %}\n {#-- We think previous grants may have carried over --#}\n {#-- Show current grants and calculate diffs --#}\n {% set current_grants_table = run_query(get_show_grant_sql(relation)) %}\n {% set current_grants_dict = adapter.standardize_grants_dict(current_grants_table) %}\n {% set needs_granting = diff_of_two_dicts(grant_config, current_grants_dict) %}\n {% set needs_revoking = diff_of_two_dicts(current_grants_dict, grant_config) %}\n {% if not (needs_granting or needs_revoking) %}\n {{ log('On ' ~ relation ~': All grants are in place, no revocation or granting needed.')}}\n {% endif %}\n {% else %}\n {#-- We don't think there's any chance of previous grants having carried over. --#}\n {#-- Jump straight to granting what the user has configured. --#}\n {% set needs_revoking = {} %}\n {% set needs_granting = grant_config %}\n {% endif %}\n {% if needs_granting or needs_revoking %}\n {% set revoke_statement_list = get_dcl_statement_list(relation, needs_revoking, get_revoke_sql) %}\n {% set grant_statement_list = get_dcl_statement_list(relation, needs_granting, get_grant_sql) %}\n {% set dcl_statement_list = revoke_statement_list + grant_statement_list %}\n {% if dcl_statement_list %}\n {{ call_dcl_statements(dcl_statement_list) }}\n {% endif %}\n {% endif %}\n {% endif %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.run_query", "macro.dbt.get_show_grant_sql", "macro.dbt.get_dcl_statement_list", "macro.dbt.call_dcl_statements"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.8324008}, "macro.dbt.alter_column_comment": {"unique_id": "macro.dbt.alter_column_comment", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "name": "alter_column_comment", "macro_sql": "{% macro alter_column_comment(relation, column_dict) -%}\n {{ return(adapter.dispatch('alter_column_comment', 'dbt')(relation, column_dict)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.834379}, "macro.dbt.default__alter_column_comment": {"unique_id": "macro.dbt.default__alter_column_comment", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "name": "default__alter_column_comment", "macro_sql": "{% macro default__alter_column_comment(relation, column_dict) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_column_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.834809}, "macro.dbt.alter_relation_comment": {"unique_id": "macro.dbt.alter_relation_comment", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "name": "alter_relation_comment", "macro_sql": "{% macro alter_relation_comment(relation, relation_comment) -%}\n {{ return(adapter.dispatch('alter_relation_comment', 'dbt')(relation, relation_comment)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__alter_relation_comment"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.835356}, "macro.dbt.default__alter_relation_comment": {"unique_id": "macro.dbt.default__alter_relation_comment", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "name": "default__alter_relation_comment", "macro_sql": "{% macro default__alter_relation_comment(relation, relation_comment) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_relation_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.835799}, "macro.dbt.persist_docs": {"unique_id": "macro.dbt.persist_docs", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "name": "persist_docs", "macro_sql": "{% macro persist_docs(relation, model, for_relation=true, for_columns=true) -%}\n {{ return(adapter.dispatch('persist_docs', 'dbt')(relation, model, for_relation, for_columns)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.8365178}, "macro.dbt.default__persist_docs": {"unique_id": "macro.dbt.default__persist_docs", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "name": "default__persist_docs", "macro_sql": "{% macro default__persist_docs(relation, model, for_relation, for_columns) -%}\n {% if for_relation and config.persist_relation_docs() and model.description %}\n {% do run_query(alter_relation_comment(relation, model.description)) %}\n {% endif %}\n\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {% do run_query(alter_column_comment(relation, model.columns)) %}\n {% endif %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.run_query", "macro.dbt.alter_relation_comment", "macro.dbt.alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.837826}, "macro.dbt.get_catalog": {"unique_id": "macro.dbt.get_catalog", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "get_catalog", "macro_sql": "{% macro get_catalog(information_schema, schemas) -%}\n {{ return(adapter.dispatch('get_catalog', 'dbt')(information_schema, schemas)) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_catalog"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.841973}, "macro.dbt.default__get_catalog": {"unique_id": "macro.dbt.default__get_catalog", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "default__get_catalog", "macro_sql": "{% macro default__get_catalog(information_schema, schemas) -%}\n\n {% set typename = adapter.type() %}\n {% set msg -%}\n get_catalog not implemented for {{ typename }}\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.842654}, "macro.dbt.information_schema_name": {"unique_id": "macro.dbt.information_schema_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "information_schema_name", "macro_sql": "{% macro information_schema_name(database) %}\n {{ return(adapter.dispatch('information_schema_name', 'dbt')(database)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__information_schema_name"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.843149}, "macro.dbt.default__information_schema_name": {"unique_id": "macro.dbt.default__information_schema_name", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "default__information_schema_name", "macro_sql": "{% macro default__information_schema_name(database) -%}\n {%- if database -%}\n {{ database }}.INFORMATION_SCHEMA\n {%- else -%}\n INFORMATION_SCHEMA\n {%- endif -%}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.8435562}, "macro.dbt.list_schemas": {"unique_id": "macro.dbt.list_schemas", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "list_schemas", "macro_sql": "{% macro list_schemas(database) -%}\n {{ return(adapter.dispatch('list_schemas', 'dbt')(database)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__list_schemas"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.8440409}, "macro.dbt.default__list_schemas": {"unique_id": "macro.dbt.default__list_schemas", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "default__list_schemas", "macro_sql": "{% macro default__list_schemas(database) -%}\n {% set sql %}\n select distinct schema_name\n from {{ information_schema_name(database) }}.SCHEMATA\n where catalog_name ilike '{{ database }}'\n {% endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.information_schema_name", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.8447008}, "macro.dbt.check_schema_exists": {"unique_id": "macro.dbt.check_schema_exists", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "check_schema_exists", "macro_sql": "{% macro check_schema_exists(information_schema, schema) -%}\n {{ return(adapter.dispatch('check_schema_exists', 'dbt')(information_schema, schema)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__check_schema_exists"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.845243}, "macro.dbt.default__check_schema_exists": {"unique_id": "macro.dbt.default__check_schema_exists", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "default__check_schema_exists", "macro_sql": "{% macro default__check_schema_exists(information_schema, schema) -%}\n {% set sql -%}\n select count(*)\n from {{ information_schema.replace(information_schema_view='SCHEMATA') }}\n where catalog_name='{{ information_schema.database }}'\n and schema_name='{{ schema }}'\n {%- endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.replace", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.846046}, "macro.dbt.list_relations_without_caching": {"unique_id": "macro.dbt.list_relations_without_caching", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "list_relations_without_caching", "macro_sql": "{% macro list_relations_without_caching(schema_relation) %}\n {{ return(adapter.dispatch('list_relations_without_caching', 'dbt')(schema_relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__list_relations_without_caching"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.846539}, "macro.dbt.default__list_relations_without_caching": {"unique_id": "macro.dbt.default__list_relations_without_caching", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "default__list_relations_without_caching", "macro_sql": "{% macro default__list_relations_without_caching(schema_relation) %}\n {{ exceptions.raise_not_implemented(\n 'list_relations_without_caching macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.846957}, "macro.dbt.get_columns_in_relation": {"unique_id": "macro.dbt.get_columns_in_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "get_columns_in_relation", "macro_sql": "{% macro get_columns_in_relation(relation) -%}\n {{ return(adapter.dispatch('get_columns_in_relation', 'dbt')(relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_columns_in_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.851595}, "macro.dbt.default__get_columns_in_relation": {"unique_id": "macro.dbt.default__get_columns_in_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "default__get_columns_in_relation", "macro_sql": "{% macro default__get_columns_in_relation(relation) -%}\n {{ exceptions.raise_not_implemented(\n 'get_columns_in_relation macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.852285}, "macro.dbt.sql_convert_columns_in_relation": {"unique_id": "macro.dbt.sql_convert_columns_in_relation", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "sql_convert_columns_in_relation", "macro_sql": "{% macro sql_convert_columns_in_relation(table) -%}\n {% set columns = [] %}\n {% for row in table %}\n {% do columns.append(api.Column(*row)) %}\n {% endfor %}\n {{ return(columns) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.853091}, "macro.dbt.get_columns_in_query": {"unique_id": "macro.dbt.get_columns_in_query", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "get_columns_in_query", "macro_sql": "{% macro get_columns_in_query(select_sql) -%}\n {{ return(adapter.dispatch('get_columns_in_query', 'dbt')(select_sql)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_columns_in_query"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.853559}, "macro.dbt.default__get_columns_in_query": {"unique_id": "macro.dbt.default__get_columns_in_query", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "default__get_columns_in_query", "macro_sql": "{% macro default__get_columns_in_query(select_sql) %}\n {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%}\n select * from (\n {{ select_sql }}\n ) as __dbt_sbq\n where false\n limit 0\n {% endcall %}\n\n {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.854439}, "macro.dbt.alter_column_type": {"unique_id": "macro.dbt.alter_column_type", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "alter_column_type", "macro_sql": "{% macro alter_column_type(relation, column_name, new_column_type) -%}\n {{ return(adapter.dispatch('alter_column_type', 'dbt')(relation, column_name, new_column_type)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__alter_column_type"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.85503}, "macro.dbt.default__alter_column_type": {"unique_id": "macro.dbt.default__alter_column_type", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "default__alter_column_type", "macro_sql": "{% macro default__alter_column_type(relation, column_name, new_column_type) -%}\n {#\n 1. Create a new column (w/ temp name and correct type)\n 2. Copy data over to it\n 3. Drop the existing column (cascade!)\n 4. Rename the new column to existing column\n #}\n {%- set tmp_column = column_name + \"__dbt_alter\" -%}\n\n {% call statement('alter_column_type') %}\n alter table {{ relation }} add column {{ adapter.quote(tmp_column) }} {{ new_column_type }};\n update {{ relation }} set {{ adapter.quote(tmp_column) }} = {{ adapter.quote(column_name) }};\n alter table {{ relation }} drop column {{ adapter.quote(column_name) }} cascade;\n alter table {{ relation }} rename column {{ adapter.quote(tmp_column) }} to {{ adapter.quote(column_name) }}\n {% endcall %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.856618}, "macro.dbt.alter_relation_add_remove_columns": {"unique_id": "macro.dbt.alter_relation_add_remove_columns", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "alter_relation_add_remove_columns", "macro_sql": "{% macro alter_relation_add_remove_columns(relation, add_columns = none, remove_columns = none) -%}\n {{ return(adapter.dispatch('alter_relation_add_remove_columns', 'dbt')(relation, add_columns, remove_columns)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__alter_relation_add_remove_columns"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.8572822}, "macro.dbt.default__alter_relation_add_remove_columns": {"unique_id": "macro.dbt.default__alter_relation_add_remove_columns", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "default__alter_relation_add_remove_columns", "macro_sql": "{% macro default__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}\n\n {% if add_columns is none %}\n {% set add_columns = [] %}\n {% endif %}\n {% if remove_columns is none %}\n {% set remove_columns = [] %}\n {% endif %}\n\n {% set sql -%}\n\n alter {{ relation.type }} {{ relation }}\n\n {% for column in add_columns %}\n add column {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }}\n {% endfor %}{{ ',' if add_columns and remove_columns }}\n\n {% for column in remove_columns %}\n drop column {{ column.name }}{{ ',' if not loop.last }}\n {% endfor %}\n\n {%- endset -%}\n\n {% do run_query(sql) %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.8593512}, "macro.dbt.test_unique": {"unique_id": "macro.dbt.test_unique", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "name": "test_unique", "macro_sql": "{% test unique(model, column_name) %}\n {% set macro = adapter.dispatch('test_unique', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__test_unique"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.87712}, "macro.dbt.test_not_null": {"unique_id": "macro.dbt.test_not_null", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "name": "test_not_null", "macro_sql": "{% test not_null(model, column_name) %}\n {% set macro = adapter.dispatch('test_not_null', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__test_not_null"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.877723}, "macro.dbt.test_accepted_values": {"unique_id": "macro.dbt.test_accepted_values", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "name": "test_accepted_values", "macro_sql": "{% test accepted_values(model, column_name, values, quote=True) %}\n {% set macro = adapter.dispatch('test_accepted_values', 'dbt') %}\n {{ macro(model, column_name, values, quote) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__test_accepted_values"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.878474}, "macro.dbt.test_relationships": {"unique_id": "macro.dbt.test_relationships", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "name": "test_relationships", "macro_sql": "{% test relationships(model, column_name, to, field) %}\n {% set macro = adapter.dispatch('test_relationships', 'dbt') %}\n {{ macro(model, column_name, to, field) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__test_relationships"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1663058262.879189}}, "docs": {"dbt.__overview__": {"unique_id": "dbt.__overview__", "package_name": "dbt", "root_path": "/Users/jerco/dev/scratch/testy/env/lib/python3.9/site-packages/dbt/include/global_project", "path": "overview.md", "original_file_path": "docs/overview.md", "name": "__overview__", "block_contents": "### Welcome!\n\nWelcome to the auto-generated documentation for your dbt project!\n\n### Navigation\n\nYou can use the `Project` and `Database` navigation tabs on the left side of the window to explore the models\nin your project.\n\n#### Project Tab\nThe `Project` tab mirrors the directory structure of your dbt project. In this tab, you can see all of the\nmodels defined in your dbt project, as well as models imported from dbt packages.\n\n#### Database Tab\nThe `Database` tab also exposes your models, but in a format that looks more like a database explorer. This view\nshows relations (tables and views) grouped into database schemas. Note that ephemeral models are _not_ shown\nin this interface, as they do not exist in the database.\n\n### Graph Exploration\nYou can click the blue icon on the bottom-right corner of the page to view the lineage graph of your models.\n\nOn model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the `Expand`\nbutton at the top-right of this lineage pane, you'll be able to see all of the models that are used to build,\nor are built from, the model you're exploring.\n\nOnce expanded, you'll be able to use the `--select` and `--exclude` model selection syntax to filter the\nmodels in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax).\n\nNote that you can also right-click on models to interactively filter and explore the graph.\n\n---\n\n### More information\n\n- [What is dbt](https://docs.getdbt.com/docs/introduction)?\n- Read the [dbt viewpoint](https://docs.getdbt.com/docs/viewpoint)\n- [Installation](https://docs.getdbt.com/docs/installation)\n- Join the [dbt Community](https://www.getdbt.com/community/) for questions and discussion"}}, "exposures": {}, "metrics": {"metric.test.my_metric": {"fqn": ["test", "my_metric"], "unique_id": "metric.test.my_metric", "package_name": "test", "root_path": "/Users/jerco/dev/scratch/testy", "path": "metric.yml", "original_file_path": "models/metric.yml", "name": "my_metric", "description": "", "label": "Count records", "type": "count", "sql": "*", "timestamp": "updated_at", "filters": [], "time_grains": ["day"], "dimensions": [], "model": "ref('my_model')", "model_unique_id": null, "resource_type": "metric", "meta": {}, "tags": [], "sources": [], "depends_on": {"macros": [], "nodes": ["model.test.my_model"]}, "refs": [["my_model"]], "metrics": [], "created_at": 1663058517.2551522}}, "selectors": {}, "disabled": {}, "parent_map": {"model.test.my_model": [], "metric.test.my_metric": ["model.test.my_model"]}, "child_map": {"model.test.my_model": ["metric.test.my_metric"], "metric.test.my_metric": []}} ================================================ FILE: tests/functional/artifacts/data/state/v7/manifest.json ================================================ {"metadata": {"dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v7.json", "dbt_version": "1.3.2", "generated_at": "2023-02-13T21:34:36.870255Z", "invocation_id": "96c0aa43-0ccd-4420-a50c-05c0f22a0df1", "env": {}, "project_id": "098f6bcd4621d373cade4e832627b4f6", "user_id": null, "send_anonymous_usage_stats": false, "adapter_type": "postgres"}, "nodes": {"model.test.my_model": {"resource_type": "model", "depends_on": {"macros": [], "nodes": []}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "post-hook": [], "pre-hook": []}, "database": "dbt", "schema": "test16763240740000063267_test_previous_version_state", "fqn": ["test", "my_model"], "unique_id": "model.test.my_model", "raw_code": "select 1 as id", "language": "sql", "package_name": "test", "root_path": "/private/var/folders/qt/vw8wqdgx4w381wh14b9y25m40000gn/T/pytest-of-gerda/pytest-126/project0", "path": "my_model.sql", "original_file_path": "models/my_model.sql", "name": "my_model", "alias": "my_model", "checksum": {"name": "sha256", "checksum": "2b9123e04ab8bb798f7c565afdc3ee0e56fcd66b4bfbdb435b4891c878d947c5"}, "tags": [], "refs": [], "sources": [], "metrics": [], "description": "Example model", "columns": {"id": {"name": "id", "description": "", "meta": {}, "data_type": null, "quote": null, "tags": []}}, "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": "test://models/schema.yml", "compiled_path": null, "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1676324075.566336}, "snapshot.test.snapshot_seed": {"resource_type": "snapshot", "depends_on": {"macros": [], "nodes": ["seed.test.my_seed"]}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "snapshot", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": "id", "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "strategy": "check", "target_schema": "test16763240740000063267_test_previous_version_state", "target_database": null, "updated_at": null, "check_cols": "all", "post-hook": [], "pre-hook": []}, "database": "dbt", "schema": "test16763240740000063267_test_previous_version_state", "fqn": ["test", "snapshot_seed", "snapshot_seed"], "unique_id": "snapshot.test.snapshot_seed", "raw_code": "\n{{\n config(\n unique_key='id',\n strategy='check',\n check_cols='all',\n target_schema=schema,\n )\n}}\nselect * from {{ ref('my_seed') }}\n", "language": "sql", "package_name": "test", "root_path": "/private/var/folders/qt/vw8wqdgx4w381wh14b9y25m40000gn/T/pytest-of-gerda/pytest-126/project0", "path": "snapshot_seed.sql", "original_file_path": "snapshots/snapshot_seed.sql", "name": "snapshot_seed", "alias": "snapshot_seed", "checksum": {"name": "sha256", "checksum": "00c13c306831070996970605fbc4c901aa456e1ed1c028725a932e4e6a4ffb0a"}, "tags": [], "refs": [["my_seed"]], "sources": [], "metrics": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "compiled_path": null, "build_path": null, "deferred": false, "unrendered_config": {"unique_key": "id", "strategy": "check", "check_cols": "all", "target_schema": "test16763240740000063267_test_previous_version_state"}, "created_at": 1676324075.423856}, "analysis.test.a": {"resource_type": "analysis", "depends_on": {"macros": [], "nodes": []}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "post-hook": [], "pre-hook": []}, "database": "dbt", "schema": "test16763240740000063267_test_previous_version_state", "fqn": ["test", "analysis", "a"], "unique_id": "analysis.test.a", "raw_code": "select 4 as id", "language": "sql", "package_name": "test", "root_path": "/private/var/folders/qt/vw8wqdgx4w381wh14b9y25m40000gn/T/pytest-of-gerda/pytest-126/project0", "path": "analysis/a.sql", "original_file_path": "analyses/a.sql", "name": "a", "alias": "a", "checksum": {"name": "sha256", "checksum": "bd1ee600e4e80d03f488fee52a66e8d51b5be2b98acc20df1cf8be4670d86ae5"}, "tags": [], "refs": [], "sources": [], "metrics": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "compiled_path": null, "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1676324075.453177}, "test.test.just_my": {"resource_type": "test", "depends_on": {"macros": [], "nodes": ["model.test.my_model"]}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": ["data_test_tag"], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "database": "dbt", "schema": "test16763240740000063267_test_previous_version_state_dbt_test__audit", "fqn": ["test", "just_my"], "unique_id": "test.test.just_my", "raw_code": "{{ config(tags = ['data_test_tag']) }}\n\nselect * from {{ ref('my_model') }}\nwhere false", "language": "sql", "package_name": "test", "root_path": "/private/var/folders/qt/vw8wqdgx4w381wh14b9y25m40000gn/T/pytest-of-gerda/pytest-126/project0", "path": "just_my.sql", "original_file_path": "tests/just_my.sql", "name": "just_my", "alias": "just_my", "checksum": {"name": "sha256", "checksum": "f30b7a814e0e3761d1a8042aa40d658d6c33affb28cd92782b0f56559c414fd8"}, "tags": ["data_test_tag"], "refs": [["my_model"]], "sources": [], "metrics": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "compiled_path": null, "build_path": null, "deferred": false, "unrendered_config": {"tags": ["data_test_tag"]}, "created_at": 1676324075.520421}, "seed.test.my_seed": {"resource_type": "seed", "depends_on": {"macros": [], "nodes": []}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "seed", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "quote_columns": null, "post-hook": [], "pre-hook": []}, "database": "dbt", "schema": "test16763240740000063267_test_previous_version_state", "fqn": ["test", "my_seed"], "unique_id": "seed.test.my_seed", "raw_code": "", "language": "sql", "package_name": "test", "root_path": "/private/var/folders/qt/vw8wqdgx4w381wh14b9y25m40000gn/T/pytest-of-gerda/pytest-126/project0", "path": "my_seed.csv", "original_file_path": "seeds/my_seed.csv", "name": "my_seed", "alias": "my_seed", "checksum": {"name": "sha256", "checksum": "22697c9b76d73a6c7561554ddb2ce101428ea2737ba8dc500d52ebcfdcfcfc13"}, "tags": [], "refs": [], "sources": [], "metrics": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "compiled_path": null, "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1676324075.542836}, "test.test.not_null_my_model_id.43e0e9183a": {"test_metadata": {"name": "not_null", "kwargs": {"column_name": "id", "model": "{{ get_where_subquery(ref('my_model')) }}"}, "namespace": null}, "resource_type": "test", "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.test.my_model"]}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "database": "dbt", "schema": "test16763240740000063267_test_previous_version_state_dbt_test__audit", "fqn": ["test", "not_null_my_model_id"], "unique_id": "test.test.not_null_my_model_id.43e0e9183a", "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "package_name": "test", "root_path": "/private/var/folders/qt/vw8wqdgx4w381wh14b9y25m40000gn/T/pytest-of-gerda/pytest-126/project0", "path": "not_null_my_model_id.sql", "original_file_path": "models/schema.yml", "name": "not_null_my_model_id", "alias": "not_null_my_model_id", "checksum": {"name": "none", "checksum": ""}, "tags": [], "refs": [["my_model"]], "sources": [], "metrics": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "compiled_path": null, "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1676324075.575407, "column_name": "id", "file_key_name": "models.my_model"}, "test.test.check_nothing_my_model_.d5a5e66110": {"test_metadata": {"name": "check_nothing", "kwargs": {"model": "{{ get_where_subquery(ref('my_model')) }}"}, "namespace": null}, "resource_type": "test", "depends_on": {"macros": ["macro.test.test_check_nothing", "macro.dbt.get_where_subquery"], "nodes": ["model.test.my_model"]}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "database": "dbt", "schema": "test16763240740000063267_test_previous_version_state_dbt_test__audit", "fqn": ["test", "check_nothing_my_model_"], "unique_id": "test.test.check_nothing_my_model_.d5a5e66110", "raw_code": "{{ test_check_nothing(**_dbt_generic_test_kwargs) }}", "language": "sql", "package_name": "test", "root_path": "/private/var/folders/qt/vw8wqdgx4w381wh14b9y25m40000gn/T/pytest-of-gerda/pytest-126/project0", "path": "check_nothing_my_model_.sql", "original_file_path": "models/schema.yml", "name": "check_nothing_my_model_", "alias": "check_nothing_my_model_", "checksum": {"name": "none", "checksum": ""}, "tags": [], "refs": [["my_model"]], "sources": [], "metrics": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "compiled_path": null, "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1676324075.577614, "column_name": null, "file_key_name": "models.my_model"}}, "sources": {"source.test.my_source.my_table": {"fqn": ["test", "my_source", "my_table"], "database": "dbt", "schema": "my_source", "unique_id": "source.test.my_source.my_table", "package_name": "test", "root_path": "/private/var/folders/qt/vw8wqdgx4w381wh14b9y25m40000gn/T/pytest-of-gerda/pytest-126/project0", "path": "models/schema.yml", "original_file_path": "models/schema.yml", "name": "my_table", "source_name": "my_source", "source_description": "My source", "loader": "a_loader", "identifier": "my_seed", "resource_type": "source", "quoting": {"database": null, "schema": null, "identifier": null, "column": null}, "loaded_at_field": null, "freshness": {"warn_after": {"count": null, "period": null}, "error_after": {"count": null, "period": null}, "filter": null}, "external": null, "description": "My table", "columns": {}, "meta": {}, "source_meta": {}, "tags": [], "config": {"enabled": true}, "patch_path": null, "unrendered_config": {}, "relation_name": "\"dbt\".\"my_source\".\"my_seed\"", "created_at": 1676324075.624893}}, "macros": {"macro.test.test_check_nothing": {"unique_id": "macro.test.test_check_nothing", "package_name": "test", "root_path": "/private/var/folders/qt/vw8wqdgx4w381wh14b9y25m40000gn/T/pytest-of-gerda/pytest-126/project0", "path": "macros/dummy_test.sql", "original_file_path": "macros/dummy_test.sql", "name": "test_check_nothing", "macro_sql": "{% test check_nothing(model) %}\n-- a silly test to make sure that table-level tests show up in the manifest\n-- without a column_name field\n\nselect 0\n\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.712246, "supported_languages": null}, "macro.test.test_disabled_check_nothing": {"unique_id": "macro.test.test_disabled_check_nothing", "package_name": "test", "root_path": "/private/var/folders/qt/vw8wqdgx4w381wh14b9y25m40000gn/T/pytest-of-gerda/pytest-126/project0", "path": "macros/disabled_dummy_test.sql", "original_file_path": "macros/disabled_dummy_test.sql", "name": "test_disabled_check_nothing", "macro_sql": "{% test disabled_check_nothing(model) %}\n-- a silly test to make sure that table-level tests show up in the manifest\n-- without a column_name field\n\n{{ config(enabled=False) }}\nselect 0\n\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.713166, "supported_languages": null}, "macro.test.do_nothing": {"unique_id": "macro.test.do_nothing", "package_name": "test", "root_path": "/private/var/folders/qt/vw8wqdgx4w381wh14b9y25m40000gn/T/pytest-of-gerda/pytest-126/project0", "path": "macros/do_nothing.sql", "original_file_path": "macros/do_nothing.sql", "name": "do_nothing", "macro_sql": "{% macro do_nothing(foo2, bar2) %}\n select\n '{{ foo2 }}' as foo2,\n '{{ bar2 }}' as bar2\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.7140381, "supported_languages": null}, "macro.dbt_postgres.postgres__current_timestamp": {"unique_id": "macro.dbt_postgres.postgres__current_timestamp", "package_name": "dbt_postgres", "root_path": "/Users/gerda/FTA/dbt/plugins/postgres/dbt/include/postgres", "path": "macros/timestamps.sql", "original_file_path": "macros/timestamps.sql", "name": "postgres__current_timestamp", "macro_sql": "{% macro postgres__current_timestamp() -%}\n now()\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.714999, "supported_languages": null}, "macro.dbt_postgres.postgres__snapshot_string_as_time": {"unique_id": "macro.dbt_postgres.postgres__snapshot_string_as_time", "package_name": "dbt_postgres", "root_path": "/Users/gerda/FTA/dbt/plugins/postgres/dbt/include/postgres", "path": "macros/timestamps.sql", "original_file_path": "macros/timestamps.sql", "name": "postgres__snapshot_string_as_time", "macro_sql": "{% macro postgres__snapshot_string_as_time(timestamp) -%}\n {%- set result = \"'\" ~ timestamp ~ \"'::timestamp without time zone\" -%}\n {{ return(result) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.715521, "supported_languages": null}, "macro.dbt_postgres.postgres__snapshot_get_time": {"unique_id": "macro.dbt_postgres.postgres__snapshot_get_time", "package_name": "dbt_postgres", "root_path": "/Users/gerda/FTA/dbt/plugins/postgres/dbt/include/postgres", "path": "macros/timestamps.sql", "original_file_path": "macros/timestamps.sql", "name": "postgres__snapshot_get_time", "macro_sql": "{% macro postgres__snapshot_get_time() -%}\n {{ current_timestamp() }}::timestamp without time zone\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.7158241, "supported_languages": null}, "macro.dbt_postgres.postgres__current_timestamp_backcompat": {"unique_id": "macro.dbt_postgres.postgres__current_timestamp_backcompat", "package_name": "dbt_postgres", "root_path": "/Users/gerda/FTA/dbt/plugins/postgres/dbt/include/postgres", "path": "macros/timestamps.sql", "original_file_path": "macros/timestamps.sql", "name": "postgres__current_timestamp_backcompat", "macro_sql": "{% macro postgres__current_timestamp_backcompat() %}\n current_timestamp::{{ type_timestamp() }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.type_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.716109, "supported_languages": null}, "macro.dbt_postgres.postgres__current_timestamp_in_utc_backcompat": {"unique_id": "macro.dbt_postgres.postgres__current_timestamp_in_utc_backcompat", "package_name": "dbt_postgres", "root_path": "/Users/gerda/FTA/dbt/plugins/postgres/dbt/include/postgres", "path": "macros/timestamps.sql", "original_file_path": "macros/timestamps.sql", "name": "postgres__current_timestamp_in_utc_backcompat", "macro_sql": "{% macro postgres__current_timestamp_in_utc_backcompat() %}\n (current_timestamp at time zone 'utc')::{{ type_timestamp() }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.type_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.7164018, "supported_languages": null}, "macro.dbt_postgres.postgres__get_catalog": {"unique_id": "macro.dbt_postgres.postgres__get_catalog", "package_name": "dbt_postgres", "root_path": "/Users/gerda/FTA/dbt/plugins/postgres/dbt/include/postgres", "path": "macros/catalog.sql", "original_file_path": "macros/catalog.sql", "name": "postgres__get_catalog", "macro_sql": "{% macro postgres__get_catalog(information_schema, schemas) -%}\n\n {%- call statement('catalog', fetch_result=True) -%}\n {#\n If the user has multiple databases set and the first one is wrong, this will fail.\n But we won't fail in the case where there are multiple quoting-difference-only dbs, which is better.\n #}\n {% set database = information_schema.database %}\n {{ adapter.verify_database(database) }}\n\n select\n '{{ database }}' as table_database,\n sch.nspname as table_schema,\n tbl.relname as table_name,\n case tbl.relkind\n when 'v' then 'VIEW'\n else 'BASE TABLE'\n end as table_type,\n tbl_desc.description as table_comment,\n col.attname as column_name,\n col.attnum as column_index,\n pg_catalog.format_type(col.atttypid, col.atttypmod) as column_type,\n col_desc.description as column_comment,\n pg_get_userbyid(tbl.relowner) as table_owner\n\n from pg_catalog.pg_namespace sch\n join pg_catalog.pg_class tbl on tbl.relnamespace = sch.oid\n join pg_catalog.pg_attribute col on col.attrelid = tbl.oid\n left outer join pg_catalog.pg_description tbl_desc on (tbl_desc.objoid = tbl.oid and tbl_desc.objsubid = 0)\n left outer join pg_catalog.pg_description col_desc on (col_desc.objoid = tbl.oid and col_desc.objsubid = col.attnum)\n\n where (\n {%- for schema in schemas -%}\n upper(sch.nspname) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n )\n and not pg_is_other_temp_schema(sch.oid) -- not a temporary schema belonging to another session\n and tbl.relpersistence in ('p', 'u') -- [p]ermanent table or [u]nlogged table. Exclude [t]emporary tables\n and tbl.relkind in ('r', 'v', 'f', 'p') -- o[r]dinary table, [v]iew, [f]oreign table, [p]artitioned table. Other values are [i]ndex, [S]equence, [c]omposite type, [t]OAST table, [m]aterialized view\n and col.attnum > 0 -- negative numbers are used for system columns such as oid\n and not col.attisdropped -- column as not been dropped\n\n order by\n sch.nspname,\n tbl.relname,\n col.attnum\n\n {%- endcall -%}\n\n {{ return(load_result('catalog').table) }}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.719163, "supported_languages": null}, "macro.dbt_postgres.postgres_get_relations": {"unique_id": "macro.dbt_postgres.postgres_get_relations", "package_name": "dbt_postgres", "root_path": "/Users/gerda/FTA/dbt/plugins/postgres/dbt/include/postgres", "path": "macros/relations.sql", "original_file_path": "macros/relations.sql", "name": "postgres_get_relations", "macro_sql": "{% macro postgres_get_relations () -%}\n\n {#\n -- in pg_depend, objid is the dependent, refobjid is the referenced object\n -- > a pg_depend entry indicates that the referenced object cannot be\n -- > dropped without also dropping the dependent object.\n #}\n\n {%- call statement('relations', fetch_result=True) -%}\n with relation as (\n select\n pg_rewrite.ev_class as class,\n pg_rewrite.oid as id\n from pg_rewrite\n ),\n class as (\n select\n oid as id,\n relname as name,\n relnamespace as schema,\n relkind as kind\n from pg_class\n ),\n dependency as (\n select distinct\n pg_depend.objid as id,\n pg_depend.refobjid as ref\n from pg_depend\n ),\n schema as (\n select\n pg_namespace.oid as id,\n pg_namespace.nspname as name\n from pg_namespace\n where nspname != 'information_schema' and nspname not like 'pg\\_%'\n ),\n referenced as (\n select\n relation.id AS id,\n referenced_class.name ,\n referenced_class.schema ,\n referenced_class.kind\n from relation\n join class as referenced_class on relation.class=referenced_class.id\n where referenced_class.kind in ('r', 'v')\n ),\n relationships as (\n select\n referenced.name as referenced_name,\n referenced.schema as referenced_schema_id,\n dependent_class.name as dependent_name,\n dependent_class.schema as dependent_schema_id,\n referenced.kind as kind\n from referenced\n join dependency on referenced.id=dependency.id\n join class as dependent_class on dependency.ref=dependent_class.id\n where\n (referenced.name != dependent_class.name or\n referenced.schema != dependent_class.schema)\n )\n\n select\n referenced_schema.name as referenced_schema,\n relationships.referenced_name as referenced_name,\n dependent_schema.name as dependent_schema,\n relationships.dependent_name as dependent_name\n from relationships\n join schema as dependent_schema on relationships.dependent_schema_id=dependent_schema.id\n join schema as referenced_schema on relationships.referenced_schema_id=referenced_schema.id\n group by referenced_schema, referenced_name, dependent_schema, dependent_name\n order by referenced_schema, referenced_name, dependent_schema, dependent_name;\n\n {%- endcall -%}\n\n {{ return(load_result('relations').table) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.7208161, "supported_languages": null}, "macro.dbt_postgres.postgres__create_table_as": {"unique_id": "macro.dbt_postgres.postgres__create_table_as", "package_name": "dbt_postgres", "root_path": "/Users/gerda/FTA/dbt/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__create_table_as", "macro_sql": "{% macro postgres__create_table_as(temporary, relation, sql) -%}\n {%- set unlogged = config.get('unlogged', default=false) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary -%}\n temporary\n {%- elif unlogged -%}\n unlogged\n {%- endif %} table {{ relation }}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.733047, "supported_languages": null}, "macro.dbt_postgres.postgres__get_create_index_sql": {"unique_id": "macro.dbt_postgres.postgres__get_create_index_sql", "package_name": "dbt_postgres", "root_path": "/Users/gerda/FTA/dbt/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__get_create_index_sql", "macro_sql": "{% macro postgres__get_create_index_sql(relation, index_dict) -%}\n {%- set index_config = adapter.parse_index(index_dict) -%}\n {%- set comma_separated_columns = \", \".join(index_config.columns) -%}\n {%- set index_name = index_config.render(relation) -%}\n\n create {% if index_config.unique -%}\n unique\n {%- endif %} index if not exists\n \"{{ index_name }}\"\n on {{ relation }} {% if index_config.type -%}\n using {{ index_config.type }}\n {%- endif %}\n ({{ comma_separated_columns }});\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.734891, "supported_languages": null}, "macro.dbt_postgres.postgres__create_schema": {"unique_id": "macro.dbt_postgres.postgres__create_schema", "package_name": "dbt_postgres", "root_path": "/Users/gerda/FTA/dbt/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__create_schema", "macro_sql": "{% macro postgres__create_schema(relation) -%}\n {% if relation.database -%}\n {{ adapter.verify_database(relation.database) }}\n {%- endif -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier().include(database=False) }}\n {%- endcall -%}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.735866, "supported_languages": null}, "macro.dbt_postgres.postgres__drop_schema": {"unique_id": "macro.dbt_postgres.postgres__drop_schema", "package_name": "dbt_postgres", "root_path": "/Users/gerda/FTA/dbt/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__drop_schema", "macro_sql": "{% macro postgres__drop_schema(relation) -%}\n {% if relation.database -%}\n {{ adapter.verify_database(relation.database) }}\n {%- endif -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier().include(database=False) }} cascade\n {%- endcall -%}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.736712, "supported_languages": null}, "macro.dbt_postgres.postgres__get_columns_in_relation": {"unique_id": "macro.dbt_postgres.postgres__get_columns_in_relation", "package_name": "dbt_postgres", "root_path": "/Users/gerda/FTA/dbt/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__get_columns_in_relation", "macro_sql": "{% macro postgres__get_columns_in_relation(relation) -%}\n {% call statement('get_columns_in_relation', fetch_result=True) %}\n select\n column_name,\n data_type,\n character_maximum_length,\n numeric_precision,\n numeric_scale\n\n from {{ relation.information_schema('columns') }}\n where table_name = '{{ relation.identifier }}'\n {% if relation.schema %}\n and table_schema = '{{ relation.schema }}'\n {% endif %}\n order by ordinal_position\n\n {% endcall %}\n {% set table = load_result('get_columns_in_relation').table %}\n {{ return(sql_convert_columns_in_relation(table)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.sql_convert_columns_in_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.738183, "supported_languages": null}, "macro.dbt_postgres.postgres__list_relations_without_caching": {"unique_id": "macro.dbt_postgres.postgres__list_relations_without_caching", "package_name": "dbt_postgres", "root_path": "/Users/gerda/FTA/dbt/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__list_relations_without_caching", "macro_sql": "{% macro postgres__list_relations_without_caching(schema_relation) %}\n {% call statement('list_relations_without_caching', fetch_result=True) -%}\n select\n '{{ schema_relation.database }}' as database,\n tablename as name,\n schemaname as schema,\n 'table' as type\n from pg_tables\n where schemaname ilike '{{ schema_relation.schema }}'\n union all\n select\n '{{ schema_relation.database }}' as database,\n viewname as name,\n schemaname as schema,\n 'view' as type\n from pg_views\n where schemaname ilike '{{ schema_relation.schema }}'\n {% endcall %}\n {{ return(load_result('list_relations_without_caching').table) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.739969, "supported_languages": null}, "macro.dbt_postgres.postgres__information_schema_name": {"unique_id": "macro.dbt_postgres.postgres__information_schema_name", "package_name": "dbt_postgres", "root_path": "/Users/gerda/FTA/dbt/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__information_schema_name", "macro_sql": "{% macro postgres__information_schema_name(database) -%}\n {% if database_name -%}\n {{ adapter.verify_database(database_name) }}\n {%- endif -%}\n information_schema\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.740651, "supported_languages": null}, "macro.dbt_postgres.postgres__list_schemas": {"unique_id": "macro.dbt_postgres.postgres__list_schemas", "package_name": "dbt_postgres", "root_path": "/Users/gerda/FTA/dbt/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__list_schemas", "macro_sql": "{% macro postgres__list_schemas(database) %}\n {% if database -%}\n {{ adapter.verify_database(database) }}\n {%- endif -%}\n {% call statement('list_schemas', fetch_result=True, auto_begin=False) %}\n select distinct nspname from pg_namespace\n {% endcall %}\n {{ return(load_result('list_schemas').table) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.741567, "supported_languages": null}, "macro.dbt_postgres.postgres__check_schema_exists": {"unique_id": "macro.dbt_postgres.postgres__check_schema_exists", "package_name": "dbt_postgres", "root_path": "/Users/gerda/FTA/dbt/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__check_schema_exists", "macro_sql": "{% macro postgres__check_schema_exists(information_schema, schema) -%}\n {% if information_schema.database -%}\n {{ adapter.verify_database(information_schema.database) }}\n {%- endif -%}\n {% call statement('check_schema_exists', fetch_result=True, auto_begin=False) %}\n select count(*) from pg_namespace where nspname = '{{ schema }}'\n {% endcall %}\n {{ return(load_result('check_schema_exists').table) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.742538, "supported_languages": null}, "macro.dbt_postgres.postgres__make_relation_with_suffix": {"unique_id": "macro.dbt_postgres.postgres__make_relation_with_suffix", "package_name": "dbt_postgres", "root_path": "/Users/gerda/FTA/dbt/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__make_relation_with_suffix", "macro_sql": "{% macro postgres__make_relation_with_suffix(base_relation, suffix, dstring) %}\n {% if dstring %}\n {% set dt = modules.datetime.datetime.now() %}\n {% set dtstring = dt.strftime(\"%H%M%S%f\") %}\n {% set suffix = suffix ~ dtstring %}\n {% endif %}\n {% set suffix_length = suffix|length %}\n {% set relation_max_name_length = base_relation.relation_max_name_length() %}\n {% if suffix_length > relation_max_name_length %}\n {% do exceptions.raise_compiler_error('Relation suffix is too long (' ~ suffix_length ~ ' characters). Maximum length is ' ~ relation_max_name_length ~ ' characters.') %}\n {% endif %}\n {% set identifier = base_relation.identifier[:relation_max_name_length - suffix_length] ~ suffix %}\n\n {{ return(base_relation.incorporate(path={\"identifier\": identifier })) }}\n\n {% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.744768, "supported_languages": null}, "macro.dbt_postgres.postgres__make_intermediate_relation": {"unique_id": "macro.dbt_postgres.postgres__make_intermediate_relation", "package_name": "dbt_postgres", "root_path": "/Users/gerda/FTA/dbt/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__make_intermediate_relation", "macro_sql": "{% macro postgres__make_intermediate_relation(base_relation, suffix) %}\n {{ return(postgres__make_relation_with_suffix(base_relation, suffix, dstring=False)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_relation_with_suffix"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.745294, "supported_languages": null}, "macro.dbt_postgres.postgres__make_temp_relation": {"unique_id": "macro.dbt_postgres.postgres__make_temp_relation", "package_name": "dbt_postgres", "root_path": "/Users/gerda/FTA/dbt/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__make_temp_relation", "macro_sql": "{% macro postgres__make_temp_relation(base_relation, suffix) %}\n {% set temp_relation = postgres__make_relation_with_suffix(base_relation, suffix, dstring=True) %}\n {{ return(temp_relation.incorporate(path={\"schema\": none,\n \"database\": none})) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_relation_with_suffix"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.746094, "supported_languages": null}, "macro.dbt_postgres.postgres__make_backup_relation": {"unique_id": "macro.dbt_postgres.postgres__make_backup_relation", "package_name": "dbt_postgres", "root_path": "/Users/gerda/FTA/dbt/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__make_backup_relation", "macro_sql": "{% macro postgres__make_backup_relation(base_relation, backup_relation_type, suffix) %}\n {% set backup_relation = postgres__make_relation_with_suffix(base_relation, suffix, dstring=False) %}\n {{ return(backup_relation.incorporate(type=backup_relation_type)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_relation_with_suffix"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.746795, "supported_languages": null}, "macro.dbt_postgres.postgres_escape_comment": {"unique_id": "macro.dbt_postgres.postgres_escape_comment", "package_name": "dbt_postgres", "root_path": "/Users/gerda/FTA/dbt/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres_escape_comment", "macro_sql": "{% macro postgres_escape_comment(comment) -%}\n {% if comment is not string %}\n {% do exceptions.raise_compiler_error('cannot escape a non-string: ' ~ comment) %}\n {% endif %}\n {%- set magic = '$dbt_comment_literal_block$' -%}\n {%- if magic in comment -%}\n {%- do exceptions.raise_compiler_error('The string ' ~ magic ~ ' is not allowed in comments.') -%}\n {%- endif -%}\n {{ magic }}{{ comment }}{{ magic }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.747864, "supported_languages": null}, "macro.dbt_postgres.postgres__alter_relation_comment": {"unique_id": "macro.dbt_postgres.postgres__alter_relation_comment", "package_name": "dbt_postgres", "root_path": "/Users/gerda/FTA/dbt/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__alter_relation_comment", "macro_sql": "{% macro postgres__alter_relation_comment(relation, comment) %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n comment on {{ relation.type }} {{ relation }} is {{ escaped_comment }};\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres_escape_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.7484329, "supported_languages": null}, "macro.dbt_postgres.postgres__alter_column_comment": {"unique_id": "macro.dbt_postgres.postgres__alter_column_comment", "package_name": "dbt_postgres", "root_path": "/Users/gerda/FTA/dbt/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__alter_column_comment", "macro_sql": "{% macro postgres__alter_column_comment(relation, column_dict) %}\n {% set existing_columns = adapter.get_columns_in_relation(relation) | map(attribute=\"name\") | list %}\n {% for column_name in column_dict if (column_name in existing_columns) %}\n {% set comment = column_dict[column_name]['description'] %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n comment on column {{ relation }}.{{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }} is {{ escaped_comment }};\n {% endfor %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres_escape_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.7500181, "supported_languages": null}, "macro.dbt_postgres.postgres__get_show_grant_sql": {"unique_id": "macro.dbt_postgres.postgres__get_show_grant_sql", "package_name": "dbt_postgres", "root_path": "/Users/gerda/FTA/dbt/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__get_show_grant_sql", "macro_sql": "\n\n{%- macro postgres__get_show_grant_sql(relation) -%}\n select grantee, privilege_type\n from {{ relation.information_schema('role_table_grants') }}\n where grantor = current_role\n and grantee != current_role\n and table_schema = '{{ relation.schema }}'\n and table_name = '{{ relation.identifier }}'\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.750537, "supported_languages": null}, "macro.dbt_postgres.postgres__copy_grants": {"unique_id": "macro.dbt_postgres.postgres__copy_grants", "package_name": "dbt_postgres", "root_path": "/Users/gerda/FTA/dbt/plugins/postgres/dbt/include/postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "postgres__copy_grants", "macro_sql": "{% macro postgres__copy_grants() %}\n {{ return(False) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.750845, "supported_languages": null}, "macro.dbt_postgres.postgres__get_incremental_default_sql": {"unique_id": "macro.dbt_postgres.postgres__get_incremental_default_sql", "package_name": "dbt_postgres", "root_path": "/Users/gerda/FTA/dbt/plugins/postgres/dbt/include/postgres", "path": "macros/materializations/incremental_strategies.sql", "original_file_path": "macros/materializations/incremental_strategies.sql", "name": "postgres__get_incremental_default_sql", "macro_sql": "{% macro postgres__get_incremental_default_sql(arg_dict) %}\n\n {% if arg_dict[\"unique_key\"] %}\n {% do return(get_incremental_delete_insert_sql(arg_dict)) %}\n {% else %}\n {% do return(get_incremental_append_sql(arg_dict)) %}\n {% endif %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_incremental_delete_insert_sql", "macro.dbt.get_incremental_append_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.752257, "supported_languages": null}, "macro.dbt_postgres.postgres__snapshot_merge_sql": {"unique_id": "macro.dbt_postgres.postgres__snapshot_merge_sql", "package_name": "dbt_postgres", "root_path": "/Users/gerda/FTA/dbt/plugins/postgres/dbt/include/postgres", "path": "macros/materializations/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshot_merge.sql", "name": "postgres__snapshot_merge_sql", "macro_sql": "{% macro postgres__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n update {{ target }}\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n from {{ source }} as DBT_INTERNAL_SOURCE\n where DBT_INTERNAL_SOURCE.dbt_scd_id::text = {{ target }}.dbt_scd_id::text\n and DBT_INTERNAL_SOURCE.dbt_change_type::text in ('update'::text, 'delete'::text)\n and {{ target }}.dbt_valid_to is null;\n\n insert into {{ target }} ({{ insert_cols_csv }})\n select {% for column in insert_cols -%}\n DBT_INTERNAL_SOURCE.{{ column }} {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n from {{ source }} as DBT_INTERNAL_SOURCE\n where DBT_INTERNAL_SOURCE.dbt_change_type::text = 'insert'::text;\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.754163, "supported_languages": null}, "macro.dbt_postgres.postgres__dateadd": {"unique_id": "macro.dbt_postgres.postgres__dateadd", "package_name": "dbt_postgres", "root_path": "/Users/gerda/FTA/dbt/plugins/postgres/dbt/include/postgres", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "name": "postgres__dateadd", "macro_sql": "{% macro postgres__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n {{ from_date_or_timestamp }} + ((interval '1 {{ datepart }}') * ({{ interval }}))\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.755116, "supported_languages": null}, "macro.dbt_postgres.postgres__listagg": {"unique_id": "macro.dbt_postgres.postgres__listagg", "package_name": "dbt_postgres", "root_path": "/Users/gerda/FTA/dbt/plugins/postgres/dbt/include/postgres", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "name": "postgres__listagg", "macro_sql": "{% macro postgres__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n {% if limit_num -%}\n array_to_string(\n (array_agg(\n {{ measure }}\n {% if order_by_clause -%}\n {{ order_by_clause }}\n {%- endif %}\n ))[1:{{ limit_num }}],\n {{ delimiter_text }}\n )\n {%- else %}\n string_agg(\n {{ measure }},\n {{ delimiter_text }}\n {% if order_by_clause -%}\n {{ order_by_clause }}\n {%- endif %}\n )\n {%- endif %}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.757173, "supported_languages": null}, "macro.dbt_postgres.postgres__datediff": {"unique_id": "macro.dbt_postgres.postgres__datediff", "package_name": "dbt_postgres", "root_path": "/Users/gerda/FTA/dbt/plugins/postgres/dbt/include/postgres", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "name": "postgres__datediff", "macro_sql": "{% macro postgres__datediff(first_date, second_date, datepart) -%}\n\n {% if datepart == 'year' %}\n (date_part('year', ({{second_date}})::date) - date_part('year', ({{first_date}})::date))\n {% elif datepart == 'quarter' %}\n ({{ datediff(first_date, second_date, 'year') }} * 4 + date_part('quarter', ({{second_date}})::date) - date_part('quarter', ({{first_date}})::date))\n {% elif datepart == 'month' %}\n ({{ datediff(first_date, second_date, 'year') }} * 12 + date_part('month', ({{second_date}})::date) - date_part('month', ({{first_date}})::date))\n {% elif datepart == 'day' %}\n (({{second_date}})::date - ({{first_date}})::date)\n {% elif datepart == 'week' %}\n ({{ datediff(first_date, second_date, 'day') }} / 7 + case\n when date_part('dow', ({{first_date}})::timestamp) <= date_part('dow', ({{second_date}})::timestamp) then\n case when {{first_date}} <= {{second_date}} then 0 else -1 end\n else\n case when {{first_date}} <= {{second_date}} then 1 else 0 end\n end)\n {% elif datepart == 'hour' %}\n ({{ datediff(first_date, second_date, 'day') }} * 24 + date_part('hour', ({{second_date}})::timestamp) - date_part('hour', ({{first_date}})::timestamp))\n {% elif datepart == 'minute' %}\n ({{ datediff(first_date, second_date, 'hour') }} * 60 + date_part('minute', ({{second_date}})::timestamp) - date_part('minute', ({{first_date}})::timestamp))\n {% elif datepart == 'second' %}\n ({{ datediff(first_date, second_date, 'minute') }} * 60 + floor(date_part('second', ({{second_date}})::timestamp)) - floor(date_part('second', ({{first_date}})::timestamp)))\n {% elif datepart == 'millisecond' %}\n ({{ datediff(first_date, second_date, 'minute') }} * 60000 + floor(date_part('millisecond', ({{second_date}})::timestamp)) - floor(date_part('millisecond', ({{first_date}})::timestamp)))\n {% elif datepart == 'microsecond' %}\n ({{ datediff(first_date, second_date, 'minute') }} * 60000000 + floor(date_part('microsecond', ({{second_date}})::timestamp)) - floor(date_part('microsecond', ({{first_date}})::timestamp)))\n {% else %}\n {{ exceptions.raise_compiler_error(\"Unsupported datepart for macro datediff in postgres: {!r}\".format(datepart)) }}\n {% endif %}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.datediff"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.765253, "supported_languages": null}, "macro.dbt_postgres.postgres__any_value": {"unique_id": "macro.dbt_postgres.postgres__any_value", "package_name": "dbt_postgres", "root_path": "/Users/gerda/FTA/dbt/plugins/postgres/dbt/include/postgres", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "name": "postgres__any_value", "macro_sql": "{% macro postgres__any_value(expression) -%}\n\n min({{ expression }})\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.766171, "supported_languages": null}, "macro.dbt_postgres.postgres__last_day": {"unique_id": "macro.dbt_postgres.postgres__last_day", "package_name": "dbt_postgres", "root_path": "/Users/gerda/FTA/dbt/plugins/postgres/dbt/include/postgres", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "name": "postgres__last_day", "macro_sql": "{% macro postgres__last_day(date, datepart) -%}\n\n {%- if datepart == 'quarter' -%}\n -- postgres dateadd does not support quarter interval.\n cast(\n {{dbt.dateadd('day', '-1',\n dbt.dateadd('month', '3', dbt.date_trunc(datepart, date))\n )}}\n as date)\n {%- else -%}\n {{dbt.default_last_day(date, datepart)}}\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.dateadd", "macro.dbt.date_trunc", "macro.dbt.default_last_day"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.767721, "supported_languages": null}, "macro.dbt_postgres.postgres__split_part": {"unique_id": "macro.dbt_postgres.postgres__split_part", "package_name": "dbt_postgres", "root_path": "/Users/gerda/FTA/dbt/plugins/postgres/dbt/include/postgres", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "name": "postgres__split_part", "macro_sql": "{% macro postgres__split_part(string_text, delimiter_text, part_number) %}\n\n {% if part_number >= 0 %}\n {{ dbt.default__split_part(string_text, delimiter_text, part_number) }}\n {% else %}\n {{ dbt._split_part_negative(string_text, delimiter_text, part_number) }}\n {% endif %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__split_part", "macro.dbt._split_part_negative"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.7690408, "supported_languages": null}, "macro.dbt.run_hooks": {"unique_id": "macro.dbt.run_hooks", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "name": "run_hooks", "macro_sql": "{% macro run_hooks(hooks, inside_transaction=True) %}\n {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %}\n {% if not inside_transaction and loop.first %}\n {% call statement(auto_begin=inside_transaction) %}\n commit;\n {% endcall %}\n {% endif %}\n {% set rendered = render(hook.get('sql')) | trim %}\n {% if (rendered | length) > 0 %}\n {% call statement(auto_begin=inside_transaction) %}\n {{ rendered }}\n {% endcall %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.7719731, "supported_languages": null}, "macro.dbt.make_hook_config": {"unique_id": "macro.dbt.make_hook_config", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "name": "make_hook_config", "macro_sql": "{% macro make_hook_config(sql, inside_transaction) %}\n {{ tojson({\"sql\": sql, \"transaction\": inside_transaction}) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.772469, "supported_languages": null}, "macro.dbt.before_begin": {"unique_id": "macro.dbt.before_begin", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "name": "before_begin", "macro_sql": "{% macro before_begin(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.7729542, "supported_languages": null}, "macro.dbt.in_transaction": {"unique_id": "macro.dbt.in_transaction", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "name": "in_transaction", "macro_sql": "{% macro in_transaction(sql) %}\n {{ make_hook_config(sql, inside_transaction=True) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.773334, "supported_languages": null}, "macro.dbt.after_commit": {"unique_id": "macro.dbt.after_commit", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "name": "after_commit", "macro_sql": "{% macro after_commit(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.773783, "supported_languages": null}, "macro.dbt.set_sql_header": {"unique_id": "macro.dbt.set_sql_header", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "name": "set_sql_header", "macro_sql": "{% macro set_sql_header(config) -%}\n {{ config.set('sql_header', caller()) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.7750452, "supported_languages": null}, "macro.dbt.should_full_refresh": {"unique_id": "macro.dbt.should_full_refresh", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "name": "should_full_refresh", "macro_sql": "{% macro should_full_refresh() %}\n {% set config_full_refresh = config.get('full_refresh') %}\n {% if config_full_refresh is none %}\n {% set config_full_refresh = flags.FULL_REFRESH %}\n {% endif %}\n {% do return(config_full_refresh) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.775828, "supported_languages": null}, "macro.dbt.should_store_failures": {"unique_id": "macro.dbt.should_store_failures", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "name": "should_store_failures", "macro_sql": "{% macro should_store_failures() %}\n {% set config_store_failures = config.get('store_failures') %}\n {% if config_store_failures is none %}\n {% set config_store_failures = flags.STORE_FAILURES %}\n {% endif %}\n {% do return(config_store_failures) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.776683, "supported_languages": null}, "macro.dbt.snapshot_merge_sql": {"unique_id": "macro.dbt.snapshot_merge_sql", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/snapshots/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", "name": "snapshot_merge_sql", "macro_sql": "{% macro snapshot_merge_sql(target, source, insert_cols) -%}\n {{ adapter.dispatch('snapshot_merge_sql', 'dbt')(target, source, insert_cols) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__snapshot_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.778024, "supported_languages": null}, "macro.dbt.default__snapshot_merge_sql": {"unique_id": "macro.dbt.default__snapshot_merge_sql", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/snapshots/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", "name": "default__snapshot_merge_sql", "macro_sql": "{% macro default__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on DBT_INTERNAL_SOURCE.dbt_scd_id = DBT_INTERNAL_DEST.dbt_scd_id\n\n when matched\n and DBT_INTERNAL_DEST.dbt_valid_to is null\n and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')\n then update\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n\n when not matched\n and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'\n then insert ({{ insert_cols_csv }})\n values ({{ insert_cols_csv }})\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.778701, "supported_languages": null}, "macro.dbt.strategy_dispatch": {"unique_id": "macro.dbt.strategy_dispatch", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "strategy_dispatch", "macro_sql": "{% macro strategy_dispatch(name) -%}\n{% set original_name = name %}\n {% if '.' in name %}\n {% set package_name, name = name.split(\".\", 1) %}\n {% else %}\n {% set package_name = none %}\n {% endif %}\n\n {% if package_name is none %}\n {% set package_context = context %}\n {% elif package_name in context %}\n {% set package_context = context[package_name] %}\n {% else %}\n {% set error_msg %}\n Could not find package '{{package_name}}', called with '{{original_name}}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n\n {%- set search_name = 'snapshot_' ~ name ~ '_strategy' -%}\n\n {% if search_name not in package_context %}\n {% set error_msg %}\n The specified strategy macro '{{name}}' was not found in package '{{ package_name }}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n {{ return(package_context[search_name]) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.786231, "supported_languages": null}, "macro.dbt.snapshot_hash_arguments": {"unique_id": "macro.dbt.snapshot_hash_arguments", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "snapshot_hash_arguments", "macro_sql": "{% macro snapshot_hash_arguments(args) -%}\n {{ adapter.dispatch('snapshot_hash_arguments', 'dbt')(args) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.78667, "supported_languages": null}, "macro.dbt.default__snapshot_hash_arguments": {"unique_id": "macro.dbt.default__snapshot_hash_arguments", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "default__snapshot_hash_arguments", "macro_sql": "{% macro default__snapshot_hash_arguments(args) -%}\n md5({%- for arg in args -%}\n coalesce(cast({{ arg }} as varchar ), '')\n {% if not loop.last %} || '|' || {% endif %}\n {%- endfor -%})\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.787231, "supported_languages": null}, "macro.dbt.snapshot_timestamp_strategy": {"unique_id": "macro.dbt.snapshot_timestamp_strategy", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "snapshot_timestamp_strategy", "macro_sql": "{% macro snapshot_timestamp_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}\n {% set primary_key = config['unique_key'] %}\n {% set updated_at = config['updated_at'] %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %}\n\n {#/*\n The snapshot relation might not have an {{ updated_at }} value if the\n snapshot strategy is changed from `check` to `timestamp`. We\n should use a dbt-created column for the comparison in the snapshot\n table instead of assuming that the user-supplied {{ updated_at }}\n will be present in the historical data.\n\n See https://github.com/dbt-labs/dbt-core/issues/2350\n */ #}\n {% set row_changed_expr -%}\n ({{ snapshotted_rel }}.dbt_valid_from < {{ current_rel }}.{{ updated_at }})\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.788982, "supported_languages": null}, "macro.dbt.snapshot_string_as_time": {"unique_id": "macro.dbt.snapshot_string_as_time", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "snapshot_string_as_time", "macro_sql": "{% macro snapshot_string_as_time(timestamp) -%}\n {{ adapter.dispatch('snapshot_string_as_time', 'dbt')(timestamp) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__snapshot_string_as_time"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.789396, "supported_languages": null}, "macro.dbt.default__snapshot_string_as_time": {"unique_id": "macro.dbt.default__snapshot_string_as_time", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "default__snapshot_string_as_time", "macro_sql": "{% macro default__snapshot_string_as_time(timestamp) %}\n {% do exceptions.raise_not_implemented(\n 'snapshot_string_as_time macro not implemented for adapter '+adapter.type()\n ) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.789981, "supported_languages": null}, "macro.dbt.snapshot_check_all_get_existing_columns": {"unique_id": "macro.dbt.snapshot_check_all_get_existing_columns", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "snapshot_check_all_get_existing_columns", "macro_sql": "{% macro snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) -%}\n {%- if not target_exists -%}\n {#-- no table yet -> return whatever the query does --#}\n {{ return((false, query_columns)) }}\n {%- endif -%}\n\n {#-- handle any schema changes --#}\n {%- set target_relation = adapter.get_relation(database=node.database, schema=node.schema, identifier=node.alias) -%}\n\n {% if check_cols_config == 'all' %}\n {%- set query_columns = get_columns_in_query(node['compiled_code']) -%}\n\n {% elif check_cols_config is iterable and (check_cols_config | length) > 0 %}\n {#-- query for proper casing/quoting, to support comparison below --#}\n {%- set select_check_cols_from_target -%}\n select {{ check_cols_config | join(', ') }} from ({{ node['compiled_code'] }}) subq\n {%- endset -%}\n {% set query_columns = get_columns_in_query(select_check_cols_from_target) %}\n\n {% else %}\n {% do exceptions.raise_compiler_error(\"Invalid value for 'check_cols': \" ~ check_cols_config) %}\n {% endif %}\n\n {%- set existing_cols = adapter.get_columns_in_relation(target_relation) | map(attribute = 'name') | list -%}\n {%- set ns = namespace() -%} {#-- handle for-loop scoping with a namespace --#}\n {%- set ns.column_added = false -%}\n\n {%- set intersection = [] -%}\n {%- for col in query_columns -%}\n {%- if col in existing_cols -%}\n {%- do intersection.append(adapter.quote(col)) -%}\n {%- else -%}\n {% set ns.column_added = true %}\n {%- endif -%}\n {%- endfor -%}\n {{ return((ns.column_added, intersection)) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_columns_in_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.793568, "supported_languages": null}, "macro.dbt.snapshot_check_strategy": {"unique_id": "macro.dbt.snapshot_check_strategy", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "snapshot_check_strategy", "macro_sql": "{% macro snapshot_check_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}\n {% set check_cols_config = config['check_cols'] %}\n {% set primary_key = config['unique_key'] %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %}\n {% set updated_at = config.get('updated_at', snapshot_get_time()) %}\n\n {% set column_added = false %}\n\n {% set column_added, check_cols = snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) %}\n\n {%- set row_changed_expr -%}\n (\n {%- if column_added -%}\n {{ get_true_sql() }}\n {%- else -%}\n {%- for col in check_cols -%}\n {{ snapshotted_rel }}.{{ col }} != {{ current_rel }}.{{ col }}\n or\n (\n (({{ snapshotted_rel }}.{{ col }} is null) and not ({{ current_rel }}.{{ col }} is null))\n or\n ((not {{ snapshotted_rel }}.{{ col }} is null) and ({{ current_rel }}.{{ col }} is null))\n )\n {%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n {%- endif -%}\n )\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.snapshot_get_time", "macro.dbt.snapshot_check_all_get_existing_columns", "macro.dbt.get_true_sql", "macro.dbt.snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.7968981, "supported_languages": null}, "macro.dbt.create_columns": {"unique_id": "macro.dbt.create_columns", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "create_columns", "macro_sql": "{% macro create_columns(relation, columns) %}\n {{ adapter.dispatch('create_columns', 'dbt')(relation, columns) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__create_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.80455, "supported_languages": null}, "macro.dbt.default__create_columns": {"unique_id": "macro.dbt.default__create_columns", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "default__create_columns", "macro_sql": "{% macro default__create_columns(relation, columns) %}\n {% for column in columns %}\n {% call statement() %}\n alter table {{ relation }} add column \"{{ column.name }}\" {{ column.data_type }};\n {% endcall %}\n {% endfor %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.805295, "supported_languages": null}, "macro.dbt.post_snapshot": {"unique_id": "macro.dbt.post_snapshot", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "post_snapshot", "macro_sql": "{% macro post_snapshot(staging_relation) %}\n {{ adapter.dispatch('post_snapshot', 'dbt')(staging_relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.805722, "supported_languages": null}, "macro.dbt.default__post_snapshot": {"unique_id": "macro.dbt.default__post_snapshot", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "default__post_snapshot", "macro_sql": "{% macro default__post_snapshot(staging_relation) %}\n {# no-op #}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.805967, "supported_languages": null}, "macro.dbt.get_true_sql": {"unique_id": "macro.dbt.get_true_sql", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "get_true_sql", "macro_sql": "{% macro get_true_sql() %}\n {{ adapter.dispatch('get_true_sql', 'dbt')() }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_true_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.8063412, "supported_languages": null}, "macro.dbt.default__get_true_sql": {"unique_id": "macro.dbt.default__get_true_sql", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "default__get_true_sql", "macro_sql": "{% macro default__get_true_sql() %}\n {{ return('TRUE') }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.806646, "supported_languages": null}, "macro.dbt.snapshot_staging_table": {"unique_id": "macro.dbt.snapshot_staging_table", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "snapshot_staging_table", "macro_sql": "{% macro snapshot_staging_table(strategy, source_sql, target_relation) -%}\n {{ adapter.dispatch('snapshot_staging_table', 'dbt')(strategy, source_sql, target_relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__snapshot_staging_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.807165, "supported_languages": null}, "macro.dbt.default__snapshot_staging_table": {"unique_id": "macro.dbt.default__snapshot_staging_table", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "default__snapshot_staging_table", "macro_sql": "{% macro default__snapshot_staging_table(strategy, source_sql, target_relation) -%}\n\n with snapshot_query as (\n\n {{ source_sql }}\n\n ),\n\n snapshotted_data as (\n\n select *,\n {{ strategy.unique_key }} as dbt_unique_key\n\n from {{ target_relation }}\n where dbt_valid_to is null\n\n ),\n\n insertions_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to,\n {{ strategy.scd_id }} as dbt_scd_id\n\n from snapshot_query\n ),\n\n updates_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n {{ strategy.updated_at }} as dbt_valid_to\n\n from snapshot_query\n ),\n\n {%- if strategy.invalidate_hard_deletes %}\n\n deletes_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key\n from snapshot_query\n ),\n {% endif %}\n\n insertions as (\n\n select\n 'insert' as dbt_change_type,\n source_data.*\n\n from insertions_source_data as source_data\n left outer join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where snapshotted_data.dbt_unique_key is null\n or (\n snapshotted_data.dbt_unique_key is not null\n and (\n {{ strategy.row_changed }}\n )\n )\n\n ),\n\n updates as (\n\n select\n 'update' as dbt_change_type,\n source_data.*,\n snapshotted_data.dbt_scd_id\n\n from updates_source_data as source_data\n join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where (\n {{ strategy.row_changed }}\n )\n )\n\n {%- if strategy.invalidate_hard_deletes -%}\n ,\n\n deletes as (\n\n select\n 'delete' as dbt_change_type,\n source_data.*,\n {{ snapshot_get_time() }} as dbt_valid_from,\n {{ snapshot_get_time() }} as dbt_updated_at,\n {{ snapshot_get_time() }} as dbt_valid_to,\n snapshotted_data.dbt_scd_id\n\n from snapshotted_data\n left join deletes_source_data as source_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where source_data.dbt_unique_key is null\n )\n {%- endif %}\n\n select * from insertions\n union all\n select * from updates\n {%- if strategy.invalidate_hard_deletes %}\n union all\n select * from deletes\n {%- endif %}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.snapshot_get_time"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.80939, "supported_languages": null}, "macro.dbt.build_snapshot_table": {"unique_id": "macro.dbt.build_snapshot_table", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "build_snapshot_table", "macro_sql": "{% macro build_snapshot_table(strategy, sql) -%}\n {{ adapter.dispatch('build_snapshot_table', 'dbt')(strategy, sql) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__build_snapshot_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.809878, "supported_languages": null}, "macro.dbt.default__build_snapshot_table": {"unique_id": "macro.dbt.default__build_snapshot_table", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "default__build_snapshot_table", "macro_sql": "{% macro default__build_snapshot_table(strategy, sql) %}\n\n select *,\n {{ strategy.scd_id }} as dbt_scd_id,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to\n from (\n {{ sql }}\n ) sbq\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.810522, "supported_languages": null}, "macro.dbt.build_snapshot_staging_table": {"unique_id": "macro.dbt.build_snapshot_staging_table", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "build_snapshot_staging_table", "macro_sql": "{% macro build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% set temp_relation = make_temp_relation(target_relation) %}\n\n {% set select = snapshot_staging_table(strategy, sql, target_relation) %}\n\n {% call statement('build_snapshot_staging_relation') %}\n {{ create_table_as(True, temp_relation, select) }}\n {% endcall %}\n\n {% do return(temp_relation) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.make_temp_relation", "macro.dbt.snapshot_staging_table", "macro.dbt.statement", "macro.dbt.create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.811589, "supported_languages": null}, "macro.dbt.materialization_snapshot_default": {"unique_id": "macro.dbt.materialization_snapshot_default", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/snapshots/snapshot.sql", "original_file_path": "macros/materializations/snapshots/snapshot.sql", "name": "materialization_snapshot_default", "macro_sql": "{% materialization snapshot, default %}\n {%- set config = model['config'] -%}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n -- grab current tables grants config for comparision later on\n {%- set grant_config = config.get('grants') -%}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=model.database,\n schema=model.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", config, target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_code']) %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% else %}\n\n {{ adapter.valid_snapshot_target(target_relation) }}\n\n {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% endif %}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode=False) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if not target_relation_exists %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if staging_table is defined %}\n {% do post_snapshot(staging_table) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_or_create_relation", "macro.dbt.run_hooks", "macro.dbt.strategy_dispatch", "macro.dbt.build_snapshot_table", "macro.dbt.create_table_as", "macro.dbt.build_snapshot_staging_table", "macro.dbt.create_columns", "macro.dbt.snapshot_merge_sql", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes", "macro.dbt.post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.826415, "supported_languages": ["sql"]}, "macro.dbt.materialization_test_default": {"unique_id": "macro.dbt.materialization_test_default", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/tests/test.sql", "original_file_path": "macros/materializations/tests/test.sql", "name": "materialization_test_default", "macro_sql": "{%- materialization test, default -%}\n\n {% set relations = [] %}\n\n {% if should_store_failures() %}\n\n {% set identifier = model['alias'] %}\n {% set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n {% set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database, type='table') -%} %}\n\n {% if old_relation %}\n {% do adapter.drop_relation(old_relation) %}\n {% endif %}\n\n {% call statement(auto_begin=True) %}\n {{ create_table_as(False, target_relation, sql) }}\n {% endcall %}\n\n {% do relations.append(target_relation) %}\n\n {% set main_sql %}\n select *\n from {{ target_relation }}\n {% endset %}\n\n {{ adapter.commit() }}\n\n {% else %}\n\n {% set main_sql = sql %}\n\n {% endif %}\n\n {% set limit = config.get('limit') %}\n {% set fail_calc = config.get('fail_calc') %}\n {% set warn_if = config.get('warn_if') %}\n {% set error_if = config.get('error_if') %}\n\n {% call statement('main', fetch_result=True) -%}\n\n {{ get_test_sql(main_sql, fail_calc, warn_if, error_if, limit)}}\n\n {%- endcall %}\n\n {{ return({'relations': relations}) }}\n\n{%- endmaterialization -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.should_store_failures", "macro.dbt.statement", "macro.dbt.create_table_as", "macro.dbt.get_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.831668, "supported_languages": ["sql"]}, "macro.dbt.get_test_sql": {"unique_id": "macro.dbt.get_test_sql", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "name": "get_test_sql", "macro_sql": "{% macro get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n {{ adapter.dispatch('get_test_sql', 'dbt')(main_sql, fail_calc, warn_if, error_if, limit) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.8331032, "supported_languages": null}, "macro.dbt.default__get_test_sql": {"unique_id": "macro.dbt.default__get_test_sql", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "name": "default__get_test_sql", "macro_sql": "{% macro default__get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n select\n {{ fail_calc }} as failures,\n {{ fail_calc }} {{ warn_if }} as should_warn,\n {{ fail_calc }} {{ error_if }} as should_error\n from (\n {{ main_sql }}\n {{ \"limit \" ~ limit if limit != none }}\n ) dbt_internal_test\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.8338718, "supported_languages": null}, "macro.dbt.get_where_subquery": {"unique_id": "macro.dbt.get_where_subquery", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/tests/where_subquery.sql", "original_file_path": "macros/materializations/tests/where_subquery.sql", "name": "get_where_subquery", "macro_sql": "{% macro get_where_subquery(relation) -%}\n {% do return(adapter.dispatch('get_where_subquery', 'dbt')(relation)) %}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_where_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.8350341, "supported_languages": null}, "macro.dbt.default__get_where_subquery": {"unique_id": "macro.dbt.default__get_where_subquery", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/tests/where_subquery.sql", "original_file_path": "macros/materializations/tests/where_subquery.sql", "name": "default__get_where_subquery", "macro_sql": "{% macro default__get_where_subquery(relation) -%}\n {% set where = config.get('where', '') %}\n {% if where %}\n {%- set filtered -%}\n (select * from {{ relation }} where {{ where }}) dbt_subquery\n {%- endset -%}\n {% do return(filtered) %}\n {%- else -%}\n {% do return(relation) %}\n {%- endif -%}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.835979, "supported_languages": null}, "macro.dbt.get_quoted_csv": {"unique_id": "macro.dbt.get_quoted_csv", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "name": "get_quoted_csv", "macro_sql": "{% macro get_quoted_csv(column_names) %}\n\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote(col)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.8394618, "supported_languages": null}, "macro.dbt.diff_columns": {"unique_id": "macro.dbt.diff_columns", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "name": "diff_columns", "macro_sql": "{% macro diff_columns(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% set source_names = source_columns | map(attribute = 'column') | list %}\n {% set target_names = target_columns | map(attribute = 'column') | list %}\n\n {# --check whether the name attribute exists in the target - this does not perform a data type check #}\n {% for sc in source_columns %}\n {% if sc.name not in target_names %}\n {{ result.append(sc) }}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.8409832, "supported_languages": null}, "macro.dbt.diff_column_data_types": {"unique_id": "macro.dbt.diff_column_data_types", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "name": "diff_column_data_types", "macro_sql": "{% macro diff_column_data_types(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% for sc in source_columns %}\n {% set tc = target_columns | selectattr(\"name\", \"equalto\", sc.name) | list | first %}\n {% if tc %}\n {% if sc.data_type != tc.data_type and not sc.can_expand_to(other_column=tc) %}\n {{ result.append( { 'column_name': tc.name, 'new_type': sc.data_type } ) }}\n {% endif %}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.842655, "supported_languages": null}, "macro.dbt.get_merge_update_columns": {"unique_id": "macro.dbt.get_merge_update_columns", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "name": "get_merge_update_columns", "macro_sql": "{% macro get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {{ return(adapter.dispatch('get_merge_update_columns', 'dbt')(merge_update_columns, merge_exclude_columns, dest_columns)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_merge_update_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.8432481, "supported_languages": null}, "macro.dbt.default__get_merge_update_columns": {"unique_id": "macro.dbt.default__get_merge_update_columns", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "name": "default__get_merge_update_columns", "macro_sql": "{% macro default__get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {%- set default_cols = dest_columns | map(attribute=\"quoted\") | list -%}\n\n {%- if merge_update_columns and merge_exclude_columns -%}\n {{ exceptions.raise_compiler_error(\n 'Model cannot specify merge_update_columns and merge_exclude_columns. Please update model to use only one config'\n )}}\n {%- elif merge_update_columns -%}\n {%- set update_columns = merge_update_columns -%}\n {%- elif merge_exclude_columns -%}\n {%- set update_columns = [] -%}\n {%- for column in dest_columns -%}\n {% if column.column | lower not in merge_exclude_columns | map(\"lower\") | list %}\n {%- do update_columns.append(column.quoted) -%}\n {% endif %}\n {%- endfor -%}\n {%- else -%}\n {%- set update_columns = default_cols -%}\n {%- endif -%}\n\n {{ return(update_columns) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.845012, "supported_languages": null}, "macro.dbt.get_merge_sql": {"unique_id": "macro.dbt.get_merge_sql", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "name": "get_merge_sql", "macro_sql": "{% macro get_merge_sql(target, source, unique_key, dest_columns, predicates=none) -%}\n {{ adapter.dispatch('get_merge_sql', 'dbt')(target, source, unique_key, dest_columns, predicates) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.8534558, "supported_languages": null}, "macro.dbt.default__get_merge_sql": {"unique_id": "macro.dbt.default__get_merge_sql", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "name": "default__get_merge_sql", "macro_sql": "{% macro default__get_merge_sql(target, source, unique_key, dest_columns, predicates) -%}\n {%- set predicates = [] if predicates is none else [] + predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set merge_update_columns = config.get('merge_update_columns') -%}\n {%- set merge_exclude_columns = config.get('merge_exclude_columns') -%}\n {%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}\n {% for key in unique_key %}\n {% set this_key_match %}\n DBT_INTERNAL_SOURCE.{{ key }} = DBT_INTERNAL_DEST.{{ key }}\n {% endset %}\n {% do predicates.append(this_key_match) %}\n {% endfor %}\n {% else %}\n {% set unique_key_match %}\n DBT_INTERNAL_SOURCE.{{ unique_key }} = DBT_INTERNAL_DEST.{{ unique_key }}\n {% endset %}\n {% do predicates.append(unique_key_match) %}\n {% endif %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n {{ sql_header if sql_header is not none }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{ predicates | join(' and ') }}\n\n {% if unique_key %}\n when matched then update set\n {% for column_name in update_columns -%}\n {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}\n {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n {% endif %}\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_quoted_csv", "macro.dbt.get_merge_update_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.857812, "supported_languages": null}, "macro.dbt.get_delete_insert_merge_sql": {"unique_id": "macro.dbt.get_delete_insert_merge_sql", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "name": "get_delete_insert_merge_sql", "macro_sql": "{% macro get_delete_insert_merge_sql(target, source, unique_key, dest_columns) -%}\n {{ adapter.dispatch('get_delete_insert_merge_sql', 'dbt')(target, source, unique_key, dest_columns) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_delete_insert_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.85844, "supported_languages": null}, "macro.dbt.default__get_delete_insert_merge_sql": {"unique_id": "macro.dbt.default__get_delete_insert_merge_sql", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "name": "default__get_delete_insert_merge_sql", "macro_sql": "{% macro default__get_delete_insert_merge_sql(target, source, unique_key, dest_columns) -%}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not string %}\n delete from {{target }}\n using {{ source }}\n where (\n {% for key in unique_key %}\n {{ source }}.{{ key }} = {{ target }}.{{ key }}\n {{ \"and \" if not loop.last }}\n {% endfor %}\n );\n {% else %}\n delete from {{ target }}\n where (\n {{ unique_key }}) in (\n select ({{ unique_key }})\n from {{ source }}\n );\n\n {% endif %}\n {% endif %}\n\n insert into {{ target }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ source }}\n )\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.8602839, "supported_languages": null}, "macro.dbt.get_insert_overwrite_merge_sql": {"unique_id": "macro.dbt.get_insert_overwrite_merge_sql", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "name": "get_insert_overwrite_merge_sql", "macro_sql": "{% macro get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header=false) -%}\n {{ adapter.dispatch('get_insert_overwrite_merge_sql', 'dbt')(target, source, dest_columns, predicates, include_sql_header) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_insert_overwrite_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.860947, "supported_languages": null}, "macro.dbt.default__get_insert_overwrite_merge_sql": {"unique_id": "macro.dbt.default__get_insert_overwrite_merge_sql", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "name": "default__get_insert_overwrite_merge_sql", "macro_sql": "{% macro default__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) -%}\n {#-- The only time include_sql_header is True: --#}\n {#-- BigQuery + insert_overwrite strategy + \"static\" partitions config --#}\n {#-- We should consider including the sql header at the materialization level instead --#}\n\n {%- set predicates = [] if predicates is none else [] + predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none and include_sql_header }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on FALSE\n\n when not matched by source\n {% if predicates %} and {{ predicates | join(' and ') }} {% endif %}\n then delete\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.862558, "supported_languages": null}, "macro.dbt.is_incremental": {"unique_id": "macro.dbt.is_incremental", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/incremental/is_incremental.sql", "original_file_path": "macros/materializations/models/incremental/is_incremental.sql", "name": "is_incremental", "macro_sql": "{% macro is_incremental() %}\n {#-- do not run introspective queries in parsing #}\n {% if not execute %}\n {{ return(False) }}\n {% else %}\n {% set relation = adapter.get_relation(this.database, this.schema, this.table) %}\n {{ return(relation is not none\n and relation.type == 'table'\n and model.config.materialized == 'incremental'\n and not should_full_refresh()) }}\n {% endif %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.should_full_refresh"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.864465, "supported_languages": null}, "macro.dbt.get_incremental_append_sql": {"unique_id": "macro.dbt.get_incremental_append_sql", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "name": "get_incremental_append_sql", "macro_sql": "{% macro get_incremental_append_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_append_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_incremental_append_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.866665, "supported_languages": null}, "macro.dbt.default__get_incremental_append_sql": {"unique_id": "macro.dbt.default__get_incremental_append_sql", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "name": "default__get_incremental_append_sql", "macro_sql": "{% macro default__get_incremental_append_sql(arg_dict) %}\n\n {% do return(get_insert_into_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"dest_columns\"])) %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_insert_into_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.867279, "supported_languages": null}, "macro.dbt.get_incremental_delete_insert_sql": {"unique_id": "macro.dbt.get_incremental_delete_insert_sql", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "name": "get_incremental_delete_insert_sql", "macro_sql": "{% macro get_incremental_delete_insert_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_delete_insert_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_incremental_delete_insert_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.867753, "supported_languages": null}, "macro.dbt.default__get_incremental_delete_insert_sql": {"unique_id": "macro.dbt.default__get_incremental_delete_insert_sql", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "name": "default__get_incremental_delete_insert_sql", "macro_sql": "{% macro default__get_incremental_delete_insert_sql(arg_dict) %}\n\n {% do return(get_delete_insert_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"unique_key\"], arg_dict[\"dest_columns\"])) %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_delete_insert_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.868414, "supported_languages": null}, "macro.dbt.get_incremental_merge_sql": {"unique_id": "macro.dbt.get_incremental_merge_sql", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "name": "get_incremental_merge_sql", "macro_sql": "{% macro get_incremental_merge_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_merge_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_incremental_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.868883, "supported_languages": null}, "macro.dbt.default__get_incremental_merge_sql": {"unique_id": "macro.dbt.default__get_incremental_merge_sql", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "name": "default__get_incremental_merge_sql", "macro_sql": "{% macro default__get_incremental_merge_sql(arg_dict) %}\n\n {% do return(get_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"unique_key\"], arg_dict[\"dest_columns\"])) %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.8695412, "supported_languages": null}, "macro.dbt.get_incremental_insert_overwrite_sql": {"unique_id": "macro.dbt.get_incremental_insert_overwrite_sql", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "name": "get_incremental_insert_overwrite_sql", "macro_sql": "{% macro get_incremental_insert_overwrite_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_insert_overwrite_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_incremental_insert_overwrite_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.870015, "supported_languages": null}, "macro.dbt.default__get_incremental_insert_overwrite_sql": {"unique_id": "macro.dbt.default__get_incremental_insert_overwrite_sql", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "name": "default__get_incremental_insert_overwrite_sql", "macro_sql": "{% macro default__get_incremental_insert_overwrite_sql(arg_dict) %}\n\n {% do return(get_insert_overwrite_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"dest_columns\"], arg_dict[\"predicates\"])) %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_insert_overwrite_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.8706748, "supported_languages": null}, "macro.dbt.get_incremental_default_sql": {"unique_id": "macro.dbt.get_incremental_default_sql", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "name": "get_incremental_default_sql", "macro_sql": "{% macro get_incremental_default_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_default_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_incremental_default_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.8712032, "supported_languages": null}, "macro.dbt.default__get_incremental_default_sql": {"unique_id": "macro.dbt.default__get_incremental_default_sql", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "name": "default__get_incremental_default_sql", "macro_sql": "{% macro default__get_incremental_default_sql(arg_dict) %}\n\n {% do return(get_incremental_append_sql(arg_dict)) %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_incremental_append_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.871608, "supported_languages": null}, "macro.dbt.get_insert_into_sql": {"unique_id": "macro.dbt.get_insert_into_sql", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "name": "get_insert_into_sql", "macro_sql": "{% macro get_insert_into_sql(target_relation, temp_relation, dest_columns) %}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n insert into {{ target_relation }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ temp_relation }}\n )\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.872406, "supported_languages": null}, "macro.dbt.materialization_incremental_default": {"unique_id": "macro.dbt.materialization_incremental_default", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/incremental/incremental.sql", "original_file_path": "macros/materializations/models/incremental/incremental.sql", "name": "materialization_incremental_default", "macro_sql": "{% materialization incremental, default -%}\n\n -- relations\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') -%}\n {%- set temp_relation = make_temp_relation(target_relation)-%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation)-%}\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n\n -- configs\n {%- set unique_key = config.get('unique_key') -%}\n {%- set full_refresh_mode = (should_full_refresh() or existing_relation.is_view) -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n\n -- the temp_ and backup_ relations should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation. This has to happen before\n -- BEGIN, in a separate transaction\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation)-%}\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set to_drop = [] %}\n\n {% if existing_relation is none %}\n {% set build_sql = get_create_table_as_sql(False, target_relation, sql) %}\n {% elif full_refresh_mode %}\n {% set build_sql = get_create_table_as_sql(False, intermediate_relation, sql) %}\n {% set need_swap = true %}\n {% else %}\n {% do run_query(get_create_table_as_sql(True, temp_relation, sql)) %}\n {% do adapter.expand_target_column_types(\n from_relation=temp_relation,\n to_relation=target_relation) %}\n {#-- Process schema changes. Returns dict of changes if successful. Use source columns for upserting/merging --#}\n {% set dest_columns = process_schema_changes(on_schema_change, temp_relation, existing_relation) %}\n {% if not dest_columns %}\n {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %}\n {% endif %}\n\n {#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#}\n {% set incremental_strategy = config.get('incremental_strategy') or 'default' %}\n {% set incremental_predicates = config.get('incremental_predicates', none) %}\n {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %}\n {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': temp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'predicates': incremental_predicates }) %}\n {% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %}\n\n {% endif %}\n\n {% call statement(\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% if need_swap %}\n {% do adapter.rename_relation(target_relation, backup_relation) %}\n {% do adapter.rename_relation(intermediate_relation, target_relation) %}\n {% do to_drop.append(backup_relation) %}\n {% endif %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if existing_relation is none or existing_relation.is_view or should_full_refresh() %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {% do adapter.commit() %}\n\n {% for rel in to_drop %}\n {% do adapter.drop_relation(rel) %}\n {% endfor %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_temp_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.should_full_refresh", "macro.dbt.incremental_validate_on_schema_change", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks", "macro.dbt.get_create_table_as_sql", "macro.dbt.run_query", "macro.dbt.process_schema_changes", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.8839822, "supported_languages": ["sql"]}, "macro.dbt.incremental_validate_on_schema_change": {"unique_id": "macro.dbt.incremental_validate_on_schema_change", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "name": "incremental_validate_on_schema_change", "macro_sql": "{% macro incremental_validate_on_schema_change(on_schema_change, default='ignore') %}\n\n {% if on_schema_change not in ['sync_all_columns', 'append_new_columns', 'fail', 'ignore'] %}\n\n {% set log_message = 'Invalid value for on_schema_change (%s) specified. Setting default value of %s.' % (on_schema_change, default) %}\n {% do log(log_message) %}\n\n {{ return(default) }}\n\n {% else %}\n\n {{ return(on_schema_change) }}\n\n {% endif %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.894743, "supported_languages": null}, "macro.dbt.check_for_schema_changes": {"unique_id": "macro.dbt.check_for_schema_changes", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "name": "check_for_schema_changes", "macro_sql": "{% macro check_for_schema_changes(source_relation, target_relation) %}\n\n {% set schema_changed = False %}\n\n {%- set source_columns = adapter.get_columns_in_relation(source_relation) -%}\n {%- set target_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set source_not_in_target = diff_columns(source_columns, target_columns) -%}\n {%- set target_not_in_source = diff_columns(target_columns, source_columns) -%}\n\n {% set new_target_types = diff_column_data_types(source_columns, target_columns) %}\n\n {% if source_not_in_target != [] %}\n {% set schema_changed = True %}\n {% elif target_not_in_source != [] or new_target_types != [] %}\n {% set schema_changed = True %}\n {% elif new_target_types != [] %}\n {% set schema_changed = True %}\n {% endif %}\n\n {% set changes_dict = {\n 'schema_changed': schema_changed,\n 'source_not_in_target': source_not_in_target,\n 'target_not_in_source': target_not_in_source,\n 'source_columns': source_columns,\n 'target_columns': target_columns,\n 'new_target_types': new_target_types\n } %}\n\n {% set msg %}\n In {{ target_relation }}:\n Schema changed: {{ schema_changed }}\n Source columns not in target: {{ source_not_in_target }}\n Target columns not in source: {{ target_not_in_source }}\n New column types: {{ new_target_types }}\n {% endset %}\n\n {% do log(msg) %}\n\n {{ return(changes_dict) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.diff_columns", "macro.dbt.diff_column_data_types"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.8977559, "supported_languages": null}, "macro.dbt.sync_column_schemas": {"unique_id": "macro.dbt.sync_column_schemas", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "name": "sync_column_schemas", "macro_sql": "{% macro sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {%- set add_to_target_arr = schema_changes_dict['source_not_in_target'] -%}\n\n {%- if on_schema_change == 'append_new_columns'-%}\n {%- if add_to_target_arr | length > 0 -%}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, none) -%}\n {%- endif -%}\n\n {% elif on_schema_change == 'sync_all_columns' %}\n {%- set remove_from_target_arr = schema_changes_dict['target_not_in_source'] -%}\n {%- set new_target_types = schema_changes_dict['new_target_types'] -%}\n\n {% if add_to_target_arr | length > 0 or remove_from_target_arr | length > 0 %}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, remove_from_target_arr) -%}\n {% endif %}\n\n {% if new_target_types != [] %}\n {% for ntt in new_target_types %}\n {% set column_name = ntt['column_name'] %}\n {% set new_type = ntt['new_type'] %}\n {% do alter_column_type(target_relation, column_name, new_type) %}\n {% endfor %}\n {% endif %}\n\n {% endif %}\n\n {% set schema_change_message %}\n In {{ target_relation }}:\n Schema change approach: {{ on_schema_change }}\n Columns added: {{ add_to_target_arr }}\n Columns removed: {{ remove_from_target_arr }}\n Data types changed: {{ new_target_types }}\n {% endset %}\n\n {% do log(schema_change_message) %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.alter_relation_add_remove_columns", "macro.dbt.alter_column_type"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.900897, "supported_languages": null}, "macro.dbt.process_schema_changes": {"unique_id": "macro.dbt.process_schema_changes", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "name": "process_schema_changes", "macro_sql": "{% macro process_schema_changes(on_schema_change, source_relation, target_relation) %}\n\n {% if on_schema_change == 'ignore' %}\n\n {{ return({}) }}\n\n {% else %}\n\n {% set schema_changes_dict = check_for_schema_changes(source_relation, target_relation) %}\n\n {% if schema_changes_dict['schema_changed'] %}\n\n {% if on_schema_change == 'fail' %}\n\n {% set fail_msg %}\n The source and target schemas on this incremental model are out of sync!\n They can be reconciled in several ways:\n - set the `on_schema_change` config to either append_new_columns or sync_all_columns, depending on your situation.\n - Re-run the incremental model with `full_refresh: True` to update the target schema.\n - update the schema manually and re-run the process.\n\n Additional troubleshooting context:\n Source columns not in target: {{ schema_changes_dict['source_not_in_target'] }}\n Target columns not in source: {{ schema_changes_dict['target_not_in_source'] }}\n New column types: {{ schema_changes_dict['new_target_types'] }}\n {% endset %}\n\n {% do exceptions.raise_compiler_error(fail_msg) %}\n\n {# -- unless we ignore, run the sync operation per the config #}\n {% else %}\n\n {% do sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {% endif %}\n\n {% endif %}\n\n {{ return(schema_changes_dict['source_columns']) }}\n\n {% endif %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.check_for_schema_changes", "macro.dbt.sync_column_schemas"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.9030159, "supported_languages": null}, "macro.dbt.materialization_table_default": {"unique_id": "macro.dbt.materialization_table_default", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/table/table.sql", "original_file_path": "macros/materializations/models/table/table.sql", "name": "materialization_table_default", "macro_sql": "{% materialization table, default %}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') %}\n {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}\n /*\n See ../view/view.sql for more information about this relation.\n */\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_table_as_sql(False, intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n {% if existing_relation is not none %}\n {{ adapter.rename_relation(existing_relation, backup_relation) }}\n {% endif %}\n\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% do create_indexes(target_relation) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n -- finally, drop the existing/backup relation after the commit\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n{% endmaterialization %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.get_create_table_as_sql", "macro.dbt.create_indexes", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.909476, "supported_languages": ["sql"]}, "macro.dbt.get_create_table_as_sql": {"unique_id": "macro.dbt.get_create_table_as_sql", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/table/create_table_as.sql", "original_file_path": "macros/materializations/models/table/create_table_as.sql", "name": "get_create_table_as_sql", "macro_sql": "{% macro get_create_table_as_sql(temporary, relation, sql) -%}\n {{ adapter.dispatch('get_create_table_as_sql', 'dbt')(temporary, relation, sql) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_create_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.911144, "supported_languages": null}, "macro.dbt.default__get_create_table_as_sql": {"unique_id": "macro.dbt.default__get_create_table_as_sql", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/table/create_table_as.sql", "original_file_path": "macros/materializations/models/table/create_table_as.sql", "name": "default__get_create_table_as_sql", "macro_sql": "{% macro default__get_create_table_as_sql(temporary, relation, sql) -%}\n {{ return(create_table_as(temporary, relation, sql)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.911638, "supported_languages": null}, "macro.dbt.create_table_as": {"unique_id": "macro.dbt.create_table_as", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/table/create_table_as.sql", "original_file_path": "macros/materializations/models/table/create_table_as.sql", "name": "create_table_as", "macro_sql": "{% macro create_table_as(temporary, relation, compiled_code, language='sql') -%}\n {# backward compatibility for create_table_as that does not support language #}\n {% if language == \"sql\" %}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, compiled_code)}}\n {% else %}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, compiled_code, language) }}\n {% endif %}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.912726, "supported_languages": null}, "macro.dbt.default__create_table_as": {"unique_id": "macro.dbt.default__create_table_as", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/table/create_table_as.sql", "original_file_path": "macros/materializations/models/table/create_table_as.sql", "name": "default__create_table_as", "macro_sql": "{% macro default__create_table_as(temporary, relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary: -%}temporary{%- endif %} table\n {{ relation.include(database=(not temporary), schema=(not temporary)) }}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.9137652, "supported_languages": null}, "macro.dbt.materialization_view_default": {"unique_id": "macro.dbt.materialization_view_default", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/view/view.sql", "original_file_path": "macros/materializations/models/view/view.sql", "name": "materialization_view_default", "macro_sql": "{%- materialization view, default -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='view') -%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}\n\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}\n /*\n This relation (probably) doesn't exist yet. If it does exist, it's a leftover from\n a previous run, and we're going to try to drop it immediately. At the end of this\n materialization, we're going to rename the \"existing_relation\" to this identifier,\n and then we're going to drop it. In order to make sure we run the correct one of:\n - drop view ...\n - drop table ...\n\n We need to set the type of this relation to be the type of the existing_relation, if it exists,\n or else \"view\" as a sane default if it does not. Note that if the existing_relation does not\n exist, then there is nothing to move out of the way and subsequentally drop. In that case,\n this relation will be effectively unused.\n */\n {%- set backup_relation_type = 'view' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n -- move the existing view out of the way\n {% if existing_relation is not none %}\n {{ adapter.rename_relation(existing_relation, backup_relation) }}\n {% endif %}\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.run_hooks", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.get_create_view_as_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.919893, "supported_languages": ["sql"]}, "macro.dbt.handle_existing_table": {"unique_id": "macro.dbt.handle_existing_table", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/view/helpers.sql", "original_file_path": "macros/materializations/models/view/helpers.sql", "name": "handle_existing_table", "macro_sql": "{% macro handle_existing_table(full_refresh, old_relation) %}\n {{ adapter.dispatch('handle_existing_table', 'dbt')(full_refresh, old_relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__handle_existing_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.921067, "supported_languages": null}, "macro.dbt.default__handle_existing_table": {"unique_id": "macro.dbt.default__handle_existing_table", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/view/helpers.sql", "original_file_path": "macros/materializations/models/view/helpers.sql", "name": "default__handle_existing_table", "macro_sql": "{% macro default__handle_existing_table(full_refresh, old_relation) %}\n {{ log(\"Dropping relation \" ~ old_relation ~ \" because it is of type \" ~ old_relation.type) }}\n {{ adapter.drop_relation(old_relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.921632, "supported_languages": null}, "macro.dbt.create_or_replace_view": {"unique_id": "macro.dbt.create_or_replace_view", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/view/create_or_replace_view.sql", "original_file_path": "macros/materializations/models/view/create_or_replace_view.sql", "name": "create_or_replace_view", "macro_sql": "{% macro create_or_replace_view() %}\n {%- set identifier = model['alias'] -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database,\n type='view') -%}\n {% set grant_config = config.get('grants') %}\n\n {{ run_hooks(pre_hooks) }}\n\n -- If there's a table with the same name and we weren't told to full refresh,\n -- that's an error. If we were told to full refresh, drop it. This behavior differs\n -- for Snowflake and BigQuery, so multiple dispatch is used.\n {%- if old_relation is not none and old_relation.is_table -%}\n {{ handle_existing_table(should_full_refresh(), old_relation) }}\n {%- endif -%}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(target_relation, sql) }}\n {%- endcall %}\n\n {% set should_revoke = should_revoke(exists_as_view, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=True) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.handle_existing_table", "macro.dbt.should_full_refresh", "macro.dbt.statement", "macro.dbt.get_create_view_as_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.925656, "supported_languages": null}, "macro.dbt.get_create_view_as_sql": {"unique_id": "macro.dbt.get_create_view_as_sql", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/view/create_view_as.sql", "original_file_path": "macros/materializations/models/view/create_view_as.sql", "name": "get_create_view_as_sql", "macro_sql": "{% macro get_create_view_as_sql(relation, sql) -%}\n {{ adapter.dispatch('get_create_view_as_sql', 'dbt')(relation, sql) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_create_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.926919, "supported_languages": null}, "macro.dbt.default__get_create_view_as_sql": {"unique_id": "macro.dbt.default__get_create_view_as_sql", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/view/create_view_as.sql", "original_file_path": "macros/materializations/models/view/create_view_as.sql", "name": "default__get_create_view_as_sql", "macro_sql": "{% macro default__get_create_view_as_sql(relation, sql) -%}\n {{ return(create_view_as(relation, sql)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.927346, "supported_languages": null}, "macro.dbt.create_view_as": {"unique_id": "macro.dbt.create_view_as", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/view/create_view_as.sql", "original_file_path": "macros/materializations/models/view/create_view_as.sql", "name": "create_view_as", "macro_sql": "{% macro create_view_as(relation, sql) -%}\n {{ adapter.dispatch('create_view_as', 'dbt')(relation, sql) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.9278772, "supported_languages": null}, "macro.dbt.default__create_view_as": {"unique_id": "macro.dbt.default__create_view_as", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/models/view/create_view_as.sql", "original_file_path": "macros/materializations/models/view/create_view_as.sql", "name": "default__create_view_as", "macro_sql": "{% macro default__create_view_as(relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n create view {{ relation }} as (\n {{ sql }}\n );\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.928556, "supported_languages": null}, "macro.dbt.materialization_seed_default": {"unique_id": "macro.dbt.materialization_seed_default", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/seeds/seed.sql", "original_file_path": "macros/materializations/seeds/seed.sql", "name": "materialization_seed_default", "macro_sql": "{% materialization seed, default %}\n\n {%- set identifier = model['alias'] -%}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set grant_config = config.get('grants') -%}\n {%- set agate_table = load_agate_table() -%}\n -- grab current tables grants config for comparision later on\n\n {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% set create_table_sql = \"\" %}\n {% if exists_as_view %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a view\".format(old_relation)) }}\n {% elif exists_as_table %}\n {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}\n {% else %}\n {% set create_table_sql = create_csv_table(model, agate_table) %}\n {% endif %}\n\n {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}\n {% set rows_affected = (agate_table.rows | length) %}\n {% set sql = load_csv_rows(model, agate_table) %}\n\n {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}\n {{ get_csv_sql(create_table_sql, sql) }};\n {% endcall %}\n\n {% set target_relation = this.incorporate(type='table') %}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if full_refresh_mode or not exists_as_table %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.run_hooks", "macro.dbt.reset_csv_table", "macro.dbt.create_csv_table", "macro.dbt.load_csv_rows", "macro.dbt.noop_statement", "macro.dbt.get_csv_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.9364128, "supported_languages": ["sql"]}, "macro.dbt.create_csv_table": {"unique_id": "macro.dbt.create_csv_table", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "create_csv_table", "macro_sql": "{% macro create_csv_table(model, agate_table) -%}\n {{ adapter.dispatch('create_csv_table', 'dbt')(model, agate_table) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.9457762, "supported_languages": null}, "macro.dbt.default__create_csv_table": {"unique_id": "macro.dbt.default__create_csv_table", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "default__create_csv_table", "macro_sql": "{% macro default__create_csv_table(model, agate_table) %}\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n\n {% set sql %}\n create table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.948132, "supported_languages": null}, "macro.dbt.reset_csv_table": {"unique_id": "macro.dbt.reset_csv_table", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "reset_csv_table", "macro_sql": "{% macro reset_csv_table(model, full_refresh, old_relation, agate_table) -%}\n {{ adapter.dispatch('reset_csv_table', 'dbt')(model, full_refresh, old_relation, agate_table) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__reset_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.948717, "supported_languages": null}, "macro.dbt.default__reset_csv_table": {"unique_id": "macro.dbt.default__reset_csv_table", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "default__reset_csv_table", "macro_sql": "{% macro default__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% set sql = \"\" %}\n {% if full_refresh %}\n {{ adapter.drop_relation(old_relation) }}\n {% set sql = create_csv_table(model, agate_table) %}\n {% else %}\n {{ adapter.truncate_relation(old_relation) }}\n {% set sql = \"truncate table \" ~ old_relation %}\n {% endif %}\n\n {{ return(sql) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.9499142, "supported_languages": null}, "macro.dbt.get_csv_sql": {"unique_id": "macro.dbt.get_csv_sql", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "get_csv_sql", "macro_sql": "{% macro get_csv_sql(create_or_truncate_sql, insert_sql) %}\n {{ adapter.dispatch('get_csv_sql', 'dbt')(create_or_truncate_sql, insert_sql) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_csv_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.9503942, "supported_languages": null}, "macro.dbt.default__get_csv_sql": {"unique_id": "macro.dbt.default__get_csv_sql", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "default__get_csv_sql", "macro_sql": "{% macro default__get_csv_sql(create_or_truncate_sql, insert_sql) %}\n {{ create_or_truncate_sql }};\n -- dbt seed --\n {{ insert_sql }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.950737, "supported_languages": null}, "macro.dbt.get_binding_char": {"unique_id": "macro.dbt.get_binding_char", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "get_binding_char", "macro_sql": "{% macro get_binding_char() -%}\n {{ adapter.dispatch('get_binding_char', 'dbt')() }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.9510899, "supported_languages": null}, "macro.dbt.default__get_binding_char": {"unique_id": "macro.dbt.default__get_binding_char", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "default__get_binding_char", "macro_sql": "{% macro default__get_binding_char() %}\n {{ return('%s') }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.951392, "supported_languages": null}, "macro.dbt.get_batch_size": {"unique_id": "macro.dbt.get_batch_size", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "get_batch_size", "macro_sql": "{% macro get_batch_size() -%}\n {{ return(adapter.dispatch('get_batch_size', 'dbt')()) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_batch_size"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.9517949, "supported_languages": null}, "macro.dbt.default__get_batch_size": {"unique_id": "macro.dbt.default__get_batch_size", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "default__get_batch_size", "macro_sql": "{% macro default__get_batch_size() %}\n {{ return(10000) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.9521081, "supported_languages": null}, "macro.dbt.get_seed_column_quoted_csv": {"unique_id": "macro.dbt.get_seed_column_quoted_csv", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "get_seed_column_quoted_csv", "macro_sql": "{% macro get_seed_column_quoted_csv(model, column_names) %}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote_seed_column(col, quote_seed_column)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.9532878, "supported_languages": null}, "macro.dbt.load_csv_rows": {"unique_id": "macro.dbt.load_csv_rows", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "load_csv_rows", "macro_sql": "{% macro load_csv_rows(model, agate_table) -%}\n {{ adapter.dispatch('load_csv_rows', 'dbt')(model, agate_table) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__load_csv_rows"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.9537542, "supported_languages": null}, "macro.dbt.default__load_csv_rows": {"unique_id": "macro.dbt.default__load_csv_rows", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "default__load_csv_rows", "macro_sql": "{% macro default__load_csv_rows(model, agate_table) %}\n\n {% set batch_size = get_batch_size() %}\n\n {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}\n {% set bindings = [] %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert into {{ this.render() }} ({{ cols_sql }}) values\n {% for row in chunk -%}\n ({%- for column in agate_table.column_names -%}\n {{ get_binding_char() }}\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_batch_size", "macro.dbt.get_seed_column_quoted_csv", "macro.dbt.get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.957227, "supported_languages": null}, "macro.dbt.generate_alias_name": {"unique_id": "macro.dbt.generate_alias_name", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/get_custom_name/get_custom_alias.sql", "original_file_path": "macros/get_custom_name/get_custom_alias.sql", "name": "generate_alias_name", "macro_sql": "{% macro generate_alias_name(custom_alias_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_alias_name', 'dbt')(custom_alias_name, node)) %}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__generate_alias_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.958654, "supported_languages": null}, "macro.dbt.default__generate_alias_name": {"unique_id": "macro.dbt.default__generate_alias_name", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/get_custom_name/get_custom_alias.sql", "original_file_path": "macros/get_custom_name/get_custom_alias.sql", "name": "default__generate_alias_name", "macro_sql": "{% macro default__generate_alias_name(custom_alias_name=none, node=none) -%}\n\n {%- if custom_alias_name is none -%}\n\n {{ node.name }}\n\n {%- else -%}\n\n {{ custom_alias_name | trim }}\n\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.959223, "supported_languages": null}, "macro.dbt.generate_schema_name": {"unique_id": "macro.dbt.generate_schema_name", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "name": "generate_schema_name", "macro_sql": "{% macro generate_schema_name(custom_schema_name=none, node=none) -%}\n {{ return(adapter.dispatch('generate_schema_name', 'dbt')(custom_schema_name, node)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__generate_schema_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.9607859, "supported_languages": null}, "macro.dbt.default__generate_schema_name": {"unique_id": "macro.dbt.default__generate_schema_name", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "name": "default__generate_schema_name", "macro_sql": "{% macro default__generate_schema_name(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if custom_schema_name is none -%}\n\n {{ default_schema }}\n\n {%- else -%}\n\n {{ default_schema }}_{{ custom_schema_name | trim }}\n\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.961487, "supported_languages": null}, "macro.dbt.generate_schema_name_for_env": {"unique_id": "macro.dbt.generate_schema_name_for_env", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "name": "generate_schema_name_for_env", "macro_sql": "{% macro generate_schema_name_for_env(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if target.name == 'prod' and custom_schema_name is not none -%}\n\n {{ custom_schema_name | trim }}\n\n {%- else -%}\n\n {{ default_schema }}\n\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.962187, "supported_languages": null}, "macro.dbt.generate_database_name": {"unique_id": "macro.dbt.generate_database_name", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "name": "generate_database_name", "macro_sql": "{% macro generate_database_name(custom_database_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_database_name', 'dbt')(custom_database_name, node)) %}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__generate_database_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.963528, "supported_languages": null}, "macro.dbt.default__generate_database_name": {"unique_id": "macro.dbt.default__generate_database_name", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "name": "default__generate_database_name", "macro_sql": "{% macro default__generate_database_name(custom_database_name=none, node=none) -%}\n {%- set default_database = target.database -%}\n {%- if custom_database_name is none -%}\n\n {{ default_database }}\n\n {%- else -%}\n\n {{ custom_database_name }}\n\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.9641569, "supported_languages": null}, "macro.dbt.default__test_relationships": {"unique_id": "macro.dbt.default__test_relationships", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/generic_test_sql/relationships.sql", "original_file_path": "macros/generic_test_sql/relationships.sql", "name": "default__test_relationships", "macro_sql": "{% macro default__test_relationships(model, column_name, to, field) %}\n\nwith child as (\n select {{ column_name }} as from_field\n from {{ model }}\n where {{ column_name }} is not null\n),\n\nparent as (\n select {{ field }} as to_field\n from {{ to }}\n)\n\nselect\n from_field\n\nfrom child\nleft join parent\n on child.from_field = parent.to_field\n\nwhere parent.to_field is null\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.9653509, "supported_languages": null}, "macro.dbt.default__test_not_null": {"unique_id": "macro.dbt.default__test_not_null", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/generic_test_sql/not_null.sql", "original_file_path": "macros/generic_test_sql/not_null.sql", "name": "default__test_not_null", "macro_sql": "{% macro default__test_not_null(model, column_name) %}\n\n{% set column_list = '*' if should_store_failures() else column_name %}\n\nselect {{ column_list }}\nfrom {{ model }}\nwhere {{ column_name }} is null\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.should_store_failures"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.966414, "supported_languages": null}, "macro.dbt.default__test_unique": {"unique_id": "macro.dbt.default__test_unique", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/generic_test_sql/unique.sql", "original_file_path": "macros/generic_test_sql/unique.sql", "name": "default__test_unique", "macro_sql": "{% macro default__test_unique(model, column_name) %}\n\nselect\n {{ column_name }} as unique_field,\n count(*) as n_records\n\nfrom {{ model }}\nwhere {{ column_name }} is not null\ngroup by {{ column_name }}\nhaving count(*) > 1\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.967387, "supported_languages": null}, "macro.dbt.default__test_accepted_values": {"unique_id": "macro.dbt.default__test_accepted_values", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/generic_test_sql/accepted_values.sql", "original_file_path": "macros/generic_test_sql/accepted_values.sql", "name": "default__test_accepted_values", "macro_sql": "{% macro default__test_accepted_values(model, column_name, values, quote=True) %}\n\nwith all_values as (\n\n select\n {{ column_name }} as value_field,\n count(*) as n_records\n\n from {{ model }}\n group by {{ column_name }}\n\n)\n\nselect *\nfrom all_values\nwhere value_field not in (\n {% for value in values -%}\n {% if quote -%}\n '{{ value }}'\n {%- else -%}\n {{ value }}\n {%- endif -%}\n {%- if not loop.last -%},{%- endif %}\n {%- endfor %}\n)\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.969207, "supported_languages": null}, "macro.dbt.statement": {"unique_id": "macro.dbt.statement", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "name": "statement", "macro_sql": "\n{%- macro statement(name=None, fetch_result=False, auto_begin=True, language='sql') -%}\n {%- if execute: -%}\n {%- set compiled_code = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime {} for node \"{}\"'.format(language, model['unique_id'])) }}\n {{ write(compiled_code) }}\n {%- endif -%}\n {%- if language == 'sql'-%}\n {%- set res, table = adapter.execute(compiled_code, auto_begin=auto_begin, fetch=fetch_result) -%}\n {%- elif language == 'python' -%}\n {%- set res = submit_python_job(model, compiled_code) -%}\n {#-- TODO: What should table be for python models? --#}\n {%- set table = None -%}\n {%- else -%}\n {% do exceptions.raise_compiler_error(\"statement macro didn't get supported language\") %}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_result(name, response=res, agate_table=table) }}\n {%- endif -%}\n\n {%- endif -%}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.973081, "supported_languages": null}, "macro.dbt.noop_statement": {"unique_id": "macro.dbt.noop_statement", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "name": "noop_statement", "macro_sql": "{% macro noop_statement(name=None, message=None, code=None, rows_affected=None, res=None) -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_raw_result(name, message=message, code=code, rows_affected=rows_affected, agate_table=res) }}\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.974716, "supported_languages": null}, "macro.dbt.run_query": {"unique_id": "macro.dbt.run_query", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "name": "run_query", "macro_sql": "{% macro run_query(sql) %}\n {% call statement(\"run_query_statement\", fetch_result=true, auto_begin=false) %}\n {{ sql }}\n {% endcall %}\n\n {% do return(load_result(\"run_query_statement\").table) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.9754639, "supported_languages": null}, "macro.dbt.convert_datetime": {"unique_id": "macro.dbt.convert_datetime", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "name": "convert_datetime", "macro_sql": "{% macro convert_datetime(date_str, date_fmt) %}\n\n {% set error_msg -%}\n The provided partition date '{{ date_str }}' does not match the expected format '{{ date_fmt }}'\n {%- endset %}\n\n {% set res = try_or_compiler_error(error_msg, modules.datetime.datetime.strptime, date_str.strip(), date_fmt) %}\n {{ return(res) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.979558, "supported_languages": null}, "macro.dbt.dates_in_range": {"unique_id": "macro.dbt.dates_in_range", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "name": "dates_in_range", "macro_sql": "{% macro dates_in_range(start_date_str, end_date_str=none, in_fmt=\"%Y%m%d\", out_fmt=\"%Y%m%d\") %}\n {% set end_date_str = start_date_str if end_date_str is none else end_date_str %}\n\n {% set start_date = convert_datetime(start_date_str, in_fmt) %}\n {% set end_date = convert_datetime(end_date_str, in_fmt) %}\n\n {% set day_count = (end_date - start_date).days %}\n {% if day_count < 0 %}\n {% set msg -%}\n Partition start date is after the end date ({{ start_date }}, {{ end_date }})\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg, model) }}\n {% endif %}\n\n {% set date_list = [] %}\n {% for i in range(0, day_count + 1) %}\n {% set the_date = (modules.datetime.timedelta(days=i) + start_date) %}\n {% if not out_fmt %}\n {% set _ = date_list.append(the_date) %}\n {% else %}\n {% set _ = date_list.append(the_date.strftime(out_fmt)) %}\n {% endif %}\n {% endfor %}\n\n {{ return(date_list) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.convert_datetime"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.982544, "supported_languages": null}, "macro.dbt.partition_range": {"unique_id": "macro.dbt.partition_range", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "name": "partition_range", "macro_sql": "{% macro partition_range(raw_partition_date, date_fmt='%Y%m%d') %}\n {% set partition_range = (raw_partition_date | string).split(\",\") %}\n\n {% if (partition_range | length) == 1 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = none %}\n {% elif (partition_range | length) == 2 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = partition_range[1] %}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Invalid partition time. Expected format: {Start Date}[,{End Date}]. Got: \" ~ raw_partition_date) }}\n {% endif %}\n\n {{ return(dates_in_range(start_date, end_date, in_fmt=date_fmt)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.dates_in_range"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.984403, "supported_languages": null}, "macro.dbt.py_current_timestring": {"unique_id": "macro.dbt.py_current_timestring", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "name": "py_current_timestring", "macro_sql": "{% macro py_current_timestring() %}\n {% set dt = modules.datetime.datetime.now() %}\n {% do return(dt.strftime(\"%Y%m%d%H%M%S%f\")) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.9849951, "supported_languages": null}, "macro.dbt.except": {"unique_id": "macro.dbt.except", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/except.sql", "original_file_path": "macros/utils/except.sql", "name": "except", "macro_sql": "{% macro except() %}\n {{ return(adapter.dispatch('except', 'dbt')()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__except"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.98597, "supported_languages": null}, "macro.dbt.default__except": {"unique_id": "macro.dbt.default__except", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/except.sql", "original_file_path": "macros/utils/except.sql", "name": "default__except", "macro_sql": "{% macro default__except() %}\n\n except\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.986175, "supported_languages": null}, "macro.dbt.replace": {"unique_id": "macro.dbt.replace", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/replace.sql", "original_file_path": "macros/utils/replace.sql", "name": "replace", "macro_sql": "{% macro replace(field, old_chars, new_chars) -%}\n {{ return(adapter.dispatch('replace', 'dbt') (field, old_chars, new_chars)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__replace"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.9873059, "supported_languages": null}, "macro.dbt.default__replace": {"unique_id": "macro.dbt.default__replace", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/replace.sql", "original_file_path": "macros/utils/replace.sql", "name": "default__replace", "macro_sql": "{% macro default__replace(field, old_chars, new_chars) %}\n\n replace(\n {{ field }},\n {{ old_chars }},\n {{ new_chars }}\n )\n\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.987733, "supported_languages": null}, "macro.dbt.concat": {"unique_id": "macro.dbt.concat", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/concat.sql", "original_file_path": "macros/utils/concat.sql", "name": "concat", "macro_sql": "{% macro concat(fields) -%}\n {{ return(adapter.dispatch('concat', 'dbt')(fields)) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__concat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.98864, "supported_languages": null}, "macro.dbt.default__concat": {"unique_id": "macro.dbt.default__concat", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/concat.sql", "original_file_path": "macros/utils/concat.sql", "name": "default__concat", "macro_sql": "{% macro default__concat(fields) -%}\n {{ fields|join(' || ') }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.988965, "supported_languages": null}, "macro.dbt.length": {"unique_id": "macro.dbt.length", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/length.sql", "original_file_path": "macros/utils/length.sql", "name": "length", "macro_sql": "{% macro length(expression) -%}\n {{ return(adapter.dispatch('length', 'dbt') (expression)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__length"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.990331, "supported_languages": null}, "macro.dbt.default__length": {"unique_id": "macro.dbt.default__length", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/length.sql", "original_file_path": "macros/utils/length.sql", "name": "default__length", "macro_sql": "{% macro default__length(expression) %}\n\n length(\n {{ expression }}\n )\n\n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.9906292, "supported_languages": null}, "macro.dbt.dateadd": {"unique_id": "macro.dbt.dateadd", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "name": "dateadd", "macro_sql": "{% macro dateadd(datepart, interval, from_date_or_timestamp) %}\n {{ return(adapter.dispatch('dateadd', 'dbt')(datepart, interval, from_date_or_timestamp)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__dateadd"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.991911, "supported_languages": null}, "macro.dbt.default__dateadd": {"unique_id": "macro.dbt.default__dateadd", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "name": "default__dateadd", "macro_sql": "{% macro default__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n dateadd(\n {{ datepart }},\n {{ interval }},\n {{ from_date_or_timestamp }}\n )\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.9923341, "supported_languages": null}, "macro.dbt.intersect": {"unique_id": "macro.dbt.intersect", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/intersect.sql", "original_file_path": "macros/utils/intersect.sql", "name": "intersect", "macro_sql": "{% macro intersect() %}\n {{ return(adapter.dispatch('intersect', 'dbt')()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__intersect"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.993244, "supported_languages": null}, "macro.dbt.default__intersect": {"unique_id": "macro.dbt.default__intersect", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/intersect.sql", "original_file_path": "macros/utils/intersect.sql", "name": "default__intersect", "macro_sql": "{% macro default__intersect() %}\n\n intersect\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.993451, "supported_languages": null}, "macro.dbt.escape_single_quotes": {"unique_id": "macro.dbt.escape_single_quotes", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/escape_single_quotes.sql", "original_file_path": "macros/utils/escape_single_quotes.sql", "name": "escape_single_quotes", "macro_sql": "{% macro escape_single_quotes(expression) %}\n {{ return(adapter.dispatch('escape_single_quotes', 'dbt') (expression)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__escape_single_quotes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.994604, "supported_languages": null}, "macro.dbt.default__escape_single_quotes": {"unique_id": "macro.dbt.default__escape_single_quotes", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/escape_single_quotes.sql", "original_file_path": "macros/utils/escape_single_quotes.sql", "name": "default__escape_single_quotes", "macro_sql": "{% macro default__escape_single_quotes(expression) -%}\n{{ expression | replace(\"'\",\"''\") }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.995014, "supported_languages": null}, "macro.dbt.right": {"unique_id": "macro.dbt.right", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/right.sql", "original_file_path": "macros/utils/right.sql", "name": "right", "macro_sql": "{% macro right(string_text, length_expression) -%}\n {{ return(adapter.dispatch('right', 'dbt') (string_text, length_expression)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__right"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.9960818, "supported_languages": null}, "macro.dbt.default__right": {"unique_id": "macro.dbt.default__right", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/right.sql", "original_file_path": "macros/utils/right.sql", "name": "default__right", "macro_sql": "{% macro default__right(string_text, length_expression) %}\n\n right(\n {{ string_text }},\n {{ length_expression }}\n )\n\n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.9965801, "supported_languages": null}, "macro.dbt.listagg": {"unique_id": "macro.dbt.listagg", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "name": "listagg", "macro_sql": "{% macro listagg(measure, delimiter_text=\"','\", order_by_clause=none, limit_num=none) -%}\n {{ return(adapter.dispatch('listagg', 'dbt') (measure, delimiter_text, order_by_clause, limit_num)) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__listagg"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.9982479, "supported_languages": null}, "macro.dbt.default__listagg": {"unique_id": "macro.dbt.default__listagg", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "name": "default__listagg", "macro_sql": "{% macro default__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n {% if limit_num -%}\n array_to_string(\n array_slice(\n array_agg(\n {{ measure }}\n ){% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n ,0\n ,{{ limit_num }}\n ),\n {{ delimiter_text }}\n )\n {%- else %}\n listagg(\n {{ measure }},\n {{ delimiter_text }}\n )\n {% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n {%- endif %}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324074.999271, "supported_languages": null}, "macro.dbt.datediff": {"unique_id": "macro.dbt.datediff", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "name": "datediff", "macro_sql": "{% macro datediff(first_date, second_date, datepart) %}\n {{ return(adapter.dispatch('datediff', 'dbt')(first_date, second_date, datepart)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__datediff"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.000368, "supported_languages": null}, "macro.dbt.default__datediff": {"unique_id": "macro.dbt.default__datediff", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "name": "default__datediff", "macro_sql": "{% macro default__datediff(first_date, second_date, datepart) -%}\n\n datediff(\n {{ datepart }},\n {{ first_date }},\n {{ second_date }}\n )\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.0007942, "supported_languages": null}, "macro.dbt.safe_cast": {"unique_id": "macro.dbt.safe_cast", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "name": "safe_cast", "macro_sql": "{% macro safe_cast(field, type) %}\n {{ return(adapter.dispatch('safe_cast', 'dbt') (field, type)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__safe_cast"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.0018618, "supported_languages": null}, "macro.dbt.default__safe_cast": {"unique_id": "macro.dbt.default__safe_cast", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "name": "default__safe_cast", "macro_sql": "{% macro default__safe_cast(field, type) %}\n {# most databases don't support this function yet\n so we just need to use cast #}\n cast({{field}} as {{type}})\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.002232, "supported_languages": null}, "macro.dbt.hash": {"unique_id": "macro.dbt.hash", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/hash.sql", "original_file_path": "macros/utils/hash.sql", "name": "hash", "macro_sql": "{% macro hash(field) -%}\n {{ return(adapter.dispatch('hash', 'dbt') (field)) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__hash"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.003175, "supported_languages": null}, "macro.dbt.default__hash": {"unique_id": "macro.dbt.default__hash", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/hash.sql", "original_file_path": "macros/utils/hash.sql", "name": "default__hash", "macro_sql": "{% macro default__hash(field) -%}\n md5(cast({{ field }} as {{ api.Column.translate_type('string') }}))\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.0035892, "supported_languages": null}, "macro.dbt.cast_bool_to_text": {"unique_id": "macro.dbt.cast_bool_to_text", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/cast_bool_to_text.sql", "original_file_path": "macros/utils/cast_bool_to_text.sql", "name": "cast_bool_to_text", "macro_sql": "{% macro cast_bool_to_text(field) %}\n {{ adapter.dispatch('cast_bool_to_text', 'dbt') (field) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__cast_bool_to_text"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.004607, "supported_languages": null}, "macro.dbt.default__cast_bool_to_text": {"unique_id": "macro.dbt.default__cast_bool_to_text", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/cast_bool_to_text.sql", "original_file_path": "macros/utils/cast_bool_to_text.sql", "name": "default__cast_bool_to_text", "macro_sql": "{% macro default__cast_bool_to_text(field) %}\n cast({{ field }} as {{ api.Column.translate_type('string') }})\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.005036, "supported_languages": null}, "macro.dbt.any_value": {"unique_id": "macro.dbt.any_value", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "name": "any_value", "macro_sql": "{% macro any_value(expression) -%}\n {{ return(adapter.dispatch('any_value', 'dbt') (expression)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__any_value"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.00601, "supported_languages": null}, "macro.dbt.default__any_value": {"unique_id": "macro.dbt.default__any_value", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "name": "default__any_value", "macro_sql": "{% macro default__any_value(expression) -%}\n\n any_value({{ expression }})\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.0063071, "supported_languages": null}, "macro.dbt.position": {"unique_id": "macro.dbt.position", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/position.sql", "original_file_path": "macros/utils/position.sql", "name": "position", "macro_sql": "{% macro position(substring_text, string_text) -%}\n {{ return(adapter.dispatch('position', 'dbt') (substring_text, string_text)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__position"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.007398, "supported_languages": null}, "macro.dbt.default__position": {"unique_id": "macro.dbt.default__position", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/position.sql", "original_file_path": "macros/utils/position.sql", "name": "default__position", "macro_sql": "{% macro default__position(substring_text, string_text) %}\n\n position(\n {{ substring_text }} in {{ string_text }}\n )\n\n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.007754, "supported_languages": null}, "macro.dbt.string_literal": {"unique_id": "macro.dbt.string_literal", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/literal.sql", "original_file_path": "macros/utils/literal.sql", "name": "string_literal", "macro_sql": "{%- macro string_literal(value) -%}\n {{ return(adapter.dispatch('string_literal', 'dbt') (value)) }}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__string_literal"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.008699, "supported_languages": null}, "macro.dbt.default__string_literal": {"unique_id": "macro.dbt.default__string_literal", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/literal.sql", "original_file_path": "macros/utils/literal.sql", "name": "default__string_literal", "macro_sql": "{% macro default__string_literal(value) -%}\n '{{ value }}'\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.008971, "supported_languages": null}, "macro.dbt.type_string": {"unique_id": "macro.dbt.type_string", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "name": "type_string", "macro_sql": "\n\n{%- macro type_string() -%}\n {{ return(adapter.dispatch('type_string', 'dbt')()) }}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__type_string"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.011225, "supported_languages": null}, "macro.dbt.default__type_string": {"unique_id": "macro.dbt.default__type_string", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "name": "default__type_string", "macro_sql": "{% macro default__type_string() %}\n {{ return(api.Column.translate_type(\"string\")) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.011629, "supported_languages": null}, "macro.dbt.type_timestamp": {"unique_id": "macro.dbt.type_timestamp", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "name": "type_timestamp", "macro_sql": "\n\n{%- macro type_timestamp() -%}\n {{ return(adapter.dispatch('type_timestamp', 'dbt')()) }}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__type_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.012175, "supported_languages": null}, "macro.dbt.default__type_timestamp": {"unique_id": "macro.dbt.default__type_timestamp", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "name": "default__type_timestamp", "macro_sql": "{% macro default__type_timestamp() %}\n {{ return(api.Column.translate_type(\"timestamp\")) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.0125592, "supported_languages": null}, "macro.dbt.type_float": {"unique_id": "macro.dbt.type_float", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "name": "type_float", "macro_sql": "\n\n{%- macro type_float() -%}\n {{ return(adapter.dispatch('type_float', 'dbt')()) }}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__type_float"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.012963, "supported_languages": null}, "macro.dbt.default__type_float": {"unique_id": "macro.dbt.default__type_float", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "name": "default__type_float", "macro_sql": "{% macro default__type_float() %}\n {{ return(api.Column.translate_type(\"float\")) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.013345, "supported_languages": null}, "macro.dbt.type_numeric": {"unique_id": "macro.dbt.type_numeric", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "name": "type_numeric", "macro_sql": "\n\n{%- macro type_numeric() -%}\n {{ return(adapter.dispatch('type_numeric', 'dbt')()) }}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__type_numeric"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.013751, "supported_languages": null}, "macro.dbt.default__type_numeric": {"unique_id": "macro.dbt.default__type_numeric", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "name": "default__type_numeric", "macro_sql": "{% macro default__type_numeric() %}\n {{ return(api.Column.numeric_type(\"numeric\", 28, 6)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.0141928, "supported_languages": null}, "macro.dbt.type_bigint": {"unique_id": "macro.dbt.type_bigint", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "name": "type_bigint", "macro_sql": "\n\n{%- macro type_bigint() -%}\n {{ return(adapter.dispatch('type_bigint', 'dbt')()) }}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__type_bigint"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.014601, "supported_languages": null}, "macro.dbt.default__type_bigint": {"unique_id": "macro.dbt.default__type_bigint", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "name": "default__type_bigint", "macro_sql": "{% macro default__type_bigint() %}\n {{ return(api.Column.translate_type(\"bigint\")) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.014984, "supported_languages": null}, "macro.dbt.type_int": {"unique_id": "macro.dbt.type_int", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "name": "type_int", "macro_sql": "\n\n{%- macro type_int() -%}\n {{ return(adapter.dispatch('type_int', 'dbt')()) }}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__type_int"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.015394, "supported_languages": null}, "macro.dbt.default__type_int": {"unique_id": "macro.dbt.default__type_int", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "name": "default__type_int", "macro_sql": "{%- macro default__type_int() -%}\n {{ return(api.Column.translate_type(\"integer\")) }}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.015761, "supported_languages": null}, "macro.dbt.type_boolean": {"unique_id": "macro.dbt.type_boolean", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "name": "type_boolean", "macro_sql": "\n\n{%- macro type_boolean() -%}\n {{ return(adapter.dispatch('type_boolean', 'dbt')()) }}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__type_boolean"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.016257, "supported_languages": null}, "macro.dbt.default__type_boolean": {"unique_id": "macro.dbt.default__type_boolean", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "name": "default__type_boolean", "macro_sql": "{%- macro default__type_boolean() -%}\n {{ return(api.Column.translate_type(\"boolean\")) }}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.01665, "supported_languages": null}, "macro.dbt.array_concat": {"unique_id": "macro.dbt.array_concat", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/array_concat.sql", "original_file_path": "macros/utils/array_concat.sql", "name": "array_concat", "macro_sql": "{% macro array_concat(array_1, array_2) -%}\n {{ return(adapter.dispatch('array_concat', 'dbt')(array_1, array_2)) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__array_concat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.017704, "supported_languages": null}, "macro.dbt.default__array_concat": {"unique_id": "macro.dbt.default__array_concat", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/array_concat.sql", "original_file_path": "macros/utils/array_concat.sql", "name": "default__array_concat", "macro_sql": "{% macro default__array_concat(array_1, array_2) -%}\n array_cat({{ array_1 }}, {{ array_2 }})\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.0180538, "supported_languages": null}, "macro.dbt.bool_or": {"unique_id": "macro.dbt.bool_or", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/bool_or.sql", "original_file_path": "macros/utils/bool_or.sql", "name": "bool_or", "macro_sql": "{% macro bool_or(expression) -%}\n {{ return(adapter.dispatch('bool_or', 'dbt') (expression)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__bool_or"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.01898, "supported_languages": null}, "macro.dbt.default__bool_or": {"unique_id": "macro.dbt.default__bool_or", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/bool_or.sql", "original_file_path": "macros/utils/bool_or.sql", "name": "default__bool_or", "macro_sql": "{% macro default__bool_or(expression) -%}\n\n bool_or({{ expression }})\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.019255, "supported_languages": null}, "macro.dbt.last_day": {"unique_id": "macro.dbt.last_day", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "name": "last_day", "macro_sql": "{% macro last_day(date, datepart) %}\n {{ return(adapter.dispatch('last_day', 'dbt') (date, datepart)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__last_day"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.020355, "supported_languages": null}, "macro.dbt.default_last_day": {"unique_id": "macro.dbt.default_last_day", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "name": "default_last_day", "macro_sql": "\n\n{%- macro default_last_day(date, datepart) -%}\n cast(\n {{dbt.dateadd('day', '-1',\n dbt.dateadd(datepart, '1', dbt.date_trunc(datepart, date))\n )}}\n as date)\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.dateadd", "macro.dbt.date_trunc"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.0210161, "supported_languages": null}, "macro.dbt.default__last_day": {"unique_id": "macro.dbt.default__last_day", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "name": "default__last_day", "macro_sql": "{% macro default__last_day(date, datepart) -%}\n {{dbt.default_last_day(date, datepart)}}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default_last_day"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.021465, "supported_languages": null}, "macro.dbt.split_part": {"unique_id": "macro.dbt.split_part", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "name": "split_part", "macro_sql": "{% macro split_part(string_text, delimiter_text, part_number) %}\n {{ return(adapter.dispatch('split_part', 'dbt') (string_text, delimiter_text, part_number)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__split_part"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.023027, "supported_languages": null}, "macro.dbt.default__split_part": {"unique_id": "macro.dbt.default__split_part", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "name": "default__split_part", "macro_sql": "{% macro default__split_part(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n {{ part_number }}\n )\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.023608, "supported_languages": null}, "macro.dbt._split_part_negative": {"unique_id": "macro.dbt._split_part_negative", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "name": "_split_part_negative", "macro_sql": "{% macro _split_part_negative(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n length({{ string_text }})\n - length(\n replace({{ string_text }}, {{ delimiter_text }}, '')\n ) + 2 {{ part_number }}\n )\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.024339, "supported_languages": null}, "macro.dbt.date_trunc": {"unique_id": "macro.dbt.date_trunc", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/date_trunc.sql", "original_file_path": "macros/utils/date_trunc.sql", "name": "date_trunc", "macro_sql": "{% macro date_trunc(datepart, date) -%}\n {{ return(adapter.dispatch('date_trunc', 'dbt') (datepart, date)) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__date_trunc"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.02547, "supported_languages": null}, "macro.dbt.default__date_trunc": {"unique_id": "macro.dbt.default__date_trunc", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/date_trunc.sql", "original_file_path": "macros/utils/date_trunc.sql", "name": "default__date_trunc", "macro_sql": "{% macro default__date_trunc(datepart, date) -%}\n date_trunc('{{datepart}}', {{date}})\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.0258088, "supported_languages": null}, "macro.dbt.array_construct": {"unique_id": "macro.dbt.array_construct", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/array_construct.sql", "original_file_path": "macros/utils/array_construct.sql", "name": "array_construct", "macro_sql": "{% macro array_construct(inputs=[], data_type=api.Column.translate_type('integer')) -%}\n {{ return(adapter.dispatch('array_construct', 'dbt')(inputs, data_type)) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__array_construct"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.0271108, "supported_languages": null}, "macro.dbt.default__array_construct": {"unique_id": "macro.dbt.default__array_construct", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/array_construct.sql", "original_file_path": "macros/utils/array_construct.sql", "name": "default__array_construct", "macro_sql": "{% macro default__array_construct(inputs, data_type) -%}\n {% if inputs|length > 0 %}\n array[ {{ inputs|join(' , ') }} ]\n {% else %}\n array[]::{{data_type}}[]\n {% endif %}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.027744, "supported_languages": null}, "macro.dbt.array_append": {"unique_id": "macro.dbt.array_append", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/array_append.sql", "original_file_path": "macros/utils/array_append.sql", "name": "array_append", "macro_sql": "{% macro array_append(array, new_element) -%}\n {{ return(adapter.dispatch('array_append', 'dbt')(array, new_element)) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__array_append"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.028975, "supported_languages": null}, "macro.dbt.default__array_append": {"unique_id": "macro.dbt.default__array_append", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/utils/array_append.sql", "original_file_path": "macros/utils/array_append.sql", "name": "default__array_append", "macro_sql": "{% macro default__array_append(array, new_element) -%}\n array_append({{ array }}, {{ new_element }})\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.0293288, "supported_languages": null}, "macro.dbt.create_schema": {"unique_id": "macro.dbt.create_schema", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "name": "create_schema", "macro_sql": "{% macro create_schema(relation) -%}\n {{ adapter.dispatch('create_schema', 'dbt')(relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__create_schema"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.030483, "supported_languages": null}, "macro.dbt.default__create_schema": {"unique_id": "macro.dbt.default__create_schema", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "name": "default__create_schema", "macro_sql": "{% macro default__create_schema(relation) -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier() }}\n {% endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.0309591, "supported_languages": null}, "macro.dbt.drop_schema": {"unique_id": "macro.dbt.drop_schema", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "name": "drop_schema", "macro_sql": "{% macro drop_schema(relation) -%}\n {{ adapter.dispatch('drop_schema', 'dbt')(relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__drop_schema"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.031368, "supported_languages": null}, "macro.dbt.default__drop_schema": {"unique_id": "macro.dbt.default__drop_schema", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "name": "default__drop_schema", "macro_sql": "{% macro default__drop_schema(relation) -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier() }} cascade\n {% endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.0318341, "supported_languages": null}, "macro.dbt.current_timestamp": {"unique_id": "macro.dbt.current_timestamp", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "name": "current_timestamp", "macro_sql": "{%- macro current_timestamp() -%}\n {{ adapter.dispatch('current_timestamp', 'dbt')() }}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.033173, "supported_languages": null}, "macro.dbt.default__current_timestamp": {"unique_id": "macro.dbt.default__current_timestamp", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "name": "default__current_timestamp", "macro_sql": "{% macro default__current_timestamp() -%}\n {{ exceptions.raise_not_implemented(\n 'current_timestamp macro not implemented for adapter ' + adapter.type()) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.033555, "supported_languages": null}, "macro.dbt.snapshot_get_time": {"unique_id": "macro.dbt.snapshot_get_time", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "name": "snapshot_get_time", "macro_sql": "\n\n{%- macro snapshot_get_time() -%}\n {{ adapter.dispatch('snapshot_get_time', 'dbt')() }}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__snapshot_get_time"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.0339258, "supported_languages": null}, "macro.dbt.default__snapshot_get_time": {"unique_id": "macro.dbt.default__snapshot_get_time", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "name": "default__snapshot_get_time", "macro_sql": "{% macro default__snapshot_get_time() %}\n {{ current_timestamp() }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.034244, "supported_languages": null}, "macro.dbt.current_timestamp_backcompat": {"unique_id": "macro.dbt.current_timestamp_backcompat", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "name": "current_timestamp_backcompat", "macro_sql": "{% macro current_timestamp_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_backcompat', 'dbt')()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__current_timestamp_backcompat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.034715, "supported_languages": null}, "macro.dbt.default__current_timestamp_backcompat": {"unique_id": "macro.dbt.default__current_timestamp_backcompat", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "name": "default__current_timestamp_backcompat", "macro_sql": "{% macro default__current_timestamp_backcompat() %}\n current_timestamp::timestamp\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.0349262, "supported_languages": null}, "macro.dbt.current_timestamp_in_utc_backcompat": {"unique_id": "macro.dbt.current_timestamp_in_utc_backcompat", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "name": "current_timestamp_in_utc_backcompat", "macro_sql": "{% macro current_timestamp_in_utc_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_in_utc_backcompat', 'dbt')()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__current_timestamp_in_utc_backcompat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.035347, "supported_languages": null}, "macro.dbt.default__current_timestamp_in_utc_backcompat": {"unique_id": "macro.dbt.default__current_timestamp_in_utc_backcompat", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "name": "default__current_timestamp_in_utc_backcompat", "macro_sql": "{% macro default__current_timestamp_in_utc_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_backcompat', 'dbt')()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.current_timestamp_backcompat", "macro.dbt_postgres.postgres__current_timestamp_backcompat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.035771, "supported_languages": null}, "macro.dbt.get_create_index_sql": {"unique_id": "macro.dbt.get_create_index_sql", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "name": "get_create_index_sql", "macro_sql": "{% macro get_create_index_sql(relation, index_dict) -%}\n {{ return(adapter.dispatch('get_create_index_sql', 'dbt')(relation, index_dict)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_create_index_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.0371048, "supported_languages": null}, "macro.dbt.default__get_create_index_sql": {"unique_id": "macro.dbt.default__get_create_index_sql", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "name": "default__get_create_index_sql", "macro_sql": "{% macro default__get_create_index_sql(relation, index_dict) -%}\n {% do return(None) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.037695, "supported_languages": null}, "macro.dbt.create_indexes": {"unique_id": "macro.dbt.create_indexes", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "name": "create_indexes", "macro_sql": "{% macro create_indexes(relation) -%}\n {{ adapter.dispatch('create_indexes', 'dbt')(relation) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.038136, "supported_languages": null}, "macro.dbt.default__create_indexes": {"unique_id": "macro.dbt.default__create_indexes", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "name": "default__create_indexes", "macro_sql": "{% macro default__create_indexes(relation) -%}\n {%- set _indexes = config.get('indexes', default=[]) -%}\n\n {% for _index_dict in _indexes %}\n {% set create_index_sql = get_create_index_sql(relation, _index_dict) %}\n {% if create_index_sql %}\n {% do run_query(create_index_sql) %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_create_index_sql", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.039148, "supported_languages": null}, "macro.dbt.make_intermediate_relation": {"unique_id": "macro.dbt.make_intermediate_relation", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "make_intermediate_relation", "macro_sql": "{% macro make_intermediate_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_intermediate_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_intermediate_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.0456681, "supported_languages": null}, "macro.dbt.default__make_intermediate_relation": {"unique_id": "macro.dbt.default__make_intermediate_relation", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "default__make_intermediate_relation", "macro_sql": "{% macro default__make_intermediate_relation(base_relation, suffix) %}\n {{ return(default__make_temp_relation(base_relation, suffix)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__make_temp_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.046104, "supported_languages": null}, "macro.dbt.make_temp_relation": {"unique_id": "macro.dbt.make_temp_relation", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "make_temp_relation", "macro_sql": "{% macro make_temp_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_temp_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_temp_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.0466611, "supported_languages": null}, "macro.dbt.default__make_temp_relation": {"unique_id": "macro.dbt.default__make_temp_relation", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "default__make_temp_relation", "macro_sql": "{% macro default__make_temp_relation(base_relation, suffix) %}\n {%- set temp_identifier = base_relation.identifier ~ suffix -%}\n {%- set temp_relation = base_relation.incorporate(\n path={\"identifier\": temp_identifier}) -%}\n\n {{ return(temp_relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.047403, "supported_languages": null}, "macro.dbt.make_backup_relation": {"unique_id": "macro.dbt.make_backup_relation", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "make_backup_relation", "macro_sql": "{% macro make_backup_relation(base_relation, backup_relation_type, suffix='__dbt_backup') %}\n {{ return(adapter.dispatch('make_backup_relation', 'dbt')(base_relation, backup_relation_type, suffix)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_backup_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.0480168, "supported_languages": null}, "macro.dbt.default__make_backup_relation": {"unique_id": "macro.dbt.default__make_backup_relation", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "default__make_backup_relation", "macro_sql": "{% macro default__make_backup_relation(base_relation, backup_relation_type, suffix) %}\n {%- set backup_identifier = base_relation.identifier ~ suffix -%}\n {%- set backup_relation = base_relation.incorporate(\n path={\"identifier\": backup_identifier},\n type=backup_relation_type\n ) -%}\n {{ return(backup_relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.048809, "supported_languages": null}, "macro.dbt.drop_relation": {"unique_id": "macro.dbt.drop_relation", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "drop_relation", "macro_sql": "{% macro drop_relation(relation) -%}\n {{ return(adapter.dispatch('drop_relation', 'dbt')(relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__drop_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.049268, "supported_languages": null}, "macro.dbt.default__drop_relation": {"unique_id": "macro.dbt.default__drop_relation", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "default__drop_relation", "macro_sql": "{% macro default__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n drop {{ relation.type }} if exists {{ relation }} cascade\n {%- endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.049806, "supported_languages": null}, "macro.dbt.truncate_relation": {"unique_id": "macro.dbt.truncate_relation", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "truncate_relation", "macro_sql": "{% macro truncate_relation(relation) -%}\n {{ return(adapter.dispatch('truncate_relation', 'dbt')(relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__truncate_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.050257, "supported_languages": null}, "macro.dbt.default__truncate_relation": {"unique_id": "macro.dbt.default__truncate_relation", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "default__truncate_relation", "macro_sql": "{% macro default__truncate_relation(relation) -%}\n {% call statement('truncate_relation') -%}\n truncate table {{ relation }}\n {%- endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.050667, "supported_languages": null}, "macro.dbt.rename_relation": {"unique_id": "macro.dbt.rename_relation", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "rename_relation", "macro_sql": "{% macro rename_relation(from_relation, to_relation) -%}\n {{ return(adapter.dispatch('rename_relation', 'dbt')(from_relation, to_relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__rename_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.051177, "supported_languages": null}, "macro.dbt.default__rename_relation": {"unique_id": "macro.dbt.default__rename_relation", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "default__rename_relation", "macro_sql": "{% macro default__rename_relation(from_relation, to_relation) -%}\n {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %}\n {% call statement('rename_relation') -%}\n alter table {{ from_relation }} rename to {{ target_name }}\n {%- endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.051893, "supported_languages": null}, "macro.dbt.get_or_create_relation": {"unique_id": "macro.dbt.get_or_create_relation", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "get_or_create_relation", "macro_sql": "{% macro get_or_create_relation(database, schema, identifier, type) -%}\n {{ return(adapter.dispatch('get_or_create_relation', 'dbt')(database, schema, identifier, type)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_or_create_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.052501, "supported_languages": null}, "macro.dbt.default__get_or_create_relation": {"unique_id": "macro.dbt.default__get_or_create_relation", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "default__get_or_create_relation", "macro_sql": "{% macro default__get_or_create_relation(database, schema, identifier, type) %}\n {%- set target_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n\n {% if target_relation %}\n {% do return([true, target_relation]) %}\n {% endif %}\n\n {%- set new_relation = api.Relation.create(\n database=database,\n schema=schema,\n identifier=identifier,\n type=type\n ) -%}\n {% do return([false, new_relation]) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.053866, "supported_languages": null}, "macro.dbt.load_cached_relation": {"unique_id": "macro.dbt.load_cached_relation", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "load_cached_relation", "macro_sql": "{% macro load_cached_relation(relation) %}\n {% do return(adapter.get_relation(\n database=relation.database,\n schema=relation.schema,\n identifier=relation.identifier\n )) -%}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.054573, "supported_languages": null}, "macro.dbt.load_relation": {"unique_id": "macro.dbt.load_relation", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "load_relation", "macro_sql": "{% macro load_relation(relation) %}\n {{ return(load_cached_relation(relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.load_cached_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.0549362, "supported_languages": null}, "macro.dbt.drop_relation_if_exists": {"unique_id": "macro.dbt.drop_relation_if_exists", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "drop_relation_if_exists", "macro_sql": "{% macro drop_relation_if_exists(relation) %}\n {% if relation is not none %}\n {{ adapter.drop_relation(relation) }}\n {% endif %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.055425, "supported_languages": null}, "macro.dbt.collect_freshness": {"unique_id": "macro.dbt.collect_freshness", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "name": "collect_freshness", "macro_sql": "{% macro collect_freshness(source, loaded_at_field, filter) %}\n {{ return(adapter.dispatch('collect_freshness', 'dbt')(source, loaded_at_field, filter))}}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__collect_freshness"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.0570502, "supported_languages": null}, "macro.dbt.default__collect_freshness": {"unique_id": "macro.dbt.default__collect_freshness", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "name": "default__collect_freshness", "macro_sql": "{% macro default__collect_freshness(source, loaded_at_field, filter) %}\n {% call statement('collect_freshness', fetch_result=True, auto_begin=False) -%}\n select\n max({{ loaded_at_field }}) as max_loaded_at,\n {{ current_timestamp() }} as snapshotted_at\n from {{ source }}\n {% if filter %}\n where {{ filter }}\n {% endif %}\n {% endcall %}\n {{ return(load_result('collect_freshness').table) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.0581172, "supported_languages": null}, "macro.dbt.copy_grants": {"unique_id": "macro.dbt.copy_grants", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "name": "copy_grants", "macro_sql": "{% macro copy_grants() %}\n {{ return(adapter.dispatch('copy_grants', 'dbt')()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__copy_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.06205, "supported_languages": null}, "macro.dbt.default__copy_grants": {"unique_id": "macro.dbt.default__copy_grants", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "name": "default__copy_grants", "macro_sql": "{% macro default__copy_grants() %}\n {{ return(True) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.062446, "supported_languages": null}, "macro.dbt.support_multiple_grantees_per_dcl_statement": {"unique_id": "macro.dbt.support_multiple_grantees_per_dcl_statement", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "name": "support_multiple_grantees_per_dcl_statement", "macro_sql": "{% macro support_multiple_grantees_per_dcl_statement() %}\n {{ return(adapter.dispatch('support_multiple_grantees_per_dcl_statement', 'dbt')()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__support_multiple_grantees_per_dcl_statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.062894, "supported_languages": null}, "macro.dbt.default__support_multiple_grantees_per_dcl_statement": {"unique_id": "macro.dbt.default__support_multiple_grantees_per_dcl_statement", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "name": "default__support_multiple_grantees_per_dcl_statement", "macro_sql": "\n\n{%- macro default__support_multiple_grantees_per_dcl_statement() -%}\n {{ return(True) }}\n{%- endmacro -%}\n\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.063191, "supported_languages": null}, "macro.dbt.should_revoke": {"unique_id": "macro.dbt.should_revoke", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "name": "should_revoke", "macro_sql": "{% macro should_revoke(existing_relation, full_refresh_mode=True) %}\n\n {% if not existing_relation %}\n {#-- The table doesn't already exist, so no grants to copy over --#}\n {{ return(False) }}\n {% elif full_refresh_mode %}\n {#-- The object is being REPLACED -- whether grants are copied over depends on the value of user config --#}\n {{ return(copy_grants()) }}\n {% else %}\n {#-- The table is being merged/upserted/inserted -- grants will be carried over --#}\n {{ return(True) }}\n {% endif %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.copy_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.064117, "supported_languages": null}, "macro.dbt.get_show_grant_sql": {"unique_id": "macro.dbt.get_show_grant_sql", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "name": "get_show_grant_sql", "macro_sql": "{% macro get_show_grant_sql(relation) %}\n {{ return(adapter.dispatch(\"get_show_grant_sql\", \"dbt\")(relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_show_grant_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.0646179, "supported_languages": null}, "macro.dbt.default__get_show_grant_sql": {"unique_id": "macro.dbt.default__get_show_grant_sql", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "name": "default__get_show_grant_sql", "macro_sql": "{% macro default__get_show_grant_sql(relation) %}\n show grants on {{ relation }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.0649018, "supported_languages": null}, "macro.dbt.get_grant_sql": {"unique_id": "macro.dbt.get_grant_sql", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "name": "get_grant_sql", "macro_sql": "{% macro get_grant_sql(relation, privilege, grantees) %}\n {{ return(adapter.dispatch('get_grant_sql', 'dbt')(relation, privilege, grantees)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_grant_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.065465, "supported_languages": null}, "macro.dbt.default__get_grant_sql": {"unique_id": "macro.dbt.default__get_grant_sql", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "name": "default__get_grant_sql", "macro_sql": "\n\n{%- macro default__get_grant_sql(relation, privilege, grantees) -%}\n grant {{ privilege }} on {{ relation }} to {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.0659552, "supported_languages": null}, "macro.dbt.get_revoke_sql": {"unique_id": "macro.dbt.get_revoke_sql", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "name": "get_revoke_sql", "macro_sql": "{% macro get_revoke_sql(relation, privilege, grantees) %}\n {{ return(adapter.dispatch('get_revoke_sql', 'dbt')(relation, privilege, grantees)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_revoke_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.066529, "supported_languages": null}, "macro.dbt.default__get_revoke_sql": {"unique_id": "macro.dbt.default__get_revoke_sql", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "name": "default__get_revoke_sql", "macro_sql": "\n\n{%- macro default__get_revoke_sql(relation, privilege, grantees) -%}\n revoke {{ privilege }} on {{ relation }} from {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.0670002, "supported_languages": null}, "macro.dbt.get_dcl_statement_list": {"unique_id": "macro.dbt.get_dcl_statement_list", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "name": "get_dcl_statement_list", "macro_sql": "{% macro get_dcl_statement_list(relation, grant_config, get_dcl_macro) %}\n {{ return(adapter.dispatch('get_dcl_statement_list', 'dbt')(relation, grant_config, get_dcl_macro)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_dcl_statement_list"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.0675662, "supported_languages": null}, "macro.dbt.default__get_dcl_statement_list": {"unique_id": "macro.dbt.default__get_dcl_statement_list", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "name": "default__get_dcl_statement_list", "macro_sql": "\n\n{%- macro default__get_dcl_statement_list(relation, grant_config, get_dcl_macro) -%}\n {#\n -- Unpack grant_config into specific privileges and the set of users who need them granted/revoked.\n -- Depending on whether this database supports multiple grantees per statement, pass in the list of\n -- all grantees per privilege, or (if not) template one statement per privilege-grantee pair.\n -- `get_dcl_macro` will be either `get_grant_sql` or `get_revoke_sql`\n #}\n {%- set dcl_statements = [] -%}\n {%- for privilege, grantees in grant_config.items() %}\n {%- if support_multiple_grantees_per_dcl_statement() and grantees -%}\n {%- set dcl = get_dcl_macro(relation, privilege, grantees) -%}\n {%- do dcl_statements.append(dcl) -%}\n {%- else -%}\n {%- for grantee in grantees -%}\n {% set dcl = get_dcl_macro(relation, privilege, [grantee]) %}\n {%- do dcl_statements.append(dcl) -%}\n {% endfor -%}\n {%- endif -%}\n {%- endfor -%}\n {{ return(dcl_statements) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.support_multiple_grantees_per_dcl_statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.069387, "supported_languages": null}, "macro.dbt.call_dcl_statements": {"unique_id": "macro.dbt.call_dcl_statements", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "name": "call_dcl_statements", "macro_sql": "{% macro call_dcl_statements(dcl_statement_list) %}\n {{ return(adapter.dispatch(\"call_dcl_statements\", \"dbt\")(dcl_statement_list)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__call_dcl_statements"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.070018, "supported_languages": null}, "macro.dbt.default__call_dcl_statements": {"unique_id": "macro.dbt.default__call_dcl_statements", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "name": "default__call_dcl_statements", "macro_sql": "{% macro default__call_dcl_statements(dcl_statement_list) %}\n {#\n -- By default, supply all grant + revoke statements in a single semicolon-separated block,\n -- so that they're all processed together.\n\n -- Some databases do not support this. Those adapters will need to override this macro\n -- to run each statement individually.\n #}\n {% call statement('grants') %}\n {% for dcl_statement in dcl_statement_list %}\n {{ dcl_statement }};\n {% endfor %}\n {% endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.070652, "supported_languages": null}, "macro.dbt.apply_grants": {"unique_id": "macro.dbt.apply_grants", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "name": "apply_grants", "macro_sql": "{% macro apply_grants(relation, grant_config, should_revoke) %}\n {{ return(adapter.dispatch(\"apply_grants\", \"dbt\")(relation, grant_config, should_revoke)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__apply_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.071306, "supported_languages": null}, "macro.dbt.default__apply_grants": {"unique_id": "macro.dbt.default__apply_grants", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "name": "default__apply_grants", "macro_sql": "{% macro default__apply_grants(relation, grant_config, should_revoke=True) %}\n {#-- If grant_config is {} or None, this is a no-op --#}\n {% if grant_config %}\n {% if should_revoke %}\n {#-- We think previous grants may have carried over --#}\n {#-- Show current grants and calculate diffs --#}\n {% set current_grants_table = run_query(get_show_grant_sql(relation)) %}\n {% set current_grants_dict = adapter.standardize_grants_dict(current_grants_table) %}\n {% set needs_granting = diff_of_two_dicts(grant_config, current_grants_dict) %}\n {% set needs_revoking = diff_of_two_dicts(current_grants_dict, grant_config) %}\n {% if not (needs_granting or needs_revoking) %}\n {{ log('On ' ~ relation ~': All grants are in place, no revocation or granting needed.')}}\n {% endif %}\n {% else %}\n {#-- We don't think there's any chance of previous grants having carried over. --#}\n {#-- Jump straight to granting what the user has configured. --#}\n {% set needs_revoking = {} %}\n {% set needs_granting = grant_config %}\n {% endif %}\n {% if needs_granting or needs_revoking %}\n {% set revoke_statement_list = get_dcl_statement_list(relation, needs_revoking, get_revoke_sql) %}\n {% set grant_statement_list = get_dcl_statement_list(relation, needs_granting, get_grant_sql) %}\n {% set dcl_statement_list = revoke_statement_list + grant_statement_list %}\n {% if dcl_statement_list %}\n {{ call_dcl_statements(dcl_statement_list) }}\n {% endif %}\n {% endif %}\n {% endif %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.run_query", "macro.dbt.get_show_grant_sql", "macro.dbt.get_dcl_statement_list", "macro.dbt.call_dcl_statements"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.074133, "supported_languages": null}, "macro.dbt.alter_column_comment": {"unique_id": "macro.dbt.alter_column_comment", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "name": "alter_column_comment", "macro_sql": "{% macro alter_column_comment(relation, column_dict) -%}\n {{ return(adapter.dispatch('alter_column_comment', 'dbt')(relation, column_dict)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.07601, "supported_languages": null}, "macro.dbt.default__alter_column_comment": {"unique_id": "macro.dbt.default__alter_column_comment", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "name": "default__alter_column_comment", "macro_sql": "{% macro default__alter_column_comment(relation, column_dict) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_column_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.076446, "supported_languages": null}, "macro.dbt.alter_relation_comment": {"unique_id": "macro.dbt.alter_relation_comment", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "name": "alter_relation_comment", "macro_sql": "{% macro alter_relation_comment(relation, relation_comment) -%}\n {{ return(adapter.dispatch('alter_relation_comment', 'dbt')(relation, relation_comment)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__alter_relation_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.0769558, "supported_languages": null}, "macro.dbt.default__alter_relation_comment": {"unique_id": "macro.dbt.default__alter_relation_comment", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "name": "default__alter_relation_comment", "macro_sql": "{% macro default__alter_relation_comment(relation, relation_comment) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_relation_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.077386, "supported_languages": null}, "macro.dbt.persist_docs": {"unique_id": "macro.dbt.persist_docs", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "name": "persist_docs", "macro_sql": "{% macro persist_docs(relation, model, for_relation=true, for_columns=true) -%}\n {{ return(adapter.dispatch('persist_docs', 'dbt')(relation, model, for_relation, for_columns)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.078047, "supported_languages": null}, "macro.dbt.default__persist_docs": {"unique_id": "macro.dbt.default__persist_docs", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "name": "default__persist_docs", "macro_sql": "{% macro default__persist_docs(relation, model, for_relation, for_columns) -%}\n {% if for_relation and config.persist_relation_docs() and model.description %}\n {% do run_query(alter_relation_comment(relation, model.description)) %}\n {% endif %}\n\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {% do run_query(alter_column_comment(relation, model.columns)) %}\n {% endif %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.run_query", "macro.dbt.alter_relation_comment", "macro.dbt.alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.0792341, "supported_languages": null}, "macro.dbt.get_catalog": {"unique_id": "macro.dbt.get_catalog", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "get_catalog", "macro_sql": "{% macro get_catalog(information_schema, schemas) -%}\n {{ return(adapter.dispatch('get_catalog', 'dbt')(information_schema, schemas)) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_catalog"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.0824351, "supported_languages": null}, "macro.dbt.default__get_catalog": {"unique_id": "macro.dbt.default__get_catalog", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "default__get_catalog", "macro_sql": "{% macro default__get_catalog(information_schema, schemas) -%}\n\n {% set typename = adapter.type() %}\n {% set msg -%}\n get_catalog not implemented for {{ typename }}\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.0830872, "supported_languages": null}, "macro.dbt.information_schema_name": {"unique_id": "macro.dbt.information_schema_name", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "information_schema_name", "macro_sql": "{% macro information_schema_name(database) %}\n {{ return(adapter.dispatch('information_schema_name', 'dbt')(database)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__information_schema_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.083549, "supported_languages": null}, "macro.dbt.default__information_schema_name": {"unique_id": "macro.dbt.default__information_schema_name", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "default__information_schema_name", "macro_sql": "{% macro default__information_schema_name(database) -%}\n {%- if database -%}\n {{ database }}.INFORMATION_SCHEMA\n {%- else -%}\n INFORMATION_SCHEMA\n {%- endif -%}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.083961, "supported_languages": null}, "macro.dbt.list_schemas": {"unique_id": "macro.dbt.list_schemas", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "list_schemas", "macro_sql": "{% macro list_schemas(database) -%}\n {{ return(adapter.dispatch('list_schemas', 'dbt')(database)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__list_schemas"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.0844128, "supported_languages": null}, "macro.dbt.default__list_schemas": {"unique_id": "macro.dbt.default__list_schemas", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "default__list_schemas", "macro_sql": "{% macro default__list_schemas(database) -%}\n {% set sql %}\n select distinct schema_name\n from {{ information_schema_name(database) }}.SCHEMATA\n where catalog_name ilike '{{ database }}'\n {% endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.information_schema_name", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.085026, "supported_languages": null}, "macro.dbt.check_schema_exists": {"unique_id": "macro.dbt.check_schema_exists", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "check_schema_exists", "macro_sql": "{% macro check_schema_exists(information_schema, schema) -%}\n {{ return(adapter.dispatch('check_schema_exists', 'dbt')(information_schema, schema)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__check_schema_exists"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.0855522, "supported_languages": null}, "macro.dbt.default__check_schema_exists": {"unique_id": "macro.dbt.default__check_schema_exists", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "default__check_schema_exists", "macro_sql": "{% macro default__check_schema_exists(information_schema, schema) -%}\n {% set sql -%}\n select count(*)\n from {{ information_schema.replace(information_schema_view='SCHEMATA') }}\n where catalog_name='{{ information_schema.database }}'\n and schema_name='{{ schema }}'\n {%- endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.replace", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.0863092, "supported_languages": null}, "macro.dbt.list_relations_without_caching": {"unique_id": "macro.dbt.list_relations_without_caching", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "list_relations_without_caching", "macro_sql": "{% macro list_relations_without_caching(schema_relation) %}\n {{ return(adapter.dispatch('list_relations_without_caching', 'dbt')(schema_relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__list_relations_without_caching"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.086777, "supported_languages": null}, "macro.dbt.default__list_relations_without_caching": {"unique_id": "macro.dbt.default__list_relations_without_caching", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "default__list_relations_without_caching", "macro_sql": "{% macro default__list_relations_without_caching(schema_relation) %}\n {{ exceptions.raise_not_implemented(\n 'list_relations_without_caching macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.087187, "supported_languages": null}, "macro.dbt.get_columns_in_relation": {"unique_id": "macro.dbt.get_columns_in_relation", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "get_columns_in_relation", "macro_sql": "{% macro get_columns_in_relation(relation) -%}\n {{ return(adapter.dispatch('get_columns_in_relation', 'dbt')(relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_columns_in_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.09098, "supported_languages": null}, "macro.dbt.default__get_columns_in_relation": {"unique_id": "macro.dbt.default__get_columns_in_relation", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "default__get_columns_in_relation", "macro_sql": "{% macro default__get_columns_in_relation(relation) -%}\n {{ exceptions.raise_not_implemented(\n 'get_columns_in_relation macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.0918489, "supported_languages": null}, "macro.dbt.sql_convert_columns_in_relation": {"unique_id": "macro.dbt.sql_convert_columns_in_relation", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "sql_convert_columns_in_relation", "macro_sql": "{% macro sql_convert_columns_in_relation(table) -%}\n {% set columns = [] %}\n {% for row in table %}\n {% do columns.append(api.Column(*row)) %}\n {% endfor %}\n {{ return(columns) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.092635, "supported_languages": null}, "macro.dbt.get_columns_in_query": {"unique_id": "macro.dbt.get_columns_in_query", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "get_columns_in_query", "macro_sql": "{% macro get_columns_in_query(select_sql) -%}\n {{ return(adapter.dispatch('get_columns_in_query', 'dbt')(select_sql)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_columns_in_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.0931032, "supported_languages": null}, "macro.dbt.default__get_columns_in_query": {"unique_id": "macro.dbt.default__get_columns_in_query", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "default__get_columns_in_query", "macro_sql": "{% macro default__get_columns_in_query(select_sql) %}\n {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%}\n select * from (\n {{ select_sql }}\n ) as __dbt_sbq\n where false\n limit 0\n {% endcall %}\n\n {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.09394, "supported_languages": null}, "macro.dbt.alter_column_type": {"unique_id": "macro.dbt.alter_column_type", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "alter_column_type", "macro_sql": "{% macro alter_column_type(relation, column_name, new_column_type) -%}\n {{ return(adapter.dispatch('alter_column_type', 'dbt')(relation, column_name, new_column_type)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__alter_column_type"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.094506, "supported_languages": null}, "macro.dbt.default__alter_column_type": {"unique_id": "macro.dbt.default__alter_column_type", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "default__alter_column_type", "macro_sql": "{% macro default__alter_column_type(relation, column_name, new_column_type) -%}\n {#\n 1. Create a new column (w/ temp name and correct type)\n 2. Copy data over to it\n 3. Drop the existing column (cascade!)\n 4. Rename the new column to existing column\n #}\n {%- set tmp_column = column_name + \"__dbt_alter\" -%}\n\n {% call statement('alter_column_type') %}\n alter table {{ relation }} add column {{ adapter.quote(tmp_column) }} {{ new_column_type }};\n update {{ relation }} set {{ adapter.quote(tmp_column) }} = {{ adapter.quote(column_name) }};\n alter table {{ relation }} drop column {{ adapter.quote(column_name) }} cascade;\n alter table {{ relation }} rename column {{ adapter.quote(tmp_column) }} to {{ adapter.quote(column_name) }}\n {% endcall %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.0959759, "supported_languages": null}, "macro.dbt.alter_relation_add_remove_columns": {"unique_id": "macro.dbt.alter_relation_add_remove_columns", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "alter_relation_add_remove_columns", "macro_sql": "{% macro alter_relation_add_remove_columns(relation, add_columns = none, remove_columns = none) -%}\n {{ return(adapter.dispatch('alter_relation_add_remove_columns', 'dbt')(relation, add_columns, remove_columns)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__alter_relation_add_remove_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.096619, "supported_languages": null}, "macro.dbt.default__alter_relation_add_remove_columns": {"unique_id": "macro.dbt.default__alter_relation_add_remove_columns", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "default__alter_relation_add_remove_columns", "macro_sql": "{% macro default__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}\n\n {% if add_columns is none %}\n {% set add_columns = [] %}\n {% endif %}\n {% if remove_columns is none %}\n {% set remove_columns = [] %}\n {% endif %}\n\n {% set sql -%}\n\n alter {{ relation.type }} {{ relation }}\n\n {% for column in add_columns %}\n add column {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }}\n {% endfor %}{{ ',' if add_columns and remove_columns }}\n\n {% for column in remove_columns %}\n drop column {{ column.name }}{{ ',' if not loop.last }}\n {% endfor %}\n\n {%- endset -%}\n\n {% do run_query(sql) %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.098658, "supported_languages": null}, "macro.dbt.build_ref_function": {"unique_id": "macro.dbt.build_ref_function", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "name": "build_ref_function", "macro_sql": "{% macro build_ref_function(model) %}\n\n {%- set ref_dict = {} -%}\n {%- for _ref in model.refs -%}\n {%- set resolved = ref(*_ref) -%}\n {%- do ref_dict.update({_ref | join(\".\"): resolved.quote(database=False, schema=False, identifier=False) | string}) -%}\n {%- endfor -%}\n\ndef ref(*args,dbt_load_df_function):\n refs = {{ ref_dict | tojson }}\n key = \".\".join(args)\n return dbt_load_df_function(refs[key])\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.10208, "supported_languages": null}, "macro.dbt.build_source_function": {"unique_id": "macro.dbt.build_source_function", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "name": "build_source_function", "macro_sql": "{% macro build_source_function(model) %}\n\n {%- set source_dict = {} -%}\n {%- for _source in model.sources -%}\n {%- set resolved = source(*_source) -%}\n {%- do source_dict.update({_source | join(\".\"): resolved.quote(database=False, schema=False, identifier=False) | string}) -%}\n {%- endfor -%}\n\ndef source(*args, dbt_load_df_function):\n sources = {{ source_dict | tojson }}\n key = \".\".join(args)\n return dbt_load_df_function(sources[key])\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.103234, "supported_languages": null}, "macro.dbt.build_config_dict": {"unique_id": "macro.dbt.build_config_dict", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "name": "build_config_dict", "macro_sql": "{% macro build_config_dict(model) %}\n {%- set config_dict = {} -%}\n {%- for key in model.config.config_keys_used -%}\n {# weird type testing with enum, would be much easier to write this logic in Python! #}\n {%- if key == 'language' -%}\n {%- set value = 'python' -%}\n {%- endif -%}\n {%- set value = model.config[key] -%}\n {%- do config_dict.update({key: value}) -%}\n {%- endfor -%}\nconfig_dict = {{ config_dict }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.104405, "supported_languages": null}, "macro.dbt.py_script_postfix": {"unique_id": "macro.dbt.py_script_postfix", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "name": "py_script_postfix", "macro_sql": "{% macro py_script_postfix(model) %}\n# This part is user provided model code\n# you will need to copy the next section to run the code\n# COMMAND ----------\n# this part is dbt logic for get ref work, do not modify\n\n{{ build_ref_function(model ) }}\n{{ build_source_function(model ) }}\n{{ build_config_dict(model) }}\n\nclass config:\n def __init__(self, *args, **kwargs):\n pass\n\n @staticmethod\n def get(key, default=None):\n return config_dict.get(key, default)\n\nclass this:\n \"\"\"dbt.this() or dbt.this.identifier\"\"\"\n database = '{{ this.database }}'\n schema = '{{ this.schema }}'\n identifier = '{{ this.identifier }}'\n def __repr__(self):\n return '{{ this }}'\n\n\nclass dbtObj:\n def __init__(self, load_df_function) -> None:\n self.source = lambda *args: source(*args, dbt_load_df_function=load_df_function)\n self.ref = lambda *args: ref(*args, dbt_load_df_function=load_df_function)\n self.config = config\n self.this = this()\n self.is_incremental = {{ is_incremental() }}\n\n# COMMAND ----------\n{{py_script_comment()}}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.build_ref_function", "macro.dbt.build_source_function", "macro.dbt.build_config_dict", "macro.dbt.is_incremental", "macro.dbt.py_script_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.105411, "supported_languages": null}, "macro.dbt.py_script_comment": {"unique_id": "macro.dbt.py_script_comment", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "name": "py_script_comment", "macro_sql": "{%macro py_script_comment()%}\n{%endmacro%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.105611, "supported_languages": null}, "macro.dbt.test_unique": {"unique_id": "macro.dbt.test_unique", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "name": "test_unique", "macro_sql": "{% test unique(model, column_name) %}\n {% set macro = adapter.dispatch('test_unique', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__test_unique"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.107241, "supported_languages": null}, "macro.dbt.test_not_null": {"unique_id": "macro.dbt.test_not_null", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "name": "test_not_null", "macro_sql": "{% test not_null(model, column_name) %}\n {% set macro = adapter.dispatch('test_not_null', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__test_not_null"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.10783, "supported_languages": null}, "macro.dbt.test_accepted_values": {"unique_id": "macro.dbt.test_accepted_values", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "name": "test_accepted_values", "macro_sql": "{% test accepted_values(model, column_name, values, quote=True) %}\n {% set macro = adapter.dispatch('test_accepted_values', 'dbt') %}\n {{ macro(model, column_name, values, quote) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__test_accepted_values"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.108544, "supported_languages": null}, "macro.dbt.test_relationships": {"unique_id": "macro.dbt.test_relationships", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "name": "test_relationships", "macro_sql": "{% test relationships(model, column_name, to, field) %}\n {% set macro = adapter.dispatch('test_relationships', 'dbt') %}\n {{ macro(model, column_name, to, field) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__test_relationships"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676324075.109239, "supported_languages": null}}, "docs": {"test.somedoc": {"unique_id": "test.somedoc", "package_name": "test", "root_path": "/private/var/folders/qt/vw8wqdgx4w381wh14b9y25m40000gn/T/pytest-of-gerda/pytest-126/project0", "path": "somedoc.md", "original_file_path": "models/somedoc.md", "name": "somedoc", "block_contents": "Testing, testing"}, "dbt.__overview__": {"unique_id": "dbt.__overview__", "package_name": "dbt", "root_path": "/Users/gerda/FTA/dbt/core/dbt/include/global_project", "path": "overview.md", "original_file_path": "docs/overview.md", "name": "__overview__", "block_contents": "### Welcome!\n\nWelcome to the auto-generated documentation for your dbt project!\n\n### Navigation\n\nYou can use the `Project` and `Database` navigation tabs on the left side of the window to explore the models\nin your project.\n\n#### Project Tab\nThe `Project` tab mirrors the directory structure of your dbt project. In this tab, you can see all of the\nmodels defined in your dbt project, as well as models imported from dbt packages.\n\n#### Database Tab\nThe `Database` tab also exposes your models, but in a format that looks more like a database explorer. This view\nshows relations (tables and views) grouped into database schemas. Note that ephemeral models are _not_ shown\nin this interface, as they do not exist in the database.\n\n### Graph Exploration\nYou can click the blue icon on the bottom-right corner of the page to view the lineage graph of your models.\n\nOn model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the `Expand`\nbutton at the top-right of this lineage pane, you'll be able to see all of the models that are used to build,\nor are built from, the model you're exploring.\n\nOnce expanded, you'll be able to use the `--select` and `--exclude` model selection syntax to filter the\nmodels in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax).\n\nNote that you can also right-click on models to interactively filter and explore the graph.\n\n---\n\n### More information\n\n- [What is dbt](https://docs.getdbt.com/docs/introduction)?\n- Read the [dbt viewpoint](https://docs.getdbt.com/docs/viewpoint)\n- [Installation](https://docs.getdbt.com/docs/installation)\n- Join the [dbt Community](https://www.getdbt.com/community/) for questions and discussion"}}, "exposures": {"exposure.test.simple_exposure": {"fqn": ["test", "simple_exposure"], "unique_id": "exposure.test.simple_exposure", "package_name": "test", "root_path": "/private/var/folders/qt/vw8wqdgx4w381wh14b9y25m40000gn/T/pytest-of-gerda/pytest-126/project0", "path": "schema.yml", "original_file_path": "models/schema.yml", "name": "simple_exposure", "type": "dashboard", "owner": {"email": "something@example.com", "name": null}, "resource_type": "exposure", "description": "", "label": null, "maturity": null, "meta": {}, "tags": [], "config": {"enabled": true}, "unrendered_config": {}, "url": null, "depends_on": {"macros": [], "nodes": ["source.test.my_source.my_table", "model.test.my_model"]}, "refs": [["my_model"]], "sources": [["my_source", "my_table"]], "created_at": 1676324075.609121}}, "metrics": {"metric.test.my_metric": {"fqn": ["test", "my_metric"], "unique_id": "metric.test.my_metric", "package_name": "test", "root_path": "/private/var/folders/qt/vw8wqdgx4w381wh14b9y25m40000gn/T/pytest-of-gerda/pytest-126/project0", "path": "schema.yml", "original_file_path": "models/schema.yml", "name": "my_metric", "description": "", "label": "Count records", "calculation_method": "count", "timestamp": "updated_at", "expression": "*", "filters": [], "time_grains": ["day"], "dimensions": [], "window": null, "model": "ref('my_model')", "model_unique_id": null, "resource_type": "metric", "meta": {}, "tags": [], "config": {"enabled": true}, "unrendered_config": {}, "sources": [], "depends_on": {"macros": [], "nodes": ["model.test.my_model"]}, "refs": [["my_model"]], "metrics": [], "created_at": 1676324075.618992}}, "selectors": {}, "disabled": {"model.test.disabled_model": [{"resource_type": "model", "depends_on": {"macros": [], "nodes": []}, "config": {"enabled": false, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "post-hook": [], "pre-hook": []}, "database": "dbt", "schema": "test16763240740000063267_test_previous_version_state", "fqn": ["test", "disabled_model"], "unique_id": "model.test.disabled_model", "raw_code": "{{ config(enabled=False) }}\nselect 2 as id", "language": "sql", "package_name": "test", "root_path": "/private/var/folders/qt/vw8wqdgx4w381wh14b9y25m40000gn/T/pytest-of-gerda/pytest-126/project0", "path": "disabled_model.sql", "original_file_path": "models/disabled_model.sql", "name": "disabled_model", "alias": "disabled_model", "checksum": {"name": "sha256", "checksum": "34f7b8e60d9e7933469c48d6c92b0a53918d0ba626a9ce2c30ab2f1532145827"}, "tags": [], "refs": [], "sources": [], "metrics": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "compiled_path": null, "build_path": null, "deferred": false, "unrendered_config": {"enabled": false}, "created_at": 1676324075.4071748, "config_call_dict": {"enabled": false}}], "snapshot.test.disabled_snapshot_seed": [{"resource_type": "snapshot", "depends_on": {"macros": [], "nodes": []}, "config": {"enabled": false, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "snapshot", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": "id", "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "strategy": "check", "target_schema": "test16763240740000063267_test_previous_version_state", "target_database": null, "updated_at": null, "check_cols": "all", "post-hook": [], "pre-hook": []}, "database": "dbt", "schema": "test16763240740000063267_test_previous_version_state", "fqn": ["test", "disabled_snapshot_seed", "disabled_snapshot_seed"], "unique_id": "snapshot.test.disabled_snapshot_seed", "raw_code": "\n{{\n config(\n unique_key='id',\n strategy='check',\n check_cols='all',\n target_schema=schema,\n enabled=False,\n )\n}}\nselect * from {{ ref('my_seed') }}\n", "language": "sql", "package_name": "test", "root_path": "/private/var/folders/qt/vw8wqdgx4w381wh14b9y25m40000gn/T/pytest-of-gerda/pytest-126/project0", "path": "disabled_snapshot_seed.sql", "original_file_path": "snapshots/disabled_snapshot_seed.sql", "name": "disabled_snapshot_seed", "alias": "disabled_snapshot_seed", "checksum": {"name": "sha256", "checksum": "52b08465e16dcbc364162dfbdb34cf25e04295bc13d63ab0b420f60d15234c76"}, "tags": [], "refs": [["my_seed"]], "sources": [], "metrics": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "compiled_path": null, "build_path": null, "deferred": false, "unrendered_config": {"unique_key": "id", "strategy": "check", "check_cols": "all", "target_schema": "test16763240740000063267_test_previous_version_state", "enabled": false}, "created_at": 1676324075.4334059, "config_call_dict": {"unique_key": "id", "strategy": "check", "check_cols": "all", "target_schema": "test16763240740000063267_test_previous_version_state", "enabled": false}}], "analysis.test.disabled_al": [{"resource_type": "analysis", "depends_on": {"macros": [], "nodes": []}, "config": {"enabled": false, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "post-hook": [], "pre-hook": []}, "database": "dbt", "schema": "test16763240740000063267_test_previous_version_state", "fqn": ["test", "analysis", "disabled_al"], "unique_id": "analysis.test.disabled_al", "raw_code": "{{ config(enabled=False) }}\nselect 9 as id", "language": "sql", "package_name": "test", "root_path": "/private/var/folders/qt/vw8wqdgx4w381wh14b9y25m40000gn/T/pytest-of-gerda/pytest-126/project0", "path": "analysis/disabled_al.sql", "original_file_path": "analyses/disabled_al.sql", "name": "disabled_al", "alias": "disabled_al", "checksum": {"name": "sha256", "checksum": "76b8579816eac97721616fd429dcd1a93c311c6358830a65d40ebe5661572610"}, "tags": [], "refs": [], "sources": [], "metrics": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "compiled_path": null, "build_path": null, "deferred": false, "unrendered_config": {"enabled": false}, "created_at": 1676324075.4483929, "config_call_dict": {"enabled": false}}], "test.test.disabled_just_my": [{"resource_type": "test", "depends_on": {"macros": [], "nodes": []}, "config": {"enabled": false, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "database": "dbt", "schema": "test16763240740000063267_test_previous_version_state_dbt_test__audit", "fqn": ["test", "disabled_just_my"], "unique_id": "test.test.disabled_just_my", "raw_code": "{{ config(enabled=False) }}\n\nselect * from {{ ref('my_model') }}\nwhere false", "language": "sql", "package_name": "test", "root_path": "/private/var/folders/qt/vw8wqdgx4w381wh14b9y25m40000gn/T/pytest-of-gerda/pytest-126/project0", "path": "disabled_just_my.sql", "original_file_path": "tests/disabled_just_my.sql", "name": "disabled_just_my", "alias": "disabled_just_my", "checksum": {"name": "sha256", "checksum": "0b5827d08d1e3c97e8fb865bea00031b2e90ecef7884a42429cc48d0f48b8c20"}, "tags": [], "refs": [["my_model"]], "sources": [], "metrics": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "compiled_path": null, "build_path": null, "deferred": false, "unrendered_config": {"enabled": false}, "created_at": 1676324075.5147672, "config_call_dict": {"enabled": false}}], "test.test.disabled_check_nothing_my_model_.f2c6a72d37": [{"test_metadata": {"name": "disabled_check_nothing", "kwargs": {"model": "{{ get_where_subquery(ref('my_model')) }}"}, "namespace": null}, "resource_type": "test", "depends_on": {"macros": ["macro.test.test_disabled_check_nothing", "macro.dbt.get_where_subquery"], "nodes": []}, "config": {"enabled": false, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "database": "dbt", "schema": "test16763240740000063267_test_previous_version_state_dbt_test__audit", "fqn": ["test", "disabled_check_nothing_my_model_"], "unique_id": "test.test.disabled_check_nothing_my_model_.f2c6a72d37", "raw_code": "{{ test_disabled_check_nothing(**_dbt_generic_test_kwargs) }}", "language": "sql", "package_name": "test", "root_path": "/private/var/folders/qt/vw8wqdgx4w381wh14b9y25m40000gn/T/pytest-of-gerda/pytest-126/project0", "path": "disabled_check_nothing_my_model_.sql", "original_file_path": "models/schema.yml", "name": "disabled_check_nothing_my_model_", "alias": "disabled_check_nothing_my_model_", "checksum": {"name": "none", "checksum": ""}, "tags": [], "refs": [["my_model"]], "sources": [], "metrics": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "compiled_path": null, "build_path": null, "deferred": false, "unrendered_config": {"enabled": false}, "created_at": 1676324075.587028, "config_call_dict": {"enabled": false}, "column_name": null, "file_key_name": "models.my_model"}], "exposure.test.disabled_exposure": [{"fqn": ["test", "disabled_exposure"], "unique_id": "exposure.test.disabled_exposure", "package_name": "test", "root_path": "/private/var/folders/qt/vw8wqdgx4w381wh14b9y25m40000gn/T/pytest-of-gerda/pytest-126/project0", "path": "schema.yml", "original_file_path": "models/schema.yml", "name": "disabled_exposure", "type": "dashboard", "owner": {"email": "something@example.com", "name": null}, "resource_type": "exposure", "description": "", "label": null, "maturity": null, "meta": {}, "tags": [], "config": {"enabled": false}, "unrendered_config": {"enabled": false}, "url": null, "depends_on": {"macros": [], "nodes": []}, "refs": [["my_model"]], "sources": [], "created_at": 1676324075.612152}], "metric.test.disabled_metric": [{"fqn": ["test", "disabled_metric"], "unique_id": "metric.test.disabled_metric", "package_name": "test", "root_path": "/private/var/folders/qt/vw8wqdgx4w381wh14b9y25m40000gn/T/pytest-of-gerda/pytest-126/project0", "path": "schema.yml", "original_file_path": "models/schema.yml", "name": "disabled_metric", "description": "", "label": "Count records", "calculation_method": "count", "timestamp": "updated_at", "expression": "*", "filters": [], "time_grains": ["day"], "dimensions": [], "window": null, "model": "ref('my_model')", "model_unique_id": null, "resource_type": "metric", "meta": {}, "tags": [], "config": {"enabled": false}, "unrendered_config": {"enabled": false}, "sources": [], "depends_on": {"macros": [], "nodes": []}, "refs": [["my_model"]], "metrics": [], "created_at": 1676324075.622605}], "seed.test.disabled_seed": [{"resource_type": "seed", "depends_on": {"macros": [], "nodes": []}, "config": {"enabled": false, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "seed", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "quote_columns": null, "post-hook": [], "pre-hook": []}, "database": "dbt", "schema": "test16763240740000063267_test_previous_version_state", "fqn": ["test", "disabled_seed"], "unique_id": "seed.test.disabled_seed", "raw_code": "", "language": "sql", "package_name": "test", "root_path": "/private/var/folders/qt/vw8wqdgx4w381wh14b9y25m40000gn/T/pytest-of-gerda/pytest-126/project0", "path": "disabled_seed.csv", "original_file_path": "seeds/disabled_seed.csv", "name": "disabled_seed", "alias": "disabled_seed", "checksum": {"name": "sha256", "checksum": "c6c08a913b5a382014ef0ba248d97b12fc801beb369fdbd24aff1a3912ee3773"}, "tags": [], "refs": [], "sources": [], "metrics": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": "test://models/schema.yml", "compiled_path": null, "build_path": null, "deferred": false, "unrendered_config": {"enabled": false}, "created_at": 1676324075.594186, "config_call_dict": {}}], "source.test.my_source.disabled_table": [{"fqn": ["test", "my_source", "disabled_table"], "database": "dbt", "schema": "my_source", "unique_id": "source.test.my_source.disabled_table", "package_name": "test", "root_path": "/private/var/folders/qt/vw8wqdgx4w381wh14b9y25m40000gn/T/pytest-of-gerda/pytest-126/project0", "path": "models/schema.yml", "original_file_path": "models/schema.yml", "name": "disabled_table", "source_name": "my_source", "source_description": "My source", "loader": "a_loader", "identifier": "disabled_table", "resource_type": "source", "quoting": {"database": null, "schema": null, "identifier": null, "column": null}, "loaded_at_field": null, "freshness": {"warn_after": {"count": null, "period": null}, "error_after": {"count": null, "period": null}, "filter": null}, "external": null, "description": "Disabled table", "columns": {}, "meta": {}, "source_meta": {}, "tags": [], "config": {"enabled": false}, "patch_path": null, "unrendered_config": {"enabled": false}, "relation_name": "\"dbt\".\"my_source\".\"disabled_table\"", "created_at": 1676324075.625102}]}, "parent_map": {"model.test.my_model": [], "snapshot.test.snapshot_seed": ["seed.test.my_seed"], "analysis.test.a": [], "test.test.just_my": ["model.test.my_model"], "seed.test.my_seed": [], "test.test.not_null_my_model_id.43e0e9183a": ["model.test.my_model"], "test.test.check_nothing_my_model_.d5a5e66110": ["model.test.my_model"], "source.test.my_source.my_table": [], "exposure.test.simple_exposure": ["model.test.my_model", "source.test.my_source.my_table"], "metric.test.my_metric": ["model.test.my_model"]}, "child_map": {"model.test.my_model": ["exposure.test.simple_exposure", "metric.test.my_metric", "test.test.check_nothing_my_model_.d5a5e66110", "test.test.just_my", "test.test.not_null_my_model_id.43e0e9183a"], "snapshot.test.snapshot_seed": [], "analysis.test.a": [], "test.test.just_my": [], "seed.test.my_seed": ["snapshot.test.snapshot_seed"], "test.test.not_null_my_model_id.43e0e9183a": [], "test.test.check_nothing_my_model_.d5a5e66110": [], "source.test.my_source.my_table": ["exposure.test.simple_exposure"], "exposure.test.simple_exposure": [], "metric.test.my_metric": []}} ================================================ FILE: tests/functional/artifacts/data/state/v8/manifest.json ================================================ {"metadata": {"dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v8.json", "dbt_version": "1.5.0a1", "generated_at": "2023-02-13T21:04:43.788883Z", "invocation_id": "c7896040-31e1-487d-8438-19d703edb137", "env": {}, "project_id": "098f6bcd4621d373cade4e832627b4f6", "user_id": null, "send_anonymous_usage_stats": false, "adapter_type": "postgres"}, "nodes": {"model.test.my_model": {"database": "dbt", "schema": "test16763222812618906995_test_previous_version_state", "name": "my_model", "resource_type": "model", "package_name": "test", "path": "my_model.sql", "original_file_path": "models/my_model.sql", "unique_id": "model.test.my_model", "fqn": ["test", "my_model"], "alias": "my_model", "checksum": {"name": "sha256", "checksum": "2b9123e04ab8bb798f7c565afdc3ee0e56fcd66b4bfbdb435b4891c878d947c5"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "post-hook": [], "pre-hook": []}, "tags": [], "description": "Example model", "columns": {"id": {"name": "id", "description": "", "meta": {}, "data_type": null, "quote": null, "tags": []}}, "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": "test://models/schema.yml", "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1676322282.4291918, "relation_name": "\"dbt\".\"test16763222812618906995_test_previous_version_state\".\"my_model\"", "raw_code": "select 1 as id", "language": "sql", "refs": [], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": null}, "snapshot.test.snapshot_seed": {"database": "dbt", "schema": "test16763222812618906995_test_previous_version_state", "name": "snapshot_seed", "resource_type": "snapshot", "package_name": "test", "path": "snapshot_seed.sql", "original_file_path": "snapshots/snapshot_seed.sql", "unique_id": "snapshot.test.snapshot_seed", "fqn": ["test", "snapshot_seed", "snapshot_seed"], "alias": "snapshot_seed", "checksum": {"name": "sha256", "checksum": "00c13c306831070996970605fbc4c901aa456e1ed1c028725a932e4e6a4ffb0a"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "snapshot", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": "id", "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "strategy": "check", "target_schema": "test16763222812618906995_test_previous_version_state", "target_database": null, "updated_at": null, "check_cols": "all", "post-hook": [], "pre-hook": []}, "tags": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {"unique_key": "id", "strategy": "check", "check_cols": "all", "target_schema": "test16763222812618906995_test_previous_version_state"}, "created_at": 1676322282.28191, "relation_name": "\"dbt\".\"test16763222812618906995_test_previous_version_state\".\"snapshot_seed\"", "raw_code": "\n{{\n config(\n unique_key='id',\n strategy='check',\n check_cols='all',\n target_schema=schema,\n )\n}}\nselect * from {{ ref('my_seed') }}\n", "language": "sql", "refs": [["my_seed"]], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["seed.test.my_seed"]}, "compiled_path": null}, "analysis.test.a": {"database": "dbt", "schema": "test16763222812618906995_test_previous_version_state", "name": "a", "resource_type": "analysis", "package_name": "test", "path": "analysis/a.sql", "original_file_path": "analyses/a.sql", "unique_id": "analysis.test.a", "fqn": ["test", "analysis", "a"], "alias": "a", "checksum": {"name": "sha256", "checksum": "bd1ee600e4e80d03f488fee52a66e8d51b5be2b98acc20df1cf8be4670d86ae5"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "post-hook": [], "pre-hook": []}, "tags": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1676322282.338664, "relation_name": null, "raw_code": "select 4 as id", "language": "sql", "refs": [], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": null}, "test.test.just_my": {"database": "dbt", "schema": "test16763222812618906995_test_previous_version_state_dbt_test__audit", "name": "just_my", "resource_type": "test", "package_name": "test", "path": "just_my.sql", "original_file_path": "tests/just_my.sql", "unique_id": "test.test.just_my", "fqn": ["test", "just_my"], "alias": "just_my", "checksum": {"name": "sha256", "checksum": "f30b7a814e0e3761d1a8042aa40d658d6c33affb28cd92782b0f56559c414fd8"}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": ["data_test_tag"], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": ["data_test_tag"], "description": "", "columns": {}, "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {"tags": ["data_test_tag"]}, "created_at": 1676322282.365304, "relation_name": null, "raw_code": "{{ config(tags = ['data_test_tag']) }}\n\nselect * from {{ ref('my_model') }}\nwhere false", "language": "sql", "refs": [["my_model"]], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["model.test.my_model"]}, "compiled_path": null}, "seed.test.my_seed": {"database": "dbt", "schema": "test16763222812618906995_test_previous_version_state", "name": "my_seed", "resource_type": "seed", "package_name": "test", "path": "my_seed.csv", "original_file_path": "seeds/my_seed.csv", "unique_id": "seed.test.my_seed", "fqn": ["test", "my_seed"], "alias": "my_seed", "checksum": {"name": "sha256", "checksum": "22697c9b76d73a6c7561554ddb2ce101428ea2737ba8dc500d52ebcfdcfcfc13"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "seed", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "quote_columns": null, "post-hook": [], "pre-hook": []}, "tags": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1676322282.395373, "relation_name": "\"dbt\".\"test16763222812618906995_test_previous_version_state\".\"my_seed\"", "raw_code": "", "root_path": "/private/var/folders/qt/vw8wqdgx4w381wh14b9y25m40000gn/T/pytest-of-gerda/pytest-115/project0", "depends_on": {"macros": []}}, "test.test.not_null_my_model_id.43e0e9183a": {"test_metadata": {"name": "not_null", "kwargs": {"column_name": "id", "model": "{{ get_where_subquery(ref('my_model')) }}"}, "namespace": null}, "database": "dbt", "schema": "test16763222812618906995_test_previous_version_state_dbt_test__audit", "name": "not_null_my_model_id", "resource_type": "test", "package_name": "test", "path": "not_null_my_model_id.sql", "original_file_path": "models/schema.yml", "unique_id": "test.test.not_null_my_model_id.43e0e9183a", "fqn": ["test", "not_null_my_model_id"], "alias": "not_null_my_model_id", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1676322282.439473, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [["my_model"]], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.test.my_model"]}, "compiled_path": null, "column_name": "id", "file_key_name": "models.my_model"}, "test.test.check_nothing_my_model_.d5a5e66110": {"test_metadata": {"name": "check_nothing", "kwargs": {"model": "{{ get_where_subquery(ref('my_model')) }}"}, "namespace": null}, "database": "dbt", "schema": "test16763222812618906995_test_previous_version_state_dbt_test__audit", "name": "check_nothing_my_model_", "resource_type": "test", "package_name": "test", "path": "check_nothing_my_model_.sql", "original_file_path": "models/schema.yml", "unique_id": "test.test.check_nothing_my_model_.d5a5e66110", "fqn": ["test", "check_nothing_my_model_"], "alias": "check_nothing_my_model_", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1676322282.4446359, "relation_name": null, "raw_code": "{{ test_check_nothing(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [["my_model"]], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.test.test_check_nothing", "macro.dbt.get_where_subquery"], "nodes": ["model.test.my_model"]}, "compiled_path": null, "column_name": null, "file_key_name": "models.my_model"}}, "sources": {"source.test.my_source.my_table": {"database": "dbt", "schema": "my_source", "name": "my_table", "resource_type": "source", "package_name": "test", "path": "models/schema.yml", "original_file_path": "models/schema.yml", "unique_id": "source.test.my_source.my_table", "fqn": ["test", "my_source", "my_table"], "source_name": "my_source", "source_description": "My source", "loader": "a_loader", "identifier": "my_seed", "quoting": {"database": null, "schema": null, "identifier": null, "column": null}, "loaded_at_field": null, "freshness": {"warn_after": {"count": null, "period": null}, "error_after": {"count": null, "period": null}, "filter": null}, "external": null, "description": "My table", "columns": {}, "meta": {}, "source_meta": {}, "tags": [], "config": {"enabled": true}, "patch_path": null, "unrendered_config": {}, "relation_name": "\"dbt\".\"my_source\".\"my_seed\"", "created_at": 1676322282.498101}}, "macros": {"macro.test.test_check_nothing": {"name": "test_check_nothing", "resource_type": "macro", "package_name": "test", "path": "macros/dummy_test.sql", "original_file_path": "macros/dummy_test.sql", "unique_id": "macro.test.test_check_nothing", "macro_sql": "{% test check_nothing(model) %}\n-- a silly test to make sure that table-level tests show up in the manifest\n-- without a column_name field\n\nselect 0\n\n{% endtest %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.4094772, "supported_languages": null}, "macro.test.test_disabled_check_nothing": {"name": "test_disabled_check_nothing", "resource_type": "macro", "package_name": "test", "path": "macros/disabled_dummy_test.sql", "original_file_path": "macros/disabled_dummy_test.sql", "unique_id": "macro.test.test_disabled_check_nothing", "macro_sql": "{% test disabled_check_nothing(model) %}\n-- a silly test to make sure that table-level tests show up in the manifest\n-- without a column_name field\n\n{{ config(enabled=False) }}\nselect 0\n\n{% endtest %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.410033, "supported_languages": null}, "macro.test.do_nothing": {"name": "do_nothing", "resource_type": "macro", "package_name": "test", "path": "macros/do_nothing.sql", "original_file_path": "macros/do_nothing.sql", "unique_id": "macro.test.do_nothing", "macro_sql": "{% macro do_nothing(foo2, bar2) %}\n select\n '{{ foo2 }}' as foo2,\n '{{ bar2 }}' as bar2\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.41051, "supported_languages": null}, "macro.dbt_postgres.postgres__current_timestamp": {"name": "postgres__current_timestamp", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/timestamps.sql", "original_file_path": "macros/timestamps.sql", "unique_id": "macro.dbt_postgres.postgres__current_timestamp", "macro_sql": "{% macro postgres__current_timestamp() -%}\n now()\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.411176, "supported_languages": null}, "macro.dbt_postgres.postgres__snapshot_string_as_time": {"name": "postgres__snapshot_string_as_time", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/timestamps.sql", "original_file_path": "macros/timestamps.sql", "unique_id": "macro.dbt_postgres.postgres__snapshot_string_as_time", "macro_sql": "{% macro postgres__snapshot_string_as_time(timestamp) -%}\n {%- set result = \"'\" ~ timestamp ~ \"'::timestamp without time zone\" -%}\n {{ return(result) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.411718, "supported_languages": null}, "macro.dbt_postgres.postgres__snapshot_get_time": {"name": "postgres__snapshot_get_time", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/timestamps.sql", "original_file_path": "macros/timestamps.sql", "unique_id": "macro.dbt_postgres.postgres__snapshot_get_time", "macro_sql": "{% macro postgres__snapshot_get_time() -%}\n {{ current_timestamp() }}::timestamp without time zone\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.412009, "supported_languages": null}, "macro.dbt_postgres.postgres__current_timestamp_backcompat": {"name": "postgres__current_timestamp_backcompat", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/timestamps.sql", "original_file_path": "macros/timestamps.sql", "unique_id": "macro.dbt_postgres.postgres__current_timestamp_backcompat", "macro_sql": "{% macro postgres__current_timestamp_backcompat() %}\n current_timestamp::{{ type_timestamp() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.type_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.41232, "supported_languages": null}, "macro.dbt_postgres.postgres__current_timestamp_in_utc_backcompat": {"name": "postgres__current_timestamp_in_utc_backcompat", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/timestamps.sql", "original_file_path": "macros/timestamps.sql", "unique_id": "macro.dbt_postgres.postgres__current_timestamp_in_utc_backcompat", "macro_sql": "{% macro postgres__current_timestamp_in_utc_backcompat() %}\n (current_timestamp at time zone 'utc')::{{ type_timestamp() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.type_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.412619, "supported_languages": null}, "macro.dbt_postgres.postgres__get_catalog": {"name": "postgres__get_catalog", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/catalog.sql", "original_file_path": "macros/catalog.sql", "unique_id": "macro.dbt_postgres.postgres__get_catalog", "macro_sql": "{% macro postgres__get_catalog(information_schema, schemas) -%}\n\n {%- call statement('catalog', fetch_result=True) -%}\n {#\n If the user has multiple databases set and the first one is wrong, this will fail.\n But we won't fail in the case where there are multiple quoting-difference-only dbs, which is better.\n #}\n {% set database = information_schema.database %}\n {{ adapter.verify_database(database) }}\n\n select\n '{{ database }}' as table_database,\n sch.nspname as table_schema,\n tbl.relname as table_name,\n case tbl.relkind\n when 'v' then 'VIEW'\n else 'BASE TABLE'\n end as table_type,\n tbl_desc.description as table_comment,\n col.attname as column_name,\n col.attnum as column_index,\n pg_catalog.format_type(col.atttypid, col.atttypmod) as column_type,\n col_desc.description as column_comment,\n pg_get_userbyid(tbl.relowner) as table_owner\n\n from pg_catalog.pg_namespace sch\n join pg_catalog.pg_class tbl on tbl.relnamespace = sch.oid\n join pg_catalog.pg_attribute col on col.attrelid = tbl.oid\n left outer join pg_catalog.pg_description tbl_desc on (tbl_desc.objoid = tbl.oid and tbl_desc.objsubid = 0)\n left outer join pg_catalog.pg_description col_desc on (col_desc.objoid = tbl.oid and col_desc.objsubid = col.attnum)\n\n where (\n {%- for schema in schemas -%}\n upper(sch.nspname) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n )\n and not pg_is_other_temp_schema(sch.oid) -- not a temporary schema belonging to another session\n and tbl.relpersistence in ('p', 'u') -- [p]ermanent table or [u]nlogged table. Exclude [t]emporary tables\n and tbl.relkind in ('r', 'v', 'f', 'p') -- o[r]dinary table, [v]iew, [f]oreign table, [p]artitioned table. Other values are [i]ndex, [S]equence, [c]omposite type, [t]OAST table, [m]aterialized view\n and col.attnum > 0 -- negative numbers are used for system columns such as oid\n and not col.attisdropped -- column as not been dropped\n\n order by\n sch.nspname,\n tbl.relname,\n col.attnum\n\n {%- endcall -%}\n\n {{ return(load_result('catalog').table) }}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.4152992, "supported_languages": null}, "macro.dbt_postgres.postgres_get_relations": {"name": "postgres_get_relations", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations.sql", "original_file_path": "macros/relations.sql", "unique_id": "macro.dbt_postgres.postgres_get_relations", "macro_sql": "{% macro postgres_get_relations () -%}\n\n {#\n -- in pg_depend, objid is the dependent, refobjid is the referenced object\n -- > a pg_depend entry indicates that the referenced object cannot be\n -- > dropped without also dropping the dependent object.\n #}\n\n {%- call statement('relations', fetch_result=True) -%}\n with relation as (\n select\n pg_rewrite.ev_class as class,\n pg_rewrite.oid as id\n from pg_rewrite\n ),\n class as (\n select\n oid as id,\n relname as name,\n relnamespace as schema,\n relkind as kind\n from pg_class\n ),\n dependency as (\n select distinct\n pg_depend.objid as id,\n pg_depend.refobjid as ref\n from pg_depend\n ),\n schema as (\n select\n pg_namespace.oid as id,\n pg_namespace.nspname as name\n from pg_namespace\n where nspname != 'information_schema' and nspname not like 'pg\\_%'\n ),\n referenced as (\n select\n relation.id AS id,\n referenced_class.name ,\n referenced_class.schema ,\n referenced_class.kind\n from relation\n join class as referenced_class on relation.class=referenced_class.id\n where referenced_class.kind in ('r', 'v')\n ),\n relationships as (\n select\n referenced.name as referenced_name,\n referenced.schema as referenced_schema_id,\n dependent_class.name as dependent_name,\n dependent_class.schema as dependent_schema_id,\n referenced.kind as kind\n from referenced\n join dependency on referenced.id=dependency.id\n join class as dependent_class on dependency.ref=dependent_class.id\n where\n (referenced.name != dependent_class.name or\n referenced.schema != dependent_class.schema)\n )\n\n select\n referenced_schema.name as referenced_schema,\n relationships.referenced_name as referenced_name,\n dependent_schema.name as dependent_schema,\n relationships.dependent_name as dependent_name\n from relationships\n join schema as dependent_schema on relationships.dependent_schema_id=dependent_schema.id\n join schema as referenced_schema on relationships.referenced_schema_id=referenced_schema.id\n group by referenced_schema, referenced_name, dependent_schema, dependent_name\n order by referenced_schema, referenced_name, dependent_schema, dependent_name;\n\n {%- endcall -%}\n\n {{ return(load_result('relations').table) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.4168088, "supported_languages": null}, "macro.dbt_postgres.postgres__create_table_as": {"name": "postgres__create_table_as", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__create_table_as", "macro_sql": "{% macro postgres__create_table_as(temporary, relation, sql) -%}\n {%- set unlogged = config.get('unlogged', default=false) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary -%}\n temporary\n {%- elif unlogged -%}\n unlogged\n {%- endif %} table {{ relation }}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.428651, "supported_languages": null}, "macro.dbt_postgres.postgres__get_create_index_sql": {"name": "postgres__get_create_index_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__get_create_index_sql", "macro_sql": "{% macro postgres__get_create_index_sql(relation, index_dict) -%}\n {%- set index_config = adapter.parse_index(index_dict) -%}\n {%- set comma_separated_columns = \", \".join(index_config.columns) -%}\n {%- set index_name = index_config.render(relation) -%}\n\n create {% if index_config.unique -%}\n unique\n {%- endif %} index if not exists\n \"{{ index_name }}\"\n on {{ relation }} {% if index_config.type -%}\n using {{ index_config.type }}\n {%- endif %}\n ({{ comma_separated_columns }});\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.430589, "supported_languages": null}, "macro.dbt_postgres.postgres__create_schema": {"name": "postgres__create_schema", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__create_schema", "macro_sql": "{% macro postgres__create_schema(relation) -%}\n {% if relation.database -%}\n {{ adapter.verify_database(relation.database) }}\n {%- endif -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier().include(database=False) }}\n {%- endcall -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.4315221, "supported_languages": null}, "macro.dbt_postgres.postgres__drop_schema": {"name": "postgres__drop_schema", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__drop_schema", "macro_sql": "{% macro postgres__drop_schema(relation) -%}\n {% if relation.database -%}\n {{ adapter.verify_database(relation.database) }}\n {%- endif -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier().include(database=False) }} cascade\n {%- endcall -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.432323, "supported_languages": null}, "macro.dbt_postgres.postgres__get_columns_in_relation": {"name": "postgres__get_columns_in_relation", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__get_columns_in_relation", "macro_sql": "{% macro postgres__get_columns_in_relation(relation) -%}\n {% call statement('get_columns_in_relation', fetch_result=True) %}\n select\n column_name,\n data_type,\n character_maximum_length,\n numeric_precision,\n numeric_scale\n\n from {{ relation.information_schema('columns') }}\n where table_name = '{{ relation.identifier }}'\n {% if relation.schema %}\n and table_schema = '{{ relation.schema }}'\n {% endif %}\n order by ordinal_position\n\n {% endcall %}\n {% set table = load_result('get_columns_in_relation').table %}\n {{ return(sql_convert_columns_in_relation(table)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.sql_convert_columns_in_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.433569, "supported_languages": null}, "macro.dbt_postgres.postgres__list_relations_without_caching": {"name": "postgres__list_relations_without_caching", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__list_relations_without_caching", "macro_sql": "{% macro postgres__list_relations_without_caching(schema_relation) %}\n {% call statement('list_relations_without_caching', fetch_result=True) -%}\n select\n '{{ schema_relation.database }}' as database,\n tablename as name,\n schemaname as schema,\n 'table' as type\n from pg_tables\n where schemaname ilike '{{ schema_relation.schema }}'\n union all\n select\n '{{ schema_relation.database }}' as database,\n viewname as name,\n schemaname as schema,\n 'view' as type\n from pg_views\n where schemaname ilike '{{ schema_relation.schema }}'\n {% endcall %}\n {{ return(load_result('list_relations_without_caching').table) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.434568, "supported_languages": null}, "macro.dbt_postgres.postgres__information_schema_name": {"name": "postgres__information_schema_name", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__information_schema_name", "macro_sql": "{% macro postgres__information_schema_name(database) -%}\n {% if database_name -%}\n {{ adapter.verify_database(database_name) }}\n {%- endif -%}\n information_schema\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.4350138, "supported_languages": null}, "macro.dbt_postgres.postgres__list_schemas": {"name": "postgres__list_schemas", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__list_schemas", "macro_sql": "{% macro postgres__list_schemas(database) %}\n {% if database -%}\n {{ adapter.verify_database(database) }}\n {%- endif -%}\n {% call statement('list_schemas', fetch_result=True, auto_begin=False) %}\n select distinct nspname from pg_namespace\n {% endcall %}\n {{ return(load_result('list_schemas').table) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.435891, "supported_languages": null}, "macro.dbt_postgres.postgres__check_schema_exists": {"name": "postgres__check_schema_exists", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__check_schema_exists", "macro_sql": "{% macro postgres__check_schema_exists(information_schema, schema) -%}\n {% if information_schema.database -%}\n {{ adapter.verify_database(information_schema.database) }}\n {%- endif -%}\n {% call statement('check_schema_exists', fetch_result=True, auto_begin=False) %}\n select count(*) from pg_namespace where nspname = '{{ schema }}'\n {% endcall %}\n {{ return(load_result('check_schema_exists').table) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.436857, "supported_languages": null}, "macro.dbt_postgres.postgres__make_relation_with_suffix": {"name": "postgres__make_relation_with_suffix", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__make_relation_with_suffix", "macro_sql": "{% macro postgres__make_relation_with_suffix(base_relation, suffix, dstring) %}\n {% if dstring %}\n {% set dt = modules.datetime.datetime.now() %}\n {% set dtstring = dt.strftime(\"%H%M%S%f\") %}\n {% set suffix = suffix ~ dtstring %}\n {% endif %}\n {% set suffix_length = suffix|length %}\n {% set relation_max_name_length = base_relation.relation_max_name_length() %}\n {% if suffix_length > relation_max_name_length %}\n {% do exceptions.raise_compiler_error('Relation suffix is too long (' ~ suffix_length ~ ' characters). Maximum length is ' ~ relation_max_name_length ~ ' characters.') %}\n {% endif %}\n {% set identifier = base_relation.identifier[:relation_max_name_length - suffix_length] ~ suffix %}\n\n {{ return(base_relation.incorporate(path={\"identifier\": identifier })) }}\n\n {% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.4389682, "supported_languages": null}, "macro.dbt_postgres.postgres__make_intermediate_relation": {"name": "postgres__make_intermediate_relation", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__make_intermediate_relation", "macro_sql": "{% macro postgres__make_intermediate_relation(base_relation, suffix) %}\n {{ return(postgres__make_relation_with_suffix(base_relation, suffix, dstring=False)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_relation_with_suffix"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.439469, "supported_languages": null}, "macro.dbt_postgres.postgres__make_temp_relation": {"name": "postgres__make_temp_relation", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__make_temp_relation", "macro_sql": "{% macro postgres__make_temp_relation(base_relation, suffix) %}\n {% set temp_relation = postgres__make_relation_with_suffix(base_relation, suffix, dstring=True) %}\n {{ return(temp_relation.incorporate(path={\"schema\": none,\n \"database\": none})) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_relation_with_suffix"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.4407659, "supported_languages": null}, "macro.dbt_postgres.postgres__make_backup_relation": {"name": "postgres__make_backup_relation", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__make_backup_relation", "macro_sql": "{% macro postgres__make_backup_relation(base_relation, backup_relation_type, suffix) %}\n {% set backup_relation = postgres__make_relation_with_suffix(base_relation, suffix, dstring=False) %}\n {{ return(backup_relation.incorporate(type=backup_relation_type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_relation_with_suffix"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.441967, "supported_languages": null}, "macro.dbt_postgres.postgres_escape_comment": {"name": "postgres_escape_comment", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres_escape_comment", "macro_sql": "{% macro postgres_escape_comment(comment) -%}\n {% if comment is not string %}\n {% do exceptions.raise_compiler_error('cannot escape a non-string: ' ~ comment) %}\n {% endif %}\n {%- set magic = '$dbt_comment_literal_block$' -%}\n {%- if magic in comment -%}\n {%- do exceptions.raise_compiler_error('The string ' ~ magic ~ ' is not allowed in comments.') -%}\n {%- endif -%}\n {{ magic }}{{ comment }}{{ magic }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.443386, "supported_languages": null}, "macro.dbt_postgres.postgres__alter_relation_comment": {"name": "postgres__alter_relation_comment", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__alter_relation_comment", "macro_sql": "{% macro postgres__alter_relation_comment(relation, comment) %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n comment on {{ relation.type }} {{ relation }} is {{ escaped_comment }};\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres_escape_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.4441102, "supported_languages": null}, "macro.dbt_postgres.postgres__alter_column_comment": {"name": "postgres__alter_column_comment", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__alter_column_comment", "macro_sql": "{% macro postgres__alter_column_comment(relation, column_dict) %}\n {% set existing_columns = adapter.get_columns_in_relation(relation) | map(attribute=\"name\") | list %}\n {% for column_name in column_dict if (column_name in existing_columns) %}\n {% set comment = column_dict[column_name]['description'] %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n comment on column {{ relation }}.{{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }} is {{ escaped_comment }};\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres_escape_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.446302, "supported_languages": null}, "macro.dbt_postgres.postgres__get_show_grant_sql": {"name": "postgres__get_show_grant_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__get_show_grant_sql", "macro_sql": "\n\n{%- macro postgres__get_show_grant_sql(relation) -%}\n select grantee, privilege_type\n from {{ relation.information_schema('role_table_grants') }}\n where grantor = current_role\n and grantee != current_role\n and table_schema = '{{ relation.schema }}'\n and table_name = '{{ relation.identifier }}'\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.4471622, "supported_languages": null}, "macro.dbt_postgres.postgres__copy_grants": {"name": "postgres__copy_grants", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__copy_grants", "macro_sql": "{% macro postgres__copy_grants() %}\n {{ return(False) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.447847, "supported_languages": null}, "macro.dbt_postgres.postgres__get_incremental_default_sql": {"name": "postgres__get_incremental_default_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/materializations/incremental_strategies.sql", "original_file_path": "macros/materializations/incremental_strategies.sql", "unique_id": "macro.dbt_postgres.postgres__get_incremental_default_sql", "macro_sql": "{% macro postgres__get_incremental_default_sql(arg_dict) %}\n\n {% if arg_dict[\"unique_key\"] %}\n {% do return(get_incremental_delete_insert_sql(arg_dict)) %}\n {% else %}\n {% do return(get_incremental_append_sql(arg_dict)) %}\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_incremental_delete_insert_sql", "macro.dbt.get_incremental_append_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.449656, "supported_languages": null}, "macro.dbt_postgres.postgres__snapshot_merge_sql": {"name": "postgres__snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/materializations/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshot_merge.sql", "unique_id": "macro.dbt_postgres.postgres__snapshot_merge_sql", "macro_sql": "{% macro postgres__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n update {{ target }}\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n from {{ source }} as DBT_INTERNAL_SOURCE\n where DBT_INTERNAL_SOURCE.dbt_scd_id::text = {{ target }}.dbt_scd_id::text\n and DBT_INTERNAL_SOURCE.dbt_change_type::text in ('update'::text, 'delete'::text)\n and {{ target }}.dbt_valid_to is null;\n\n insert into {{ target }} ({{ insert_cols_csv }})\n select {% for column in insert_cols -%}\n DBT_INTERNAL_SOURCE.{{ column }} {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n from {{ source }} as DBT_INTERNAL_SOURCE\n where DBT_INTERNAL_SOURCE.dbt_change_type::text = 'insert'::text;\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.452299, "supported_languages": null}, "macro.dbt_postgres.postgres__dateadd": {"name": "postgres__dateadd", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt_postgres.postgres__dateadd", "macro_sql": "{% macro postgres__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n {{ from_date_or_timestamp }} + ((interval '1 {{ datepart }}') * ({{ interval }}))\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.453088, "supported_languages": null}, "macro.dbt_postgres.postgres__listagg": {"name": "postgres__listagg", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "unique_id": "macro.dbt_postgres.postgres__listagg", "macro_sql": "{% macro postgres__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n {% if limit_num -%}\n array_to_string(\n (array_agg(\n {{ measure }}\n {% if order_by_clause -%}\n {{ order_by_clause }}\n {%- endif %}\n ))[1:{{ limit_num }}],\n {{ delimiter_text }}\n )\n {%- else %}\n string_agg(\n {{ measure }},\n {{ delimiter_text }}\n {% if order_by_clause -%}\n {{ order_by_clause }}\n {%- endif %}\n )\n {%- endif %}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.455125, "supported_languages": null}, "macro.dbt_postgres.postgres__datediff": {"name": "postgres__datediff", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt_postgres.postgres__datediff", "macro_sql": "{% macro postgres__datediff(first_date, second_date, datepart) -%}\n\n {% if datepart == 'year' %}\n (date_part('year', ({{second_date}})::date) - date_part('year', ({{first_date}})::date))\n {% elif datepart == 'quarter' %}\n ({{ datediff(first_date, second_date, 'year') }} * 4 + date_part('quarter', ({{second_date}})::date) - date_part('quarter', ({{first_date}})::date))\n {% elif datepart == 'month' %}\n ({{ datediff(first_date, second_date, 'year') }} * 12 + date_part('month', ({{second_date}})::date) - date_part('month', ({{first_date}})::date))\n {% elif datepart == 'day' %}\n (({{second_date}})::date - ({{first_date}})::date)\n {% elif datepart == 'week' %}\n ({{ datediff(first_date, second_date, 'day') }} / 7 + case\n when date_part('dow', ({{first_date}})::timestamp) <= date_part('dow', ({{second_date}})::timestamp) then\n case when {{first_date}} <= {{second_date}} then 0 else -1 end\n else\n case when {{first_date}} <= {{second_date}} then 1 else 0 end\n end)\n {% elif datepart == 'hour' %}\n ({{ datediff(first_date, second_date, 'day') }} * 24 + date_part('hour', ({{second_date}})::timestamp) - date_part('hour', ({{first_date}})::timestamp))\n {% elif datepart == 'minute' %}\n ({{ datediff(first_date, second_date, 'hour') }} * 60 + date_part('minute', ({{second_date}})::timestamp) - date_part('minute', ({{first_date}})::timestamp))\n {% elif datepart == 'second' %}\n ({{ datediff(first_date, second_date, 'minute') }} * 60 + floor(date_part('second', ({{second_date}})::timestamp)) - floor(date_part('second', ({{first_date}})::timestamp)))\n {% elif datepart == 'millisecond' %}\n ({{ datediff(first_date, second_date, 'minute') }} * 60000 + floor(date_part('millisecond', ({{second_date}})::timestamp)) - floor(date_part('millisecond', ({{first_date}})::timestamp)))\n {% elif datepart == 'microsecond' %}\n ({{ datediff(first_date, second_date, 'minute') }} * 60000000 + floor(date_part('microsecond', ({{second_date}})::timestamp)) - floor(date_part('microsecond', ({{first_date}})::timestamp)))\n {% else %}\n {{ exceptions.raise_compiler_error(\"Unsupported datepart for macro datediff in postgres: {!r}\".format(datepart)) }}\n {% endif %}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.datediff"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.462395, "supported_languages": null}, "macro.dbt_postgres.postgres__any_value": {"name": "postgres__any_value", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "unique_id": "macro.dbt_postgres.postgres__any_value", "macro_sql": "{% macro postgres__any_value(expression) -%}\n\n min({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.463126, "supported_languages": null}, "macro.dbt_postgres.postgres__last_day": {"name": "postgres__last_day", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt_postgres.postgres__last_day", "macro_sql": "{% macro postgres__last_day(date, datepart) -%}\n\n {%- if datepart == 'quarter' -%}\n -- postgres dateadd does not support quarter interval.\n cast(\n {{dbt.dateadd('day', '-1',\n dbt.dateadd('month', '3', dbt.date_trunc(datepart, date))\n )}}\n as date)\n {%- else -%}\n {{dbt.default_last_day(date, datepart)}}\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.dateadd", "macro.dbt.date_trunc", "macro.dbt.default_last_day"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.464517, "supported_languages": null}, "macro.dbt_postgres.postgres__split_part": {"name": "postgres__split_part", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt_postgres.postgres__split_part", "macro_sql": "{% macro postgres__split_part(string_text, delimiter_text, part_number) %}\n\n {% if part_number >= 0 %}\n {{ dbt.default__split_part(string_text, delimiter_text, part_number) }}\n {% else %}\n {{ dbt._split_part_negative(string_text, delimiter_text, part_number) }}\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__split_part", "macro.dbt._split_part_negative"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.4655108, "supported_languages": null}, "macro.dbt.run_hooks": {"name": "run_hooks", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.run_hooks", "macro_sql": "{% macro run_hooks(hooks, inside_transaction=True) %}\n {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %}\n {% if not inside_transaction and loop.first %}\n {% call statement(auto_begin=inside_transaction) %}\n commit;\n {% endcall %}\n {% endif %}\n {% set rendered = render(hook.get('sql')) | trim %}\n {% if (rendered | length) > 0 %}\n {% call statement(auto_begin=inside_transaction) %}\n {{ rendered }}\n {% endcall %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.468192, "supported_languages": null}, "macro.dbt.make_hook_config": {"name": "make_hook_config", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.make_hook_config", "macro_sql": "{% macro make_hook_config(sql, inside_transaction) %}\n {{ tojson({\"sql\": sql, \"transaction\": inside_transaction}) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.468693, "supported_languages": null}, "macro.dbt.before_begin": {"name": "before_begin", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.before_begin", "macro_sql": "{% macro before_begin(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.469058, "supported_languages": null}, "macro.dbt.in_transaction": {"name": "in_transaction", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.in_transaction", "macro_sql": "{% macro in_transaction(sql) %}\n {{ make_hook_config(sql, inside_transaction=True) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.4694211, "supported_languages": null}, "macro.dbt.after_commit": {"name": "after_commit", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.after_commit", "macro_sql": "{% macro after_commit(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.469785, "supported_languages": null}, "macro.dbt.set_sql_header": {"name": "set_sql_header", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "unique_id": "macro.dbt.set_sql_header", "macro_sql": "{% macro set_sql_header(config) -%}\n {{ config.set('sql_header', caller()) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.470596, "supported_languages": null}, "macro.dbt.should_full_refresh": {"name": "should_full_refresh", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "unique_id": "macro.dbt.should_full_refresh", "macro_sql": "{% macro should_full_refresh() %}\n {% set config_full_refresh = config.get('full_refresh') %}\n {% if config_full_refresh is none %}\n {% set config_full_refresh = flags.FULL_REFRESH %}\n {% endif %}\n {% do return(config_full_refresh) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.471361, "supported_languages": null}, "macro.dbt.should_store_failures": {"name": "should_store_failures", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "unique_id": "macro.dbt.should_store_failures", "macro_sql": "{% macro should_store_failures() %}\n {% set config_store_failures = config.get('store_failures') %}\n {% if config_store_failures is none %}\n {% set config_store_failures = flags.STORE_FAILURES %}\n {% endif %}\n {% do return(config_store_failures) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.472131, "supported_languages": null}, "macro.dbt.snapshot_merge_sql": {"name": "snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", "unique_id": "macro.dbt.snapshot_merge_sql", "macro_sql": "{% macro snapshot_merge_sql(target, source, insert_cols) -%}\n {{ adapter.dispatch('snapshot_merge_sql', 'dbt')(target, source, insert_cols) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__snapshot_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.473068, "supported_languages": null}, "macro.dbt.default__snapshot_merge_sql": {"name": "default__snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", "unique_id": "macro.dbt.default__snapshot_merge_sql", "macro_sql": "{% macro default__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on DBT_INTERNAL_SOURCE.dbt_scd_id = DBT_INTERNAL_DEST.dbt_scd_id\n\n when matched\n and DBT_INTERNAL_DEST.dbt_valid_to is null\n and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')\n then update\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n\n when not matched\n and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'\n then insert ({{ insert_cols_csv }})\n values ({{ insert_cols_csv }})\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.4737349, "supported_languages": null}, "macro.dbt.strategy_dispatch": {"name": "strategy_dispatch", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.strategy_dispatch", "macro_sql": "{% macro strategy_dispatch(name) -%}\n{% set original_name = name %}\n {% if '.' in name %}\n {% set package_name, name = name.split(\".\", 1) %}\n {% else %}\n {% set package_name = none %}\n {% endif %}\n\n {% if package_name is none %}\n {% set package_context = context %}\n {% elif package_name in context %}\n {% set package_context = context[package_name] %}\n {% else %}\n {% set error_msg %}\n Could not find package '{{package_name}}', called with '{{original_name}}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n\n {%- set search_name = 'snapshot_' ~ name ~ '_strategy' -%}\n\n {% if search_name not in package_context %}\n {% set error_msg %}\n The specified strategy macro '{{name}}' was not found in package '{{ package_name }}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n {{ return(package_context[search_name]) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.481837, "supported_languages": null}, "macro.dbt.snapshot_hash_arguments": {"name": "snapshot_hash_arguments", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_hash_arguments", "macro_sql": "{% macro snapshot_hash_arguments(args) -%}\n {{ adapter.dispatch('snapshot_hash_arguments', 'dbt')(args) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.4828649, "supported_languages": null}, "macro.dbt.default__snapshot_hash_arguments": {"name": "default__snapshot_hash_arguments", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.default__snapshot_hash_arguments", "macro_sql": "{% macro default__snapshot_hash_arguments(args) -%}\n md5({%- for arg in args -%}\n coalesce(cast({{ arg }} as varchar ), '')\n {% if not loop.last %} || '|' || {% endif %}\n {%- endfor -%})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.4835358, "supported_languages": null}, "macro.dbt.snapshot_timestamp_strategy": {"name": "snapshot_timestamp_strategy", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_timestamp_strategy", "macro_sql": "{% macro snapshot_timestamp_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}\n {% set primary_key = config['unique_key'] %}\n {% set updated_at = config['updated_at'] %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %}\n\n {#/*\n The snapshot relation might not have an {{ updated_at }} value if the\n snapshot strategy is changed from `check` to `timestamp`. We\n should use a dbt-created column for the comparison in the snapshot\n table instead of assuming that the user-supplied {{ updated_at }}\n will be present in the historical data.\n\n See https://github.com/dbt-labs/dbt-core/issues/2350\n */ #}\n {% set row_changed_expr -%}\n ({{ snapshotted_rel }}.dbt_valid_from < {{ current_rel }}.{{ updated_at }})\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.4857202, "supported_languages": null}, "macro.dbt.snapshot_string_as_time": {"name": "snapshot_string_as_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_string_as_time", "macro_sql": "{% macro snapshot_string_as_time(timestamp) -%}\n {{ adapter.dispatch('snapshot_string_as_time', 'dbt')(timestamp) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__snapshot_string_as_time"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.4861922, "supported_languages": null}, "macro.dbt.default__snapshot_string_as_time": {"name": "default__snapshot_string_as_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.default__snapshot_string_as_time", "macro_sql": "{% macro default__snapshot_string_as_time(timestamp) %}\n {% do exceptions.raise_not_implemented(\n 'snapshot_string_as_time macro not implemented for adapter '+adapter.type()\n ) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.486644, "supported_languages": null}, "macro.dbt.snapshot_check_all_get_existing_columns": {"name": "snapshot_check_all_get_existing_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_check_all_get_existing_columns", "macro_sql": "{% macro snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) -%}\n {%- if not target_exists -%}\n {#-- no table yet -> return whatever the query does --#}\n {{ return((false, query_columns)) }}\n {%- endif -%}\n\n {#-- handle any schema changes --#}\n {%- set target_relation = adapter.get_relation(database=node.database, schema=node.schema, identifier=node.alias) -%}\n\n {% if check_cols_config == 'all' %}\n {%- set query_columns = get_columns_in_query(node['compiled_code']) -%}\n\n {% elif check_cols_config is iterable and (check_cols_config | length) > 0 %}\n {#-- query for proper casing/quoting, to support comparison below --#}\n {%- set select_check_cols_from_target -%}\n select {{ check_cols_config | join(', ') }} from ({{ node['compiled_code'] }}) subq\n {%- endset -%}\n {% set query_columns = get_columns_in_query(select_check_cols_from_target) %}\n\n {% else %}\n {% do exceptions.raise_compiler_error(\"Invalid value for 'check_cols': \" ~ check_cols_config) %}\n {% endif %}\n\n {%- set existing_cols = adapter.get_columns_in_relation(target_relation) | map(attribute = 'name') | list -%}\n {%- set ns = namespace() -%} {#-- handle for-loop scoping with a namespace --#}\n {%- set ns.column_added = false -%}\n\n {%- set intersection = [] -%}\n {%- for col in query_columns -%}\n {%- if col in existing_cols -%}\n {%- do intersection.append(adapter.quote(col)) -%}\n {%- else -%}\n {% set ns.column_added = true %}\n {%- endif -%}\n {%- endfor -%}\n {{ return((ns.column_added, intersection)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_columns_in_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.490088, "supported_languages": null}, "macro.dbt.snapshot_check_strategy": {"name": "snapshot_check_strategy", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_check_strategy", "macro_sql": "{% macro snapshot_check_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}\n {% set check_cols_config = config['check_cols'] %}\n {% set primary_key = config['unique_key'] %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %}\n {% set updated_at = config.get('updated_at', snapshot_get_time()) %}\n\n {% set column_added = false %}\n\n {% set column_added, check_cols = snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) %}\n\n {%- set row_changed_expr -%}\n (\n {%- if column_added -%}\n {{ get_true_sql() }}\n {%- else -%}\n {%- for col in check_cols -%}\n {{ snapshotted_rel }}.{{ col }} != {{ current_rel }}.{{ col }}\n or\n (\n (({{ snapshotted_rel }}.{{ col }} is null) and not ({{ current_rel }}.{{ col }} is null))\n or\n ((not {{ snapshotted_rel }}.{{ col }} is null) and ({{ current_rel }}.{{ col }} is null))\n )\n {%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n {%- endif -%}\n )\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.snapshot_get_time", "macro.dbt.snapshot_check_all_get_existing_columns", "macro.dbt.get_true_sql", "macro.dbt.snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.4935129, "supported_languages": null}, "macro.dbt.create_columns": {"name": "create_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.create_columns", "macro_sql": "{% macro create_columns(relation, columns) %}\n {{ adapter.dispatch('create_columns', 'dbt')(relation, columns) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__create_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.5012, "supported_languages": null}, "macro.dbt.default__create_columns": {"name": "default__create_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__create_columns", "macro_sql": "{% macro default__create_columns(relation, columns) %}\n {% for column in columns %}\n {% call statement() %}\n alter table {{ relation }} add column \"{{ column.name }}\" {{ column.data_type }};\n {% endcall %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.501941, "supported_languages": null}, "macro.dbt.post_snapshot": {"name": "post_snapshot", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.post_snapshot", "macro_sql": "{% macro post_snapshot(staging_relation) %}\n {{ adapter.dispatch('post_snapshot', 'dbt')(staging_relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.502363, "supported_languages": null}, "macro.dbt.default__post_snapshot": {"name": "default__post_snapshot", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__post_snapshot", "macro_sql": "{% macro default__post_snapshot(staging_relation) %}\n {# no-op #}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.5025961, "supported_languages": null}, "macro.dbt.get_true_sql": {"name": "get_true_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.get_true_sql", "macro_sql": "{% macro get_true_sql() %}\n {{ adapter.dispatch('get_true_sql', 'dbt')() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_true_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.502961, "supported_languages": null}, "macro.dbt.default__get_true_sql": {"name": "default__get_true_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__get_true_sql", "macro_sql": "{% macro default__get_true_sql() %}\n {{ return('TRUE') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.5032582, "supported_languages": null}, "macro.dbt.snapshot_staging_table": {"name": "snapshot_staging_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.snapshot_staging_table", "macro_sql": "{% macro snapshot_staging_table(strategy, source_sql, target_relation) -%}\n {{ adapter.dispatch('snapshot_staging_table', 'dbt')(strategy, source_sql, target_relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__snapshot_staging_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.50385, "supported_languages": null}, "macro.dbt.default__snapshot_staging_table": {"name": "default__snapshot_staging_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__snapshot_staging_table", "macro_sql": "{% macro default__snapshot_staging_table(strategy, source_sql, target_relation) -%}\n\n with snapshot_query as (\n\n {{ source_sql }}\n\n ),\n\n snapshotted_data as (\n\n select *,\n {{ strategy.unique_key }} as dbt_unique_key\n\n from {{ target_relation }}\n where dbt_valid_to is null\n\n ),\n\n insertions_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to,\n {{ strategy.scd_id }} as dbt_scd_id\n\n from snapshot_query\n ),\n\n updates_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n {{ strategy.updated_at }} as dbt_valid_to\n\n from snapshot_query\n ),\n\n {%- if strategy.invalidate_hard_deletes %}\n\n deletes_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key\n from snapshot_query\n ),\n {% endif %}\n\n insertions as (\n\n select\n 'insert' as dbt_change_type,\n source_data.*\n\n from insertions_source_data as source_data\n left outer join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where snapshotted_data.dbt_unique_key is null\n or (\n snapshotted_data.dbt_unique_key is not null\n and (\n {{ strategy.row_changed }}\n )\n )\n\n ),\n\n updates as (\n\n select\n 'update' as dbt_change_type,\n source_data.*,\n snapshotted_data.dbt_scd_id\n\n from updates_source_data as source_data\n join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where (\n {{ strategy.row_changed }}\n )\n )\n\n {%- if strategy.invalidate_hard_deletes -%}\n ,\n\n deletes as (\n\n select\n 'delete' as dbt_change_type,\n source_data.*,\n {{ snapshot_get_time() }} as dbt_valid_from,\n {{ snapshot_get_time() }} as dbt_updated_at,\n {{ snapshot_get_time() }} as dbt_valid_to,\n snapshotted_data.dbt_scd_id\n\n from snapshotted_data\n left join deletes_source_data as source_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where source_data.dbt_unique_key is null\n )\n {%- endif %}\n\n select * from insertions\n union all\n select * from updates\n {%- if strategy.invalidate_hard_deletes %}\n union all\n select * from deletes\n {%- endif %}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.snapshot_get_time"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.5059588, "supported_languages": null}, "macro.dbt.build_snapshot_table": {"name": "build_snapshot_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.build_snapshot_table", "macro_sql": "{% macro build_snapshot_table(strategy, sql) -%}\n {{ adapter.dispatch('build_snapshot_table', 'dbt')(strategy, sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__build_snapshot_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.5064478, "supported_languages": null}, "macro.dbt.default__build_snapshot_table": {"name": "default__build_snapshot_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__build_snapshot_table", "macro_sql": "{% macro default__build_snapshot_table(strategy, sql) %}\n\n select *,\n {{ strategy.scd_id }} as dbt_scd_id,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to\n from (\n {{ sql }}\n ) sbq\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.5070798, "supported_languages": null}, "macro.dbt.build_snapshot_staging_table": {"name": "build_snapshot_staging_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.build_snapshot_staging_table", "macro_sql": "{% macro build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% set temp_relation = make_temp_relation(target_relation) %}\n\n {% set select = snapshot_staging_table(strategy, sql, target_relation) %}\n\n {% call statement('build_snapshot_staging_relation') %}\n {{ create_table_as(True, temp_relation, select) }}\n {% endcall %}\n\n {% do return(temp_relation) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_temp_relation", "macro.dbt.snapshot_staging_table", "macro.dbt.statement", "macro.dbt.create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.508459, "supported_languages": null}, "macro.dbt.materialization_snapshot_default": {"name": "materialization_snapshot_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/snapshot.sql", "original_file_path": "macros/materializations/snapshots/snapshot.sql", "unique_id": "macro.dbt.materialization_snapshot_default", "macro_sql": "{% materialization snapshot, default %}\n {%- set config = model['config'] -%}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n -- grab current tables grants config for comparision later on\n {%- set grant_config = config.get('grants') -%}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=model.database,\n schema=model.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", config, target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_code']) %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% else %}\n\n {{ adapter.valid_snapshot_target(target_relation) }}\n\n {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% endif %}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode=False) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if not target_relation_exists %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if staging_table is defined %}\n {% do post_snapshot(staging_table) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.get_or_create_relation", "macro.dbt.run_hooks", "macro.dbt.strategy_dispatch", "macro.dbt.build_snapshot_table", "macro.dbt.create_table_as", "macro.dbt.build_snapshot_staging_table", "macro.dbt.create_columns", "macro.dbt.snapshot_merge_sql", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes", "macro.dbt.post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.523073, "supported_languages": ["sql"]}, "macro.dbt.materialization_test_default": {"name": "materialization_test_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/test.sql", "original_file_path": "macros/materializations/tests/test.sql", "unique_id": "macro.dbt.materialization_test_default", "macro_sql": "{%- materialization test, default -%}\n\n {% set relations = [] %}\n\n {% if should_store_failures() %}\n\n {% set identifier = model['alias'] %}\n {% set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n {% set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database, type='table') -%} %}\n\n {% if old_relation %}\n {% do adapter.drop_relation(old_relation) %}\n {% endif %}\n\n {% call statement(auto_begin=True) %}\n {{ create_table_as(False, target_relation, sql) }}\n {% endcall %}\n\n {% do relations.append(target_relation) %}\n\n {% set main_sql %}\n select *\n from {{ target_relation }}\n {% endset %}\n\n {{ adapter.commit() }}\n\n {% else %}\n\n {% set main_sql = sql %}\n\n {% endif %}\n\n {% set limit = config.get('limit') %}\n {% set fail_calc = config.get('fail_calc') %}\n {% set warn_if = config.get('warn_if') %}\n {% set error_if = config.get('error_if') %}\n\n {% call statement('main', fetch_result=True) -%}\n\n {{ get_test_sql(main_sql, fail_calc, warn_if, error_if, limit)}}\n\n {%- endcall %}\n\n {{ return({'relations': relations}) }}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.should_store_failures", "macro.dbt.statement", "macro.dbt.create_table_as", "macro.dbt.get_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.528616, "supported_languages": ["sql"]}, "macro.dbt.get_test_sql": {"name": "get_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.get_test_sql", "macro_sql": "{% macro get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n {{ adapter.dispatch('get_test_sql', 'dbt')(main_sql, fail_calc, warn_if, error_if, limit) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.52976, "supported_languages": null}, "macro.dbt.default__get_test_sql": {"name": "default__get_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.default__get_test_sql", "macro_sql": "{% macro default__get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n select\n {{ fail_calc }} as failures,\n {{ fail_calc }} {{ warn_if }} as should_warn,\n {{ fail_calc }} {{ error_if }} as should_error\n from (\n {{ main_sql }}\n {{ \"limit \" ~ limit if limit != none }}\n ) dbt_internal_test\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.531104, "supported_languages": null}, "macro.dbt.get_where_subquery": {"name": "get_where_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/where_subquery.sql", "original_file_path": "macros/materializations/tests/where_subquery.sql", "unique_id": "macro.dbt.get_where_subquery", "macro_sql": "{% macro get_where_subquery(relation) -%}\n {% do return(adapter.dispatch('get_where_subquery', 'dbt')(relation)) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_where_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.5320342, "supported_languages": null}, "macro.dbt.default__get_where_subquery": {"name": "default__get_where_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/where_subquery.sql", "original_file_path": "macros/materializations/tests/where_subquery.sql", "unique_id": "macro.dbt.default__get_where_subquery", "macro_sql": "{% macro default__get_where_subquery(relation) -%}\n {% set where = config.get('where', '') %}\n {% if where %}\n {%- set filtered -%}\n (select * from {{ relation }} where {{ where }}) dbt_subquery\n {%- endset -%}\n {% do return(filtered) %}\n {%- else -%}\n {% do return(relation) %}\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.533005, "supported_languages": null}, "macro.dbt.get_quoted_csv": {"name": "get_quoted_csv", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.get_quoted_csv", "macro_sql": "{% macro get_quoted_csv(column_names) %}\n\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote(col)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.536449, "supported_languages": null}, "macro.dbt.diff_columns": {"name": "diff_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.diff_columns", "macro_sql": "{% macro diff_columns(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% set source_names = source_columns | map(attribute = 'column') | list %}\n {% set target_names = target_columns | map(attribute = 'column') | list %}\n\n {# --check whether the name attribute exists in the target - this does not perform a data type check #}\n {% for sc in source_columns %}\n {% if sc.name not in target_names %}\n {{ result.append(sc) }}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.537867, "supported_languages": null}, "macro.dbt.diff_column_data_types": {"name": "diff_column_data_types", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.diff_column_data_types", "macro_sql": "{% macro diff_column_data_types(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% for sc in source_columns %}\n {% set tc = target_columns | selectattr(\"name\", \"equalto\", sc.name) | list | first %}\n {% if tc %}\n {% if sc.data_type != tc.data_type and not sc.can_expand_to(other_column=tc) %}\n {{ result.append( { 'column_name': tc.name, 'new_type': sc.data_type } ) }}\n {% endif %}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.539545, "supported_languages": null}, "macro.dbt.get_merge_update_columns": {"name": "get_merge_update_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.get_merge_update_columns", "macro_sql": "{% macro get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {{ return(adapter.dispatch('get_merge_update_columns', 'dbt')(merge_update_columns, merge_exclude_columns, dest_columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_merge_update_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.5401359, "supported_languages": null}, "macro.dbt.default__get_merge_update_columns": {"name": "default__get_merge_update_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.default__get_merge_update_columns", "macro_sql": "{% macro default__get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {%- set default_cols = dest_columns | map(attribute=\"quoted\") | list -%}\n\n {%- if merge_update_columns and merge_exclude_columns -%}\n {{ exceptions.raise_compiler_error(\n 'Model cannot specify merge_update_columns and merge_exclude_columns. Please update model to use only one config'\n )}}\n {%- elif merge_update_columns -%}\n {%- set update_columns = merge_update_columns -%}\n {%- elif merge_exclude_columns -%}\n {%- set update_columns = [] -%}\n {%- for column in dest_columns -%}\n {% if column.column | lower not in merge_exclude_columns | map(\"lower\") | list %}\n {%- do update_columns.append(column.quoted) -%}\n {% endif %}\n {%- endfor -%}\n {%- else -%}\n {%- set update_columns = default_cols -%}\n {%- endif -%}\n\n {{ return(update_columns) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.5423229, "supported_languages": null}, "macro.dbt.get_merge_sql": {"name": "get_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.get_merge_sql", "macro_sql": "{% macro get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) -%}\n -- back compat for old kwarg name\n {% set incremental_predicates = kwargs.get('predicates', incremental_predicates) %}\n {{ adapter.dispatch('get_merge_sql', 'dbt')(target, source, unique_key, dest_columns, incremental_predicates) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.554441, "supported_languages": null}, "macro.dbt.default__get_merge_sql": {"name": "default__get_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.default__get_merge_sql", "macro_sql": "{% macro default__get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) -%}\n {%- set predicates = [] if incremental_predicates is none else [] + incremental_predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set merge_update_columns = config.get('merge_update_columns') -%}\n {%- set merge_exclude_columns = config.get('merge_exclude_columns') -%}\n {%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}\n {% for key in unique_key %}\n {% set this_key_match %}\n DBT_INTERNAL_SOURCE.{{ key }} = DBT_INTERNAL_DEST.{{ key }}\n {% endset %}\n {% do predicates.append(this_key_match) %}\n {% endfor %}\n {% else %}\n {% set unique_key_match %}\n DBT_INTERNAL_SOURCE.{{ unique_key }} = DBT_INTERNAL_DEST.{{ unique_key }}\n {% endset %}\n {% do predicates.append(unique_key_match) %}\n {% endif %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n {{ sql_header if sql_header is not none }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{\"(\" ~ predicates | join(\") and (\") ~ \")\"}}\n\n {% if unique_key %}\n when matched then update set\n {% for column_name in update_columns -%}\n {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}\n {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n {% endif %}\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv", "macro.dbt.get_merge_update_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.558579, "supported_languages": null}, "macro.dbt.get_delete_insert_merge_sql": {"name": "get_delete_insert_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.get_delete_insert_merge_sql", "macro_sql": "{% macro get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}\n {{ adapter.dispatch('get_delete_insert_merge_sql', 'dbt')(target, source, unique_key, dest_columns, incremental_predicates) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_delete_insert_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.559229, "supported_languages": null}, "macro.dbt.default__get_delete_insert_merge_sql": {"name": "default__get_delete_insert_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.default__get_delete_insert_merge_sql", "macro_sql": "{% macro default__get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not string %}\n delete from {{target }}\n using {{ source }}\n where (\n {% for key in unique_key %}\n {{ source }}.{{ key }} = {{ target }}.{{ key }}\n {{ \"and \" if not loop.last}}\n {% endfor %}\n {% if incremental_predicates %}\n {% for predicate in incremental_predicates %}\n and {{ predicate }}\n {% endfor %}\n {% endif %}\n );\n {% else %}\n delete from {{ target }}\n where (\n {{ unique_key }}) in (\n select ({{ unique_key }})\n from {{ source }}\n )\n {%- if incremental_predicates %}\n {% for predicate in incremental_predicates %}\n and {{ predicate }}\n {% endfor %}\n {%- endif -%};\n\n {% endif %}\n {% endif %}\n\n insert into {{ target }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ source }}\n )\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.561732, "supported_languages": null}, "macro.dbt.get_insert_overwrite_merge_sql": {"name": "get_insert_overwrite_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.get_insert_overwrite_merge_sql", "macro_sql": "{% macro get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header=false) -%}\n {{ adapter.dispatch('get_insert_overwrite_merge_sql', 'dbt')(target, source, dest_columns, predicates, include_sql_header) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_insert_overwrite_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.56239, "supported_languages": null}, "macro.dbt.default__get_insert_overwrite_merge_sql": {"name": "default__get_insert_overwrite_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.default__get_insert_overwrite_merge_sql", "macro_sql": "{% macro default__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) -%}\n {#-- The only time include_sql_header is True: --#}\n {#-- BigQuery + insert_overwrite strategy + \"static\" partitions config --#}\n {#-- We should consider including the sql header at the materialization level instead --#}\n\n {%- set predicates = [] if predicates is none else [] + predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none and include_sql_header }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on FALSE\n\n when not matched by source\n {% if predicates %} and {{ predicates | join(' and ') }} {% endif %}\n then delete\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.564075, "supported_languages": null}, "macro.dbt.is_incremental": {"name": "is_incremental", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/is_incremental.sql", "original_file_path": "macros/materializations/models/incremental/is_incremental.sql", "unique_id": "macro.dbt.is_incremental", "macro_sql": "{% macro is_incremental() %}\n {#-- do not run introspective queries in parsing #}\n {% if not execute %}\n {{ return(False) }}\n {% else %}\n {% set relation = adapter.get_relation(this.database, this.schema, this.table) %}\n {{ return(relation is not none\n and relation.type == 'table'\n and model.config.materialized == 'incremental'\n and not should_full_refresh()) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.565938, "supported_languages": null}, "macro.dbt.get_incremental_append_sql": {"name": "get_incremental_append_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_append_sql", "macro_sql": "{% macro get_incremental_append_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_append_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_append_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.567849, "supported_languages": null}, "macro.dbt.default__get_incremental_append_sql": {"name": "default__get_incremental_append_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_append_sql", "macro_sql": "{% macro default__get_incremental_append_sql(arg_dict) %}\n\n {% do return(get_insert_into_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"dest_columns\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_insert_into_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.568505, "supported_languages": null}, "macro.dbt.get_incremental_delete_insert_sql": {"name": "get_incremental_delete_insert_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_delete_insert_sql", "macro_sql": "{% macro get_incremental_delete_insert_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_delete_insert_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_delete_insert_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.5689778, "supported_languages": null}, "macro.dbt.default__get_incremental_delete_insert_sql": {"name": "default__get_incremental_delete_insert_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_delete_insert_sql", "macro_sql": "{% macro default__get_incremental_delete_insert_sql(arg_dict) %}\n\n {% do return(get_delete_insert_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"unique_key\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_delete_insert_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.569709, "supported_languages": null}, "macro.dbt.get_incremental_merge_sql": {"name": "get_incremental_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_merge_sql", "macro_sql": "{% macro get_incremental_merge_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_merge_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.570188, "supported_languages": null}, "macro.dbt.default__get_incremental_merge_sql": {"name": "default__get_incremental_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_merge_sql", "macro_sql": "{% macro default__get_incremental_merge_sql(arg_dict) %}\n\n {% do return(get_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"unique_key\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.5709162, "supported_languages": null}, "macro.dbt.get_incremental_insert_overwrite_sql": {"name": "get_incremental_insert_overwrite_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_insert_overwrite_sql", "macro_sql": "{% macro get_incremental_insert_overwrite_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_insert_overwrite_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_insert_overwrite_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.57139, "supported_languages": null}, "macro.dbt.default__get_incremental_insert_overwrite_sql": {"name": "default__get_incremental_insert_overwrite_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_insert_overwrite_sql", "macro_sql": "{% macro default__get_incremental_insert_overwrite_sql(arg_dict) %}\n\n {% do return(get_insert_overwrite_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_insert_overwrite_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.572049, "supported_languages": null}, "macro.dbt.get_incremental_default_sql": {"name": "get_incremental_default_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_default_sql", "macro_sql": "{% macro get_incremental_default_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_default_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_incremental_default_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.572522, "supported_languages": null}, "macro.dbt.default__get_incremental_default_sql": {"name": "default__get_incremental_default_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_default_sql", "macro_sql": "{% macro default__get_incremental_default_sql(arg_dict) %}\n\n {% do return(get_incremental_append_sql(arg_dict)) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_incremental_append_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.5729191, "supported_languages": null}, "macro.dbt.get_insert_into_sql": {"name": "get_insert_into_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_insert_into_sql", "macro_sql": "{% macro get_insert_into_sql(target_relation, temp_relation, dest_columns) %}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n insert into {{ target_relation }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ temp_relation }}\n )\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.57362, "supported_languages": null}, "macro.dbt.materialization_incremental_default": {"name": "materialization_incremental_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/incremental.sql", "original_file_path": "macros/materializations/models/incremental/incremental.sql", "unique_id": "macro.dbt.materialization_incremental_default", "macro_sql": "{% materialization incremental, default -%}\n\n -- relations\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') -%}\n {%- set temp_relation = make_temp_relation(target_relation)-%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation)-%}\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n\n -- configs\n {%- set unique_key = config.get('unique_key') -%}\n {%- set full_refresh_mode = (should_full_refresh() or existing_relation.is_view) -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n\n -- the temp_ and backup_ relations should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation. This has to happen before\n -- BEGIN, in a separate transaction\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation)-%}\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set to_drop = [] %}\n\n {% if existing_relation is none %}\n {% set build_sql = get_create_table_as_sql(False, target_relation, sql) %}\n {% elif full_refresh_mode %}\n {% set build_sql = get_create_table_as_sql(False, intermediate_relation, sql) %}\n {% set need_swap = true %}\n {% else %}\n {% do run_query(get_create_table_as_sql(True, temp_relation, sql)) %}\n {% do adapter.expand_target_column_types(\n from_relation=temp_relation,\n to_relation=target_relation) %}\n {#-- Process schema changes. Returns dict of changes if successful. Use source columns for upserting/merging --#}\n {% set dest_columns = process_schema_changes(on_schema_change, temp_relation, existing_relation) %}\n {% if not dest_columns %}\n {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %}\n {% endif %}\n\n {#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#}\n {% set incremental_strategy = config.get('incremental_strategy') or 'default' %}\n {% set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) %}\n {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %}\n {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': temp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'incremental_predicates': incremental_predicates }) %}\n {% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %}\n\n {% endif %}\n\n {% call statement(\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% if need_swap %}\n {% do adapter.rename_relation(target_relation, backup_relation) %}\n {% do adapter.rename_relation(intermediate_relation, target_relation) %}\n {% do to_drop.append(backup_relation) %}\n {% endif %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if existing_relation is none or existing_relation.is_view or should_full_refresh() %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {% do adapter.commit() %}\n\n {% for rel in to_drop %}\n {% do adapter.drop_relation(rel) %}\n {% endfor %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_temp_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.should_full_refresh", "macro.dbt.incremental_validate_on_schema_change", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks", "macro.dbt.get_create_table_as_sql", "macro.dbt.run_query", "macro.dbt.process_schema_changes", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.585784, "supported_languages": ["sql"]}, "macro.dbt.incremental_validate_on_schema_change": {"name": "incremental_validate_on_schema_change", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.incremental_validate_on_schema_change", "macro_sql": "{% macro incremental_validate_on_schema_change(on_schema_change, default='ignore') %}\n\n {% if on_schema_change not in ['sync_all_columns', 'append_new_columns', 'fail', 'ignore'] %}\n\n {% set log_message = 'Invalid value for on_schema_change (%s) specified. Setting default value of %s.' % (on_schema_change, default) %}\n {% do log(log_message) %}\n\n {{ return(default) }}\n\n {% else %}\n\n {{ return(on_schema_change) }}\n\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.59618, "supported_languages": null}, "macro.dbt.check_for_schema_changes": {"name": "check_for_schema_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.check_for_schema_changes", "macro_sql": "{% macro check_for_schema_changes(source_relation, target_relation) %}\n\n {% set schema_changed = False %}\n\n {%- set source_columns = adapter.get_columns_in_relation(source_relation) -%}\n {%- set target_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set source_not_in_target = diff_columns(source_columns, target_columns) -%}\n {%- set target_not_in_source = diff_columns(target_columns, source_columns) -%}\n\n {% set new_target_types = diff_column_data_types(source_columns, target_columns) %}\n\n {% if source_not_in_target != [] %}\n {% set schema_changed = True %}\n {% elif target_not_in_source != [] or new_target_types != [] %}\n {% set schema_changed = True %}\n {% elif new_target_types != [] %}\n {% set schema_changed = True %}\n {% endif %}\n\n {% set changes_dict = {\n 'schema_changed': schema_changed,\n 'source_not_in_target': source_not_in_target,\n 'target_not_in_source': target_not_in_source,\n 'source_columns': source_columns,\n 'target_columns': target_columns,\n 'new_target_types': new_target_types\n } %}\n\n {% set msg %}\n In {{ target_relation }}:\n Schema changed: {{ schema_changed }}\n Source columns not in target: {{ source_not_in_target }}\n Target columns not in source: {{ target_not_in_source }}\n New column types: {{ new_target_types }}\n {% endset %}\n\n {% do log(msg) %}\n\n {{ return(changes_dict) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.diff_columns", "macro.dbt.diff_column_data_types"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.60002, "supported_languages": null}, "macro.dbt.sync_column_schemas": {"name": "sync_column_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.sync_column_schemas", "macro_sql": "{% macro sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {%- set add_to_target_arr = schema_changes_dict['source_not_in_target'] -%}\n\n {%- if on_schema_change == 'append_new_columns'-%}\n {%- if add_to_target_arr | length > 0 -%}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, none) -%}\n {%- endif -%}\n\n {% elif on_schema_change == 'sync_all_columns' %}\n {%- set remove_from_target_arr = schema_changes_dict['target_not_in_source'] -%}\n {%- set new_target_types = schema_changes_dict['new_target_types'] -%}\n\n {% if add_to_target_arr | length > 0 or remove_from_target_arr | length > 0 %}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, remove_from_target_arr) -%}\n {% endif %}\n\n {% if new_target_types != [] %}\n {% for ntt in new_target_types %}\n {% set column_name = ntt['column_name'] %}\n {% set new_type = ntt['new_type'] %}\n {% do alter_column_type(target_relation, column_name, new_type) %}\n {% endfor %}\n {% endif %}\n\n {% endif %}\n\n {% set schema_change_message %}\n In {{ target_relation }}:\n Schema change approach: {{ on_schema_change }}\n Columns added: {{ add_to_target_arr }}\n Columns removed: {{ remove_from_target_arr }}\n Data types changed: {{ new_target_types }}\n {% endset %}\n\n {% do log(schema_change_message) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.alter_relation_add_remove_columns", "macro.dbt.alter_column_type"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.603049, "supported_languages": null}, "macro.dbt.process_schema_changes": {"name": "process_schema_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.process_schema_changes", "macro_sql": "{% macro process_schema_changes(on_schema_change, source_relation, target_relation) %}\n\n {% if on_schema_change == 'ignore' %}\n\n {{ return({}) }}\n\n {% else %}\n\n {% set schema_changes_dict = check_for_schema_changes(source_relation, target_relation) %}\n\n {% if schema_changes_dict['schema_changed'] %}\n\n {% if on_schema_change == 'fail' %}\n\n {% set fail_msg %}\n The source and target schemas on this incremental model are out of sync!\n They can be reconciled in several ways:\n - set the `on_schema_change` config to either append_new_columns or sync_all_columns, depending on your situation.\n - Re-run the incremental model with `full_refresh: True` to update the target schema.\n - update the schema manually and re-run the process.\n\n Additional troubleshooting context:\n Source columns not in target: {{ schema_changes_dict['source_not_in_target'] }}\n Target columns not in source: {{ schema_changes_dict['target_not_in_source'] }}\n New column types: {{ schema_changes_dict['new_target_types'] }}\n {% endset %}\n\n {% do exceptions.raise_compiler_error(fail_msg) %}\n\n {# -- unless we ignore, run the sync operation per the config #}\n {% else %}\n\n {% do sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {% endif %}\n\n {% endif %}\n\n {{ return(schema_changes_dict['source_columns']) }}\n\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.check_for_schema_changes", "macro.dbt.sync_column_schemas"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.605203, "supported_languages": null}, "macro.dbt.materialization_table_default": {"name": "materialization_table_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table/table.sql", "original_file_path": "macros/materializations/models/table/table.sql", "unique_id": "macro.dbt.materialization_table_default", "macro_sql": "{% materialization table, default %}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') %}\n {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}\n /*\n See ../view/view.sql for more information about this relation.\n */\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_table_as_sql(False, intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n {% if existing_relation is not none %}\n {{ adapter.rename_relation(existing_relation, backup_relation) }}\n {% endif %}\n\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% do create_indexes(target_relation) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n -- finally, drop the existing/backup relation after the commit\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.get_create_table_as_sql", "macro.dbt.create_indexes", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.611255, "supported_languages": ["sql"]}, "macro.dbt.get_create_table_as_sql": {"name": "get_create_table_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table/create_table_as.sql", "original_file_path": "macros/materializations/models/table/create_table_as.sql", "unique_id": "macro.dbt.get_create_table_as_sql", "macro_sql": "{% macro get_create_table_as_sql(temporary, relation, sql) -%}\n {{ adapter.dispatch('get_create_table_as_sql', 'dbt')(temporary, relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_create_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.612431, "supported_languages": null}, "macro.dbt.default__get_create_table_as_sql": {"name": "default__get_create_table_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table/create_table_as.sql", "original_file_path": "macros/materializations/models/table/create_table_as.sql", "unique_id": "macro.dbt.default__get_create_table_as_sql", "macro_sql": "{% macro default__get_create_table_as_sql(temporary, relation, sql) -%}\n {{ return(create_table_as(temporary, relation, sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.6129012, "supported_languages": null}, "macro.dbt.create_table_as": {"name": "create_table_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table/create_table_as.sql", "original_file_path": "macros/materializations/models/table/create_table_as.sql", "unique_id": "macro.dbt.create_table_as", "macro_sql": "{% macro create_table_as(temporary, relation, compiled_code, language='sql') -%}\n {# backward compatibility for create_table_as that does not support language #}\n {% if language == \"sql\" %}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, compiled_code)}}\n {% else %}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, compiled_code, language) }}\n {% endif %}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.613973, "supported_languages": null}, "macro.dbt.default__create_table_as": {"name": "default__create_table_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table/create_table_as.sql", "original_file_path": "macros/materializations/models/table/create_table_as.sql", "unique_id": "macro.dbt.default__create_table_as", "macro_sql": "{% macro default__create_table_as(temporary, relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary: -%}temporary{%- endif %} table\n {{ relation.include(database=(not temporary), schema=(not temporary)) }}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.614995, "supported_languages": null}, "macro.dbt.materialization_view_default": {"name": "materialization_view_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/view/view.sql", "original_file_path": "macros/materializations/models/view/view.sql", "unique_id": "macro.dbt.materialization_view_default", "macro_sql": "{%- materialization view, default -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='view') -%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}\n\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}\n /*\n This relation (probably) doesn't exist yet. If it does exist, it's a leftover from\n a previous run, and we're going to try to drop it immediately. At the end of this\n materialization, we're going to rename the \"existing_relation\" to this identifier,\n and then we're going to drop it. In order to make sure we run the correct one of:\n - drop view ...\n - drop table ...\n\n We need to set the type of this relation to be the type of the existing_relation, if it exists,\n or else \"view\" as a sane default if it does not. Note that if the existing_relation does not\n exist, then there is nothing to move out of the way and subsequentally drop. In that case,\n this relation will be effectively unused.\n */\n {%- set backup_relation_type = 'view' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n -- move the existing view out of the way\n {% if existing_relation is not none %}\n {{ adapter.rename_relation(existing_relation, backup_relation) }}\n {% endif %}\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.run_hooks", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.get_create_view_as_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.621283, "supported_languages": ["sql"]}, "macro.dbt.handle_existing_table": {"name": "handle_existing_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/view/helpers.sql", "original_file_path": "macros/materializations/models/view/helpers.sql", "unique_id": "macro.dbt.handle_existing_table", "macro_sql": "{% macro handle_existing_table(full_refresh, old_relation) %}\n {{ adapter.dispatch('handle_existing_table', 'dbt')(full_refresh, old_relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__handle_existing_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.6220968, "supported_languages": null}, "macro.dbt.default__handle_existing_table": {"name": "default__handle_existing_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/view/helpers.sql", "original_file_path": "macros/materializations/models/view/helpers.sql", "unique_id": "macro.dbt.default__handle_existing_table", "macro_sql": "{% macro default__handle_existing_table(full_refresh, old_relation) %}\n {{ log(\"Dropping relation \" ~ old_relation ~ \" because it is of type \" ~ old_relation.type) }}\n {{ adapter.drop_relation(old_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.622665, "supported_languages": null}, "macro.dbt.create_or_replace_view": {"name": "create_or_replace_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/view/create_or_replace_view.sql", "original_file_path": "macros/materializations/models/view/create_or_replace_view.sql", "unique_id": "macro.dbt.create_or_replace_view", "macro_sql": "{% macro create_or_replace_view() %}\n {%- set identifier = model['alias'] -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database,\n type='view') -%}\n {% set grant_config = config.get('grants') %}\n\n {{ run_hooks(pre_hooks) }}\n\n -- If there's a table with the same name and we weren't told to full refresh,\n -- that's an error. If we were told to full refresh, drop it. This behavior differs\n -- for Snowflake and BigQuery, so multiple dispatch is used.\n {%- if old_relation is not none and old_relation.is_table -%}\n {{ handle_existing_table(should_full_refresh(), old_relation) }}\n {%- endif -%}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(target_relation, sql) }}\n {%- endcall %}\n\n {% set should_revoke = should_revoke(exists_as_view, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=True) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.handle_existing_table", "macro.dbt.should_full_refresh", "macro.dbt.statement", "macro.dbt.get_create_view_as_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.626231, "supported_languages": null}, "macro.dbt.get_create_view_as_sql": {"name": "get_create_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/view/create_view_as.sql", "original_file_path": "macros/materializations/models/view/create_view_as.sql", "unique_id": "macro.dbt.get_create_view_as_sql", "macro_sql": "{% macro get_create_view_as_sql(relation, sql) -%}\n {{ adapter.dispatch('get_create_view_as_sql', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_create_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.627094, "supported_languages": null}, "macro.dbt.default__get_create_view_as_sql": {"name": "default__get_create_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/view/create_view_as.sql", "original_file_path": "macros/materializations/models/view/create_view_as.sql", "unique_id": "macro.dbt.default__get_create_view_as_sql", "macro_sql": "{% macro default__get_create_view_as_sql(relation, sql) -%}\n {{ return(create_view_as(relation, sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.627518, "supported_languages": null}, "macro.dbt.create_view_as": {"name": "create_view_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/view/create_view_as.sql", "original_file_path": "macros/materializations/models/view/create_view_as.sql", "unique_id": "macro.dbt.create_view_as", "macro_sql": "{% macro create_view_as(relation, sql) -%}\n {{ adapter.dispatch('create_view_as', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.627975, "supported_languages": null}, "macro.dbt.default__create_view_as": {"name": "default__create_view_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/view/create_view_as.sql", "original_file_path": "macros/materializations/models/view/create_view_as.sql", "unique_id": "macro.dbt.default__create_view_as", "macro_sql": "{% macro default__create_view_as(relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n create view {{ relation }} as (\n {{ sql }}\n );\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.628621, "supported_languages": null}, "macro.dbt.materialization_seed_default": {"name": "materialization_seed_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/seed.sql", "original_file_path": "macros/materializations/seeds/seed.sql", "unique_id": "macro.dbt.materialization_seed_default", "macro_sql": "{% materialization seed, default %}\n\n {%- set identifier = model['alias'] -%}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set grant_config = config.get('grants') -%}\n {%- set agate_table = load_agate_table() -%}\n -- grab current tables grants config for comparision later on\n\n {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% set create_table_sql = \"\" %}\n {% if exists_as_view %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a view\".format(old_relation)) }}\n {% elif exists_as_table %}\n {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}\n {% else %}\n {% set create_table_sql = create_csv_table(model, agate_table) %}\n {% endif %}\n\n {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}\n {% set rows_affected = (agate_table.rows | length) %}\n {% set sql = load_csv_rows(model, agate_table) %}\n\n {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}\n {{ get_csv_sql(create_table_sql, sql) }};\n {% endcall %}\n\n {% set target_relation = this.incorporate(type='table') %}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if full_refresh_mode or not exists_as_table %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.run_hooks", "macro.dbt.reset_csv_table", "macro.dbt.create_csv_table", "macro.dbt.load_csv_rows", "macro.dbt.noop_statement", "macro.dbt.get_csv_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.6366222, "supported_languages": ["sql"]}, "macro.dbt.create_csv_table": {"name": "create_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.create_csv_table", "macro_sql": "{% macro create_csv_table(model, agate_table) -%}\n {{ adapter.dispatch('create_csv_table', 'dbt')(model, agate_table) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.6457422, "supported_languages": null}, "macro.dbt.default__create_csv_table": {"name": "default__create_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__create_csv_table", "macro_sql": "{% macro default__create_csv_table(model, agate_table) %}\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n\n {% set sql %}\n create table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.648115, "supported_languages": null}, "macro.dbt.reset_csv_table": {"name": "reset_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.reset_csv_table", "macro_sql": "{% macro reset_csv_table(model, full_refresh, old_relation, agate_table) -%}\n {{ adapter.dispatch('reset_csv_table', 'dbt')(model, full_refresh, old_relation, agate_table) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__reset_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.648722, "supported_languages": null}, "macro.dbt.default__reset_csv_table": {"name": "default__reset_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__reset_csv_table", "macro_sql": "{% macro default__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% set sql = \"\" %}\n {% if full_refresh %}\n {{ adapter.drop_relation(old_relation) }}\n {% set sql = create_csv_table(model, agate_table) %}\n {% else %}\n {{ adapter.truncate_relation(old_relation) }}\n {% set sql = \"truncate table \" ~ old_relation %}\n {% endif %}\n\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.650075, "supported_languages": null}, "macro.dbt.get_csv_sql": {"name": "get_csv_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_csv_sql", "macro_sql": "{% macro get_csv_sql(create_or_truncate_sql, insert_sql) %}\n {{ adapter.dispatch('get_csv_sql', 'dbt')(create_or_truncate_sql, insert_sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_csv_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.65061, "supported_languages": null}, "macro.dbt.default__get_csv_sql": {"name": "default__get_csv_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__get_csv_sql", "macro_sql": "{% macro default__get_csv_sql(create_or_truncate_sql, insert_sql) %}\n {{ create_or_truncate_sql }};\n -- dbt seed --\n {{ insert_sql }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.6509619, "supported_languages": null}, "macro.dbt.get_binding_char": {"name": "get_binding_char", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_binding_char", "macro_sql": "{% macro get_binding_char() -%}\n {{ adapter.dispatch('get_binding_char', 'dbt')() }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.6513228, "supported_languages": null}, "macro.dbt.default__get_binding_char": {"name": "default__get_binding_char", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__get_binding_char", "macro_sql": "{% macro default__get_binding_char() %}\n {{ return('%s') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.6516201, "supported_languages": null}, "macro.dbt.get_batch_size": {"name": "get_batch_size", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_batch_size", "macro_sql": "{% macro get_batch_size() -%}\n {{ return(adapter.dispatch('get_batch_size', 'dbt')()) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_batch_size"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.652027, "supported_languages": null}, "macro.dbt.default__get_batch_size": {"name": "default__get_batch_size", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__get_batch_size", "macro_sql": "{% macro default__get_batch_size() %}\n {{ return(10000) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.6523268, "supported_languages": null}, "macro.dbt.get_seed_column_quoted_csv": {"name": "get_seed_column_quoted_csv", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_seed_column_quoted_csv", "macro_sql": "{% macro get_seed_column_quoted_csv(model, column_names) %}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote_seed_column(col, quote_seed_column)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.6536052, "supported_languages": null}, "macro.dbt.load_csv_rows": {"name": "load_csv_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.load_csv_rows", "macro_sql": "{% macro load_csv_rows(model, agate_table) -%}\n {{ adapter.dispatch('load_csv_rows', 'dbt')(model, agate_table) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__load_csv_rows"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.654092, "supported_languages": null}, "macro.dbt.default__load_csv_rows": {"name": "default__load_csv_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__load_csv_rows", "macro_sql": "{% macro default__load_csv_rows(model, agate_table) %}\n\n {% set batch_size = get_batch_size() %}\n\n {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}\n {% set bindings = [] %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert into {{ this.render() }} ({{ cols_sql }}) values\n {% for row in chunk -%}\n ({%- for column in agate_table.column_names -%}\n {{ get_binding_char() }}\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_batch_size", "macro.dbt.get_seed_column_quoted_csv", "macro.dbt.get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.657616, "supported_languages": null}, "macro.dbt.generate_alias_name": {"name": "generate_alias_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_alias.sql", "original_file_path": "macros/get_custom_name/get_custom_alias.sql", "unique_id": "macro.dbt.generate_alias_name", "macro_sql": "{% macro generate_alias_name(custom_alias_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_alias_name', 'dbt')(custom_alias_name, node)) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__generate_alias_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.658516, "supported_languages": null}, "macro.dbt.default__generate_alias_name": {"name": "default__generate_alias_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_alias.sql", "original_file_path": "macros/get_custom_name/get_custom_alias.sql", "unique_id": "macro.dbt.default__generate_alias_name", "macro_sql": "{% macro default__generate_alias_name(custom_alias_name=none, node=none) -%}\n\n {%- if custom_alias_name is none -%}\n\n {{ node.name }}\n\n {%- else -%}\n\n {{ custom_alias_name | trim }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.659103, "supported_languages": null}, "macro.dbt.generate_schema_name": {"name": "generate_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "unique_id": "macro.dbt.generate_schema_name", "macro_sql": "{% macro generate_schema_name(custom_schema_name=none, node=none) -%}\n {{ return(adapter.dispatch('generate_schema_name', 'dbt')(custom_schema_name, node)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__generate_schema_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.660212, "supported_languages": null}, "macro.dbt.default__generate_schema_name": {"name": "default__generate_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "unique_id": "macro.dbt.default__generate_schema_name", "macro_sql": "{% macro default__generate_schema_name(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if custom_schema_name is none -%}\n\n {{ default_schema }}\n\n {%- else -%}\n\n {{ default_schema }}_{{ custom_schema_name | trim }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.660863, "supported_languages": null}, "macro.dbt.generate_schema_name_for_env": {"name": "generate_schema_name_for_env", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "unique_id": "macro.dbt.generate_schema_name_for_env", "macro_sql": "{% macro generate_schema_name_for_env(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if target.name == 'prod' and custom_schema_name is not none -%}\n\n {{ custom_schema_name | trim }}\n\n {%- else -%}\n\n {{ default_schema }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.661577, "supported_languages": null}, "macro.dbt.generate_database_name": {"name": "generate_database_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "unique_id": "macro.dbt.generate_database_name", "macro_sql": "{% macro generate_database_name(custom_database_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_database_name', 'dbt')(custom_database_name, node)) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__generate_database_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.662461, "supported_languages": null}, "macro.dbt.default__generate_database_name": {"name": "default__generate_database_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "unique_id": "macro.dbt.default__generate_database_name", "macro_sql": "{% macro default__generate_database_name(custom_database_name=none, node=none) -%}\n {%- set default_database = target.database -%}\n {%- if custom_database_name is none -%}\n\n {{ default_database }}\n\n {%- else -%}\n\n {{ custom_database_name }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.6631, "supported_languages": null}, "macro.dbt.default__test_relationships": {"name": "default__test_relationships", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/relationships.sql", "original_file_path": "macros/generic_test_sql/relationships.sql", "unique_id": "macro.dbt.default__test_relationships", "macro_sql": "{% macro default__test_relationships(model, column_name, to, field) %}\n\nwith child as (\n select {{ column_name }} as from_field\n from {{ model }}\n where {{ column_name }} is not null\n),\n\nparent as (\n select {{ field }} as to_field\n from {{ to }}\n)\n\nselect\n from_field\n\nfrom child\nleft join parent\n on child.from_field = parent.to_field\n\nwhere parent.to_field is null\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.664021, "supported_languages": null}, "macro.dbt.default__test_not_null": {"name": "default__test_not_null", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/not_null.sql", "original_file_path": "macros/generic_test_sql/not_null.sql", "unique_id": "macro.dbt.default__test_not_null", "macro_sql": "{% macro default__test_not_null(model, column_name) %}\n\n{% set column_list = '*' if should_store_failures() else column_name %}\n\nselect {{ column_list }}\nfrom {{ model }}\nwhere {{ column_name }} is null\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_store_failures"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.664862, "supported_languages": null}, "macro.dbt.default__test_unique": {"name": "default__test_unique", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/unique.sql", "original_file_path": "macros/generic_test_sql/unique.sql", "unique_id": "macro.dbt.default__test_unique", "macro_sql": "{% macro default__test_unique(model, column_name) %}\n\nselect\n {{ column_name }} as unique_field,\n count(*) as n_records\n\nfrom {{ model }}\nwhere {{ column_name }} is not null\ngroup by {{ column_name }}\nhaving count(*) > 1\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.665466, "supported_languages": null}, "macro.dbt.default__test_accepted_values": {"name": "default__test_accepted_values", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/accepted_values.sql", "original_file_path": "macros/generic_test_sql/accepted_values.sql", "unique_id": "macro.dbt.default__test_accepted_values", "macro_sql": "{% macro default__test_accepted_values(model, column_name, values, quote=True) %}\n\nwith all_values as (\n\n select\n {{ column_name }} as value_field,\n count(*) as n_records\n\n from {{ model }}\n group by {{ column_name }}\n\n)\n\nselect *\nfrom all_values\nwhere value_field not in (\n {% for value in values -%}\n {% if quote -%}\n '{{ value }}'\n {%- else -%}\n {{ value }}\n {%- endif -%}\n {%- if not loop.last -%},{%- endif %}\n {%- endfor %}\n)\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.666805, "supported_languages": null}, "macro.dbt.statement": {"name": "statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt.statement", "macro_sql": "\n{%- macro statement(name=None, fetch_result=False, auto_begin=True, language='sql') -%}\n {%- if execute: -%}\n {%- set compiled_code = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime {} for node \"{}\"'.format(language, model['unique_id'])) }}\n {{ write(compiled_code) }}\n {%- endif -%}\n {%- if language == 'sql'-%}\n {%- set res, table = adapter.execute(compiled_code, auto_begin=auto_begin, fetch=fetch_result) -%}\n {%- elif language == 'python' -%}\n {%- set res = submit_python_job(model, compiled_code) -%}\n {#-- TODO: What should table be for python models? --#}\n {%- set table = None -%}\n {%- else -%}\n {% do exceptions.raise_compiler_error(\"statement macro didn't get supported language\") %}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_result(name, response=res, agate_table=table) }}\n {%- endif -%}\n\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.670157, "supported_languages": null}, "macro.dbt.noop_statement": {"name": "noop_statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt.noop_statement", "macro_sql": "{% macro noop_statement(name=None, message=None, code=None, rows_affected=None, res=None) -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_raw_result(name, message=message, code=code, rows_affected=rows_affected, agate_table=res) }}\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.671575, "supported_languages": null}, "macro.dbt.run_query": {"name": "run_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt.run_query", "macro_sql": "{% macro run_query(sql) %}\n {% call statement(\"run_query_statement\", fetch_result=true, auto_begin=false) %}\n {{ sql }}\n {% endcall %}\n\n {% do return(load_result(\"run_query_statement\").table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.672309, "supported_languages": null}, "macro.dbt.convert_datetime": {"name": "convert_datetime", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.convert_datetime", "macro_sql": "{% macro convert_datetime(date_str, date_fmt) %}\n\n {% set error_msg -%}\n The provided partition date '{{ date_str }}' does not match the expected format '{{ date_fmt }}'\n {%- endset %}\n\n {% set res = try_or_compiler_error(error_msg, modules.datetime.datetime.strptime, date_str.strip(), date_fmt) %}\n {{ return(res) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.675864, "supported_languages": null}, "macro.dbt.dates_in_range": {"name": "dates_in_range", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.dates_in_range", "macro_sql": "{% macro dates_in_range(start_date_str, end_date_str=none, in_fmt=\"%Y%m%d\", out_fmt=\"%Y%m%d\") %}\n {% set end_date_str = start_date_str if end_date_str is none else end_date_str %}\n\n {% set start_date = convert_datetime(start_date_str, in_fmt) %}\n {% set end_date = convert_datetime(end_date_str, in_fmt) %}\n\n {% set day_count = (end_date - start_date).days %}\n {% if day_count < 0 %}\n {% set msg -%}\n Partition start date is after the end date ({{ start_date }}, {{ end_date }})\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg, model) }}\n {% endif %}\n\n {% set date_list = [] %}\n {% for i in range(0, day_count + 1) %}\n {% set the_date = (modules.datetime.timedelta(days=i) + start_date) %}\n {% if not out_fmt %}\n {% set _ = date_list.append(the_date) %}\n {% else %}\n {% set _ = date_list.append(the_date.strftime(out_fmt)) %}\n {% endif %}\n {% endfor %}\n\n {{ return(date_list) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.convert_datetime"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.678925, "supported_languages": null}, "macro.dbt.partition_range": {"name": "partition_range", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.partition_range", "macro_sql": "{% macro partition_range(raw_partition_date, date_fmt='%Y%m%d') %}\n {% set partition_range = (raw_partition_date | string).split(\",\") %}\n\n {% if (partition_range | length) == 1 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = none %}\n {% elif (partition_range | length) == 2 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = partition_range[1] %}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Invalid partition time. Expected format: {Start Date}[,{End Date}]. Got: \" ~ raw_partition_date) }}\n {% endif %}\n\n {{ return(dates_in_range(start_date, end_date, in_fmt=date_fmt)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.dates_in_range"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.680963, "supported_languages": null}, "macro.dbt.py_current_timestring": {"name": "py_current_timestring", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.py_current_timestring", "macro_sql": "{% macro py_current_timestring() %}\n {% set dt = modules.datetime.datetime.now() %}\n {% do return(dt.strftime(\"%Y%m%d%H%M%S%f\")) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.681591, "supported_languages": null}, "macro.dbt.except": {"name": "except", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/except.sql", "original_file_path": "macros/utils/except.sql", "unique_id": "macro.dbt.except", "macro_sql": "{% macro except() %}\n {{ return(adapter.dispatch('except', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__except"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.682165, "supported_languages": null}, "macro.dbt.default__except": {"name": "default__except", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/except.sql", "original_file_path": "macros/utils/except.sql", "unique_id": "macro.dbt.default__except", "macro_sql": "{% macro default__except() %}\n\n except\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.682366, "supported_languages": null}, "macro.dbt.replace": {"name": "replace", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/replace.sql", "original_file_path": "macros/utils/replace.sql", "unique_id": "macro.dbt.replace", "macro_sql": "{% macro replace(field, old_chars, new_chars) -%}\n {{ return(adapter.dispatch('replace', 'dbt') (field, old_chars, new_chars)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__replace"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.6831412, "supported_languages": null}, "macro.dbt.default__replace": {"name": "default__replace", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/replace.sql", "original_file_path": "macros/utils/replace.sql", "unique_id": "macro.dbt.default__replace", "macro_sql": "{% macro default__replace(field, old_chars, new_chars) %}\n\n replace(\n {{ field }},\n {{ old_chars }},\n {{ new_chars }}\n )\n\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.683658, "supported_languages": null}, "macro.dbt.concat": {"name": "concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/concat.sql", "original_file_path": "macros/utils/concat.sql", "unique_id": "macro.dbt.concat", "macro_sql": "{% macro concat(fields) -%}\n {{ return(adapter.dispatch('concat', 'dbt')(fields)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__concat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.684277, "supported_languages": null}, "macro.dbt.default__concat": {"name": "default__concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/concat.sql", "original_file_path": "macros/utils/concat.sql", "unique_id": "macro.dbt.default__concat", "macro_sql": "{% macro default__concat(fields) -%}\n {{ fields|join(' || ') }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.6846101, "supported_languages": null}, "macro.dbt.length": {"name": "length", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/length.sql", "original_file_path": "macros/utils/length.sql", "unique_id": "macro.dbt.length", "macro_sql": "{% macro length(expression) -%}\n {{ return(adapter.dispatch('length', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__length"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.685228, "supported_languages": null}, "macro.dbt.default__length": {"name": "default__length", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/length.sql", "original_file_path": "macros/utils/length.sql", "unique_id": "macro.dbt.default__length", "macro_sql": "{% macro default__length(expression) %}\n\n length(\n {{ expression }}\n )\n\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.685508, "supported_languages": null}, "macro.dbt.dateadd": {"name": "dateadd", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt.dateadd", "macro_sql": "{% macro dateadd(datepart, interval, from_date_or_timestamp) %}\n {{ return(adapter.dispatch('dateadd', 'dbt')(datepart, interval, from_date_or_timestamp)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__dateadd"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.6863098, "supported_languages": null}, "macro.dbt.default__dateadd": {"name": "default__dateadd", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt.default__dateadd", "macro_sql": "{% macro default__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n dateadd(\n {{ datepart }},\n {{ interval }},\n {{ from_date_or_timestamp }}\n )\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.6870232, "supported_languages": null}, "macro.dbt.intersect": {"name": "intersect", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/intersect.sql", "original_file_path": "macros/utils/intersect.sql", "unique_id": "macro.dbt.intersect", "macro_sql": "{% macro intersect() %}\n {{ return(adapter.dispatch('intersect', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__intersect"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.687619, "supported_languages": null}, "macro.dbt.default__intersect": {"name": "default__intersect", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/intersect.sql", "original_file_path": "macros/utils/intersect.sql", "unique_id": "macro.dbt.default__intersect", "macro_sql": "{% macro default__intersect() %}\n\n intersect\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.6878238, "supported_languages": null}, "macro.dbt.escape_single_quotes": {"name": "escape_single_quotes", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/escape_single_quotes.sql", "original_file_path": "macros/utils/escape_single_quotes.sql", "unique_id": "macro.dbt.escape_single_quotes", "macro_sql": "{% macro escape_single_quotes(expression) %}\n {{ return(adapter.dispatch('escape_single_quotes', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__escape_single_quotes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.688459, "supported_languages": null}, "macro.dbt.default__escape_single_quotes": {"name": "default__escape_single_quotes", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/escape_single_quotes.sql", "original_file_path": "macros/utils/escape_single_quotes.sql", "unique_id": "macro.dbt.default__escape_single_quotes", "macro_sql": "{% macro default__escape_single_quotes(expression) -%}\n{{ expression | replace(\"'\",\"''\") }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.688828, "supported_languages": null}, "macro.dbt.right": {"name": "right", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/right.sql", "original_file_path": "macros/utils/right.sql", "unique_id": "macro.dbt.right", "macro_sql": "{% macro right(string_text, length_expression) -%}\n {{ return(adapter.dispatch('right', 'dbt') (string_text, length_expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__right"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.689515, "supported_languages": null}, "macro.dbt.default__right": {"name": "default__right", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/right.sql", "original_file_path": "macros/utils/right.sql", "unique_id": "macro.dbt.default__right", "macro_sql": "{% macro default__right(string_text, length_expression) %}\n\n right(\n {{ string_text }},\n {{ length_expression }}\n )\n\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.689863, "supported_languages": null}, "macro.dbt.listagg": {"name": "listagg", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "unique_id": "macro.dbt.listagg", "macro_sql": "{% macro listagg(measure, delimiter_text=\"','\", order_by_clause=none, limit_num=none) -%}\n {{ return(adapter.dispatch('listagg', 'dbt') (measure, delimiter_text, order_by_clause, limit_num)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__listagg"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.691304, "supported_languages": null}, "macro.dbt.default__listagg": {"name": "default__listagg", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "unique_id": "macro.dbt.default__listagg", "macro_sql": "{% macro default__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n {% if limit_num -%}\n array_to_string(\n array_slice(\n array_agg(\n {{ measure }}\n ){% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n ,0\n ,{{ limit_num }}\n ),\n {{ delimiter_text }}\n )\n {%- else %}\n listagg(\n {{ measure }},\n {{ delimiter_text }}\n )\n {% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n {%- endif %}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.6924748, "supported_languages": null}, "macro.dbt.datediff": {"name": "datediff", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt.datediff", "macro_sql": "{% macro datediff(first_date, second_date, datepart) %}\n {{ return(adapter.dispatch('datediff', 'dbt')(first_date, second_date, datepart)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__datediff"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.693249, "supported_languages": null}, "macro.dbt.default__datediff": {"name": "default__datediff", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt.default__datediff", "macro_sql": "{% macro default__datediff(first_date, second_date, datepart) -%}\n\n datediff(\n {{ datepart }},\n {{ first_date }},\n {{ second_date }}\n )\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.69366, "supported_languages": null}, "macro.dbt.safe_cast": {"name": "safe_cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "unique_id": "macro.dbt.safe_cast", "macro_sql": "{% macro safe_cast(field, type) %}\n {{ return(adapter.dispatch('safe_cast', 'dbt') (field, type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__safe_cast"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.694339, "supported_languages": null}, "macro.dbt.default__safe_cast": {"name": "default__safe_cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "unique_id": "macro.dbt.default__safe_cast", "macro_sql": "{% macro default__safe_cast(field, type) %}\n {# most databases don't support this function yet\n so we just need to use cast #}\n cast({{field}} as {{type}})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.6946921, "supported_languages": null}, "macro.dbt.hash": {"name": "hash", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/hash.sql", "original_file_path": "macros/utils/hash.sql", "unique_id": "macro.dbt.hash", "macro_sql": "{% macro hash(field) -%}\n {{ return(adapter.dispatch('hash', 'dbt') (field)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__hash"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.695292, "supported_languages": null}, "macro.dbt.default__hash": {"name": "default__hash", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/hash.sql", "original_file_path": "macros/utils/hash.sql", "unique_id": "macro.dbt.default__hash", "macro_sql": "{% macro default__hash(field) -%}\n md5(cast({{ field }} as {{ api.Column.translate_type('string') }}))\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.695692, "supported_languages": null}, "macro.dbt.cast_bool_to_text": {"name": "cast_bool_to_text", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast_bool_to_text.sql", "original_file_path": "macros/utils/cast_bool_to_text.sql", "unique_id": "macro.dbt.cast_bool_to_text", "macro_sql": "{% macro cast_bool_to_text(field) %}\n {{ adapter.dispatch('cast_bool_to_text', 'dbt') (field) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__cast_bool_to_text"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.696287, "supported_languages": null}, "macro.dbt.default__cast_bool_to_text": {"name": "default__cast_bool_to_text", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast_bool_to_text.sql", "original_file_path": "macros/utils/cast_bool_to_text.sql", "unique_id": "macro.dbt.default__cast_bool_to_text", "macro_sql": "{% macro default__cast_bool_to_text(field) %}\n cast({{ field }} as {{ api.Column.translate_type('string') }})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.696754, "supported_languages": null}, "macro.dbt.any_value": {"name": "any_value", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "unique_id": "macro.dbt.any_value", "macro_sql": "{% macro any_value(expression) -%}\n {{ return(adapter.dispatch('any_value', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__any_value"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.697381, "supported_languages": null}, "macro.dbt.default__any_value": {"name": "default__any_value", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "unique_id": "macro.dbt.default__any_value", "macro_sql": "{% macro default__any_value(expression) -%}\n\n any_value({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.6978078, "supported_languages": null}, "macro.dbt.position": {"name": "position", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/position.sql", "original_file_path": "macros/utils/position.sql", "unique_id": "macro.dbt.position", "macro_sql": "{% macro position(substring_text, string_text) -%}\n {{ return(adapter.dispatch('position', 'dbt') (substring_text, string_text)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__position"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.698524, "supported_languages": null}, "macro.dbt.default__position": {"name": "default__position", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/position.sql", "original_file_path": "macros/utils/position.sql", "unique_id": "macro.dbt.default__position", "macro_sql": "{% macro default__position(substring_text, string_text) %}\n\n position(\n {{ substring_text }} in {{ string_text }}\n )\n\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.6990662, "supported_languages": null}, "macro.dbt.string_literal": {"name": "string_literal", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/literal.sql", "original_file_path": "macros/utils/literal.sql", "unique_id": "macro.dbt.string_literal", "macro_sql": "{%- macro string_literal(value) -%}\n {{ return(adapter.dispatch('string_literal', 'dbt') (value)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__string_literal"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.69984, "supported_languages": null}, "macro.dbt.default__string_literal": {"name": "default__string_literal", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/literal.sql", "original_file_path": "macros/utils/literal.sql", "unique_id": "macro.dbt.default__string_literal", "macro_sql": "{% macro default__string_literal(value) -%}\n '{{ value }}'\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.700127, "supported_languages": null}, "macro.dbt.type_string": {"name": "type_string", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_string", "macro_sql": "\n\n{%- macro type_string() -%}\n {{ return(adapter.dispatch('type_string', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_string"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.702124, "supported_languages": null}, "macro.dbt.default__type_string": {"name": "default__type_string", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_string", "macro_sql": "{% macro default__type_string() %}\n {{ return(api.Column.translate_type(\"string\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.702521, "supported_languages": null}, "macro.dbt.type_timestamp": {"name": "type_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_timestamp", "macro_sql": "\n\n{%- macro type_timestamp() -%}\n {{ return(adapter.dispatch('type_timestamp', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.702935, "supported_languages": null}, "macro.dbt.default__type_timestamp": {"name": "default__type_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_timestamp", "macro_sql": "{% macro default__type_timestamp() %}\n {{ return(api.Column.translate_type(\"timestamp\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.703326, "supported_languages": null}, "macro.dbt.type_float": {"name": "type_float", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_float", "macro_sql": "\n\n{%- macro type_float() -%}\n {{ return(adapter.dispatch('type_float', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_float"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.703733, "supported_languages": null}, "macro.dbt.default__type_float": {"name": "default__type_float", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_float", "macro_sql": "{% macro default__type_float() %}\n {{ return(api.Column.translate_type(\"float\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.704247, "supported_languages": null}, "macro.dbt.type_numeric": {"name": "type_numeric", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_numeric", "macro_sql": "\n\n{%- macro type_numeric() -%}\n {{ return(adapter.dispatch('type_numeric', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_numeric"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.7046552, "supported_languages": null}, "macro.dbt.default__type_numeric": {"name": "default__type_numeric", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_numeric", "macro_sql": "{% macro default__type_numeric() %}\n {{ return(api.Column.numeric_type(\"numeric\", 28, 6)) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.705098, "supported_languages": null}, "macro.dbt.type_bigint": {"name": "type_bigint", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_bigint", "macro_sql": "\n\n{%- macro type_bigint() -%}\n {{ return(adapter.dispatch('type_bigint', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_bigint"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.705503, "supported_languages": null}, "macro.dbt.default__type_bigint": {"name": "default__type_bigint", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_bigint", "macro_sql": "{% macro default__type_bigint() %}\n {{ return(api.Column.translate_type(\"bigint\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.705885, "supported_languages": null}, "macro.dbt.type_int": {"name": "type_int", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_int", "macro_sql": "\n\n{%- macro type_int() -%}\n {{ return(adapter.dispatch('type_int', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_int"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.7063448, "supported_languages": null}, "macro.dbt.default__type_int": {"name": "default__type_int", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_int", "macro_sql": "{%- macro default__type_int() -%}\n {{ return(api.Column.translate_type(\"integer\")) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.7067552, "supported_languages": null}, "macro.dbt.type_boolean": {"name": "type_boolean", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_boolean", "macro_sql": "\n\n{%- macro type_boolean() -%}\n {{ return(adapter.dispatch('type_boolean', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_boolean"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.7071831, "supported_languages": null}, "macro.dbt.default__type_boolean": {"name": "default__type_boolean", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_boolean", "macro_sql": "{%- macro default__type_boolean() -%}\n {{ return(api.Column.translate_type(\"boolean\")) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.707561, "supported_languages": null}, "macro.dbt.array_concat": {"name": "array_concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_concat.sql", "original_file_path": "macros/utils/array_concat.sql", "unique_id": "macro.dbt.array_concat", "macro_sql": "{% macro array_concat(array_1, array_2) -%}\n {{ return(adapter.dispatch('array_concat', 'dbt')(array_1, array_2)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__array_concat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.708241, "supported_languages": null}, "macro.dbt.default__array_concat": {"name": "default__array_concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_concat.sql", "original_file_path": "macros/utils/array_concat.sql", "unique_id": "macro.dbt.default__array_concat", "macro_sql": "{% macro default__array_concat(array_1, array_2) -%}\n array_cat({{ array_1 }}, {{ array_2 }})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.7085838, "supported_languages": null}, "macro.dbt.bool_or": {"name": "bool_or", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/bool_or.sql", "original_file_path": "macros/utils/bool_or.sql", "unique_id": "macro.dbt.bool_or", "macro_sql": "{% macro bool_or(expression) -%}\n {{ return(adapter.dispatch('bool_or', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__bool_or"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.709186, "supported_languages": null}, "macro.dbt.default__bool_or": {"name": "default__bool_or", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/bool_or.sql", "original_file_path": "macros/utils/bool_or.sql", "unique_id": "macro.dbt.default__bool_or", "macro_sql": "{% macro default__bool_or(expression) -%}\n\n bool_or({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.7094588, "supported_languages": null}, "macro.dbt.last_day": {"name": "last_day", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt.last_day", "macro_sql": "{% macro last_day(date, datepart) %}\n {{ return(adapter.dispatch('last_day', 'dbt') (date, datepart)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__last_day"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.7102468, "supported_languages": null}, "macro.dbt.default_last_day": {"name": "default_last_day", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt.default_last_day", "macro_sql": "\n\n{%- macro default_last_day(date, datepart) -%}\n cast(\n {{dbt.dateadd('day', '-1',\n dbt.dateadd(datepart, '1', dbt.date_trunc(datepart, date))\n )}}\n as date)\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.dateadd", "macro.dbt.date_trunc"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.710925, "supported_languages": null}, "macro.dbt.default__last_day": {"name": "default__last_day", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt.default__last_day", "macro_sql": "{% macro default__last_day(date, datepart) -%}\n {{dbt.default_last_day(date, datepart)}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default_last_day"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.711309, "supported_languages": null}, "macro.dbt.split_part": {"name": "split_part", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt.split_part", "macro_sql": "{% macro split_part(string_text, delimiter_text, part_number) %}\n {{ return(adapter.dispatch('split_part', 'dbt') (string_text, delimiter_text, part_number)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__split_part"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.712547, "supported_languages": null}, "macro.dbt.default__split_part": {"name": "default__split_part", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt.default__split_part", "macro_sql": "{% macro default__split_part(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n {{ part_number }}\n )\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.71298, "supported_languages": null}, "macro.dbt._split_part_negative": {"name": "_split_part_negative", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt._split_part_negative", "macro_sql": "{% macro _split_part_negative(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n length({{ string_text }})\n - length(\n replace({{ string_text }}, {{ delimiter_text }}, '')\n ) + 2 {{ part_number }}\n )\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.713571, "supported_languages": null}, "macro.dbt.date_trunc": {"name": "date_trunc", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_trunc.sql", "original_file_path": "macros/utils/date_trunc.sql", "unique_id": "macro.dbt.date_trunc", "macro_sql": "{% macro date_trunc(datepart, date) -%}\n {{ return(adapter.dispatch('date_trunc', 'dbt') (datepart, date)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__date_trunc"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.714258, "supported_languages": null}, "macro.dbt.default__date_trunc": {"name": "default__date_trunc", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_trunc.sql", "original_file_path": "macros/utils/date_trunc.sql", "unique_id": "macro.dbt.default__date_trunc", "macro_sql": "{% macro default__date_trunc(datepart, date) -%}\n date_trunc('{{datepart}}', {{date}})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.714612, "supported_languages": null}, "macro.dbt.array_construct": {"name": "array_construct", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_construct.sql", "original_file_path": "macros/utils/array_construct.sql", "unique_id": "macro.dbt.array_construct", "macro_sql": "{% macro array_construct(inputs=[], data_type=api.Column.translate_type('integer')) -%}\n {{ return(adapter.dispatch('array_construct', 'dbt')(inputs, data_type)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__array_construct"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.7155101, "supported_languages": null}, "macro.dbt.default__array_construct": {"name": "default__array_construct", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_construct.sql", "original_file_path": "macros/utils/array_construct.sql", "unique_id": "macro.dbt.default__array_construct", "macro_sql": "{% macro default__array_construct(inputs, data_type) -%}\n {% if inputs|length > 0 %}\n array[ {{ inputs|join(' , ') }} ]\n {% else %}\n array[]::{{data_type}}[]\n {% endif %}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.716278, "supported_languages": null}, "macro.dbt.array_append": {"name": "array_append", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_append.sql", "original_file_path": "macros/utils/array_append.sql", "unique_id": "macro.dbt.array_append", "macro_sql": "{% macro array_append(array, new_element) -%}\n {{ return(adapter.dispatch('array_append', 'dbt')(array, new_element)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__array_append"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.7171369, "supported_languages": null}, "macro.dbt.default__array_append": {"name": "default__array_append", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_append.sql", "original_file_path": "macros/utils/array_append.sql", "unique_id": "macro.dbt.default__array_append", "macro_sql": "{% macro default__array_append(array, new_element) -%}\n array_append({{ array }}, {{ new_element }})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.717506, "supported_languages": null}, "macro.dbt.create_schema": {"name": "create_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.create_schema", "macro_sql": "{% macro create_schema(relation) -%}\n {{ adapter.dispatch('create_schema', 'dbt')(relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__create_schema"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.718344, "supported_languages": null}, "macro.dbt.default__create_schema": {"name": "default__create_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.default__create_schema", "macro_sql": "{% macro default__create_schema(relation) -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier() }}\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.71883, "supported_languages": null}, "macro.dbt.drop_schema": {"name": "drop_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.drop_schema", "macro_sql": "{% macro drop_schema(relation) -%}\n {{ adapter.dispatch('drop_schema', 'dbt')(relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__drop_schema"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.7192378, "supported_languages": null}, "macro.dbt.default__drop_schema": {"name": "default__drop_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.default__drop_schema", "macro_sql": "{% macro default__drop_schema(relation) -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier() }} cascade\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.719709, "supported_languages": null}, "macro.dbt.current_timestamp": {"name": "current_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.current_timestamp", "macro_sql": "{%- macro current_timestamp() -%}\n {{ adapter.dispatch('current_timestamp', 'dbt')() }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_postgres.postgres__current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.720729, "supported_languages": null}, "macro.dbt.default__current_timestamp": {"name": "default__current_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__current_timestamp", "macro_sql": "{% macro default__current_timestamp() -%}\n {{ exceptions.raise_not_implemented(\n 'current_timestamp macro not implemented for adapter ' + adapter.type()) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.7211258, "supported_languages": null}, "macro.dbt.snapshot_get_time": {"name": "snapshot_get_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.snapshot_get_time", "macro_sql": "\n\n{%- macro snapshot_get_time() -%}\n {{ adapter.dispatch('snapshot_get_time', 'dbt')() }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_postgres.postgres__snapshot_get_time"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.721498, "supported_languages": null}, "macro.dbt.default__snapshot_get_time": {"name": "default__snapshot_get_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__snapshot_get_time", "macro_sql": "{% macro default__snapshot_get_time() %}\n {{ current_timestamp() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.721788, "supported_languages": null}, "macro.dbt.current_timestamp_backcompat": {"name": "current_timestamp_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.current_timestamp_backcompat", "macro_sql": "{% macro current_timestamp_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_backcompat', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__current_timestamp_backcompat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.722215, "supported_languages": null}, "macro.dbt.default__current_timestamp_backcompat": {"name": "default__current_timestamp_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__current_timestamp_backcompat", "macro_sql": "{% macro default__current_timestamp_backcompat() %}\n current_timestamp::timestamp\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.7224221, "supported_languages": null}, "macro.dbt.current_timestamp_in_utc_backcompat": {"name": "current_timestamp_in_utc_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.current_timestamp_in_utc_backcompat", "macro_sql": "{% macro current_timestamp_in_utc_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_in_utc_backcompat', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__current_timestamp_in_utc_backcompat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.7228422, "supported_languages": null}, "macro.dbt.default__current_timestamp_in_utc_backcompat": {"name": "default__current_timestamp_in_utc_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__current_timestamp_in_utc_backcompat", "macro_sql": "{% macro default__current_timestamp_in_utc_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_backcompat', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.current_timestamp_backcompat", "macro.dbt_postgres.postgres__current_timestamp_backcompat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.723277, "supported_languages": null}, "macro.dbt.get_create_index_sql": {"name": "get_create_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.get_create_index_sql", "macro_sql": "{% macro get_create_index_sql(relation, index_dict) -%}\n {{ return(adapter.dispatch('get_create_index_sql', 'dbt')(relation, index_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_create_index_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.724287, "supported_languages": null}, "macro.dbt.default__get_create_index_sql": {"name": "default__get_create_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__get_create_index_sql", "macro_sql": "{% macro default__get_create_index_sql(relation, index_dict) -%}\n {% do return(None) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.7246542, "supported_languages": null}, "macro.dbt.create_indexes": {"name": "create_indexes", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.create_indexes", "macro_sql": "{% macro create_indexes(relation) -%}\n {{ adapter.dispatch('create_indexes', 'dbt')(relation) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.725062, "supported_languages": null}, "macro.dbt.default__create_indexes": {"name": "default__create_indexes", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__create_indexes", "macro_sql": "{% macro default__create_indexes(relation) -%}\n {%- set _indexes = config.get('indexes', default=[]) -%}\n\n {% for _index_dict in _indexes %}\n {% set create_index_sql = get_create_index_sql(relation, _index_dict) %}\n {% if create_index_sql %}\n {% do run_query(create_index_sql) %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_create_index_sql", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.726084, "supported_languages": null}, "macro.dbt.make_intermediate_relation": {"name": "make_intermediate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.make_intermediate_relation", "macro_sql": "{% macro make_intermediate_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_intermediate_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_intermediate_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.732279, "supported_languages": null}, "macro.dbt.default__make_intermediate_relation": {"name": "default__make_intermediate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__make_intermediate_relation", "macro_sql": "{% macro default__make_intermediate_relation(base_relation, suffix) %}\n {{ return(default__make_temp_relation(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__make_temp_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.732851, "supported_languages": null}, "macro.dbt.make_temp_relation": {"name": "make_temp_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.make_temp_relation", "macro_sql": "{% macro make_temp_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_temp_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_temp_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.7335029, "supported_languages": null}, "macro.dbt.default__make_temp_relation": {"name": "default__make_temp_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__make_temp_relation", "macro_sql": "{% macro default__make_temp_relation(base_relation, suffix) %}\n {%- set temp_identifier = base_relation.identifier ~ suffix -%}\n {%- set temp_relation = base_relation.incorporate(\n path={\"identifier\": temp_identifier}) -%}\n\n {{ return(temp_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.734247, "supported_languages": null}, "macro.dbt.make_backup_relation": {"name": "make_backup_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.make_backup_relation", "macro_sql": "{% macro make_backup_relation(base_relation, backup_relation_type, suffix='__dbt_backup') %}\n {{ return(adapter.dispatch('make_backup_relation', 'dbt')(base_relation, backup_relation_type, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_backup_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.7348611, "supported_languages": null}, "macro.dbt.default__make_backup_relation": {"name": "default__make_backup_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__make_backup_relation", "macro_sql": "{% macro default__make_backup_relation(base_relation, backup_relation_type, suffix) %}\n {%- set backup_identifier = base_relation.identifier ~ suffix -%}\n {%- set backup_relation = base_relation.incorporate(\n path={\"identifier\": backup_identifier},\n type=backup_relation_type\n ) -%}\n {{ return(backup_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.7356532, "supported_languages": null}, "macro.dbt.drop_relation": {"name": "drop_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.drop_relation", "macro_sql": "{% macro drop_relation(relation) -%}\n {{ return(adapter.dispatch('drop_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__drop_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.7361112, "supported_languages": null}, "macro.dbt.default__drop_relation": {"name": "default__drop_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__drop_relation", "macro_sql": "{% macro default__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n drop {{ relation.type }} if exists {{ relation }} cascade\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.7366579, "supported_languages": null}, "macro.dbt.truncate_relation": {"name": "truncate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.truncate_relation", "macro_sql": "{% macro truncate_relation(relation) -%}\n {{ return(adapter.dispatch('truncate_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__truncate_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.7371142, "supported_languages": null}, "macro.dbt.default__truncate_relation": {"name": "default__truncate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__truncate_relation", "macro_sql": "{% macro default__truncate_relation(relation) -%}\n {% call statement('truncate_relation') -%}\n truncate table {{ relation }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.737532, "supported_languages": null}, "macro.dbt.rename_relation": {"name": "rename_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.rename_relation", "macro_sql": "{% macro rename_relation(from_relation, to_relation) -%}\n {{ return(adapter.dispatch('rename_relation', 'dbt')(from_relation, to_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__rename_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.738038, "supported_languages": null}, "macro.dbt.default__rename_relation": {"name": "default__rename_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__rename_relation", "macro_sql": "{% macro default__rename_relation(from_relation, to_relation) -%}\n {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %}\n {% call statement('rename_relation') -%}\n alter table {{ from_relation }} rename to {{ target_name }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.7387478, "supported_languages": null}, "macro.dbt.get_or_create_relation": {"name": "get_or_create_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.get_or_create_relation", "macro_sql": "{% macro get_or_create_relation(database, schema, identifier, type) -%}\n {{ return(adapter.dispatch('get_or_create_relation', 'dbt')(database, schema, identifier, type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_or_create_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.7393658, "supported_languages": null}, "macro.dbt.default__get_or_create_relation": {"name": "default__get_or_create_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__get_or_create_relation", "macro_sql": "{% macro default__get_or_create_relation(database, schema, identifier, type) %}\n {%- set target_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n\n {% if target_relation %}\n {% do return([true, target_relation]) %}\n {% endif %}\n\n {%- set new_relation = api.Relation.create(\n database=database,\n schema=schema,\n identifier=identifier,\n type=type\n ) -%}\n {% do return([false, new_relation]) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.740736, "supported_languages": null}, "macro.dbt.load_cached_relation": {"name": "load_cached_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.load_cached_relation", "macro_sql": "{% macro load_cached_relation(relation) %}\n {% do return(adapter.get_relation(\n database=relation.database,\n schema=relation.schema,\n identifier=relation.identifier\n )) -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.741308, "supported_languages": null}, "macro.dbt.load_relation": {"name": "load_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.load_relation", "macro_sql": "{% macro load_relation(relation) %}\n {{ return(load_cached_relation(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.741668, "supported_languages": null}, "macro.dbt.drop_relation_if_exists": {"name": "drop_relation_if_exists", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.drop_relation_if_exists", "macro_sql": "{% macro drop_relation_if_exists(relation) %}\n {% if relation is not none %}\n {{ adapter.drop_relation(relation) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.7421598, "supported_languages": null}, "macro.dbt.collect_freshness": {"name": "collect_freshness", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "unique_id": "macro.dbt.collect_freshness", "macro_sql": "{% macro collect_freshness(source, loaded_at_field, filter) %}\n {{ return(adapter.dispatch('collect_freshness', 'dbt')(source, loaded_at_field, filter))}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__collect_freshness"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.743146, "supported_languages": null}, "macro.dbt.default__collect_freshness": {"name": "default__collect_freshness", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "unique_id": "macro.dbt.default__collect_freshness", "macro_sql": "{% macro default__collect_freshness(source, loaded_at_field, filter) %}\n {% call statement('collect_freshness', fetch_result=True, auto_begin=False) -%}\n select\n max({{ loaded_at_field }}) as max_loaded_at,\n {{ current_timestamp() }} as snapshotted_at\n from {{ source }}\n {% if filter %}\n where {{ filter }}\n {% endif %}\n {% endcall %}\n {{ return(load_result('collect_freshness')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.7441761, "supported_languages": null}, "macro.dbt.copy_grants": {"name": "copy_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.copy_grants", "macro_sql": "{% macro copy_grants() %}\n {{ return(adapter.dispatch('copy_grants', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__copy_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.7475011, "supported_languages": null}, "macro.dbt.default__copy_grants": {"name": "default__copy_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__copy_grants", "macro_sql": "{% macro default__copy_grants() %}\n {{ return(True) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.7478158, "supported_languages": null}, "macro.dbt.support_multiple_grantees_per_dcl_statement": {"name": "support_multiple_grantees_per_dcl_statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.support_multiple_grantees_per_dcl_statement", "macro_sql": "{% macro support_multiple_grantees_per_dcl_statement() %}\n {{ return(adapter.dispatch('support_multiple_grantees_per_dcl_statement', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__support_multiple_grantees_per_dcl_statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.748252, "supported_languages": null}, "macro.dbt.default__support_multiple_grantees_per_dcl_statement": {"name": "default__support_multiple_grantees_per_dcl_statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__support_multiple_grantees_per_dcl_statement", "macro_sql": "\n\n{%- macro default__support_multiple_grantees_per_dcl_statement() -%}\n {{ return(True) }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.7485409, "supported_languages": null}, "macro.dbt.should_revoke": {"name": "should_revoke", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.should_revoke", "macro_sql": "{% macro should_revoke(existing_relation, full_refresh_mode=True) %}\n\n {% if not existing_relation %}\n {#-- The table doesn't already exist, so no grants to copy over --#}\n {{ return(False) }}\n {% elif full_refresh_mode %}\n {#-- The object is being REPLACED -- whether grants are copied over depends on the value of user config --#}\n {{ return(copy_grants()) }}\n {% else %}\n {#-- The table is being merged/upserted/inserted -- grants will be carried over --#}\n {{ return(True) }}\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.copy_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.749697, "supported_languages": null}, "macro.dbt.get_show_grant_sql": {"name": "get_show_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_show_grant_sql", "macro_sql": "{% macro get_show_grant_sql(relation) %}\n {{ return(adapter.dispatch(\"get_show_grant_sql\", \"dbt\")(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_show_grant_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.750331, "supported_languages": null}, "macro.dbt.default__get_show_grant_sql": {"name": "default__get_show_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_show_grant_sql", "macro_sql": "{% macro default__get_show_grant_sql(relation) %}\n show grants on {{ relation }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.7506151, "supported_languages": null}, "macro.dbt.get_grant_sql": {"name": "get_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_grant_sql", "macro_sql": "{% macro get_grant_sql(relation, privilege, grantees) %}\n {{ return(adapter.dispatch('get_grant_sql', 'dbt')(relation, privilege, grantees)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_grant_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.751184, "supported_languages": null}, "macro.dbt.default__get_grant_sql": {"name": "default__get_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_grant_sql", "macro_sql": "\n\n{%- macro default__get_grant_sql(relation, privilege, grantees) -%}\n grant {{ privilege }} on {{ relation }} to {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.7516642, "supported_languages": null}, "macro.dbt.get_revoke_sql": {"name": "get_revoke_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_revoke_sql", "macro_sql": "{% macro get_revoke_sql(relation, privilege, grantees) %}\n {{ return(adapter.dispatch('get_revoke_sql', 'dbt')(relation, privilege, grantees)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_revoke_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.752241, "supported_languages": null}, "macro.dbt.default__get_revoke_sql": {"name": "default__get_revoke_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_revoke_sql", "macro_sql": "\n\n{%- macro default__get_revoke_sql(relation, privilege, grantees) -%}\n revoke {{ privilege }} on {{ relation }} from {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.752723, "supported_languages": null}, "macro.dbt.get_dcl_statement_list": {"name": "get_dcl_statement_list", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_dcl_statement_list", "macro_sql": "{% macro get_dcl_statement_list(relation, grant_config, get_dcl_macro) %}\n {{ return(adapter.dispatch('get_dcl_statement_list', 'dbt')(relation, grant_config, get_dcl_macro)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_dcl_statement_list"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.7532978, "supported_languages": null}, "macro.dbt.default__get_dcl_statement_list": {"name": "default__get_dcl_statement_list", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_dcl_statement_list", "macro_sql": "\n\n{%- macro default__get_dcl_statement_list(relation, grant_config, get_dcl_macro) -%}\n {#\n -- Unpack grant_config into specific privileges and the set of users who need them granted/revoked.\n -- Depending on whether this database supports multiple grantees per statement, pass in the list of\n -- all grantees per privilege, or (if not) template one statement per privilege-grantee pair.\n -- `get_dcl_macro` will be either `get_grant_sql` or `get_revoke_sql`\n #}\n {%- set dcl_statements = [] -%}\n {%- for privilege, grantees in grant_config.items() %}\n {%- if support_multiple_grantees_per_dcl_statement() and grantees -%}\n {%- set dcl = get_dcl_macro(relation, privilege, grantees) -%}\n {%- do dcl_statements.append(dcl) -%}\n {%- else -%}\n {%- for grantee in grantees -%}\n {% set dcl = get_dcl_macro(relation, privilege, [grantee]) %}\n {%- do dcl_statements.append(dcl) -%}\n {% endfor -%}\n {%- endif -%}\n {%- endfor -%}\n {{ return(dcl_statements) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.support_multiple_grantees_per_dcl_statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.7553658, "supported_languages": null}, "macro.dbt.call_dcl_statements": {"name": "call_dcl_statements", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.call_dcl_statements", "macro_sql": "{% macro call_dcl_statements(dcl_statement_list) %}\n {{ return(adapter.dispatch(\"call_dcl_statements\", \"dbt\")(dcl_statement_list)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__call_dcl_statements"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.756046, "supported_languages": null}, "macro.dbt.default__call_dcl_statements": {"name": "default__call_dcl_statements", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__call_dcl_statements", "macro_sql": "{% macro default__call_dcl_statements(dcl_statement_list) %}\n {#\n -- By default, supply all grant + revoke statements in a single semicolon-separated block,\n -- so that they're all processed together.\n\n -- Some databases do not support this. Those adapters will need to override this macro\n -- to run each statement individually.\n #}\n {% call statement('grants') %}\n {% for dcl_statement in dcl_statement_list %}\n {{ dcl_statement }};\n {% endfor %}\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.7566988, "supported_languages": null}, "macro.dbt.apply_grants": {"name": "apply_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.apply_grants", "macro_sql": "{% macro apply_grants(relation, grant_config, should_revoke) %}\n {{ return(adapter.dispatch(\"apply_grants\", \"dbt\")(relation, grant_config, should_revoke)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__apply_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.7572742, "supported_languages": null}, "macro.dbt.default__apply_grants": {"name": "default__apply_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__apply_grants", "macro_sql": "{% macro default__apply_grants(relation, grant_config, should_revoke=True) %}\n {#-- If grant_config is {} or None, this is a no-op --#}\n {% if grant_config %}\n {% if should_revoke %}\n {#-- We think previous grants may have carried over --#}\n {#-- Show current grants and calculate diffs --#}\n {% set current_grants_table = run_query(get_show_grant_sql(relation)) %}\n {% set current_grants_dict = adapter.standardize_grants_dict(current_grants_table) %}\n {% set needs_granting = diff_of_two_dicts(grant_config, current_grants_dict) %}\n {% set needs_revoking = diff_of_two_dicts(current_grants_dict, grant_config) %}\n {% if not (needs_granting or needs_revoking) %}\n {{ log('On ' ~ relation ~': All grants are in place, no revocation or granting needed.')}}\n {% endif %}\n {% else %}\n {#-- We don't think there's any chance of previous grants having carried over. --#}\n {#-- Jump straight to granting what the user has configured. --#}\n {% set needs_revoking = {} %}\n {% set needs_granting = grant_config %}\n {% endif %}\n {% if needs_granting or needs_revoking %}\n {% set revoke_statement_list = get_dcl_statement_list(relation, needs_revoking, get_revoke_sql) %}\n {% set grant_statement_list = get_dcl_statement_list(relation, needs_granting, get_grant_sql) %}\n {% set dcl_statement_list = revoke_statement_list + grant_statement_list %}\n {% if dcl_statement_list %}\n {{ call_dcl_statements(dcl_statement_list) }}\n {% endif %}\n {% endif %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query", "macro.dbt.get_show_grant_sql", "macro.dbt.get_dcl_statement_list", "macro.dbt.call_dcl_statements"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.760213, "supported_languages": null}, "macro.dbt.alter_column_comment": {"name": "alter_column_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.alter_column_comment", "macro_sql": "{% macro alter_column_comment(relation, column_dict) -%}\n {{ return(adapter.dispatch('alter_column_comment', 'dbt')(relation, column_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.761635, "supported_languages": null}, "macro.dbt.default__alter_column_comment": {"name": "default__alter_column_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.default__alter_column_comment", "macro_sql": "{% macro default__alter_column_comment(relation, column_dict) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_column_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.762066, "supported_languages": null}, "macro.dbt.alter_relation_comment": {"name": "alter_relation_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.alter_relation_comment", "macro_sql": "{% macro alter_relation_comment(relation, relation_comment) -%}\n {{ return(adapter.dispatch('alter_relation_comment', 'dbt')(relation, relation_comment)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__alter_relation_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.762573, "supported_languages": null}, "macro.dbt.default__alter_relation_comment": {"name": "default__alter_relation_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.default__alter_relation_comment", "macro_sql": "{% macro default__alter_relation_comment(relation, relation_comment) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_relation_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.762995, "supported_languages": null}, "macro.dbt.persist_docs": {"name": "persist_docs", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.persist_docs", "macro_sql": "{% macro persist_docs(relation, model, for_relation=true, for_columns=true) -%}\n {{ return(adapter.dispatch('persist_docs', 'dbt')(relation, model, for_relation, for_columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.7636638, "supported_languages": null}, "macro.dbt.default__persist_docs": {"name": "default__persist_docs", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.default__persist_docs", "macro_sql": "{% macro default__persist_docs(relation, model, for_relation, for_columns) -%}\n {% if for_relation and config.persist_relation_docs() and model.description %}\n {% do run_query(alter_relation_comment(relation, model.description)) %}\n {% endif %}\n\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {% do run_query(alter_column_comment(relation, model.columns)) %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query", "macro.dbt.alter_relation_comment", "macro.dbt.alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.76503, "supported_languages": null}, "macro.dbt.get_catalog": {"name": "get_catalog", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_catalog", "macro_sql": "{% macro get_catalog(information_schema, schemas) -%}\n {{ return(adapter.dispatch('get_catalog', 'dbt')(information_schema, schemas)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_catalog"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.767971, "supported_languages": null}, "macro.dbt.default__get_catalog": {"name": "default__get_catalog", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_catalog", "macro_sql": "{% macro default__get_catalog(information_schema, schemas) -%}\n\n {% set typename = adapter.type() %}\n {% set msg -%}\n get_catalog not implemented for {{ typename }}\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.7686348, "supported_languages": null}, "macro.dbt.information_schema_name": {"name": "information_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.information_schema_name", "macro_sql": "{% macro information_schema_name(database) %}\n {{ return(adapter.dispatch('information_schema_name', 'dbt')(database)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__information_schema_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.769115, "supported_languages": null}, "macro.dbt.default__information_schema_name": {"name": "default__information_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__information_schema_name", "macro_sql": "{% macro default__information_schema_name(database) -%}\n {%- if database -%}\n {{ database }}.INFORMATION_SCHEMA\n {%- else -%}\n INFORMATION_SCHEMA\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.769671, "supported_languages": null}, "macro.dbt.list_schemas": {"name": "list_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.list_schemas", "macro_sql": "{% macro list_schemas(database) -%}\n {{ return(adapter.dispatch('list_schemas', 'dbt')(database)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__list_schemas"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.770122, "supported_languages": null}, "macro.dbt.default__list_schemas": {"name": "default__list_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__list_schemas", "macro_sql": "{% macro default__list_schemas(database) -%}\n {% set sql %}\n select distinct schema_name\n from {{ information_schema_name(database) }}.SCHEMATA\n where catalog_name ilike '{{ database }}'\n {% endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.information_schema_name", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.770835, "supported_languages": null}, "macro.dbt.check_schema_exists": {"name": "check_schema_exists", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.check_schema_exists", "macro_sql": "{% macro check_schema_exists(information_schema, schema) -%}\n {{ return(adapter.dispatch('check_schema_exists', 'dbt')(information_schema, schema)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__check_schema_exists"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.771361, "supported_languages": null}, "macro.dbt.default__check_schema_exists": {"name": "default__check_schema_exists", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__check_schema_exists", "macro_sql": "{% macro default__check_schema_exists(information_schema, schema) -%}\n {% set sql -%}\n select count(*)\n from {{ information_schema.replace(information_schema_view='SCHEMATA') }}\n where catalog_name='{{ information_schema.database }}'\n and schema_name='{{ schema }}'\n {%- endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.replace", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.772129, "supported_languages": null}, "macro.dbt.list_relations_without_caching": {"name": "list_relations_without_caching", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.list_relations_without_caching", "macro_sql": "{% macro list_relations_without_caching(schema_relation) %}\n {{ return(adapter.dispatch('list_relations_without_caching', 'dbt')(schema_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__list_relations_without_caching"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.772596, "supported_languages": null}, "macro.dbt.default__list_relations_without_caching": {"name": "default__list_relations_without_caching", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__list_relations_without_caching", "macro_sql": "{% macro default__list_relations_without_caching(schema_relation) %}\n {{ exceptions.raise_not_implemented(\n 'list_relations_without_caching macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.7730088, "supported_languages": null}, "macro.dbt.get_columns_in_relation": {"name": "get_columns_in_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_columns_in_relation", "macro_sql": "{% macro get_columns_in_relation(relation) -%}\n {{ return(adapter.dispatch('get_columns_in_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_columns_in_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.776206, "supported_languages": null}, "macro.dbt.default__get_columns_in_relation": {"name": "default__get_columns_in_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_columns_in_relation", "macro_sql": "{% macro default__get_columns_in_relation(relation) -%}\n {{ exceptions.raise_not_implemented(\n 'get_columns_in_relation macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.776611, "supported_languages": null}, "macro.dbt.sql_convert_columns_in_relation": {"name": "sql_convert_columns_in_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.sql_convert_columns_in_relation", "macro_sql": "{% macro sql_convert_columns_in_relation(table) -%}\n {% set columns = [] %}\n {% for row in table %}\n {% do columns.append(api.Column(*row)) %}\n {% endfor %}\n {{ return(columns) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.777584, "supported_languages": null}, "macro.dbt.get_columns_in_query": {"name": "get_columns_in_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_columns_in_query", "macro_sql": "{% macro get_columns_in_query(select_sql) -%}\n {{ return(adapter.dispatch('get_columns_in_query', 'dbt')(select_sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_columns_in_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.77807, "supported_languages": null}, "macro.dbt.default__get_columns_in_query": {"name": "default__get_columns_in_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_columns_in_query", "macro_sql": "{% macro default__get_columns_in_query(select_sql) %}\n {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%}\n select * from (\n {{ select_sql }}\n ) as __dbt_sbq\n where false\n limit 0\n {% endcall %}\n\n {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.77891, "supported_languages": null}, "macro.dbt.alter_column_type": {"name": "alter_column_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.alter_column_type", "macro_sql": "{% macro alter_column_type(relation, column_name, new_column_type) -%}\n {{ return(adapter.dispatch('alter_column_type', 'dbt')(relation, column_name, new_column_type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__alter_column_type"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.77949, "supported_languages": null}, "macro.dbt.default__alter_column_type": {"name": "default__alter_column_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__alter_column_type", "macro_sql": "{% macro default__alter_column_type(relation, column_name, new_column_type) -%}\n {#\n 1. Create a new column (w/ temp name and correct type)\n 2. Copy data over to it\n 3. Drop the existing column (cascade!)\n 4. Rename the new column to existing column\n #}\n {%- set tmp_column = column_name + \"__dbt_alter\" -%}\n\n {% call statement('alter_column_type') %}\n alter table {{ relation }} add column {{ adapter.quote(tmp_column) }} {{ new_column_type }};\n update {{ relation }} set {{ adapter.quote(tmp_column) }} = {{ adapter.quote(column_name) }};\n alter table {{ relation }} drop column {{ adapter.quote(column_name) }} cascade;\n alter table {{ relation }} rename column {{ adapter.quote(tmp_column) }} to {{ adapter.quote(column_name) }}\n {% endcall %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.7810528, "supported_languages": null}, "macro.dbt.alter_relation_add_remove_columns": {"name": "alter_relation_add_remove_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.alter_relation_add_remove_columns", "macro_sql": "{% macro alter_relation_add_remove_columns(relation, add_columns = none, remove_columns = none) -%}\n {{ return(adapter.dispatch('alter_relation_add_remove_columns', 'dbt')(relation, add_columns, remove_columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__alter_relation_add_remove_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.781748, "supported_languages": null}, "macro.dbt.default__alter_relation_add_remove_columns": {"name": "default__alter_relation_add_remove_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__alter_relation_add_remove_columns", "macro_sql": "{% macro default__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}\n\n {% if add_columns is none %}\n {% set add_columns = [] %}\n {% endif %}\n {% if remove_columns is none %}\n {% set remove_columns = [] %}\n {% endif %}\n\n {% set sql -%}\n\n alter {{ relation.type }} {{ relation }}\n\n {% for column in add_columns %}\n add column {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }}\n {% endfor %}{{ ',' if add_columns and remove_columns }}\n\n {% for column in remove_columns %}\n drop column {{ column.name }}{{ ',' if not loop.last }}\n {% endfor %}\n\n {%- endset -%}\n\n {% do run_query(sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.7839968, "supported_languages": null}, "macro.dbt.build_ref_function": {"name": "build_ref_function", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.build_ref_function", "macro_sql": "{% macro build_ref_function(model) %}\n\n {%- set ref_dict = {} -%}\n {%- for _ref in model.refs -%}\n {%- set resolved = ref(*_ref) -%}\n {%- do ref_dict.update({_ref | join(\".\"): resolved | string | replace('\"', '\\\"')}) -%}\n {%- endfor -%}\n\ndef ref(*args,dbt_load_df_function):\n refs = {{ ref_dict | tojson }}\n key = \".\".join(args)\n return dbt_load_df_function(refs[key])\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.787274, "supported_languages": null}, "macro.dbt.build_source_function": {"name": "build_source_function", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.build_source_function", "macro_sql": "{% macro build_source_function(model) %}\n\n {%- set source_dict = {} -%}\n {%- for _source in model.sources -%}\n {%- set resolved = source(*_source) -%}\n {%- do source_dict.update({_source | join(\".\"): resolved | string | replace('\"', '\\\"')}) -%}\n {%- endfor -%}\n\ndef source(*args, dbt_load_df_function):\n sources = {{ source_dict | tojson }}\n key = \".\".join(args)\n return dbt_load_df_function(sources[key])\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.788432, "supported_languages": null}, "macro.dbt.build_config_dict": {"name": "build_config_dict", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.build_config_dict", "macro_sql": "{% macro build_config_dict(model) %}\n {%- set config_dict = {} -%}\n {% set config_dbt_used = zip(model.config.config_keys_used, model.config.config_keys_defaults) | list %}\n {%- for key, default in config_dbt_used -%}\n {# weird type testing with enum, would be much easier to write this logic in Python! #}\n {%- if key == \"language\" -%}\n {%- set value = \"python\" -%}\n {%- endif -%}\n {%- set value = model.config.get(key, default) -%}\n {%- do config_dict.update({key: value}) -%}\n {%- endfor -%}\nconfig_dict = {{ config_dict }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.790293, "supported_languages": null}, "macro.dbt.py_script_postfix": {"name": "py_script_postfix", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.py_script_postfix", "macro_sql": "{% macro py_script_postfix(model) %}\n# This part is user provided model code\n# you will need to copy the next section to run the code\n# COMMAND ----------\n# this part is dbt logic for get ref work, do not modify\n\n{{ build_ref_function(model ) }}\n{{ build_source_function(model ) }}\n{{ build_config_dict(model) }}\n\nclass config:\n def __init__(self, *args, **kwargs):\n pass\n\n @staticmethod\n def get(key, default=None):\n return config_dict.get(key, default)\n\nclass this:\n \"\"\"dbt.this() or dbt.this.identifier\"\"\"\n database = \"{{ this.database }}\"\n schema = \"{{ this.schema }}\"\n identifier = \"{{ this.identifier }}\"\n {% set this_relation_name = this | string | replace('\"', '\\\\\"') %}\n def __repr__(self):\n return \"{{ this_relation_name }}\"\n\n\nclass dbtObj:\n def __init__(self, load_df_function) -> None:\n self.source = lambda *args: source(*args, dbt_load_df_function=load_df_function)\n self.ref = lambda *args: ref(*args, dbt_load_df_function=load_df_function)\n self.config = config\n self.this = this()\n self.is_incremental = {{ is_incremental() }}\n\n# COMMAND ----------\n{{py_script_comment()}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.build_ref_function", "macro.dbt.build_source_function", "macro.dbt.build_config_dict", "macro.dbt.is_incremental", "macro.dbt.py_script_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.791504, "supported_languages": null}, "macro.dbt.py_script_comment": {"name": "py_script_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.py_script_comment", "macro_sql": "{%macro py_script_comment()%}\n{%endmacro%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.791784, "supported_languages": null}, "macro.dbt.test_unique": {"name": "test_unique", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_unique", "macro_sql": "{% test unique(model, column_name) %}\n {% set macro = adapter.dispatch('test_unique', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_unique"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.7929192, "supported_languages": null}, "macro.dbt.test_not_null": {"name": "test_not_null", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_not_null", "macro_sql": "{% test not_null(model, column_name) %}\n {% set macro = adapter.dispatch('test_not_null', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_not_null"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.793498, "supported_languages": null}, "macro.dbt.test_accepted_values": {"name": "test_accepted_values", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_accepted_values", "macro_sql": "{% test accepted_values(model, column_name, values, quote=True) %}\n {% set macro = adapter.dispatch('test_accepted_values', 'dbt') %}\n {{ macro(model, column_name, values, quote) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_accepted_values"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.794215, "supported_languages": null}, "macro.dbt.test_relationships": {"name": "test_relationships", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_relationships", "macro_sql": "{% test relationships(model, column_name, to, field) %}\n {% set macro = adapter.dispatch('test_relationships', 'dbt') %}\n {{ macro(model, column_name, to, field) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_relationships"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1676322281.7948952, "supported_languages": null}}, "docs": {"doc.test.somedoc": {"name": "somedoc", "resource_type": "doc", "package_name": "test", "path": "somedoc.md", "original_file_path": "models/somedoc.md", "unique_id": "doc.test.somedoc", "block_contents": "Testing, testing"}, "doc.dbt.__overview__": {"name": "__overview__", "resource_type": "doc", "package_name": "dbt", "path": "overview.md", "original_file_path": "docs/overview.md", "unique_id": "doc.dbt.__overview__", "block_contents": "### Welcome!\n\nWelcome to the auto-generated documentation for your dbt project!\n\n### Navigation\n\nYou can use the `Project` and `Database` navigation tabs on the left side of the window to explore the models\nin your project.\n\n#### Project Tab\nThe `Project` tab mirrors the directory structure of your dbt project. In this tab, you can see all of the\nmodels defined in your dbt project, as well as models imported from dbt packages.\n\n#### Database Tab\nThe `Database` tab also exposes your models, but in a format that looks more like a database explorer. This view\nshows relations (tables and views) grouped into database schemas. Note that ephemeral models are _not_ shown\nin this interface, as they do not exist in the database.\n\n### Graph Exploration\nYou can click the blue icon on the bottom-right corner of the page to view the lineage graph of your models.\n\nOn model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the `Expand`\nbutton at the top-right of this lineage pane, you'll be able to see all of the models that are used to build,\nor are built from, the model you're exploring.\n\nOnce expanded, you'll be able to use the `--select` and `--exclude` model selection syntax to filter the\nmodels in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax).\n\nNote that you can also right-click on models to interactively filter and explore the graph.\n\n---\n\n### More information\n\n- [What is dbt](https://docs.getdbt.com/docs/introduction)?\n- Read the [dbt viewpoint](https://docs.getdbt.com/docs/viewpoint)\n- [Installation](https://docs.getdbt.com/docs/installation)\n- Join the [dbt Community](https://www.getdbt.com/community/) for questions and discussion"}}, "exposures": {"exposure.test.simple_exposure": {"name": "simple_exposure", "resource_type": "exposure", "package_name": "test", "path": "schema.yml", "original_file_path": "models/schema.yml", "unique_id": "exposure.test.simple_exposure", "fqn": ["test", "simple_exposure"], "type": "dashboard", "owner": {"email": "something@example.com", "name": null}, "description": "", "label": null, "maturity": null, "meta": {}, "tags": [], "config": {"enabled": true}, "unrendered_config": {}, "url": null, "depends_on": {"macros": [], "nodes": ["source.test.my_source.my_table", "model.test.my_model"]}, "refs": [["my_model"]], "sources": [["my_source", "my_table"]], "metrics": [], "created_at": 1676322282.478955}}, "metrics": {"metric.test.my_metric": {"name": "my_metric", "resource_type": "metric", "package_name": "test", "path": "schema.yml", "original_file_path": "models/schema.yml", "unique_id": "metric.test.my_metric", "fqn": ["test", "my_metric"], "description": "", "label": "Count records", "calculation_method": "count", "expression": "*", "filters": [], "time_grains": ["day"], "dimensions": [], "timestamp": "updated_at", "window": null, "model": "ref('my_model')", "model_unique_id": null, "meta": {}, "tags": [], "config": {"enabled": true}, "unrendered_config": {}, "sources": [], "depends_on": {"macros": [], "nodes": ["model.test.my_model"]}, "refs": [["my_model"]], "metrics": [], "created_at": 1676322282.491698}}, "selectors": {}, "disabled": {"model.test.disabled_model": [{"database": "dbt", "schema": "test16763222812618906995_test_previous_version_state", "name": "disabled_model", "resource_type": "model", "package_name": "test", "path": "disabled_model.sql", "original_file_path": "models/disabled_model.sql", "unique_id": "model.test.disabled_model", "fqn": ["test", "disabled_model"], "alias": "disabled_model", "checksum": {"name": "sha256", "checksum": "34f7b8e60d9e7933469c48d6c92b0a53918d0ba626a9ce2c30ab2f1532145827"}, "config": {"enabled": false, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "post-hook": [], "pre-hook": []}, "tags": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {"enabled": false}, "created_at": 1676322282.224511, "config_call_dict": {"enabled": false}, "relation_name": "\"dbt\".\"test16763222812618906995_test_previous_version_state\".\"disabled_model\"", "raw_code": "{{ config(enabled=False) }}\nselect 2 as id", "language": "sql", "refs": [], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": null}], "snapshot.test.disabled_snapshot_seed": [{"database": "dbt", "schema": "test16763222812618906995_test_previous_version_state", "name": "disabled_snapshot_seed", "resource_type": "snapshot", "package_name": "test", "path": "disabled_snapshot_seed.sql", "original_file_path": "snapshots/disabled_snapshot_seed.sql", "unique_id": "snapshot.test.disabled_snapshot_seed", "fqn": ["test", "disabled_snapshot_seed", "disabled_snapshot_seed"], "alias": "disabled_snapshot_seed", "checksum": {"name": "sha256", "checksum": "52b08465e16dcbc364162dfbdb34cf25e04295bc13d63ab0b420f60d15234c76"}, "config": {"enabled": false, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "snapshot", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": "id", "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "strategy": "check", "target_schema": "test16763222812618906995_test_previous_version_state", "target_database": null, "updated_at": null, "check_cols": "all", "post-hook": [], "pre-hook": []}, "tags": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {"unique_key": "id", "strategy": "check", "check_cols": "all", "target_schema": "test16763222812618906995_test_previous_version_state", "enabled": false}, "created_at": 1676322282.303265, "config_call_dict": {"unique_key": "id", "strategy": "check", "check_cols": "all", "target_schema": "test16763222812618906995_test_previous_version_state", "enabled": false}, "relation_name": "\"dbt\".\"test16763222812618906995_test_previous_version_state\".\"disabled_snapshot_seed\"", "raw_code": "\n{{\n config(\n unique_key='id',\n strategy='check',\n check_cols='all',\n target_schema=schema,\n enabled=False,\n )\n}}\nselect * from {{ ref('my_seed') }}\n", "language": "sql", "refs": [["my_seed"]], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": null}], "analysis.test.disabled_al": [{"database": "dbt", "schema": "test16763222812618906995_test_previous_version_state", "name": "disabled_al", "resource_type": "analysis", "package_name": "test", "path": "analysis/disabled_al.sql", "original_file_path": "analyses/disabled_al.sql", "unique_id": "analysis.test.disabled_al", "fqn": ["test", "analysis", "disabled_al"], "alias": "disabled_al", "checksum": {"name": "sha256", "checksum": "76b8579816eac97721616fd429dcd1a93c311c6358830a65d40ebe5661572610"}, "config": {"enabled": false, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "post-hook": [], "pre-hook": []}, "tags": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {"enabled": false}, "created_at": 1676322282.3320582, "config_call_dict": {"enabled": false}, "relation_name": null, "raw_code": "{{ config(enabled=False) }}\nselect 9 as id", "language": "sql", "refs": [], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": null}], "test.test.disabled_just_my": [{"database": "dbt", "schema": "test16763222812618906995_test_previous_version_state_dbt_test__audit", "name": "disabled_just_my", "resource_type": "test", "package_name": "test", "path": "disabled_just_my.sql", "original_file_path": "tests/disabled_just_my.sql", "unique_id": "test.test.disabled_just_my", "fqn": ["test", "disabled_just_my"], "alias": "disabled_just_my", "checksum": {"name": "sha256", "checksum": "0b5827d08d1e3c97e8fb865bea00031b2e90ecef7884a42429cc48d0f48b8c20"}, "config": {"enabled": false, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {"enabled": false}, "created_at": 1676322282.359573, "config_call_dict": {"enabled": false}, "relation_name": null, "raw_code": "{{ config(enabled=False) }}\n\nselect * from {{ ref('my_model') }}\nwhere false", "language": "sql", "refs": [["my_model"]], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": null}], "test.test.disabled_check_nothing_my_model_.f2c6a72d37": [{"test_metadata": {"name": "disabled_check_nothing", "kwargs": {"model": "{{ get_where_subquery(ref('my_model')) }}"}, "namespace": null}, "database": "dbt", "schema": "test16763222812618906995_test_previous_version_state_dbt_test__audit", "name": "disabled_check_nothing_my_model_", "resource_type": "test", "package_name": "test", "path": "disabled_check_nothing_my_model_.sql", "original_file_path": "models/schema.yml", "unique_id": "test.test.disabled_check_nothing_my_model_.f2c6a72d37", "fqn": ["test", "disabled_check_nothing_my_model_"], "alias": "disabled_check_nothing_my_model_", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": false, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {"enabled": false}, "created_at": 1676322282.455549, "config_call_dict": {"enabled": false}, "relation_name": null, "raw_code": "{{ test_disabled_check_nothing(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [["my_model"]], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.test.test_disabled_check_nothing", "macro.dbt.get_where_subquery"], "nodes": []}, "compiled_path": null, "column_name": null, "file_key_name": "models.my_model"}], "exposure.test.disabled_exposure": [{"name": "disabled_exposure", "resource_type": "exposure", "package_name": "test", "path": "schema.yml", "original_file_path": "models/schema.yml", "unique_id": "exposure.test.disabled_exposure", "fqn": ["test", "disabled_exposure"], "type": "dashboard", "owner": {"email": "something@example.com", "name": null}, "description": "", "label": null, "maturity": null, "meta": {}, "tags": [], "config": {"enabled": false}, "unrendered_config": {"enabled": false}, "url": null, "depends_on": {"macros": [], "nodes": []}, "refs": [["my_model"]], "sources": [], "metrics": [], "created_at": 1676322282.482795}], "metric.test.disabled_metric": [{"name": "disabled_metric", "resource_type": "metric", "package_name": "test", "path": "schema.yml", "original_file_path": "models/schema.yml", "unique_id": "metric.test.disabled_metric", "fqn": ["test", "disabled_metric"], "description": "", "label": "Count records", "calculation_method": "count", "expression": "*", "filters": [], "time_grains": ["day"], "dimensions": [], "timestamp": "updated_at", "window": null, "model": "ref('my_model')", "model_unique_id": null, "meta": {}, "tags": [], "config": {"enabled": false}, "unrendered_config": {"enabled": false}, "sources": [], "depends_on": {"macros": [], "nodes": []}, "refs": [["my_model"]], "metrics": [], "created_at": 1676322282.495338}], "seed.test.disabled_seed": [{"database": "dbt", "schema": "test16763222812618906995_test_previous_version_state", "name": "disabled_seed", "resource_type": "seed", "package_name": "test", "path": "disabled_seed.csv", "original_file_path": "seeds/disabled_seed.csv", "unique_id": "seed.test.disabled_seed", "fqn": ["test", "disabled_seed"], "alias": "disabled_seed", "checksum": {"name": "sha256", "checksum": "c6c08a913b5a382014ef0ba248d97b12fc801beb369fdbd24aff1a3912ee3773"}, "config": {"enabled": false, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "seed", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "quote_columns": null, "post-hook": [], "pre-hook": []}, "tags": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": "test://models/schema.yml", "build_path": null, "deferred": false, "unrendered_config": {"enabled": false}, "created_at": 1676322282.462719, "config_call_dict": {}, "relation_name": "\"dbt\".\"test16763222812618906995_test_previous_version_state\".\"disabled_seed\"", "raw_code": "", "root_path": "/private/var/folders/qt/vw8wqdgx4w381wh14b9y25m40000gn/T/pytest-of-gerda/pytest-115/project0", "depends_on": {"macros": []}}], "source.test.my_source.disabled_table": [{"database": "dbt", "schema": "my_source", "name": "disabled_table", "resource_type": "source", "package_name": "test", "path": "models/schema.yml", "original_file_path": "models/schema.yml", "unique_id": "source.test.my_source.disabled_table", "fqn": ["test", "my_source", "disabled_table"], "source_name": "my_source", "source_description": "My source", "loader": "a_loader", "identifier": "disabled_table", "quoting": {"database": null, "schema": null, "identifier": null, "column": null}, "loaded_at_field": null, "freshness": {"warn_after": {"count": null, "period": null}, "error_after": {"count": null, "period": null}, "filter": null}, "external": null, "description": "Disabled table", "columns": {}, "meta": {}, "source_meta": {}, "tags": [], "config": {"enabled": false}, "patch_path": null, "unrendered_config": {"enabled": false}, "relation_name": "\"dbt\".\"my_source\".\"disabled_table\"", "created_at": 1676322282.498409}]}, "parent_map": {"model.test.my_model": [], "snapshot.test.snapshot_seed": ["seed.test.my_seed"], "analysis.test.a": [], "test.test.just_my": ["model.test.my_model"], "seed.test.my_seed": [], "test.test.not_null_my_model_id.43e0e9183a": ["model.test.my_model"], "test.test.check_nothing_my_model_.d5a5e66110": ["model.test.my_model"], "source.test.my_source.my_table": [], "exposure.test.simple_exposure": ["model.test.my_model", "source.test.my_source.my_table"], "metric.test.my_metric": ["model.test.my_model"]}, "child_map": {"model.test.my_model": ["exposure.test.simple_exposure", "metric.test.my_metric", "test.test.check_nothing_my_model_.d5a5e66110", "test.test.just_my", "test.test.not_null_my_model_id.43e0e9183a"], "snapshot.test.snapshot_seed": [], "analysis.test.a": [], "test.test.just_my": [], "seed.test.my_seed": ["snapshot.test.snapshot_seed"], "test.test.not_null_my_model_id.43e0e9183a": [], "test.test.check_nothing_my_model_.d5a5e66110": [], "source.test.my_source.my_table": ["exposure.test.simple_exposure"], "exposure.test.simple_exposure": [], "metric.test.my_metric": []}} ================================================ FILE: tests/functional/artifacts/data/state/v9/manifest.json ================================================ {"metadata": {"dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v9.json", "dbt_version": "1.5.0b5", "generated_at": "2023-04-10T02:53:50.434615Z", "invocation_id": "7e6390ca-c227-4a45-b9e0-85eeb260e9a8", "env": {}, "project_id": "098f6bcd4621d373cade4e832627b4f6", "user_id": null, "send_anonymous_usage_stats": false, "adapter_type": "postgres"}, "nodes": {"model.test.my_model": {"database": "dbt", "schema": "test16810952296205305560_test_previous_version_state", "name": "my_model", "resource_type": "model", "package_name": "test", "path": "my_model.sql", "original_file_path": "models/my_model.sql", "unique_id": "model.test.my_model", "fqn": ["test", "my_model"], "alias": "my_model", "checksum": {"name": "sha256", "checksum": "3ea0f972fa1b56aa2dc2f56ee784b6a5796312f9a813d59ae70fd8855f10d16d"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false}, "post-hook": [], "pre-hook": []}, "tags": [], "description": "Example model", "columns": {"id": {"name": "id", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": "test://models/schema.yml", "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1681095229.898038, "relation_name": "\"dbt\".\"test16810952296205305560_test_previous_version_state\".\"my_model\"", "raw_code": "select 1 as id", "language": "sql", "refs": [], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": null, "contract": {"enforced": false, "checksum": null}, "access": "protected", "version": null, "is_latest_version": null}, "snapshot.test.snapshot_seed": {"database": "dbt", "schema": "test16810952296205305560_test_previous_version_state", "name": "snapshot_seed", "resource_type": "snapshot", "package_name": "test", "path": "snapshot_seed.sql", "original_file_path": "snapshots/snapshot_seed.sql", "unique_id": "snapshot.test.snapshot_seed", "fqn": ["test", "snapshot_seed", "snapshot_seed"], "alias": "snapshot_seed", "checksum": {"name": "sha256", "checksum": "5fc998f39655f8fe52443a919e749b6e23883ef90202b040412baac13c6bfe18"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "snapshot", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": "id", "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false}, "strategy": "check", "target_schema": "test16810952296205305560_test_previous_version_state", "target_database": null, "updated_at": null, "check_cols": "all", "post-hook": [], "pre-hook": []}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {"unique_key": "id", "strategy": "check", "check_cols": "all", "target_schema": "test16810952296205305560_test_previous_version_state"}, "created_at": 1681095229.843765, "relation_name": "\"dbt\".\"test16810952296205305560_test_previous_version_state\".\"snapshot_seed\"", "raw_code": "\n{{\n config(\n unique_key='id',\n strategy='check',\n check_cols='all',\n target_schema=schema,\n )\n}}\nselect * from {{ ref('my_seed') }}\n", "language": "sql", "refs": [{"name": "my_seed", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["seed.test.my_seed"]}, "compiled_path": null, "contract": {"enforced": false, "checksum": null}}, "analysis.test.a": {"database": "dbt", "schema": "test16810952296205305560_test_previous_version_state", "name": "a", "resource_type": "analysis", "package_name": "test", "path": "analysis/a.sql", "original_file_path": "analyses/a.sql", "unique_id": "analysis.test.a", "fqn": ["test", "analysis", "a"], "alias": "a", "checksum": {"name": "sha256", "checksum": "a389c282f569f0bbdc2a8a4f174dea746c28582fdaf2048d31d9226af9feab23"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false}, "post-hook": [], "pre-hook": []}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1681095229.8655732, "relation_name": null, "raw_code": "select 4 as id", "language": "sql", "refs": [], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": null, "contract": {"enforced": false, "checksum": null}}, "test.test.just_my": {"database": "dbt", "schema": "test16810952296205305560_test_previous_version_state_dbt_test__audit", "name": "just_my", "resource_type": "test", "package_name": "test", "path": "just_my.sql", "original_file_path": "tests/just_my.sql", "unique_id": "test.test.just_my", "fqn": ["test", "just_my"], "alias": "just_my", "checksum": {"name": "sha256", "checksum": "744889a2e2d9ce380619265e1217d7ccf6e6ca896c048d42ebe0f9cfb74d7156"}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": ["data_test_tag"], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": ["data_test_tag"], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {"tags": ["data_test_tag"]}, "created_at": 1681095229.884334, "relation_name": null, "raw_code": "{{ config(tags = ['data_test_tag']) }}\n\nselect * from {{ ref('my_model') }}\nwhere false", "language": "sql", "refs": [{"name": "my_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["model.test.my_model"]}, "compiled_path": null, "contract": {"enforced": false, "checksum": null}}, "seed.test.my_seed": {"database": "dbt", "schema": "test16810952296205305560_test_previous_version_state", "name": "my_seed", "resource_type": "seed", "package_name": "test", "path": "my_seed.csv", "original_file_path": "seeds/my_seed.csv", "unique_id": "seed.test.my_seed", "fqn": ["test", "my_seed"], "alias": "my_seed", "checksum": {"name": "sha256", "checksum": "f7ede83f36165ac6b7a047aa2c3f212dff385bfa9f35f395108cd06fc8e96943"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "seed", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false}, "quote_columns": null, "post-hook": [], "pre-hook": []}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1681095229.889285, "relation_name": "\"dbt\".\"test16810952296205305560_test_previous_version_state\".\"my_seed\"", "raw_code": "", "root_path": "/private/var/folders/k6/gtt07v8j2vn51m_z05xk_fjc0000gp/T/pytest-of-michelleark/pytest-80/project5", "depends_on": {"macros": []}}, "test.test.not_null_my_model_id.43e0e9183a": {"test_metadata": {"name": "not_null", "kwargs": {"column_name": "id", "model": "{{ get_where_subquery(ref('my_model')) }}"}, "namespace": null}, "database": "dbt", "schema": "test16810952296205305560_test_previous_version_state_dbt_test__audit", "name": "not_null_my_model_id", "resource_type": "test", "package_name": "test", "path": "not_null_my_model_id.sql", "original_file_path": "models/schema.yml", "unique_id": "test.test.not_null_my_model_id.43e0e9183a", "fqn": ["test", "not_null_my_model_id"], "alias": "not_null_my_model_id", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1681095229.898516, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "my_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.test.my_model"]}, "compiled_path": null, "contract": {"enforced": false, "checksum": null}, "column_name": "id", "file_key_name": "models.my_model", "attached_node": "model.test.my_model"}, "test.test.check_nothing_my_model_.d5a5e66110": {"test_metadata": {"name": "check_nothing", "kwargs": {"model": "{{ get_where_subquery(ref('my_model')) }}"}, "namespace": null}, "database": "dbt", "schema": "test16810952296205305560_test_previous_version_state_dbt_test__audit", "name": "check_nothing_my_model_", "resource_type": "test", "package_name": "test", "path": "check_nothing_my_model_.sql", "original_file_path": "models/schema.yml", "unique_id": "test.test.check_nothing_my_model_.d5a5e66110", "fqn": ["test", "check_nothing_my_model_"], "alias": "check_nothing_my_model_", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1681095229.900049, "relation_name": null, "raw_code": "{{ test_check_nothing(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "my_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.test.test_check_nothing", "macro.dbt.get_where_subquery"], "nodes": ["model.test.my_model"]}, "compiled_path": null, "contract": {"enforced": false, "checksum": null}, "column_name": null, "file_key_name": "models.my_model", "attached_node": "model.test.my_model"}}, "sources": {"source.test.my_source.my_table": {"database": "dbt", "schema": "my_source", "name": "my_table", "resource_type": "source", "package_name": "test", "path": "models/schema.yml", "original_file_path": "models/schema.yml", "unique_id": "source.test.my_source.my_table", "fqn": ["test", "my_source", "my_table"], "source_name": "my_source", "source_description": "My source", "loader": "a_loader", "identifier": "my_seed", "quoting": {"database": null, "schema": null, "identifier": null, "column": null}, "loaded_at_field": null, "freshness": {"warn_after": {"count": null, "period": null}, "error_after": {"count": null, "period": null}, "filter": null}, "external": null, "description": "My table", "columns": {}, "meta": {}, "source_meta": {}, "tags": [], "config": {"enabled": true}, "patch_path": null, "unrendered_config": {}, "relation_name": "\"dbt\".\"my_source\".\"my_seed\"", "created_at": 1681095229.938866}}, "macros": {"macro.test.test_check_nothing": {"name": "test_check_nothing", "resource_type": "macro", "package_name": "test", "path": "macros/dummy_test.sql", "original_file_path": "macros/dummy_test.sql", "unique_id": "macro.test.test_check_nothing", "macro_sql": "{% test check_nothing(model) %}\n-- a silly test to make sure that table-level tests show up in the manifest\n-- without a column_name field\n\nselect 0\n\n{% endtest %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.549095, "supported_languages": null}, "macro.test.test_disabled_check_nothing": {"name": "test_disabled_check_nothing", "resource_type": "macro", "package_name": "test", "path": "macros/disabled_dummy_test.sql", "original_file_path": "macros/disabled_dummy_test.sql", "unique_id": "macro.test.test_disabled_check_nothing", "macro_sql": "{% test disabled_check_nothing(model) %}\n-- a silly test to make sure that table-level tests show up in the manifest\n-- without a column_name field\n\n{{ config(enabled=False) }}\nselect 0\n\n{% endtest %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.549314, "supported_languages": null}, "macro.test.do_nothing": {"name": "do_nothing", "resource_type": "macro", "package_name": "test", "path": "macros/do_nothing.sql", "original_file_path": "macros/do_nothing.sql", "unique_id": "macro.test.do_nothing", "macro_sql": "{% macro do_nothing(foo2, bar2) %}\n select\n '{{ foo2 }}' as foo2,\n '{{ bar2 }}' as bar2\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.549501, "supported_languages": null}, "macro.dbt_postgres.postgres__current_timestamp": {"name": "postgres__current_timestamp", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/timestamps.sql", "original_file_path": "macros/timestamps.sql", "unique_id": "macro.dbt_postgres.postgres__current_timestamp", "macro_sql": "{% macro postgres__current_timestamp() -%}\n now()\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.5497909, "supported_languages": null}, "macro.dbt_postgres.postgres__snapshot_string_as_time": {"name": "postgres__snapshot_string_as_time", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/timestamps.sql", "original_file_path": "macros/timestamps.sql", "unique_id": "macro.dbt_postgres.postgres__snapshot_string_as_time", "macro_sql": "{% macro postgres__snapshot_string_as_time(timestamp) -%}\n {%- set result = \"'\" ~ timestamp ~ \"'::timestamp without time zone\" -%}\n {{ return(result) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.549994, "supported_languages": null}, "macro.dbt_postgres.postgres__snapshot_get_time": {"name": "postgres__snapshot_get_time", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/timestamps.sql", "original_file_path": "macros/timestamps.sql", "unique_id": "macro.dbt_postgres.postgres__snapshot_get_time", "macro_sql": "{% macro postgres__snapshot_get_time() -%}\n {{ current_timestamp() }}::timestamp without time zone\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.5501041, "supported_languages": null}, "macro.dbt_postgres.postgres__current_timestamp_backcompat": {"name": "postgres__current_timestamp_backcompat", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/timestamps.sql", "original_file_path": "macros/timestamps.sql", "unique_id": "macro.dbt_postgres.postgres__current_timestamp_backcompat", "macro_sql": "{% macro postgres__current_timestamp_backcompat() %}\n current_timestamp::{{ type_timestamp() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.type_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.5502121, "supported_languages": null}, "macro.dbt_postgres.postgres__current_timestamp_in_utc_backcompat": {"name": "postgres__current_timestamp_in_utc_backcompat", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/timestamps.sql", "original_file_path": "macros/timestamps.sql", "unique_id": "macro.dbt_postgres.postgres__current_timestamp_in_utc_backcompat", "macro_sql": "{% macro postgres__current_timestamp_in_utc_backcompat() %}\n (current_timestamp at time zone 'utc')::{{ type_timestamp() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.type_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.5503209, "supported_languages": null}, "macro.dbt_postgres.postgres__get_catalog": {"name": "postgres__get_catalog", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/catalog.sql", "original_file_path": "macros/catalog.sql", "unique_id": "macro.dbt_postgres.postgres__get_catalog", "macro_sql": "{% macro postgres__get_catalog(information_schema, schemas) -%}\n\n {%- call statement('catalog', fetch_result=True) -%}\n {#\n If the user has multiple databases set and the first one is wrong, this will fail.\n But we won't fail in the case where there are multiple quoting-difference-only dbs, which is better.\n #}\n {% set database = information_schema.database %}\n {{ adapter.verify_database(database) }}\n\n select\n '{{ database }}' as table_database,\n sch.nspname as table_schema,\n tbl.relname as table_name,\n case tbl.relkind\n when 'v' then 'VIEW'\n else 'BASE TABLE'\n end as table_type,\n tbl_desc.description as table_comment,\n col.attname as column_name,\n col.attnum as column_index,\n pg_catalog.format_type(col.atttypid, col.atttypmod) as column_type,\n col_desc.description as column_comment,\n pg_get_userbyid(tbl.relowner) as table_owner\n\n from pg_catalog.pg_namespace sch\n join pg_catalog.pg_class tbl on tbl.relnamespace = sch.oid\n join pg_catalog.pg_attribute col on col.attrelid = tbl.oid\n left outer join pg_catalog.pg_description tbl_desc on (tbl_desc.objoid = tbl.oid and tbl_desc.objsubid = 0)\n left outer join pg_catalog.pg_description col_desc on (col_desc.objoid = tbl.oid and col_desc.objsubid = col.attnum)\n\n where (\n {%- for schema in schemas -%}\n upper(sch.nspname) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n )\n and not pg_is_other_temp_schema(sch.oid) -- not a temporary schema belonging to another session\n and tbl.relpersistence in ('p', 'u') -- [p]ermanent table or [u]nlogged table. Exclude [t]emporary tables\n and tbl.relkind in ('r', 'v', 'f', 'p') -- o[r]dinary table, [v]iew, [f]oreign table, [p]artitioned table. Other values are [i]ndex, [S]equence, [c]omposite type, [t]OAST table, [m]aterialized view\n and col.attnum > 0 -- negative numbers are used for system columns such as oid\n and not col.attisdropped -- column as not been dropped\n\n order by\n sch.nspname,\n tbl.relname,\n col.attnum\n\n {%- endcall -%}\n\n {{ return(load_result('catalog').table) }}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.551399, "supported_languages": null}, "macro.dbt_postgres.postgres_get_relations": {"name": "postgres_get_relations", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations.sql", "original_file_path": "macros/relations.sql", "unique_id": "macro.dbt_postgres.postgres_get_relations", "macro_sql": "{% macro postgres_get_relations () -%}\n\n {#\n -- in pg_depend, objid is the dependent, refobjid is the referenced object\n -- > a pg_depend entry indicates that the referenced object cannot be\n -- > dropped without also dropping the dependent object.\n #}\n\n {%- call statement('relations', fetch_result=True) -%}\n with relation as (\n select\n pg_rewrite.ev_class as class,\n pg_rewrite.oid as id\n from pg_rewrite\n ),\n class as (\n select\n oid as id,\n relname as name,\n relnamespace as schema,\n relkind as kind\n from pg_class\n ),\n dependency as (\n select distinct\n pg_depend.objid as id,\n pg_depend.refobjid as ref\n from pg_depend\n ),\n schema as (\n select\n pg_namespace.oid as id,\n pg_namespace.nspname as name\n from pg_namespace\n where nspname != 'information_schema' and nspname not like 'pg\\_%'\n ),\n referenced as (\n select\n relation.id AS id,\n referenced_class.name ,\n referenced_class.schema ,\n referenced_class.kind\n from relation\n join class as referenced_class on relation.class=referenced_class.id\n where referenced_class.kind in ('r', 'v')\n ),\n relationships as (\n select\n referenced.name as referenced_name,\n referenced.schema as referenced_schema_id,\n dependent_class.name as dependent_name,\n dependent_class.schema as dependent_schema_id,\n referenced.kind as kind\n from referenced\n join dependency on referenced.id=dependency.id\n join class as dependent_class on dependency.ref=dependent_class.id\n where\n (referenced.name != dependent_class.name or\n referenced.schema != dependent_class.schema)\n )\n\n select\n referenced_schema.name as referenced_schema,\n relationships.referenced_name as referenced_name,\n dependent_schema.name as dependent_schema,\n relationships.dependent_name as dependent_name\n from relationships\n join schema as dependent_schema on relationships.dependent_schema_id=dependent_schema.id\n join schema as referenced_schema on relationships.referenced_schema_id=referenced_schema.id\n group by referenced_schema, referenced_name, dependent_schema, dependent_name\n order by referenced_schema, referenced_name, dependent_schema, dependent_name;\n\n {%- endcall -%}\n\n {{ return(load_result('relations').table) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.552023, "supported_languages": null}, "macro.dbt_postgres.postgres__create_table_as": {"name": "postgres__create_table_as", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__create_table_as", "macro_sql": "{% macro postgres__create_table_as(temporary, relation, sql) -%}\n {%- set unlogged = config.get('unlogged', default=false) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary -%}\n temporary\n {%- elif unlogged -%}\n unlogged\n {%- endif %} table {{ relation }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {{ get_columns_spec_ddl() }} ;\n insert into {{ relation }} {{ get_column_names() }}\n {%- set sql = get_select_subquery(sql) %}\n {% else %}\n as\n {% endif %}\n (\n {{ sql }}\n );\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_assert_columns_equivalent", "macro.dbt.get_columns_spec_ddl", "macro.dbt_postgres.get_column_names", "macro.dbt.get_select_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.5590951, "supported_languages": null}, "macro.dbt_postgres.postgres__get_create_index_sql": {"name": "postgres__get_create_index_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__get_create_index_sql", "macro_sql": "{% macro postgres__get_create_index_sql(relation, index_dict) -%}\n {%- set index_config = adapter.parse_index(index_dict) -%}\n {%- set comma_separated_columns = \", \".join(index_config.columns) -%}\n {%- set index_name = index_config.render(relation) -%}\n\n create {% if index_config.unique -%}\n unique\n {%- endif %} index if not exists\n \"{{ index_name }}\"\n on {{ relation }} {% if index_config.type -%}\n using {{ index_config.type }}\n {%- endif %}\n ({{ comma_separated_columns }});\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.559607, "supported_languages": null}, "macro.dbt_postgres.postgres__create_schema": {"name": "postgres__create_schema", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__create_schema", "macro_sql": "{% macro postgres__create_schema(relation) -%}\n {% if relation.database -%}\n {{ adapter.verify_database(relation.database) }}\n {%- endif -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier().include(database=False) }}\n {%- endcall -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.5599282, "supported_languages": null}, "macro.dbt_postgres.postgres__drop_schema": {"name": "postgres__drop_schema", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__drop_schema", "macro_sql": "{% macro postgres__drop_schema(relation) -%}\n {% if relation.database -%}\n {{ adapter.verify_database(relation.database) }}\n {%- endif -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier().include(database=False) }} cascade\n {%- endcall -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.560247, "supported_languages": null}, "macro.dbt_postgres.postgres__get_columns_in_relation": {"name": "postgres__get_columns_in_relation", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__get_columns_in_relation", "macro_sql": "{% macro postgres__get_columns_in_relation(relation) -%}\n {% call statement('get_columns_in_relation', fetch_result=True) %}\n select\n column_name,\n data_type,\n character_maximum_length,\n numeric_precision,\n numeric_scale\n\n from {{ relation.information_schema('columns') }}\n where table_name = '{{ relation.identifier }}'\n {% if relation.schema %}\n and table_schema = '{{ relation.schema }}'\n {% endif %}\n order by ordinal_position\n\n {% endcall %}\n {% set table = load_result('get_columns_in_relation').table %}\n {{ return(sql_convert_columns_in_relation(table)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.sql_convert_columns_in_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.5607271, "supported_languages": null}, "macro.dbt_postgres.postgres__list_relations_without_caching": {"name": "postgres__list_relations_without_caching", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__list_relations_without_caching", "macro_sql": "{% macro postgres__list_relations_without_caching(schema_relation) %}\n {% call statement('list_relations_without_caching', fetch_result=True) -%}\n select\n '{{ schema_relation.database }}' as database,\n tablename as name,\n schemaname as schema,\n 'table' as type\n from pg_tables\n where schemaname ilike '{{ schema_relation.schema }}'\n union all\n select\n '{{ schema_relation.database }}' as database,\n viewname as name,\n schemaname as schema,\n 'view' as type\n from pg_views\n where schemaname ilike '{{ schema_relation.schema }}'\n {% endcall %}\n {{ return(load_result('list_relations_without_caching').table) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.5611079, "supported_languages": null}, "macro.dbt_postgres.postgres__information_schema_name": {"name": "postgres__information_schema_name", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__information_schema_name", "macro_sql": "{% macro postgres__information_schema_name(database) -%}\n {% if database_name -%}\n {{ adapter.verify_database(database_name) }}\n {%- endif -%}\n information_schema\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.561283, "supported_languages": null}, "macro.dbt_postgres.postgres__list_schemas": {"name": "postgres__list_schemas", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__list_schemas", "macro_sql": "{% macro postgres__list_schemas(database) %}\n {% if database -%}\n {{ adapter.verify_database(database) }}\n {%- endif -%}\n {% call statement('list_schemas', fetch_result=True, auto_begin=False) %}\n select distinct nspname from pg_namespace\n {% endcall %}\n {{ return(load_result('list_schemas').table) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.561631, "supported_languages": null}, "macro.dbt_postgres.postgres__check_schema_exists": {"name": "postgres__check_schema_exists", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__check_schema_exists", "macro_sql": "{% macro postgres__check_schema_exists(information_schema, schema) -%}\n {% if information_schema.database -%}\n {{ adapter.verify_database(information_schema.database) }}\n {%- endif -%}\n {% call statement('check_schema_exists', fetch_result=True, auto_begin=False) %}\n select count(*) from pg_namespace where nspname = '{{ schema }}'\n {% endcall %}\n {{ return(load_result('check_schema_exists').table) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.56203, "supported_languages": null}, "macro.dbt_postgres.postgres__make_relation_with_suffix": {"name": "postgres__make_relation_with_suffix", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__make_relation_with_suffix", "macro_sql": "{% macro postgres__make_relation_with_suffix(base_relation, suffix, dstring) %}\n {% if dstring %}\n {% set dt = modules.datetime.datetime.now() %}\n {% set dtstring = dt.strftime(\"%H%M%S%f\") %}\n {% set suffix = suffix ~ dtstring %}\n {% endif %}\n {% set suffix_length = suffix|length %}\n {% set relation_max_name_length = base_relation.relation_max_name_length() %}\n {% if suffix_length > relation_max_name_length %}\n {% do exceptions.raise_compiler_error('Relation suffix is too long (' ~ suffix_length ~ ' characters). Maximum length is ' ~ relation_max_name_length ~ ' characters.') %}\n {% endif %}\n {% set identifier = base_relation.identifier[:relation_max_name_length - suffix_length] ~ suffix %}\n\n {{ return(base_relation.incorporate(path={\"identifier\": identifier })) }}\n\n {% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.562905, "supported_languages": null}, "macro.dbt_postgres.postgres__make_intermediate_relation": {"name": "postgres__make_intermediate_relation", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__make_intermediate_relation", "macro_sql": "{% macro postgres__make_intermediate_relation(base_relation, suffix) %}\n {{ return(postgres__make_relation_with_suffix(base_relation, suffix, dstring=False)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_relation_with_suffix"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.5631082, "supported_languages": null}, "macro.dbt_postgres.postgres__make_temp_relation": {"name": "postgres__make_temp_relation", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__make_temp_relation", "macro_sql": "{% macro postgres__make_temp_relation(base_relation, suffix) %}\n {% set temp_relation = postgres__make_relation_with_suffix(base_relation, suffix, dstring=True) %}\n {{ return(temp_relation.incorporate(path={\"schema\": none,\n \"database\": none})) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_relation_with_suffix"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.563437, "supported_languages": null}, "macro.dbt_postgres.postgres__make_backup_relation": {"name": "postgres__make_backup_relation", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__make_backup_relation", "macro_sql": "{% macro postgres__make_backup_relation(base_relation, backup_relation_type, suffix) %}\n {% set backup_relation = postgres__make_relation_with_suffix(base_relation, suffix, dstring=False) %}\n {{ return(backup_relation.incorporate(type=backup_relation_type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_relation_with_suffix"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.563714, "supported_languages": null}, "macro.dbt_postgres.postgres_escape_comment": {"name": "postgres_escape_comment", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres_escape_comment", "macro_sql": "{% macro postgres_escape_comment(comment) -%}\n {% if comment is not string %}\n {% do exceptions.raise_compiler_error('cannot escape a non-string: ' ~ comment) %}\n {% endif %}\n {%- set magic = '$dbt_comment_literal_block$' -%}\n {%- if magic in comment -%}\n {%- do exceptions.raise_compiler_error('The string ' ~ magic ~ ' is not allowed in comments.') -%}\n {%- endif -%}\n {{ magic }}{{ comment }}{{ magic }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.564138, "supported_languages": null}, "macro.dbt_postgres.postgres__alter_relation_comment": {"name": "postgres__alter_relation_comment", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__alter_relation_comment", "macro_sql": "{% macro postgres__alter_relation_comment(relation, comment) %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n comment on {{ relation.type }} {{ relation }} is {{ escaped_comment }};\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres_escape_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.564367, "supported_languages": null}, "macro.dbt_postgres.postgres__alter_column_comment": {"name": "postgres__alter_column_comment", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__alter_column_comment", "macro_sql": "{% macro postgres__alter_column_comment(relation, column_dict) %}\n {% set existing_columns = adapter.get_columns_in_relation(relation) | map(attribute=\"name\") | list %}\n {% for column_name in column_dict if (column_name in existing_columns) %}\n {% set comment = column_dict[column_name]['description'] %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n comment on column {{ relation }}.{{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }} is {{ escaped_comment }};\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres_escape_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.5649762, "supported_languages": null}, "macro.dbt_postgres.postgres__get_show_grant_sql": {"name": "postgres__get_show_grant_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__get_show_grant_sql", "macro_sql": "\n\n{%- macro postgres__get_show_grant_sql(relation) -%}\n select grantee, privilege_type\n from {{ relation.information_schema('role_table_grants') }}\n where grantor = current_role\n and grantee != current_role\n and table_schema = '{{ relation.schema }}'\n and table_name = '{{ relation.identifier }}'\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.565191, "supported_languages": null}, "macro.dbt_postgres.postgres__copy_grants": {"name": "postgres__copy_grants", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__copy_grants", "macro_sql": "{% macro postgres__copy_grants() %}\n {{ return(False) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.565307, "supported_languages": null}, "macro.dbt_postgres.postgres__get_incremental_default_sql": {"name": "postgres__get_incremental_default_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/materializations/incremental_strategies.sql", "original_file_path": "macros/materializations/incremental_strategies.sql", "unique_id": "macro.dbt_postgres.postgres__get_incremental_default_sql", "macro_sql": "{% macro postgres__get_incremental_default_sql(arg_dict) %}\n\n {% if arg_dict[\"unique_key\"] %}\n {% do return(get_incremental_delete_insert_sql(arg_dict)) %}\n {% else %}\n {% do return(get_incremental_append_sql(arg_dict)) %}\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_incremental_delete_insert_sql", "macro.dbt.get_incremental_append_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.565693, "supported_languages": null}, "macro.dbt_postgres.postgres__snapshot_merge_sql": {"name": "postgres__snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/materializations/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshot_merge.sql", "unique_id": "macro.dbt_postgres.postgres__snapshot_merge_sql", "macro_sql": "{% macro postgres__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n update {{ target }}\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n from {{ source }} as DBT_INTERNAL_SOURCE\n where DBT_INTERNAL_SOURCE.dbt_scd_id::text = {{ target }}.dbt_scd_id::text\n and DBT_INTERNAL_SOURCE.dbt_change_type::text in ('update'::text, 'delete'::text)\n and {{ target }}.dbt_valid_to is null;\n\n insert into {{ target }} ({{ insert_cols_csv }})\n select {% for column in insert_cols -%}\n DBT_INTERNAL_SOURCE.{{ column }} {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n from {{ source }} as DBT_INTERNAL_SOURCE\n where DBT_INTERNAL_SOURCE.dbt_change_type::text = 'insert'::text;\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.5664032, "supported_languages": null}, "macro.dbt_postgres.get_column_names": {"name": "get_column_names", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/utils/columns_spec_ddl.sql", "original_file_path": "macros/utils/columns_spec_ddl.sql", "unique_id": "macro.dbt_postgres.get_column_names", "macro_sql": "{% macro get_column_names() %}\n {# loop through user_provided_columns to get column names #}\n {%- set user_provided_columns = model['columns'] -%}\n (\n {% for i in user_provided_columns %}\n {% set col = user_provided_columns[i] %}\n {{ col['name'] }} {{ \",\" if not loop.last }}\n {% endfor %}\n )\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.566866, "supported_languages": null}, "macro.dbt_postgres.postgres__dateadd": {"name": "postgres__dateadd", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt_postgres.postgres__dateadd", "macro_sql": "{% macro postgres__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n {{ from_date_or_timestamp }} + ((interval '1 {{ datepart }}') * ({{ interval }}))\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.567093, "supported_languages": null}, "macro.dbt_postgres.postgres__listagg": {"name": "postgres__listagg", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "unique_id": "macro.dbt_postgres.postgres__listagg", "macro_sql": "{% macro postgres__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n {% if limit_num -%}\n array_to_string(\n (array_agg(\n {{ measure }}\n {% if order_by_clause -%}\n {{ order_by_clause }}\n {%- endif %}\n ))[1:{{ limit_num }}],\n {{ delimiter_text }}\n )\n {%- else %}\n string_agg(\n {{ measure }},\n {{ delimiter_text }}\n {% if order_by_clause -%}\n {{ order_by_clause }}\n {%- endif %}\n )\n {%- endif %}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.5678, "supported_languages": null}, "macro.dbt_postgres.postgres__datediff": {"name": "postgres__datediff", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt_postgres.postgres__datediff", "macro_sql": "{% macro postgres__datediff(first_date, second_date, datepart) -%}\n\n {% if datepart == 'year' %}\n (date_part('year', ({{second_date}})::date) - date_part('year', ({{first_date}})::date))\n {% elif datepart == 'quarter' %}\n ({{ datediff(first_date, second_date, 'year') }} * 4 + date_part('quarter', ({{second_date}})::date) - date_part('quarter', ({{first_date}})::date))\n {% elif datepart == 'month' %}\n ({{ datediff(first_date, second_date, 'year') }} * 12 + date_part('month', ({{second_date}})::date) - date_part('month', ({{first_date}})::date))\n {% elif datepart == 'day' %}\n (({{second_date}})::date - ({{first_date}})::date)\n {% elif datepart == 'week' %}\n ({{ datediff(first_date, second_date, 'day') }} / 7 + case\n when date_part('dow', ({{first_date}})::timestamp) <= date_part('dow', ({{second_date}})::timestamp) then\n case when {{first_date}} <= {{second_date}} then 0 else -1 end\n else\n case when {{first_date}} <= {{second_date}} then 1 else 0 end\n end)\n {% elif datepart == 'hour' %}\n ({{ datediff(first_date, second_date, 'day') }} * 24 + date_part('hour', ({{second_date}})::timestamp) - date_part('hour', ({{first_date}})::timestamp))\n {% elif datepart == 'minute' %}\n ({{ datediff(first_date, second_date, 'hour') }} * 60 + date_part('minute', ({{second_date}})::timestamp) - date_part('minute', ({{first_date}})::timestamp))\n {% elif datepart == 'second' %}\n ({{ datediff(first_date, second_date, 'minute') }} * 60 + floor(date_part('second', ({{second_date}})::timestamp)) - floor(date_part('second', ({{first_date}})::timestamp)))\n {% elif datepart == 'millisecond' %}\n ({{ datediff(first_date, second_date, 'minute') }} * 60000 + floor(date_part('millisecond', ({{second_date}})::timestamp)) - floor(date_part('millisecond', ({{first_date}})::timestamp)))\n {% elif datepart == 'microsecond' %}\n ({{ datediff(first_date, second_date, 'minute') }} * 60000000 + floor(date_part('microsecond', ({{second_date}})::timestamp)) - floor(date_part('microsecond', ({{first_date}})::timestamp)))\n {% else %}\n {{ exceptions.raise_compiler_error(\"Unsupported datepart for macro datediff in postgres: {!r}\".format(datepart)) }}\n {% endif %}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.datediff"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.571049, "supported_languages": null}, "macro.dbt_postgres.postgres__any_value": {"name": "postgres__any_value", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "unique_id": "macro.dbt_postgres.postgres__any_value", "macro_sql": "{% macro postgres__any_value(expression) -%}\n\n min({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.5712, "supported_languages": null}, "macro.dbt_postgres.postgres__last_day": {"name": "postgres__last_day", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt_postgres.postgres__last_day", "macro_sql": "{% macro postgres__last_day(date, datepart) -%}\n\n {%- if datepart == 'quarter' -%}\n -- postgres dateadd does not support quarter interval.\n cast(\n {{dbt.dateadd('day', '-1',\n dbt.dateadd('month', '3', dbt.date_trunc(datepart, date))\n )}}\n as date)\n {%- else -%}\n {{dbt.default_last_day(date, datepart)}}\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.dateadd", "macro.dbt.date_trunc", "macro.dbt.default_last_day"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.571703, "supported_languages": null}, "macro.dbt_postgres.postgres__split_part": {"name": "postgres__split_part", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt_postgres.postgres__split_part", "macro_sql": "{% macro postgres__split_part(string_text, delimiter_text, part_number) %}\n\n {% if part_number >= 0 %}\n {{ dbt.default__split_part(string_text, delimiter_text, part_number) }}\n {% else %}\n {{ dbt._split_part_negative(string_text, delimiter_text, part_number) }}\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__split_part", "macro.dbt._split_part_negative"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.572116, "supported_languages": null}, "macro.dbt.run_hooks": {"name": "run_hooks", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.run_hooks", "macro_sql": "{% macro run_hooks(hooks, inside_transaction=True) %}\n {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %}\n {% if not inside_transaction and loop.first %}\n {% call statement(auto_begin=inside_transaction) %}\n commit;\n {% endcall %}\n {% endif %}\n {% set rendered = render(hook.get('sql')) | trim %}\n {% if (rendered | length) > 0 %}\n {% call statement(auto_begin=inside_transaction) %}\n {{ rendered }}\n {% endcall %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.573218, "supported_languages": null}, "macro.dbt.make_hook_config": {"name": "make_hook_config", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.make_hook_config", "macro_sql": "{% macro make_hook_config(sql, inside_transaction) %}\n {{ tojson({\"sql\": sql, \"transaction\": inside_transaction}) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.573418, "supported_languages": null}, "macro.dbt.before_begin": {"name": "before_begin", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.before_begin", "macro_sql": "{% macro before_begin(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.57356, "supported_languages": null}, "macro.dbt.in_transaction": {"name": "in_transaction", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.in_transaction", "macro_sql": "{% macro in_transaction(sql) %}\n {{ make_hook_config(sql, inside_transaction=True) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.5737019, "supported_languages": null}, "macro.dbt.after_commit": {"name": "after_commit", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.after_commit", "macro_sql": "{% macro after_commit(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.573847, "supported_languages": null}, "macro.dbt.set_sql_header": {"name": "set_sql_header", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "unique_id": "macro.dbt.set_sql_header", "macro_sql": "{% macro set_sql_header(config) -%}\n {{ config.set('sql_header', caller()) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.5742688, "supported_languages": null}, "macro.dbt.should_full_refresh": {"name": "should_full_refresh", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "unique_id": "macro.dbt.should_full_refresh", "macro_sql": "{% macro should_full_refresh() %}\n {% set config_full_refresh = config.get('full_refresh') %}\n {% if config_full_refresh is none %}\n {% set config_full_refresh = flags.FULL_REFRESH %}\n {% endif %}\n {% do return(config_full_refresh) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.5745878, "supported_languages": null}, "macro.dbt.should_store_failures": {"name": "should_store_failures", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "unique_id": "macro.dbt.should_store_failures", "macro_sql": "{% macro should_store_failures() %}\n {% set config_store_failures = config.get('store_failures') %}\n {% if config_store_failures is none %}\n {% set config_store_failures = flags.STORE_FAILURES %}\n {% endif %}\n {% do return(config_store_failures) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.574906, "supported_languages": null}, "macro.dbt.snapshot_merge_sql": {"name": "snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", "unique_id": "macro.dbt.snapshot_merge_sql", "macro_sql": "{% macro snapshot_merge_sql(target, source, insert_cols) -%}\n {{ adapter.dispatch('snapshot_merge_sql', 'dbt')(target, source, insert_cols) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__snapshot_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.575354, "supported_languages": null}, "macro.dbt.default__snapshot_merge_sql": {"name": "default__snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", "unique_id": "macro.dbt.default__snapshot_merge_sql", "macro_sql": "{% macro default__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on DBT_INTERNAL_SOURCE.dbt_scd_id = DBT_INTERNAL_DEST.dbt_scd_id\n\n when matched\n and DBT_INTERNAL_DEST.dbt_valid_to is null\n and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')\n then update\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n\n when not matched\n and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'\n then insert ({{ insert_cols_csv }})\n values ({{ insert_cols_csv }})\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.5756302, "supported_languages": null}, "macro.dbt.strategy_dispatch": {"name": "strategy_dispatch", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.strategy_dispatch", "macro_sql": "{% macro strategy_dispatch(name) -%}\n{% set original_name = name %}\n {% if '.' in name %}\n {% set package_name, name = name.split(\".\", 1) %}\n {% else %}\n {% set package_name = none %}\n {% endif %}\n\n {% if package_name is none %}\n {% set package_context = context %}\n {% elif package_name in context %}\n {% set package_context = context[package_name] %}\n {% else %}\n {% set error_msg %}\n Could not find package '{{package_name}}', called with '{{original_name}}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n\n {%- set search_name = 'snapshot_' ~ name ~ '_strategy' -%}\n\n {% if search_name not in package_context %}\n {% set error_msg %}\n The specified strategy macro '{{name}}' was not found in package '{{ package_name }}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n {{ return(package_context[search_name]) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.579268, "supported_languages": null}, "macro.dbt.snapshot_hash_arguments": {"name": "snapshot_hash_arguments", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_hash_arguments", "macro_sql": "{% macro snapshot_hash_arguments(args) -%}\n {{ adapter.dispatch('snapshot_hash_arguments', 'dbt')(args) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.579456, "supported_languages": null}, "macro.dbt.default__snapshot_hash_arguments": {"name": "default__snapshot_hash_arguments", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.default__snapshot_hash_arguments", "macro_sql": "{% macro default__snapshot_hash_arguments(args) -%}\n md5({%- for arg in args -%}\n coalesce(cast({{ arg }} as varchar ), '')\n {% if not loop.last %} || '|' || {% endif %}\n {%- endfor -%})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.579693, "supported_languages": null}, "macro.dbt.snapshot_timestamp_strategy": {"name": "snapshot_timestamp_strategy", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_timestamp_strategy", "macro_sql": "{% macro snapshot_timestamp_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}\n {% set primary_key = config['unique_key'] %}\n {% set updated_at = config['updated_at'] %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %}\n\n {#/*\n The snapshot relation might not have an {{ updated_at }} value if the\n snapshot strategy is changed from `check` to `timestamp`. We\n should use a dbt-created column for the comparison in the snapshot\n table instead of assuming that the user-supplied {{ updated_at }}\n will be present in the historical data.\n\n See https://github.com/dbt-labs/dbt-core/issues/2350\n */ #}\n {% set row_changed_expr -%}\n ({{ snapshotted_rel }}.dbt_valid_from < {{ current_rel }}.{{ updated_at }})\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.5804448, "supported_languages": null}, "macro.dbt.snapshot_string_as_time": {"name": "snapshot_string_as_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_string_as_time", "macro_sql": "{% macro snapshot_string_as_time(timestamp) -%}\n {{ adapter.dispatch('snapshot_string_as_time', 'dbt')(timestamp) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__snapshot_string_as_time"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.5806148, "supported_languages": null}, "macro.dbt.default__snapshot_string_as_time": {"name": "default__snapshot_string_as_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.default__snapshot_string_as_time", "macro_sql": "{% macro default__snapshot_string_as_time(timestamp) %}\n {% do exceptions.raise_not_implemented(\n 'snapshot_string_as_time macro not implemented for adapter '+adapter.type()\n ) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.5807948, "supported_languages": null}, "macro.dbt.snapshot_check_all_get_existing_columns": {"name": "snapshot_check_all_get_existing_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_check_all_get_existing_columns", "macro_sql": "{% macro snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) -%}\n {%- if not target_exists -%}\n {#-- no table yet -> return whatever the query does --#}\n {{ return((false, query_columns)) }}\n {%- endif -%}\n\n {#-- handle any schema changes --#}\n {%- set target_relation = adapter.get_relation(database=node.database, schema=node.schema, identifier=node.alias) -%}\n\n {% if check_cols_config == 'all' %}\n {%- set query_columns = get_columns_in_query(node['compiled_code']) -%}\n\n {% elif check_cols_config is iterable and (check_cols_config | length) > 0 %}\n {#-- query for proper casing/quoting, to support comparison below --#}\n {%- set select_check_cols_from_target -%}\n select {{ check_cols_config | join(', ') }} from ({{ node['compiled_code'] }}) subq\n {%- endset -%}\n {% set query_columns = get_columns_in_query(select_check_cols_from_target) %}\n\n {% else %}\n {% do exceptions.raise_compiler_error(\"Invalid value for 'check_cols': \" ~ check_cols_config) %}\n {% endif %}\n\n {%- set existing_cols = adapter.get_columns_in_relation(target_relation) | map(attribute = 'name') | list -%}\n {%- set ns = namespace() -%} {#-- handle for-loop scoping with a namespace --#}\n {%- set ns.column_added = false -%}\n\n {%- set intersection = [] -%}\n {%- for col in query_columns -%}\n {%- if col in existing_cols -%}\n {%- do intersection.append(adapter.quote(col)) -%}\n {%- else -%}\n {% set ns.column_added = true %}\n {%- endif -%}\n {%- endfor -%}\n {{ return((ns.column_added, intersection)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_columns_in_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.5822341, "supported_languages": null}, "macro.dbt.snapshot_check_strategy": {"name": "snapshot_check_strategy", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_check_strategy", "macro_sql": "{% macro snapshot_check_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}\n {% set check_cols_config = config['check_cols'] %}\n {% set primary_key = config['unique_key'] %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %}\n {% set updated_at = config.get('updated_at', snapshot_get_time()) %}\n\n {% set column_added = false %}\n\n {% set column_added, check_cols = snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) %}\n\n {%- set row_changed_expr -%}\n (\n {%- if column_added -%}\n {{ get_true_sql() }}\n {%- else -%}\n {%- for col in check_cols -%}\n {{ snapshotted_rel }}.{{ col }} != {{ current_rel }}.{{ col }}\n or\n (\n (({{ snapshotted_rel }}.{{ col }} is null) and not ({{ current_rel }}.{{ col }} is null))\n or\n ((not {{ snapshotted_rel }}.{{ col }} is null) and ({{ current_rel }}.{{ col }} is null))\n )\n {%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n {%- endif -%}\n )\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.snapshot_get_time", "macro.dbt.snapshot_check_all_get_existing_columns", "macro.dbt.get_true_sql", "macro.dbt.snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.583667, "supported_languages": null}, "macro.dbt.create_columns": {"name": "create_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.create_columns", "macro_sql": "{% macro create_columns(relation, columns) %}\n {{ adapter.dispatch('create_columns', 'dbt')(relation, columns) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__create_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.587839, "supported_languages": null}, "macro.dbt.default__create_columns": {"name": "default__create_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__create_columns", "macro_sql": "{% macro default__create_columns(relation, columns) %}\n {% for column in columns %}\n {% call statement() %}\n alter table {{ relation }} add column \"{{ column.name }}\" {{ column.data_type }};\n {% endcall %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.58813, "supported_languages": null}, "macro.dbt.post_snapshot": {"name": "post_snapshot", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.post_snapshot", "macro_sql": "{% macro post_snapshot(staging_relation) %}\n {{ adapter.dispatch('post_snapshot', 'dbt')(staging_relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.5883, "supported_languages": null}, "macro.dbt.default__post_snapshot": {"name": "default__post_snapshot", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__post_snapshot", "macro_sql": "{% macro default__post_snapshot(staging_relation) %}\n {# no-op #}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.588392, "supported_languages": null}, "macro.dbt.get_true_sql": {"name": "get_true_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.get_true_sql", "macro_sql": "{% macro get_true_sql() %}\n {{ adapter.dispatch('get_true_sql', 'dbt')() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_true_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.5885382, "supported_languages": null}, "macro.dbt.default__get_true_sql": {"name": "default__get_true_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__get_true_sql", "macro_sql": "{% macro default__get_true_sql() %}\n {{ return('TRUE') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.588657, "supported_languages": null}, "macro.dbt.snapshot_staging_table": {"name": "snapshot_staging_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.snapshot_staging_table", "macro_sql": "{% macro snapshot_staging_table(strategy, source_sql, target_relation) -%}\n {{ adapter.dispatch('snapshot_staging_table', 'dbt')(strategy, source_sql, target_relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__snapshot_staging_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.588872, "supported_languages": null}, "macro.dbt.default__snapshot_staging_table": {"name": "default__snapshot_staging_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__snapshot_staging_table", "macro_sql": "{% macro default__snapshot_staging_table(strategy, source_sql, target_relation) -%}\n\n with snapshot_query as (\n\n {{ source_sql }}\n\n ),\n\n snapshotted_data as (\n\n select *,\n {{ strategy.unique_key }} as dbt_unique_key\n\n from {{ target_relation }}\n where dbt_valid_to is null\n\n ),\n\n insertions_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to,\n {{ strategy.scd_id }} as dbt_scd_id\n\n from snapshot_query\n ),\n\n updates_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n {{ strategy.updated_at }} as dbt_valid_to\n\n from snapshot_query\n ),\n\n {%- if strategy.invalidate_hard_deletes %}\n\n deletes_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key\n from snapshot_query\n ),\n {% endif %}\n\n insertions as (\n\n select\n 'insert' as dbt_change_type,\n source_data.*\n\n from insertions_source_data as source_data\n left outer join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where snapshotted_data.dbt_unique_key is null\n or (\n snapshotted_data.dbt_unique_key is not null\n and (\n {{ strategy.row_changed }}\n )\n )\n\n ),\n\n updates as (\n\n select\n 'update' as dbt_change_type,\n source_data.*,\n snapshotted_data.dbt_scd_id\n\n from updates_source_data as source_data\n join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where (\n {{ strategy.row_changed }}\n )\n )\n\n {%- if strategy.invalidate_hard_deletes -%}\n ,\n\n deletes as (\n\n select\n 'delete' as dbt_change_type,\n source_data.*,\n {{ snapshot_get_time() }} as dbt_valid_from,\n {{ snapshot_get_time() }} as dbt_updated_at,\n {{ snapshot_get_time() }} as dbt_valid_to,\n snapshotted_data.dbt_scd_id\n\n from snapshotted_data\n left join deletes_source_data as source_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where source_data.dbt_unique_key is null\n )\n {%- endif %}\n\n select * from insertions\n union all\n select * from updates\n {%- if strategy.invalidate_hard_deletes %}\n union all\n select * from deletes\n {%- endif %}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.snapshot_get_time"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.5897758, "supported_languages": null}, "macro.dbt.build_snapshot_table": {"name": "build_snapshot_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.build_snapshot_table", "macro_sql": "{% macro build_snapshot_table(strategy, sql) -%}\n {{ adapter.dispatch('build_snapshot_table', 'dbt')(strategy, sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__build_snapshot_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.589972, "supported_languages": null}, "macro.dbt.default__build_snapshot_table": {"name": "default__build_snapshot_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__build_snapshot_table", "macro_sql": "{% macro default__build_snapshot_table(strategy, sql) %}\n\n select *,\n {{ strategy.scd_id }} as dbt_scd_id,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to\n from (\n {{ sql }}\n ) sbq\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.590228, "supported_languages": null}, "macro.dbt.build_snapshot_staging_table": {"name": "build_snapshot_staging_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.build_snapshot_staging_table", "macro_sql": "{% macro build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% set temp_relation = make_temp_relation(target_relation) %}\n\n {% set select = snapshot_staging_table(strategy, sql, target_relation) %}\n\n {% call statement('build_snapshot_staging_relation') %}\n {{ create_table_as(True, temp_relation, select) }}\n {% endcall %}\n\n {% do return(temp_relation) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_temp_relation", "macro.dbt.snapshot_staging_table", "macro.dbt.statement", "macro.dbt.create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.5906692, "supported_languages": null}, "macro.dbt.materialization_snapshot_default": {"name": "materialization_snapshot_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/snapshot.sql", "original_file_path": "macros/materializations/snapshots/snapshot.sql", "unique_id": "macro.dbt.materialization_snapshot_default", "macro_sql": "{% materialization snapshot, default %}\n {%- set config = model['config'] -%}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n -- grab current tables grants config for comparision later on\n {%- set grant_config = config.get('grants') -%}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=model.database,\n schema=model.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", config, target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_code']) %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% else %}\n\n {{ adapter.valid_snapshot_target(target_relation) }}\n\n {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% endif %}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode=False) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if not target_relation_exists %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if staging_table is defined %}\n {% do post_snapshot(staging_table) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.get_or_create_relation", "macro.dbt.run_hooks", "macro.dbt.strategy_dispatch", "macro.dbt.build_snapshot_table", "macro.dbt.create_table_as", "macro.dbt.build_snapshot_staging_table", "macro.dbt.create_columns", "macro.dbt.snapshot_merge_sql", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes", "macro.dbt.post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.5969589, "supported_languages": ["sql"]}, "macro.dbt.materialization_test_default": {"name": "materialization_test_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/test.sql", "original_file_path": "macros/materializations/tests/test.sql", "unique_id": "macro.dbt.materialization_test_default", "macro_sql": "{%- materialization test, default -%}\n\n {% set relations = [] %}\n\n {% if should_store_failures() %}\n\n {% set identifier = model['alias'] %}\n {% set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n {% set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database, type='table') -%} %}\n\n {% if old_relation %}\n {% do adapter.drop_relation(old_relation) %}\n {% endif %}\n\n {% call statement(auto_begin=True) %}\n {{ create_table_as(False, target_relation, sql) }}\n {% endcall %}\n\n {% do relations.append(target_relation) %}\n\n {% set main_sql %}\n select *\n from {{ target_relation }}\n {% endset %}\n\n {{ adapter.commit() }}\n\n {% else %}\n\n {% set main_sql = sql %}\n\n {% endif %}\n\n {% set limit = config.get('limit') %}\n {% set fail_calc = config.get('fail_calc') %}\n {% set warn_if = config.get('warn_if') %}\n {% set error_if = config.get('error_if') %}\n\n {% call statement('main', fetch_result=True) -%}\n\n {{ get_test_sql(main_sql, fail_calc, warn_if, error_if, limit)}}\n\n {%- endcall %}\n\n {{ return({'relations': relations}) }}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.should_store_failures", "macro.dbt.statement", "macro.dbt.create_table_as", "macro.dbt.get_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.599081, "supported_languages": ["sql"]}, "macro.dbt.get_test_sql": {"name": "get_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.get_test_sql", "macro_sql": "{% macro get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n {{ adapter.dispatch('get_test_sql', 'dbt')(main_sql, fail_calc, warn_if, error_if, limit) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.5995462, "supported_languages": null}, "macro.dbt.default__get_test_sql": {"name": "default__get_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.default__get_test_sql", "macro_sql": "{% macro default__get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n select\n {{ fail_calc }} as failures,\n {{ fail_calc }} {{ warn_if }} as should_warn,\n {{ fail_calc }} {{ error_if }} as should_error\n from (\n {{ main_sql }}\n {{ \"limit \" ~ limit if limit != none }}\n ) dbt_internal_test\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.599873, "supported_languages": null}, "macro.dbt.get_where_subquery": {"name": "get_where_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/where_subquery.sql", "original_file_path": "macros/materializations/tests/where_subquery.sql", "unique_id": "macro.dbt.get_where_subquery", "macro_sql": "{% macro get_where_subquery(relation) -%}\n {% do return(adapter.dispatch('get_where_subquery', 'dbt')(relation)) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_where_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.600277, "supported_languages": null}, "macro.dbt.default__get_where_subquery": {"name": "default__get_where_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/where_subquery.sql", "original_file_path": "macros/materializations/tests/where_subquery.sql", "unique_id": "macro.dbt.default__get_where_subquery", "macro_sql": "{% macro default__get_where_subquery(relation) -%}\n {% set where = config.get('where', '') %}\n {% if where %}\n {%- set filtered -%}\n (select * from {{ relation }} where {{ where }}) dbt_subquery\n {%- endset -%}\n {% do return(filtered) %}\n {%- else -%}\n {% do return(relation) %}\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.600673, "supported_languages": null}, "macro.dbt.get_quoted_csv": {"name": "get_quoted_csv", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.get_quoted_csv", "macro_sql": "{% macro get_quoted_csv(column_names) %}\n\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote(col)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6024802, "supported_languages": null}, "macro.dbt.diff_columns": {"name": "diff_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.diff_columns", "macro_sql": "{% macro diff_columns(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% set source_names = source_columns | map(attribute = 'column') | list %}\n {% set target_names = target_columns | map(attribute = 'column') | list %}\n\n {# --check whether the name attribute exists in the target - this does not perform a data type check #}\n {% for sc in source_columns %}\n {% if sc.name not in target_names %}\n {{ result.append(sc) }}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.603078, "supported_languages": null}, "macro.dbt.diff_column_data_types": {"name": "diff_column_data_types", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.diff_column_data_types", "macro_sql": "{% macro diff_column_data_types(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% for sc in source_columns %}\n {% set tc = target_columns | selectattr(\"name\", \"equalto\", sc.name) | list | first %}\n {% if tc %}\n {% if sc.data_type != tc.data_type and not sc.can_expand_to(other_column=tc) %}\n {{ result.append( { 'column_name': tc.name, 'new_type': sc.data_type } ) }}\n {% endif %}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.603765, "supported_languages": null}, "macro.dbt.get_merge_update_columns": {"name": "get_merge_update_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.get_merge_update_columns", "macro_sql": "{% macro get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {{ return(adapter.dispatch('get_merge_update_columns', 'dbt')(merge_update_columns, merge_exclude_columns, dest_columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_merge_update_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6040099, "supported_languages": null}, "macro.dbt.default__get_merge_update_columns": {"name": "default__get_merge_update_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.default__get_merge_update_columns", "macro_sql": "{% macro default__get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {%- set default_cols = dest_columns | map(attribute=\"quoted\") | list -%}\n\n {%- if merge_update_columns and merge_exclude_columns -%}\n {{ exceptions.raise_compiler_error(\n 'Model cannot specify merge_update_columns and merge_exclude_columns. Please update model to use only one config'\n )}}\n {%- elif merge_update_columns -%}\n {%- set update_columns = merge_update_columns -%}\n {%- elif merge_exclude_columns -%}\n {%- set update_columns = [] -%}\n {%- for column in dest_columns -%}\n {% if column.column | lower not in merge_exclude_columns | map(\"lower\") | list %}\n {%- do update_columns.append(column.quoted) -%}\n {% endif %}\n {%- endfor -%}\n {%- else -%}\n {%- set update_columns = default_cols -%}\n {%- endif -%}\n\n {{ return(update_columns) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.604749, "supported_languages": null}, "macro.dbt.get_merge_sql": {"name": "get_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.get_merge_sql", "macro_sql": "{% macro get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) -%}\n -- back compat for old kwarg name\n {% set incremental_predicates = kwargs.get('predicates', incremental_predicates) %}\n {{ adapter.dispatch('get_merge_sql', 'dbt')(target, source, unique_key, dest_columns, incremental_predicates) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.61131, "supported_languages": null}, "macro.dbt.default__get_merge_sql": {"name": "default__get_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.default__get_merge_sql", "macro_sql": "{% macro default__get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) -%}\n {%- set predicates = [] if incremental_predicates is none else [] + incremental_predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set merge_update_columns = config.get('merge_update_columns') -%}\n {%- set merge_exclude_columns = config.get('merge_exclude_columns') -%}\n {%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}\n {% for key in unique_key %}\n {% set this_key_match %}\n DBT_INTERNAL_SOURCE.{{ key }} = DBT_INTERNAL_DEST.{{ key }}\n {% endset %}\n {% do predicates.append(this_key_match) %}\n {% endfor %}\n {% else %}\n {% set unique_key_match %}\n DBT_INTERNAL_SOURCE.{{ unique_key }} = DBT_INTERNAL_DEST.{{ unique_key }}\n {% endset %}\n {% do predicates.append(unique_key_match) %}\n {% endif %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n {{ sql_header if sql_header is not none }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{\"(\" ~ predicates | join(\") and (\") ~ \")\"}}\n\n {% if unique_key %}\n when matched then update set\n {% for column_name in update_columns -%}\n {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}\n {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n {% endif %}\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv", "macro.dbt.get_merge_update_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.613036, "supported_languages": null}, "macro.dbt.get_delete_insert_merge_sql": {"name": "get_delete_insert_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.get_delete_insert_merge_sql", "macro_sql": "{% macro get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}\n {{ adapter.dispatch('get_delete_insert_merge_sql', 'dbt')(target, source, unique_key, dest_columns, incremental_predicates) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_delete_insert_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6133099, "supported_languages": null}, "macro.dbt.default__get_delete_insert_merge_sql": {"name": "default__get_delete_insert_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.default__get_delete_insert_merge_sql", "macro_sql": "{% macro default__get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not string %}\n delete from {{target }}\n using {{ source }}\n where (\n {% for key in unique_key %}\n {{ source }}.{{ key }} = {{ target }}.{{ key }}\n {{ \"and \" if not loop.last}}\n {% endfor %}\n {% if incremental_predicates %}\n {% for predicate in incremental_predicates %}\n and {{ predicate }}\n {% endfor %}\n {% endif %}\n );\n {% else %}\n delete from {{ target }}\n where (\n {{ unique_key }}) in (\n select ({{ unique_key }})\n from {{ source }}\n )\n {%- if incremental_predicates %}\n {% for predicate in incremental_predicates %}\n and {{ predicate }}\n {% endfor %}\n {%- endif -%};\n\n {% endif %}\n {% endif %}\n\n insert into {{ target }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ source }}\n )\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.614329, "supported_languages": null}, "macro.dbt.get_insert_overwrite_merge_sql": {"name": "get_insert_overwrite_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.get_insert_overwrite_merge_sql", "macro_sql": "{% macro get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header=false) -%}\n {{ adapter.dispatch('get_insert_overwrite_merge_sql', 'dbt')(target, source, dest_columns, predicates, include_sql_header) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_insert_overwrite_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6146078, "supported_languages": null}, "macro.dbt.default__get_insert_overwrite_merge_sql": {"name": "default__get_insert_overwrite_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.default__get_insert_overwrite_merge_sql", "macro_sql": "{% macro default__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) -%}\n {#-- The only time include_sql_header is True: --#}\n {#-- BigQuery + insert_overwrite strategy + \"static\" partitions config --#}\n {#-- We should consider including the sql header at the materialization level instead --#}\n\n {%- set predicates = [] if predicates is none else [] + predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none and include_sql_header }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on FALSE\n\n when not matched by source\n {% if predicates %} and {{ predicates | join(' and ') }} {% endif %}\n then delete\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.615274, "supported_languages": null}, "macro.dbt.is_incremental": {"name": "is_incremental", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/is_incremental.sql", "original_file_path": "macros/materializations/models/incremental/is_incremental.sql", "unique_id": "macro.dbt.is_incremental", "macro_sql": "{% macro is_incremental() %}\n {#-- do not run introspective queries in parsing #}\n {% if not execute %}\n {{ return(False) }}\n {% else %}\n {% set relation = adapter.get_relation(this.database, this.schema, this.table) %}\n {{ return(relation is not none\n and relation.type == 'table'\n and model.config.materialized == 'incremental'\n and not should_full_refresh()) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6159291, "supported_languages": null}, "macro.dbt.get_incremental_append_sql": {"name": "get_incremental_append_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_append_sql", "macro_sql": "{% macro get_incremental_append_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_append_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_append_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.616825, "supported_languages": null}, "macro.dbt.default__get_incremental_append_sql": {"name": "default__get_incremental_append_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_append_sql", "macro_sql": "{% macro default__get_incremental_append_sql(arg_dict) %}\n\n {% do return(get_insert_into_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"dest_columns\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_insert_into_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.617065, "supported_languages": null}, "macro.dbt.get_incremental_delete_insert_sql": {"name": "get_incremental_delete_insert_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_delete_insert_sql", "macro_sql": "{% macro get_incremental_delete_insert_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_delete_insert_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_delete_insert_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6172569, "supported_languages": null}, "macro.dbt.default__get_incremental_delete_insert_sql": {"name": "default__get_incremental_delete_insert_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_delete_insert_sql", "macro_sql": "{% macro default__get_incremental_delete_insert_sql(arg_dict) %}\n\n {% do return(get_delete_insert_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"unique_key\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_delete_insert_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.617557, "supported_languages": null}, "macro.dbt.get_incremental_merge_sql": {"name": "get_incremental_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_merge_sql", "macro_sql": "{% macro get_incremental_merge_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_merge_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6177459, "supported_languages": null}, "macro.dbt.default__get_incremental_merge_sql": {"name": "default__get_incremental_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_merge_sql", "macro_sql": "{% macro default__get_incremental_merge_sql(arg_dict) %}\n\n {% do return(get_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"unique_key\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.618047, "supported_languages": null}, "macro.dbt.get_incremental_insert_overwrite_sql": {"name": "get_incremental_insert_overwrite_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_insert_overwrite_sql", "macro_sql": "{% macro get_incremental_insert_overwrite_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_insert_overwrite_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_insert_overwrite_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.618237, "supported_languages": null}, "macro.dbt.default__get_incremental_insert_overwrite_sql": {"name": "default__get_incremental_insert_overwrite_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_insert_overwrite_sql", "macro_sql": "{% macro default__get_incremental_insert_overwrite_sql(arg_dict) %}\n\n {% do return(get_insert_overwrite_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_insert_overwrite_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.618508, "supported_languages": null}, "macro.dbt.get_incremental_default_sql": {"name": "get_incremental_default_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_default_sql", "macro_sql": "{% macro get_incremental_default_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_default_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_incremental_default_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6187038, "supported_languages": null}, "macro.dbt.default__get_incremental_default_sql": {"name": "default__get_incremental_default_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_default_sql", "macro_sql": "{% macro default__get_incremental_default_sql(arg_dict) %}\n\n {% do return(get_incremental_append_sql(arg_dict)) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_incremental_append_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.618857, "supported_languages": null}, "macro.dbt.get_insert_into_sql": {"name": "get_insert_into_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_insert_into_sql", "macro_sql": "{% macro get_insert_into_sql(target_relation, temp_relation, dest_columns) %}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n insert into {{ target_relation }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ temp_relation }}\n )\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.619138, "supported_languages": null}, "macro.dbt.materialization_incremental_default": {"name": "materialization_incremental_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/incremental.sql", "original_file_path": "macros/materializations/models/incremental/incremental.sql", "unique_id": "macro.dbt.materialization_incremental_default", "macro_sql": "{% materialization incremental, default -%}\n\n -- relations\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') -%}\n {%- set temp_relation = make_temp_relation(target_relation)-%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation)-%}\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n\n -- configs\n {%- set unique_key = config.get('unique_key') -%}\n {%- set full_refresh_mode = (should_full_refresh() or existing_relation.is_view) -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n\n -- the temp_ and backup_ relations should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation. This has to happen before\n -- BEGIN, in a separate transaction\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation)-%}\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set to_drop = [] %}\n\n {% if existing_relation is none %}\n {% set build_sql = get_create_table_as_sql(False, target_relation, sql) %}\n {% elif full_refresh_mode %}\n {% set build_sql = get_create_table_as_sql(False, intermediate_relation, sql) %}\n {% set need_swap = true %}\n {% else %}\n {% do run_query(get_create_table_as_sql(True, temp_relation, sql)) %}\n {% do adapter.expand_target_column_types(\n from_relation=temp_relation,\n to_relation=target_relation) %}\n {#-- Process schema changes. Returns dict of changes if successful. Use source columns for upserting/merging --#}\n {% set dest_columns = process_schema_changes(on_schema_change, temp_relation, existing_relation) %}\n {% if not dest_columns %}\n {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %}\n {% endif %}\n\n {#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#}\n {% set incremental_strategy = config.get('incremental_strategy') or 'default' %}\n {% set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) %}\n {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %}\n {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': temp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'incremental_predicates': incremental_predicates }) %}\n {% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %}\n\n {% endif %}\n\n {% call statement(\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% if need_swap %}\n {% do adapter.rename_relation(target_relation, backup_relation) %}\n {% do adapter.rename_relation(intermediate_relation, target_relation) %}\n {% do to_drop.append(backup_relation) %}\n {% endif %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if existing_relation is none or existing_relation.is_view or should_full_refresh() %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {% do adapter.commit() %}\n\n {% for rel in to_drop %}\n {% do adapter.drop_relation(rel) %}\n {% endfor %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_temp_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.should_full_refresh", "macro.dbt.incremental_validate_on_schema_change", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks", "macro.dbt.get_create_table_as_sql", "macro.dbt.run_query", "macro.dbt.process_schema_changes", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.624626, "supported_languages": ["sql"]}, "macro.dbt.incremental_validate_on_schema_change": {"name": "incremental_validate_on_schema_change", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.incremental_validate_on_schema_change", "macro_sql": "{% macro incremental_validate_on_schema_change(on_schema_change, default='ignore') %}\n\n {% if on_schema_change not in ['sync_all_columns', 'append_new_columns', 'fail', 'ignore'] %}\n\n {% set log_message = 'Invalid value for on_schema_change (%s) specified. Setting default value of %s.' % (on_schema_change, default) %}\n {% do log(log_message) %}\n\n {{ return(default) }}\n\n {% else %}\n\n {{ return(on_schema_change) }}\n\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.630311, "supported_languages": null}, "macro.dbt.check_for_schema_changes": {"name": "check_for_schema_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.check_for_schema_changes", "macro_sql": "{% macro check_for_schema_changes(source_relation, target_relation) %}\n\n {% set schema_changed = False %}\n\n {%- set source_columns = adapter.get_columns_in_relation(source_relation) -%}\n {%- set target_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set source_not_in_target = diff_columns(source_columns, target_columns) -%}\n {%- set target_not_in_source = diff_columns(target_columns, source_columns) -%}\n\n {% set new_target_types = diff_column_data_types(source_columns, target_columns) %}\n\n {% if source_not_in_target != [] %}\n {% set schema_changed = True %}\n {% elif target_not_in_source != [] or new_target_types != [] %}\n {% set schema_changed = True %}\n {% elif new_target_types != [] %}\n {% set schema_changed = True %}\n {% endif %}\n\n {% set changes_dict = {\n 'schema_changed': schema_changed,\n 'source_not_in_target': source_not_in_target,\n 'target_not_in_source': target_not_in_source,\n 'source_columns': source_columns,\n 'target_columns': target_columns,\n 'new_target_types': new_target_types\n } %}\n\n {% set msg %}\n In {{ target_relation }}:\n Schema changed: {{ schema_changed }}\n Source columns not in target: {{ source_not_in_target }}\n Target columns not in source: {{ target_not_in_source }}\n New column types: {{ new_target_types }}\n {% endset %}\n\n {% do log(msg) %}\n\n {{ return(changes_dict) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.diff_columns", "macro.dbt.diff_column_data_types"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.631569, "supported_languages": null}, "macro.dbt.sync_column_schemas": {"name": "sync_column_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.sync_column_schemas", "macro_sql": "{% macro sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {%- set add_to_target_arr = schema_changes_dict['source_not_in_target'] -%}\n\n {%- if on_schema_change == 'append_new_columns'-%}\n {%- if add_to_target_arr | length > 0 -%}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, none) -%}\n {%- endif -%}\n\n {% elif on_schema_change == 'sync_all_columns' %}\n {%- set remove_from_target_arr = schema_changes_dict['target_not_in_source'] -%}\n {%- set new_target_types = schema_changes_dict['new_target_types'] -%}\n\n {% if add_to_target_arr | length > 0 or remove_from_target_arr | length > 0 %}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, remove_from_target_arr) -%}\n {% endif %}\n\n {% if new_target_types != [] %}\n {% for ntt in new_target_types %}\n {% set column_name = ntt['column_name'] %}\n {% set new_type = ntt['new_type'] %}\n {% do alter_column_type(target_relation, column_name, new_type) %}\n {% endfor %}\n {% endif %}\n\n {% endif %}\n\n {% set schema_change_message %}\n In {{ target_relation }}:\n Schema change approach: {{ on_schema_change }}\n Columns added: {{ add_to_target_arr }}\n Columns removed: {{ remove_from_target_arr }}\n Data types changed: {{ new_target_types }}\n {% endset %}\n\n {% do log(schema_change_message) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.alter_relation_add_remove_columns", "macro.dbt.alter_column_type"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6327949, "supported_languages": null}, "macro.dbt.process_schema_changes": {"name": "process_schema_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.process_schema_changes", "macro_sql": "{% macro process_schema_changes(on_schema_change, source_relation, target_relation) %}\n\n {% if on_schema_change == 'ignore' %}\n\n {{ return({}) }}\n\n {% else %}\n\n {% set schema_changes_dict = check_for_schema_changes(source_relation, target_relation) %}\n\n {% if schema_changes_dict['schema_changed'] %}\n\n {% if on_schema_change == 'fail' %}\n\n {% set fail_msg %}\n The source and target schemas on this incremental model are out of sync!\n They can be reconciled in several ways:\n - set the `on_schema_change` config to either append_new_columns or sync_all_columns, depending on your situation.\n - Re-run the incremental model with `full_refresh: True` to update the target schema.\n - update the schema manually and re-run the process.\n\n Additional troubleshooting context:\n Source columns not in target: {{ schema_changes_dict['source_not_in_target'] }}\n Target columns not in source: {{ schema_changes_dict['target_not_in_source'] }}\n New column types: {{ schema_changes_dict['new_target_types'] }}\n {% endset %}\n\n {% do exceptions.raise_compiler_error(fail_msg) %}\n\n {# -- unless we ignore, run the sync operation per the config #}\n {% else %}\n\n {% do sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {% endif %}\n\n {% endif %}\n\n {{ return(schema_changes_dict['source_columns']) }}\n\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.check_for_schema_changes", "macro.dbt.sync_column_schemas"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.633666, "supported_languages": null}, "macro.dbt.get_columns_spec_ddl": {"name": "get_columns_spec_ddl", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table/columns_spec_ddl.sql", "original_file_path": "macros/materializations/models/table/columns_spec_ddl.sql", "unique_id": "macro.dbt.get_columns_spec_ddl", "macro_sql": "{%- macro get_columns_spec_ddl() -%}\n {{ adapter.dispatch('get_columns_spec_ddl', 'dbt')() }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__get_columns_spec_ddl"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.634799, "supported_languages": null}, "macro.dbt.default__get_columns_spec_ddl": {"name": "default__get_columns_spec_ddl", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table/columns_spec_ddl.sql", "original_file_path": "macros/materializations/models/table/columns_spec_ddl.sql", "unique_id": "macro.dbt.default__get_columns_spec_ddl", "macro_sql": "{% macro default__get_columns_spec_ddl() -%}\n {{ return(columns_spec_ddl()) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.columns_spec_ddl"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.634922, "supported_languages": null}, "macro.dbt.columns_spec_ddl": {"name": "columns_spec_ddl", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table/columns_spec_ddl.sql", "original_file_path": "macros/materializations/models/table/columns_spec_ddl.sql", "unique_id": "macro.dbt.columns_spec_ddl", "macro_sql": "{% macro columns_spec_ddl() %}\n {# loop through user_provided_columns to create DDL with data types and constraints #}\n {%- set user_provided_columns = model['columns'] -%}\n (\n {% for i in user_provided_columns %}\n {%- set col = user_provided_columns[i] -%}\n {%- set constraints = col['constraints'] -%}\n {{ col['name'] }} {{ col['data_type'] }}{% for c in constraints %} {{ adapter.render_raw_column_constraint(c) }}{% endfor %}{{ \",\" if not loop.last }}\n {% endfor -%}\n )\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6354618, "supported_languages": null}, "macro.dbt.get_assert_columns_equivalent": {"name": "get_assert_columns_equivalent", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table/columns_spec_ddl.sql", "original_file_path": "macros/materializations/models/table/columns_spec_ddl.sql", "unique_id": "macro.dbt.get_assert_columns_equivalent", "macro_sql": "\n\n{%- macro get_assert_columns_equivalent(sql) -%}\n {{ adapter.dispatch('get_assert_columns_equivalent', 'dbt')(sql) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__get_assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6356301, "supported_languages": null}, "macro.dbt.default__get_assert_columns_equivalent": {"name": "default__get_assert_columns_equivalent", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table/columns_spec_ddl.sql", "original_file_path": "macros/materializations/models/table/columns_spec_ddl.sql", "unique_id": "macro.dbt.default__get_assert_columns_equivalent", "macro_sql": "{% macro default__get_assert_columns_equivalent(sql) -%}\n {{ return(assert_columns_equivalent(sql)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.635766, "supported_languages": null}, "macro.dbt.assert_columns_equivalent": {"name": "assert_columns_equivalent", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table/columns_spec_ddl.sql", "original_file_path": "macros/materializations/models/table/columns_spec_ddl.sql", "unique_id": "macro.dbt.assert_columns_equivalent", "macro_sql": "{% macro assert_columns_equivalent(sql) %}\n {#-- Obtain the column schema provided by sql file. #}\n {%- set sql_file_provided_columns = get_column_schema_from_query(sql) -%}\n {#--Obtain the column schema provided by the schema file by generating an 'empty schema' query from the model's columns. #}\n {%- set schema_file_provided_columns = get_column_schema_from_query(get_empty_schema_sql(model['columns'])) -%}\n\n {#-- create dictionaries with name and formatted data type and strings for exception #}\n {%- set sql_columns = format_columns(sql_file_provided_columns) -%}\n {%- set string_sql_columns = stringify_formatted_columns(sql_columns) -%}\n {%- set yaml_columns = format_columns(schema_file_provided_columns) -%}\n {%- set string_yaml_columns = stringify_formatted_columns(yaml_columns) -%}\n\n {%- if sql_columns|length != yaml_columns|length -%}\n {%- do exceptions.raise_contract_error(string_yaml_columns, string_sql_columns) -%}\n {%- endif -%}\n\n {%- for sql_col in sql_columns -%}\n {%- set yaml_col = [] -%}\n {%- for this_col in yaml_columns -%}\n {%- if this_col['name'] == sql_col['name'] -%}\n {%- do yaml_col.append(this_col) -%}\n {%- break -%}\n {%- endif -%}\n {%- endfor -%}\n {%- if not yaml_col -%}\n {#-- Column with name not found in yaml #}\n {%- do exceptions.raise_contract_error(string_yaml_columns, string_sql_columns) -%}\n {%- endif -%}\n {%- if sql_col['formatted'] != yaml_col[0]['formatted'] -%}\n {#-- Column data types don't match #}\n {%- do exceptions.raise_contract_error(string_yaml_columns, string_sql_columns) -%}\n {%- endif -%}\n {%- endfor -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_column_schema_from_query", "macro.dbt.get_empty_schema_sql", "macro.dbt.format_columns", "macro.dbt.stringify_formatted_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.636966, "supported_languages": null}, "macro.dbt.format_columns": {"name": "format_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table/columns_spec_ddl.sql", "original_file_path": "macros/materializations/models/table/columns_spec_ddl.sql", "unique_id": "macro.dbt.format_columns", "macro_sql": "{% macro format_columns(columns) %}\n {% set formatted_columns = [] %}\n {% for column in columns %}\n {%- set formatted_column = adapter.dispatch('format_column', 'dbt')(column) -%}\n {%- do formatted_columns.append({'name': column.name, 'formatted': formatted_column}) -%}\n {% endfor %}\n {{ return(formatted_columns) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__format_column"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.637417, "supported_languages": null}, "macro.dbt.stringify_formatted_columns": {"name": "stringify_formatted_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table/columns_spec_ddl.sql", "original_file_path": "macros/materializations/models/table/columns_spec_ddl.sql", "unique_id": "macro.dbt.stringify_formatted_columns", "macro_sql": "{% macro stringify_formatted_columns(formatted_columns) %}\n {% set column_strings = [] %}\n {% for column in formatted_columns %}\n {% do column_strings.append(column['formatted']) %}\n {% endfor %}\n {{ return(column_strings|join(', ')) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6377542, "supported_languages": null}, "macro.dbt.default__format_column": {"name": "default__format_column", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table/columns_spec_ddl.sql", "original_file_path": "macros/materializations/models/table/columns_spec_ddl.sql", "unique_id": "macro.dbt.default__format_column", "macro_sql": "{% macro default__format_column(column) -%}\n {{ return(column.column.lower() ~ \" \" ~ column.dtype) }}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6379352, "supported_languages": null}, "macro.dbt.materialization_table_default": {"name": "materialization_table_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table/table.sql", "original_file_path": "macros/materializations/models/table/table.sql", "unique_id": "macro.dbt.materialization_table_default", "macro_sql": "{% materialization table, default %}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') %}\n {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}\n /*\n See ../view/view.sql for more information about this relation.\n */\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_table_as_sql(False, intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n {% if existing_relation is not none %}\n {{ adapter.rename_relation(existing_relation, backup_relation) }}\n {% endif %}\n\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% do create_indexes(target_relation) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n -- finally, drop the existing/backup relation after the commit\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.get_create_table_as_sql", "macro.dbt.create_indexes", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.640608, "supported_languages": ["sql"]}, "macro.dbt.get_create_table_as_sql": {"name": "get_create_table_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table/create_table_as.sql", "original_file_path": "macros/materializations/models/table/create_table_as.sql", "unique_id": "macro.dbt.get_create_table_as_sql", "macro_sql": "{% macro get_create_table_as_sql(temporary, relation, sql) -%}\n {{ adapter.dispatch('get_create_table_as_sql', 'dbt')(temporary, relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_create_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.641506, "supported_languages": null}, "macro.dbt.default__get_create_table_as_sql": {"name": "default__get_create_table_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table/create_table_as.sql", "original_file_path": "macros/materializations/models/table/create_table_as.sql", "unique_id": "macro.dbt.default__get_create_table_as_sql", "macro_sql": "{% macro default__get_create_table_as_sql(temporary, relation, sql) -%}\n {{ return(create_table_as(temporary, relation, sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.641694, "supported_languages": null}, "macro.dbt.create_table_as": {"name": "create_table_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table/create_table_as.sql", "original_file_path": "macros/materializations/models/table/create_table_as.sql", "unique_id": "macro.dbt.create_table_as", "macro_sql": "{% macro create_table_as(temporary, relation, compiled_code, language='sql') -%}\n {# backward compatibility for create_table_as that does not support language #}\n {% if language == \"sql\" %}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, compiled_code)}}\n {% else %}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, compiled_code, language) }}\n {% endif %}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.642134, "supported_languages": null}, "macro.dbt.default__create_table_as": {"name": "default__create_table_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table/create_table_as.sql", "original_file_path": "macros/materializations/models/table/create_table_as.sql", "unique_id": "macro.dbt.default__create_table_as", "macro_sql": "{% macro default__create_table_as(temporary, relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary: -%}temporary{%- endif %} table\n {{ relation.include(database=(not temporary), schema=(not temporary)) }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {{ get_columns_spec_ddl() }}\n {%- set sql = get_select_subquery(sql) %}\n {% endif %}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_assert_columns_equivalent", "macro.dbt.get_columns_spec_ddl", "macro.dbt.get_select_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.642807, "supported_languages": null}, "macro.dbt.get_select_subquery": {"name": "get_select_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table/create_table_as.sql", "original_file_path": "macros/materializations/models/table/create_table_as.sql", "unique_id": "macro.dbt.get_select_subquery", "macro_sql": "{% macro get_select_subquery(sql) %}\n {{ return(adapter.dispatch('get_select_subquery', 'dbt')(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_select_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.643002, "supported_languages": null}, "macro.dbt.default__get_select_subquery": {"name": "default__get_select_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table/create_table_as.sql", "original_file_path": "macros/materializations/models/table/create_table_as.sql", "unique_id": "macro.dbt.default__get_select_subquery", "macro_sql": "{% macro default__get_select_subquery(sql) %}\n select\n {% for column in model['columns'] %}\n {{ column }}{{ \", \" if not loop.last }}\n {% endfor %}\n from (\n {{ sql }}\n ) as model_subq\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.643258, "supported_languages": null}, "macro.dbt.materialization_view_default": {"name": "materialization_view_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/view/view.sql", "original_file_path": "macros/materializations/models/view/view.sql", "unique_id": "macro.dbt.materialization_view_default", "macro_sql": "{%- materialization view, default -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='view') -%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}\n\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}\n /*\n This relation (probably) doesn't exist yet. If it does exist, it's a leftover from\n a previous run, and we're going to try to drop it immediately. At the end of this\n materialization, we're going to rename the \"existing_relation\" to this identifier,\n and then we're going to drop it. In order to make sure we run the correct one of:\n - drop view ...\n - drop table ...\n\n We need to set the type of this relation to be the type of the existing_relation, if it exists,\n or else \"view\" as a sane default if it does not. Note that if the existing_relation does not\n exist, then there is nothing to move out of the way and subsequentally drop. In that case,\n this relation will be effectively unused.\n */\n {%- set backup_relation_type = 'view' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n -- move the existing view out of the way\n {% if existing_relation is not none %}\n {{ adapter.rename_relation(existing_relation, backup_relation) }}\n {% endif %}\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.run_hooks", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.get_create_view_as_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.645857, "supported_languages": ["sql"]}, "macro.dbt.handle_existing_table": {"name": "handle_existing_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/view/helpers.sql", "original_file_path": "macros/materializations/models/view/helpers.sql", "unique_id": "macro.dbt.handle_existing_table", "macro_sql": "{% macro handle_existing_table(full_refresh, old_relation) %}\n {{ adapter.dispatch('handle_existing_table', 'dbt')(full_refresh, old_relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__handle_existing_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.64616, "supported_languages": null}, "macro.dbt.default__handle_existing_table": {"name": "default__handle_existing_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/view/helpers.sql", "original_file_path": "macros/materializations/models/view/helpers.sql", "unique_id": "macro.dbt.default__handle_existing_table", "macro_sql": "{% macro default__handle_existing_table(full_refresh, old_relation) %}\n {{ log(\"Dropping relation \" ~ old_relation ~ \" because it is of type \" ~ old_relation.type) }}\n {{ adapter.drop_relation(old_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6463842, "supported_languages": null}, "macro.dbt.create_or_replace_view": {"name": "create_or_replace_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/view/create_or_replace_view.sql", "original_file_path": "macros/materializations/models/view/create_or_replace_view.sql", "unique_id": "macro.dbt.create_or_replace_view", "macro_sql": "{% macro create_or_replace_view() %}\n {%- set identifier = model['alias'] -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database,\n type='view') -%}\n {% set grant_config = config.get('grants') %}\n\n {{ run_hooks(pre_hooks) }}\n\n -- If there's a table with the same name and we weren't told to full refresh,\n -- that's an error. If we were told to full refresh, drop it. This behavior differs\n -- for Snowflake and BigQuery, so multiple dispatch is used.\n {%- if old_relation is not none and old_relation.is_table -%}\n {{ handle_existing_table(should_full_refresh(), old_relation) }}\n {%- endif -%}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(target_relation, sql) }}\n {%- endcall %}\n\n {% set should_revoke = should_revoke(exists_as_view, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=True) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.handle_existing_table", "macro.dbt.should_full_refresh", "macro.dbt.statement", "macro.dbt.get_create_view_as_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6479082, "supported_languages": null}, "macro.dbt.get_create_view_as_sql": {"name": "get_create_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/view/create_view_as.sql", "original_file_path": "macros/materializations/models/view/create_view_as.sql", "unique_id": "macro.dbt.get_create_view_as_sql", "macro_sql": "{% macro get_create_view_as_sql(relation, sql) -%}\n {{ adapter.dispatch('get_create_view_as_sql', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_create_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.648345, "supported_languages": null}, "macro.dbt.default__get_create_view_as_sql": {"name": "default__get_create_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/view/create_view_as.sql", "original_file_path": "macros/materializations/models/view/create_view_as.sql", "unique_id": "macro.dbt.default__get_create_view_as_sql", "macro_sql": "{% macro default__get_create_view_as_sql(relation, sql) -%}\n {{ return(create_view_as(relation, sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.648515, "supported_languages": null}, "macro.dbt.create_view_as": {"name": "create_view_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/view/create_view_as.sql", "original_file_path": "macros/materializations/models/view/create_view_as.sql", "unique_id": "macro.dbt.create_view_as", "macro_sql": "{% macro create_view_as(relation, sql) -%}\n {{ adapter.dispatch('create_view_as', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6487029, "supported_languages": null}, "macro.dbt.default__create_view_as": {"name": "default__create_view_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/view/create_view_as.sql", "original_file_path": "macros/materializations/models/view/create_view_as.sql", "unique_id": "macro.dbt.default__create_view_as", "macro_sql": "{% macro default__create_view_as(relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n create view {{ relation }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {%- endif %}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6491442, "supported_languages": null}, "macro.dbt.materialization_seed_default": {"name": "materialization_seed_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/seed.sql", "original_file_path": "macros/materializations/seeds/seed.sql", "unique_id": "macro.dbt.materialization_seed_default", "macro_sql": "{% materialization seed, default %}\n\n {%- set identifier = model['alias'] -%}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set grant_config = config.get('grants') -%}\n {%- set agate_table = load_agate_table() -%}\n -- grab current tables grants config for comparision later on\n\n {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% set create_table_sql = \"\" %}\n {% if exists_as_view %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a view\".format(old_relation)) }}\n {% elif exists_as_table %}\n {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}\n {% else %}\n {% set create_table_sql = create_csv_table(model, agate_table) %}\n {% endif %}\n\n {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}\n {% set rows_affected = (agate_table.rows | length) %}\n {% set sql = load_csv_rows(model, agate_table) %}\n\n {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}\n {{ get_csv_sql(create_table_sql, sql) }};\n {% endcall %}\n\n {% set target_relation = this.incorporate(type='table') %}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if full_refresh_mode or not exists_as_table %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.run_hooks", "macro.dbt.reset_csv_table", "macro.dbt.create_csv_table", "macro.dbt.load_csv_rows", "macro.dbt.noop_statement", "macro.dbt.get_csv_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.652572, "supported_languages": ["sql"]}, "macro.dbt.create_csv_table": {"name": "create_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.create_csv_table", "macro_sql": "{% macro create_csv_table(model, agate_table) -%}\n {{ adapter.dispatch('create_csv_table', 'dbt')(model, agate_table) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.657813, "supported_languages": null}, "macro.dbt.default__create_csv_table": {"name": "default__create_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__create_csv_table", "macro_sql": "{% macro default__create_csv_table(model, agate_table) %}\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n\n {% set sql %}\n create table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.65873, "supported_languages": null}, "macro.dbt.reset_csv_table": {"name": "reset_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.reset_csv_table", "macro_sql": "{% macro reset_csv_table(model, full_refresh, old_relation, agate_table) -%}\n {{ adapter.dispatch('reset_csv_table', 'dbt')(model, full_refresh, old_relation, agate_table) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__reset_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.658967, "supported_languages": null}, "macro.dbt.default__reset_csv_table": {"name": "default__reset_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__reset_csv_table", "macro_sql": "{% macro default__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% set sql = \"\" %}\n {% if full_refresh %}\n {{ adapter.drop_relation(old_relation) }}\n {% set sql = create_csv_table(model, agate_table) %}\n {% else %}\n {{ adapter.truncate_relation(old_relation) }}\n {% set sql = \"truncate table \" ~ old_relation %}\n {% endif %}\n\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.659456, "supported_languages": null}, "macro.dbt.get_csv_sql": {"name": "get_csv_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_csv_sql", "macro_sql": "{% macro get_csv_sql(create_or_truncate_sql, insert_sql) %}\n {{ adapter.dispatch('get_csv_sql', 'dbt')(create_or_truncate_sql, insert_sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_csv_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.659651, "supported_languages": null}, "macro.dbt.default__get_csv_sql": {"name": "default__get_csv_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__get_csv_sql", "macro_sql": "{% macro default__get_csv_sql(create_or_truncate_sql, insert_sql) %}\n {{ create_or_truncate_sql }};\n -- dbt seed --\n {{ insert_sql }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.659791, "supported_languages": null}, "macro.dbt.get_binding_char": {"name": "get_binding_char", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_binding_char", "macro_sql": "{% macro get_binding_char() -%}\n {{ adapter.dispatch('get_binding_char', 'dbt')() }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.659935, "supported_languages": null}, "macro.dbt.default__get_binding_char": {"name": "default__get_binding_char", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__get_binding_char", "macro_sql": "{% macro default__get_binding_char() %}\n {{ return('%s') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6600509, "supported_languages": null}, "macro.dbt.get_batch_size": {"name": "get_batch_size", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_batch_size", "macro_sql": "{% macro get_batch_size() -%}\n {{ return(adapter.dispatch('get_batch_size', 'dbt')()) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_batch_size"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6602108, "supported_languages": null}, "macro.dbt.default__get_batch_size": {"name": "default__get_batch_size", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__get_batch_size", "macro_sql": "{% macro default__get_batch_size() %}\n {{ return(10000) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6603289, "supported_languages": null}, "macro.dbt.get_seed_column_quoted_csv": {"name": "get_seed_column_quoted_csv", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_seed_column_quoted_csv", "macro_sql": "{% macro get_seed_column_quoted_csv(model, column_names) %}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote_seed_column(col, quote_seed_column)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.660815, "supported_languages": null}, "macro.dbt.load_csv_rows": {"name": "load_csv_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.load_csv_rows", "macro_sql": "{% macro load_csv_rows(model, agate_table) -%}\n {{ adapter.dispatch('load_csv_rows', 'dbt')(model, agate_table) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__load_csv_rows"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6610072, "supported_languages": null}, "macro.dbt.default__load_csv_rows": {"name": "default__load_csv_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__load_csv_rows", "macro_sql": "{% macro default__load_csv_rows(model, agate_table) %}\n\n {% set batch_size = get_batch_size() %}\n\n {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}\n {% set bindings = [] %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert into {{ this.render() }} ({{ cols_sql }}) values\n {% for row in chunk -%}\n ({%- for column in agate_table.column_names -%}\n {{ get_binding_char() }}\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_batch_size", "macro.dbt.get_seed_column_quoted_csv", "macro.dbt.get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.66231, "supported_languages": null}, "macro.dbt.generate_alias_name": {"name": "generate_alias_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_alias.sql", "original_file_path": "macros/get_custom_name/get_custom_alias.sql", "unique_id": "macro.dbt.generate_alias_name", "macro_sql": "{% macro generate_alias_name(custom_alias_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_alias_name', 'dbt')(custom_alias_name, node)) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__generate_alias_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.662733, "supported_languages": null}, "macro.dbt.default__generate_alias_name": {"name": "default__generate_alias_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_alias.sql", "original_file_path": "macros/get_custom_name/get_custom_alias.sql", "unique_id": "macro.dbt.default__generate_alias_name", "macro_sql": "{% macro default__generate_alias_name(custom_alias_name=none, node=none) -%}\n\n {%- if custom_alias_name -%}\n\n {{ custom_alias_name | trim }}\n\n {%- elif node.version -%}\n\n {{ return(node.name ~ \"_v\" ~ node.version) }}\n\n {%- else -%}\n\n {{ node.name }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6630669, "supported_languages": null}, "macro.dbt.generate_schema_name": {"name": "generate_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "unique_id": "macro.dbt.generate_schema_name", "macro_sql": "{% macro generate_schema_name(custom_schema_name=none, node=none) -%}\n {{ return(adapter.dispatch('generate_schema_name', 'dbt')(custom_schema_name, node)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__generate_schema_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6635892, "supported_languages": null}, "macro.dbt.default__generate_schema_name": {"name": "default__generate_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "unique_id": "macro.dbt.default__generate_schema_name", "macro_sql": "{% macro default__generate_schema_name(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if custom_schema_name is none -%}\n\n {{ default_schema }}\n\n {%- else -%}\n\n {{ default_schema }}_{{ custom_schema_name | trim }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.663862, "supported_languages": null}, "macro.dbt.generate_schema_name_for_env": {"name": "generate_schema_name_for_env", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "unique_id": "macro.dbt.generate_schema_name_for_env", "macro_sql": "{% macro generate_schema_name_for_env(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if target.name == 'prod' and custom_schema_name is not none -%}\n\n {{ custom_schema_name | trim }}\n\n {%- else -%}\n\n {{ default_schema }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.664153, "supported_languages": null}, "macro.dbt.generate_database_name": {"name": "generate_database_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "unique_id": "macro.dbt.generate_database_name", "macro_sql": "{% macro generate_database_name(custom_database_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_database_name', 'dbt')(custom_database_name, node)) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__generate_database_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6645498, "supported_languages": null}, "macro.dbt.default__generate_database_name": {"name": "default__generate_database_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "unique_id": "macro.dbt.default__generate_database_name", "macro_sql": "{% macro default__generate_database_name(custom_database_name=none, node=none) -%}\n {%- set default_database = target.database -%}\n {%- if custom_database_name is none -%}\n\n {{ default_database }}\n\n {%- else -%}\n\n {{ custom_database_name }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.664808, "supported_languages": null}, "macro.dbt.default__test_relationships": {"name": "default__test_relationships", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/relationships.sql", "original_file_path": "macros/generic_test_sql/relationships.sql", "unique_id": "macro.dbt.default__test_relationships", "macro_sql": "{% macro default__test_relationships(model, column_name, to, field) %}\n\nwith child as (\n select {{ column_name }} as from_field\n from {{ model }}\n where {{ column_name }} is not null\n),\n\nparent as (\n select {{ field }} as to_field\n from {{ to }}\n)\n\nselect\n from_field\n\nfrom child\nleft join parent\n on child.from_field = parent.to_field\n\nwhere parent.to_field is null\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6651552, "supported_languages": null}, "macro.dbt.default__test_not_null": {"name": "default__test_not_null", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/not_null.sql", "original_file_path": "macros/generic_test_sql/not_null.sql", "unique_id": "macro.dbt.default__test_not_null", "macro_sql": "{% macro default__test_not_null(model, column_name) %}\n\n{% set column_list = '*' if should_store_failures() else column_name %}\n\nselect {{ column_list }}\nfrom {{ model }}\nwhere {{ column_name }} is null\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_store_failures"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.66545, "supported_languages": null}, "macro.dbt.default__test_unique": {"name": "default__test_unique", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/unique.sql", "original_file_path": "macros/generic_test_sql/unique.sql", "unique_id": "macro.dbt.default__test_unique", "macro_sql": "{% macro default__test_unique(model, column_name) %}\n\nselect\n {{ column_name }} as unique_field,\n count(*) as n_records\n\nfrom {{ model }}\nwhere {{ column_name }} is not null\ngroup by {{ column_name }}\nhaving count(*) > 1\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.665698, "supported_languages": null}, "macro.dbt.default__test_accepted_values": {"name": "default__test_accepted_values", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/accepted_values.sql", "original_file_path": "macros/generic_test_sql/accepted_values.sql", "unique_id": "macro.dbt.default__test_accepted_values", "macro_sql": "{% macro default__test_accepted_values(model, column_name, values, quote=True) %}\n\nwith all_values as (\n\n select\n {{ column_name }} as value_field,\n count(*) as n_records\n\n from {{ model }}\n group by {{ column_name }}\n\n)\n\nselect *\nfrom all_values\nwhere value_field not in (\n {% for value in values -%}\n {% if quote -%}\n '{{ value }}'\n {%- else -%}\n {{ value }}\n {%- endif -%}\n {%- if not loop.last -%},{%- endif %}\n {%- endfor %}\n)\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6662662, "supported_languages": null}, "macro.dbt.statement": {"name": "statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt.statement", "macro_sql": "\n{%- macro statement(name=None, fetch_result=False, auto_begin=True, language='sql') -%}\n {%- if execute: -%}\n {%- set compiled_code = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime {} for node \"{}\"'.format(language, model['unique_id'])) }}\n {{ write(compiled_code) }}\n {%- endif -%}\n {%- if language == 'sql'-%}\n {%- set res, table = adapter.execute(compiled_code, auto_begin=auto_begin, fetch=fetch_result) -%}\n {%- elif language == 'python' -%}\n {%- set res = submit_python_job(model, compiled_code) -%}\n {#-- TODO: What should table be for python models? --#}\n {%- set table = None -%}\n {%- else -%}\n {% do exceptions.raise_compiler_error(\"statement macro didn't get supported language\") %}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_result(name, response=res, agate_table=table) }}\n {%- endif -%}\n\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.667764, "supported_languages": null}, "macro.dbt.noop_statement": {"name": "noop_statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt.noop_statement", "macro_sql": "{% macro noop_statement(name=None, message=None, code=None, rows_affected=None, res=None) -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_raw_result(name, message=message, code=code, rows_affected=rows_affected, agate_table=res) }}\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6684191, "supported_languages": null}, "macro.dbt.run_query": {"name": "run_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt.run_query", "macro_sql": "{% macro run_query(sql) %}\n {% call statement(\"run_query_statement\", fetch_result=true, auto_begin=false) %}\n {{ sql }}\n {% endcall %}\n\n {% do return(load_result(\"run_query_statement\").table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.668716, "supported_languages": null}, "macro.dbt.convert_datetime": {"name": "convert_datetime", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.convert_datetime", "macro_sql": "{% macro convert_datetime(date_str, date_fmt) %}\n\n {% set error_msg -%}\n The provided partition date '{{ date_str }}' does not match the expected format '{{ date_fmt }}'\n {%- endset %}\n\n {% set res = try_or_compiler_error(error_msg, modules.datetime.datetime.strptime, date_str.strip(), date_fmt) %}\n {{ return(res) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.670603, "supported_languages": null}, "macro.dbt.dates_in_range": {"name": "dates_in_range", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.dates_in_range", "macro_sql": "{% macro dates_in_range(start_date_str, end_date_str=none, in_fmt=\"%Y%m%d\", out_fmt=\"%Y%m%d\") %}\n {% set end_date_str = start_date_str if end_date_str is none else end_date_str %}\n\n {% set start_date = convert_datetime(start_date_str, in_fmt) %}\n {% set end_date = convert_datetime(end_date_str, in_fmt) %}\n\n {% set day_count = (end_date - start_date).days %}\n {% if day_count < 0 %}\n {% set msg -%}\n Partition start date is after the end date ({{ start_date }}, {{ end_date }})\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg, model) }}\n {% endif %}\n\n {% set date_list = [] %}\n {% for i in range(0, day_count + 1) %}\n {% set the_date = (modules.datetime.timedelta(days=i) + start_date) %}\n {% if not out_fmt %}\n {% set _ = date_list.append(the_date) %}\n {% else %}\n {% set _ = date_list.append(the_date.strftime(out_fmt)) %}\n {% endif %}\n {% endfor %}\n\n {{ return(date_list) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.convert_datetime"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.671814, "supported_languages": null}, "macro.dbt.partition_range": {"name": "partition_range", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.partition_range", "macro_sql": "{% macro partition_range(raw_partition_date, date_fmt='%Y%m%d') %}\n {% set partition_range = (raw_partition_date | string).split(\",\") %}\n\n {% if (partition_range | length) == 1 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = none %}\n {% elif (partition_range | length) == 2 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = partition_range[1] %}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Invalid partition time. Expected format: {Start Date}[,{End Date}]. Got: \" ~ raw_partition_date) }}\n {% endif %}\n\n {{ return(dates_in_range(start_date, end_date, in_fmt=date_fmt)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.dates_in_range"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6726348, "supported_languages": null}, "macro.dbt.py_current_timestring": {"name": "py_current_timestring", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.py_current_timestring", "macro_sql": "{% macro py_current_timestring() %}\n {% set dt = modules.datetime.datetime.now() %}\n {% do return(dt.strftime(\"%Y%m%d%H%M%S%f\")) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.672904, "supported_languages": null}, "macro.dbt.except": {"name": "except", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/except.sql", "original_file_path": "macros/utils/except.sql", "unique_id": "macro.dbt.except", "macro_sql": "{% macro except() %}\n {{ return(adapter.dispatch('except', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__except"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6731641, "supported_languages": null}, "macro.dbt.default__except": {"name": "default__except", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/except.sql", "original_file_path": "macros/utils/except.sql", "unique_id": "macro.dbt.default__except", "macro_sql": "{% macro default__except() %}\n\n except\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.673249, "supported_languages": null}, "macro.dbt.replace": {"name": "replace", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/replace.sql", "original_file_path": "macros/utils/replace.sql", "unique_id": "macro.dbt.replace", "macro_sql": "{% macro replace(field, old_chars, new_chars) -%}\n {{ return(adapter.dispatch('replace', 'dbt') (field, old_chars, new_chars)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__replace"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.673606, "supported_languages": null}, "macro.dbt.default__replace": {"name": "default__replace", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/replace.sql", "original_file_path": "macros/utils/replace.sql", "unique_id": "macro.dbt.default__replace", "macro_sql": "{% macro default__replace(field, old_chars, new_chars) %}\n\n replace(\n {{ field }},\n {{ old_chars }},\n {{ new_chars }}\n )\n\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.673785, "supported_languages": null}, "macro.dbt.concat": {"name": "concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/concat.sql", "original_file_path": "macros/utils/concat.sql", "unique_id": "macro.dbt.concat", "macro_sql": "{% macro concat(fields) -%}\n {{ return(adapter.dispatch('concat', 'dbt')(fields)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__concat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.674051, "supported_languages": null}, "macro.dbt.default__concat": {"name": "default__concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/concat.sql", "original_file_path": "macros/utils/concat.sql", "unique_id": "macro.dbt.default__concat", "macro_sql": "{% macro default__concat(fields) -%}\n {{ fields|join(' || ') }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6741881, "supported_languages": null}, "macro.dbt.length": {"name": "length", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/length.sql", "original_file_path": "macros/utils/length.sql", "unique_id": "macro.dbt.length", "macro_sql": "{% macro length(expression) -%}\n {{ return(adapter.dispatch('length', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__length"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.674554, "supported_languages": null}, "macro.dbt.default__length": {"name": "default__length", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/length.sql", "original_file_path": "macros/utils/length.sql", "unique_id": "macro.dbt.default__length", "macro_sql": "{% macro default__length(expression) %}\n\n length(\n {{ expression }}\n )\n\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.674687, "supported_languages": null}, "macro.dbt.dateadd": {"name": "dateadd", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt.dateadd", "macro_sql": "{% macro dateadd(datepart, interval, from_date_or_timestamp) %}\n {{ return(adapter.dispatch('dateadd', 'dbt')(datepart, interval, from_date_or_timestamp)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__dateadd"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.675072, "supported_languages": null}, "macro.dbt.default__dateadd": {"name": "default__dateadd", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt.default__dateadd", "macro_sql": "{% macro default__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n dateadd(\n {{ datepart }},\n {{ interval }},\n {{ from_date_or_timestamp }}\n )\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6752498, "supported_languages": null}, "macro.dbt.intersect": {"name": "intersect", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/intersect.sql", "original_file_path": "macros/utils/intersect.sql", "unique_id": "macro.dbt.intersect", "macro_sql": "{% macro intersect() %}\n {{ return(adapter.dispatch('intersect', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__intersect"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.675481, "supported_languages": null}, "macro.dbt.default__intersect": {"name": "default__intersect", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/intersect.sql", "original_file_path": "macros/utils/intersect.sql", "unique_id": "macro.dbt.default__intersect", "macro_sql": "{% macro default__intersect() %}\n\n intersect\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.675561, "supported_languages": null}, "macro.dbt.escape_single_quotes": {"name": "escape_single_quotes", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/escape_single_quotes.sql", "original_file_path": "macros/utils/escape_single_quotes.sql", "unique_id": "macro.dbt.escape_single_quotes", "macro_sql": "{% macro escape_single_quotes(expression) %}\n {{ return(adapter.dispatch('escape_single_quotes', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__escape_single_quotes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6758258, "supported_languages": null}, "macro.dbt.default__escape_single_quotes": {"name": "default__escape_single_quotes", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/escape_single_quotes.sql", "original_file_path": "macros/utils/escape_single_quotes.sql", "unique_id": "macro.dbt.default__escape_single_quotes", "macro_sql": "{% macro default__escape_single_quotes(expression) -%}\n{{ expression | replace(\"'\",\"''\") }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6759732, "supported_languages": null}, "macro.dbt.right": {"name": "right", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/right.sql", "original_file_path": "macros/utils/right.sql", "unique_id": "macro.dbt.right", "macro_sql": "{% macro right(string_text, length_expression) -%}\n {{ return(adapter.dispatch('right', 'dbt') (string_text, length_expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__right"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.676275, "supported_languages": null}, "macro.dbt.default__right": {"name": "default__right", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/right.sql", "original_file_path": "macros/utils/right.sql", "unique_id": "macro.dbt.default__right", "macro_sql": "{% macro default__right(string_text, length_expression) %}\n\n right(\n {{ string_text }},\n {{ length_expression }}\n )\n\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6765099, "supported_languages": null}, "macro.dbt.listagg": {"name": "listagg", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "unique_id": "macro.dbt.listagg", "macro_sql": "{% macro listagg(measure, delimiter_text=\"','\", order_by_clause=none, limit_num=none) -%}\n {{ return(adapter.dispatch('listagg', 'dbt') (measure, delimiter_text, order_by_clause, limit_num)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__listagg"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6771588, "supported_languages": null}, "macro.dbt.default__listagg": {"name": "default__listagg", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "unique_id": "macro.dbt.default__listagg", "macro_sql": "{% macro default__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n {% if limit_num -%}\n array_to_string(\n array_slice(\n array_agg(\n {{ measure }}\n ){% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n ,0\n ,{{ limit_num }}\n ),\n {{ delimiter_text }}\n )\n {%- else %}\n listagg(\n {{ measure }},\n {{ delimiter_text }}\n )\n {% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n {%- endif %}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.677581, "supported_languages": null}, "macro.dbt.datediff": {"name": "datediff", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt.datediff", "macro_sql": "{% macro datediff(first_date, second_date, datepart) %}\n {{ return(adapter.dispatch('datediff', 'dbt')(first_date, second_date, datepart)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__datediff"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.677927, "supported_languages": null}, "macro.dbt.default__datediff": {"name": "default__datediff", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt.default__datediff", "macro_sql": "{% macro default__datediff(first_date, second_date, datepart) -%}\n\n datediff(\n {{ datepart }},\n {{ first_date }},\n {{ second_date }}\n )\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.678091, "supported_languages": null}, "macro.dbt.safe_cast": {"name": "safe_cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "unique_id": "macro.dbt.safe_cast", "macro_sql": "{% macro safe_cast(field, type) %}\n {{ return(adapter.dispatch('safe_cast', 'dbt') (field, type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__safe_cast"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6783779, "supported_languages": null}, "macro.dbt.default__safe_cast": {"name": "default__safe_cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "unique_id": "macro.dbt.default__safe_cast", "macro_sql": "{% macro default__safe_cast(field, type) %}\n {# most databases don't support this function yet\n so we just need to use cast #}\n cast({{field}} as {{type}})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6785228, "supported_languages": null}, "macro.dbt.hash": {"name": "hash", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/hash.sql", "original_file_path": "macros/utils/hash.sql", "unique_id": "macro.dbt.hash", "macro_sql": "{% macro hash(field) -%}\n {{ return(adapter.dispatch('hash', 'dbt') (field)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__hash"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6787798, "supported_languages": null}, "macro.dbt.default__hash": {"name": "default__hash", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/hash.sql", "original_file_path": "macros/utils/hash.sql", "unique_id": "macro.dbt.default__hash", "macro_sql": "{% macro default__hash(field) -%}\n md5(cast({{ field }} as {{ api.Column.translate_type('string') }}))\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6789498, "supported_languages": null}, "macro.dbt.cast_bool_to_text": {"name": "cast_bool_to_text", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast_bool_to_text.sql", "original_file_path": "macros/utils/cast_bool_to_text.sql", "unique_id": "macro.dbt.cast_bool_to_text", "macro_sql": "{% macro cast_bool_to_text(field) %}\n {{ adapter.dispatch('cast_bool_to_text', 'dbt') (field) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__cast_bool_to_text"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.679196, "supported_languages": null}, "macro.dbt.default__cast_bool_to_text": {"name": "default__cast_bool_to_text", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast_bool_to_text.sql", "original_file_path": "macros/utils/cast_bool_to_text.sql", "unique_id": "macro.dbt.default__cast_bool_to_text", "macro_sql": "{% macro default__cast_bool_to_text(field) %}\n cast({{ field }} as {{ api.Column.translate_type('string') }})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6793652, "supported_languages": null}, "macro.dbt.any_value": {"name": "any_value", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "unique_id": "macro.dbt.any_value", "macro_sql": "{% macro any_value(expression) -%}\n {{ return(adapter.dispatch('any_value', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__any_value"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6796181, "supported_languages": null}, "macro.dbt.default__any_value": {"name": "default__any_value", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "unique_id": "macro.dbt.default__any_value", "macro_sql": "{% macro default__any_value(expression) -%}\n\n any_value({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6797252, "supported_languages": null}, "macro.dbt.position": {"name": "position", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/position.sql", "original_file_path": "macros/utils/position.sql", "unique_id": "macro.dbt.position", "macro_sql": "{% macro position(substring_text, string_text) -%}\n {{ return(adapter.dispatch('position', 'dbt') (substring_text, string_text)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__position"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.680025, "supported_languages": null}, "macro.dbt.default__position": {"name": "default__position", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/position.sql", "original_file_path": "macros/utils/position.sql", "unique_id": "macro.dbt.default__position", "macro_sql": "{% macro default__position(substring_text, string_text) %}\n\n position(\n {{ substring_text }} in {{ string_text }}\n )\n\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6801689, "supported_languages": null}, "macro.dbt.string_literal": {"name": "string_literal", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/literal.sql", "original_file_path": "macros/utils/literal.sql", "unique_id": "macro.dbt.string_literal", "macro_sql": "{%- macro string_literal(value) -%}\n {{ return(adapter.dispatch('string_literal', 'dbt') (value)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__string_literal"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.680419, "supported_languages": null}, "macro.dbt.default__string_literal": {"name": "default__string_literal", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/literal.sql", "original_file_path": "macros/utils/literal.sql", "unique_id": "macro.dbt.default__string_literal", "macro_sql": "{% macro default__string_literal(value) -%}\n '{{ value }}'\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6805272, "supported_languages": null}, "macro.dbt.type_string": {"name": "type_string", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_string", "macro_sql": "\n\n{%- macro type_string() -%}\n {{ return(adapter.dispatch('type_string', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_string"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.681462, "supported_languages": null}, "macro.dbt.default__type_string": {"name": "default__type_string", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_string", "macro_sql": "{% macro default__type_string() %}\n {{ return(api.Column.translate_type(\"string\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.681621, "supported_languages": null}, "macro.dbt.type_timestamp": {"name": "type_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_timestamp", "macro_sql": "\n\n{%- macro type_timestamp() -%}\n {{ return(adapter.dispatch('type_timestamp', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6818528, "supported_languages": null}, "macro.dbt.default__type_timestamp": {"name": "default__type_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_timestamp", "macro_sql": "{% macro default__type_timestamp() %}\n {{ return(api.Column.translate_type(\"timestamp\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6820118, "supported_languages": null}, "macro.dbt.type_float": {"name": "type_float", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_float", "macro_sql": "\n\n{%- macro type_float() -%}\n {{ return(adapter.dispatch('type_float', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_float"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6821811, "supported_languages": null}, "macro.dbt.default__type_float": {"name": "default__type_float", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_float", "macro_sql": "{% macro default__type_float() %}\n {{ return(api.Column.translate_type(\"float\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.682334, "supported_languages": null}, "macro.dbt.type_numeric": {"name": "type_numeric", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_numeric", "macro_sql": "\n\n{%- macro type_numeric() -%}\n {{ return(adapter.dispatch('type_numeric', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_numeric"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6826391, "supported_languages": null}, "macro.dbt.default__type_numeric": {"name": "default__type_numeric", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_numeric", "macro_sql": "{% macro default__type_numeric() %}\n {{ return(api.Column.numeric_type(\"numeric\", 28, 6)) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.682859, "supported_languages": null}, "macro.dbt.type_bigint": {"name": "type_bigint", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_bigint", "macro_sql": "\n\n{%- macro type_bigint() -%}\n {{ return(adapter.dispatch('type_bigint', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_bigint"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.683039, "supported_languages": null}, "macro.dbt.default__type_bigint": {"name": "default__type_bigint", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_bigint", "macro_sql": "{% macro default__type_bigint() %}\n {{ return(api.Column.translate_type(\"bigint\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.683199, "supported_languages": null}, "macro.dbt.type_int": {"name": "type_int", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_int", "macro_sql": "\n\n{%- macro type_int() -%}\n {{ return(adapter.dispatch('type_int', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_int"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.683383, "supported_languages": null}, "macro.dbt.default__type_int": {"name": "default__type_int", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_int", "macro_sql": "{%- macro default__type_int() -%}\n {{ return(api.Column.translate_type(\"integer\")) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.683554, "supported_languages": null}, "macro.dbt.type_boolean": {"name": "type_boolean", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_boolean", "macro_sql": "\n\n{%- macro type_boolean() -%}\n {{ return(adapter.dispatch('type_boolean', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_boolean"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.683734, "supported_languages": null}, "macro.dbt.default__type_boolean": {"name": "default__type_boolean", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_boolean", "macro_sql": "{%- macro default__type_boolean() -%}\n {{ return(api.Column.translate_type(\"boolean\")) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.683891, "supported_languages": null}, "macro.dbt.array_concat": {"name": "array_concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_concat.sql", "original_file_path": "macros/utils/array_concat.sql", "unique_id": "macro.dbt.array_concat", "macro_sql": "{% macro array_concat(array_1, array_2) -%}\n {{ return(adapter.dispatch('array_concat', 'dbt')(array_1, array_2)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__array_concat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.684203, "supported_languages": null}, "macro.dbt.default__array_concat": {"name": "default__array_concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_concat.sql", "original_file_path": "macros/utils/array_concat.sql", "unique_id": "macro.dbt.default__array_concat", "macro_sql": "{% macro default__array_concat(array_1, array_2) -%}\n array_cat({{ array_1 }}, {{ array_2 }})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.684342, "supported_languages": null}, "macro.dbt.bool_or": {"name": "bool_or", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/bool_or.sql", "original_file_path": "macros/utils/bool_or.sql", "unique_id": "macro.dbt.bool_or", "macro_sql": "{% macro bool_or(expression) -%}\n {{ return(adapter.dispatch('bool_or', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__bool_or"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6845891, "supported_languages": null}, "macro.dbt.default__bool_or": {"name": "default__bool_or", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/bool_or.sql", "original_file_path": "macros/utils/bool_or.sql", "unique_id": "macro.dbt.default__bool_or", "macro_sql": "{% macro default__bool_or(expression) -%}\n\n bool_or({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.684702, "supported_languages": null}, "macro.dbt.last_day": {"name": "last_day", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt.last_day", "macro_sql": "{% macro last_day(date, datepart) %}\n {{ return(adapter.dispatch('last_day', 'dbt') (date, datepart)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__last_day"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.685059, "supported_languages": null}, "macro.dbt.default_last_day": {"name": "default_last_day", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt.default_last_day", "macro_sql": "\n\n{%- macro default_last_day(date, datepart) -%}\n cast(\n {{dbt.dateadd('day', '-1',\n dbt.dateadd(datepart, '1', dbt.date_trunc(datepart, date))\n )}}\n as date)\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.dateadd", "macro.dbt.date_trunc"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.685353, "supported_languages": null}, "macro.dbt.default__last_day": {"name": "default__last_day", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt.default__last_day", "macro_sql": "{% macro default__last_day(date, datepart) -%}\n {{dbt.default_last_day(date, datepart)}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default_last_day"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.685507, "supported_languages": null}, "macro.dbt.split_part": {"name": "split_part", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt.split_part", "macro_sql": "{% macro split_part(string_text, delimiter_text, part_number) %}\n {{ return(adapter.dispatch('split_part', 'dbt') (string_text, delimiter_text, part_number)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__split_part"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6860402, "supported_languages": null}, "macro.dbt.default__split_part": {"name": "default__split_part", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt.default__split_part", "macro_sql": "{% macro default__split_part(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n {{ part_number }}\n )\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.686302, "supported_languages": null}, "macro.dbt._split_part_negative": {"name": "_split_part_negative", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt._split_part_negative", "macro_sql": "{% macro _split_part_negative(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n length({{ string_text }})\n - length(\n replace({{ string_text }}, {{ delimiter_text }}, '')\n ) + 2 {{ part_number }}\n )\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.686583, "supported_languages": null}, "macro.dbt.date_trunc": {"name": "date_trunc", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_trunc.sql", "original_file_path": "macros/utils/date_trunc.sql", "unique_id": "macro.dbt.date_trunc", "macro_sql": "{% macro date_trunc(datepart, date) -%}\n {{ return(adapter.dispatch('date_trunc', 'dbt') (datepart, date)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__date_trunc"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6868732, "supported_languages": null}, "macro.dbt.default__date_trunc": {"name": "default__date_trunc", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_trunc.sql", "original_file_path": "macros/utils/date_trunc.sql", "unique_id": "macro.dbt.default__date_trunc", "macro_sql": "{% macro default__date_trunc(datepart, date) -%}\n date_trunc('{{datepart}}', {{date}})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.687006, "supported_languages": null}, "macro.dbt.array_construct": {"name": "array_construct", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_construct.sql", "original_file_path": "macros/utils/array_construct.sql", "unique_id": "macro.dbt.array_construct", "macro_sql": "{% macro array_construct(inputs=[], data_type=api.Column.translate_type('integer')) -%}\n {{ return(adapter.dispatch('array_construct', 'dbt')(inputs, data_type)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__array_construct"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.687397, "supported_languages": null}, "macro.dbt.default__array_construct": {"name": "default__array_construct", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_construct.sql", "original_file_path": "macros/utils/array_construct.sql", "unique_id": "macro.dbt.default__array_construct", "macro_sql": "{% macro default__array_construct(inputs, data_type) -%}\n {% if inputs|length > 0 %}\n array[ {{ inputs|join(' , ') }} ]\n {% else %}\n array[]::{{data_type}}[]\n {% endif %}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.687667, "supported_languages": null}, "macro.dbt.array_append": {"name": "array_append", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_append.sql", "original_file_path": "macros/utils/array_append.sql", "unique_id": "macro.dbt.array_append", "macro_sql": "{% macro array_append(array, new_element) -%}\n {{ return(adapter.dispatch('array_append', 'dbt')(array, new_element)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__array_append"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.687964, "supported_languages": null}, "macro.dbt.default__array_append": {"name": "default__array_append", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_append.sql", "original_file_path": "macros/utils/array_append.sql", "unique_id": "macro.dbt.default__array_append", "macro_sql": "{% macro default__array_append(array, new_element) -%}\n array_append({{ array }}, {{ new_element }})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.688101, "supported_languages": null}, "macro.dbt.create_schema": {"name": "create_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.create_schema", "macro_sql": "{% macro create_schema(relation) -%}\n {{ adapter.dispatch('create_schema', 'dbt')(relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__create_schema"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.688492, "supported_languages": null}, "macro.dbt.default__create_schema": {"name": "default__create_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.default__create_schema", "macro_sql": "{% macro default__create_schema(relation) -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier() }}\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.688687, "supported_languages": null}, "macro.dbt.drop_schema": {"name": "drop_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.drop_schema", "macro_sql": "{% macro drop_schema(relation) -%}\n {{ adapter.dispatch('drop_schema', 'dbt')(relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__drop_schema"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.688858, "supported_languages": null}, "macro.dbt.default__drop_schema": {"name": "default__drop_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.default__drop_schema", "macro_sql": "{% macro default__drop_schema(relation) -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier() }} cascade\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.689048, "supported_languages": null}, "macro.dbt.current_timestamp": {"name": "current_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.current_timestamp", "macro_sql": "{%- macro current_timestamp() -%}\n {{ adapter.dispatch('current_timestamp', 'dbt')() }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_postgres.postgres__current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6895611, "supported_languages": null}, "macro.dbt.default__current_timestamp": {"name": "default__current_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__current_timestamp", "macro_sql": "{% macro default__current_timestamp() -%}\n {{ exceptions.raise_not_implemented(\n 'current_timestamp macro not implemented for adapter ' + adapter.type()) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.689718, "supported_languages": null}, "macro.dbt.snapshot_get_time": {"name": "snapshot_get_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.snapshot_get_time", "macro_sql": "\n\n{%- macro snapshot_get_time() -%}\n {{ adapter.dispatch('snapshot_get_time', 'dbt')() }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_postgres.postgres__snapshot_get_time"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.689871, "supported_languages": null}, "macro.dbt.default__snapshot_get_time": {"name": "default__snapshot_get_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__snapshot_get_time", "macro_sql": "{% macro default__snapshot_get_time() %}\n {{ current_timestamp() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6899788, "supported_languages": null}, "macro.dbt.current_timestamp_backcompat": {"name": "current_timestamp_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.current_timestamp_backcompat", "macro_sql": "{% macro current_timestamp_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_backcompat', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__current_timestamp_backcompat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.690153, "supported_languages": null}, "macro.dbt.default__current_timestamp_backcompat": {"name": "default__current_timestamp_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__current_timestamp_backcompat", "macro_sql": "{% macro default__current_timestamp_backcompat() %}\n current_timestamp::timestamp\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.690235, "supported_languages": null}, "macro.dbt.current_timestamp_in_utc_backcompat": {"name": "current_timestamp_in_utc_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.current_timestamp_in_utc_backcompat", "macro_sql": "{% macro current_timestamp_in_utc_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_in_utc_backcompat', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__current_timestamp_in_utc_backcompat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6904068, "supported_languages": null}, "macro.dbt.default__current_timestamp_in_utc_backcompat": {"name": "default__current_timestamp_in_utc_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__current_timestamp_in_utc_backcompat", "macro_sql": "{% macro default__current_timestamp_in_utc_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_backcompat', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.current_timestamp_backcompat", "macro.dbt_postgres.postgres__current_timestamp_backcompat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.690679, "supported_languages": null}, "macro.dbt.get_create_index_sql": {"name": "get_create_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.get_create_index_sql", "macro_sql": "{% macro get_create_index_sql(relation, index_dict) -%}\n {{ return(adapter.dispatch('get_create_index_sql', 'dbt')(relation, index_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_create_index_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.691232, "supported_languages": null}, "macro.dbt.default__get_create_index_sql": {"name": "default__get_create_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__get_create_index_sql", "macro_sql": "{% macro default__get_create_index_sql(relation, index_dict) -%}\n {% do return(None) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6913862, "supported_languages": null}, "macro.dbt.create_indexes": {"name": "create_indexes", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.create_indexes", "macro_sql": "{% macro create_indexes(relation) -%}\n {{ adapter.dispatch('create_indexes', 'dbt')(relation) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.69156, "supported_languages": null}, "macro.dbt.default__create_indexes": {"name": "default__create_indexes", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__create_indexes", "macro_sql": "{% macro default__create_indexes(relation) -%}\n {%- set _indexes = config.get('indexes', default=[]) -%}\n\n {% for _index_dict in _indexes %}\n {% set create_index_sql = get_create_index_sql(relation, _index_dict) %}\n {% if create_index_sql %}\n {% do run_query(create_index_sql) %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_create_index_sql", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.692, "supported_languages": null}, "macro.dbt.make_intermediate_relation": {"name": "make_intermediate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.make_intermediate_relation", "macro_sql": "{% macro make_intermediate_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_intermediate_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_intermediate_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.695603, "supported_languages": null}, "macro.dbt.default__make_intermediate_relation": {"name": "default__make_intermediate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__make_intermediate_relation", "macro_sql": "{% macro default__make_intermediate_relation(base_relation, suffix) %}\n {{ return(default__make_temp_relation(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__make_temp_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6958349, "supported_languages": null}, "macro.dbt.make_temp_relation": {"name": "make_temp_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.make_temp_relation", "macro_sql": "{% macro make_temp_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_temp_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_temp_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6960871, "supported_languages": null}, "macro.dbt.default__make_temp_relation": {"name": "default__make_temp_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__make_temp_relation", "macro_sql": "{% macro default__make_temp_relation(base_relation, suffix) %}\n {%- set temp_identifier = base_relation.identifier ~ suffix -%}\n {%- set temp_relation = base_relation.incorporate(\n path={\"identifier\": temp_identifier}) -%}\n\n {{ return(temp_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.696411, "supported_languages": null}, "macro.dbt.make_backup_relation": {"name": "make_backup_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.make_backup_relation", "macro_sql": "{% macro make_backup_relation(base_relation, backup_relation_type, suffix='__dbt_backup') %}\n {{ return(adapter.dispatch('make_backup_relation', 'dbt')(base_relation, backup_relation_type, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__make_backup_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6966882, "supported_languages": null}, "macro.dbt.default__make_backup_relation": {"name": "default__make_backup_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__make_backup_relation", "macro_sql": "{% macro default__make_backup_relation(base_relation, backup_relation_type, suffix) %}\n {%- set backup_identifier = base_relation.identifier ~ suffix -%}\n {%- set backup_relation = base_relation.incorporate(\n path={\"identifier\": backup_identifier},\n type=backup_relation_type\n ) -%}\n {{ return(backup_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.697024, "supported_languages": null}, "macro.dbt.drop_relation": {"name": "drop_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.drop_relation", "macro_sql": "{% macro drop_relation(relation) -%}\n {{ return(adapter.dispatch('drop_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__drop_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6972108, "supported_languages": null}, "macro.dbt.default__drop_relation": {"name": "default__drop_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__drop_relation", "macro_sql": "{% macro default__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n drop {{ relation.type }} if exists {{ relation }} cascade\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.697429, "supported_languages": null}, "macro.dbt.truncate_relation": {"name": "truncate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.truncate_relation", "macro_sql": "{% macro truncate_relation(relation) -%}\n {{ return(adapter.dispatch('truncate_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__truncate_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6976268, "supported_languages": null}, "macro.dbt.default__truncate_relation": {"name": "default__truncate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__truncate_relation", "macro_sql": "{% macro default__truncate_relation(relation) -%}\n {% call statement('truncate_relation') -%}\n truncate table {{ relation }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.697804, "supported_languages": null}, "macro.dbt.rename_relation": {"name": "rename_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.rename_relation", "macro_sql": "{% macro rename_relation(from_relation, to_relation) -%}\n {{ return(adapter.dispatch('rename_relation', 'dbt')(from_relation, to_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__rename_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.69802, "supported_languages": null}, "macro.dbt.default__rename_relation": {"name": "default__rename_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__rename_relation", "macro_sql": "{% macro default__rename_relation(from_relation, to_relation) -%}\n {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %}\n {% call statement('rename_relation') -%}\n alter table {{ from_relation }} rename to {{ target_name }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6983092, "supported_languages": null}, "macro.dbt.get_or_create_relation": {"name": "get_or_create_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.get_or_create_relation", "macro_sql": "{% macro get_or_create_relation(database, schema, identifier, type) -%}\n {{ return(adapter.dispatch('get_or_create_relation', 'dbt')(database, schema, identifier, type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_or_create_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.698685, "supported_languages": null}, "macro.dbt.default__get_or_create_relation": {"name": "default__get_or_create_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__get_or_create_relation", "macro_sql": "{% macro default__get_or_create_relation(database, schema, identifier, type) %}\n {%- set target_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n\n {% if target_relation %}\n {% do return([true, target_relation]) %}\n {% endif %}\n\n {%- set new_relation = api.Relation.create(\n database=database,\n schema=schema,\n identifier=identifier,\n type=type\n ) -%}\n {% do return([false, new_relation]) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.6992798, "supported_languages": null}, "macro.dbt.load_cached_relation": {"name": "load_cached_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.load_cached_relation", "macro_sql": "{% macro load_cached_relation(relation) %}\n {% do return(adapter.get_relation(\n database=relation.database,\n schema=relation.schema,\n identifier=relation.identifier\n )) -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.699531, "supported_languages": null}, "macro.dbt.load_relation": {"name": "load_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.load_relation", "macro_sql": "{% macro load_relation(relation) %}\n {{ return(load_cached_relation(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.699724, "supported_languages": null}, "macro.dbt.drop_relation_if_exists": {"name": "drop_relation_if_exists", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.drop_relation_if_exists", "macro_sql": "{% macro drop_relation_if_exists(relation) %}\n {% if relation is not none %}\n {{ adapter.drop_relation(relation) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.699935, "supported_languages": null}, "macro.dbt.collect_freshness": {"name": "collect_freshness", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "unique_id": "macro.dbt.collect_freshness", "macro_sql": "{% macro collect_freshness(source, loaded_at_field, filter) %}\n {{ return(adapter.dispatch('collect_freshness', 'dbt')(source, loaded_at_field, filter))}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__collect_freshness"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.700392, "supported_languages": null}, "macro.dbt.default__collect_freshness": {"name": "default__collect_freshness", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "unique_id": "macro.dbt.default__collect_freshness", "macro_sql": "{% macro default__collect_freshness(source, loaded_at_field, filter) %}\n {% call statement('collect_freshness', fetch_result=True, auto_begin=False) -%}\n select\n max({{ loaded_at_field }}) as max_loaded_at,\n {{ current_timestamp() }} as snapshotted_at\n from {{ source }}\n {% if filter %}\n where {{ filter }}\n {% endif %}\n {% endcall %}\n {{ return(load_result('collect_freshness')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.7008178, "supported_languages": null}, "macro.dbt.copy_grants": {"name": "copy_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.copy_grants", "macro_sql": "{% macro copy_grants() %}\n {{ return(adapter.dispatch('copy_grants', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__copy_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.7027688, "supported_languages": null}, "macro.dbt.default__copy_grants": {"name": "default__copy_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__copy_grants", "macro_sql": "{% macro default__copy_grants() %}\n {{ return(True) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.702912, "supported_languages": null}, "macro.dbt.support_multiple_grantees_per_dcl_statement": {"name": "support_multiple_grantees_per_dcl_statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.support_multiple_grantees_per_dcl_statement", "macro_sql": "{% macro support_multiple_grantees_per_dcl_statement() %}\n {{ return(adapter.dispatch('support_multiple_grantees_per_dcl_statement', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__support_multiple_grantees_per_dcl_statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.7031, "supported_languages": null}, "macro.dbt.default__support_multiple_grantees_per_dcl_statement": {"name": "default__support_multiple_grantees_per_dcl_statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__support_multiple_grantees_per_dcl_statement", "macro_sql": "\n\n{%- macro default__support_multiple_grantees_per_dcl_statement() -%}\n {{ return(True) }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.703357, "supported_languages": null}, "macro.dbt.should_revoke": {"name": "should_revoke", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.should_revoke", "macro_sql": "{% macro should_revoke(existing_relation, full_refresh_mode=True) %}\n\n {% if not existing_relation %}\n {#-- The table doesn't already exist, so no grants to copy over --#}\n {{ return(False) }}\n {% elif full_refresh_mode %}\n {#-- The object is being REPLACED -- whether grants are copied over depends on the value of user config --#}\n {{ return(copy_grants()) }}\n {% else %}\n {#-- The table is being merged/upserted/inserted -- grants will be carried over --#}\n {{ return(True) }}\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.copy_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.7037492, "supported_languages": null}, "macro.dbt.get_show_grant_sql": {"name": "get_show_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_show_grant_sql", "macro_sql": "{% macro get_show_grant_sql(relation) %}\n {{ return(adapter.dispatch(\"get_show_grant_sql\", \"dbt\")(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_show_grant_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.7039511, "supported_languages": null}, "macro.dbt.default__get_show_grant_sql": {"name": "default__get_show_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_show_grant_sql", "macro_sql": "{% macro default__get_show_grant_sql(relation) %}\n show grants on {{ relation }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.7040598, "supported_languages": null}, "macro.dbt.get_grant_sql": {"name": "get_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_grant_sql", "macro_sql": "{% macro get_grant_sql(relation, privilege, grantees) %}\n {{ return(adapter.dispatch('get_grant_sql', 'dbt')(relation, privilege, grantees)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_grant_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.704402, "supported_languages": null}, "macro.dbt.default__get_grant_sql": {"name": "default__get_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_grant_sql", "macro_sql": "\n\n{%- macro default__get_grant_sql(relation, privilege, grantees) -%}\n grant {{ privilege }} on {{ relation }} to {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.704654, "supported_languages": null}, "macro.dbt.get_revoke_sql": {"name": "get_revoke_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_revoke_sql", "macro_sql": "{% macro get_revoke_sql(relation, privilege, grantees) %}\n {{ return(adapter.dispatch('get_revoke_sql', 'dbt')(relation, privilege, grantees)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_revoke_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.704913, "supported_languages": null}, "macro.dbt.default__get_revoke_sql": {"name": "default__get_revoke_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_revoke_sql", "macro_sql": "\n\n{%- macro default__get_revoke_sql(relation, privilege, grantees) -%}\n revoke {{ privilege }} on {{ relation }} from {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.705131, "supported_languages": null}, "macro.dbt.get_dcl_statement_list": {"name": "get_dcl_statement_list", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_dcl_statement_list", "macro_sql": "{% macro get_dcl_statement_list(relation, grant_config, get_dcl_macro) %}\n {{ return(adapter.dispatch('get_dcl_statement_list', 'dbt')(relation, grant_config, get_dcl_macro)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_dcl_statement_list"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.705385, "supported_languages": null}, "macro.dbt.default__get_dcl_statement_list": {"name": "default__get_dcl_statement_list", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_dcl_statement_list", "macro_sql": "\n\n{%- macro default__get_dcl_statement_list(relation, grant_config, get_dcl_macro) -%}\n {#\n -- Unpack grant_config into specific privileges and the set of users who need them granted/revoked.\n -- Depending on whether this database supports multiple grantees per statement, pass in the list of\n -- all grantees per privilege, or (if not) template one statement per privilege-grantee pair.\n -- `get_dcl_macro` will be either `get_grant_sql` or `get_revoke_sql`\n #}\n {%- set dcl_statements = [] -%}\n {%- for privilege, grantees in grant_config.items() %}\n {%- if support_multiple_grantees_per_dcl_statement() and grantees -%}\n {%- set dcl = get_dcl_macro(relation, privilege, grantees) -%}\n {%- do dcl_statements.append(dcl) -%}\n {%- else -%}\n {%- for grantee in grantees -%}\n {% set dcl = get_dcl_macro(relation, privilege, [grantee]) %}\n {%- do dcl_statements.append(dcl) -%}\n {% endfor -%}\n {%- endif -%}\n {%- endfor -%}\n {{ return(dcl_statements) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.support_multiple_grantees_per_dcl_statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.7062578, "supported_languages": null}, "macro.dbt.call_dcl_statements": {"name": "call_dcl_statements", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.call_dcl_statements", "macro_sql": "{% macro call_dcl_statements(dcl_statement_list) %}\n {{ return(adapter.dispatch(\"call_dcl_statements\", \"dbt\")(dcl_statement_list)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__call_dcl_statements"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.706471, "supported_languages": null}, "macro.dbt.default__call_dcl_statements": {"name": "default__call_dcl_statements", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__call_dcl_statements", "macro_sql": "{% macro default__call_dcl_statements(dcl_statement_list) %}\n {#\n -- By default, supply all grant + revoke statements in a single semicolon-separated block,\n -- so that they're all processed together.\n\n -- Some databases do not support this. Those adapters will need to override this macro\n -- to run each statement individually.\n #}\n {% call statement('grants') %}\n {% for dcl_statement in dcl_statement_list %}\n {{ dcl_statement }};\n {% endfor %}\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.706742, "supported_languages": null}, "macro.dbt.apply_grants": {"name": "apply_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.apply_grants", "macro_sql": "{% macro apply_grants(relation, grant_config, should_revoke) %}\n {{ return(adapter.dispatch(\"apply_grants\", \"dbt\")(relation, grant_config, should_revoke)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__apply_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.707036, "supported_languages": null}, "macro.dbt.default__apply_grants": {"name": "default__apply_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__apply_grants", "macro_sql": "{% macro default__apply_grants(relation, grant_config, should_revoke=True) %}\n {#-- If grant_config is {} or None, this is a no-op --#}\n {% if grant_config %}\n {% if should_revoke %}\n {#-- We think previous grants may have carried over --#}\n {#-- Show current grants and calculate diffs --#}\n {% set current_grants_table = run_query(get_show_grant_sql(relation)) %}\n {% set current_grants_dict = adapter.standardize_grants_dict(current_grants_table) %}\n {% set needs_granting = diff_of_two_dicts(grant_config, current_grants_dict) %}\n {% set needs_revoking = diff_of_two_dicts(current_grants_dict, grant_config) %}\n {% if not (needs_granting or needs_revoking) %}\n {{ log('On ' ~ relation ~': All grants are in place, no revocation or granting needed.')}}\n {% endif %}\n {% else %}\n {#-- We don't think there's any chance of previous grants having carried over. --#}\n {#-- Jump straight to granting what the user has configured. --#}\n {% set needs_revoking = {} %}\n {% set needs_granting = grant_config %}\n {% endif %}\n {% if needs_granting or needs_revoking %}\n {% set revoke_statement_list = get_dcl_statement_list(relation, needs_revoking, get_revoke_sql) %}\n {% set grant_statement_list = get_dcl_statement_list(relation, needs_granting, get_grant_sql) %}\n {% set dcl_statement_list = revoke_statement_list + grant_statement_list %}\n {% if dcl_statement_list %}\n {{ call_dcl_statements(dcl_statement_list) }}\n {% endif %}\n {% endif %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query", "macro.dbt.get_show_grant_sql", "macro.dbt.get_dcl_statement_list", "macro.dbt.call_dcl_statements"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.708344, "supported_languages": null}, "macro.dbt.alter_column_comment": {"name": "alter_column_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.alter_column_comment", "macro_sql": "{% macro alter_column_comment(relation, column_dict) -%}\n {{ return(adapter.dispatch('alter_column_comment', 'dbt')(relation, column_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.7090821, "supported_languages": null}, "macro.dbt.default__alter_column_comment": {"name": "default__alter_column_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.default__alter_column_comment", "macro_sql": "{% macro default__alter_column_comment(relation, column_dict) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_column_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.709265, "supported_languages": null}, "macro.dbt.alter_relation_comment": {"name": "alter_relation_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.alter_relation_comment", "macro_sql": "{% macro alter_relation_comment(relation, relation_comment) -%}\n {{ return(adapter.dispatch('alter_relation_comment', 'dbt')(relation, relation_comment)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__alter_relation_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.7094889, "supported_languages": null}, "macro.dbt.default__alter_relation_comment": {"name": "default__alter_relation_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.default__alter_relation_comment", "macro_sql": "{% macro default__alter_relation_comment(relation, relation_comment) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_relation_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.7096682, "supported_languages": null}, "macro.dbt.persist_docs": {"name": "persist_docs", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.persist_docs", "macro_sql": "{% macro persist_docs(relation, model, for_relation=true, for_columns=true) -%}\n {{ return(adapter.dispatch('persist_docs', 'dbt')(relation, model, for_relation, for_columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.709949, "supported_languages": null}, "macro.dbt.default__persist_docs": {"name": "default__persist_docs", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.default__persist_docs", "macro_sql": "{% macro default__persist_docs(relation, model, for_relation, for_columns) -%}\n {% if for_relation and config.persist_relation_docs() and model.description %}\n {% do run_query(alter_relation_comment(relation, model.description)) %}\n {% endif %}\n\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {% do run_query(alter_column_comment(relation, model.columns)) %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query", "macro.dbt.alter_relation_comment", "macro.dbt.alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.7104452, "supported_languages": null}, "macro.dbt.get_catalog": {"name": "get_catalog", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_catalog", "macro_sql": "{% macro get_catalog(information_schema, schemas) -%}\n {{ return(adapter.dispatch('get_catalog', 'dbt')(information_schema, schemas)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_catalog"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.711974, "supported_languages": null}, "macro.dbt.default__get_catalog": {"name": "default__get_catalog", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_catalog", "macro_sql": "{% macro default__get_catalog(information_schema, schemas) -%}\n\n {% set typename = adapter.type() %}\n {% set msg -%}\n get_catalog not implemented for {{ typename }}\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.712246, "supported_languages": null}, "macro.dbt.information_schema_name": {"name": "information_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.information_schema_name", "macro_sql": "{% macro information_schema_name(database) %}\n {{ return(adapter.dispatch('information_schema_name', 'dbt')(database)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__information_schema_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.712435, "supported_languages": null}, "macro.dbt.default__information_schema_name": {"name": "default__information_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__information_schema_name", "macro_sql": "{% macro default__information_schema_name(database) -%}\n {%- if database -%}\n {{ database }}.INFORMATION_SCHEMA\n {%- else -%}\n INFORMATION_SCHEMA\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.7125928, "supported_languages": null}, "macro.dbt.list_schemas": {"name": "list_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.list_schemas", "macro_sql": "{% macro list_schemas(database) -%}\n {{ return(adapter.dispatch('list_schemas', 'dbt')(database)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__list_schemas"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.712777, "supported_languages": null}, "macro.dbt.default__list_schemas": {"name": "default__list_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__list_schemas", "macro_sql": "{% macro default__list_schemas(database) -%}\n {% set sql %}\n select distinct schema_name\n from {{ information_schema_name(database) }}.SCHEMATA\n where catalog_name ilike '{{ database }}'\n {% endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.information_schema_name", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.7130291, "supported_languages": null}, "macro.dbt.check_schema_exists": {"name": "check_schema_exists", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.check_schema_exists", "macro_sql": "{% macro check_schema_exists(information_schema, schema) -%}\n {{ return(adapter.dispatch('check_schema_exists', 'dbt')(information_schema, schema)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__check_schema_exists"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.7132502, "supported_languages": null}, "macro.dbt.default__check_schema_exists": {"name": "default__check_schema_exists", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__check_schema_exists", "macro_sql": "{% macro default__check_schema_exists(information_schema, schema) -%}\n {% set sql -%}\n select count(*)\n from {{ information_schema.replace(information_schema_view='SCHEMATA') }}\n where catalog_name='{{ information_schema.database }}'\n and schema_name='{{ schema }}'\n {%- endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.replace", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.713568, "supported_languages": null}, "macro.dbt.list_relations_without_caching": {"name": "list_relations_without_caching", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.list_relations_without_caching", "macro_sql": "{% macro list_relations_without_caching(schema_relation) %}\n {{ return(adapter.dispatch('list_relations_without_caching', 'dbt')(schema_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__list_relations_without_caching"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.713761, "supported_languages": null}, "macro.dbt.default__list_relations_without_caching": {"name": "default__list_relations_without_caching", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__list_relations_without_caching", "macro_sql": "{% macro default__list_relations_without_caching(schema_relation) %}\n {{ exceptions.raise_not_implemented(\n 'list_relations_without_caching macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.713928, "supported_languages": null}, "macro.dbt.get_columns_in_relation": {"name": "get_columns_in_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_columns_in_relation", "macro_sql": "{% macro get_columns_in_relation(relation) -%}\n {{ return(adapter.dispatch('get_columns_in_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_postgres.postgres__get_columns_in_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.716088, "supported_languages": null}, "macro.dbt.default__get_columns_in_relation": {"name": "default__get_columns_in_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_columns_in_relation", "macro_sql": "{% macro default__get_columns_in_relation(relation) -%}\n {{ exceptions.raise_not_implemented(\n 'get_columns_in_relation macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.7163389, "supported_languages": null}, "macro.dbt.sql_convert_columns_in_relation": {"name": "sql_convert_columns_in_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.sql_convert_columns_in_relation", "macro_sql": "{% macro sql_convert_columns_in_relation(table) -%}\n {% set columns = [] %}\n {% for row in table %}\n {% do columns.append(api.Column(*row)) %}\n {% endfor %}\n {{ return(columns) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.716653, "supported_languages": null}, "macro.dbt.get_empty_subquery_sql": {"name": "get_empty_subquery_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_empty_subquery_sql", "macro_sql": "{% macro get_empty_subquery_sql(select_sql) -%}\n {{ return(adapter.dispatch('get_empty_subquery_sql', 'dbt')(select_sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_empty_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.71684, "supported_languages": null}, "macro.dbt.default__get_empty_subquery_sql": {"name": "default__get_empty_subquery_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_empty_subquery_sql", "macro_sql": "{% macro default__get_empty_subquery_sql(select_sql) %}\n select * from (\n {{ select_sql }}\n ) as __dbt_sbq\n where false\n limit 0\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.71695, "supported_languages": null}, "macro.dbt.get_empty_schema_sql": {"name": "get_empty_schema_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_empty_schema_sql", "macro_sql": "{% macro get_empty_schema_sql(columns) -%}\n {{ return(adapter.dispatch('get_empty_schema_sql', 'dbt')(columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_empty_schema_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.717133, "supported_languages": null}, "macro.dbt.default__get_empty_schema_sql": {"name": "default__get_empty_schema_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_empty_schema_sql", "macro_sql": "{% macro default__get_empty_schema_sql(columns) %}\n {%- set col_err = [] -%}\n select\n {% for i in columns %}\n {%- set col = columns[i] -%}\n {%- if col['data_type'] is not defined -%}\n {{ col_err.append(col['name']) }}\n {%- endif -%}\n cast(null as {{ col['data_type'] }}) as {{ col['name'] }}{{ \", \" if not loop.last }}\n {%- endfor -%}\n {%- if (col_err | length) > 0 -%}\n {{ exceptions.column_type_missing(column_names=col_err) }}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.717796, "supported_languages": null}, "macro.dbt.get_column_schema_from_query": {"name": "get_column_schema_from_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_column_schema_from_query", "macro_sql": "{% macro get_column_schema_from_query(select_sql) -%}\n {% set columns = [] %}\n {# -- Using an 'empty subquery' here to get the same schema as the given select_sql statement, without necessitating a data scan.#}\n {% set sql = get_empty_subquery_sql(select_sql) %}\n {% set column_schema = adapter.get_column_schema_from_query(sql) %}\n {{ return(column_schema) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_empty_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.718113, "supported_languages": null}, "macro.dbt.get_columns_in_query": {"name": "get_columns_in_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_columns_in_query", "macro_sql": "{% macro get_columns_in_query(select_sql) -%}\n {{ return(adapter.dispatch('get_columns_in_query', 'dbt')(select_sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_columns_in_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.7183008, "supported_languages": null}, "macro.dbt.default__get_columns_in_query": {"name": "default__get_columns_in_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_columns_in_query", "macro_sql": "{% macro default__get_columns_in_query(select_sql) %}\n {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%}\n {{ get_empty_subquery_sql(select_sql) }}\n {% endcall %}\n {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.get_empty_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.7186701, "supported_languages": null}, "macro.dbt.alter_column_type": {"name": "alter_column_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.alter_column_type", "macro_sql": "{% macro alter_column_type(relation, column_name, new_column_type) -%}\n {{ return(adapter.dispatch('alter_column_type', 'dbt')(relation, column_name, new_column_type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__alter_column_type"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.718902, "supported_languages": null}, "macro.dbt.default__alter_column_type": {"name": "default__alter_column_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__alter_column_type", "macro_sql": "{% macro default__alter_column_type(relation, column_name, new_column_type) -%}\n {#\n 1. Create a new column (w/ temp name and correct type)\n 2. Copy data over to it\n 3. Drop the existing column (cascade!)\n 4. Rename the new column to existing column\n #}\n {%- set tmp_column = column_name + \"__dbt_alter\" -%}\n\n {% call statement('alter_column_type') %}\n alter table {{ relation }} add column {{ adapter.quote(tmp_column) }} {{ new_column_type }};\n update {{ relation }} set {{ adapter.quote(tmp_column) }} = {{ adapter.quote(column_name) }};\n alter table {{ relation }} drop column {{ adapter.quote(column_name) }} cascade;\n alter table {{ relation }} rename column {{ adapter.quote(tmp_column) }} to {{ adapter.quote(column_name) }}\n {% endcall %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.719517, "supported_languages": null}, "macro.dbt.alter_relation_add_remove_columns": {"name": "alter_relation_add_remove_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.alter_relation_add_remove_columns", "macro_sql": "{% macro alter_relation_add_remove_columns(relation, add_columns = none, remove_columns = none) -%}\n {{ return(adapter.dispatch('alter_relation_add_remove_columns', 'dbt')(relation, add_columns, remove_columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__alter_relation_add_remove_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.7197812, "supported_languages": null}, "macro.dbt.default__alter_relation_add_remove_columns": {"name": "default__alter_relation_add_remove_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__alter_relation_add_remove_columns", "macro_sql": "{% macro default__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}\n\n {% if add_columns is none %}\n {% set add_columns = [] %}\n {% endif %}\n {% if remove_columns is none %}\n {% set remove_columns = [] %}\n {% endif %}\n\n {% set sql -%}\n\n alter {{ relation.type }} {{ relation }}\n\n {% for column in add_columns %}\n add column {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }}\n {% endfor %}{{ ',' if add_columns and remove_columns }}\n\n {% for column in remove_columns %}\n drop column {{ column.name }}{{ ',' if not loop.last }}\n {% endfor %}\n\n {%- endset -%}\n\n {% do run_query(sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.720577, "supported_languages": null}, "macro.dbt.resolve_model_name": {"name": "resolve_model_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.resolve_model_name", "macro_sql": "{% macro resolve_model_name(input_model_name) %}\n {{ return(adapter.dispatch('resolve_model_name', 'dbt')(input_model_name)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__resolve_model_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.7219672, "supported_languages": null}, "macro.dbt.default__resolve_model_name": {"name": "default__resolve_model_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.default__resolve_model_name", "macro_sql": "\n\n{%- macro default__resolve_model_name(input_model_name) -%}\n {{ input_model_name | string | replace('\"', '\\\"') }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.722124, "supported_languages": null}, "macro.dbt.build_ref_function": {"name": "build_ref_function", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.build_ref_function", "macro_sql": "{% macro build_ref_function(model) %}\n\n {%- set ref_dict = {} -%}\n {%- for _ref in model.refs -%}\n {%- set resolved = ref(*_ref) -%}\n {%- do ref_dict.update({_ref | join('.'): resolve_model_name(resolved)}) -%}\n {%- endfor -%}\n\ndef ref(*args,dbt_load_df_function):\n refs = {{ ref_dict | tojson }}\n key = '.'.join(args)\n return dbt_load_df_function(refs[key])\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.resolve_model_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.722541, "supported_languages": null}, "macro.dbt.build_source_function": {"name": "build_source_function", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.build_source_function", "macro_sql": "{% macro build_source_function(model) %}\n\n {%- set source_dict = {} -%}\n {%- for _source in model.sources -%}\n {%- set resolved = source(*_source) -%}\n {%- do source_dict.update({_source | join('.'): resolve_model_name(resolved)}) -%}\n {%- endfor -%}\n\ndef source(*args, dbt_load_df_function):\n sources = {{ source_dict | tojson }}\n key = '.'.join(args)\n return dbt_load_df_function(sources[key])\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.resolve_model_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.722951, "supported_languages": null}, "macro.dbt.build_config_dict": {"name": "build_config_dict", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.build_config_dict", "macro_sql": "{% macro build_config_dict(model) %}\n {%- set config_dict = {} -%}\n {% set config_dbt_used = zip(model.config.config_keys_used, model.config.config_keys_defaults) | list %}\n {%- for key, default in config_dbt_used -%}\n {# weird type testing with enum, would be much easier to write this logic in Python! #}\n {%- if key == \"language\" -%}\n {%- set value = \"python\" -%}\n {%- endif -%}\n {%- set value = model.config.get(key, default) -%}\n {%- do config_dict.update({key: value}) -%}\n {%- endfor -%}\nconfig_dict = {{ config_dict }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.723784, "supported_languages": null}, "macro.dbt.py_script_postfix": {"name": "py_script_postfix", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.py_script_postfix", "macro_sql": "{% macro py_script_postfix(model) %}\n# This part is user provided model code\n# you will need to copy the next section to run the code\n# COMMAND ----------\n# this part is dbt logic for get ref work, do not modify\n\n{{ build_ref_function(model ) }}\n{{ build_source_function(model ) }}\n{{ build_config_dict(model) }}\n\nclass config:\n def __init__(self, *args, **kwargs):\n pass\n\n @staticmethod\n def get(key, default=None):\n return config_dict.get(key, default)\n\nclass this:\n \"\"\"dbt.this() or dbt.this.identifier\"\"\"\n database = \"{{ this.database }}\"\n schema = \"{{ this.schema }}\"\n identifier = \"{{ this.identifier }}\"\n {% set this_relation_name = resolve_model_name(this) %}\n def __repr__(self):\n return '{{ this_relation_name }}'\n\n\nclass dbtObj:\n def __init__(self, load_df_function) -> None:\n self.source = lambda *args: source(*args, dbt_load_df_function=load_df_function)\n self.ref = lambda *args: ref(*args, dbt_load_df_function=load_df_function)\n self.config = config\n self.this = this()\n self.is_incremental = {{ is_incremental() }}\n\n# COMMAND ----------\n{{py_script_comment()}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.build_ref_function", "macro.dbt.build_source_function", "macro.dbt.build_config_dict", "macro.dbt.resolve_model_name", "macro.dbt.is_incremental", "macro.dbt.py_script_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.724266, "supported_languages": null}, "macro.dbt.py_script_comment": {"name": "py_script_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.py_script_comment", "macro_sql": "{%macro py_script_comment()%}\n{%endmacro%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.724343, "supported_languages": null}, "macro.dbt.test_unique": {"name": "test_unique", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_unique", "macro_sql": "{% test unique(model, column_name) %}\n {% set macro = adapter.dispatch('test_unique', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_unique"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.724848, "supported_languages": null}, "macro.dbt.test_not_null": {"name": "test_not_null", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_not_null", "macro_sql": "{% test not_null(model, column_name) %}\n {% set macro = adapter.dispatch('test_not_null', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_not_null"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.7250812, "supported_languages": null}, "macro.dbt.test_accepted_values": {"name": "test_accepted_values", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_accepted_values", "macro_sql": "{% test accepted_values(model, column_name, values, quote=True) %}\n {% set macro = adapter.dispatch('test_accepted_values', 'dbt') %}\n {{ macro(model, column_name, values, quote) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_accepted_values"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.725373, "supported_languages": null}, "macro.dbt.test_relationships": {"name": "test_relationships", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_relationships", "macro_sql": "{% test relationships(model, column_name, to, field) %}\n {% set macro = adapter.dispatch('test_relationships', 'dbt') %}\n {{ macro(model, column_name, to, field) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_relationships"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1681095229.725664, "supported_languages": null}}, "docs": {"doc.test.somedoc": {"name": "somedoc", "resource_type": "doc", "package_name": "test", "path": "somedoc.md", "original_file_path": "models/somedoc.md", "unique_id": "doc.test.somedoc", "block_contents": "Testing, testing"}, "doc.dbt.__overview__": {"name": "__overview__", "resource_type": "doc", "package_name": "dbt", "path": "overview.md", "original_file_path": "docs/overview.md", "unique_id": "doc.dbt.__overview__", "block_contents": "### Welcome!\n\nWelcome to the auto-generated documentation for your dbt project!\n\n### Navigation\n\nYou can use the `Project` and `Database` navigation tabs on the left side of the window to explore the models\nin your project.\n\n#### Project Tab\nThe `Project` tab mirrors the directory structure of your dbt project. In this tab, you can see all of the\nmodels defined in your dbt project, as well as models imported from dbt packages.\n\n#### Database Tab\nThe `Database` tab also exposes your models, but in a format that looks more like a database explorer. This view\nshows relations (tables and views) grouped into database schemas. Note that ephemeral models are _not_ shown\nin this interface, as they do not exist in the database.\n\n### Graph Exploration\nYou can click the blue icon on the bottom-right corner of the page to view the lineage graph of your models.\n\nOn model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the `Expand`\nbutton at the top-right of this lineage pane, you'll be able to see all of the models that are used to build,\nor are built from, the model you're exploring.\n\nOnce expanded, you'll be able to use the `--select` and `--exclude` model selection syntax to filter the\nmodels in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax).\n\nNote that you can also right-click on models to interactively filter and explore the graph.\n\n---\n\n### More information\n\n- [What is dbt](https://docs.getdbt.com/docs/introduction)?\n- Read the [dbt viewpoint](https://docs.getdbt.com/docs/viewpoint)\n- [Installation](https://docs.getdbt.com/docs/installation)\n- Join the [dbt Community](https://www.getdbt.com/community/) for questions and discussion"}}, "exposures": {"exposure.test.simple_exposure": {"name": "simple_exposure", "resource_type": "exposure", "package_name": "test", "path": "schema.yml", "original_file_path": "models/schema.yml", "unique_id": "exposure.test.simple_exposure", "fqn": ["test", "simple_exposure"], "type": "dashboard", "owner": {"email": "something@example.com", "name": null}, "description": "", "label": null, "maturity": null, "meta": {}, "tags": [], "config": {"enabled": true}, "unrendered_config": {}, "url": null, "depends_on": {"macros": [], "nodes": ["source.test.my_source.my_table", "model.test.my_model"]}, "refs": [{"name": "my_model", "package": null, "version": null}], "sources": [["my_source", "my_table"]], "metrics": [], "created_at": 1681095229.907179}}, "metrics": {"metric.test.my_metric": {"name": "my_metric", "resource_type": "metric", "package_name": "test", "path": "schema.yml", "original_file_path": "models/schema.yml", "unique_id": "metric.test.my_metric", "fqn": ["test", "my_metric"], "description": "", "label": "Count records", "calculation_method": "count", "expression": "*", "filters": [], "time_grains": ["day"], "dimensions": [], "timestamp": "updated_at", "window": null, "model": "ref('my_model')", "model_unique_id": null, "meta": {}, "tags": [], "config": {"enabled": true, "group": null}, "unrendered_config": {}, "sources": [], "depends_on": {"macros": [], "nodes": ["model.test.my_model"]}, "refs": [{"name": "my_model", "package": null, "version": null}], "metrics": [], "created_at": 1681095229.936167, "group": null}}, "groups": {}, "selectors": {}, "disabled": {"model.test.disabled_model": [{"database": "dbt", "schema": "test16810952296205305560_test_previous_version_state", "name": "disabled_model", "resource_type": "model", "package_name": "test", "path": "disabled_model.sql", "original_file_path": "models/disabled_model.sql", "unique_id": "model.test.disabled_model", "fqn": ["test", "disabled_model"], "alias": "disabled_model", "checksum": {"name": "sha256", "checksum": "597106d23ce34e3cd2430588e5c1cf474ebdd138fc47e09b925a4ab258a27acc"}, "config": {"enabled": false, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false}, "post-hook": [], "pre-hook": []}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {"enabled": false}, "created_at": 1681095229.840684, "config_call_dict": {"enabled": false}, "relation_name": "\"dbt\".\"test16810952296205305560_test_previous_version_state\".\"disabled_model\"", "raw_code": "{{ config(enabled=False) }}\nselect 2 as id", "language": "sql", "refs": [], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": null, "contract": {"enforced": false, "checksum": null}, "access": "protected", "version": null, "is_latest_version": null}], "snapshot.test.disabled_snapshot_seed": [{"database": "dbt", "schema": "test16810952296205305560_test_previous_version_state", "name": "disabled_snapshot_seed", "resource_type": "snapshot", "package_name": "test", "path": "disabled_snapshot_seed.sql", "original_file_path": "snapshots/disabled_snapshot_seed.sql", "unique_id": "snapshot.test.disabled_snapshot_seed", "fqn": ["test", "disabled_snapshot_seed", "disabled_snapshot_seed"], "alias": "disabled_snapshot_seed", "checksum": {"name": "sha256", "checksum": "fe76c9dd437341c9e82a0f2a8baf3148f961b768eaa0a4410cd27d3c071bd617"}, "config": {"enabled": false, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "snapshot", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": "id", "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false}, "strategy": "check", "target_schema": "test16810952296205305560_test_previous_version_state", "target_database": null, "updated_at": null, "check_cols": "all", "post-hook": [], "pre-hook": []}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {"unique_key": "id", "strategy": "check", "check_cols": "all", "target_schema": "test16810952296205305560_test_previous_version_state", "enabled": false}, "created_at": 1681095229.846447, "config_call_dict": {"unique_key": "id", "strategy": "check", "check_cols": "all", "target_schema": "test16810952296205305560_test_previous_version_state", "enabled": false}, "relation_name": "\"dbt\".\"test16810952296205305560_test_previous_version_state\".\"disabled_snapshot_seed\"", "raw_code": "\n{{\n config(\n unique_key='id',\n strategy='check',\n check_cols='all',\n target_schema=schema,\n enabled=False,\n )\n}}\nselect * from {{ ref('my_seed') }}\n", "language": "sql", "refs": [{"name": "my_seed", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": null, "contract": {"enforced": false, "checksum": null}}], "analysis.test.disabled_al": [{"database": "dbt", "schema": "test16810952296205305560_test_previous_version_state", "name": "disabled_al", "resource_type": "analysis", "package_name": "test", "path": "analysis/disabled_al.sql", "original_file_path": "analyses/disabled_al.sql", "unique_id": "analysis.test.disabled_al", "fqn": ["test", "analysis", "disabled_al"], "alias": "disabled_al", "checksum": {"name": "sha256", "checksum": "32d36ad6cff0786eb562440ba60ef6c9b9a7f4c282dfb7a52eaf19d36370f0e1"}, "config": {"enabled": false, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false}, "post-hook": [], "pre-hook": []}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {"enabled": false}, "created_at": 1681095229.863457, "config_call_dict": {"enabled": false}, "relation_name": null, "raw_code": "{{ config(enabled=False) }}\nselect 9 as id", "language": "sql", "refs": [], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": null, "contract": {"enforced": false, "checksum": null}}], "test.test.disabled_just_my": [{"database": "dbt", "schema": "test16810952296205305560_test_previous_version_state_dbt_test__audit", "name": "disabled_just_my", "resource_type": "test", "package_name": "test", "path": "disabled_just_my.sql", "original_file_path": "tests/disabled_just_my.sql", "unique_id": "test.test.disabled_just_my", "fqn": ["test", "disabled_just_my"], "alias": "disabled_just_my", "checksum": {"name": "sha256", "checksum": "4f2268fd89a3b4ef899264ada6d7aa33603671cbc5d5acead7dc2eadf1add985"}, "config": {"enabled": false, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {"enabled": false}, "created_at": 1681095229.882241, "config_call_dict": {"enabled": false}, "relation_name": null, "raw_code": "{{ config(enabled=False) }}\n\nselect * from {{ ref('my_model') }}\nwhere false", "language": "sql", "refs": [{"name": "my_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": null, "contract": {"enforced": false, "checksum": null}}], "test.test.disabled_check_nothing_my_model_.f2c6a72d37": [{"test_metadata": {"name": "disabled_check_nothing", "kwargs": {"model": "{{ get_where_subquery(ref('my_model')) }}"}, "namespace": null}, "database": "dbt", "schema": "test16810952296205305560_test_previous_version_state_dbt_test__audit", "name": "disabled_check_nothing_my_model_", "resource_type": "test", "package_name": "test", "path": "disabled_check_nothing_my_model_.sql", "original_file_path": "models/schema.yml", "unique_id": "test.test.disabled_check_nothing_my_model_.f2c6a72d37", "fqn": ["test", "disabled_check_nothing_my_model_"], "alias": "disabled_check_nothing_my_model_", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": false, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "deferred": false, "unrendered_config": {"enabled": false}, "created_at": 1681095229.9022238, "config_call_dict": {"enabled": false}, "relation_name": null, "raw_code": "{{ test_disabled_check_nothing(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "my_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.test.test_disabled_check_nothing", "macro.dbt.get_where_subquery"], "nodes": []}, "compiled_path": null, "contract": {"enforced": false, "checksum": null}, "column_name": null, "file_key_name": "models.my_model", "attached_node": "model.test.my_model"}], "exposure.test.disabled_exposure": [{"name": "disabled_exposure", "resource_type": "exposure", "package_name": "test", "path": "schema.yml", "original_file_path": "models/schema.yml", "unique_id": "exposure.test.disabled_exposure", "fqn": ["test", "disabled_exposure"], "type": "dashboard", "owner": {"email": "something@example.com", "name": null}, "description": "", "label": null, "maturity": null, "meta": {}, "tags": [], "config": {"enabled": false}, "unrendered_config": {"enabled": false}, "url": null, "depends_on": {"macros": [], "nodes": []}, "refs": [{"name": "my_model", "package": null, "version": null}], "sources": [], "metrics": [], "created_at": 1681095229.9083421}], "metric.test.disabled_metric": [{"name": "disabled_metric", "resource_type": "metric", "package_name": "test", "path": "schema.yml", "original_file_path": "models/schema.yml", "unique_id": "metric.test.disabled_metric", "fqn": ["test", "disabled_metric"], "description": "", "label": "Count records", "calculation_method": "count", "expression": "*", "filters": [], "time_grains": ["day"], "dimensions": [], "timestamp": "updated_at", "window": null, "model": "ref('my_model')", "model_unique_id": null, "meta": {}, "tags": [], "config": {"enabled": false, "group": null}, "unrendered_config": {"enabled": false}, "sources": [], "depends_on": {"macros": [], "nodes": []}, "refs": [{"name": "my_model", "package": null, "version": null}], "metrics": [], "created_at": 1681095229.937823, "group": null}], "seed.test.disabled_seed": [{"database": "dbt", "schema": "test16810952296205305560_test_previous_version_state", "name": "disabled_seed", "resource_type": "seed", "package_name": "test", "path": "disabled_seed.csv", "original_file_path": "seeds/disabled_seed.csv", "unique_id": "seed.test.disabled_seed", "fqn": ["test", "disabled_seed"], "alias": "disabled_seed", "checksum": {"name": "sha256", "checksum": "31fddd8ec40c6aba6a3a8e7d83fedea2fd0a56c47b64ea3df1847ec1b018e2d1"}, "config": {"enabled": false, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "seed", "incremental_strategy": null, "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false}, "quote_columns": null, "post-hook": [], "pre-hook": []}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": "test://models/schema.yml", "build_path": null, "deferred": false, "unrendered_config": {"enabled": false}, "created_at": 1681095229.905121, "config_call_dict": {}, "relation_name": "\"dbt\".\"test16810952296205305560_test_previous_version_state\".\"disabled_seed\"", "raw_code": "", "root_path": "/private/var/folders/k6/gtt07v8j2vn51m_z05xk_fjc0000gp/T/pytest-of-michelleark/pytest-80/project5", "depends_on": {"macros": []}}], "source.test.my_source.disabled_table": [{"database": "dbt", "schema": "my_source", "name": "disabled_table", "resource_type": "source", "package_name": "test", "path": "models/schema.yml", "original_file_path": "models/schema.yml", "unique_id": "source.test.my_source.disabled_table", "fqn": ["test", "my_source", "disabled_table"], "source_name": "my_source", "source_description": "My source", "loader": "a_loader", "identifier": "disabled_table", "quoting": {"database": null, "schema": null, "identifier": null, "column": null}, "loaded_at_field": null, "freshness": {"warn_after": {"count": null, "period": null}, "error_after": {"count": null, "period": null}, "filter": null}, "external": null, "description": "Disabled table", "columns": {}, "meta": {}, "source_meta": {}, "tags": [], "config": {"enabled": false}, "patch_path": null, "unrendered_config": {"enabled": false}, "relation_name": "\"dbt\".\"my_source\".\"disabled_table\"", "created_at": 1681095229.939002}]}, "parent_map": {"model.test.my_model": [], "snapshot.test.snapshot_seed": ["seed.test.my_seed"], "analysis.test.a": [], "test.test.just_my": ["model.test.my_model"], "seed.test.my_seed": [], "test.test.not_null_my_model_id.43e0e9183a": ["model.test.my_model"], "test.test.check_nothing_my_model_.d5a5e66110": ["model.test.my_model"], "source.test.my_source.my_table": [], "exposure.test.simple_exposure": ["model.test.my_model", "source.test.my_source.my_table"], "metric.test.my_metric": ["model.test.my_model"]}, "child_map": {"model.test.my_model": ["exposure.test.simple_exposure", "metric.test.my_metric", "test.test.check_nothing_my_model_.d5a5e66110", "test.test.just_my", "test.test.not_null_my_model_id.43e0e9183a"], "snapshot.test.snapshot_seed": [], "analysis.test.a": [], "test.test.just_my": [], "seed.test.my_seed": ["snapshot.test.snapshot_seed"], "test.test.not_null_my_model_id.43e0e9183a": [], "test.test.check_nothing_my_model_.d5a5e66110": [], "source.test.my_source.my_table": ["exposure.test.simple_exposure"], "exposure.test.simple_exposure": [], "metric.test.my_metric": []}, "group_map": {}} ================================================ FILE: tests/functional/artifacts/expected_manifest.py ================================================ import hashlib import os from unittest.mock import ANY import dbt from dbt.tests.util import AnyStringWith # This produces an "expected manifest", with a number of the fields # modified to avoid ephemeral changes. # ANY # AnyStringWith # LineIndifferent # It also uses some convenience methods to generate the # various config dictionaries. def get_rendered_model_config(**updates): result = { "database": None, "schema": None, "alias": None, "enabled": True, "group": None, "materialized": "view", "pre-hook": [], "post-hook": [], "column_types": {}, "quoting": {}, "tags": [], "persist_docs": {}, "full_refresh": None, "on_schema_change": "ignore", "on_configuration_change": "apply", "meta": {}, "unique_key": None, "grants": {}, "packages": [], "incremental_strategy": None, "docs": {"node_color": None, "show": True}, "contract": {"enforced": False, "alias_types": True}, "access": "protected", "event_time": None, "lookback": 1, "batch_size": None, "begin": None, "concurrent_batches": None, "freshness": None, } result.update(updates) return result def get_unrendered_model_config(**updates): return updates def get_rendered_seed_config(**updates): result = { "enabled": True, "group": None, "materialized": "seed", "persist_docs": {}, "pre-hook": [], "post-hook": [], "column_types": {}, "delimiter": ",", "quoting": {}, "tags": [], "quote_columns": True, "full_refresh": None, "on_schema_change": "ignore", "on_configuration_change": "apply", "database": None, "schema": None, "alias": None, "meta": {}, "unique_key": None, "grants": {}, "packages": [], "incremental_strategy": None, "docs": {"node_color": None, "show": True}, "contract": {"enforced": False, "alias_types": True}, "event_time": None, "lookback": 1, "batch_size": None, "begin": None, "concurrent_batches": None, } result.update(updates) return result def get_unrendered_seed_config(**updates): result = {"quote_columns": True} result.update(updates) return result def get_rendered_snapshot_config(**updates): result = { "database": None, "schema": None, "alias": None, "enabled": True, "group": None, "materialized": "snapshot", "pre-hook": [], "post-hook": [], "column_types": {}, "quoting": {}, "snapshot_meta_column_names": { "dbt_valid_to": None, "dbt_valid_from": None, "dbt_updated_at": None, "dbt_scd_id": None, "dbt_is_deleted": None, }, "dbt_valid_to_current": None, "tags": [], "persist_docs": {}, "full_refresh": None, "on_schema_change": "ignore", "on_configuration_change": "apply", "strategy": "check", "check_cols": "all", "unique_key": "id", "target_database": None, "target_schema": None, "updated_at": None, "meta": {}, "grants": {}, "packages": [], "incremental_strategy": None, "docs": {"node_color": None, "show": True}, "contract": {"enforced": False, "alias_types": True}, "event_time": None, "lookback": 1, "batch_size": None, "begin": None, "concurrent_batches": None, } result.update(updates) return result def get_unrendered_snapshot_config(**updates): result = {"check_cols": "all", "strategy": "check", "target_schema": None, "unique_key": "id"} result.update(updates) return result def get_rendered_tst_config(**updates): result = { "enabled": True, "group": None, "materialized": "test", "tags": [], "severity": "ERROR", "store_failures": None, "store_failures_as": None, "sql_header": None, "warn_if": "!= 0", "error_if": "!= 0", "fail_calc": "count(*)", "where": None, "limit": None, "database": None, "schema": "dbt_test__audit", "alias": None, "meta": {}, } result.update(updates) return result def get_unrendered_tst_config(**updates): result = {} result.update(updates) return result def quote(value): quote_char = '"' return "{0}{1}{0}".format(quote_char, value) def relation_name_format(quote_database: bool, quote_schema: bool, quote_identifier: bool): return ".".join( ( quote("{0}") if quote_database else "{0}", quote("{1}") if quote_schema else "{1}", quote("{2}") if quote_identifier else "{2}", ) ) def checksum_file(path): """windows has silly git behavior that adds newlines, and python does silly things if we just open(..., 'r').encode('utf-8'). seed files should not have their contents normalized to mirror the normalize_file_contents usage during file loading """ with open(path, "rb") as fp: # We strip the file contents because we want the checksum to match the stored contents file_contents = fp.read().strip() # Normalize non-seed contents if not path.endswith(".csv"): file_contents = " ".join(file_contents.decode("utf-8").split()).encode("utf-8") hashed = hashlib.sha256(file_contents).hexdigest() return { "name": "sha256", "checksum": hashed, } def read_file_replace_returns(path): with open(path, "r") as fp: return fp.read().replace("\r", "").replace("\n", "") class LineIndifferent: def __init__(self, expected): self.expected = expected.replace("\r", "") def __eq__(self, other): got = other.replace("\r", "").replace("\n", "") return self.expected == got def __repr__(self): return "LineIndifferent({!r})".format(self.expected) def __str__(self): return self.__repr__() def expected_seeded_manifest(project, model_database=None, quote_model=False): model_sql_path = os.path.join("models", "model.sql") second_model_sql_path = os.path.join("models", "second_model.sql") model_schema_yml_path = os.path.join("models", "schema.yml") seed_schema_yml_path = os.path.join("seeds", "schema.yml") seed_path = os.path.join("seeds", "seed.csv") snapshot_path = os.path.join("snapshots", "snapshot_seed.sql") my_schema_name = project.test_schema alternate_schema = project.test_schema + "_test" test_audit_schema = my_schema_name + "_dbt_test__audit" model_database = project.database model_config = get_rendered_model_config(docs={"node_color": None, "show": False}) second_config = get_rendered_model_config( schema="test", docs={"node_color": None, "show": False} ) unrendered_model_config = get_unrendered_model_config( materialized="view", docs={"show": False} ) unrendered_second_config = get_unrendered_model_config( schema="test", materialized="view", docs={"show": False} ) seed_config = get_rendered_seed_config() unrendered_seed_config = get_unrendered_seed_config() test_config = get_rendered_tst_config() unrendered_test_config = get_unrendered_tst_config() snapshot_config = get_rendered_snapshot_config(target_schema=alternate_schema) unrendered_snapshot_config = get_unrendered_snapshot_config(target_schema=alternate_schema) quote_database = quote_schema = True relation_name_node_format = relation_name_format(quote_database, quote_schema, quote_model) relation_name_source_format = relation_name_format( quote_database, quote_schema, quote_identifier=True ) compiled_model_path = os.path.join("target", "compiled", "test", "models") model_raw_code = read_file_replace_returns(model_sql_path).rstrip("\r\n") return { "dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v7.json", "dbt_version": dbt.version.__version__, "nodes": { "model.test.model": { "compiled_path": os.path.join(compiled_model_path, "model.sql"), "build_path": None, "created_at": ANY, "name": "model", "relation_name": relation_name_node_format.format( model_database, my_schema_name, "model" ), "resource_type": "model", "path": "model.sql", "original_file_path": model_sql_path, "package_name": "test", "raw_code": LineIndifferent(model_raw_code), "language": "sql", "refs": [{"name": "seed", "package": None, "version": None}], "sources": [], "functions": [], "depends_on": {"nodes": ["seed.test.seed"], "macros": []}, "deprecation_date": None, "unique_id": "model.test.model", "fqn": ["test", "model"], "metrics": [], "tags": [], "meta": {}, "config": model_config, "group": None, "schema": my_schema_name, "database": model_database, "alias": "model", "description": "The test model", "primary_key": ["id"], "columns": { "id": { "name": "id", "description": "The user ID number", "dimension": None, "entity": None, "data_type": None, "meta": {}, "quote": None, "tags": [], "constraints": [], "granularity": None, "doc_blocks": [], "config": {"meta": {}, "tags": []}, }, "first_name": { "name": "first_name", "description": "The user's first name", "dimension": None, "entity": None, "data_type": None, "meta": {}, "quote": None, "tags": [], "constraints": [], "granularity": None, "doc_blocks": [], "config": {"meta": {}, "tags": []}, }, "email": { "name": "email", "description": "The user's email", "dimension": None, "entity": None, "data_type": None, "meta": {}, "quote": None, "tags": [], "constraints": [], "granularity": None, "doc_blocks": [], "config": {"meta": {}, "tags": []}, }, "ip_address": { "name": "ip_address", "description": "The user's IP address", "dimension": None, "entity": None, "data_type": None, "meta": {}, "quote": None, "tags": [], "constraints": [], "granularity": None, "doc_blocks": [], "config": {"meta": {}, "tags": []}, }, "updated_at": { "name": "updated_at", "description": "The last time this user's email was updated", "dimension": None, "entity": None, "data_type": None, "meta": {}, "quote": None, "tags": [], "constraints": [], "granularity": None, "doc_blocks": [], "config": {"meta": {}, "tags": []}, }, }, "contract": {"checksum": None, "enforced": False, "alias_types": True}, "constraints": [], "patch_path": "test://" + model_schema_yml_path, "docs": {"node_color": None, "show": False}, "compiled": True, "compiled_code": ANY, "extra_ctes_injected": True, "extra_ctes": [], "checksum": checksum_file(model_sql_path), "unrendered_config": unrendered_model_config, "access": "protected", "version": None, "latest_version": None, "time_spine": None, "doc_blocks": [], }, "model.test.second_model": { "compiled_path": os.path.join(compiled_model_path, "second_model.sql"), "build_path": None, "created_at": ANY, "name": "second_model", "relation_name": relation_name_node_format.format( project.database, alternate_schema, "second_model" ), "resource_type": "model", "path": "second_model.sql", "original_file_path": second_model_sql_path, "package_name": "test", "raw_code": LineIndifferent( read_file_replace_returns(second_model_sql_path).rstrip("\r\n") ), "language": "sql", "refs": [{"name": "seed", "package": None, "version": None}], "sources": [], "functions": [], "depends_on": {"nodes": ["seed.test.seed"], "macros": []}, "deprecation_date": None, "unique_id": "model.test.second_model", "fqn": ["test", "second_model"], "metrics": [], "tags": [], "meta": {}, "config": second_config, "group": None, "schema": alternate_schema, "database": project.database, "alias": "second_model", "description": "The second test model", "primary_key": [], "columns": { "id": { "name": "id", "description": "The user ID number", "dimension": None, "entity": None, "data_type": None, "meta": {}, "quote": None, "tags": [], "constraints": [], "granularity": None, "doc_blocks": [], "config": {"meta": {}, "tags": []}, }, "first_name": { "name": "first_name", "description": "The user's first name", "dimension": None, "entity": None, "data_type": None, "meta": {}, "quote": None, "tags": [], "constraints": [], "granularity": None, "doc_blocks": [], "config": {"meta": {}, "tags": []}, }, "email": { "name": "email", "description": "The user's email", "dimension": None, "entity": None, "data_type": None, "meta": {}, "quote": None, "tags": [], "constraints": [], "granularity": None, "doc_blocks": [], "config": {"meta": {}, "tags": []}, }, "ip_address": { "name": "ip_address", "description": "The user's IP address", "dimension": None, "entity": None, "data_type": None, "meta": {}, "quote": None, "tags": [], "constraints": [], "granularity": None, "doc_blocks": [], "config": {"meta": {}, "tags": []}, }, "updated_at": { "name": "updated_at", "description": "The last time this user's email was updated", "dimension": None, "entity": None, "data_type": None, "meta": {}, "quote": None, "tags": [], "constraints": [], "granularity": None, "doc_blocks": [], "config": {"meta": {}, "tags": []}, }, }, "contract": {"checksum": None, "enforced": False, "alias_types": True}, "constraints": [], "patch_path": "test://" + model_schema_yml_path, "docs": {"node_color": None, "show": False}, "compiled": True, "compiled_code": ANY, "extra_ctes_injected": True, "extra_ctes": [], "checksum": checksum_file(second_model_sql_path), "unrendered_config": unrendered_second_config, "access": "protected", "version": None, "latest_version": None, "time_spine": None, "doc_blocks": [], }, "seed.test.seed": { "build_path": None, "created_at": ANY, "config": seed_config, "group": None, "patch_path": "test://" + seed_schema_yml_path, "path": "seed.csv", "name": "seed", "root_path": project.project_root, "resource_type": "seed", "raw_code": "", "package_name": "test", "original_file_path": seed_path, "unique_id": "seed.test.seed", "fqn": ["test", "seed"], "tags": [], "meta": {}, "depends_on": {"macros": []}, "schema": my_schema_name, "database": project.database, "alias": "seed", "description": "The test seed", "columns": { "id": { "name": "id", "description": "The user ID number", "dimension": None, "entity": None, "data_type": None, "meta": {}, "quote": None, "tags": [], "constraints": [], "granularity": None, "doc_blocks": [], "config": {"meta": {}, "tags": []}, }, "first_name": { "name": "first_name", "description": "The user's first name", "dimension": None, "entity": None, "data_type": None, "meta": {}, "quote": None, "tags": [], "constraints": [], "granularity": None, "doc_blocks": [], "config": {"meta": {}, "tags": []}, }, "email": { "name": "email", "description": "The user's email", "dimension": None, "entity": None, "data_type": None, "meta": {}, "quote": None, "tags": [], "constraints": [], "granularity": None, "doc_blocks": [], "config": {"meta": {}, "tags": []}, }, "ip_address": { "name": "ip_address", "description": "The user's IP address", "dimension": None, "entity": None, "data_type": None, "meta": {}, "quote": None, "tags": [], "constraints": [], "granularity": None, "doc_blocks": [], "config": {"meta": {}, "tags": []}, }, "updated_at": { "name": "updated_at", "description": "The last time this user's email was updated", "dimension": None, "entity": None, "data_type": None, "meta": {}, "quote": None, "tags": [], "constraints": [], "granularity": None, "doc_blocks": [], "config": {"meta": {}, "tags": []}, }, }, "docs": {"node_color": None, "show": True}, "checksum": checksum_file(seed_path), "unrendered_config": unrendered_seed_config, "relation_name": relation_name_node_format.format( project.database, my_schema_name, "seed" ), "doc_blocks": [], }, "test.test.not_null_model_id.d01cc630e6": { "alias": "not_null_model_id", "attached_node": "model.test.model", "compiled_path": os.path.join( compiled_model_path, "schema.yml", "not_null_model_id.sql" ), "build_path": None, "created_at": ANY, "column_name": "id", "columns": {}, "config": test_config, "sources": [], "functions": [], "group": None, "depends_on": { "macros": ["macro.dbt.test_not_null", "macro.dbt.get_where_subquery"], "nodes": ["model.test.model"], }, "description": "", "file_key_name": "models.model", "fqn": ["test", "not_null_model_id"], "metrics": [], "name": "not_null_model_id", "original_file_path": model_schema_yml_path, "package_name": "test", "patch_path": None, "path": "not_null_model_id.sql", "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "model", "package": None, "version": None}], "relation_name": None, "resource_type": "test", "schema": test_audit_schema, "database": project.database, "tags": [], "meta": {}, "unique_id": "test.test.not_null_model_id.d01cc630e6", "docs": {"node_color": None, "show": True}, "compiled": True, "compiled_code": AnyStringWith("where id is null"), "extra_ctes_injected": True, "extra_ctes": [], "test_metadata": { "namespace": None, "name": "not_null", "kwargs": { "column_name": "id", "model": "{{ get_where_subquery(ref('model')) }}", }, }, "checksum": {"name": "none", "checksum": ""}, "unrendered_config": unrendered_test_config, "contract": {"checksum": None, "enforced": False, "alias_types": True}, "doc_blocks": [], }, "snapshot.test.snapshot_seed": { "alias": "snapshot_seed", "compiled_path": os.path.join( "target", "compiled", "test", "snapshots", "snapshot_seed.sql" ), "build_path": None, "created_at": ANY, "checksum": checksum_file(snapshot_path), "columns": {}, "compiled": True, "compiled_code": ANY, "config": snapshot_config, "contract": {"checksum": None, "enforced": False, "alias_types": True}, "database": project.database, "group": None, "depends_on": { "macros": [], "nodes": ["seed.test.seed"], }, "description": "", "docs": {"node_color": None, "show": True}, "extra_ctes": [], "extra_ctes_injected": True, "fqn": ["test", "snapshot_seed", "snapshot_seed"], "metrics": [], "meta": {}, "name": "snapshot_seed", "original_file_path": snapshot_path, "package_name": "test", "patch_path": None, "path": "snapshot_seed.sql", "raw_code": LineIndifferent( read_file_replace_returns(snapshot_path) .replace("{% snapshot snapshot_seed %}", "") .replace("{% endsnapshot %}", "") ), "language": "sql", "refs": [{"name": "seed", "package": None, "version": None}], "relation_name": relation_name_node_format.format( project.database, alternate_schema, "snapshot_seed" ), "resource_type": "snapshot", "schema": alternate_schema, "sources": [], "functions": [], "tags": [], "unique_id": "snapshot.test.snapshot_seed", "unrendered_config": unrendered_snapshot_config, "doc_blocks": [], }, "test.test.test_nothing_model_.5d38568946": { "alias": "test_nothing_model_", "attached_node": "model.test.model", "compiled_path": os.path.join( compiled_model_path, "schema.yml", "test_nothing_model_.sql" ), "build_path": None, "created_at": ANY, "column_name": None, "columns": {}, "config": test_config, "group": None, "contract": {"checksum": None, "enforced": False, "alias_types": True}, "sources": [], "functions": [], "depends_on": { "macros": ["macro.test.test_nothing", "macro.dbt.get_where_subquery"], "nodes": ["model.test.model"], }, "description": "", "file_key_name": "models.model", "fqn": ["test", "test_nothing_model_"], "metrics": [], "name": "test_nothing_model_", "original_file_path": model_schema_yml_path, "package_name": "test", "patch_path": None, "path": "test_nothing_model_.sql", "raw_code": "{{ test.test_nothing(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "model", "package": None, "version": None}], "relation_name": None, "resource_type": "test", "schema": test_audit_schema, "database": project.database, "tags": [], "meta": {}, "unique_id": "test.test.test_nothing_model_.5d38568946", "docs": {"node_color": None, "show": True}, "compiled": True, "compiled_code": AnyStringWith("select 0"), "extra_ctes_injected": True, "extra_ctes": [], "test_metadata": { "namespace": "test", "name": "nothing", "kwargs": { "model": "{{ get_where_subquery(ref('model')) }}", }, }, "checksum": {"name": "none", "checksum": ""}, "unrendered_config": unrendered_test_config, "doc_blocks": [], }, "test.test.unique_model_id.67b76558ff": { "alias": "unique_model_id", "attached_node": "model.test.model", "compiled_path": os.path.join( compiled_model_path, "schema.yml", "unique_model_id.sql" ), "build_path": None, "created_at": ANY, "column_name": "id", "columns": {}, "config": test_config, "group": None, "contract": {"checksum": None, "enforced": False, "alias_types": True}, "sources": [], "functions": [], "depends_on": { "macros": ["macro.dbt.test_unique", "macro.dbt.get_where_subquery"], "nodes": ["model.test.model"], }, "description": "", "file_key_name": "models.model", "fqn": ["test", "unique_model_id"], "metrics": [], "name": "unique_model_id", "original_file_path": model_schema_yml_path, "package_name": "test", "patch_path": None, "path": "unique_model_id.sql", "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "model", "package": None, "version": None}], "relation_name": None, "resource_type": "test", "schema": test_audit_schema, "database": project.database, "tags": [], "meta": {}, "unique_id": "test.test.unique_model_id.67b76558ff", "docs": {"node_color": None, "show": True}, "compiled": True, "compiled_code": AnyStringWith("count(*)"), "extra_ctes_injected": True, "extra_ctes": [], "test_metadata": { "namespace": None, "name": "unique", "kwargs": { "column_name": "id", "model": "{{ get_where_subquery(ref('model')) }}", }, }, "checksum": {"name": "none", "checksum": ""}, "unrendered_config": unrendered_test_config, "doc_blocks": [], }, }, "sources": { "source.test.my_source.my_table": { "created_at": ANY, "columns": { "id": { "description": "An ID field", "name": "id", "dimension": None, "entity": None, "data_type": None, "meta": {}, "quote": None, "tags": [], "constraints": [], "granularity": None, "doc_blocks": [], "config": {"meta": {}, "tags": []}, } }, "config": { "enabled": True, "event_time": None, "freshness": { "error_after": {"count": None, "period": None}, "warn_after": {"count": None, "period": None}, "filter": None, }, "loaded_at_query": None, "loaded_at_field": None, "meta": {}, "tags": [], }, "quoting": { "database": None, "schema": None, "identifier": True, "column": None, }, "database": project.database, "description": "My table", "external": None, "freshness": { "error_after": {"count": None, "period": None}, "warn_after": {"count": None, "period": None}, "filter": None, }, "identifier": "seed", "loaded_at_field": None, "loaded_at_query": None, "loader": "a_loader", "meta": {}, "name": "my_table", "original_file_path": os.path.join("models", "schema.yml"), "package_name": "test", "path": os.path.join("models", "schema.yml"), "patch_path": None, "relation_name": relation_name_source_format.format( project.database, my_schema_name, "seed" ), "resource_type": "source", "schema": my_schema_name, "source_description": "My source", "source_name": "my_source", "source_meta": {}, "tags": [], "unique_id": "source.test.my_source.my_table", "fqn": ["test", "my_source", "my_table"], "unrendered_config": { "loaded_at_field": None, "loaded_at_query": None, "meta": {}, "tags": [], }, "unrendered_database": None, "unrendered_schema": "{{ var('test_schema') }}", "doc_blocks": [], }, }, "exposures": { "exposure.test.notebook_exposure": { "created_at": ANY, "depends_on": { "macros": [], "nodes": ["model.test.model", "model.test.second_model"], }, "description": "A description of the complex exposure\n", "label": None, "config": { "enabled": True, "meta": {"tool": "my_tool", "languages": ["python"]}, "tags": ["my_department"], }, "fqn": ["test", "notebook_exposure"], "maturity": "medium", "meta": {"tool": "my_tool", "languages": ["python"]}, "metrics": [], "tags": ["my_department"], "name": "notebook_exposure", "original_file_path": os.path.join("models", "schema.yml"), "owner": {"email": "something@example.com", "name": "Some name"}, "package_name": "test", "path": "schema.yml", "refs": [ {"name": "model", "package": None, "version": None}, {"name": "second_model", "package": None, "version": None}, ], "resource_type": "exposure", "sources": [], "type": "notebook", "unique_id": "exposure.test.notebook_exposure", "url": "http://example.com/notebook/1", "unrendered_config": {}, }, "exposure.test.simple_exposure": { "created_at": ANY, "depends_on": { "macros": [], "nodes": ["source.test.my_source.my_table", "model.test.model"], }, "description": "", "label": None, "config": { "enabled": True, "meta": {}, "tags": [], }, "fqn": ["test", "simple_exposure"], "metrics": [], "name": "simple_exposure", "original_file_path": os.path.join("models", "schema.yml"), "owner": { "email": "something@example.com", "name": None, }, "package_name": "test", "path": "schema.yml", "refs": [{"name": "model", "package": None, "version": None}], "resource_type": "exposure", "sources": [["my_source", "my_table"]], "type": "dashboard", "unique_id": "exposure.test.simple_exposure", "url": None, "maturity": None, "meta": {}, "tags": [], "unrendered_config": {}, }, }, "metrics": {}, "groups": {}, "selectors": {}, "parent_map": { "model.test.model": ["seed.test.seed"], "model.test.second_model": ["seed.test.seed"], "exposure.test.notebook_exposure": ["model.test.model", "model.test.second_model"], "exposure.test.simple_exposure": [ "model.test.model", "source.test.my_source.my_table", ], "seed.test.seed": [], "snapshot.test.snapshot_seed": ["seed.test.seed"], "source.test.my_source.my_table": [], "test.test.not_null_model_id.d01cc630e6": ["model.test.model"], "test.test.test_nothing_model_.5d38568946": ["model.test.model"], "test.test.unique_model_id.67b76558ff": ["model.test.model"], }, "child_map": { "model.test.model": [ "exposure.test.notebook_exposure", "exposure.test.simple_exposure", "test.test.not_null_model_id.d01cc630e6", "test.test.test_nothing_model_.5d38568946", "test.test.unique_model_id.67b76558ff", ], "model.test.second_model": ["exposure.test.notebook_exposure"], "exposure.test.notebook_exposure": [], "exposure.test.simple_exposure": [], "seed.test.seed": [ "model.test.model", "model.test.second_model", "snapshot.test.snapshot_seed", ], "snapshot.test.snapshot_seed": [], "source.test.my_source.my_table": ["exposure.test.simple_exposure"], "test.test.not_null_model_id.d01cc630e6": [], "test.test.test_nothing_model_.5d38568946": [], "test.test.unique_model_id.67b76558ff": [], }, "group_map": {}, "docs": { "doc.dbt.__overview__": ANY, "doc.test.macro_info": ANY, "doc.test.macro_arg_info": ANY, }, "disabled": {}, "semantic_models": {}, "unit_tests": {}, "saved_queries": {}, "functions": {}, } def expected_references_manifest(project): model_database = project.database my_schema_name = project.test_schema docs_path = os.path.join("models", "docs.md") ephemeral_copy_path = os.path.join("models", "ephemeral_copy.sql") ephemeral_summary_path = os.path.join("models", "ephemeral_summary.sql") view_summary_path = os.path.join("models", "view_summary.sql") seed_path = os.path.join("seeds", "seed.csv") snapshot_path = os.path.join("snapshots", "snapshot_seed.sql") compiled_model_path = os.path.join("target", "compiled", "test", "models") schema_yml_path = os.path.join("models", "schema.yml") ephemeral_copy_sql = read_file_replace_returns(ephemeral_copy_path).rstrip("\r\n") ephemeral_summary_sql = read_file_replace_returns(ephemeral_summary_path).rstrip("\r\n") view_summary_sql = read_file_replace_returns(view_summary_path).rstrip("\r\n") alternate_schema = project.test_schema + "_test" return { "dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v7.json", "dbt_version": dbt.version.__version__, "nodes": { "model.test.ephemeral_copy": { "alias": "ephemeral_copy", "compiled_path": os.path.join(compiled_model_path, "ephemeral_copy.sql"), "build_path": None, "created_at": ANY, "columns": {}, "config": get_rendered_model_config(materialized="ephemeral"), "sources": [["my_source", "my_table"]], "depends_on": { "macros": [], "nodes": ["source.test.my_source.my_table"], }, "deprecation_date": None, "description": "", "primary_key": [], "docs": {"node_color": None, "show": True}, "fqn": ["test", "ephemeral_copy"], "group": None, "metrics": [], "functions": [], "name": "ephemeral_copy", "original_file_path": ephemeral_copy_path, "package_name": "test", "patch_path": None, "path": "ephemeral_copy.sql", "raw_code": LineIndifferent(ephemeral_copy_sql), "language": "sql", "refs": [], "relation_name": None, "resource_type": "model", "schema": my_schema_name, "database": project.database, "tags": [], "meta": {}, "unique_id": "model.test.ephemeral_copy", "compiled": True, "compiled_code": ANY, "contract": {"checksum": None, "enforced": False, "alias_types": True}, "extra_ctes_injected": True, "extra_ctes": [], "checksum": checksum_file(ephemeral_copy_path), "unrendered_config": get_unrendered_model_config(materialized="ephemeral"), "access": "protected", "version": None, "latest_version": None, "constraints": [], "time_spine": None, "doc_blocks": [], }, "model.test.ephemeral_summary": { "alias": "ephemeral_summary", "compiled_path": os.path.join(compiled_model_path, "ephemeral_summary.sql"), "build_path": None, "created_at": ANY, "columns": { "first_name": { "description": "The first name being summarized", "name": "first_name", "dimension": None, "entity": None, "data_type": None, "meta": {}, "quote": None, "tags": [], "constraints": [], "granularity": None, "doc_blocks": ["doc.test.summary_first_name"], "config": {"meta": {}, "tags": []}, }, "ct": { "description": "The number of instances of the first name", "name": "ct", "dimension": None, "entity": None, "data_type": None, "meta": {}, "quote": None, "tags": [], "constraints": [], "granularity": None, "doc_blocks": ["doc.test.summary_count"], "config": {"meta": {}, "tags": []}, }, }, "config": get_rendered_model_config(materialized="table", group="test_group"), "contract": {"checksum": None, "enforced": False, "alias_types": True}, "sources": [], "functions": [], "depends_on": { "macros": [], "nodes": ["model.test.ephemeral_copy"], }, "deprecation_date": None, "description": "A summmary table of the ephemeral copy of the seed data", "primary_key": [], "docs": {"node_color": None, "show": True}, "fqn": ["test", "ephemeral_summary"], "group": "test_group", "metrics": [], "name": "ephemeral_summary", "original_file_path": ephemeral_summary_path, "package_name": "test", "patch_path": "test://" + os.path.join("models", "schema.yml"), "path": "ephemeral_summary.sql", "raw_code": LineIndifferent(ephemeral_summary_sql), "language": "sql", "refs": [{"name": "ephemeral_copy", "package": None, "version": None}], "relation_name": '"{0}"."{1}".ephemeral_summary'.format( model_database, my_schema_name ), "resource_type": "model", "schema": my_schema_name, "database": project.database, "tags": [], "meta": {}, "unique_id": "model.test.ephemeral_summary", "compiled": True, "compiled_code": ANY, "extra_ctes_injected": True, "extra_ctes": [ANY], "checksum": checksum_file(ephemeral_summary_path), "unrendered_config": get_unrendered_model_config( materialized="table", group="test_group" ), "access": "protected", "version": None, "latest_version": None, "constraints": [], "time_spine": None, "doc_blocks": ["doc.test.ephemeral_summary"], }, "model.test.view_summary": { "alias": "view_summary", "compiled_path": os.path.join(compiled_model_path, "view_summary.sql"), "build_path": None, "created_at": ANY, "columns": { "first_name": { "description": "The first name being summarized", "name": "first_name", "dimension": None, "entity": None, "data_type": None, "meta": {}, "quote": None, "tags": [], "constraints": [], "granularity": None, "doc_blocks": ["doc.test.summary_first_name"], "config": {"meta": {}, "tags": []}, }, "ct": { "description": "The number of instances of the first name", "name": "ct", "dimension": None, "entity": None, "data_type": None, "meta": {}, "quote": None, "tags": [], "constraints": [], "granularity": None, "doc_blocks": ["doc.test.summary_count"], "config": {"meta": {}, "tags": []}, }, }, "config": get_rendered_model_config(), "contract": {"checksum": None, "enforced": False, "alias_types": True}, "database": project.database, "depends_on": { "macros": [], "nodes": ["model.test.ephemeral_summary"], }, "deprecation_date": None, "description": "A view of the summary of the ephemeral copy of the seed data", "primary_key": [], "docs": {"node_color": None, "show": True}, "fqn": ["test", "view_summary"], "group": None, "metrics": [], "name": "view_summary", "original_file_path": view_summary_path, "package_name": "test", "patch_path": "test://" + schema_yml_path, "path": "view_summary.sql", "raw_code": LineIndifferent(view_summary_sql), "language": "sql", "refs": [{"name": "ephemeral_summary", "package": None, "version": None}], "relation_name": '"{0}"."{1}".view_summary'.format(model_database, my_schema_name), "resource_type": "model", "schema": my_schema_name, "sources": [], "functions": [], "tags": [], "meta": {}, "unique_id": "model.test.view_summary", "compiled": True, "compiled_code": ANY, "extra_ctes_injected": True, "extra_ctes": [], "checksum": checksum_file(view_summary_path), "unrendered_config": get_unrendered_model_config(materialized="view"), "access": "protected", "version": None, "latest_version": None, "constraints": [], "time_spine": None, "doc_blocks": ["doc.test.view_summary"], }, "seed.test.seed": { "alias": "seed", "build_path": None, "created_at": ANY, "columns": { "id": { "name": "id", "description": "The user ID number", "dimension": None, "entity": None, "data_type": None, "meta": {}, "quote": None, "tags": [], "constraints": [], "granularity": None, "doc_blocks": [], "config": {"meta": {}, "tags": []}, }, "first_name": { "name": "first_name", "description": "The user's first name", "dimension": None, "entity": None, "data_type": None, "meta": {}, "quote": None, "tags": [], "constraints": [], "granularity": None, "doc_blocks": [], "config": {"meta": {}, "tags": []}, }, "email": { "name": "email", "description": "The user's email", "dimension": None, "entity": None, "data_type": None, "meta": {}, "quote": None, "tags": [], "constraints": [], "granularity": None, "doc_blocks": [], "config": {"meta": {}, "tags": []}, }, "ip_address": { "name": "ip_address", "description": "The user's IP address", "dimension": None, "entity": None, "data_type": None, "meta": {}, "quote": None, "tags": [], "constraints": [], "granularity": None, "doc_blocks": [], "config": {"meta": {}, "tags": []}, }, "updated_at": { "name": "updated_at", "description": "The last time this user's email was updated", "dimension": None, "entity": None, "data_type": None, "meta": {}, "quote": None, "tags": [], "constraints": [], "granularity": None, "doc_blocks": [], "config": {"meta": {}, "tags": []}, }, }, "config": get_rendered_seed_config(), "depends_on": {"macros": []}, "description": "The test seed", "docs": {"node_color": None, "show": True}, "fqn": ["test", "seed"], "group": None, "name": "seed", "original_file_path": seed_path, "package_name": "test", "patch_path": "test://" + os.path.join("seeds", "schema.yml"), "path": "seed.csv", "raw_code": "", "resource_type": "seed", "root_path": project.project_root, "schema": my_schema_name, "database": project.database, "tags": [], "meta": {}, "unique_id": "seed.test.seed", "checksum": checksum_file(seed_path), "unrendered_config": get_unrendered_seed_config(), "relation_name": '"{0}"."{1}".seed'.format(project.database, my_schema_name), "doc_blocks": [], }, "snapshot.test.snapshot_seed": { "alias": "snapshot_seed", "compiled_path": os.path.join( "target", "compiled", "test", "snapshots", "snapshot_seed.sql" ), "build_path": None, "created_at": ANY, "checksum": checksum_file(snapshot_path), "columns": {}, "compiled": True, "compiled_code": ANY, "config": get_rendered_snapshot_config(target_schema=alternate_schema), "contract": {"checksum": None, "enforced": False, "alias_types": True}, "database": model_database, "depends_on": {"macros": [], "nodes": ["seed.test.seed"]}, "description": "", "docs": {"node_color": None, "show": True}, "extra_ctes": [], "extra_ctes_injected": True, "fqn": ["test", "snapshot_seed", "snapshot_seed"], "group": None, "metrics": [], "functions": [], "meta": {}, "name": "snapshot_seed", "original_file_path": snapshot_path, "package_name": "test", "patch_path": None, "path": "snapshot_seed.sql", "raw_code": ANY, "language": "sql", "refs": [{"name": "seed", "package": None, "version": None}], "relation_name": '"{0}"."{1}".snapshot_seed'.format( model_database, alternate_schema ), "resource_type": "snapshot", "schema": alternate_schema, "sources": [], "tags": [], "unique_id": "snapshot.test.snapshot_seed", "unrendered_config": get_unrendered_snapshot_config( target_schema=alternate_schema ), "doc_blocks": [], }, }, "sources": { "source.test.my_source.my_table": { "columns": { "id": { "description": "An ID field", "dimension": None, "entity": None, "name": "id", "data_type": None, "meta": {}, "quote": None, "tags": [], "constraints": [], "granularity": None, "doc_blocks": ["doc.test.column_info"], "config": {"meta": {}, "tags": []}, } }, "config": { "enabled": True, "event_time": None, "freshness": { "error_after": {"count": None, "period": None}, "warn_after": {"count": None, "period": None}, "filter": None, }, "loaded_at_field": None, "loaded_at_query": None, "meta": {}, "tags": [], }, "quoting": { "database": False, "schema": None, "identifier": True, "column": None, }, "created_at": ANY, "database": project.database, "description": "My table", "external": None, "freshness": { "error_after": {"count": None, "period": None}, "warn_after": {"count": None, "period": None}, "filter": None, }, "identifier": "seed", "loaded_at_field": None, "loaded_at_query": None, "loader": "a_loader", "meta": {}, "name": "my_table", "original_file_path": os.path.join("models", "schema.yml"), "package_name": "test", "path": os.path.join("models", "schema.yml"), "patch_path": None, "relation_name": '{0}."{1}"."seed"'.format(project.database, my_schema_name), "resource_type": "source", "schema": my_schema_name, "source_description": "My source", "source_name": "my_source", "source_meta": {}, "tags": [], "unique_id": "source.test.my_source.my_table", "fqn": ["test", "my_source", "my_table"], "unrendered_config": { "loaded_at_field": None, "loaded_at_query": None, "meta": {}, "tags": [], }, "unrendered_database": None, "unrendered_schema": "{{ var('test_schema') }}", "doc_blocks": ["doc.test.table_info"], }, }, "exposures": { "exposure.test.notebook_exposure": { "created_at": ANY, "depends_on": { "macros": [], "nodes": ["model.test.view_summary"], }, "description": "A description of the complex exposure", "label": None, "config": { "enabled": True, "meta": {"tool": "my_tool", "languages": ["python"]}, "tags": ["my_department"], }, "fqn": ["test", "notebook_exposure"], "maturity": "medium", "meta": {"tool": "my_tool", "languages": ["python"]}, "metrics": [], "tags": ["my_department"], "name": "notebook_exposure", "original_file_path": os.path.join("models", "schema.yml"), "owner": {"email": "something@example.com", "name": "Some name"}, "package_name": "test", "path": "schema.yml", "refs": [{"name": "view_summary", "package": None, "version": None}], "resource_type": "exposure", "sources": [], "type": "notebook", "unique_id": "exposure.test.notebook_exposure", "url": "http://example.com/notebook/1", "unrendered_config": {}, }, }, "metrics": {}, "groups": { "group.test.test_group": { "name": "test_group", "resource_type": "group", "original_file_path": os.path.join("models", "schema.yml"), "owner": {"email": "test_group@test.com", "name": None}, "package_name": "test", "path": "schema.yml", "unique_id": "group.test.test_group", "description": None, "config": {"meta": {}}, } }, "selectors": {}, "docs": { "doc.dbt.__overview__": ANY, "doc.test.column_info": { "block_contents": "An ID field", "resource_type": "doc", "name": "column_info", "original_file_path": docs_path, "package_name": "test", "path": "docs.md", "unique_id": "doc.test.column_info", }, "doc.test.ephemeral_summary": { "block_contents": ("A summmary table of the ephemeral copy of the seed data"), "resource_type": "doc", "name": "ephemeral_summary", "original_file_path": docs_path, "package_name": "test", "path": "docs.md", "unique_id": "doc.test.ephemeral_summary", }, "doc.test.source_info": { "block_contents": "My source", "resource_type": "doc", "name": "source_info", "original_file_path": docs_path, "package_name": "test", "path": "docs.md", "unique_id": "doc.test.source_info", }, "doc.test.summary_count": { "block_contents": "The number of instances of the first name", "resource_type": "doc", "name": "summary_count", "original_file_path": docs_path, "package_name": "test", "path": "docs.md", "unique_id": "doc.test.summary_count", }, "doc.test.summary_first_name": { "block_contents": "The first name being summarized", "resource_type": "doc", "name": "summary_first_name", "original_file_path": docs_path, "package_name": "test", "path": "docs.md", "unique_id": "doc.test.summary_first_name", }, "doc.test.table_info": { "block_contents": "My table", "resource_type": "doc", "name": "table_info", "original_file_path": docs_path, "package_name": "test", "path": "docs.md", "unique_id": "doc.test.table_info", }, "doc.test.view_summary": { "block_contents": ("A view of the summary of the ephemeral copy of the seed data"), "resource_type": "doc", "name": "view_summary", "original_file_path": docs_path, "package_name": "test", "path": "docs.md", "unique_id": "doc.test.view_summary", }, "doc.test.macro_info": { "block_contents": "My custom test that I wrote that does nothing", "resource_type": "doc", "name": "macro_info", "original_file_path": os.path.join("macros", "macro.md"), "package_name": "test", "path": "macro.md", "unique_id": "doc.test.macro_info", }, "doc.test.notebook_info": { "block_contents": "A description of the complex exposure", "resource_type": "doc", "name": "notebook_info", "original_file_path": docs_path, "package_name": "test", "path": "docs.md", "unique_id": "doc.test.notebook_info", }, "doc.test.macro_arg_info": { "block_contents": "The model for my custom test", "resource_type": "doc", "name": "macro_arg_info", "original_file_path": os.path.join("macros", "macro.md"), "package_name": "test", "path": "macro.md", "unique_id": "doc.test.macro_arg_info", }, }, "child_map": { "model.test.ephemeral_copy": ["model.test.ephemeral_summary"], "exposure.test.notebook_exposure": [], "model.test.ephemeral_summary": ["model.test.view_summary"], "model.test.view_summary": ["exposure.test.notebook_exposure"], "seed.test.seed": ["snapshot.test.snapshot_seed"], "snapshot.test.snapshot_seed": [], "source.test.my_source.my_table": ["model.test.ephemeral_copy"], }, "parent_map": { "model.test.ephemeral_copy": ["source.test.my_source.my_table"], "model.test.ephemeral_summary": ["model.test.ephemeral_copy"], "model.test.view_summary": ["model.test.ephemeral_summary"], "exposure.test.notebook_exposure": ["model.test.view_summary"], "seed.test.seed": [], "snapshot.test.snapshot_seed": ["seed.test.seed"], "source.test.my_source.my_table": [], }, "group_map": {"test_group": ["model.test.ephemeral_summary"]}, "disabled": {}, "macros": { "macro.test.test_nothing": { "name": "test_nothing", "depends_on": {"macros": []}, "created_at": ANY, "description": "My custom test that I wrote that does nothing", "docs": {"node_color": None, "show": True}, "macro_sql": AnyStringWith("test nothing"), "original_file_path": os.path.join("macros", "dummy_test.sql"), "path": os.path.join("macros", "dummy_test.sql"), "package_name": "test", "meta": { "some_key": 100, }, "config": { "meta": { "some_key": 100, }, "docs": {"node_color": None, "show": True}, }, "patch_path": "test://" + os.path.join("macros", "schema.yml"), "resource_type": "macro", "unique_id": "macro.test.test_nothing", "supported_languages": None, "arguments": [ { "name": "model", "type": "Relation", "description": "The model for my custom test", }, ], } }, "semantic_models": {}, "unit_tests": {}, "saved_queries": {}, "functions": {}, } def expected_versions_manifest(project): model_database = project.database my_schema_name = project.test_schema versioned_model_v1_path = os.path.join("models", "arbitrary_file_name.sql") versioned_model_v2_path = os.path.join("models", "versioned_model_v2.sql") ref_versioned_model_path = os.path.join("models", "ref_versioned_model.sql") compiled_model_path = os.path.join("target", "compiled", "test", "models") schema_yml_path = os.path.join("models", "schema.yml") versioned_model_v1_sql = read_file_replace_returns(versioned_model_v1_path).rstrip("\r\n") versioned_model_v2_sql = read_file_replace_returns(versioned_model_v2_path).rstrip("\r\n") ref_versioned_model_sql = read_file_replace_returns(ref_versioned_model_path).rstrip("\r\n") test_config = get_rendered_tst_config() unrendered_test_config = get_unrendered_tst_config() test_audit_schema = my_schema_name + "_dbt_test__audit" model_schema_yml_path = os.path.join("models", "schema.yml") return { "dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v7.json", "dbt_version": dbt.version.__version__, "nodes": { "model.test.versioned_model.v1": { "alias": "versioned_model_v1", "compiled_path": os.path.join(compiled_model_path, "arbitrary_file_name.sql"), "build_path": None, "created_at": ANY, "columns": { "first_name": { "description": "The first name being summarized", "name": "first_name", "dimension": None, "entity": None, "data_type": None, "meta": {}, "quote": None, "tags": [], "constraints": [], "granularity": None, "doc_blocks": [], "config": { "meta": {}, "tags": [], }, }, "ct": { "description": "The number of instances of the first name", "name": "ct", "dimension": None, "entity": None, "data_type": None, "meta": {}, "quote": None, "tags": [], "constraints": [], "granularity": None, "doc_blocks": [], "config": { "meta": {}, "tags": [], }, }, }, "config": get_rendered_model_config( materialized="table", group="test_group", meta={"size": "large", "color": "blue"}, ), "constraints": [], "sources": [], "functions": [], "depends_on": {"macros": [], "nodes": []}, "description": "A versioned model", "primary_key": ["count", "first_name"], "deprecation_date": ANY, "docs": {"node_color": None, "show": True}, "fqn": ["test", "versioned_model", "v1"], "group": "test_group", "metrics": [], "name": "versioned_model", "original_file_path": versioned_model_v1_path, "package_name": "test", "patch_path": "test://" + os.path.join("models", "schema.yml"), "path": "arbitrary_file_name.sql", "raw_code": LineIndifferent(versioned_model_v1_sql), "language": "sql", "refs": [], "relation_name": '"{0}"."{1}".versioned_model_v1'.format( model_database, my_schema_name ), "resource_type": "model", "schema": my_schema_name, "database": project.database, "tags": [], "meta": {"size": "large", "color": "blue"}, "unique_id": "model.test.versioned_model.v1", "compiled": True, "compiled_code": ANY, "contract": {"checksum": None, "enforced": False, "alias_types": True}, "extra_ctes_injected": True, "extra_ctes": [], "checksum": checksum_file(versioned_model_v1_path), "unrendered_config": get_unrendered_model_config( materialized="table", group="test_group", meta={"size": "large", "color": "blue"}, ), "access": "protected", "version": 1, "latest_version": 2, "time_spine": None, "doc_blocks": [], }, "model.test.versioned_model.v2": { "alias": "versioned_model_v2", "compiled_path": os.path.join(compiled_model_path, "versioned_model_v2.sql"), "build_path": None, "created_at": ANY, "columns": { "first_name": { "description": "The first name being summarized", "name": "first_name", "dimension": None, "entity": None, "data_type": None, "meta": {}, "quote": None, "tags": [], "constraints": [], "granularity": None, "doc_blocks": [], "config": { "meta": {}, "tags": [], }, }, "extra": { "description": "", "name": "extra", "dimension": None, "entity": None, "data_type": None, "meta": {}, "quote": None, "tags": [], "constraints": [], "granularity": None, "doc_blocks": [], "config": { "meta": {}, "tags": [], }, }, }, "config": get_rendered_model_config( materialized="view", group="test_group", meta={"size": "large", "color": "red"} ), "constraints": [], "contract": {"checksum": None, "enforced": False, "alias_types": True}, "sources": [], "functions": [], "depends_on": {"macros": [], "nodes": []}, "description": "A versioned model", "primary_key": ["first_name"], "deprecation_date": None, "docs": {"node_color": None, "show": True}, "fqn": ["test", "versioned_model", "v2"], "group": "test_group", "metrics": [], "name": "versioned_model", "original_file_path": versioned_model_v2_path, "package_name": "test", "patch_path": "test://" + os.path.join("models", "schema.yml"), "path": "versioned_model_v2.sql", "raw_code": LineIndifferent(versioned_model_v2_sql), "language": "sql", "refs": [], "relation_name": '"{0}"."{1}".versioned_model_v2'.format( model_database, my_schema_name ), "resource_type": "model", "schema": my_schema_name, "database": project.database, "tags": [], "meta": {"size": "large", "color": "red"}, "unique_id": "model.test.versioned_model.v2", "compiled": True, "compiled_code": ANY, "extra_ctes_injected": True, "extra_ctes": [], "checksum": checksum_file(versioned_model_v2_path), "unrendered_config": get_unrendered_model_config( materialized="view", group="test_group", meta={"size": "large", "color": "red"} ), "access": "protected", "version": 2, "latest_version": 2, "time_spine": None, "doc_blocks": [], }, "model.test.ref_versioned_model": { "alias": "ref_versioned_model", "compiled_path": os.path.join(compiled_model_path, "ref_versioned_model.sql"), "build_path": None, "created_at": ANY, "columns": {}, "config": get_rendered_model_config(), "constraints": [], "contract": {"checksum": None, "enforced": False, "alias_types": True}, "database": project.database, "depends_on": { "macros": [], "nodes": [ "model.test.versioned_model.v2", "model.test.versioned_model.v1", ], }, "deprecation_date": None, "description": "", "primary_key": [], "docs": {"node_color": None, "show": True}, "fqn": ["test", "ref_versioned_model"], "group": None, "metrics": [], "name": "ref_versioned_model", "original_file_path": ref_versioned_model_path, "package_name": "test", "patch_path": "test://" + schema_yml_path, "path": "ref_versioned_model.sql", "raw_code": LineIndifferent(ref_versioned_model_sql), "language": "sql", "refs": [ {"name": "versioned_model", "package": None, "version": 2}, {"name": "versioned_model", "package": None, "version": "2"}, {"name": "versioned_model", "package": None, "version": 2}, {"name": "versioned_model", "package": None, "version": None}, {"name": "versioned_model", "package": None, "version": 1}, ], "relation_name": '"{0}"."{1}".ref_versioned_model'.format( model_database, my_schema_name ), "resource_type": "model", "schema": my_schema_name, "sources": [], "functions": [], "tags": [], "meta": {}, "unique_id": "model.test.ref_versioned_model", "compiled": True, "compiled_code": ANY, "extra_ctes_injected": True, "extra_ctes": [], "checksum": checksum_file(ref_versioned_model_path), "unrendered_config": get_unrendered_model_config(), "access": "protected", "version": None, "latest_version": None, "time_spine": None, "doc_blocks": [], }, "test.test.unique_versioned_model_v1_first_name.6138195dec": { "alias": "unique_versioned_model_v1_first_name", "attached_node": "model.test.versioned_model.v1", "compiled_path": os.path.join( compiled_model_path, "schema.yml", "unique_versioned_model_v1_first_name.sql" ), "build_path": None, "created_at": ANY, "column_name": "first_name", "columns": {}, "config": test_config, "group": "test_group", "contract": {"checksum": None, "enforced": False, "alias_types": True}, "sources": [], "functions": [], "depends_on": { "macros": ["macro.dbt.test_unique", "macro.dbt.get_where_subquery"], "nodes": ["model.test.versioned_model.v1"], }, "description": "", "file_key_name": "models.versioned_model", "fqn": ["test", "unique_versioned_model_v1_first_name"], "metrics": [], "name": "unique_versioned_model_v1_first_name", "original_file_path": model_schema_yml_path, "package_name": "test", "patch_path": None, "path": "unique_versioned_model_v1_first_name.sql", "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "versioned_model", "package": None, "version": 1}], "relation_name": None, "resource_type": "test", "schema": test_audit_schema, "database": project.database, "tags": [], "meta": {}, "unique_id": "test.test.unique_versioned_model_v1_first_name.6138195dec", "docs": {"node_color": None, "show": True}, "compiled": True, "compiled_code": AnyStringWith("count(*)"), "extra_ctes_injected": True, "extra_ctes": [], "test_metadata": { "namespace": None, "name": "unique", "kwargs": { "column_name": "first_name", "model": "{{ get_where_subquery(ref('versioned_model', version='1')) }}", }, }, "checksum": {"name": "none", "checksum": ""}, "unrendered_config": unrendered_test_config, "doc_blocks": [], }, "test.test.unique_versioned_model_v1_count.0b4c0b688a": { "alias": "unique_versioned_model_v1_count", "attached_node": "model.test.versioned_model.v1", "compiled_path": os.path.join( compiled_model_path, "schema.yml", "unique_versioned_model_v1_count.sql" ), "build_path": None, "created_at": ANY, "column_name": None, "columns": {}, "config": test_config, "group": "test_group", "contract": {"checksum": None, "enforced": False, "alias_types": True}, "sources": [], "functions": [], "depends_on": { "macros": ["macro.dbt.test_unique", "macro.dbt.get_where_subquery"], "nodes": ["model.test.versioned_model.v1"], }, "description": "", "file_key_name": "models.versioned_model", "fqn": ["test", "unique_versioned_model_v1_count"], "metrics": [], "name": "unique_versioned_model_v1_count", "original_file_path": model_schema_yml_path, "package_name": "test", "patch_path": None, "path": "unique_versioned_model_v1_count.sql", "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "versioned_model", "package": None, "version": 1}], "relation_name": None, "resource_type": "test", "schema": test_audit_schema, "database": project.database, "tags": [], "meta": {}, "unique_id": "test.test.unique_versioned_model_v1_count.0b4c0b688a", "docs": {"node_color": None, "show": True}, "compiled": True, "compiled_code": AnyStringWith("count(*)"), "extra_ctes_injected": True, "extra_ctes": [], "test_metadata": { "namespace": None, "name": "unique", "kwargs": { "column_name": "count", "model": "{{ get_where_subquery(ref('versioned_model', version='1')) }}", }, }, "checksum": {"name": "none", "checksum": ""}, "unrendered_config": unrendered_test_config, "doc_blocks": [], }, "test.test.unique_versioned_model_v2_first_name.998430d28e": { "alias": "unique_versioned_model_v2_first_name", "attached_node": "model.test.versioned_model.v2", "compiled_path": os.path.join( compiled_model_path, "schema.yml", "unique_versioned_model_v2_first_name.sql" ), "build_path": None, "created_at": ANY, "column_name": "first_name", "columns": {}, "config": test_config, "group": "test_group", "contract": {"checksum": None, "enforced": False, "alias_types": True}, "sources": [], "functions": [], "depends_on": { "macros": ["macro.dbt.test_unique", "macro.dbt.get_where_subquery"], "nodes": ["model.test.versioned_model.v2"], }, "description": "", "file_key_name": "models.versioned_model", "fqn": ["test", "unique_versioned_model_v2_first_name"], "metrics": [], "name": "unique_versioned_model_v2_first_name", "original_file_path": model_schema_yml_path, "package_name": "test", "patch_path": None, "path": "unique_versioned_model_v2_first_name.sql", "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "versioned_model", "package": None, "version": 2}], "relation_name": None, "resource_type": "test", "schema": test_audit_schema, "database": project.database, "tags": [], "meta": {}, "unique_id": "test.test.unique_versioned_model_v2_first_name.998430d28e", "docs": {"node_color": None, "show": True}, "compiled": True, "compiled_code": AnyStringWith("count(*)"), "extra_ctes_injected": True, "extra_ctes": [], "test_metadata": { "namespace": None, "name": "unique", "kwargs": { "column_name": "first_name", "model": "{{ get_where_subquery(ref('versioned_model', version='2')) }}", }, }, "checksum": {"name": "none", "checksum": ""}, "unrendered_config": unrendered_test_config, "doc_blocks": [], }, }, "exposures": { "exposure.test.notebook_exposure": { "created_at": ANY, "depends_on": { "macros": [], "nodes": ["model.test.versioned_model.v2"], }, "description": "notebook_info", "label": None, "config": { "enabled": True, "meta": {}, "tags": [], }, "fqn": ["test", "notebook_exposure"], "maturity": None, "meta": {}, "metrics": [], "tags": [], "name": "notebook_exposure", "original_file_path": os.path.join("models", "schema.yml"), "owner": {"email": "something@example.com", "name": "Some name"}, "package_name": "test", "path": "schema.yml", "refs": [{"name": "versioned_model", "package": None, "version": 2}], "resource_type": "exposure", "sources": [], "type": "notebook", "unique_id": "exposure.test.notebook_exposure", "url": None, "unrendered_config": {}, }, }, "metrics": {}, "groups": { "group.test.test_group": { "name": "test_group", "resource_type": "group", "original_file_path": os.path.join("models", "schema.yml"), "owner": {"email": "test_group@test.com", "name": None}, "package_name": "test", "path": "schema.yml", "unique_id": "group.test.test_group", "description": None, "config": {"meta": {}}, } }, "sources": {}, "functions": {}, "selectors": {}, "docs": {}, "child_map": { "model.test.versioned_model.v1": [ "model.test.ref_versioned_model", "test.test.unique_versioned_model_v1_count.0b4c0b688a", "test.test.unique_versioned_model_v1_first_name.6138195dec", ], "model.test.versioned_model.v2": [ "exposure.test.notebook_exposure", "model.test.ref_versioned_model", "test.test.unique_versioned_model_v2_first_name.998430d28e", ], "model.test.ref_versioned_model": [], "exposure.test.notebook_exposure": [], "test.test.unique_versioned_model_v1_first_name.6138195dec": [], "test.test.unique_versioned_model_v1_count.0b4c0b688a": [], "test.test.unique_versioned_model_v2_first_name.998430d28e": [], }, "parent_map": { "model.test.versioned_model.v1": [], "model.test.versioned_model.v2": [], "model.test.ref_versioned_model": [ "model.test.versioned_model.v1", "model.test.versioned_model.v2", ], "exposure.test.notebook_exposure": ["model.test.versioned_model.v2"], "test.test.unique_versioned_model_v1_first_name.6138195dec": [ "model.test.versioned_model.v1" ], "test.test.unique_versioned_model_v1_count.0b4c0b688a": [ "model.test.versioned_model.v1" ], "test.test.unique_versioned_model_v2_first_name.998430d28e": [ "model.test.versioned_model.v2" ], }, "group_map": { "test_group": [ "model.test.versioned_model.v1", "model.test.versioned_model.v2", "test.test.unique_versioned_model_v1_first_name.6138195dec", "test.test.unique_versioned_model_v1_count.0b4c0b688a", "test.test.unique_versioned_model_v2_first_name.998430d28e", ] }, "disabled": {}, "macros": {}, "semantic_models": {}, "unit_tests": {}, "saved_queries": {}, } ================================================ FILE: tests/functional/artifacts/expected_run_results.py ================================================ from unittest.mock import ANY from dbt.tests.util import AnyFloat def expected_run_results(): """ The expected results of this run. """ return [ { "status": "success", "message": None, "execution_time": AnyFloat(), "unique_id": "model.test.model", "adapter_response": ANY, "thread_id": ANY, "timing": [ANY, ANY], "failures": ANY, "compiled": True, "compiled_code": ANY, "relation_name": ANY, "batch_results": None, }, { "status": "success", "message": None, "execution_time": AnyFloat(), "unique_id": "model.test.second_model", "adapter_response": ANY, "thread_id": ANY, "timing": [ANY, ANY], "failures": ANY, "compiled": True, "compiled_code": ANY, "relation_name": ANY, "batch_results": None, }, { "status": "success", "message": None, "execution_time": AnyFloat(), "unique_id": "seed.test.seed", "adapter_response": ANY, "thread_id": ANY, "timing": [ANY, ANY], "failures": ANY, "compiled": None, "compiled_code": ANY, "relation_name": None, "batch_results": None, }, { "status": "success", "message": None, "execution_time": AnyFloat(), "unique_id": "snapshot.test.snapshot_seed", "adapter_response": ANY, "thread_id": ANY, "timing": [ANY, ANY], "failures": ANY, "compiled": True, "compiled_code": ANY, "relation_name": ANY, "batch_results": None, }, { "status": "success", "message": None, "execution_time": AnyFloat(), "unique_id": "test.test.not_null_model_id.d01cc630e6", "adapter_response": ANY, "thread_id": ANY, "timing": [ANY, ANY], "failures": ANY, "compiled": True, "compiled_code": ANY, "relation_name": None, "batch_results": None, }, { "status": "success", "message": None, "execution_time": AnyFloat(), "unique_id": "test.test.test_nothing_model_.5d38568946", "adapter_response": ANY, "thread_id": ANY, "timing": [ANY, ANY], "failures": ANY, "compiled": True, "compiled_code": ANY, "relation_name": None, "batch_results": None, }, { "status": "success", "message": None, "execution_time": AnyFloat(), "unique_id": "test.test.unique_model_id.67b76558ff", "adapter_response": ANY, "thread_id": ANY, "timing": [ANY, ANY], "failures": ANY, "compiled": True, "compiled_code": ANY, "relation_name": None, "batch_results": None, }, ] def expected_references_run_results(): return [ { "status": "success", "message": None, "execution_time": AnyFloat(), "unique_id": "model.test.ephemeral_summary", "adapter_response": ANY, "thread_id": ANY, "timing": [ANY, ANY], "failures": ANY, "compiled": True, "compiled_code": ANY, "relation_name": ANY, "batch_results": None, }, { "status": "success", "message": None, "execution_time": AnyFloat(), "unique_id": "model.test.view_summary", "adapter_response": ANY, "thread_id": ANY, "timing": [ANY, ANY], "failures": ANY, "compiled": True, "compiled_code": ANY, "relation_name": ANY, "batch_results": None, }, { "status": "success", "message": None, "execution_time": AnyFloat(), "unique_id": "seed.test.seed", "adapter_response": ANY, "thread_id": ANY, "timing": [ANY, ANY], "failures": ANY, "compiled": None, "compiled_code": ANY, "relation_name": ANY, "batch_results": None, }, { "status": "success", "message": None, "execution_time": AnyFloat(), "unique_id": "snapshot.test.snapshot_seed", "adapter_response": ANY, "thread_id": ANY, "timing": [ANY, ANY], "failures": ANY, "compiled": True, "compiled_code": ANY, "relation_name": ANY, "batch_results": None, }, ] def expected_versions_run_results(): return [ { "status": "success", "message": None, "execution_time": AnyFloat(), "unique_id": "model.test.ref_versioned_model", "adapter_response": ANY, "thread_id": ANY, "timing": [ANY, ANY], "failures": ANY, "compiled": True, "compiled_code": ANY, "relation_name": ANY, "batch_results": None, }, { "status": "success", "message": None, "execution_time": AnyFloat(), "unique_id": "model.test.versioned_model.v1", "adapter_response": ANY, "thread_id": ANY, "timing": [ANY, ANY], "failures": ANY, "compiled": True, "compiled_code": ANY, "relation_name": ANY, "batch_results": None, }, { "status": "success", "message": None, "execution_time": AnyFloat(), "unique_id": "model.test.versioned_model.v2", "adapter_response": ANY, "thread_id": ANY, "timing": [ANY, ANY], "failures": ANY, "compiled": True, "compiled_code": ANY, "relation_name": ANY, "batch_results": None, }, { "status": "success", "message": None, "execution_time": AnyFloat(), "unique_id": "test.test.unique_versioned_model_v1_count.0b4c0b688a", "adapter_response": ANY, "thread_id": ANY, "timing": [ANY, ANY], "failures": ANY, "compiled": True, "compiled_code": ANY, "relation_name": ANY, "batch_results": None, }, { "status": "success", "message": None, "execution_time": AnyFloat(), "unique_id": "test.test.unique_versioned_model_v1_first_name.6138195dec", "adapter_response": ANY, "thread_id": ANY, "timing": [ANY, ANY], "failures": ANY, "compiled": True, "compiled_code": ANY, "relation_name": ANY, "batch_results": None, }, { "status": "success", "message": None, "execution_time": AnyFloat(), "unique_id": "test.test.unique_versioned_model_v2_first_name.998430d28e", "adapter_response": ANY, "thread_id": ANY, "timing": [ANY, ANY], "failures": ANY, "compiled": True, "compiled_code": ANY, "relation_name": ANY, "batch_results": None, }, ] ================================================ FILE: tests/functional/artifacts/test_artifact_fields.py ================================================ import pytest from dbt.tests.util import get_artifact, get_manifest, run_dbt # This is a place to put specific tests for contents of artifacts that we # don't want to bother putting in the big artifact output test, which is # hard to update. my_model_sql = "select 1 as fun" schema_yml = """ version: 2 models: - name: my_model columns: - name: fun data_tests: - not_null """ class TestRelationNameInTests: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_model_sql, "schema.yml": schema_yml, } def test_relation_name_in_tests(self, project): results = run_dbt(["run"]) assert len(results) == 1 manifest = get_manifest(project.project_root) test_id = "test.test.not_null_my_model_fun.bf3b032a01" assert test_id in manifest.nodes assert manifest.nodes[test_id].relation_name is None results = run_dbt(["test", "--store-failures"]) assert len(results) == 1 # The relation_name for tests with previously generated manifest and # store_failures passed in on the command line, will be in the manifest.json # but not in the parsed manifest. manifest = get_manifest(project.project_root) assert manifest.nodes[test_id].relation_name is None manifest_json = get_artifact(project.project_root, "target", "manifest.json") assert test_id in manifest_json["nodes"] relation_name = manifest_json["nodes"][test_id]["relation_name"] assert relation_name assert '"not_null_my_model_fun"' in relation_name ================================================ FILE: tests/functional/artifacts/test_artifacts.py ================================================ import os from datetime import datetime, timezone import jsonschema import pytest import dbt from dbt.artifacts.schemas.results import RunStatus from dbt.artifacts.schemas.run import RunResultsArtifact from dbt.contracts.graph.manifest import WritableManifest from dbt.events.types import ArtifactWritten from dbt.tests.util import ( check_datetime_between, get_artifact, run_dbt, run_dbt_and_capture, ) from dbt_common.events.event_catcher import EventCatcher from tests.functional.artifacts.expected_manifest import ( expected_references_manifest, expected_seeded_manifest, expected_versions_manifest, ) from tests.functional.artifacts.expected_run_results import ( expected_references_run_results, expected_run_results, expected_versions_run_results, ) models__schema_yml = """ version: 2 models: - name: model description: "The test model" docs: show: false columns: - name: id description: The user ID number data_tests: - unique - not_null - name: first_name description: The user's first name - name: email description: The user's email - name: ip_address description: The user's IP address - name: updated_at description: The last time this user's email was updated data_tests: - test.nothing - name: second_model description: "The second test model" docs: show: false columns: - name: id description: The user ID number - name: first_name description: The user's first name - name: email description: The user's email - name: ip_address description: The user's IP address - name: updated_at description: The last time this user's email was updated sources: - name: my_source description: "My source" loader: a_loader schema: "{{ var('test_schema') }}" tables: - name: my_table description: "My table" identifier: seed quoting: identifier: True columns: - name: id description: "An ID field" exposures: - name: simple_exposure type: dashboard depends_on: - ref('model') - source('my_source', 'my_table') owner: email: something@example.com - name: notebook_exposure type: notebook depends_on: - ref('model') - ref('second_model') owner: email: something@example.com name: Some name description: > A description of the complex exposure maturity: medium meta: tool: 'my_tool' languages: - python tags: ['my_department'] url: http://example.com/notebook/1 """ models__second_model_sql = """ {{ config( materialized='view', schema='test', ) }} select * from {{ ref('seed') }} """ models__readme_md = """ This is a readme.md file with {{ invalid-ish jinja }} in it """ models__model_sql = """ {{ config( materialized='view', ) }} select * from {{ ref('seed') }} """ models__model_with_pre_hook_sql = """ {{ config( pre_hook={ "sql": "{{ alter_timezone(timezone='Etc/UTC') }}" } ) }} select current_setting('timezone') as timezone """ seed__schema_yml = """ version: 2 seeds: - name: seed description: "The test seed" columns: - name: id description: The user ID number - name: first_name description: The user's first name - name: email description: The user's email - name: ip_address description: The user's IP address - name: updated_at description: The last time this user's email was updated """ seed__seed_csv = """id,first_name,email,ip_address,updated_at 1,Larry,lking0@miitbeian.gov.cn,69.135.206.194,2008-09-12 19:08:31 """ macros__schema_yml = """ version: 2 macros: - name: test_nothing description: "{{ doc('macro_info') }}" meta: some_key: 100 arguments: - name: model type: Relation description: "{{ doc('macro_arg_info') }}" """ macros__macro_md = """ {% docs macro_info %} My custom test that I wrote that does nothing {% enddocs %} {% docs macro_arg_info %} The model for my custom test {% enddocs %} """ macros__dummy_test_sql = """ {% test nothing(model) %} -- a silly test to make sure that table-level tests show up in the manifest -- without a column_name field select 0 {% endtest %} """ macros__alter_timezone_sql = """ {% macro alter_timezone(timezone='America/Los_Angeles') %} {% set sql %} SET TimeZone='{{ timezone }}'; {% endset %} {% do run_query(sql) %} {% do log("Timezone set to: " + timezone, info=True) %} {% endmacro %} """ snapshot__snapshot_seed_sql = """ {% snapshot snapshot_seed %} {{ config( unique_key='id', strategy='check', check_cols='all', target_schema=var('alternate_schema') ) }} select * from {{ ref('seed') }} {% endsnapshot %} """ ref_models__schema_yml = """ version: 2 groups: - name: test_group owner: email: test_group@test.com models: - name: ephemeral_summary description: "{{ doc('ephemeral_summary') }}" config: group: test_group columns: &summary_columns - name: first_name description: "{{ doc('summary_first_name') }}" - name: ct description: "{{ doc('summary_count') }}" - name: view_summary description: "{{ doc('view_summary') }}" columns: *summary_columns sources: - name: my_source description: "{{ doc('source_info') }}" loader: a_loader schema: "{{ var('test_schema') }}" quoting: database: False identifier: False tables: - name: my_table description: "{{ doc('table_info') }}" identifier: seed quoting: identifier: True columns: - name: id description: "{{ doc('column_info') }}" exposures: - name: notebook_exposure type: notebook depends_on: - ref('view_summary') owner: email: something@example.com name: Some name description: "{{ doc('notebook_info') }}" maturity: medium url: http://example.com/notebook/1 meta: tool: 'my_tool' languages: - python tags: ['my_department'] """ ref_models__view_summary_sql = """ {{ config( materialized = "view" ) }} select first_name, ct from {{ref('ephemeral_summary')}} order by ct asc """ ref_models__ephemeral_summary_sql = """ {{ config( materialized = "table" ) }} select first_name, count(*) as ct from {{ref('ephemeral_copy')}} group by first_name order by first_name asc """ ref_models__ephemeral_copy_sql = """ {{ config( materialized = "ephemeral" ) }} select * from {{ source("my_source", "my_table") }} """ ref_models__docs_md = """ {% docs ephemeral_summary %} A summmary table of the ephemeral copy of the seed data {% enddocs %} {% docs summary_first_name %} The first name being summarized {% enddocs %} {% docs summary_count %} The number of instances of the first name {% enddocs %} {% docs view_summary %} A view of the summary of the ephemeral copy of the seed data {% enddocs %} {% docs source_info %} My source {% enddocs %} {% docs table_info %} My table {% enddocs %} {% docs column_info %} An ID field {% enddocs %} {% docs notebook_info %} A description of the complex exposure {% enddocs %} """ versioned_models__schema_yml = """ version: 2 groups: - name: test_group owner: email: test_group@test.com models: - name: versioned_model description: "A versioned model" latest_version: 2 config: group: test_group materialized: table meta: color: blue size: large data_tests: - unique: column_name: count columns: - name: first_name description: "The first name being summarized" data_tests: - unique - name: ct description: "The number of instances of the first name" versions: - v: 1 defined_in: arbitrary_file_name deprecation_date: 2022-07-11 - v: 2 config: materialized: view meta: color: red data_tests: [] columns: - include: '*' exclude: ['ct'] - name: extra - name: ref_versioned_model exposures: - name: notebook_exposure type: notebook depends_on: - ref('versioned_model', v=2) owner: email: something@example.com name: Some name description: "notebook_info" """ versioned_models__v1_sql = """ select "test first name" as first_name, 1 as ct """ versioned_models__v2_sql = """ select "test first name" as first_name, 1 as extra """ versioned_models___ref_sql = """ select first_name from {{ ref("versioned_model", version=2) }} UNION ALL select first_name from {{ ref("versioned_model", version="2") }} UNION ALL select first_name from {{ ref("versioned_model", v=2) }} UNION ALL select first_name from {{ ref("versioned_model") }} UNION ALL select first_name from {{ ref("versioned_model", version=1) }} """ def verify_metadata(metadata, dbt_schema_version, start_time): assert "generated_at" in metadata check_datetime_between(metadata["generated_at"], start=start_time) assert "dbt_version" in metadata assert metadata["dbt_version"] == dbt.version.__version__ assert "dbt_schema_version" in metadata assert metadata["dbt_schema_version"] == dbt_schema_version key = "env_key" if os.name == "nt": key = key.upper() assert metadata["env"] == {key: "env_value"} def verify_manifest(project, expected_manifest, start_time, manifest_schema_path): manifest_path = os.path.join(project.project_root, "target", "manifest.json") assert os.path.exists(manifest_path) manifest = get_artifact(manifest_path) # Verify that manifest jsonschema from WritableManifest works manifest_schema = WritableManifest.json_schema() validate(manifest_schema, manifest) # Verify that stored manifest jsonschema works. # If this fails, schemas need to be updated with: # scripts/collect-artifact-schema.py --path schemas --artifact manifest stored_manifest_schema = get_artifact(manifest_schema_path) validate(stored_manifest_schema, manifest) manifest_keys = { "nodes", "sources", "macros", "parent_map", "child_map", "group_map", "metrics", "groups", "docs", "metadata", "docs", "disabled", "exposures", "functions", "selectors", "semantic_models", "unit_tests", "saved_queries", } assert set(manifest.keys()) == manifest_keys for key in manifest_keys: if key == "macros": verify_manifest_macros(manifest, expected_manifest.get("macros")) elif key == "metadata": metadata = manifest["metadata"] dbt_schema_version = str(WritableManifest.dbt_schema_version) verify_metadata(metadata, dbt_schema_version, start_time) assert ( "project_id" in metadata and metadata["project_id"] == "098f6bcd4621d373cade4e832627b4f6" ) assert "project_name" in metadata and metadata["project_name"] == "test" assert ( "send_anonymous_usage_stats" in metadata and metadata["send_anonymous_usage_stats"] is False ) assert "adapter_type" in metadata and metadata["adapter_type"] == project.adapter_type elif key in ["nodes", "sources", "exposures", "metrics", "disabled", "docs"]: for unique_id, node in expected_manifest[key].items(): assert unique_id in manifest[key] assert manifest[key][unique_id] == node, f"{unique_id} did not match" else: # ['docs', 'parent_map', 'child_map', 'group_map', 'selectors', 'semantic_models', 'saved_queries'] assert manifest[key] == expected_manifest[key] def verify_manifest_macros(manifest, expected=None): assert "macros" in manifest if expected: for unique_id, expected_macro in expected.items(): assert unique_id in manifest["macros"] actual_macro = manifest["macros"][unique_id] assert expected_macro == actual_macro def verify_run_results(project, expected_run_results, start_time, run_results_schema_path): run_results_path = os.path.join(project.project_root, "target", "run_results.json") run_results = get_artifact(run_results_path) assert "metadata" in run_results # Verify that jsonschema for RunResultsArtifact works run_results_schema = RunResultsArtifact.json_schema() validate(run_results_schema, run_results) # Verify that stored run_results jsonschema works. # If this fails, schemas need to be updated with: # scripts/collect-artifact-schema.py --path schemas --artifact run-results stored_run_results_schema = get_artifact(run_results_schema_path) validate(stored_run_results_schema, run_results) dbt_schema_version = str(RunResultsArtifact.dbt_schema_version) verify_metadata(run_results["metadata"], dbt_schema_version, start_time) assert "elapsed_time" in run_results assert run_results["elapsed_time"] > 0 assert isinstance(run_results["elapsed_time"], float) assert "args" in run_results # sort the results so we can make reasonable assertions run_results["results"].sort(key=lambda r: r["unique_id"]) assert run_results["results"] == expected_run_results assert set(run_results) == {"elapsed_time", "results", "metadata", "args"} class BaseVerifyProject: @pytest.fixture(scope="class", autouse=True) def setup(self, project): alternate_schema_name = project.test_schema + "_test" project.create_test_schema(schema_name=alternate_schema_name) os.environ["DBT_ENV_CUSTOM_ENV_env_key"] = "env_value" run_dbt(["seed"]) yield del os.environ["DBT_ENV_CUSTOM_ENV_env_key"] @pytest.fixture(scope="class") def seeds(self): return {"schema.yml": seed__schema_yml, "seed.csv": seed__seed_csv} @pytest.fixture(scope="class") def macros(self): return { "schema.yml": macros__schema_yml, "macro.md": macros__macro_md, "dummy_test.sql": macros__dummy_test_sql, } @pytest.fixture(scope="class") def snapshots(self): return {"snapshot_seed.sql": snapshot__snapshot_seed_sql} @pytest.fixture(scope="class") def project_config_update(self, unique_schema): alternate_schema = unique_schema + "_test" return { "vars": { "test_schema": unique_schema, "alternate_schema": alternate_schema, }, "seeds": { "quote_columns": True, }, "quoting": {"identifier": False}, } @pytest.fixture(scope="class") def manifest_schema_path(self, request): schema_version_paths = WritableManifest.dbt_schema_version.path.split("/") manifest_schema_path = os.path.join( request.config.rootdir, "schemas", *schema_version_paths ) return manifest_schema_path @pytest.fixture(scope="class") def run_results_schema_path(self, request): schema_version_paths = RunResultsArtifact.dbt_schema_version.path.split("/") run_results_schema_path = os.path.join( request.config.rootdir, "schemas", *schema_version_paths ) return run_results_schema_path def validate(artifact_schema, artifact_dict): validator = jsonschema.Draft7Validator(artifact_schema) error = next(iter(validator.iter_errors(artifact_dict)), None) assert error is None class TestVerifyArtifacts(BaseVerifyProject): @pytest.fixture(scope="class") def models(self): return { "schema.yml": models__schema_yml, "second_model.sql": models__second_model_sql, "readme.md": models__readme_md, "model.sql": models__model_sql, } # Test generic "docs generate" command def test_run_and_generate(self, project, manifest_schema_path, run_results_schema_path): catcher = EventCatcher(ArtifactWritten) start_time = datetime.now(timezone.utc).replace(tzinfo=None) results = run_dbt(args=["compile"], callbacks=[catcher.catch]) assert len(results) == 7 verify_manifest( project, expected_seeded_manifest(project, quote_model=False), start_time, manifest_schema_path, ) verify_run_results(project, expected_run_results(), start_time, run_results_schema_path) # manifest written twice, semantic manifest written twice, run results written once assert len(catcher.caught_events) == 5 assert ( len( [ event for event in catcher.caught_events if event.data.artifact_type == "WritableManifest" ] ) > 0 ) assert ( len( [ event for event in catcher.caught_events if event.data.artifact_type == "SemanticManifest" ] ) > 0 ) assert ( len( [ event for event in catcher.caught_events if event.data.artifact_type == "RunExecutionResult" ] ) > 0 ) # Test artifact with additional fields load fine def test_load_artifact(self, project, manifest_schema_path, run_results_schema_path): catcher = EventCatcher(ArtifactWritten) results = run_dbt(args=["compile"], callbacks=[catcher.catch]) assert len(results) == 7 manifest_dct = get_artifact(os.path.join(project.project_root, "target", "manifest.json")) # add a field that is not in the schema for _, node in manifest_dct["nodes"].items(): node["something_else"] = "something_else" # load the manifest with the additional field loaded_manifest = WritableManifest.from_dict(manifest_dct) # successfully loaded the manifest with the additional field, but the field should not be present for _, node in loaded_manifest.nodes.items(): assert not hasattr(node, "something_else") class TestVerifyArtifactsReferences(BaseVerifyProject): @pytest.fixture(scope="class") def models(self): return { "schema.yml": ref_models__schema_yml, "view_summary.sql": ref_models__view_summary_sql, "ephemeral_summary.sql": ref_models__ephemeral_summary_sql, "ephemeral_copy.sql": ref_models__ephemeral_copy_sql, "docs.md": ref_models__docs_md, } def test_references(self, project, manifest_schema_path, run_results_schema_path): start_time = datetime.now(timezone.utc).replace(tzinfo=None) results = run_dbt(["compile"]) assert len(results) == 4 verify_manifest( project, expected_references_manifest(project), start_time, manifest_schema_path ) verify_run_results( project, expected_references_run_results(), start_time, run_results_schema_path ) class TestVerifyArtifactsVersions(BaseVerifyProject): @pytest.fixture(scope="class") def models(self): return { "schema.yml": versioned_models__schema_yml, "versioned_model_v2.sql": versioned_models__v2_sql, "arbitrary_file_name.sql": versioned_models__v1_sql, "ref_versioned_model.sql": versioned_models___ref_sql, } @pytest.fixture(scope="class") def seeds(self): return {} @pytest.fixture(scope="class") def snapshots(self): return {} def test_versions(self, project, manifest_schema_path, run_results_schema_path): start_time = datetime.now(timezone.utc).replace(tzinfo=None) results = run_dbt(["compile"]) assert len(results) == 6 verify_manifest( project, expected_versions_manifest(project), start_time, manifest_schema_path ) verify_run_results( project, expected_versions_run_results(), start_time, run_results_schema_path ) class TestVerifyRunOperation(BaseVerifyProject): @pytest.fixture(scope="class") def macros(self): return {"alter_timezone.sql": macros__alter_timezone_sql} @pytest.fixture(scope="class") def models(self): return { "model_with_pre_hook.sql": models__model_with_pre_hook_sql, } def test_run_operation(self, project): results, log_output = run_dbt_and_capture(["run-operation", "alter_timezone"]) assert len(results) == 1 assert results[0].status == RunStatus.Success assert results[0].unique_id == "macro.test.alter_timezone" assert "Timezone set to: America/Los_Angeles" in log_output def test_run_model_with_operation(self, project): # pre-hooks are not included in run_results since they are an attribute of the node and not a node in their # own right results, log_output = run_dbt_and_capture(["run", "--select", "model_with_pre_hook"]) assert len(results) == 1 assert results[0].status == RunStatus.Success assert "Timezone set to: Etc/UTC" in log_output ================================================ FILE: tests/functional/artifacts/test_docs_generate_defer.py ================================================ import os import shutil import pytest from dbt.tests.util import run_dbt model_sql = """ select 1 as id """ class TestDocsGenerateDefer: @pytest.fixture(scope="class") def models(self): return {"model.sql": model_sql} def copy_state(self): assert not os.path.exists("state") os.makedirs("state") shutil.copyfile("target/manifest.json", "state/manifest.json") def test_generate_defer( self, project, ): results = run_dbt(["run"]) assert len(results) == 1 # copy state files self.copy_state() # defer test, it succeeds catalog = run_dbt(["docs", "generate", "--state", "./state", "--defer"]) assert catalog.nodes["model.test.model"] # Check that catalog validates with jsonschema catalog_dict = catalog.to_dict() try: catalog.validate(catalog_dict) except Exception: raise pytest.fail("Catalog validation failed") ================================================ FILE: tests/functional/artifacts/test_override.py ================================================ import pytest from dbt.exceptions import CompilationError from dbt.tests.util import run_dbt model_sql = """ select 1 as id """ fail_macros__failure_sql = """ {% macro get_catalog_relations(information_schema, relations) %} {% do exceptions.raise_compiler_error('rejected: no catalogs for you') %} {% endmacro %} """ class TestDocsGenerateOverride: @pytest.fixture(scope="class") def models(self): return {"model.sql": model_sql} @pytest.fixture(scope="class") def macros(self): return {"failure.sql": fail_macros__failure_sql} def test_override_used( self, project, ): results = run_dbt(["run"]) assert len(results) == 1 # this should pick up our failure macro and raise a compilation exception with pytest.raises(CompilationError) as excinfo: run_dbt(["--warn-error", "docs", "generate"]) assert "rejected: no catalogs for you" in str(excinfo.value) ================================================ FILE: tests/functional/artifacts/test_previous_version_state.py ================================================ import json import os import shutil import pytest from dbt.artifacts.exceptions import IncompatibleSchemaError from dbt.artifacts.schemas.base import get_artifact_schema_version from dbt.artifacts.schemas.run import RunResultsArtifact from dbt.contracts.graph.manifest import WritableManifest from dbt.tests.util import get_manifest, run_dbt # This project must have one of each kind of node type, plus disabled versions, for # test coverage to be complete. models__my_model_sql = """ select 1 as id """ models__disabled_model_sql = """ {{ config(enabled=False) }} select 2 as id """ seeds__my_seed_csv = """ id,value 4,2 """ seeds__disabled_seed_csv = """ id,value 6,4 """ docs__somedoc_md = """ {% docs somedoc %} Testing, testing {% enddocs %} """ macros__do_nothing_sql = """ {% macro do_nothing(foo2, bar2) %} select '{{ foo2 }}' as foo2, '{{ bar2 }}' as bar2 {% endmacro %} """ macros__dummy_test_sql = """ {% test check_nothing(model) %} -- a silly test to make sure that table-level tests show up in the manifest -- without a column_name field select 0 {% endtest %} """ macros__disabled_dummy_test_sql = """ {% test disabled_check_nothing(model) %} -- a silly test to make sure that table-level tests show up in the manifest -- without a column_name field {{ config(enabled=False) }} select 0 {% endtest %} """ snapshot__snapshot_seed_sql = """ {% snapshot snapshot_seed %} {{ config( unique_key='id', strategy='check', check_cols='all', target_schema=schema, ) }} select * from {{ ref('my_seed') }} {% endsnapshot %} """ snapshot__disabled_snapshot_seed_sql = """ {% snapshot disabled_snapshot_seed %} {{ config( unique_key='id', strategy='check', check_cols='all', target_schema=schema, enabled=False, ) }} select * from {{ ref('my_seed') }} {% endsnapshot %} """ tests__just_my_sql = """ {{ config(tags = ['data_test_tag']) }} select * from {{ ref('my_model') }} where false """ tests__disabled_just_my_sql = """ {{ config(enabled=False) }} select * from {{ ref('my_model') }} where false """ analyses__a_sql = """ select 4 as id """ analyses__disabled_a_sql = """ {{ config(enabled=False) }} select 9 as id """ metricflow_time_spine_sql = """ SELECT to_date('02/20/2023', 'mm/dd/yyyy') as date_day """ # Use old attribute names (v1.0-1.2) to test forward/backward compatibility with the rename in v1.3 models__schema_yml = """ version: 2 models: - name: my_model description: "Example model" data_tests: - check_nothing - disabled_check_nothing columns: - name: id data_tests: - not_null semantic_models: - name: semantic_people model: ref('my_model') dimensions: - name: favorite_color type: categorical - name: created_at type: TIME type_params: time_granularity: day measures: - name: years_tenure agg: SUM expr: tenure - name: people agg: count expr: id - name: customers agg: count expr: id entities: - name: id type: primary defaults: agg_time_dimension: created_at metrics: - name: blue_customers_post_2010 label: Blue Customers since 2010 type: simple filter: "{{ TimeDimension('id__created_at', 'day') }} > '2010-01-01'" type_params: measure: name: customers filter: "{{ Dimension('id__favorite_color') }} = 'blue'" - name: customers label: Customers Metric type: simple type_params: measure: customers - name: disabled_metric label: Count records config: enabled: False filter: "{{ Dimension('id__favorite_color') }} = 'blue'" type: simple type_params: measure: customers - name: ratio_of_blue_customers_to_red_customers label: Very Important Customer Color Ratio type: ratio type_params: numerator: name: customers filter: "{{ Dimension('id__favorite_color')}} = 'blue'" denominator: name: customers filter: "{{ Dimension('id__favorite_color')}} = 'red'" - name: doubled_blue_customers type: derived label: Inflated blue customer numbers type_params: expr: 'customers * 2' metrics: - name: customers filter: "{{ Dimension('id__favorite_color')}} = 'blue'" sources: - name: my_source description: "My source" loader: a_loader tables: - name: my_table description: "My table" identifier: my_seed - name: disabled_table description: "Disabled table" config: enabled: False exposures: - name: simple_exposure type: dashboard depends_on: - ref('my_model') - source('my_source', 'my_table') owner: email: something@example.com - name: disabled_exposure type: dashboard config: enabled: False depends_on: - ref('my_model') owner: email: something@example.com seeds: - name: disabled_seed config: enabled: False """ # SETUP: Using this project, we have run past minor versions of dbt # to generate each contracted version of `manifest.json`. # Whenever we bump the manifest version, we should add a new entry for that version # into `data`, generated from this same project, and update the CURRENT_EXPECTED_MANIFEST_VERSION. # You can generate the manifest using the generate_latest_manifest() method below. # TEST: Then, using the *current* version of dbt (this branch), # we will perform a `--state` comparison against those older manifests. # Some comparisons should succeed, where we expect backward/forward compatibility. # Comparisons against older versions should fail, because the structure of the # WritableManifest class has changed in ways that prevent successful deserialization # of older JSON manifests. # We are creating enabled versions of every node type that might be in the manifest, # plus disabled versions for types that support it (everything except macros and docs). class TestPreviousVersionState: CURRENT_EXPECTED_MANIFEST_VERSION = 12 CURRENT_EXPECTED_RUN_RESULTS_VERSION = 6 @pytest.fixture(scope="class") def models(self): return { "my_model.sql": models__my_model_sql, "schema.yml": models__schema_yml, "somedoc.md": docs__somedoc_md, "disabled_model.sql": models__disabled_model_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, } @pytest.fixture(scope="class") def seeds(self): return { "my_seed.csv": seeds__my_seed_csv, "disabled_seed.csv": seeds__disabled_seed_csv, } @pytest.fixture(scope="class") def snapshots(self): return { "snapshot_seed.sql": snapshot__snapshot_seed_sql, "disabled_snapshot_seed.sql": snapshot__disabled_snapshot_seed_sql, } @pytest.fixture(scope="class") def tests(self): return { "just_my.sql": tests__just_my_sql, "disabled_just_my.sql": tests__disabled_just_my_sql, } @pytest.fixture(scope="class") def macros(self): return { "do_nothing.sql": macros__do_nothing_sql, "dummy_test.sql": macros__dummy_test_sql, "disabled_dummy_test.sql": macros__disabled_dummy_test_sql, } @pytest.fixture(scope="class") def analyses(self): return { "a.sql": analyses__a_sql, "disabled_al.sql": analyses__disabled_a_sql, } def test_project(self, project): # This is mainly used to test changes to the test project in isolation from # the other noise. results = run_dbt(["run"]) assert len(results) == 2 manifest = get_manifest(project.project_root) # model, snapshot, seed, singular test, generic test, analysis assert len(manifest.nodes) == 8 assert len(manifest.sources) == 1 assert len(manifest.exposures) == 1 assert len(manifest.metrics) == 4 # disabled model, snapshot, seed, singular test, generic test, analysis, source, exposure, metric assert len(manifest.disabled) == 9 assert "macro.test.do_nothing" in manifest.macros # Use this method when generating a new manifest version for the first time. # Once generated, we shouldn't need to re-generate or modify the manifest. def generate_latest_manifest( self, project, current_manifest_version, ): run_dbt(["parse"]) source_path = os.path.join(project.project_root, "target/manifest.json") state_path = os.path.join(project.test_data_dir, f"state/v{current_manifest_version}") target_path = os.path.join(state_path, "manifest.json") os.makedirs(state_path, exist_ok=True) shutil.copyfile(source_path, target_path) # Use this method when generating a new run_results version for the first time. # Once generated, we shouldn't need to re-generate or modify the manifest. def generate_latest_run_results( self, project, current_run_results_version, ): run_dbt(["run"]) source_path = os.path.join(project.project_root, "target/run_results.json") state_path = os.path.join(project.test_data_dir, f"results/v{current_run_results_version}") target_path = os.path.join(state_path, "run_results.json") os.makedirs(state_path, exist_ok=True) shutil.copyfile(source_path, target_path) # The actual test method. Run `dbt list --select state:modified --state ...` # once for each past manifest version. They all have the same content, but different # schema/structure, only some of which are forward-compatible with the # current WritableManifest class. def compare_previous_state( self, project, compare_manifest_version, expect_pass, num_results, ): state_path = os.path.join(project.test_data_dir, f"state/v{compare_manifest_version}") cli_args = [ "list", "--resource-types", "model", "--select", "state:modified", "--state", state_path, ] if expect_pass: results = run_dbt(cli_args, expect_pass=expect_pass) assert len(results) == num_results else: with pytest.raises(IncompatibleSchemaError): run_dbt(cli_args, expect_pass=expect_pass) # The actual test method. Run `dbt retry --state ...` # once for each past run_results version. They all have the same content, but different # schema/structure, only some of which are forward-compatible with the # current WritableManifest class. def compare_previous_results( self, project, compare_run_results_version, expect_pass, num_results, ): state_path = os.path.join(project.test_data_dir, f"results/v{compare_run_results_version}") cli_args = [ "retry", "--state", state_path, ] if expect_pass: results = run_dbt(cli_args, expect_pass=expect_pass) assert len(results) == num_results else: with pytest.raises(IncompatibleSchemaError): run_dbt(cli_args, expect_pass=expect_pass) def test_compare_state_current(self, project): current_manifest_schema_version = WritableManifest.dbt_schema_version.version assert ( current_manifest_schema_version == self.CURRENT_EXPECTED_MANIFEST_VERSION ), "Sounds like you've bumped the manifest version and need to update this test!" # If we need a newly generated manifest, uncomment the following line and commit the result # self.generate_latest_manifest(project, current_manifest_schema_version) self.compare_previous_state(project, current_manifest_schema_version, True, 0) def test_backwards_compatible_versions(self, project): # manifest schema version 4 and greater should always be forward compatible for schema_version in range(4, 10): self.compare_previous_state(project, schema_version, True, 1) for schema_version in range(10, self.CURRENT_EXPECTED_MANIFEST_VERSION): self.compare_previous_state(project, schema_version, True, 0) def test_nonbackwards_compatible_versions(self, project): # schema versions 1, 2, 3 are all not forward compatible for schema_version in range(1, 4): self.compare_previous_state(project, schema_version, False, 0) def test_get_manifest_schema_version(self, project): for schema_version in range(1, self.CURRENT_EXPECTED_MANIFEST_VERSION): manifest_path = os.path.join( project.test_data_dir, f"state/v{schema_version}/manifest.json" ) manifest = json.load(open(manifest_path)) manifest_version = get_artifact_schema_version(manifest) assert manifest_version == schema_version def test_compare_results_current(self, project): current_run_results_schema_version = RunResultsArtifact.dbt_schema_version.version assert ( current_run_results_schema_version == self.CURRENT_EXPECTED_RUN_RESULTS_VERSION ), "Sounds like you've bumped the run_results version and need to update this test!" # If we need a newly generated run_results, uncomment the following line and commit the result # self.generate_latest_run_results(project, current_run_results_schema_version) self.compare_previous_results(project, current_run_results_schema_version, True, 0) def test_backwards_compatible_run_results_versions(self, project): # run_results schema version 4 and greater should always be forward compatible for schema_version in range(4, self.CURRENT_EXPECTED_RUN_RESULTS_VERSION): self.compare_previous_results(project, schema_version, True, 0) ================================================ FILE: tests/functional/artifacts/test_run_execution_result.py ================================================ import pytest from dateutil.tz import tzutc from dbt.contracts.results import RunExecutionResult from dbt.tests.util import run_dbt, write_file sample_model_sql = """ select 1 as id """ @pytest.fixture(scope="function", autouse=True) def sample_model(project): write_file( sample_model_sql, project.project_root, "models", "model.sql", ) def test_run_execution_result_compiled_serialization(project): result = run_dbt(["compile"]) result_from_dict = RunExecutionResult.from_dict(result.to_dict()) assert isinstance(result, RunExecutionResult) assert len(result.results) > 0 assert result.results[0].status.name == "Success" assert result_from_dict.args == result.args assert len(result_from_dict.results) == len(result.results) assert result.generated_at.tzinfo is None assert result_from_dict.generated_at.tzinfo == tzutc() assert result_from_dict.generated_at.replace(tzinfo=None) == result.generated_at for original in result.results: for deserialized in result_from_dict.results: if original.node.unique_id == deserialized.node.unique_id: assert original.node.created_at == deserialized.node.created_at ================================================ FILE: tests/functional/artifacts/test_run_results.py ================================================ import json from multiprocessing import Process from pathlib import Path import pytest from dbt.tests.util import run_dbt good_model_sql = """ select 1 as id """ bad_model_sql = """ something bad """ slow_model_sql = """ {{ config(materialized='table') }} select id from {{ ref('good_model') }}, pg_sleep(5) """ class TestRunResultsTimingSuccess: @pytest.fixture(scope="class") def models(self): return {"model.sql": good_model_sql} def test_timing_exists(self, project): results = run_dbt(["run"]) assert len(results.results) == 1 assert len(results.results[0].timing) > 0 class TestRunResultsTimingFailure: @pytest.fixture(scope="class") def models(self): return {"model.sql": bad_model_sql} def test_timing_exists(self, project): results = run_dbt(["run"], expect_pass=False) assert len(results.results) == 1 assert len(results.results[0].timing) > 0 class TestRunResultsSerializableInContext: @pytest.fixture(scope="class") def models(self): return {"model.sql": good_model_sql} @pytest.fixture(scope="class") def project_config_update(self): return { "on-run-end": ["{% for result in results %}{{ log(result.to_dict()) }}{% endfor %}"] } def test_results_serializable(self, project): results = run_dbt(["run"]) assert len(results.results) == 2 # This test is failing due to the faulty assumptions that run_results.json would # be written multiple times. Temporarily disabling. @pytest.mark.skip() class TestRunResultsWritesFileOnSignal: @pytest.fixture(scope="class") def models(self): return {"good_model.sql": good_model_sql, "slow_model.sql": slow_model_sql} def test_run_results_are_written_on_signal(self, project): # Start the runner in a seperate process. external_process_dbt = Process( target=run_dbt, args=([["run"]]), kwargs={"expect_pass": False} ) external_process_dbt.start() assert external_process_dbt.is_alive() # Wait until the first file write, then kill the process. run_results_file = Path(project.project_root) / "target/run_results.json" while run_results_file.is_file() is False: pass external_process_dbt.terminate() # Wait until the process is dead, then check the file that there is only one result. while external_process_dbt.is_alive() is True: pass with run_results_file.open() as run_results_str: run_results = json.loads(run_results_str.read()) assert len(run_results["results"]) == 1 ================================================ FILE: tests/functional/assertions/test_runner.py ================================================ import os from typing import Callable, List, Optional from dbt.cli.main import dbtRunner, dbtRunnerResult from dbt.contracts.graph.manifest import Manifest from dbt.tests.util import get_run_results from dbt_common.events.base_types import EventMsg def assert_run_results_have_compiled_node_attributes( args: List[str], result: dbtRunnerResult ) -> None: commands_with_run_results = ["build", "compile", "docs", "run", "test"] if not [a for a in args if a in commands_with_run_results] or not result.success: return run_results = get_run_results(os.getcwd()) for r in run_results["results"]: if r["unique_id"].startswith("model") and r["status"] == "success": assert "compiled_code" in r assert "compiled" in r _STANDARD_ASSERTIONS = [assert_run_results_have_compiled_node_attributes] class dbtTestRunner(dbtRunner): def __init__( self, manifest: Optional[Manifest] = None, callbacks: Optional[List[Callable[[EventMsg], None]]] = None, exit_assertions: Optional[List[Callable[[List[str], dbtRunnerResult], None]]] = None, ): self.exit_assertions = exit_assertions if exit_assertions else _STANDARD_ASSERTIONS super().__init__(manifest, callbacks) def invoke(self, args: List[str], **kwargs) -> dbtRunnerResult: result = super().invoke(args, **kwargs) for assertion in self.exit_assertions: assertion(args, result) return result ================================================ FILE: tests/functional/basic/data/seed-initial.csv ================================================ id,first_name,last_name,email,gender,ip_address 1,Jack,Hunter,jhunter0@pbs.org,Male,59.80.20.168 2,Kathryn,Walker,kwalker1@ezinearticles.com,Female,194.121.179.35 3,Gerald,Ryan,gryan2@com.com,Male,11.3.212.243 4,Bonnie,Spencer,bspencer3@ameblo.jp,Female,216.32.196.175 5,Harold,Taylor,htaylor4@people.com.cn,Male,253.10.246.136 6,Jacqueline,Griffin,jgriffin5@t.co,Female,16.13.192.220 7,Wanda,Arnold,warnold6@google.nl,Female,232.116.150.64 8,Craig,Ortiz,cortiz7@sciencedaily.com,Male,199.126.106.13 9,Gary,Day,gday8@nih.gov,Male,35.81.68.186 10,Rose,Wright,rwright9@yahoo.co.jp,Female,236.82.178.100 11,Raymond,Kelley,rkelleya@fc2.com,Male,213.65.166.67 12,Gerald,Robinson,grobinsonb@disqus.com,Male,72.232.194.193 13,Mildred,Martinez,mmartinezc@samsung.com,Female,198.29.112.5 14,Dennis,Arnold,darnoldd@google.com,Male,86.96.3.250 15,Judy,Gray,jgraye@opensource.org,Female,79.218.162.245 16,Theresa,Garza,tgarzaf@epa.gov,Female,21.59.100.54 17,Gerald,Robertson,grobertsong@csmonitor.com,Male,131.134.82.96 18,Philip,Hernandez,phernandezh@adobe.com,Male,254.196.137.72 19,Julia,Gonzalez,jgonzalezi@cam.ac.uk,Female,84.240.227.174 20,Andrew,Davis,adavisj@patch.com,Male,9.255.67.25 21,Kimberly,Harper,kharperk@foxnews.com,Female,198.208.120.253 22,Mark,Martin,mmartinl@marketwatch.com,Male,233.138.182.153 23,Cynthia,Ruiz,cruizm@google.fr,Female,18.178.187.201 24,Samuel,Carroll,scarrolln@youtu.be,Male,128.113.96.122 25,Jennifer,Larson,jlarsono@vinaora.com,Female,98.234.85.95 26,Ashley,Perry,aperryp@rakuten.co.jp,Female,247.173.114.52 27,Howard,Rodriguez,hrodriguezq@shutterfly.com,Male,231.188.95.26 28,Amy,Brooks,abrooksr@theatlantic.com,Female,141.199.174.118 29,Louise,Warren,lwarrens@adobe.com,Female,96.105.158.28 30,Tina,Watson,twatsont@myspace.com,Female,251.142.118.177 31,Janice,Kelley,jkelleyu@creativecommons.org,Female,239.167.34.233 32,Terry,Mccoy,tmccoyv@bravesites.com,Male,117.201.183.203 33,Jeffrey,Morgan,jmorganw@surveymonkey.com,Male,78.101.78.149 34,Louis,Harvey,lharveyx@sina.com.cn,Male,51.50.0.167 35,Philip,Miller,pmillery@samsung.com,Male,103.255.222.110 36,Willie,Marshall,wmarshallz@ow.ly,Male,149.219.91.68 37,Patrick,Lopez,plopez10@redcross.org,Male,250.136.229.89 38,Adam,Jenkins,ajenkins11@harvard.edu,Male,7.36.112.81 39,Benjamin,Cruz,bcruz12@linkedin.com,Male,32.38.98.15 40,Ruby,Hawkins,rhawkins13@gmpg.org,Female,135.171.129.255 41,Carlos,Barnes,cbarnes14@a8.net,Male,240.197.85.140 42,Ruby,Griffin,rgriffin15@bravesites.com,Female,19.29.135.24 43,Sean,Mason,smason16@icq.com,Male,159.219.155.249 44,Anthony,Payne,apayne17@utexas.edu,Male,235.168.199.218 45,Steve,Cruz,scruz18@pcworld.com,Male,238.201.81.198 46,Anthony,Garcia,agarcia19@flavors.me,Male,25.85.10.18 47,Doris,Lopez,dlopez1a@sphinn.com,Female,245.218.51.238 48,Susan,Nichols,snichols1b@freewebs.com,Female,199.99.9.61 49,Wanda,Ferguson,wferguson1c@yahoo.co.jp,Female,236.241.135.21 50,Andrea,Pierce,apierce1d@google.co.uk,Female,132.40.10.209 51,Lawrence,Phillips,lphillips1e@jugem.jp,Male,72.226.82.87 52,Judy,Gilbert,jgilbert1f@multiply.com,Female,196.250.15.142 53,Eric,Williams,ewilliams1g@joomla.org,Male,222.202.73.126 54,Ralph,Romero,rromero1h@sogou.com,Male,123.184.125.212 55,Jean,Wilson,jwilson1i@ocn.ne.jp,Female,176.106.32.194 56,Lori,Reynolds,lreynolds1j@illinois.edu,Female,114.181.203.22 57,Donald,Moreno,dmoreno1k@bbc.co.uk,Male,233.249.97.60 58,Steven,Berry,sberry1l@eepurl.com,Male,186.193.50.50 59,Theresa,Shaw,tshaw1m@people.com.cn,Female,120.37.71.222 60,John,Stephens,jstephens1n@nationalgeographic.com,Male,191.87.127.115 61,Richard,Jacobs,rjacobs1o@state.tx.us,Male,66.210.83.155 62,Andrew,Lawson,alawson1p@over-blog.com,Male,54.98.36.94 63,Peter,Morgan,pmorgan1q@rambler.ru,Male,14.77.29.106 64,Nicole,Garrett,ngarrett1r@zimbio.com,Female,21.127.74.68 65,Joshua,Kim,jkim1s@edublogs.org,Male,57.255.207.41 66,Ralph,Roberts,rroberts1t@people.com.cn,Male,222.143.131.109 67,George,Montgomery,gmontgomery1u@smugmug.com,Male,76.75.111.77 68,Gerald,Alvarez,galvarez1v@flavors.me,Male,58.157.186.194 69,Donald,Olson,dolson1w@whitehouse.gov,Male,69.65.74.135 70,Carlos,Morgan,cmorgan1x@pbs.org,Male,96.20.140.87 71,Aaron,Stanley,astanley1y@webnode.com,Male,163.119.217.44 72,Virginia,Long,vlong1z@spiegel.de,Female,204.150.194.182 73,Robert,Berry,rberry20@tripadvisor.com,Male,104.19.48.241 74,Antonio,Brooks,abrooks21@unesco.org,Male,210.31.7.24 75,Ruby,Garcia,rgarcia22@ovh.net,Female,233.218.162.214 76,Jack,Hanson,jhanson23@blogtalkradio.com,Male,31.55.46.199 77,Kathryn,Nelson,knelson24@walmart.com,Female,14.189.146.41 78,Jason,Reed,jreed25@printfriendly.com,Male,141.189.89.255 79,George,Coleman,gcoleman26@people.com.cn,Male,81.189.221.144 80,Rose,King,rking27@ucoz.com,Female,212.123.168.231 81,Johnny,Holmes,jholmes28@boston.com,Male,177.3.93.188 82,Katherine,Gilbert,kgilbert29@altervista.org,Female,199.215.169.61 83,Joshua,Thomas,jthomas2a@ustream.tv,Male,0.8.205.30 84,Julie,Perry,jperry2b@opensource.org,Female,60.116.114.192 85,Richard,Perry,rperry2c@oracle.com,Male,181.125.70.232 86,Kenneth,Ruiz,kruiz2d@wikimedia.org,Male,189.105.137.109 87,Jose,Morgan,jmorgan2e@webnode.com,Male,101.134.215.156 88,Donald,Campbell,dcampbell2f@goo.ne.jp,Male,102.120.215.84 89,Debra,Collins,dcollins2g@uol.com.br,Female,90.13.153.235 90,Jesse,Johnson,jjohnson2h@stumbleupon.com,Male,225.178.125.53 91,Elizabeth,Stone,estone2i@histats.com,Female,123.184.126.221 92,Angela,Rogers,arogers2j@goodreads.com,Female,98.104.132.187 93,Emily,Dixon,edixon2k@mlb.com,Female,39.190.75.57 94,Albert,Scott,ascott2l@tinypic.com,Male,40.209.13.189 95,Barbara,Peterson,bpeterson2m@ow.ly,Female,75.249.136.180 96,Adam,Greene,agreene2n@fastcompany.com,Male,184.173.109.144 97,Earl,Sanders,esanders2o@hc360.com,Male,247.34.90.117 98,Angela,Brooks,abrooks2p@mtv.com,Female,10.63.249.126 99,Harold,Foster,hfoster2q@privacy.gov.au,Male,139.214.40.244 100,Carl,Meyer,cmeyer2r@disqus.com,Male,204.117.7.88 ================================================ FILE: tests/functional/basic/data/seed-update.csv ================================================ id,first_name,last_name,email,gender,ip_address 1,Jack,Hunter,jhunter0@pbs.org,Male,59.80.20.168 2,Kathryn,Walker,kwalker1@ezinearticles.com,Female,194.121.179.35 3,Gerald,Ryan,gryan2@com.com,Male,11.3.212.243 4,Bonnie,Spencer,bspencer3@ameblo.jp,Female,216.32.196.175 5,Harold,Taylor,htaylor4@people.com.cn,Male,253.10.246.136 6,Jacqueline,Griffin,jgriffin5@t.co,Female,16.13.192.220 7,Wanda,Arnold,warnold6@google.nl,Female,232.116.150.64 8,Craig,Ortiz,cortiz7@sciencedaily.com,Male,199.126.106.13 9,Gary,Day,gday8@nih.gov,Male,35.81.68.186 10,Rose,Wright,rwright9@yahoo.co.jp,Female,236.82.178.100 11,Raymond,Kelley,rkelleya@fc2.com,Male,213.65.166.67 12,Gerald,Robinson,grobinsonb@disqus.com,Male,72.232.194.193 13,Mildred,Martinez,mmartinezc@samsung.com,Female,198.29.112.5 14,Dennis,Arnold,darnoldd@google.com,Male,86.96.3.250 15,Judy,Gray,jgraye@opensource.org,Female,79.218.162.245 16,Theresa,Garza,tgarzaf@epa.gov,Female,21.59.100.54 17,Gerald,Robertson,grobertsong@csmonitor.com,Male,131.134.82.96 18,Philip,Hernandez,phernandezh@adobe.com,Male,254.196.137.72 19,Julia,Gonzalez,jgonzalezi@cam.ac.uk,Female,84.240.227.174 20,Andrew,Davis,adavisj@patch.com,Male,9.255.67.25 21,Kimberly,Harper,kharperk@foxnews.com,Female,198.208.120.253 22,Mark,Martin,mmartinl@marketwatch.com,Male,233.138.182.153 23,Cynthia,Ruiz,cruizm@google.fr,Female,18.178.187.201 24,Samuel,Carroll,scarrolln@youtu.be,Male,128.113.96.122 25,Jennifer,Larson,jlarsono@vinaora.com,Female,98.234.85.95 26,Ashley,Perry,aperryp@rakuten.co.jp,Female,247.173.114.52 27,Howard,Rodriguez,hrodriguezq@shutterfly.com,Male,231.188.95.26 28,Amy,Brooks,abrooksr@theatlantic.com,Female,141.199.174.118 29,Louise,Warren,lwarrens@adobe.com,Female,96.105.158.28 30,Tina,Watson,twatsont@myspace.com,Female,251.142.118.177 31,Janice,Kelley,jkelleyu@creativecommons.org,Female,239.167.34.233 32,Terry,Mccoy,tmccoyv@bravesites.com,Male,117.201.183.203 33,Jeffrey,Morgan,jmorganw@surveymonkey.com,Male,78.101.78.149 34,Louis,Harvey,lharveyx@sina.com.cn,Male,51.50.0.167 35,Philip,Miller,pmillery@samsung.com,Male,103.255.222.110 36,Willie,Marshall,wmarshallz@ow.ly,Male,149.219.91.68 37,Patrick,Lopez,plopez10@redcross.org,Male,250.136.229.89 38,Adam,Jenkins,ajenkins11@harvard.edu,Male,7.36.112.81 39,Benjamin,Cruz,bcruz12@linkedin.com,Male,32.38.98.15 40,Ruby,Hawkins,rhawkins13@gmpg.org,Female,135.171.129.255 41,Carlos,Barnes,cbarnes14@a8.net,Male,240.197.85.140 42,Ruby,Griffin,rgriffin15@bravesites.com,Female,19.29.135.24 43,Sean,Mason,smason16@icq.com,Male,159.219.155.249 44,Anthony,Payne,apayne17@utexas.edu,Male,235.168.199.218 45,Steve,Cruz,scruz18@pcworld.com,Male,238.201.81.198 46,Anthony,Garcia,agarcia19@flavors.me,Male,25.85.10.18 47,Doris,Lopez,dlopez1a@sphinn.com,Female,245.218.51.238 48,Susan,Nichols,snichols1b@freewebs.com,Female,199.99.9.61 49,Wanda,Ferguson,wferguson1c@yahoo.co.jp,Female,236.241.135.21 50,Andrea,Pierce,apierce1d@google.co.uk,Female,132.40.10.209 51,Lawrence,Phillips,lphillips1e@jugem.jp,Male,72.226.82.87 52,Judy,Gilbert,jgilbert1f@multiply.com,Female,196.250.15.142 53,Eric,Williams,ewilliams1g@joomla.org,Male,222.202.73.126 54,Ralph,Romero,rromero1h@sogou.com,Male,123.184.125.212 55,Jean,Wilson,jwilson1i@ocn.ne.jp,Female,176.106.32.194 56,Lori,Reynolds,lreynolds1j@illinois.edu,Female,114.181.203.22 57,Donald,Moreno,dmoreno1k@bbc.co.uk,Male,233.249.97.60 58,Steven,Berry,sberry1l@eepurl.com,Male,186.193.50.50 59,Theresa,Shaw,tshaw1m@people.com.cn,Female,120.37.71.222 60,John,Stephens,jstephens1n@nationalgeographic.com,Male,191.87.127.115 61,Richard,Jacobs,rjacobs1o@state.tx.us,Male,66.210.83.155 62,Andrew,Lawson,alawson1p@over-blog.com,Male,54.98.36.94 63,Peter,Morgan,pmorgan1q@rambler.ru,Male,14.77.29.106 64,Nicole,Garrett,ngarrett1r@zimbio.com,Female,21.127.74.68 65,Joshua,Kim,jkim1s@edublogs.org,Male,57.255.207.41 66,Ralph,Roberts,rroberts1t@people.com.cn,Male,222.143.131.109 67,George,Montgomery,gmontgomery1u@smugmug.com,Male,76.75.111.77 68,Gerald,Alvarez,galvarez1v@flavors.me,Male,58.157.186.194 69,Donald,Olson,dolson1w@whitehouse.gov,Male,69.65.74.135 70,Carlos,Morgan,cmorgan1x@pbs.org,Male,96.20.140.87 71,Aaron,Stanley,astanley1y@webnode.com,Male,163.119.217.44 72,Virginia,Long,vlong1z@spiegel.de,Female,204.150.194.182 73,Robert,Berry,rberry20@tripadvisor.com,Male,104.19.48.241 74,Antonio,Brooks,abrooks21@unesco.org,Male,210.31.7.24 75,Ruby,Garcia,rgarcia22@ovh.net,Female,233.218.162.214 76,Jack,Hanson,jhanson23@blogtalkradio.com,Male,31.55.46.199 77,Kathryn,Nelson,knelson24@walmart.com,Female,14.189.146.41 78,Jason,Reed,jreed25@printfriendly.com,Male,141.189.89.255 79,George,Coleman,gcoleman26@people.com.cn,Male,81.189.221.144 80,Rose,King,rking27@ucoz.com,Female,212.123.168.231 81,Johnny,Holmes,jholmes28@boston.com,Male,177.3.93.188 82,Katherine,Gilbert,kgilbert29@altervista.org,Female,199.215.169.61 83,Joshua,Thomas,jthomas2a@ustream.tv,Male,0.8.205.30 84,Julie,Perry,jperry2b@opensource.org,Female,60.116.114.192 85,Richard,Perry,rperry2c@oracle.com,Male,181.125.70.232 86,Kenneth,Ruiz,kruiz2d@wikimedia.org,Male,189.105.137.109 87,Jose,Morgan,jmorgan2e@webnode.com,Male,101.134.215.156 88,Donald,Campbell,dcampbell2f@goo.ne.jp,Male,102.120.215.84 89,Debra,Collins,dcollins2g@uol.com.br,Female,90.13.153.235 90,Jesse,Johnson,jjohnson2h@stumbleupon.com,Male,225.178.125.53 91,Elizabeth,Stone,estone2i@histats.com,Female,123.184.126.221 92,Angela,Rogers,arogers2j@goodreads.com,Female,98.104.132.187 93,Emily,Dixon,edixon2k@mlb.com,Female,39.190.75.57 94,Albert,Scott,ascott2l@tinypic.com,Male,40.209.13.189 95,Barbara,Peterson,bpeterson2m@ow.ly,Female,75.249.136.180 96,Adam,Greene,agreene2n@fastcompany.com,Male,184.173.109.144 97,Earl,Sanders,esanders2o@hc360.com,Male,247.34.90.117 98,Angela,Brooks,abrooks2p@mtv.com,Female,10.63.249.126 99,Harold,Foster,hfoster2q@privacy.gov.au,Male,139.214.40.244 100,Carl,Meyer,cmeyer2r@disqus.com,Male,204.117.7.88 101,Michael,Perez,mperez0@chronoengine.com,Male,106.239.70.175 102,Shawn,Mccoy,smccoy1@reddit.com,Male,24.165.76.182 103,Kathleen,Payne,kpayne2@cargocollective.com,Female,113.207.168.106 104,Jimmy,Cooper,jcooper3@cargocollective.com,Male,198.24.63.114 105,Katherine,Rice,krice4@typepad.com,Female,36.97.186.238 106,Sarah,Ryan,sryan5@gnu.org,Female,119.117.152.40 107,Martin,Mcdonald,mmcdonald6@opera.com,Male,8.76.38.115 108,Frank,Robinson,frobinson7@wunderground.com,Male,186.14.64.194 109,Jennifer,Franklin,jfranklin8@mail.ru,Female,91.216.3.131 110,Henry,Welch,hwelch9@list-manage.com,Male,176.35.182.168 111,Fred,Snyder,fsnydera@reddit.com,Male,217.106.196.54 112,Amy,Dunn,adunnb@nba.com,Female,95.39.163.195 113,Kathleen,Meyer,kmeyerc@cdc.gov,Female,164.142.188.214 114,Steve,Ferguson,sfergusond@reverbnation.com,Male,138.22.204.251 115,Teresa,Hill,thille@dion.ne.jp,Female,82.84.228.235 116,Amanda,Harper,aharperf@mail.ru,Female,16.123.56.176 117,Kimberly,Ray,krayg@xing.com,Female,48.66.48.12 118,Johnny,Knight,jknighth@jalbum.net,Male,99.30.138.123 119,Virginia,Freeman,vfreemani@tiny.cc,Female,225.172.182.63 120,Anna,Austin,aaustinj@diigo.com,Female,62.111.227.148 121,Willie,Hill,whillk@mail.ru,Male,0.86.232.249 122,Sean,Harris,sharrisl@zdnet.com,Male,117.165.133.249 123,Mildred,Adams,madamsm@usatoday.com,Female,163.44.97.46 124,David,Graham,dgrahamn@zimbio.com,Male,78.13.246.202 125,Victor,Hunter,vhuntero@ehow.com,Male,64.156.179.139 126,Aaron,Ruiz,aruizp@weebly.com,Male,34.194.68.78 127,Benjamin,Brooks,bbrooksq@jalbum.net,Male,20.192.189.107 128,Lisa,Wilson,lwilsonr@japanpost.jp,Female,199.152.130.217 129,Benjamin,King,bkings@comsenz.com,Male,29.189.189.213 130,Christina,Williamson,cwilliamsont@boston.com,Female,194.101.52.60 131,Jane,Gonzalez,jgonzalezu@networksolutions.com,Female,109.119.12.87 132,Thomas,Owens,towensv@psu.edu,Male,84.168.213.153 133,Katherine,Moore,kmoorew@naver.com,Female,183.150.65.24 134,Jennifer,Stewart,jstewartx@yahoo.com,Female,38.41.244.58 135,Sara,Tucker,stuckery@topsy.com,Female,181.130.59.184 136,Harold,Ortiz,hortizz@vkontakte.ru,Male,198.231.63.137 137,Shirley,James,sjames10@yelp.com,Female,83.27.160.104 138,Dennis,Johnson,djohnson11@slate.com,Male,183.178.246.101 139,Louise,Weaver,lweaver12@china.com.cn,Female,1.14.110.18 140,Maria,Armstrong,marmstrong13@prweb.com,Female,181.142.1.249 141,Gloria,Cruz,gcruz14@odnoklassniki.ru,Female,178.232.140.243 142,Diana,Spencer,dspencer15@ifeng.com,Female,125.153.138.244 143,Kelly,Nguyen,knguyen16@altervista.org,Female,170.13.201.119 144,Jane,Rodriguez,jrodriguez17@biblegateway.com,Female,12.102.249.81 145,Scott,Brown,sbrown18@geocities.jp,Male,108.174.99.192 146,Norma,Cruz,ncruz19@si.edu,Female,201.112.156.197 147,Marie,Peters,mpeters1a@mlb.com,Female,231.121.197.144 148,Lillian,Carr,lcarr1b@typepad.com,Female,206.179.164.163 149,Judy,Nichols,jnichols1c@t-online.de,Female,158.190.209.194 150,Billy,Long,blong1d@yahoo.com,Male,175.20.23.160 151,Howard,Reid,hreid1e@exblog.jp,Male,118.99.196.20 152,Laura,Ferguson,lferguson1f@tuttocitta.it,Female,22.77.87.110 153,Anne,Bailey,abailey1g@geocities.com,Female,58.144.159.245 154,Rose,Morgan,rmorgan1h@ehow.com,Female,118.127.97.4 155,Nicholas,Reyes,nreyes1i@google.ru,Male,50.135.10.252 156,Joshua,Kennedy,jkennedy1j@house.gov,Male,154.6.163.209 157,Paul,Watkins,pwatkins1k@upenn.edu,Male,177.236.120.87 158,Kathryn,Kelly,kkelly1l@businessweek.com,Female,70.28.61.86 159,Adam,Armstrong,aarmstrong1m@techcrunch.com,Male,133.235.24.202 160,Norma,Wallace,nwallace1n@phoca.cz,Female,241.119.227.128 161,Timothy,Reyes,treyes1o@google.cn,Male,86.28.23.26 162,Elizabeth,Patterson,epatterson1p@sun.com,Female,139.97.159.149 163,Edward,Gomez,egomez1q@google.fr,Male,158.103.108.255 164,David,Cox,dcox1r@friendfeed.com,Male,206.80.80.58 165,Brenda,Wood,bwood1s@over-blog.com,Female,217.207.44.179 166,Adam,Walker,awalker1t@blogs.com,Male,253.211.54.93 167,Michael,Hart,mhart1u@wix.com,Male,230.206.200.22 168,Jesse,Ellis,jellis1v@google.co.uk,Male,213.254.162.52 169,Janet,Powell,jpowell1w@un.org,Female,27.192.194.86 170,Helen,Ford,hford1x@creativecommons.org,Female,52.160.102.168 171,Gerald,Carpenter,gcarpenter1y@about.me,Male,36.30.194.218 172,Kathryn,Oliver,koliver1z@army.mil,Female,202.63.103.69 173,Alan,Berry,aberry20@gov.uk,Male,246.157.112.211 174,Harry,Andrews,handrews21@ameblo.jp,Male,195.108.0.12 175,Andrea,Hall,ahall22@hp.com,Female,149.162.163.28 176,Barbara,Wells,bwells23@behance.net,Female,224.70.72.1 177,Anne,Wells,awells24@apache.org,Female,180.168.81.153 178,Harry,Harper,hharper25@rediff.com,Male,151.87.130.21 179,Jack,Ray,jray26@wufoo.com,Male,220.109.38.178 180,Phillip,Hamilton,phamilton27@joomla.org,Male,166.40.47.30 181,Shirley,Hunter,shunter28@newsvine.com,Female,97.209.140.194 182,Arthur,Daniels,adaniels29@reuters.com,Male,5.40.240.86 183,Virginia,Rodriguez,vrodriguez2a@walmart.com,Female,96.80.164.184 184,Christina,Ryan,cryan2b@hibu.com,Female,56.35.5.52 185,Theresa,Mendoza,tmendoza2c@vinaora.com,Female,243.42.0.210 186,Jason,Cole,jcole2d@ycombinator.com,Male,198.248.39.129 187,Phillip,Bryant,pbryant2e@rediff.com,Male,140.39.116.251 188,Adam,Torres,atorres2f@sun.com,Male,101.75.187.135 189,Margaret,Johnston,mjohnston2g@ucsd.edu,Female,159.30.69.149 190,Paul,Payne,ppayne2h@hhs.gov,Male,199.234.140.220 191,Todd,Willis,twillis2i@businessweek.com,Male,191.59.136.214 192,Willie,Oliver,woliver2j@noaa.gov,Male,44.212.35.197 193,Frances,Robertson,frobertson2k@go.com,Female,31.117.65.136 194,Gregory,Hawkins,ghawkins2l@joomla.org,Male,91.3.22.49 195,Lisa,Perkins,lperkins2m@si.edu,Female,145.95.31.186 196,Jacqueline,Anderson,janderson2n@cargocollective.com,Female,14.176.0.187 197,Shirley,Diaz,sdiaz2o@ucla.edu,Female,207.12.95.46 198,Nicole,Meyer,nmeyer2p@flickr.com,Female,231.79.115.13 199,Mary,Gray,mgray2q@constantcontact.com,Female,210.116.64.253 200,Jean,Mcdonald,jmcdonald2r@baidu.com,Female,122.239.235.117 ================================================ FILE: tests/functional/basic/data/summary_expected.csv ================================================ gender,ct Female,40 Male,60 ================================================ FILE: tests/functional/basic/data/summary_expected_update.csv ================================================ gender,ct Female,94 Male,106 ================================================ FILE: tests/functional/basic/data/varchar10_seed.sql ================================================ create table {schema}.seed ( id BIGSERIAL PRIMARY KEY, first_name VARCHAR(50), last_name VARCHAR(50), email VARCHAR(50), "GenDEr" VARCHAR(10), ip_address VARCHAR(20) ); insert into {schema}.seed (first_name, last_name, email, "GenDEr", ip_address) values ('Jack', 'Hunter', 'jhunter0@pbs.org', 'Male', '59.80.20.168'), ('Kathryn', 'Walker', 'kwalker1@ezinearticles.com', 'Female', '194.121.179.35'), ('Gerald', 'Ryan', 'gryan2@com.com', 'Male', '11.3.212.243'), ('Bonnie', 'Spencer', 'bspencer3@ameblo.jp', 'Female', '216.32.196.175'), ('Harold', 'Taylor', 'htaylor4@people.com.cn', 'Male', '253.10.246.136'), ('Jacqueline', 'Griffin', 'jgriffin5@t.co', 'Female', '16.13.192.220'), ('Wanda', 'Arnold', 'warnold6@google.nl', 'Female', '232.116.150.64'), ('Craig', 'Ortiz', 'cortiz7@sciencedaily.com', 'Male', '199.126.106.13'), ('Gary', 'Day', 'gday8@nih.gov', 'Male', '35.81.68.186'), ('Rose', 'Wright', 'rwright9@yahoo.co.jp', 'Female', '236.82.178.100'), ('Raymond', 'Kelley', 'rkelleya@fc2.com', 'Male', '213.65.166.67'), ('Gerald', 'Robinson', 'grobinsonb@disqus.com', 'Male', '72.232.194.193'), ('Mildred', 'Martinez', 'mmartinezc@samsung.com', 'Female', '198.29.112.5'), ('Dennis', 'Arnold', 'darnoldd@google.com', 'Male', '86.96.3.250'), ('Judy', 'Gray', 'jgraye@opensource.org', 'Female', '79.218.162.245'), ('Theresa', 'Garza', 'tgarzaf@epa.gov', 'Female', '21.59.100.54'), ('Gerald', 'Robertson', 'grobertsong@csmonitor.com', 'Male', '131.134.82.96'), ('Philip', 'Hernandez', 'phernandezh@adobe.com', 'Male', '254.196.137.72'), ('Julia', 'Gonzalez', 'jgonzalezi@cam.ac.uk', 'Female', '84.240.227.174'), ('Andrew', 'Davis', 'adavisj@patch.com', 'Male', '9.255.67.25'), ('Kimberly', 'Harper', 'kharperk@foxnews.com', 'Female', '198.208.120.253'), ('Mark', 'Martin', 'mmartinl@marketwatch.com', 'Male', '233.138.182.153'), ('Cynthia', 'Ruiz', 'cruizm@google.fr', 'Female', '18.178.187.201'), ('Samuel', 'Carroll', 'scarrolln@youtu.be', 'Male', '128.113.96.122'), ('Jennifer', 'Larson', 'jlarsono@vinaora.com', 'Female', '98.234.85.95'), ('Ashley', 'Perry', 'aperryp@rakuten.co.jp', 'Female', '247.173.114.52'), ('Howard', 'Rodriguez', 'hrodriguezq@shutterfly.com', 'Male', '231.188.95.26'), ('Amy', 'Brooks', 'abrooksr@theatlantic.com', 'Female', '141.199.174.118'), ('Louise', 'Warren', 'lwarrens@adobe.com', 'Female', '96.105.158.28'), ('Tina', 'Watson', 'twatsont@myspace.com', 'Female', '251.142.118.177'), ('Janice', 'Kelley', 'jkelleyu@creativecommons.org', 'Female', '239.167.34.233'), ('Terry', 'Mccoy', 'tmccoyv@bravesites.com', 'Male', '117.201.183.203'), ('Jeffrey', 'Morgan', 'jmorganw@surveymonkey.com', 'Male', '78.101.78.149'), ('Louis', 'Harvey', 'lharveyx@sina.com.cn', 'Male', '51.50.0.167'), ('Philip', 'Miller', 'pmillery@samsung.com', 'Male', '103.255.222.110'), ('Willie', 'Marshall', 'wmarshallz@ow.ly', 'Male', '149.219.91.68'), ('Patrick', 'Lopez', 'plopez10@redcross.org', 'Male', '250.136.229.89'), ('Adam', 'Jenkins', 'ajenkins11@harvard.edu', 'Male', '7.36.112.81'), ('Benjamin', 'Cruz', 'bcruz12@linkedin.com', 'Male', '32.38.98.15'), ('Ruby', 'Hawkins', 'rhawkins13@gmpg.org', 'Female', '135.171.129.255'), ('Carlos', 'Barnes', 'cbarnes14@a8.net', 'Male', '240.197.85.140'), ('Ruby', 'Griffin', 'rgriffin15@bravesites.com', 'Female', '19.29.135.24'), ('Sean', 'Mason', 'smason16@icq.com', 'Male', '159.219.155.249'), ('Anthony', 'Payne', 'apayne17@utexas.edu', 'Male', '235.168.199.218'), ('Steve', 'Cruz', 'scruz18@pcworld.com', 'Male', '238.201.81.198'), ('Anthony', 'Garcia', 'agarcia19@flavors.me', 'Male', '25.85.10.18'), ('Doris', 'Lopez', 'dlopez1a@sphinn.com', 'Female', '245.218.51.238'), ('Susan', 'Nichols', 'snichols1b@freewebs.com', 'Female', '199.99.9.61'), ('Wanda', 'Ferguson', 'wferguson1c@yahoo.co.jp', 'Female', '236.241.135.21'), ('Andrea', 'Pierce', 'apierce1d@google.co.uk', 'Female', '132.40.10.209'), ('Lawrence', 'Phillips', 'lphillips1e@jugem.jp', 'Male', '72.226.82.87'), ('Judy', 'Gilbert', 'jgilbert1f@multiply.com', 'Female', '196.250.15.142'), ('Eric', 'Williams', 'ewilliams1g@joomla.org', 'Male', '222.202.73.126'), ('Ralph', 'Romero', 'rromero1h@sogou.com', 'Male', '123.184.125.212'), ('Jean', 'Wilson', 'jwilson1i@ocn.ne.jp', 'Female', '176.106.32.194'), ('Lori', 'Reynolds', 'lreynolds1j@illinois.edu', 'Female', '114.181.203.22'), ('Donald', 'Moreno', 'dmoreno1k@bbc.co.uk', 'Male', '233.249.97.60'), ('Steven', 'Berry', 'sberry1l@eepurl.com', 'Male', '186.193.50.50'), ('Theresa', 'Shaw', 'tshaw1m@people.com.cn', 'Female', '120.37.71.222'), ('John', 'Stephens', 'jstephens1n@nationalgeographic.com', 'Male', '191.87.127.115'), ('Richard', 'Jacobs', 'rjacobs1o@state.tx.us', 'Male', '66.210.83.155'), ('Andrew', 'Lawson', 'alawson1p@over-blog.com', 'Male', '54.98.36.94'), ('Peter', 'Morgan', 'pmorgan1q@rambler.ru', 'Male', '14.77.29.106'), ('Nicole', 'Garrett', 'ngarrett1r@zimbio.com', 'Female', '21.127.74.68'), ('Joshua', 'Kim', 'jkim1s@edublogs.org', 'Male', '57.255.207.41'), ('Ralph', 'Roberts', 'rroberts1t@people.com.cn', 'Male', '222.143.131.109'), ('George', 'Montgomery', 'gmontgomery1u@smugmug.com', 'Male', '76.75.111.77'), ('Gerald', 'Alvarez', 'galvarez1v@flavors.me', 'Male', '58.157.186.194'), ('Donald', 'Olson', 'dolson1w@whitehouse.gov', 'Male', '69.65.74.135'), ('Carlos', 'Morgan', 'cmorgan1x@pbs.org', 'Male', '96.20.140.87'), ('Aaron', 'Stanley', 'astanley1y@webnode.com', 'Male', '163.119.217.44'), ('Virginia', 'Long', 'vlong1z@spiegel.de', 'Female', '204.150.194.182'), ('Robert', 'Berry', 'rberry20@tripadvisor.com', 'Male', '104.19.48.241'), ('Antonio', 'Brooks', 'abrooks21@unesco.org', 'Male', '210.31.7.24'), ('Ruby', 'Garcia', 'rgarcia22@ovh.net', 'Female', '233.218.162.214'), ('Jack', 'Hanson', 'jhanson23@blogtalkradio.com', 'Male', '31.55.46.199'), ('Kathryn', 'Nelson', 'knelson24@walmart.com', 'Female', '14.189.146.41'), ('Jason', 'Reed', 'jreed25@printfriendly.com', 'Male', '141.189.89.255'), ('George', 'Coleman', 'gcoleman26@people.com.cn', 'Male', '81.189.221.144'), ('Rose', 'King', 'rking27@ucoz.com', 'Female', '212.123.168.231'), ('Johnny', 'Holmes', 'jholmes28@boston.com', 'Male', '177.3.93.188'), ('Katherine', 'Gilbert', 'kgilbert29@altervista.org', 'Female', '199.215.169.61'), ('Joshua', 'Thomas', 'jthomas2a@ustream.tv', 'Male', '0.8.205.30'), ('Julie', 'Perry', 'jperry2b@opensource.org', 'Female', '60.116.114.192'), ('Richard', 'Perry', 'rperry2c@oracle.com', 'Male', '181.125.70.232'), ('Kenneth', 'Ruiz', 'kruiz2d@wikimedia.org', 'Male', '189.105.137.109'), ('Jose', 'Morgan', 'jmorgan2e@webnode.com', 'Male', '101.134.215.156'), ('Donald', 'Campbell', 'dcampbell2f@goo.ne.jp', 'Male', '102.120.215.84'), ('Debra', 'Collins', 'dcollins2g@uol.com.br', 'Female', '90.13.153.235'), ('Jesse', 'Johnson', 'jjohnson2h@stumbleupon.com', 'Male', '225.178.125.53'), ('Elizabeth', 'Stone', 'estone2i@histats.com', 'Female', '123.184.126.221'), ('Angela', 'Rogers', 'arogers2j@goodreads.com', 'Female', '98.104.132.187'), ('Emily', 'Dixon', 'edixon2k@mlb.com', 'Female', '39.190.75.57'), ('Albert', 'Scott', 'ascott2l@tinypic.com', 'Male', '40.209.13.189'), ('Barbara', 'Peterson', 'bpeterson2m@ow.ly', 'Female', '75.249.136.180'), ('Adam', 'Greene', 'agreene2n@fastcompany.com', 'Male', '184.173.109.144'), ('Earl', 'Sanders', 'esanders2o@hc360.com', 'Male', '247.34.90.117'), ('Angela', 'Brooks', 'abrooks2p@mtv.com', 'Female', '10.63.249.126'), ('Harold', 'Foster', 'hfoster2q@privacy.gov.au', 'Male', '139.214.40.244'), ('Carl', 'Meyer', 'cmeyer2r@disqus.com', 'Male', '204.117.7.88'); ================================================ FILE: tests/functional/basic/data/varchar300_seed.sql ================================================ ALTER TABLE {schema}.seed ALTER COLUMN "GenDEr" TYPE varchar(300); insert into {schema}.seed (first_name, last_name, email, "GenDEr", ip_address) values ('Annie', 'Reynolds', 'areynolds0@nifty.com', 'Amerisource Bergen', '133.30.242.211'), ('Doris', 'Wood', 'dwood1@skyrock.com', 'Bliss World, LLC', '128.229.89.207'), ('Andrea', 'Ray', 'aray2@google.co.jp', 'Nelco Laboratories, Inc.', '109.74.153.45'), ('Frank', 'Morgan', 'fmorgan3@1688.com', 'ALK-Abello, Inc.', '252.211.209.9'), ('Angela', 'Stanley', 'astanley4@google.fr', 'Gemini Pharmaceuticals, Inc. dba ONDRA Pharmaceuticals', '134.142.194.184'), ('Ruby', 'Jordan', 'rjordan5@nymag.com', 'Watson Pharma, Inc.', '195.104.60.172'), ('Kathleen', 'Ryan', 'kryan6@scientificamerican.com', 'SHISEIDO AMERICAS CORPORATION', '209.110.160.192'), ('Margaret', 'Jacobs', 'mjacobs7@example.com', 'Cardinal Health', '72.36.52.20'), ('Ernest', 'Brown', 'ebrown8@360.cn', 'West-ward Pharmaceutical Corp', '138.157.61.255'), ('Elizabeth', 'Phillips', 'ephillips9@japanpost.jp', 'Cellex-C International Inc', '68.46.195.188'), ('Annie', 'Ellis', 'aellisa@weather.com', 'NATURE REPUBLIC CO., LTD.', '163.128.214.142'), ('Melissa', 'Olson', 'molsonb@theguardian.com', 'Nelco Laboratories, Inc.', '202.22.153.188'), ('Timothy', 'Martinez', 'tmartinezc@zimbio.com', 'Lake Erie Medical & Surgical Supply DBA Quality Care Products LLC', '45.64.205.47'), ('Mark', 'Nelson', 'mnelsond@bloomberg.com', '7-Eleven', '91.99.195.160'), ('Kenneth', 'Hart', 'kharte@berkeley.edu', 'Preferred Pharmaceuticals, Inc.', '207.240.9.102'), ('Kathryn', 'White', 'kwhitef@csmonitor.com', 'Cantrell Drug Company', '191.178.162.18'), ('Mary', 'Greene', 'mgreeneg@usnews.com', 'Neutrogena Corporation', '251.226.65.64'), ('Bruce', 'Peters', 'bpetersh@blogspot.com', 'Sun & Skin Care Research, LLC', '153.227.91.121'), ('Albert', 'Armstrong', 'aarmstrongi@weather.com', 'Access Business Group LLC', '199.146.159.228'), ('Beverly', 'Gray', 'bgrayj@spiegel.de', 'Church & Dwight Co., Inc.', '47.3.135.226'), ('Catherine', 'Taylor', 'ctaylork@walmart.com', 'Matrixx Initiatives, Inc.', '82.24.129.147'), ('Paula', 'Bradley', 'pbradleyl@edublogs.org', 'Nash-Finch Company', '14.145.193.163'), ('Terry', 'Campbell', 'tcampbellm@artisteer.com', 'MedVantx, Inc.', '89.181.95.177'), ('Bruce', 'Stevens', 'bstevensn@ucla.edu', 'Global Pharmaceuticals', '128.81.126.144'), ('Ruby', 'Bishop', 'rbishopo@telegraph.co.uk', 'General Injectables & Vaccines, Inc.', '191.191.17.173'), ('Denise', 'Duncan', 'dduncanp@reference.com', 'Bare Escentuals Beauty, Inc.', '150.207.3.163'), ('Dennis', 'Perkins', 'dperkinsq@1und1.de', 'Altaire Pharmaceuticals Inc.', '21.150.103.133'), ('Brandon', 'Ray', 'brayr@psu.edu', 'Meijer Distribution Inc', '216.53.187.191'), ('Ernest', 'Graham', 'egrahams@tinyurl.com', 'BioComp Pharma, Inc.', '49.85.236.162'), ('Denise', 'Matthews', 'dmatthewst@digg.com', 'Procter & Gamble Manufacturing Co.', '160.4.119.137'), ('Randy', 'Alexander', 'ralexanderu@goo.gl', 'Reckitt Benckiser Pharmaceuticals Inc', '211.72.176.12'), ('Aaron', 'Jackson', 'ajacksonv@gizmodo.com', 'Molton Brown LTD (UK)', '226.178.48.73'), ('Wanda', 'Turner', 'wturnerw@reverbnation.com', 'American Health Packaging', '43.22.122.56'), ('Stephen', 'Ferguson', 'sfergusonx@kickstarter.com', 'Amneal Pharmaceuticals of New York, LLC', '110.211.112.233'), ('Jane', 'Bradley', 'jbradleyy@usgs.gov', 'Kroger Company', '186.153.255.125'), ('Phillip', 'Wood', 'pwoodz@about.com', 'Unit Dose Services', '112.65.6.93'), ('Jeffrey', 'Howell', 'jhowell10@symantec.com', 'Midlothian Laboratories', '232.92.208.248'), ('Howard', 'Harvey', 'hharvey11@nhs.uk', 'Novartis Pharmaceuticals Corporation', '50.212.26.218'), ('Benjamin', 'Johnston', 'bjohnston12@diigo.com', 'Nelco Laboratories, Inc.', '131.109.13.9'), ('Ernest', 'Burke', 'eburke13@toplist.cz', 'Apotex Corp.', '151.176.178.175'), ('Joe', 'Wright', 'jwright14@mapy.cz', 'MULTALER & CIE S.A.', '233.55.33.63'), ('Ronald', 'Griffin', 'rgriffin15@topsy.com', 'Gavis Pharmaceuticals, LLC', '174.233.67.86'), ('Susan', 'Oliver', 'soliver16@goo.gl', 'Bath & Body Works, Inc.', '104.171.43.12'), ('Karen', 'Cox', 'kcox17@hp.com', 'Home Sweet Homeopathics', '225.51.182.192'), ('Antonio', 'Larson', 'alarson18@gov.uk', 'Eight and Company', '243.118.98.188'), ('Brandon', 'Cook', 'bcook19@mozilla.com', 'Chain Drug Consortium, LLC', '38.64.44.255'), ('Gary', 'Gray', 'ggray1a@alexa.com', 'Lil'' Drug Store Products, Inc', '43.34.161.60'), ('Doris', 'Harrison', 'dharrison1b@wiley.com', 'Dispensing Solutions, Inc.', '153.66.74.140'), ('Clarence', 'Perry', 'cperry1c@issuu.com', 'Nelco Laboratories, Inc.', '14.72.110.59'), ('Emily', 'George', 'egeorge1d@blogtalkradio.com', 'State of Florida DOH Central Pharmacy', '148.35.114.224'), ('Dennis', 'Larson', 'dlarson1e@trellian.com', 'G&W Laboratories, Inc.', '134.158.117.11'), ('Ashley', 'Peters', 'apeters1f@de.vu', 'Mylan Pharmaceuticals Inc.', '50.193.252.146'), ('Douglas', 'Andrews', 'dandrews1g@mac.com', 'Jubilant HollisterStier LLC', '159.134.237.86'), ('Craig', 'Dunn', 'cdunn1h@cornell.edu', 'Antigen Laboratories, Inc.', '227.11.100.112'), ('Heather', 'Black', 'hblack1i@harvard.edu', 'Hospira, Inc.', '61.9.121.22'), ('Shirley', 'Ruiz', 'sruiz1j@tmall.com', 'Hankuk Bowonbio Co., Ltd', '171.144.250.254'), ('Carl', 'Martinez', 'cmartinez1k@geocities.jp', 'ALK-Abello, Inc.', '128.216.69.116'), ('Stephen', 'Anderson', 'sanderson1l@odnoklassniki.ru', 'Cardinal Health', '145.154.63.186'), ('Diana', 'Payne', 'dpayne1m@ftc.gov', 'Pharmaceutical Associates, Inc.', '98.9.155.136'), ('Judy', 'Gonzalez', 'jgonzalez1n@walmart.com', 'SHISEIDO CO., LTD.', '73.96.109.149'), ('Steve', 'Cole', 'scole1o@flickr.com', 'Walgreen Company', '251.244.20.117'), ('Johnny', 'Ellis', 'jellis1p@time.com', 'Jubilant HollisterStier LLC', '188.153.76.182'), ('Andrea', 'Hamilton', 'ahamilton1q@dailymail.co.uk', 'ALK-Abello, Inc.', '229.58.149.141'), ('Sean', 'Kennedy', 'skennedy1r@nifty.com', 'Newton Laboratories, Inc.', '227.105.251.134'), ('Sara', 'Grant', 'sgrant1s@flickr.com', 'Rubbermaid Commercial Products LLC', '96.211.162.73'), ('Joan', 'Bennett', 'jbennett1t@forbes.com', 'Nelco Laboratories, Inc.', '143.27.240.163'), ('Judith', 'Daniels', 'jdaniels1u@theguardian.com', 'Newton Laboratories, Inc.', '164.99.249.153'), ('Irene', 'Bennett', 'ibennett1v@comsenz.com', 'Cellab Co., Ltd.', '112.104.12.122'), ('Katherine', 'Perez', 'kperez1w@phpbb.com', 'Temple Industrial Welding Supply Co', '211.31.214.131'), ('Jean', 'Kim', 'jkim1x@umich.edu', 'Bryant Ranch Prepack', '245.252.150.110'), ('Walter', 'Hernandez', 'whernandez1y@nbcnews.com', 'Virtus Pharmaceuticals LLC', '200.201.83.21'), ('Larry', 'Scott', 'lscott1z@quantcast.com', 'BIOKEY INC.', '122.141.109.98'), ('Gerald', 'Palmer', 'gpalmer20@usgs.gov', 'JAFRA COSMETICS INTERNATIONAL', '60.173.159.145'), ('Harry', 'Andrews', 'handrews21@alexa.com', 'NCS HealthCare of KY, Inc dba Vangard Labs', '210.64.37.91'), ('Jerry', 'Morrison', 'jmorrison22@drupal.org', 'Teva Pharmaceuticals USA Inc', '83.190.174.61'), ('Irene', 'Diaz', 'idiaz23@joomla.org', 'Dolgencorp, LLC', '214.16.44.235'), ('Brenda', 'Hansen', 'bhansen24@wisc.edu', 'REMEDYREPACK INC.', '167.231.200.232'), ('Carlos', 'Williamson', 'cwilliamson25@w3.org', 'Kroger Company', '251.202.210.204'), ('David', 'Fuller', 'dfuller26@canalblog.com', 'Supervalu Inc', '175.125.205.131'), ('Norma', 'Bishop', 'nbishop27@jugem.jp', 'Mylan Institutional Inc.', '208.162.25.149'), ('Brenda', 'Daniels', 'bdaniels28@mediafire.com', 'Space Brands Limited', '92.235.250.138'), ('Kathy', 'Reed', 'kreed29@prweb.com', 'Rugby Laboratories Inc.', '182.114.174.63'), ('Anthony', 'Long', 'along2a@dropbox.com', 'Fresenius Kabi USA, LLC', '160.146.121.173'), ('Craig', 'Palmer', 'cpalmer2b@desdev.cn', 'Bio-Pharm, Inc.', '135.77.134.24'), ('Rachel', 'Banks', 'rbanks2c@devhub.com', 'Sam''s West Inc', '35.72.5.193'), ('Kenneth', 'Peters', 'kpeters2d@ocn.ne.jp', 'International Labs, Inc.', '11.38.191.65'), ('Susan', 'Clark', 'sclark2e@ed.gov', 'Shionogi Inc.', '19.243.67.80'), ('Walter', 'Sullivan', 'wsullivan2f@vinaora.com', 'STAT Rx USA LLC', '154.137.170.227'), ('Kathleen', 'Wood', 'kwood2g@salon.com', 'Freds Inc', '155.54.131.149'), ('Phyllis', 'Henderson', 'phenderson2h@walmart.com', 'REMEDYREPACK INC.', '146.65.150.251'), ('Cheryl', 'Wells', 'cwells2i@gov.uk', 'Rebel Distributors Corp', '69.127.148.31'), ('Rose', 'Bradley', 'rbradley2j@un.org', 'Hi-Tech Pharmacal Co., Inc.', '150.101.165.102'), ('Aaron', 'Moreno', 'amoreno2k@tinypic.com', 'Pharmacia and Upjohn Company', '50.27.226.40'), ('Amy', 'Campbell', 'acampbell2l@auda.org.au', 'Chi Research, Inc.', '242.64.63.241'), ('Rebecca', 'Butler', 'rbutler2m@godaddy.com', 'Cardinal Health', '40.55.159.66'), ('Justin', 'Rodriguez', 'jrodriguez2n@meetup.com', 'Hikma Pharmaceutical', '118.9.132.156'), ('Donald', 'Nelson', 'dnelson2o@narod.ru', 'Nature''s Way Products, Inc.', '165.174.28.134'), ('Edward', 'Lawson', 'elawson2p@addtoany.com', 'Apotheca Company', '135.17.238.170'), ('Paul', 'Bell', 'pbell2q@simplemachines.org', 'Washington Homeopathic Products', '235.149.137.62'), ('Mark', 'Rose', 'mrose2r@google.pl', 'AMERICAN SALES COMPANY', '164.108.170.187'); ================================================ FILE: tests/functional/basic/test_basic.py ================================================ import pytest from dbt.tests.util import get_manifest, run_dbt my_model_sql = """ select 1 as fun """ @pytest.fixture(scope="class") def models(): return {"my_model.sql": my_model_sql} def test_basic(project): # Tests that a project with a single model works results = run_dbt(["run"]) assert len(results) == 1 manifest = get_manifest(project.project_root) assert "model.test.my_model" in manifest.nodes ================================================ FILE: tests/functional/basic/test_invalid_reference.py ================================================ import pytest from dbt.exceptions import CompilationError from dbt.tests.util import run_dbt descendant_sql = """ -- should be ref('model') select * from {{ ref(model) }} """ model_sql = """ select 1 as id """ @pytest.fixture(scope="class") def models(): return { "descendant.sql": descendant_sql, "model.sql": model_sql, } def test_undefined_value(project): # Tests that a project with an invalid reference fails with pytest.raises(CompilationError): run_dbt(["compile"]) ================================================ FILE: tests/functional/basic/test_jaffle_shop.py ================================================ from dbt.tests.util import get_manifest, run_dbt, run_dbt_and_capture, write_file from tests.fixtures.jaffle_shop import JaffleShopProject class TestBasic(JaffleShopProject): def test_basic(self, project): # test .dbtignore works write_file("models/ignore*.sql\nignore_folder", project.project_root, ".dbtignore") # Create the data from seeds results = run_dbt(["seed"]) # Tests that the jaffle_shop project runs results = run_dbt(["run"]) assert len(results) == 5 manifest = get_manifest(project.project_root) assert "model.jaffle_shop.orders" in manifest.nodes def test_execution_time_format_is_humanized(self, project): # Create the data from seeds run_dbt(["seed"]) _, log_output = run_dbt_and_capture(["run"]) assert " in 0 hours 0 minutes and " in log_output assert " seconds" in log_output ================================================ FILE: tests/functional/basic/test_mixed_case_db.py ================================================ import pytest from dbt.tests.util import get_manifest, run_dbt model_sql = """ select 1 as id """ @pytest.fixture(scope="class") def models(): return {"model.sql": model_sql} @pytest.fixture(scope="class") def dbt_profile_data(unique_schema): return { "test": { "outputs": { "default": { "type": "postgres", "threads": 4, "host": "localhost", "port": 5432, "user": "root", "pass": "password", "dbname": "dbtMixedCase", "schema": unique_schema, }, }, "target": "default", }, } def test_basic(project_root, project): assert project.database == "dbtMixedCase" # Tests that a project with a single model works results = run_dbt(["run"]) assert len(results) == 1 manifest = get_manifest(project_root) assert "model.test.model" in manifest.nodes # Running a second time works results = run_dbt(["run"]) ================================================ FILE: tests/functional/basic/test_project.py ================================================ import os from pathlib import Path import pytest import yaml from dbt.cli.main import dbtRunner from dbt.exceptions import DbtProjectError, ProjectContractError from dbt.tests.util import run_dbt, update_config_file, write_config_file simple_model_sql = """ select true as my_column """ simple_model_yml = """ models: - name: simple_model description: "is sythentic data ok? my column:" columns: - name: my_column description: asked and answered """ class TestSchemaYmlVersionMissing: @pytest.fixture(scope="class") def models(self): return {"simple_model.sql": simple_model_sql, "simple_model.yml": simple_model_yml} def test_empty_version(self, project): run_dbt(["run"], expect_pass=True) class TestProjectConfigVersionMissing: # default dbt_project.yml has config-version: 2 @pytest.fixture(scope="class") def project_config_remove(self): return ["config-version"] def test_empty_version(self, project): run_dbt(["run"], expect_pass=True) class TestProjectYamlVersionMissing: # default dbt_project.yml does not fill version def test_empty_version(self, project): run_dbt(["run"], expect_pass=True) class TestProjectYamlVersionValid: @pytest.fixture(scope="class") def project_config_update(self): return {"version": "1.0.0"} def test_valid_version(self, project): run_dbt(["run"], expect_pass=True) class TestProjectYamlVersionInvalid: def test_invalid_version(self, project): # we need to run it so the project gets set up first, otherwise we hit the semver error in setting up the test project run_dbt() update_config_file({"version": "invalid"}, "dbt_project.yml") with pytest.raises(ProjectContractError) as excinfo: run_dbt() assert "at path ['version']: 'invalid' is not valid under any of the given schemas" in str( excinfo.value ) class TestProjectDbtCloudConfig: @pytest.fixture(scope="class") def models(self): return {"simple_model.sql": simple_model_sql, "simple_model.yml": simple_model_yml} def test_dbt_cloud(self, project): run_dbt(["parse"], expect_pass=True) conf = yaml.safe_load( Path(os.path.join(project.project_root, "dbt_project.yml")).read_text() ) assert conf == { "name": "test", "profile": "test", "flags": {"send_anonymous_usage_stats": False}, } config = { "name": "test", "profile": "test", "flags": {"send_anonymous_usage_stats": False}, "dbt-cloud": { "account_id": "123", "application": "test", "environment": "test", "api_key": "test", }, } write_config_file(config, project.project_root, "dbt_project.yml") run_dbt(["parse"], expect_pass=True) conf = yaml.safe_load( Path(os.path.join(project.project_root, "dbt_project.yml")).read_text() ) assert conf == config class TestProjectDbtCloudConfigString: @pytest.fixture(scope="class") def models(self): return {"simple_model.sql": simple_model_sql, "simple_model.yml": simple_model_yml} def test_dbt_cloud_invalid(self, project): run_dbt() config = {"name": "test", "profile": "test", "dbt-cloud": "Some string"} update_config_file(config, "dbt_project.yml") expected_err = ( "at path ['dbt-cloud']: 'Some string' is not valid under any of the given schemas" ) with pytest.raises(ProjectContractError) as excinfo: run_dbt() assert expected_err in str(excinfo.value) class TestVersionSpecifierChecksComeBeforeYamlValidation: def test_version_specifier_checks_before_yaml_validation(self, project) -> None: runner = dbtRunner() # if no version specifier error, we should get a yaml validation error config_update = {"this-is-not-a-valid-key": "my-value-for-invalid-key"} update_config_file(config_update, "dbt_project.yml") result = runner.invoke(["parse"]) assert result.exception is not None assert isinstance(result.exception, ProjectContractError) assert "Additional properties are not allowed" in str(result.exception) # add bad version specifier, and assert we get the error for that update_config_file({"require-dbt-version": [">0.0.0", "<=0.0.1"]}, "dbt_project.yml") result = runner.invoke(["parse"]) assert result.exception is not None assert isinstance(result.exception, DbtProjectError) assert "This version of dbt is not supported" class TestArchiveNotAllowed: """At one point in time we supported an 'archive' key in projects, but no longer""" def test_archive_not_allowed(self, project): runner = dbtRunner() config_update = { "archive": { "source_schema": "a", "target_schema": "b", "tables": [ { "source_table": "seed", "target_table": "archive_actual", "updated_at": "updated_at", "unique_key": """id || '-' || first_name""", }, ], } } update_config_file(config_update, "dbt_project.yml") result = runner.invoke(["parse"]) assert result.exception is not None assert isinstance(result.exception, ProjectContractError) assert "Additional properties are not allowed" in str(result.exception) ================================================ FILE: tests/functional/basic/test_simple_reference.py ================================================ import pytest from dbt.tests.util import check_relations_equal, copy_file, read_file, run_dbt ephemeral_copy_sql = """ {{ config( materialized = "ephemeral" ) }} select * from {{ this.schema }}.users """ ephemeral_summary_sql = """ {{ config( materialized = "table" ) }} select gender, count(*) as ct from {{ref('ephemeral_copy')}} group by gender order by gender asc """ incremental_copy_sql = """ {{ config( materialized = "incremental" ) }} select * from {{ this.schema }}.users {% if is_incremental() %} where id > (select max(id) from {{this}}) {% endif %} """ incremental_summary_sql = """ {{ config( materialized = "table", ) }} select gender, count(*) as ct from {{ref('incremental_copy')}} group by gender order by gender asc """ materialized_copy_sql = """ {{ config( materialized = "table" ) }} select * from {{ this.schema }}.users """ materialized_summary_sql = """ {{ config( materialized = "table" ) }} select gender, count(*) as ct from {{ref('materialized_copy')}} group by gender order by gender asc """ view_copy_sql = """ {{ config( materialized = "view" ) }} select * from {{ this.schema }}.users """ view_summary_sql = """ {{ config( materialized = "view" ) }} select gender, count(*) as ct from {{ref('view_copy')}} group by gender order by gender asc """ view_using_ref_sql = """ {{ config( materialized = "view" ) }} select gender, count(*) as ct from {{ var('var_ref') }} group by gender order by gender asc """ properties_yml = """ version: 2 seeds: - name: summary_expected config: column_types: ct: BIGINT gender: text """ @pytest.fixture(scope="class") def models(): return { "ephemeral_copy.sql": ephemeral_copy_sql, "ephemeral_summary.sql": ephemeral_summary_sql, "incremental_copy.sql": incremental_copy_sql, "incremental_summary.sql": incremental_summary_sql, "materialized_copy.sql": materialized_copy_sql, "materialized_summary.sql": materialized_summary_sql, "view_copy.sql": view_copy_sql, "view_summary.sql": view_summary_sql, "view_using_ref.sql": view_using_ref_sql, } @pytest.fixture(scope="class") def seeds(test_data_dir): # Read seed file and return seeds = {"properties.yml": properties_yml} seed_csv = read_file(test_data_dir, "seed-initial.csv") seeds["users.csv"] = seed_csv summary_csv = read_file(test_data_dir, "summary_expected.csv") seeds["summary_expected.csv"] = summary_csv return seeds @pytest.fixture(scope="class") def project_config_update(): return { "vars": { "test": { "var_ref": '{{ ref("view_copy") }}', }, }, "seeds": {"quote_columns": False}, } # This test checks that with different materializations we get the right # tables copied or built. def test_simple_reference(project): results = run_dbt(["seed"]) assert len(results) == 2 # Now run dbt results = run_dbt() assert len(results) == 8 # Copies should match check_relations_equal( project.adapter, ["users", "incremental_copy", "materialized_copy", "view_copy"] ) # Summaries should match check_relations_equal( project.adapter, [ "summary_expected", "incremental_summary", "materialized_summary", "view_summary", "ephemeral_summary", "view_using_ref", ], ) # update the seed files and run seed copy_file( project.test_data_dir, "seed-update.csv", project.project_root, ["seeds", "users.csv"] ) copy_file( project.test_data_dir, "summary_expected_update.csv", project.project_root, ["seeds", "summary_expected.csv"], ) results = run_dbt(["seed"]) assert len(results) == 2 results = run_dbt() assert len(results) == 8 # Copies should match check_relations_equal( project.adapter, ["users", "incremental_copy", "materialized_copy", "view_copy"] ) # Summaries should match check_relations_equal( project.adapter, [ "summary_expected", "incremental_summary", "materialized_summary", "view_summary", "ephemeral_summary", "view_using_ref", ], ) def test_simple_reference_with_models_and_children(project): results = run_dbt(["seed"]) assert len(results) == 2 # Run materialized_copy, ephemeral_copy, and their dependents results = run_dbt(["run", "--models", "materialized_copy+", "ephemeral_copy+"]) assert len(results) == 3 # Copies should match check_relations_equal(project.adapter, ["users", "materialized_copy"]) # Summaries should match check_relations_equal( project.adapter, ["summary_expected", "materialized_summary", "ephemeral_summary"] ) created_tables = project.get_tables_in_schema() assert "incremental_copy" not in created_tables assert "incremental_summary" not in created_tables assert "view_copy" not in created_tables assert "view_summary" not in created_tables # make sure this wasn't errantly materialized assert "ephemeral_copy" not in created_tables assert "materialized_copy" in created_tables assert "materialized_summary" in created_tables assert created_tables["materialized_copy"] == "table" assert created_tables["materialized_summary"] == "table" assert "ephemeral_summary" in created_tables assert created_tables["ephemeral_summary"] == "table" def test_simple_ref_with_models(project): results = run_dbt(["seed"]) assert len(results) == 2 # Run materialized_copy, ephemeral_copy, and their dependents # ephemeral_copy should not actually be materialized b/c it is ephemeral results = run_dbt(["run", "--models", "materialized_copy", "ephemeral_copy"]) assert len(results) == 1 # Copies should match check_relations_equal(project.adapter, ["users", "materialized_copy"]) created_tables = project.get_tables_in_schema() assert "materialized_copy" in created_tables ================================================ FILE: tests/functional/basic/test_varchar_widening.py ================================================ import os import pytest from dbt.tests.util import check_relations_equal, run_dbt incremental_sql = """ {{ config( materialized = "incremental" ) }} select * from {{ this.schema }}.seed {% if is_incremental() %} where id > (select max(id) from {{this}}) {% endif %} """ materialized_sql = """ {{ config( materialized = "table" ) }} select * from {{ this.schema }}.seed """ @pytest.fixture(scope="class") def models(): return {"incremental.sql": incremental_sql, "materialized.sql": materialized_sql} def test_varchar_widening(project): path = os.path.join(project.test_data_dir, "varchar10_seed.sql") project.run_sql_file(path) results = run_dbt(["run"]) assert len(results) == 2 check_relations_equal(project.adapter, ["seed", "incremental"]) check_relations_equal(project.adapter, ["seed", "materialized"]) path = os.path.join(project.test_data_dir, "varchar300_seed.sql") project.run_sql_file(path) results = run_dbt(["run"]) assert len(results) == 2 check_relations_equal(project.adapter, ["seed", "incremental"]) check_relations_equal(project.adapter, ["seed", "materialized"]) ================================================ FILE: tests/functional/build_command/fixtures.py ================================================ seeds__country_csv = """iso3,name,iso2,iso_numeric,cow_alpha,cow_numeric,fao_code,un_code,wb_code,imf_code,fips,geonames_name,geonames_id,r_name,aiddata_name,aiddata_code,oecd_name,oecd_code,historical_name,historical_iso3,historical_iso2,historical_iso_numeric ABW,Aruba,AW,533,,,,533,ABW,314,AA,Aruba,3577279,ARUBA,Aruba,12,Aruba,373,,,, AFG,Afghanistan,AF,4,AFG,700,2,4,AFG,512,AF,Afghanistan,1149361,AFGHANISTAN,Afghanistan,1,Afghanistan,625,,,, AGO,Angola,AO,24,ANG,540,7,24,AGO,614,AO,Angola,3351879,ANGOLA,Angola,7,Angola,225,,,, AIA,Anguilla,AI,660,,,,660,AIA,312,AV,Anguilla,3573511,ANGUILLA,Anguilla,8,Anguilla,376,,,, ALA,Aland Islands,AX,248,,,,248,ALA,,,Aland Islands,661882,ALAND ISLANDS,,,,,,,, ALB,Albania,AL,8,ALB,339,3,8,ALB,914,AL,Albania,783754,ALBANIA,Albania,3,Albania,71,,,, AND,Andorra,AD,20,AND,232,6,20,ADO,,AN,Andorra,3041565,ANDORRA,,,,,,,, ANT,Netherlands Antilles,AN,530,,,,,ANT,353,NT,Netherlands Antilles,,NETHERLANDS ANTILLES,Netherlands Antilles,211,Netherlands Antilles,361,Netherlands Antilles,ANT,AN,530 ARE,United Arab Emirates,AE,784,UAE,696,225,784,ARE,466,AE,United Arab Emirates,290557,UNITED ARAB EMIRATES,United Arab Emirates,140,United Arab Emirates,576,,,, """ snapshots__snap_0 = """ {% snapshot snap_0 %} {{ config( target_database=database, target_schema=schema, unique_key='iso3', strategy='timestamp', updated_at='snap_0_updated_at', ) }} select *, current_timestamp as snap_0_updated_at from {{ ref('model_0') }} {% endsnapshot %} """ snapshots__snap_1 = """ {% snapshot snap_1 %} {{ config( target_database=database, target_schema=schema, unique_key='iso3', strategy='timestamp', updated_at='snap_1_updated_at', ) }} SELECT iso3, name, iso2, iso_numeric, cow_alpha, cow_numeric, fao_code, un_code, wb_code, imf_code, fips, geonames_name, geonames_id, r_name, aiddata_name, aiddata_code, oecd_name, oecd_code, historical_name, historical_iso3, historical_iso2, historical_iso_numeric, current_timestamp as snap_1_updated_at from {{ ref('model_1') }} {% endsnapshot %} """ snapshots__snap_99 = """ {% snapshot snap_99 %} {{ config( target_database=database, target_schema=schema, strategy='timestamp', unique_key='num', updated_at='snap_99_updated_at', ) }} select *, current_timestamp as snap_99_updated_at from {{ ref('model_99') }} {% endsnapshot %} """ models__model_0_sql = """ {{ config(materialized='table') }} select * from {{ ref('countries') }} """ models__model_1_sql = """ {{ config(materialized='table') }} select * from {{ ref('snap_0') }} """ models__model_2_sql = """ {{ config(materialized='table') }} select * from {{ ref('snap_1') }} """ models__model_3_sql = """ {{ config(materialized='table') }} select * from {{ ref('model_1') }} """ models__model_99_sql = """ {{ config(materialized='table') }} select '1' as "num" """ models__test_yml = """ version: 2 models: - name: model_0 columns: - name: iso3 data_tests: - unique - not_null - name: model_2 columns: - name: iso3 data_tests: - unique - not_null """ unit_tests__yml = """ unit_tests: - name: ut_model_3 model: model_3 given: - input: ref('model_1') rows: - {iso3: ABW, name: Aruba} expect: rows: - {iso3: ABW, name: Aruba} """ models_failing_tests__tests_yml = """ version: 2 models: - name: model_0 columns: - name: iso3 data_tests: - unique - not_null - name: historical_iso_numeric data_tests: - not_null - name: model_2 columns: - name: iso3 data_tests: - unique - not_null """ models_failing__model_1_sql = """ {{ config(materialized='table') }} select bad_column from {{ ref('snap_0') }} """ models_circular_relationship__test_yml = """ version: 2 models: - name: model_0 columns: - name: iso3 data_tests: - relationships: to: ref('model_1') field: iso3 - name: model_1 columns: - name: iso3 data_tests: - relationships: to: ref('model_0') field: iso3 """ models_simple_blocking__model_a_sql = """ select null as id """ models_simple_blocking__model_b_sql = """ select * from {{ ref('model_a') }} """ models_simple_blocking__test_yml = """ version: 2 models: - name: model_a columns: - name: id data_tests: - not_null """ models_triple_blocking__test_yml = """ version: 2 models: - name: model_a columns: - name: id data_tests: - not_null - name: model_b columns: - name: id data_tests: - not_null - name: model_c columns: - name: id data_tests: - not_null """ models_interdependent__model_a_sql = """ select 1 as id """ models_interdependent__model_b_sql = """ select * from {{ ref('model_a') }} """ models_interdependent__model_b_null_sql = """ select null from {{ ref('model_a') }} """ models_interdependent__model_c_sql = """ select * from {{ ref('model_b') }} """ models_interdependent__test_yml = """ version: 2 models: - name: model_a columns: - name: id data_tests: - unique - not_null - relationships: to: ref('model_b') field: id - relationships: to: ref('model_c') field: id - name: model_b columns: - name: id data_tests: - unique - not_null - relationships: to: ref('model_a') field: id - relationships: to: ref('model_c') field: id - name: model_c columns: - name: id data_tests: - unique - not_null - relationships: to: ref('model_a') field: id - relationships: to: ref('model_b') field: id """ ================================================ FILE: tests/functional/build_command/test_build.py ================================================ import pytest from dbt.tests.util import run_dbt from tests.functional.build_command.fixtures import ( models__model_0_sql, models__model_1_sql, models__model_2_sql, models__model_3_sql, models__model_99_sql, models__test_yml, models_circular_relationship__test_yml, models_failing__model_1_sql, models_failing_tests__tests_yml, models_interdependent__model_a_sql, models_interdependent__model_b_null_sql, models_interdependent__model_b_sql, models_interdependent__model_c_sql, models_interdependent__test_yml, models_simple_blocking__model_a_sql, models_simple_blocking__model_b_sql, models_simple_blocking__test_yml, models_triple_blocking__test_yml, seeds__country_csv, snapshots__snap_0, snapshots__snap_1, snapshots__snap_99, unit_tests__yml, ) class TestBuildBase: @pytest.fixture(scope="class") def seeds(self): return {"countries.csv": seeds__country_csv} @pytest.fixture(scope="class") def snapshots(self): return { "snap_0.sql": snapshots__snap_0, "snap_1.sql": snapshots__snap_1, "snap_99.sql": snapshots__snap_99, } @pytest.fixture(scope="class") def project_config_update(self): return { "seeds": { "quote_columns": False, }, } class TestPassingBuild(TestBuildBase): @pytest.fixture(scope="class") def models(self): return { "model_0.sql": models__model_0_sql, "model_1.sql": models__model_1_sql, "model_2.sql": models__model_2_sql, "model_3.sql": models__model_3_sql, "model_99.sql": models__model_99_sql, "test.yml": models__test_yml + unit_tests__yml, } def test_build_happy_path(self, project): run_dbt(["build"]) class TestFailingBuild(TestBuildBase): @pytest.fixture(scope="class") def models(self): return { "model_0.sql": models__model_0_sql, "model_1.sql": models_failing__model_1_sql, "model_2.sql": models__model_2_sql, "model_3.sql": models__model_3_sql, "model_99.sql": models__model_99_sql, "test.yml": models__test_yml + unit_tests__yml, } def test_failing_test_skips_downstream(self, project): results = run_dbt(["build"], expect_pass=False) assert len(results) == 14 actual = [str(r.status) for r in results] expected = ["error"] * 1 + ["skipped"] * 6 + ["pass"] * 2 + ["success"] * 5 assert sorted(actual) == sorted(expected) class TestFailingTestsBuild(TestBuildBase): @pytest.fixture(scope="class") def models(self): return { "model_0.sql": models__model_0_sql, "model_1.sql": models__model_1_sql, "model_2.sql": models__model_2_sql, "model_99.sql": models__model_99_sql, "test.yml": models_failing_tests__tests_yml, } def test_failing_test_skips_downstream(self, project): results = run_dbt(["build"], expect_pass=False) assert len(results) == 13 actual = [str(r.status) for r in results] expected = ["fail"] + ["skipped"] * 6 + ["pass"] * 2 + ["success"] * 4 assert sorted(actual) == sorted(expected) class TestCircularRelationshipTestsBuild(TestBuildBase): @pytest.fixture(scope="class") def models(self): return { "model_0.sql": models__model_0_sql, "model_1.sql": models__model_1_sql, "model_99.sql": models__model_99_sql, "test.yml": models_circular_relationship__test_yml, } def test_circular_relationship_test_success(self, project): """Ensure that tests that refer to each other's model don't create a circular dependency.""" results = run_dbt(["build"]) actual = [str(r.status) for r in results] expected = ["success"] * 7 + ["pass"] * 2 assert sorted(actual) == sorted(expected) class TestSimpleBlockingTest: @pytest.fixture(scope="class") def models(self): return { "model_a.sql": models_simple_blocking__model_a_sql, "model_b.sql": models_simple_blocking__model_b_sql, "test.yml": models_simple_blocking__test_yml, } def test_simple_blocking_test(self, project): """Ensure that a failed test on model_a always blocks model_b""" results = run_dbt(["build"], expect_pass=False) actual = [r.status for r in results] expected = ["success", "fail", "skipped"] assert sorted(actual) == sorted(expected) class TestInterdependentModels: @pytest.fixture(scope="class") def seeds(self): return {"countries.csv": seeds__country_csv} @pytest.fixture(scope="class") def project_config_update(self): return { "seeds": { "quote_columns": False, }, } @pytest.fixture(scope="class") def models(self): return { "model_a.sql": models_interdependent__model_a_sql, "model_b.sql": models_interdependent__model_b_sql, "model_c.sql": models_interdependent__model_c_sql, "test.yml": models_interdependent__test_yml, } def test_interdependent_models(self, project): results = run_dbt(["build"]) assert len(results) == 16 class TestInterdependentModelsFail: @pytest.fixture(scope="class") def seeds(self): return {"countries.csv": seeds__country_csv} @pytest.fixture(scope="class") def project_config_update(self): return { "seeds": { "quote_columns": False, }, } @pytest.fixture(scope="class") def models(self): return { "model_a.sql": models_interdependent__model_a_sql, "model_b.sql": models_interdependent__model_b_null_sql, "model_c.sql": models_interdependent__model_c_sql, "test.yml": models_interdependent__test_yml, } def test_interdependent_models_fail(self, project): results = run_dbt(["build"], expect_pass=False) assert len(results) == 16 actual = [str(r.status) for r in results] expected = ["error"] * 4 + ["skipped"] * 7 + ["pass"] * 2 + ["success"] * 3 assert sorted(actual) == sorted(expected) class TestDownstreamSelection: @pytest.fixture(scope="class") def models(self): return { "model_a.sql": models_simple_blocking__model_a_sql, "model_b.sql": models_simple_blocking__model_b_sql, "test.yml": models_simple_blocking__test_yml, } def test_downstream_selection(self, project): """Ensure that selecting test+ does not select model_a's other children""" # fails with "Got 1 result, configured to fail if != 0" # model_a is defined as select null as id results = run_dbt(["build", "--select", "model_a not_null_model_a_id+"], expect_pass=False) assert len(results) == 2 class TestLimitedUpstreamSelection: @pytest.fixture(scope="class") def models(self): return { "model_a.sql": models_interdependent__model_a_sql, "model_b.sql": models_interdependent__model_b_sql, "model_c.sql": models_interdependent__model_c_sql, "test.yml": models_triple_blocking__test_yml, } def test_limited_upstream_selection(self, project): """Ensure that selecting 1+model_c only selects up to model_b (+ tests of both)""" # Fails with "relation "test17005969872609282880_test_build.model_a" does not exist" results = run_dbt(["build", "--select", "1+model_c"], expect_pass=False) assert len(results) == 4 ================================================ FILE: tests/functional/catalogs/test_catalogs_parsing.py ================================================ from unittest import mock import pytest from dbt.adapters.catalogs import CatalogIntegration, CatalogIntegrationConfig from dbt.tests.util import run_dbt, write_config_file from dbt_common.exceptions import DbtValidationError write_integration_1 = { "name": "write_integration_1", "external_volume": "write_external_volume", "table_format": "write_format", "catalog_type": "write", "adapter_properties": {"my_custom_property": "foo_1"}, } write_integration_2 = { "name": "write_integration_2", "external_volume": "write_external_volume", "table_format": "write_format", "catalog_type": "write", "adapter_properties": {"my_custom_property": "foo_2"}, } class WriteCatalogIntegration(CatalogIntegration): catalog_type = "write" allows_writes = True def __init__(self, config: CatalogIntegrationConfig): super().__init__(config) for key, value in config.adapter_properties.items(): setattr(self, key, value) class TestSingleWriteIntegration: @pytest.fixture def catalogs(self): return { "catalogs": [ {"name": "write_catalog_1", "write_integrations": [write_integration_1]}, {"name": "write_catalog_2", "write_integrations": [write_integration_2]}, ] } def test_integration(self, project, catalogs, adapter): write_config_file(catalogs, project.project_root, "catalogs.yml") with mock.patch.object( type(project.adapter), "CATALOG_INTEGRATIONS", [WriteCatalogIntegration] ): run_dbt(["run"]) for i in range(1, 3): write_integration = project.adapter.get_catalog_integration(f"write_catalog_{i}") assert isinstance(write_integration, WriteCatalogIntegration) assert write_integration.name == f"write_catalog_{i}" assert write_integration.catalog_type == "write" assert write_integration.catalog_name == f"write_integration_{i}" assert write_integration.table_format == "write_format" assert write_integration.external_volume == "write_external_volume" assert write_integration.allows_writes is True assert write_integration.my_custom_property == f"foo_{i}" class TestMultipleWriteIntegration: @pytest.fixture def catalogs(self): return { "catalogs": [ { "name": "write_catalog", "write_integrations": [write_integration_1, write_integration_2], "active_write_integration": "write_integration_2", }, ] } def test_integration(self, project, catalogs, adapter): write_config_file(catalogs, project.project_root, "catalogs.yml") with mock.patch.object( type(project.adapter), "CATALOG_INTEGRATIONS", [WriteCatalogIntegration] ): run_dbt(["build"]) write_integration = project.adapter.get_catalog_integration("write_catalog") assert write_integration.name == "write_catalog" assert write_integration.catalog_name == "write_integration_2" assert write_integration.my_custom_property == "foo_2" class TestNoActiveWriteIntegration: @pytest.fixture def catalogs(self): return { "catalogs": [ { "name": "write_catalog", "write_integrations": [write_integration_1, write_integration_2], }, ] } def test_integration(self, project, catalogs, adapter): write_config_file(catalogs, project.project_root, "catalogs.yml") with mock.patch.object( type(project.adapter), "CATALOG_INTEGRATIONS", [WriteCatalogIntegration] ): error_msg = "Catalog 'write_catalog' must specify an 'active_write_integration' when multiple 'write_integrations' are provided." with pytest.raises(DbtValidationError, match=error_msg): run_dbt(["run"]) class TestInvalidWriteIntegration: @pytest.fixture def catalogs(self): return { "catalogs": [ { "name": "write_catalog", "write_integrations": [write_integration_1, write_integration_2], "active_write_integration": "write_integration_3", }, ] } def test_integration(self, project, catalogs, adapter): write_config_file(catalogs, project.project_root, "catalogs.yml") with mock.patch.object( type(project.adapter), "CATALOG_INTEGRATIONS", [WriteCatalogIntegration] ): error_msg = "Catalog 'write_catalog' must specify an 'active_write_integration' from its set of defined 'write_integrations'" with pytest.raises(DbtValidationError, match=error_msg): run_dbt(["run"]) class TestDuplicateWriteIntegration: @pytest.fixture def catalogs(self): return { "catalogs": [ { "name": "write_catalog", "write_integrations": [write_integration_1, write_integration_1], "active_write_integration": "write_integration_1", }, ] } def test_integration(self, project, catalogs, adapter): write_config_file(catalogs, project.project_root, "catalogs.yml") with mock.patch.object( type(project.adapter), "CATALOG_INTEGRATIONS", [WriteCatalogIntegration] ): error_msg = "Catalog 'write_catalog' cannot have multiple 'write_integrations' with the same name: 'write_integration_1'." with pytest.raises(DbtValidationError, match=error_msg): run_dbt(["run"]) ================================================ FILE: tests/functional/clean/test_clean.py ================================================ from pathlib import Path import pytest from dbt.exceptions import DbtRuntimeError from dbt.tests.util import run_dbt from tests.functional.utils import up_one class TestCleanSourcePath: @pytest.fixture(scope="class") def project_config_update(self): return "clean-targets: ['models']" def test_clean_source_path(self, project): with pytest.raises(DbtRuntimeError, match="dbt will not clean the following source paths"): run_dbt(["clean"]) class TestCleanPathOutsideProjectRelative: @pytest.fixture(scope="class") def project_config_update(self): return "clean-targets: ['..']" def test_clean_path_outside_project(self, project): with pytest.raises( DbtRuntimeError, match="dbt will not clean the following directories outside the project", ): run_dbt(["clean"]) class TestCleanPathOutsideProjectAbsolute: @pytest.fixture(scope="class") def project_config_update(self): return "clean-targets: ['/']" def test_clean_path_outside_project(self, project): with pytest.raises( DbtRuntimeError, match="dbt will not clean the following directories outside the project", ): run_dbt(["clean"]) class TestCleanPathOutsideProjectWithFlag: @pytest.fixture(scope="class") def project_config_update(self): return "clean-targets: ['/tmp/foo']" def test_clean_path_outside_project(self, project): # Doesn't fail because flag is set run_dbt(["clean", "--no-clean-project-files-only"]) with pytest.raises( DbtRuntimeError, match="dbt will not clean the following directories outside the project", ): run_dbt(["clean", "--clean-project-files-only"]) class TestCleanRelativeProjectDir: def test_clean_relative_project_dir(self, project): with up_one(): project_dir = Path(project.project_root).relative_to(Path.cwd()) run_dbt(["clean", "--project-dir", str(project_dir)]) ================================================ FILE: tests/functional/cli/test_cli_exit_codes.py ================================================ import pytest from dbt.cli.exceptions import ResultExit from dbt.cli.main import cli good_sql = """ select 1 as fun """ bad_sql = """ someting bad """ class CliRunnerBase: def run_cli(self): ctx = cli.make_context(cli.name, ["run"]) return cli.invoke(ctx) class TestExitCodeZero(CliRunnerBase): @pytest.fixture(scope="class") def models(self): return {"model_one.sql": good_sql} def test_no_exc_thrown(self, project): self.run_cli() class TestExitCodeOne(CliRunnerBase): @pytest.fixture(scope="class") def models(self): return {"model_one.sql": bad_sql} def test_exc_thrown(self, project): with pytest.raises(ResultExit): self.run_cli() ================================================ FILE: tests/functional/cli/test_click_flags.py ================================================ import os from unittest import mock import pytest from dbt.exceptions import DbtProjectError from dbt.tests.util import run_dbt, update_config_file class TestClickCLIFlagsResolveTruthy: def test_resolve_truthy(self, project): # we can't do this in a fixture because the project will error out before the test can run update_config_file({"require-dbt-version": "0.0.0"}, "dbt_project.yml") with pytest.raises(DbtProjectError): run_dbt(["parse", "--version-check"]) class TestClickEnvVarFlagsResolveTruthy: @pytest.mark.parametrize("env_var_value", ["yes", "y", "true", "t", "on", "1"]) def test_resolve_truthy(self, project, env_var_value: str): # we can't do this in a fixture because the project will error out before the test can run update_config_file({"require-dbt-version": "0.0.0"}, "dbt_project.yml") with mock.patch.dict(os.environ, {"DBT_VERSION_CHECK": env_var_value}): # should raise with pytest.raises(DbtProjectError): run_dbt(["parse"]) class TestClickCLIFlagsResolveFalsey: def test_resolve_falsey(self, project): # we can't do this in a fixture because the project will error out before the test can run update_config_file({"require-dbt-version": "0.0.0"}, "dbt_project.yml") # shouldn't raise run_dbt(["parse", "--no-version-check"]) class TestClickEnvVarFlagsResolveFalsey: @pytest.mark.parametrize("env_var_value", ["no", "n", "false", "f", "off", "0"]) def test_resolve_falsey(self, project, env_var_value: str): # we can't do this in a fixture because the project will error out before the test can run update_config_file({"require-dbt-version": "0.0.0"}, "dbt_project.yml") with mock.patch.dict(os.environ, {"DBT_VERSION_CHECK": env_var_value}): # shouldn't raise run_dbt(["parse"]) ================================================ FILE: tests/functional/cli/test_env_var_deprecations.py ================================================ import os import pytest from dbt.tests.util import read_file, run_dbt model_one_sql = """ select 1 as fun """ class TestDeprecatedEnvVars: @pytest.fixture(scope="class") def models(self): return {"model_one.sql": model_one_sql} def test_defer(self, project, logs_dir): self.assert_deprecated( logs_dir, "DBT_DEFER_TO_STATE", "DBT_ENGINE_DEFER", ) def test_favor_state(self, project, logs_dir): self.assert_deprecated( logs_dir, "DBT_FAVOR_STATE_MODE", "DBT_ENGINE_FAVOR_STATE", command="build", ) def test_print(self, project, logs_dir): self.assert_deprecated( logs_dir, "DBT_NO_PRINT", "DBT_ENGINE_PRINT", ) def test_state(self, project, logs_dir): self.assert_deprecated( logs_dir, "DBT_ARTIFACT_STATE_PATH", "DBT_ENGINE_STATE", old_val=".", ) def assert_deprecated(self, logs_dir, old_env_var, new_env_var, command="run", old_val="0"): os.environ[old_env_var] = old_val run_dbt([command]) # replacing new lines with spaces accounts for text wrapping log_file = read_file(logs_dir, "dbt.log").replace("\n", " ").replace("\\n", " ") dep_str = f"The environment variable `{old_env_var}` has been renamed as `{new_env_var}`" try: assert dep_str in log_file except Exception as e: del os.environ[old_env_var] raise e del os.environ[old_env_var] ================================================ FILE: tests/functional/cli/test_error_handling.py ================================================ import pytest from dbt.tests.util import run_dbt model_one_sql = """ someting bad """ class TestHandledExit: @pytest.fixture(scope="class") def models(self): return {"model_one.sql": model_one_sql} def test_failed_run_does_not_throw(self, project): run_dbt(["run"], expect_pass=False) def test_fail_fast_failed_run_does_not_throw(self, project): run_dbt(["--fail-fast", "run"], expect_pass=False) ================================================ FILE: tests/functional/cli/test_multioption.py ================================================ import json import pytest from dbt.tests.util import run_dbt model_one_sql = """ select 1 as fun """ model_with_materialization_sql = """ {{ config(materialized='table') }} select 1 as fun """ model_with_meta_sql = """ {{ config( materialized='incremental', meta={'owner': 'data-team', 'criticality': 'high'} ) }} select 2 as id, 'meta_test' as name """ schema_sql = """ sources: - name: my_source description: "My source" schema: test_schema tables: - name: my_table - name: my_other_table exposures: - name: weekly_jaffle_metrics label: By the Week type: dashboard maturity: high url: https://bi.tool/dashboards/1 description: > Did someone say "exponential growth"? depends_on: - ref('model_one') owner: name: dbt Labs email: data@jaffleshop.com """ class TestResourceType: @pytest.fixture(scope="class") def models(self): return {"schema.yml": schema_sql, "model_one.sql": model_one_sql} def test_resource_type_single(self, project): result = run_dbt(["-q", "ls", "--resource-types", "model"]) assert len(result) == 1 assert result == ["test.model_one"] def test_resource_type_quoted(self, project): result = run_dbt(["-q", "ls", "--resource-types", "model source"]) assert len(result) == 3 expected_result = { "test.model_one", "source:test.my_source.my_table", "source:test.my_source.my_other_table", } assert set(result) == expected_result def test_resource_type_args(self, project): result = run_dbt( [ "-q", "ls", "--resource-type", "model", "--resource-type", "source", "--resource-type", "exposure", ] ) assert len(result) == 4 expected_result = { "test.model_one", "source:test.my_source.my_table", "source:test.my_source.my_other_table", "exposure:test.weekly_jaffle_metrics", } assert set(result) == expected_result class TestOutputKeys: @pytest.fixture(scope="class") def models(self): return { "model_one.sql": model_one_sql, "model_table.sql": model_with_materialization_sql, "model_meta.sql": model_with_meta_sql, } def test_output_key_single(self, project): result = run_dbt(["-q", "ls", "--output", "json", "--output-keys", "name"]) assert len(result) == 3 expected_names = ["model_one", "model_table", "model_meta"] actual_names = [json.loads(r)["name"] for r in result] assert set(actual_names) == set(expected_names) def test_output_key_quoted(self, project): result = run_dbt(["-q", "ls", "--output", "json", "--output-keys", "name resource_type"]) assert len(result) == 3 # All should be models with names for r in result: result_json = json.loads(r) assert result_json["resource_type"] == "model" assert "name" in result_json def test_output_key_args(self, project): result = run_dbt( [ "-q", "ls", "--output", "json", "--output-keys", "name", "--output-keys", "resource_type", ] ) assert len(result) == 3 # All should be models with names for r in result: result_json = json.loads(r) assert result_json["resource_type"] == "model" assert "name" in result_json def test_output_key_nested(self, project): result = run_dbt( [ "-q", "ls", "--output", "json", "--output-keys", "name", "--output-keys", "config.materialized", "--select", "model_table", ] ) assert len(result) == 1 import json result_json = json.loads(result[0]) assert result_json["name"] == "model_table" assert result_json["config.materialized"] == "table" def test_output_key_nested_single_arg(self, project): result = run_dbt( [ "-q", "ls", "--output", "json", "--output-keys", "config.materialized name", "--select", "model_table", ] ) assert len(result) == 1 import json result_json = json.loads(result[0]) assert result_json["name"] == "model_table" assert result_json["config.materialized"] == "table" def test_output_key_nested_nonexistent(self, project): """Test that non-existent nested keys return empty objects""" result = run_dbt( [ "-q", "ls", "--output", "json", "--output-keys", "config.nonexistent", "--select", "model_table", ] ) assert len(result) == 1 import json result_json = json.loads(result[0]) assert result_json == {} # Non-existent key should result in empty object def test_output_key_nested_mixed_existent_nonexistent(self, project): """Test mixing existent and non-existent nested keys""" result = run_dbt( [ "-q", "ls", "--output", "json", "--output-keys", "name", "--output-keys", "config.materialized", "--output-keys", "config.nonexistent", "--select", "model_table", ] ) assert len(result) == 1 import json result_json = json.loads(result[0]) assert result_json["name"] == "model_table" assert result_json["config.materialized"] == "table" # Non-existent key should not appear in result assert "config.nonexistent" not in result_json def test_output_key_nested_deep_nonexistent(self, project): """Test deeply nested non-existent keys""" result = run_dbt( [ "-q", "ls", "--output", "json", "--output-keys", "config.meta.owner.nonexistent", "--select", "model_table", ] ) assert len(result) == 1 import json result_json = json.loads(result[0]) assert result_json == {} # Deep non-existent key should result in empty object def test_output_key_nested_deep_meta(self, project): """Test deeply nested meta keys that exist""" result = run_dbt( [ "-q", "ls", "--output", "json", "--output-keys", "name", "--output-keys", "config.meta.owner", "--select", "model_meta", ] ) assert len(result) == 1 result_json = json.loads(result[0]) assert result_json["name"] == "model_meta" assert result_json["config.meta.owner"] == "data-team" def test_output_key_nested_whole_meta_object(self, project): """Test getting the whole meta object as nested key""" result = run_dbt( [ "-q", "ls", "--output", "json", "--output-keys", "config.meta", "--select", "model_meta", ] ) assert len(result) == 1 result_json = json.loads(result[0]) expected_meta = {"owner": "data-team", "criticality": "high"} assert result_json["config.meta"] == expected_meta class TestSelectExclude: @pytest.fixture(scope="class") def models(self): return { "model_one.sql": model_one_sql, "model_two.sql": model_one_sql, "model_three.sql": model_one_sql, } def test_select_exclude_single(self, project): result = run_dbt(["-q", "ls", "--select", "model_one"]) assert len(result) == 1 assert result == ["test.model_one"] result = run_dbt(["-q", "ls", "--exclude", "model_one"]) assert len(result) == 2 assert "test.model_one" not in result def test_select_exclude_quoted(self, project): result = run_dbt(["-q", "ls", "--select", "model_one model_two"]) assert len(result) == 2 assert "test.model_three" not in result result = run_dbt(["-q", "ls", "--exclude", "model_one model_two"]) assert len(result) == 1 assert result == ["test.model_three"] def test_select_exclude_args(self, project): result = run_dbt(["-q", "ls", "--select", "model_one", "--select", "model_two"]) assert len(result) == 2 assert "test.model_three" not in result result = run_dbt(["-q", "ls", "--exclude", "model_one", "--exclude", "model_two"]) assert len(result) == 1 assert result == ["test.model_three"] ================================================ FILE: tests/functional/cli/test_option_interaction_validations.py ================================================ import pytest from dbt.tests.util import run_dbt class TestEventTimeEndEventTimeStart: @pytest.mark.parametrize( "event_time_start,event_time_end,expect_pass", [ ("2024-10-01", "2024-10-02", True), ("2024-10-02", "2024-10-01", False), ], ) def test_option_combo(self, project, event_time_start, event_time_end, expect_pass): try: run_dbt( [ "build", "--event-time-start", event_time_start, "--event-time-end", event_time_end, ] ) assert expect_pass except Exception as e: assert ( "Value for `--event-time-start` must be less than `--event-time-end`" in e.__str__() ) assert not expect_pass class TestEventTimeEndEventTimeStartMutuallyRequired: @pytest.mark.parametrize( "specified,missing", [ ("--event-time-start", "--event-time-end"), ("--event-time-end", "--event-time-start"), ], ) def test_option_combo(self, project, specified, missing): try: run_dbt(["build", specified, "2024-10-01"]) assert False, f"An error should have been raised for missing `{missing}` flag" except Exception as e: assert ( f"When specifying `{specified}`, `{missing}` must also be present." in e.__str__() ) ================================================ FILE: tests/functional/cli/test_requires.py ================================================ import os import pytest from pytest_mock import MockerFixture from dbt.events.types import JinjaLogInfo, PartialParsingNotEnabled from dbt.tests.util import run_dbt from dbt_common.events.event_catcher import EventCatcher model_one_sql = """ {{ log("DBT_ENGINE_SHOW_RESOURCE_REPORT: " ~ env_var('DBT_ENGINE_SHOW_RESOURCE_REPORT', default="0"), info=True) }} {{ log("DBT_SHOW_RESOURCE_REPORT: " ~ env_var('DBT_SHOW_RESOURCE_REPORT', default="0"), info=True) }} select 1 as fun """ class TestOldEngineEnvVarPropagation: @pytest.fixture(scope="class") def models(self): return {"model_one.sql": model_one_sql} @pytest.mark.parametrize( "set_old,set_new, expect", [(False, False, 0), (True, False, False), (False, True, True), (True, True, True)], ) def test_engine_env_var_propagation( self, project, mocker: MockerFixture, set_old: bool, set_new: bool, expect: bool ): # Of note, the default value for DBT_PARTIAL_PARSE is True if set_old: mocker.patch.dict(os.environ, {"DBT_SHOW_RESOURCE_REPORT": "False"}) if set_new: mocker.patch.dict(os.environ, {"DBT_ENGINE_SHOW_RESOURCE_REPORT": "True"}) event_catcher = EventCatcher(event_to_catch=JinjaLogInfo) run_dbt(["parse", "--no-partial-parse"], callbacks=[event_catcher.catch]) assert len(event_catcher.caught_events) == 2 for event in event_catcher.caught_events: if event.data.msg.startswith("DBT_ENGINE_SHOW_RESOURCE_REPORT"): assert event.data.msg.endswith(f"{expect}") elif event.data.msg.startswith("DBT_SHOW_RESOURCE_REPORT"): assert event.data.msg.endswith(f"{expect}") else: assert False, "Unexpected log message" class TestEngineEnvVarPickedUpByClick: @pytest.fixture(scope="class") def models(self): return {"model_one.sql": model_one_sql} def test_engine_env_var_picked_up_by_cli_flags(self, project, mocker: MockerFixture): event_catcher = EventCatcher(event_to_catch=PartialParsingNotEnabled) run_dbt(["parse"], callbacks=[event_catcher.catch]) assert len(event_catcher.caught_events) == 0 run_dbt(["parse"], callbacks=[event_catcher.catch]) assert len(event_catcher.caught_events) == 0 mocker.patch.dict(os.environ, {"DBT_ENGINE_PARTIAL_PARSE": "False"}) run_dbt(["parse"], callbacks=[event_catcher.catch]) assert len(event_catcher.caught_events) == 1 class TestKnownEngineEnvVarsExplicit: def test_allow_list_is_correct(self, project): run_dbt(["parse"]) hard_coded_allow_list = { "DBT_ENGINE_NO_PRINT", "DBT_ENGINE_EXCLUDE_RESOURCE_TYPES", "DBT_ENGINE_TARGET", "DBT_ENGINE_PROJECT_DIR", "DBT_ENGINE_MACRO_DEBUGGING", "DBT_ENGINE_EVENT_TIME_END", "DBT_ENGINE_PRINTER_WIDTH", "DBT_ENGINE_PACKAGE_HUB_URL", "DBT_ENGINE_TARGET_PATH", "DBT_ENGINE_EMPTY", "DBT_ENGINE_DOWNLOAD_DIR", "DBT_ENGINE_INDIRECT_SELECTION", "DBT_ENGINE_SHOW_RESOURCE_REPORT", "DBT_ENGINE_EVENT_TIME_START", "DBT_ENGINE_LOG_CACHE_EVENTS", "DBT_ENGINE_USE_COLORS_FILE", "DBT_ENGINE_LOG_FILE_MAX_BYTES", "DBT_ENGINE_DEFER_TO_STATE", "DBT_ENGINE_RESOURCE_TYPES", "DBT_ENGINE_HOST", "DBT_ENGINE_STATE", "DBT_ENGINE_PP_FILE_DIFF_TEST", "DBT_ENGINE_STORE_FAILURES", "DBT_ENGINE_LOG_PATH", "DBT_ENGINE_EXPORT_SAVED_QUERIES", "DBT_ENGINE_CLEAN_PROJECT_FILES_ONLY", "DBT_ENGINE_CACHE_SELECTED_ONLY", "DBT_ENGINE_WRITE_JSON", "DBT_ENGINE_SEND_ANONYMOUS_USAGE_STATS", "DBT_ENGINE_RECORDED_FILE_PATH", "DBT_ENGINE_PARTIAL_PARSE_FILE_DIFF", "DBT_ENGINE_PP_TEST", "DBT_ENGINE_INTROSPECT", "DBT_ENGINE_USE_FAST_TEST_EDGES", "DBT_ENGINE_VERSION_CHECK", "DBT_ENGINE_QUIET", "DBT_ENGINE_SINGLE_THREADED", "DBT_ENGINE_SQLPARSE", "DBT_ENGINE_ARTIFACT_STATE_PATH", "DBT_ENGINE_FULL_REFRESH", "DBT_ENGINE_FAIL_FAST", "DBT_ENGINE_INCLUDE_SAVED_QUERY", "DBT_ENGINE_WARN_ERROR", "DBT_ENGINE_PROFILES_DIR", "DBT_ENGINE_LOG_LEVEL", "DBT_ENGINE_STATIC_PARSER", "DBT_ENGINE_PROFILE", "DBT_ENGINE_PARTIAL_PARSE", "DBT_ENGINE_POPULATE_CACHE", "DBT_ENGINE_DEFER", "DBT_ENGINE_FAVOR_STATE_MODE", "DBT_ENGINE_USE_COLORS", "DBT_ENGINE_DEFER_STATE", "DBT_ENGINE_LOG_FORMAT", "DBT_ENGINE_PARTIAL_PARSE_FILE_PATH", "DBT_ENGINE_WARN_ERROR_OPTIONS", "DBT_ENGINE_INVOCATION_ENV", "DBT_ENGINE_FAVOR_STATE", "DBT_ENGINE_LOG_FORMAT_FILE", "DBT_ENGINE_TEST_STATE_MODIFIED", "DBT_ENGINE_LOG_LEVEL_FILE", "DBT_ENGINE_USE_EXPERIMENTAL_PARSER", "DBT_ENGINE_UPLOAD_TO_ARTIFACTS_INGEST_API", "DBT_ENGINE_DEBUG", "DBT_ENGINE_PRINT", "DBT_ENGINE_SAMPLE", } from dbt.env_vars import _ALLOWED_ENV_VARS assert hard_coded_allow_list == _ALLOWED_ENV_VARS ================================================ FILE: tests/functional/cli/test_resolvers.py ================================================ from pathlib import Path import pytest from dbt.cli.resolvers import default_log_path class TestDefaultLogPathNoProject: def test_default_log_path_no_project(self): expected_log_path = Path("logs") actual_log_path = default_log_path("nonexistent_project_dir") assert actual_log_path == expected_log_path class TestDefaultLogPathWithProject: @pytest.fixture(scope="class") def project_config_update(self): return {"log-path": "test_default_log_path"} def test_default_log_path_with_project(self, project, project_config_update): expected_log_path = Path(project.project_root) / "test_default_log_path" actual_log_path = default_log_path(project.project_root) assert actual_log_path == expected_log_path class TestDefaultLogPathWithProjectNoConfiguredLogPath: @pytest.fixture(scope="class") def project_config_update(self): return {"log-path": None} def test_default_log_path_with_project(self, project, project_config_update): expected_log_path = Path(project.project_root) / "logs" actual_log_path = default_log_path(project.project_root) assert actual_log_path == expected_log_path ================================================ FILE: tests/functional/colors/test_colors.py ================================================ import re import pytest from dbt.tests.util import run_dbt_and_capture models__do_nothing_then_fail_sql = """ select 1, """ @pytest.fixture(scope="class") def models(): return {"do_nothing_then_fail.sql": models__do_nothing_then_fail_sql} @pytest.fixture(scope="class") def project_config_update(): return {"config-version": 2} class TestColors: def test_use_colors(self, project): self.assert_colors_used( "--use-colors", expect_colors=True, ) def test_no_use_colors(self, project): self.assert_colors_used( "--no-use-colors", expect_colors=False, ) def assert_colors_used(self, flag, expect_colors): _, stdout = run_dbt_and_capture(args=[flag, "run"], expect_pass=False) # pattern to match formatted log output pattern = re.compile(r"\[31m.*|\[33m.*") stdout_contains_formatting_characters = bool(pattern.search(stdout)) if expect_colors: assert stdout_contains_formatting_characters else: assert not stdout_contains_formatting_characters ================================================ FILE: tests/functional/column_quoting/test_column_quotes.py ================================================ import pytest from dbt.tests.util import run_dbt _MODELS__COLUMN_QUOTING_DEFAULT = """ {% set col_a = '"col_A"' %} {% set col_b = '"col_B"' %} {{ config( materialized = 'incremental', unique_key = col_a, ) }} select {{ col_a }}, {{ col_b }} from {{ref('seed')}} """ _MODELS__COLUMN_QUOTING_NO_QUOTING = """ {% set col_a = '"col_a"' %} {% set col_b = '"col_b"' %} {{ config( materialized = 'incremental', unique_key = col_a, ) }} select {{ col_a }}, {{ col_b }} from {{ref('seed')}} """ _SEEDS_BASIC_SEED = """col_A,col_B 1,2 3,4 5,6 """ class BaseColumnQuotingTest: @pytest.fixture(scope="class") def models(self): return {"model.sql": _MODELS__COLUMN_QUOTING_DEFAULT} @pytest.fixture(scope="class") def seeds(self): return {"seed.csv": _SEEDS_BASIC_SEED} @pytest.fixture(scope="function") def run_column_quotes(self, project): def fixt(): results = run_dbt(["seed"]) assert len(results) == 1 results = run_dbt(["run"]) assert len(results) == 1 results = run_dbt(["run"]) assert len(results) == 1 return fixt class TestColumnQuotingDefault(BaseColumnQuotingTest): def test_column_quotes(self, run_column_quotes): run_column_quotes() class TestColumnQuotingEnabled(BaseColumnQuotingTest): @pytest.fixture(scope="class") def project_config_update(self): return { "seeds": { "quote_columns": True, }, } def test_column_quotes(self, run_column_quotes): run_column_quotes() class TestColumnQuotingDisabled(BaseColumnQuotingTest): @pytest.fixture(scope="class") def models(self): return {"model.sql": _MODELS__COLUMN_QUOTING_NO_QUOTING} @pytest.fixture(scope="class") def project_config_update(self): return { "seeds": { "quote_columns": False, }, } def test_column_quotes(self, run_column_quotes): run_column_quotes() ================================================ FILE: tests/functional/compile/fixtures.py ================================================ first_model_sql = """ select 1 as fun """ second_model_sql = """ {%- set columns = adapter.get_columns_in_relation(ref('first_model')) -%} select *, {{ this.schema }} as schema from {{ ref('first_model') }} """ first_ephemeral_model_sql = """ {{ config(materialized = 'ephemeral') }} select 1 as fun """ second_ephemeral_model_sql = """ {{ config(materialized = 'ephemeral') }} select * from {{ ref('first_ephemeral_model') }} """ third_ephemeral_model_sql = """ select * from {{ ref('second_ephemeral_model')}} union all select 2 as fun """ model_multiline_jinja = """ select {{ 1 + 1 }} as fun """ with_recursive_model_sql = """ {{ config(materialized = 'ephemeral') }} with recursive t(n) as ( select * from {{ ref('first_ephemeral_model') }} union all select n+1 from t where n < 100 ) select sum(n) from t; """ first_ephemeral_model_with_alias_sql = """ {{ config(materialized = 'ephemeral', alias = 'first_alias') }} select 1 as fun """ second_ephemeral_model_with_alias_sql = """ select * from {{ ref('first_ephemeral_model_with_alias') }} """ schema_yml = """ version: 2 models: - name: second_model description: "The second model" columns: - name: fun data_tests: - not_null - name: schema data_tests: - unique """ ================================================ FILE: tests/functional/compile/test_compile.py ================================================ import json import pathlib import re import pytest import sqlparse from dbt.tests.util import read_file, run_dbt, run_dbt_and_capture from dbt_common.exceptions import DbtBaseException as DbtException from dbt_common.exceptions import DbtRuntimeError from tests.functional.assertions.test_runner import dbtTestRunner from tests.functional.compile.fixtures import ( first_ephemeral_model_sql, first_ephemeral_model_with_alias_sql, first_model_sql, model_multiline_jinja, schema_yml, second_ephemeral_model_sql, second_ephemeral_model_with_alias_sql, second_model_sql, third_ephemeral_model_sql, with_recursive_model_sql, ) def norm_whitespace(string): _RE_COMBINE_WHITESPACE = re.compile(r"\s+") string = _RE_COMBINE_WHITESPACE.sub(" ", string).strip() return string def get_lines(model_name): f = read_file("target", "compiled", "test", "models", model_name + ".sql") return [line for line in f.splitlines() if line] def file_exists(model_name): from dbt.tests.util import file_exists return file_exists("target", "compiled", "test", "models", model_name + ".sql") class TestIntrospectFlag: @pytest.fixture(scope="class") def models(self): return { "first_model.sql": first_model_sql, "second_model.sql": second_model_sql, "schema.yml": schema_yml, } def test_default(self, project): run_dbt(["compile"]) assert get_lines("first_model") == ["select 1 as fun"] assert any("_test_compile as schema" in line for line in get_lines("second_model")) def test_no_introspect(self, project): with pytest.raises(DbtRuntimeError, match="connection never acquired for thread"): run_dbt(["compile", "--no-introspect"]) class TestEphemeralModels: @pytest.fixture(scope="class") def models(self): return { "first_ephemeral_model.sql": first_ephemeral_model_sql, "second_ephemeral_model.sql": second_ephemeral_model_sql, "third_ephemeral_model.sql": third_ephemeral_model_sql, "with_recursive_model.sql": with_recursive_model_sql, } def test_first_selector(self, project): (results, log_output) = run_dbt_and_capture( ["compile", "--select", "first_ephemeral_model"] ) assert file_exists("first_ephemeral_model") assert not file_exists("second_ephemeral_model") assert not file_exists("third_ephemeral_model") assert "Compiled node 'first_ephemeral_model' is" in log_output def test_middle_selector(self, project): (results, log_output) = run_dbt_and_capture( ["compile", "--select", "second_ephemeral_model"] ) assert file_exists("first_ephemeral_model") assert file_exists("second_ephemeral_model") assert not file_exists("third_ephemeral_model") assert "Compiled node 'second_ephemeral_model' is" in log_output def test_last_selector(self, project): (results, log_output) = run_dbt_and_capture( ["compile", "--select", "third_ephemeral_model"] ) assert file_exists("first_ephemeral_model") assert file_exists("second_ephemeral_model") assert file_exists("third_ephemeral_model") assert "Compiled node 'third_ephemeral_model' is" in log_output def test_no_selector(self, project): run_dbt(["compile"]) sql = read_file("target", "compiled", "test", "models", "first_ephemeral_model.sql") assert norm_whitespace(sql) == norm_whitespace("select 1 as fun") sql = read_file("target", "compiled", "test", "models", "second_ephemeral_model.sql") expected_sql = """with __dbt__cte__first_ephemeral_model as ( select 1 as fun ) select * from __dbt__cte__first_ephemeral_model""" assert norm_whitespace(sql) == norm_whitespace(expected_sql) sql = read_file("target", "compiled", "test", "models", "third_ephemeral_model.sql") expected_sql = """with __dbt__cte__first_ephemeral_model as ( select 1 as fun ), __dbt__cte__second_ephemeral_model as ( select * from __dbt__cte__first_ephemeral_model ) select * from __dbt__cte__second_ephemeral_model union all select 2 as fun""" assert norm_whitespace(sql) == norm_whitespace(expected_sql) def test_with_recursive_cte(self, project): run_dbt(["compile"]) assert get_lines("with_recursive_model") == [ "with recursive __dbt__cte__first_ephemeral_model as (", "select 1 as fun", "), t(n) as (", " select * from __dbt__cte__first_ephemeral_model", " union all", " select n+1 from t where n < 100", ")", "select sum(n) from t;", ] class TestEphemeralModelWithAlias: @pytest.fixture(scope="class") def models(self): return { "first_ephemeral_model_with_alias.sql": first_ephemeral_model_with_alias_sql, "second_ephemeral_model_with_alias.sql": second_ephemeral_model_with_alias_sql, } def test_compile(self, project): run_dbt(["compile"]) assert get_lines("second_ephemeral_model_with_alias") == [ "with __dbt__cte__first_alias as (", "select 1 as fun", ") select * from __dbt__cte__first_alias", ] class TestCompile: @pytest.fixture(scope="class") def models(self): return { "first_model.sql": first_model_sql, "second_model.sql": second_model_sql, "schema.yml": schema_yml, } def test_none(self, project): (results, log_output) = run_dbt_and_capture(["compile"]) assert len(results) == 4 assert "Compiled node" not in log_output def test_inline_pass(self, project): (results, log_output) = run_dbt_and_capture( ["compile", "--inline", "select * from {{ ref('first_model') }}"] ) assert len(results) == 1 assert "Compiled inline node is:" in log_output def test_inline_pass_quiet(self, project): (results, log_output) = run_dbt_and_capture( ["compile", "--quiet", "--inline", "select * from {{ ref('first_model') }}"] ) assert len(results) == 1 assert "Compiled inline node is:" not in log_output def test_select_pass(self, project): (results, log_output) = run_dbt_and_capture(["compile", "--select", "second_model"]) assert len(results) == 3 assert "Compiled node 'second_model' is:" in log_output def test_select_pass_quiet(self, project): (results, log_output) = run_dbt_and_capture( ["compile", "--quiet", "--select", "second_model"] ) assert len(results) == 3 assert "Compiled node 'second_model' is:" not in log_output def test_select_pass_empty(self, project): (results, log_output) = run_dbt_and_capture( ["compile", "--indirect-selection", "empty", "--select", "second_model"] ) assert len(results) == 1 assert "Compiled node 'second_model' is:" in log_output def test_inline_fail(self, project): with pytest.raises(DbtException, match="Error parsing inline query"): run_dbt(["compile", "--inline", "select * from {{ ref('third_model') }}"]) def test_inline_fail_database_error(self, project): with pytest.raises(DbtRuntimeError, match="Database Error"): run_dbt(["show", "--inline", "slect asdlkjfsld;j"]) def test_multiline_jinja(self, project): (results, log_output) = run_dbt_and_capture(["compile", "--inline", model_multiline_jinja]) assert len(results) == 1 assert "Compiled inline node is:" in log_output def test_output_json_select(self, project): (results, log_output) = run_dbt_and_capture( ["compile", "--select", "second_model", "--output", "json"] ) assert len(results) == 3 assert "node" in log_output assert "compiled" in log_output with pytest.raises(json.JSONDecodeError): json.loads(log_output) def test_output_json_select_quiet(self, project): (results, log_output) = run_dbt_and_capture( ["compile", "--quiet", "--select", "second_model", "--output", "json"] ) assert len(results) == 3 assert "node" in log_output assert "compiled" in log_output json.loads(log_output) def test_output_json_inline(self, project): (results, log_output) = run_dbt_and_capture( ["compile", "--inline", "select * from {{ ref('second_model') }}", "--output", "json"] ) assert len(results) == 1 assert '"node"' not in log_output assert '"compiled"' in log_output with pytest.raises(json.JSONDecodeError): json.loads(log_output) def test_output_json_inline_quiet(self, project): (results, log_output) = run_dbt_and_capture( [ "compile", "--quiet", "--inline", "select * from {{ ref('second_model') }}", "--output", "json", ] ) assert len(results) == 1 assert '"node"' not in log_output assert '"compiled"' in log_output json.loads(log_output) def test_compile_inline_not_add_node(self, project): dbt = dbtTestRunner() parse_result = dbt.invoke(["parse"]) manifest = parse_result.result assert len(manifest.nodes) == 4 dbt = dbtTestRunner(manifest=manifest) dbt.invoke( ["compile", "--inline", "select * from {{ ref('second_model') }}"], populate_cache=False, ) assert len(manifest.nodes) == 4 def test_compile_inline_syntax_error(self, project, mocker): patched_fire_event = mocker.patch("dbt.task.compile.fire_event") with pytest.raises(DbtException, match="Error parsing inline query"): run_dbt(["compile", "--inline", "select * from {{ ref(1) }}"]) # Event for parsing error fired patched_fire_event.assert_called_once() def test_compile_inline_ref_node_not_exist(self, project, mocker): patched_fire_event = mocker.patch("dbt.task.compile.fire_event") with pytest.raises(DbtException, match="Error parsing inline query"): run_dbt(["compile", "--inline", "select * from {{ ref('third_model') }}"]) # Event for parsing error fired patched_fire_event.assert_called_once() def test_graph_summary_output(self, project): """Ensure that the compile command generates a file named graph_summary.json in the target directory, that the file contains valid json, and that the json has the high level structure it should.""" dbtTestRunner().invoke(["compile"]) summary_path = pathlib.Path(project.project_root, "target/graph_summary.json") with open(summary_path, "r") as summary_file: summary = json.load(summary_file) assert "_invocation_id" in summary assert "linked" in summary class TestSqlParseGroupingTokenLimit: @pytest.fixture(scope="class") def models(self): return { "first_ephemeral_model.sql": first_ephemeral_model_sql, "second_ephemeral_model.sql": second_ephemeral_model_sql, "third_ephemeral_model.sql": third_ephemeral_model_sql, "with_recursive_model.sql": with_recursive_model_sql, } def test_sqlparse_grouping_token_limit(self, project): # No flag: compile succeeds (default is no limit) run_dbt(["compile"]) assert sqlparse.engine.grouping.MAX_GROUPING_TOKENS is None # Flag set to 0: compile fails because token limit is exceeded with pytest.raises(DbtRuntimeError, match="You may raise the limit via --sqlparse"): run_dbt(["compile", "--sqlparse", '{"MAX_GROUPING_TOKENS": "0"}']) # Flag set to 10000: compile succeeds run_dbt(["compile", "--sqlparse", '{"MAX_GROUPING_TOKENS": "10000"}']) class TestSqlParseGroupingDepthLimit: @pytest.fixture(scope="class") def models(self): return { "first_ephemeral_model.sql": first_ephemeral_model_sql, "second_ephemeral_model.sql": second_ephemeral_model_sql, "third_ephemeral_model.sql": third_ephemeral_model_sql, "with_recursive_model.sql": with_recursive_model_sql, } def test_sqlparse_grouping_depth_limit(self, project): # No flag: compile succeeds (default is no limit) run_dbt(["compile"]) assert sqlparse.engine.grouping.MAX_GROUPING_DEPTH is None # Flag set to 0: compile fails because depth limit is exceeded with pytest.raises(DbtRuntimeError, match="You may raise the limit via --sqlparse"): run_dbt(["compile", "--sqlparse", '{"MAX_GROUPING_DEPTH": "0"}']) # Flag set to 10000: compile succeeds run_dbt(["compile", "--sqlparse", '{"MAX_GROUPING_DEPTH": "10000"}']) ================================================ FILE: tests/functional/configs/fixtures.py ================================================ import pytest models__schema_yml = """ version: 2 sources: - name: raw database: "{{ target.database }}" schema: "{{ target.schema }}" tables: - name: 'seed' identifier: "{{ var('seed_name', 'invalid') }}" columns: - name: id data_tests: - unique: enabled: "{{ var('enabled_direct', None) | as_native }}" - accepted_values: enabled: "{{ var('enabled_direct', None) | as_native }}" severity: "{{ var('severity_direct', None) | as_native }}" values: [1,2] models: - name: model columns: - name: id data_tests: - unique - accepted_values: values: [1,2,3,4] """ models__untagged_sql = """ {{ config(materialized='table') }} select id, value from {{ source('raw', 'seed') }} """ models__tagged__model_sql = """ {{ config( materialized='view', tags=['tag_two'], ) }} {{ config( materialized='table', tags=['tag_three'], ) }} select 4 as id, 2 as value """ seeds__seed_csv = """id,value 4,2 """ tests__failing_sql = """ select 1 as fun """ tests__sleeper_agent_sql = """ {{ config( enabled = var('enabled_direct', False), severity = var('severity_direct', 'WARN') ) }} select 1 as fun """ my_model = """ select 1 as user """ my_model_2 = """ select * from {{ ref('my_model') }} """ my_model_3 = """ select * from {{ ref('my_model_2') }} """ my_model_2_disabled = """ {{ config(enabled=false) }} select * from {{ ref('my_model') }} """ my_model_3_disabled = """ {{ config(enabled=false) }} select * from {{ ref('my_model_2') }} """ my_model_2_enabled = """ {{ config(enabled=true) }} select * from {{ ref('my_model') }} """ my_model_3_enabled = """ {{ config(enabled=true) }} select * from {{ ref('my_model') }} """ schema_all_disabled_yml = """ version: 2 models: - name: my_model - name: my_model_2 config: enabled: false - name: my_model_3 config: enabled: false """ schema_explicit_enabled_yml = """ version: 2 models: - name: my_model - name: my_model_2 config: enabled: true - name: my_model_3 config: enabled: true """ schema_partial_disabled_yml = """ version: 2 models: - name: my_model - name: my_model_2 config: enabled: false - name: my_model_3 """ schema_partial_enabled_yml = """ version: 2 models: - name: my_model - name: my_model_2 config: enabled: True - name: my_model_3 """ schema_invalid_enabled_yml = """ version: 2 models: - name: my_model config: enabled: True and False - name: my_model_3 """ simple_snapshot = """{% snapshot mysnapshot %} {{ config( target_schema='snapshots', strategy='timestamp', unique_key='id', updated_at='updated_at' ) }} select * from dummy {% endsnapshot %}""" class BaseConfigProject: @pytest.fixture(scope="class") def models(self): return { "schema.yml": models__schema_yml, "untagged.sql": models__untagged_sql, "tagged": {"model.sql": models__tagged__model_sql}, } @pytest.fixture(scope="class") def seeds(self): return {"seed.csv": seeds__seed_csv} @pytest.fixture(scope="class") def tests(self): return { "failing.sql": tests__failing_sql, "sleeper_agent.sql": tests__sleeper_agent_sql, } ================================================ FILE: tests/functional/configs/test_configs.py ================================================ import os import pytest from dbt.exceptions import SchemaConfigError from dbt.tests.util import ( check_relations_equal, run_dbt, update_config_file, write_file, ) from dbt_common.dataclass_schema import ValidationError from tests.functional.configs.fixtures import BaseConfigProject, simple_snapshot class TestConfigs(BaseConfigProject): @pytest.fixture(scope="class") def project_config_update(self): return { "models": { "test": { "tagged": { # the model configs will override this "materialized": "invalid", # the model configs will append to these "tags": ["tag_one"], } }, }, "seeds": { "quote_columns": False, }, } def test_config_layering( self, project, ): # run seed results = run_dbt(["seed"]) assert len(results) == 1 # test the project-level tag, and both config() call tags assert len(run_dbt(["run", "--model", "tag:tag_one"])) == 1 assert len(run_dbt(["run", "--model", "tag:tag_two"])) == 1 assert len(run_dbt(["run", "--model", "tag:tag_three"])) == 1 check_relations_equal(project.adapter, ["seed", "model"]) # make sure we overwrote the materialization properly tables = project.get_tables_in_schema() assert tables["model"] == "table" # In addition to testing an alternative target-paths setting, it tests that # the attribute is jinja rendered and that the context "modules" works. class TestTargetConfigs(BaseConfigProject): @pytest.fixture(scope="class") def project_config_update(self): return { "target-path": "target_{{ modules.datetime.datetime.now(modules.pytz.timezone('Etc/UTC')).strftime('%Y%m%dT%H%M%S') }}", "seeds": { "quote_columns": False, }, } def test_alternative_target_paths(self, project): # chdir to a different directory to test creation of target directory under project_root os.chdir(project.profiles_dir) run_dbt(["seed"]) target_path = "" for d in os.listdir(project.project_root): if os.path.isdir(os.path.join(project.project_root, d)) and d.startswith("target_"): target_path = d assert os.path.exists(os.path.join(project.project_root, target_path, "manifest.json")) class TestInvalidTestsMaterializationProj(object): def test_tests_materialization_proj_config(self, project): config_patch = {"data_tests": {"materialized": "table"}} update_config_file(config_patch, project.project_root, "dbt_project.yml") tests_dir = os.path.join(project.project_root, "tests") write_file("select * from foo", tests_dir, "test.sql") with pytest.raises(SchemaConfigError): run_dbt() class TestInvalidSeedsMaterializationProj(object): def test_seeds_materialization_proj_config(self, project): config_patch = {"seeds": {"materialized": "table"}} update_config_file(config_patch, project.project_root, "dbt_project.yml") seeds_dir = os.path.join(project.project_root, "seeds") write_file("id1, id2\n1, 2", seeds_dir, "seed.csv") with pytest.raises(SchemaConfigError): run_dbt() class TestInvalidSeedsMaterializationSchema(object): def test_seeds_materialization_schema_config(self, project): seeds_dir = os.path.join(project.project_root, "seeds") write_file( "version: 2\nseeds:\n - name: myseed\n config:\n materialized: table", seeds_dir, "schema.yml", ) write_file("id1, id2\n1, 2", seeds_dir, "myseed.csv") with pytest.raises(SchemaConfigError): run_dbt() class TestInvalidSnapshotsMaterializationProj(object): def test_snapshots_materialization_proj_config(self, project): config_patch = {"snapshots": {"materialized": "table"}} update_config_file(config_patch, project.project_root, "dbt_project.yml") snapshots_dir = os.path.join(project.project_root, "snapshots") write_file(simple_snapshot, snapshots_dir, "mysnapshot.sql") with pytest.raises(ValidationError): run_dbt() class TestInvalidSnapshotsMaterializationSchema(object): def test_snapshots_materialization_schema_config(self, project): snapshots_dir = os.path.join(project.project_root, "snapshots") write_file( "version: 2\nsnapshots:\n - name: mysnapshot\n config:\n materialized: table", snapshots_dir, "schema.yml", ) write_file(simple_snapshot, snapshots_dir, "mysnapshot.sql") with pytest.raises(ValidationError): run_dbt() ================================================ FILE: tests/functional/configs/test_configs_in_schema_files.py ================================================ import pytest from dbt.exceptions import CompilationError, ParsingError from dbt.tests.util import check_relations_equal, get_manifest, run_dbt, write_file models_alt__schema_yml = """ version: 2 sources: - name: raw database: "{{ target.database }}" schema: "{{ target.schema }}" tables: - name: 'some_seed' columns: - name: id models: - name: model description: "This is a model description" config: tags: ['tag_in_schema'] meta: owner: 'Julie Smith' my_attr: "{{ var('my_var') }}" materialized: view columns: - name: id data_tests: - not_null: meta: owner: 'Simple Simon' - unique: config: meta: owner: 'John Doe' """ models_alt__untagged_sql = """ {{ config(materialized='table') }} select id, value from {{ source('raw', 'some_seed') }} """ models_alt__tagged__model_sql = """ {{ config( materialized='view', tags=['tag_1_in_model'], ) }} {{ config( materialized='table', tags=['tag_2_in_model'], ) }} select 4 as id, 2 as value """ models_no_materialized__model_sql = """ {{ config( tags=['tag_1_in_model'], ) }} {{ config( tags=['tag_2_in_model'], ) }} select 4 as id, 2 as value """ seeds_alt__some_seed_csv = """id,value 4,2 """ extra_alt__untagged_yml = """ version: 2 models: - name: untagged description: "This is a model description" meta: owner: 'Somebody Else' config: meta: owner: 'Julie Smith' """ extra_alt__untagged2_yml = """ version: 2 models: - name: untagged description: "This is a model description" data_tests: - not_null: error_if: ">2" config: error_if: ">2" """ class TestSchemaFileConfigs: @pytest.fixture(scope="class") def expected_unrendered_config(self): # my_attr is unrendered when state_modified_compare_more_unrendered_values: True return { "materialized": "view", "meta": {"my_attr": "{{ var('my_var') }}", "owner": "Julie Smith"}, "tags": ["tag_1_in_model", "tag_2_in_model"], } @pytest.fixture(scope="class") def models(self): return { "schema.yml": models_alt__schema_yml, "untagged.sql": models_alt__untagged_sql, "tagged": {"model.sql": models_alt__tagged__model_sql}, } @pytest.fixture(scope="class") def seeds(self): return {"some_seed.csv": seeds_alt__some_seed_csv} @pytest.fixture(scope="class") def project_config_update(self): return { "flags": { "state_modified_compare_more_unrendered_values": True, }, "models": { "+meta": { "company": "NuMade", }, "test": { "+meta": { "project": "test", }, "tagged": { "+meta": { "team": "Core Team", }, "tags": ["tag_in_project"], "model": { "materialized": "table", "+meta": { "owner": "Julie Dent", }, }, }, }, }, "vars": { "test": { "my_var": "TESTING", } }, "seeds": { "quote_columns": False, }, } def test_config_layering( self, project, expected_unrendered_config, ): # run seed assert len(run_dbt(["seed"])) == 1 # test the project-level tag, and both config() call tags assert len(run_dbt(["run", "--model", "tag:tag_in_project"])) == 1 assert len(run_dbt(["run", "--model", "tag:tag_1_in_model"])) == 1 assert len(run_dbt(["run", "--model", "tag:tag_2_in_model"])) == 1 assert len(run_dbt(["run", "--model", "tag:tag_in_schema"])) == 1 # Verify that model nodes have expected tags and meta manifest = get_manifest(project.project_root) model_id = "model.test.model" model_node = manifest.nodes[model_id] meta_expected = { "company": "NuMade", "project": "test", "team": "Core Team", "owner": "Julie Smith", "my_attr": "TESTING", } assert model_node.meta == meta_expected assert model_node.config.meta == meta_expected model_tags = ["tag_1_in_model", "tag_2_in_model", "tag_in_project", "tag_in_schema"] model_node_tags = model_node.tags.copy() model_node_tags.sort() assert model_node_tags == model_tags model_node_config_tags = model_node.config.tags.copy() model_node_config_tags.sort() assert model_node_config_tags == model_tags model_meta = { "company": "NuMade", "project": "test", "team": "Core Team", "owner": "Julie Smith", "my_attr": "TESTING", } assert model_node.config.meta == model_meta # make sure we overwrote the materialization properly tables = project.get_tables_in_schema() assert tables["model"] == "table" check_relations_equal(project.adapter, ["some_seed", "model"]) # Remove materialized config from model write_file( models_no_materialized__model_sql, project.project_root, "models", "tagged", "model.sql", ) results = run_dbt(["run"]) assert len(results) == 2 manifest = get_manifest(project.project_root) model_node = manifest.nodes[model_id] assert model_node.config.materialized == "view" model_unrendered_config = expected_unrendered_config assert model_node.unrendered_config == model_unrendered_config # look for test meta schema_file_id = model_node.patch_path schema_file = manifest.files[schema_file_id] tests = schema_file.get_tests("models", "model") assert tests[0] in manifest.nodes test = manifest.nodes[tests[0]] expected_meta = {"owner": "Simple Simon"} assert test.config.meta == expected_meta test = manifest.nodes[tests[1]] expected_meta = {"owner": "John Doe"} assert test.config.meta == expected_meta # copy a schema file with multiple metas # shutil.copyfile('extra-alt/untagged.yml', 'models-alt/untagged.yml') write_file(extra_alt__untagged_yml, project.project_root, "models", "untagged.yml") with pytest.raises(ParsingError): run_dbt(["run"]) # copy a schema file with config key in top-level of test and in config dict # shutil.copyfile('extra-alt/untagged2.yml', 'models-alt/untagged.yml') write_file(extra_alt__untagged2_yml, project.project_root, "models", "untagged.yml") with pytest.raises(CompilationError): run_dbt(["run"]) class TestLegacySchemaFileConfigs(TestSchemaFileConfigs): @pytest.fixture(scope="class") def expected_unrendered_config(self): # my_attr is rendered ("TESTING") when state_modified_compare_more_unrendered_values: False return { "materialized": "view", "meta": {"my_attr": "TESTING", "owner": "Julie Smith"}, "tags": ["tag_1_in_model", "tag_2_in_model"], } @pytest.fixture(scope="class") def project_config_update(self): return { # The uncommented below lines can be removed once the default behaviour is flipped. # state_modified_compare_more_unrendered_values defaults to false currently # "flags": { # "state_modified_compare_more_unrendered_values": False, # }, "models": { "+meta": { "company": "NuMade", }, "test": { "+meta": { "project": "test", }, "tagged": { "+meta": { "team": "Core Team", }, "tags": ["tag_in_project"], "model": { "materialized": "table", "+meta": { "owner": "Julie Dent", }, }, }, }, }, "vars": { "test": { "my_var": "TESTING", } }, "seeds": { "quote_columns": False, }, } list_schema_yml = """ - name: my_name - name: alt_name """ class TestListSchemaFile: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": "select 1 as id", "schema.yml": list_schema_yml, } def test_list_schema(self, project): with pytest.raises(ParsingError) as excinfo: run_dbt(["run"]) assert "Dictionary expected" in str(excinfo.value) ================================================ FILE: tests/functional/configs/test_contract_configs.py ================================================ import os import pytest from dbt.artifacts.resources.v1.components import ColumnInfo from dbt.exceptions import ParsingError, ValidationError from dbt.tests.util import ( get_artifact, get_manifest, run_dbt, run_dbt_and_capture, write_file, ) from dbt_common.contracts.constraints import ColumnLevelConstraint, ConstraintType my_model_sql = """ {{ config( materialized = "table" ) }} select 'blue' as color, 1 as id, cast('2019-01-01' as date) as date_day """ my_model_contract_sql = """ {{ config( materialized = "table", contract = {"enforced": true} ) }} select 1 as id, 'blue' as color, cast('2019-01-01' as date) as date_day """ my_model_contract_disabled_sql = """ {{ config( materialized = "table", contract = {"enforced": false} ) }} select 1 as id, 'blue' as color, cast('2019-01-01' as date) as date_day """ my_incremental_model_sql = """ {{ config( materialized = "incremental" ) }} select 1 as id, 'blue' as color, cast('2019-01-01' as date) as date_day """ my_view_model_sql = """ {{ config( materialized = "view" ) }} select 1 as id, 'blue' as color, cast('2019-01-01' as date) as date_day """ my_model_python_error = """ import holidays, s3fs def model(dbt, _): dbt.config( materialized="table", packages=["holidays", "s3fs"], # how to import python libraries in dbt's context ) df = dbt.ref("my_model") df_describe = df.describe() # basic statistics profiling return df_describe """ model_schema_yml = """ models: - name: my_model config: contract: enforced: true columns: - name: id quote: true data_type: integer description: hello constraints: - type: not_null - type: primary_key - type: check expression: (id > 0) data_tests: - unique - name: color data_type: string - name: date_day data_type: date """ model_pk_model_column_schema_yml = """ models: - name: my_model config: contract: enforced: true constraints: - type: primary_key columns: [id] columns: - name: id data_type: integer description: hello constraints: - type: not_null - type: primary_key - type: check expression: (id > 0) data_tests: - unique - name: color data_type: string - name: date_day data_type: date """ model_pk_mult_column_schema_yml = """ models: - name: my_model config: contract: enforced: true columns: - name: id quote: true data_type: integer description: hello constraints: - type: not_null - type: primary_key - type: check expression: (id > 0) data_tests: - unique - name: color data_type: string constraints: - type: not_null - type: primary_key - name: date_day data_type: date """ model_schema_alias_types_false_yml = """ models: - name: my_model config: contract: enforced: true alias_types: false columns: - name: id quote: true data_type: integer description: hello constraints: - type: not_null - type: primary_key - type: check expression: (id > 0) data_tests: - unique - name: color data_type: string - name: date_day data_type: date """ model_schema_ignore_unsupported_yml = """ models: - name: my_model config: contract: enforced: true columns: - name: id quote: true data_type: integer description: hello constraints: - type: not_null warn_unsupported: False - type: primary_key warn_unsupported: False - type: check warn_unsupported: False expression: (id > 0) data_tests: - unique - name: color data_type: text - name: date_day data_type: date """ model_schema_errors_yml = """ models: - name: my_model config: contract: enforced: true columns: - name: id data_type: integer description: hello constraints: - type: not_null - type: primary_key - type: check expression: (id > 0) data_tests: - unique - name: color data_type: text - name: date_day - name: python_model config: contract: enforced: true columns: - name: id data_type: integer description: hello constraints: - type: not_null - type: primary_key - type: check expression: (id > 0) data_tests: - unique - name: color data_type: text - name: date_day data_type: date """ model_schema_blank_yml = """ models: - name: my_model config: contract: enforced: true """ model_schema_complete_datatypes_yml = """ models: - name: my_model columns: - name: id quote: true data_type: integer description: hello constraints: - type: not_null - type: primary_key - type: check expression: (id > 0) data_tests: - unique - name: color data_type: text - name: date_day data_type: date """ model_schema_incomplete_datatypes_yml = """ models: - name: my_model columns: - name: id quote: true data_type: integer description: hello constraints: - type: not_null - type: primary_key - type: check expression: (id > 0) data_tests: - unique - name: color - name: date_day data_type: date """ class TestModelLevelContractEnabledConfigs: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_model_sql, "constraints_schema.yml": model_schema_yml, } def test__model_contract_true(self, project): run_dbt(["run"]) manifest = get_manifest(project.project_root) model_id = "model.test.my_model" model = manifest.nodes[model_id] my_model_columns = model.columns my_model_config = model.config contract_actual_config = my_model_config.contract assert contract_actual_config.enforced is True expected_columns = { "id": ColumnInfo( name="id", description="hello", meta={}, data_type="integer", doc_blocks=[], constraints=[ ColumnLevelConstraint( type=ConstraintType.not_null, name=None, expression=None, warn_unenforced=True, warn_unsupported=True, to=None, to_columns=[], ), ColumnLevelConstraint( type=ConstraintType.primary_key, name=None, expression=None, warn_unenforced=True, warn_unsupported=True, to=None, to_columns=[], ), ColumnLevelConstraint( type=ConstraintType.check, name=None, expression="(id > 0)", warn_unenforced=True, warn_unsupported=True, to=None, to_columns=[], ), ], quote=True, tags=[], _extra={}, granularity=None, ), "color": ColumnInfo( name="color", description="", doc_blocks=[], meta={}, data_type="string", constraints=[], quote=None, tags=[], _extra={}, granularity=None, ), "date_day": ColumnInfo( name="date_day", description="", doc_blocks=[], meta={}, data_type="date", constraints=[], quote=None, tags=[], _extra={}, granularity=None, ), } assert expected_columns == my_model_columns # compiled fields aren't in the manifest above because it only has parsed fields manifest_json = get_artifact(project.project_root, "target", "manifest.json") compiled_code = manifest_json["nodes"][model_id]["compiled_code"] cleaned_code = " ".join(compiled_code.split()) assert ( "select 'blue' as color, 1 as id, cast('2019-01-01' as date) as date_day" == cleaned_code ) # set alias_types to false (should fail to compile) write_file( model_schema_alias_types_false_yml, project.project_root, "models", "constraints_schema.yml", ) run_dbt(["run"], expect_pass=False) class TestProjectContractEnabledConfigs: @pytest.fixture(scope="class") def project_config_update(self): return {"models": {"test": {"+contract": {"enforced": True}}}} @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_model_sql, "constraints_schema.yml": model_schema_complete_datatypes_yml, } def test_defined_column_type(self, project): run_dbt(["run"], expect_pass=True) manifest = get_manifest(project.project_root) model_id = "model.test.my_model" my_model_config = manifest.nodes[model_id].config contract_actual_config = my_model_config.contract assert contract_actual_config.enforced is True class TestProjectContractEnabledConfigsError: @pytest.fixture(scope="class") def project_config_update(self): return { "models": { "test": { "+contract": { "enforced": True, }, } } } @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_model_sql, "constraints_schema.yml": model_schema_incomplete_datatypes_yml, } def test_undefined_column_type(self, project): _, log_output = run_dbt_and_capture(["run", "-s", "my_model"], expect_pass=False) manifest = get_manifest(project.project_root) model_id = "model.test.my_model" my_model_config = manifest.nodes[model_id].config contract_actual_config = my_model_config.contract assert contract_actual_config.enforced is True expected_compile_error = "Please ensure that the column name and data_type are defined within the YAML configuration for the ['color'] column(s)." assert expected_compile_error in log_output class TestModelContractEnabledConfigs: @pytest.fixture(scope="class") def models(self): return {"my_model.sql": my_model_contract_sql, "constraints_schema.yml": model_schema_yml} def test__model_contract(self, project): run_dbt(["run"]) manifest = get_manifest(project.project_root) model_id = "model.test.my_model" my_model_config = manifest.nodes[model_id].config contract_actual_config = my_model_config.contract assert contract_actual_config.enforced is True class TestModelContractEnabledConfigsMissingDataTypes: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_model_contract_sql, "constraints_schema.yml": model_schema_incomplete_datatypes_yml, } def test_undefined_column_type(self, project): _, log_output = run_dbt_and_capture(["run", "-s", "my_model"], expect_pass=False) manifest = get_manifest(project.project_root) model_id = "model.test.my_model" my_model_config = manifest.nodes[model_id].config contract_actual_config = my_model_config.contract assert contract_actual_config.enforced is True expected_compile_error = "Please ensure that the column name and data_type are defined within the YAML configuration for the ['color'] column(s)." assert expected_compile_error in log_output class TestModelLevelContractDisabledConfigs: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_model_contract_disabled_sql, "constraints_schema.yml": model_schema_yml, } def test__model_contract_false(self, project): run_dbt(["parse"]) manifest = get_manifest(project.project_root) model_id = "model.test.my_model" my_model_config = manifest.nodes[model_id].config contract_actual_config = my_model_config.contract assert contract_actual_config.enforced is False class TestModelLevelContractErrorMessages: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_incremental_model_sql, "constraints_schema.yml": model_schema_yml, } def test__config_errors(self, project): with pytest.raises(ValidationError) as err_info: run_dbt(["run"], expect_pass=False) exc_str = " ".join(str(err_info.value).split()) expected_materialization_error = "Invalid value for on_schema_change: ignore. Models materialized as incremental with contracts enabled must set on_schema_change to 'append_new_columns' or 'fail'" assert expected_materialization_error in str(exc_str) class TestModelLevelConstraintsErrorMessages: @pytest.fixture(scope="class") def models(self): return { "my_model.py": my_model_python_error, "constraints_schema.yml": model_schema_yml, } def test__config_errors(self, project): with pytest.raises(ParsingError) as err_info: run_dbt(["run"], expect_pass=False) exc_str = " ".join(str(err_info.value).split()) expected_materialization_error = "Language Error: Expected 'sql' but found 'python'" assert expected_materialization_error in str(exc_str) # This is a compile time error and we won't get here because the materialization check is parse time expected_empty_data_type_error = "Columns with `data_type` Blank/Null not allowed on contracted models. Columns Blank/Null: ['date_day']" assert expected_empty_data_type_error not in str(exc_str) class TestModelLevelConstraintsWarningMessages: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_view_model_sql, "constraints_schema.yml": model_schema_yml, } def test__config_warning(self, project): _, log_output = run_dbt_and_capture(["run"]) expected_materialization_warning = ( "Constraint types are not supported for view materializations" ) assert expected_materialization_warning in str(log_output) # change to not show warnings, message should not be in logs models_dir = os.path.join(project.project_root, "models") write_file(model_schema_ignore_unsupported_yml, models_dir, "constraints_schema.yml") _, log_output = run_dbt_and_capture(["run"]) expected_materialization_warning = ( "Constraint types are not supported for view materializations" ) assert expected_materialization_warning not in str(log_output) class TestSchemaContractEnabledConfigs: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_model_sql, "constraints_schema.yml": model_schema_blank_yml, } def test__schema_error(self, project): with pytest.raises(ParsingError) as err_info: run_dbt(["parse"], expect_pass=False) exc_str = " ".join(str(err_info.value).split()) schema_error_expected = "Constraints must be defined in a `yml` schema configuration file" assert schema_error_expected in str(exc_str) class TestPythonModelLevelContractErrorMessages: @pytest.fixture(scope="class") def models(self): return { "python_model.py": my_model_python_error, "constraints_schema.yml": model_schema_errors_yml, } def test__python_errors(self, project): with pytest.raises(ParsingError) as err_info: run_dbt(["parse"], expect_pass=False) exc_str = " ".join(str(err_info.value).split()) expected_python_error = "Language Error: Expected 'sql' but found 'python'" assert expected_python_error in exc_str class TestModelContractMissingYAMLColumns: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_model_contract_sql, } def test__missing_column_contract_error(self, project): results = run_dbt(["run"], expect_pass=False) expected_error = ( "This model has an enforced contract, and its 'columns' specification is missing" ) assert expected_error in results[0].message # test primary key defined across model and column level constraints, expect error class TestPrimaryKeysModelAndColumnLevelConstraints: @pytest.fixture(scope="class") def models(self): return { "constraints_schema.yml": model_pk_model_column_schema_yml, "my_model.sql": my_model_sql, } def test_model_column_pk_error(self, project): expected_error = "Primary key constraints defined at the model level and the columns level" with pytest.raises(ParsingError) as exc_info: run_dbt(["run"]) assert expected_error in str(exc_info.value) # test primary key defined across multiple columns, expect error class TestPrimaryKeysMultipleColumns: @pytest.fixture(scope="class") def models(self): return { "constraints_schema.yml": model_pk_mult_column_schema_yml, "my_model.sql": my_model_sql, } def test_pk_multiple_columns(self, project): expected_error = "Found 2 columns (['id', 'color']) with primary key constraints defined" with pytest.raises(ParsingError) as exc_info: run_dbt(["run"]) assert expected_error in str(exc_info.value) ================================================ FILE: tests/functional/configs/test_custom_node_colors_configs.py ================================================ import pytest from dbt.exceptions import ConfigUpdateError from dbt.tests.util import get_manifest, run_dbt from dbt_common.dataclass_schema import ValidationError CUSTOM_NODE_COLOR_MODEL_LEVEL = "red" CUSTOM_NODE_COLOR_SCHEMA_LEVEL = "blue" CUSTOM_NODE_COLOR_PROJECT_LEVEL_ROOT = "#121212" CUSTOM_NODE_COLOR_PROJECT_LEVEL_FOLDER = "purple" CUSTOM_NODE_COLOR_INVALID_HEX = '"#xxx111"' CUSTOM_NODE_COLOR_INVALID_NAME = "notacolor" # F strings are a pain here so replacing XXX with the config above instead models__custom_node_color__model_sql = """ {{ config(materialized='view', docs={'node_color': 'XXX'}) }} select 1 as id """.replace( "XXX", CUSTOM_NODE_COLOR_MODEL_LEVEL ) models__non_custom_node_color__model_sql = """ {{ config(materialized='view') }} select 1 as id """ models__show_docs_false__model_sql = """ {{ config(materialized='view', docs={"show": True}) }} select 1 as id """ models__custom_node_color__schema_yml = """ version: 2 models: - name: custom_color_model description: "This is a model description" config: docs: node_color: {} """.format( CUSTOM_NODE_COLOR_SCHEMA_LEVEL ) models__non_custom_node_color__schema_yml = """ version: 2 models: - name: non_custom_color_model description: "This is a model description" config: docs: node_color: {} show: True """.format( CUSTOM_NODE_COLOR_SCHEMA_LEVEL ) # To check that incorect configs are raising errors models__non_custom_node_color_invalid_config_docs__schema_yml = """ version: 2 models: - name: non_custom_node_color description: "This is a model description" config: docs: node_color: {} show: True """.format( CUSTOM_NODE_COLOR_INVALID_HEX ) models__non_custom_node_color_invalid_docs__schema_yml = """ version: 2 models: - name: non_custom_node_color description: "This is a model description" docs: node_color: {} show: True """.format( CUSTOM_NODE_COLOR_INVALID_NAME ) models__custom_node_color_invalid_hex__model_sql = """ {{ config(materialized='view', docs={"show": True, "node_color": XXX }) }} select 1 as id """.replace( "XXX", CUSTOM_NODE_COLOR_INVALID_HEX ) class BaseCustomNodeColorModelvsProject: @pytest.fixture(scope="class") def project_config_update(self): return { "models": { "test": { "+docs": {"node_color": CUSTOM_NODE_COLOR_PROJECT_LEVEL_ROOT, "show": False}, "subdirectory": { "+docs": { "node_color": CUSTOM_NODE_COLOR_PROJECT_LEVEL_FOLDER, "show": True, }, }, } } } # validation that model level node_color configs supercede dbt_project.yml class TestModelLevelProjectColorConfigs(BaseCustomNodeColorModelvsProject): @pytest.fixture(scope="class") def models(self): return {"custom_color_model.sql": models__custom_node_color__model_sql} def test__model_override_project(self, project): run_dbt(["compile"]) manifest = get_manifest(project.project_root) model_id = "model.test.custom_color_model" my_model_config = manifest.nodes[model_id].config my_model_docs = manifest.nodes[model_id].docs node_color_actual_config = my_model_config["docs"].node_color show_actual_config = my_model_config["docs"].show node_color_actual_docs = my_model_docs.node_color show_actual_docs = my_model_docs.show # check node_color config is in the right spots for each model assert node_color_actual_config == CUSTOM_NODE_COLOR_MODEL_LEVEL assert node_color_actual_docs == CUSTOM_NODE_COLOR_MODEL_LEVEL assert not show_actual_config assert not show_actual_docs # validation that model level node_color configs supercede schema.yml class TestModelLevelSchemaColorConfigs(BaseCustomNodeColorModelvsProject): @pytest.fixture(scope="class") def models(self): return { "custom_color_model.sql": models__custom_node_color__model_sql, "custom_color_schema.yml": models__custom_node_color__schema_yml, } def test__model_override_schema(self, project): run_dbt(["compile"]) manifest = get_manifest(project.project_root) model_id = "model.test.custom_color_model" my_model_config = manifest.nodes[model_id].config my_model_docs = manifest.nodes[model_id].docs node_color_actual_config = my_model_config["docs"].node_color show_actual_config = my_model_config["docs"].show node_color_actual_docs = my_model_docs.node_color show_actual_docs = my_model_docs.show # check node_color config is in the right spots for each model assert node_color_actual_config == CUSTOM_NODE_COLOR_MODEL_LEVEL assert node_color_actual_docs == CUSTOM_NODE_COLOR_MODEL_LEVEL assert not show_actual_config assert not show_actual_docs # validation that node_color configured on subdirectories in dbt_project.yml supercedes project root class TestSubdirectoryColorConfigs(BaseCustomNodeColorModelvsProject): @pytest.fixture(scope="class") def models(self): return { "subdirectory": { "non_custom_color_model_subdirectory.sql": models__non_custom_node_color__model_sql } } def test__project_folder_override_project_root(self, project): run_dbt(["compile"]) manifest = get_manifest(project.project_root) model_id = "model.test.non_custom_color_model_subdirectory" my_model_config = manifest.nodes[model_id].config my_model_docs = manifest.nodes[model_id].docs node_color_actual_config = my_model_config["docs"].node_color show_actual_config = my_model_config["docs"].show node_color_actual_docs = my_model_docs.node_color show_actual_docs = my_model_docs.show # check node_color config is in the right spots for each model assert node_color_actual_config == CUSTOM_NODE_COLOR_PROJECT_LEVEL_FOLDER assert node_color_actual_docs == CUSTOM_NODE_COLOR_PROJECT_LEVEL_FOLDER # in this case show should be True since the dbt_project.yml overrides the root setting for /subdirectory assert show_actual_config assert show_actual_docs # validation that node_color configured in schema.yml supercedes dbt_project.yml class TestSchemaOverProjectColorConfigs(BaseCustomNodeColorModelvsProject): @pytest.fixture(scope="class") def models(self): return { "non_custom_color_model.sql": models__non_custom_node_color__model_sql, "non_custom_color_schema.yml": models__non_custom_node_color__schema_yml, } def test__schema_override_project( self, project, ): run_dbt(["compile"]) manifest = get_manifest(project.project_root) model_id = "model.test.non_custom_color_model" my_model_config = manifest.nodes[model_id].config my_model_docs = manifest.nodes[model_id].docs node_color_actual_config = my_model_config["docs"].node_color show_actual_config = my_model_config["docs"].show node_color_actual_docs = my_model_docs.node_color show_actual_docs = my_model_docs.show # check node_color config is in the right spots for each model assert node_color_actual_config == CUSTOM_NODE_COLOR_SCHEMA_LEVEL assert node_color_actual_docs == CUSTOM_NODE_COLOR_SCHEMA_LEVEL # in this case show should be True since the schema.yml overrides the dbt_project.yml assert show_actual_config assert show_actual_docs # validation that docs: show configured in model file supercedes dbt_project.yml class TestModelOverProjectColorConfigs(BaseCustomNodeColorModelvsProject): @pytest.fixture(scope="class") def models(self): return {"show_docs_override_model.sql": models__show_docs_false__model_sql} def test__model_show_overrides_dbt_project( self, project, ): run_dbt(["compile"]) manifest = get_manifest(project.project_root) model_id = "model.test.show_docs_override_model" my_model_config = manifest.nodes[model_id].config my_model_docs = manifest.nodes[model_id].docs node_color_actual_config = my_model_config["docs"].node_color show_actual_config = my_model_config["docs"].show node_color_actual_docs = my_model_docs.node_color show_actual_docs = my_model_docs.show # check node_color config is in the right spots for each model assert node_color_actual_config == CUSTOM_NODE_COLOR_PROJECT_LEVEL_ROOT assert node_color_actual_docs == CUSTOM_NODE_COLOR_PROJECT_LEVEL_ROOT # in this case show should be True since the schema.yml overrides the dbt_project.yml assert show_actual_config assert show_actual_docs # validation that an incorrect color in dbt_project.yml raises an exception class TestCustomNodeColorIncorrectColorProject: @pytest.fixture(scope="class") def models(self): # noqa: F811 return {"non_custom_node_color.sql": models__non_custom_node_color__model_sql} @pytest.fixture(scope="class") def project_config_update(self): return { "models": { "test": {"+docs": {"node_color": CUSTOM_NODE_COLOR_INVALID_NAME, "show": False}} } } def test__invalid_color_project( self, project, ): with pytest.raises(ValidationError): run_dbt(["compile"]) # validation that an incorrect color in the config block raises an exception class TestCustomNodeColorIncorrectColorModelConfig: @pytest.fixture(scope="class") def models(self): return { "custom_node_color_invalid_hex.sql": models__custom_node_color_invalid_hex__model_sql } @pytest.fixture(scope="class") def project_config_update(self): return {"models": {"+docs": {"node_color": "blue", "show": False}}} def test__invalid_color_config_block( self, project, ): with pytest.raises((ValidationError, ConfigUpdateError)): run_dbt(["compile"]) # validation that an incorrect color in the YML file raises an exception class TestCustomNodeColorIncorrectColorNameYMLConfig: @pytest.fixture(scope="class") def models(self): return { "non_custom_node_color.sql": models__non_custom_node_color__model_sql, "invalid_custom_color.yml": models__non_custom_node_color_invalid_docs__schema_yml, } @pytest.fixture(scope="class") def project_config_update(self): return {"models": {"+docs": {"node_color": "blue", "show": False}}} def test__invalid_color_docs_not_under_config( self, project, ): with pytest.raises(ValidationError): run_dbt(["compile"]) class TestCustomNodeColorIncorrectColorHEXYMLConfig: @pytest.fixture(scope="class") def models(self): return { "non_custom_node_color.sql": models__non_custom_node_color__model_sql, "invalid_custom_color.yml": models__non_custom_node_color_invalid_config_docs__schema_yml, } def test__invalid_color_docs_under_config( self, project, ): with pytest.raises(ValidationError): run_dbt(["compile"]) ================================================ FILE: tests/functional/configs/test_disabled_configs.py ================================================ import pytest from dbt.tests.util import run_dbt from tests.functional.configs.fixtures import BaseConfigProject class TestDisabledConfigs(BaseConfigProject): @pytest.fixture(scope="class") def dbt_profile_data(self, unique_schema): return { "test": { "outputs": { "default": { "type": "postgres", # make sure you can do this and get an int out "threads": "{{ (1 + 3) | as_number }}", "host": "localhost", "port": "{{ (5400 + 32) | as_number }}", "user": "root", "pass": "password", "dbname": "dbt", "schema": unique_schema, }, "disabled": { "type": "postgres", # make sure you can do this and get an int out "threads": "{{ (1 + 3) | as_number }}", "host": "localhost", "port": "{{ (5400 + 32) | as_number }}", "user": "root", "pass": "password", "dbname": "dbt", "schema": unique_schema, }, }, "target": "default", }, } @pytest.fixture(scope="class") def project_config_update(self): return { "models": { "test": { "enabled": "{{ (target.name == 'default' | as_bool) }}", }, }, # set the `var` result in schema.yml to be 'seed', so that the # `source` call can suceed. "vars": { "test": { "seed_name": "seed", } }, "seeds": { "quote_columns": False, "test": { "seed": { "enabled": "{{ (target.name == 'default') | as_bool }}", }, }, }, "data_tests": { "test": { "enabled": "{{ (target.name == 'default') | as_bool }}", "severity": "WARN", }, }, } def test_disable_seed_partial_parse(self, project): run_dbt(["--partial-parse", "seed", "--target", "disabled"]) run_dbt(["--partial-parse", "seed", "--target", "disabled"]) def test_conditional_model(self, project): # no seeds/models - enabled should eval to False because of the target results = run_dbt(["seed", "--target", "disabled"]) assert len(results) == 0 results = run_dbt(["run", "--target", "disabled"]) assert len(results) == 0 results = run_dbt(["test", "--target", "disabled"]) assert len(results) == 0 # has seeds/models - enabled should eval to True because of the target results = run_dbt(["seed"]) assert len(results) == 1 results = run_dbt(["run"]) assert len(results) == 2 results = run_dbt(["test"]) assert len(results) == 5 my_analysis_sql = """ {{ config(enabled=False) }} select 1 as id """ schema_yml = """ models: - name: my_analysis description: "A Sample model" config: meta: owner: Joe analyses: - name: my_analysis description: "A sample analysis" config: enabled: false """ class TestDisabledConfigsSameName: @pytest.fixture(scope="class") def models(self): return { "my_analysis.sql": my_analysis_sql, "schema.yml": schema_yml, } @pytest.fixture(scope="class") def analyses(self): return { "my_analysis.sql": my_analysis_sql, } def test_disabled_analysis(self, project): manifest = run_dbt(["parse"]) assert len(manifest.disabled) == 2 assert len(manifest.nodes) == 0 ================================================ FILE: tests/functional/configs/test_disabled_model.py ================================================ import pytest from dbt.exceptions import CompilationError, ParsingError, SchemaConfigError from dbt.tests.util import get_manifest, run_dbt from tests.functional.configs.fixtures import ( my_model, my_model_2, my_model_2_disabled, my_model_2_enabled, my_model_3, my_model_3_disabled, my_model_3_enabled, schema_all_disabled_yml, schema_explicit_enabled_yml, schema_invalid_enabled_yml, schema_partial_disabled_yml, schema_partial_enabled_yml, ) # ensure double disabled doesn't throw error when set at schema level class TestSchemaDisabledConfigs: @pytest.fixture(scope="class") def models(self): return { "schema.yml": schema_all_disabled_yml, "my_model.sql": my_model, "my_model_2.sql": my_model_2, "my_model_3.sql": my_model_3, } def test_disabled_config(self, project): run_dbt(["parse"]) # ensure this throws a specific error that the model is disabled class TestSchemaDisabledConfigsFailure: @pytest.fixture(scope="class") def models(self): return { "schema.yml": schema_partial_disabled_yml, "my_model.sql": my_model, "my_model_2.sql": my_model_2, "my_model_3.sql": my_model_3, } def test_disabled_config(self, project): with pytest.raises(CompilationError) as exc: run_dbt(["parse"]) exc_str = " ".join(str(exc.value).split()) # flatten all whitespace expected_msg = "which is disabled" assert expected_msg in exc_str # ensure double disabled doesn't throw error when set in model configs class TestModelDisabledConfigs: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_model, "my_model_2.sql": my_model_2_disabled, "my_model_3.sql": my_model_3_disabled, } def test_disabled_config(self, project): run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "model.test.my_model_2" not in manifest.nodes assert "model.test.my_model_3" not in manifest.nodes assert "model.test.my_model_2" in manifest.disabled assert "model.test.my_model_3" in manifest.disabled # ensure config set in project.yml can be overridden in yaml file class TestOverrideProjectConfigsInYaml: @pytest.fixture(scope="class") def models(self): return { "schema.yml": schema_partial_enabled_yml, "my_model.sql": my_model, "my_model_2.sql": my_model_2, "my_model_3.sql": my_model_3, } @pytest.fixture(scope="class") def project_config_update(self): return { "models": { "test": { "my_model_2": { "enabled": False, }, "my_model_3": { "enabled": False, }, }, } } def test_override_project_yaml_config(self, project): run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "model.test.my_model_2" in manifest.nodes assert "model.test.my_model_3" not in manifest.nodes assert "model.test.my_model_2" not in manifest.disabled assert "model.test.my_model_3" in manifest.disabled # ensure config set in project.yml can be overridden in sql file class TestOverrideProjectConfigsInSQL: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_model, "my_model_2.sql": my_model_2_enabled, "my_model_3.sql": my_model_3, } @pytest.fixture(scope="class") def project_config_update(self): return { "models": { "test": { "my_model_2": { "enabled": False, }, "my_model_3": { "enabled": False, }, }, } } def test_override_project_sql_config(self, project): run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "model.test.my_model_2" in manifest.nodes assert "model.test.my_model_3" not in manifest.nodes assert "model.test.my_model_2" not in manifest.disabled assert "model.test.my_model_3" in manifest.disabled # ensure false config set in yaml file can be overridden in sql file class TestOverrideFalseYAMLConfigsInSQL: @pytest.fixture(scope="class") def models(self): return { "schema.yml": schema_all_disabled_yml, "my_model.sql": my_model, "my_model_2.sql": my_model_2_enabled, "my_model_3.sql": my_model_3, } def test_override_yaml_sql_config(self, project): run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "model.test.my_model_2" in manifest.nodes assert "model.test.my_model_3" not in manifest.nodes assert "model.test.my_model_2" not in manifest.disabled assert "model.test.my_model_3" in manifest.disabled # ensure true config set in yaml file can be overridden by false in sql file class TestOverrideTrueYAMLConfigsInSQL: @pytest.fixture(scope="class") def models(self): return { "schema.yml": schema_explicit_enabled_yml, "my_model.sql": my_model, "my_model_2.sql": my_model_2_enabled, "my_model_3.sql": my_model_3_disabled, } def test_override_yaml_sql_config(self, project): run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "model.test.my_model_2" in manifest.nodes assert "model.test.my_model_3" not in manifest.nodes assert "model.test.my_model_2" not in manifest.disabled assert "model.test.my_model_3" in manifest.disabled # ensure error when enabling in schema file when multiple nodes exist within disabled class TestMultipleDisabledNodesForUniqueIDFailure: @pytest.fixture(scope="class") def models(self): return { "schema.yml": schema_partial_enabled_yml, "my_model.sql": my_model, "folder_1": { "my_model_2.sql": my_model_2_disabled, "my_model_3.sql": my_model_3_disabled, }, "folder_2": { "my_model_2.sql": my_model_2_disabled, "my_model_3.sql": my_model_3_disabled, }, "folder_3": { "my_model_2.sql": my_model_2_disabled, "my_model_3.sql": my_model_3_disabled, }, } def test_disabled_config(self, project): with pytest.raises(ParsingError) as exc: run_dbt(["parse"]) exc_str = " ".join(str(exc.value).split()) # flatten all whitespace expected_msg = "Found 3 matching disabled nodes for model 'my_model_2'" assert expected_msg in exc_str # ensure error when enabling in schema file when multiple nodes exist within disabled class TestMultipleDisabledNodesSuccess: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_model, "folder_1": { "my_model_2.sql": my_model_2, "my_model_3.sql": my_model_3, }, "folder_2": { "my_model_2.sql": my_model_2, "my_model_3.sql": my_model_3, }, } @pytest.fixture(scope="class") def project_config_update(self): return { "models": { "test": { "folder_1": { "enabled": False, }, "folder_2": { "enabled": True, }, }, } } def test_multiple_disabled_config(self, project): run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "model.test.my_model_2" in manifest.nodes assert "model.test.my_model_3" in manifest.nodes expected_file_path = "folder_2" assert expected_file_path in manifest.nodes["model.test.my_model_2"].original_file_path assert expected_file_path in manifest.nodes["model.test.my_model_3"].original_file_path assert "model.test.my_model_2" in manifest.disabled assert "model.test.my_model_3" in manifest.disabled expected_disabled_file_path = "folder_1" assert ( expected_disabled_file_path in manifest.disabled["model.test.my_model_2"][0].original_file_path ) assert ( expected_disabled_file_path in manifest.disabled["model.test.my_model_3"][0].original_file_path ) # ensure overrides work when enabling in sql file when multiple nodes exist within disabled class TestMultipleDisabledNodesOverrideModel: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_model, "folder_1": { "my_model_2.sql": my_model_2_enabled, "my_model_3.sql": my_model_3, }, "folder_2": { "my_model_2.sql": my_model_2, "my_model_3.sql": my_model_3_enabled, }, } @pytest.fixture(scope="class") def project_config_update(self): return { "models": { "test": { "folder_1": { "enabled": False, }, "folder_2": { "enabled": False, }, }, } } def test_multiple_disabled_config(self, project): run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "model.test.my_model_2" in manifest.nodes assert "model.test.my_model_3" in manifest.nodes expected_file_path_2 = "folder_1" assert expected_file_path_2 in manifest.nodes["model.test.my_model_2"].original_file_path expected_file_path_3 = "folder_2" assert expected_file_path_3 in manifest.nodes["model.test.my_model_3"].original_file_path assert "model.test.my_model_2" in manifest.disabled assert "model.test.my_model_3" in manifest.disabled expected_disabled_file_path_2 = "folder_2" assert ( expected_disabled_file_path_2 in manifest.disabled["model.test.my_model_2"][0].original_file_path ) expected_disabled_file_path_3 = "folder_1" assert ( expected_disabled_file_path_3 in manifest.disabled["model.test.my_model_3"][0].original_file_path ) # ensure everything lands where it should when disabling multiple nodes with the same unique id class TestManyDisabledNodesSuccess: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_model, "folder_1": { "my_model_2.sql": my_model_2, "my_model_3.sql": my_model_3, }, "folder_2": { "my_model_2.sql": my_model_2, "my_model_3.sql": my_model_3, }, "folder_3": { "my_model_2.sql": my_model_2, "my_model_3.sql": my_model_3, }, "folder_4": { "my_model_2.sql": my_model_2, "my_model_3.sql": my_model_3, }, } @pytest.fixture(scope="class") def project_config_update(self): return { "models": { "test": { "folder_1": { "enabled": False, }, "folder_2": { "enabled": True, }, "folder_3": { "enabled": False, }, "folder_4": { "enabled": False, }, }, } } def test_many_disabled_config(self, project): run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "model.test.my_model_2" in manifest.nodes assert "model.test.my_model_3" in manifest.nodes expected_file_path = "folder_2" assert expected_file_path in manifest.nodes["model.test.my_model_2"].original_file_path assert expected_file_path in manifest.nodes["model.test.my_model_3"].original_file_path assert len(manifest.disabled["model.test.my_model_2"]) == 3 assert len(manifest.disabled["model.test.my_model_3"]) == 3 class TestInvalidEnabledConfig: @pytest.fixture(scope="class") def models(self): return { "schema.yml": schema_invalid_enabled_yml, "my_model.sql": my_model, } def test_invalid_config(self, project): with pytest.raises(SchemaConfigError) as exc: run_dbt(["parse"]) exc_str = " ".join(str(exc.value).split()) # flatten all whitespace expected_msg = "'True and False' is not of type 'boolean'" assert expected_msg in exc_str ================================================ FILE: tests/functional/configs/test_dupe_paths.py ================================================ import pytest from dbt.tests.util import run_dbt my_model_sql = """ select 1 as fun """ seed_csv = """id,value 4,2 """ somedoc_md = """ {% docs somedoc %} Testing, testing {% enddocs %} """ schema_yml = """ version: 2 models: - name: my_model description: testing model """ # Either a docs or a yml file is necessary to see the problem # when two of the paths in 'all_source_paths' are the same class TestDupeProjectPaths: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_model_sql, "seed.csv": seed_csv, "somedoc.md": somedoc_md, "schema.yml": schema_yml, } @pytest.fixture(scope="class") def project_config_update(self): return { "model-paths": ["models"], "seed-paths": ["models"], } def test_config_with_dupe_paths(self, project, dbt_project_yml): results = run_dbt(["seed"]) assert len(results) == 1 results = run_dbt(["run"]) assert len(results) == 1 class TestDupeStrippedProjectPaths: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_model_sql, "seed.csv": seed_csv, "somedoc.md": somedoc_md, "schema.yml": schema_yml, } @pytest.fixture(scope="class") def project_config_update(self): return { "model-paths": ["models/"], "seed-paths": ["models"], } def test_config_with_dupe_paths(self, project, dbt_project_yml): results = run_dbt(["seed"]) assert len(results) == 1 results = run_dbt(["run"]) assert len(results) == 1 ================================================ FILE: tests/functional/configs/test_get_default.py ================================================ import pytest from dbt.tests.util import run_dbt, write_file models_get__any_model_sql = """ -- models/any_model.sql select {{ config.get('made_up_nonexistent_key', 'default_value') }} as col_value """ meta_model_get_sql = """ -- models/meta_model.sql select {{ config.get('meta_key', 'meta_default_value') }} as col_value """ meta_model_meta_get_sql = """ -- models/meta_model.sql select {{ config.meta_get('meta_key', 'meta_default_value') }} as col_value """ schema_yml = """ models: - name: meta_model config: meta: meta_key: my_meta_value """ meta_model_require_sql = """ -- models/meta_model.sql select {{ config.require('meta_key') }} as col_value """ meta_model_meta_require_sql = """ -- models/meta_model.sql select {{ config.meta_require('meta_key') }} as col_value """ class TestConfigGetDefault: @pytest.fixture(scope="class") def models(self): return {"any_model.sql": models_get__any_model_sql} def test_config_with_get_default( self, project, ): # This test runs a model with a config.get(key, default) # The default value is 'default_value' and causes an error results = run_dbt(["run"], expect_pass=False) assert len(results) == 1 assert str(results[0].status) == "error" assert 'column "default_value" does not exist' in results[0].message class TestConfigGetMeta: @pytest.fixture(scope="class") def models(self): return { "meta_model.sql": meta_model_get_sql, "schema.yml": schema_yml, } def test_config_with_meta_key( self, project, ): # This test runs a model with a config.get(key, default) -> default value returned results = run_dbt(["run"], expect_pass=False) assert len(results) == 1 assert str(results[0].status) == "error" assert 'column "meta_default_value" does not exist' in results[0].message write_file(meta_model_meta_get_sql, "models", "meta_model.sql") results = run_dbt(["run"], expect_pass=False) assert len(results) == 1 assert str(results[0].status) == "error" assert 'column "my_meta_value" does not exist' in results[0].message class TestConfigGetMetaRequire: @pytest.fixture(scope="class") def models(self): return { "meta_model.sql": meta_model_require_sql, "schema.yml": schema_yml, } def test_config_with_meta_require( self, project, ): # This test runs a model with a config.require(key) results = run_dbt(["run"], expect_pass=False) assert len(results) == 1 assert str(results[0].status) == "error" assert "does not define a required config parameter 'meta_key'" in results[0].message write_file(meta_model_meta_require_sql, "models", "meta_model.sql") results = run_dbt(["run"], expect_pass=False) assert len(results) == 1 assert str(results[0].status) == "error" assert 'column "my_meta_value" does not exist' in results[0].message ================================================ FILE: tests/functional/configs/test_grant_configs.py ================================================ import pytest from dbt.tests.util import get_manifest, run_dbt, write_config_file, write_file dbt_project_yml = """ models: test: my_model: +grants: my_select: ["reporter", "bi"] """ append_schema_yml = """ version: 2 models: - name: my_model config: grants: +my_select: ["someone"] """ my_model_base_sql = """ select 1 as fun """ my_model_clobber_sql = """ {{ config(grants={'my_select': ['other_user']}) }} select 1 as fun """ my_model_extend_sql = """ {{ config(grants={'+my_select': ['other_user']}) }} select 1 as fun """ my_model_extend_string_sql = """ {{ config(grants={'+my_select': 'other_user'}) }} select 1 as fun """ my_model_extend_twice_sql = """ {{ config(grants={'+my_select': ['other_user']}) }} {{ config(grants={'+my_select': ['alt_user']}) }} select 1 as fun """ class TestGrantConfigs: @pytest.fixture(scope="class") def models(self): return {"my_model.sql": my_model_base_sql} @pytest.fixture(scope="class") def project_config_update(self): return dbt_project_yml def test_model_grant_config(self, project, logs_dir): # This test uses "my_select" instead of "select", so we need # use "parse" instead of "run" because we will get compilation # errors for the grants. run_dbt(["parse"]) manifest = get_manifest(project.project_root) model_id = "model.test.my_model" assert model_id in manifest.nodes model = manifest.nodes[model_id] model_config = model.config assert hasattr(model_config, "grants") # no schema grant, no model grant, just project expected = {"my_select": ["reporter", "bi"]} assert model_config.grants == expected # add model grant with clobber write_file(my_model_clobber_sql, project.project_root, "models", "my_model.sql") run_dbt(["parse"]) manifest = get_manifest(project.project_root) model_config = manifest.nodes[model_id].config expected = {"my_select": ["other_user"]} assert model_config.grants == expected # change model to extend grants write_file(my_model_extend_sql, project.project_root, "models", "my_model.sql") run_dbt(["parse"]) manifest = get_manifest(project.project_root) model_config = manifest.nodes[model_id].config expected = {"my_select": ["reporter", "bi", "other_user"]} assert model_config.grants == expected # add schema file with extend write_file(append_schema_yml, project.project_root, "models", "schema.yml") run_dbt(["parse"]) manifest = get_manifest(project.project_root) model_config = manifest.nodes[model_id].config expected = {"my_select": ["reporter", "bi", "someone", "other_user"]} assert model_config.grants == expected # change model file to have string instead of list write_file(my_model_extend_string_sql, project.project_root, "models", "my_model.sql") run_dbt(["parse"]) manifest = get_manifest(project.project_root) model_config = manifest.nodes[model_id].config expected = {"my_select": ["reporter", "bi", "someone", "other_user"]} assert model_config.grants == expected # change model file to have string instead of list write_file(my_model_extend_twice_sql, project.project_root, "models", "my_model.sql") run_dbt(["parse"]) manifest = get_manifest(project.project_root) model_config = manifest.nodes[model_id].config expected = {"my_select": ["reporter", "bi", "someone", "other_user", "alt_user"]} assert model_config.grants == expected # Remove grant from dbt_project config = { "config-version": 2, "name": "test", "version": "0.1.0", "profile": "test", "log-path": logs_dir, } write_config_file(config, project.project_root, "dbt_project.yml") run_dbt(["parse"]) manifest = get_manifest(project.project_root) model_config = manifest.nodes[model_id].config expected = {"my_select": ["someone", "other_user", "alt_user"]} assert model_config.grants == expected # Remove my_model config, leaving only schema file write_file(my_model_base_sql, project.project_root, "models", "my_model.sql") run_dbt(["parse"]) manifest = get_manifest(project.project_root) model_config = manifest.nodes[model_id].config expected = {"my_select": ["someone"]} assert model_config.grants == expected ================================================ FILE: tests/functional/configs/test_indiv_tests.py ================================================ import pytest from dbt.tests.util import run_dbt from tests.functional.configs.fixtures import BaseConfigProject class TestConfigIndivTests(BaseConfigProject): @pytest.fixture(scope="class") def project_config_update(self): return { "seeds": { "quote_columns": False, }, "vars": { "test": { "seed_name": "seed", } }, "data_tests": {"test": {"enabled": True, "severity": "WARN"}}, } def test_configuring_individual_tests( self, project, ): assert len(run_dbt(["seed"])) == 1 assert len(run_dbt(["run"])) == 2 # all tests on (minus sleeper_agent) + WARN assert len(run_dbt(["test"])) == 5 # turn off two of them directly assert len(run_dbt(["test", "--vars", '{"enabled_direct": False}'])) == 3 # turn on sleeper_agent data test directly assert ( len( run_dbt( ["test", "--models", "sleeper_agent", "--vars", '{"enabled_direct": True}'] ) ) == 1 ) # set three to ERROR directly results = run_dbt( [ "test", "--models", "config.severity:error", "--vars", '{"enabled_direct": True, "severity_direct": "ERROR"}', ], expect_pass=False, ) assert len(results) == 2 assert results[0].status == "fail" assert results[1].status == "fail" ================================================ FILE: tests/functional/configs/test_unused_configs.py ================================================ import pytest from dbt.exceptions import CompilationError from dbt.tests.util import run_dbt seeds__seed_csv = """id,value 4,2 """ class TestUnusedModelConfigs: @pytest.fixture(scope="class") def seeds(self): return {"seed.csv": seeds__seed_csv} @pytest.fixture(scope="class") def project_config_update(self): return { "test-paths": ["does-not-exist"], "models": { "test": { "enabled": True, } }, "seeds": { "quote_columns": False, }, "sources": { "test": { "enabled": True, } }, "data_tests": { "test": { "enabled": True, } }, } def test_warn_unused_configuration_paths( self, project, ): with pytest.raises(CompilationError) as excinfo: run_dbt(["--warn-error", "seed"]) assert "Configuration paths exist" in str(excinfo.value) assert "- sources.test" in str(excinfo.value) assert "- models.test" in str(excinfo.value) assert "- models.test" in str(excinfo.value) run_dbt(["seed"]) ================================================ FILE: tests/functional/configs/test_vars_file.py ================================================ import pytest import yaml from dbt.artifacts.schemas.results import RunStatus from dbt.exceptions import DbtProjectError from dbt.tests.util import get_manifest, relation_from_name, run_dbt from dbt_common.exceptions import CompilationError # ============================================================================= # Fixtures - Models, Macros, etc. # ============================================================================= # SQL model that uses a var model_with_var_sql = """ {{ config(materialized='table') }} select '{{ var("my_var") }}' as my_var_value """ # SQL model that uses multiple vars model_with_multiple_vars_sql = """ {{ config(materialized='table') }} select '{{ var("var_one") }}' as var_one_value, '{{ var("var_two") }}' as var_two_value """ # SQL model that uses a package-scoped var model_with_package_var_sql = """ {{ config(materialized='table') }} select '{{ var("package_var") }}' as package_var_value """ # SQL model that calls a macro which uses a var model_calling_macro_sql = """ {{ config(materialized='table') }} select '{{ get_var_value() }}' as macro_var_value """ # Macro that uses a var macro_with_var_sql = """ {% macro get_var_value() -%} {{ var("macro_var") }} {%- endmacro %} """ class TestDbtProjectVarFromVarsFile: """dbt_project.yml with var should be rendered properly when var is set through vars.yml""" @pytest.fixture(scope="class") def models(self): return {"model_with_var.sql": model_with_var_sql} @pytest.fixture(scope="class") def project_config_update(self): return { "models": { "+meta": { "project_var": "{{ var('project_var') }}", } } } @pytest.fixture(scope="class") def vars_yml_update(self): return {"vars": {"my_var": "from_file", "project_var": "project_var_from_file"}} def test_dbt_project_var_from_vars_file(self, project): results = run_dbt(["run"]) assert len(results) == 1 assert results[0].status == RunStatus.Success manifest = get_manifest(project.project_root) assert manifest is not None model = manifest.nodes["model.test.model_with_var"] assert model.config.meta["project_var"] == "project_var_from_file" class TestDbtProjectVarCliOverridesFile: """dbt_project.yml with var should use CLI value when set in both vars.yml and cli""" @pytest.fixture(scope="class") def models(self): return {"model_with_var.sql": model_with_var_sql} @pytest.fixture(scope="class") def project_config_update(self): return { "models": { "+meta": { "project_var": "{{ var('project_var') }}", } } } @pytest.fixture(scope="class") def vars_yml_update(self): return {"vars": {"my_var": "from_file", "project_var": "from_file"}} def test_cli_overrides_vars_file(self, project): cli_vars = {"my_var": "from_cli", "project_var": "from_cli"} results = run_dbt(["run", "--vars", yaml.safe_dump(cli_vars)]) assert len(results) == 1 # Verify CLI value was used in SQL model relation = relation_from_name(project.adapter, "model_with_var") result = project.run_sql(f"select my_var_value from {relation}", fetch="one") assert result[0] == "from_cli" # Verify CLI value was used to render dbt_project.yml manifest = get_manifest(project.project_root) model = manifest.nodes["model.test.model_with_var"] assert model.config.meta["project_var"] == "from_cli" class TestDbtProjectVarMissingFromVarsFile: """dbt should throw error if dbt_project.yml expects a var not present in vars.yml""" @pytest.fixture(scope="class") def models(self): return {"model_with_var.sql": model_with_var_sql} @pytest.fixture(scope="class") def project_config_update(self): return { "models": { "+meta": { "missing_var": "{{ var('missing_var') }}", } } } @pytest.fixture(scope="class") def vars_yml_update(self): return {"vars": {"my_var": "from_file"}} def test_error_when_var_missing( self, project_root, profiles_root, profiles_yml, dbt_project_yml ): # This test expects an error during project setup, so we can't use the project fixture. # We must pass --project-dir and --profiles-dir explicitly since the adapter fixture # (which normally sets these flags) is not used. with pytest.raises( CompilationError, match="Required var 'missing_var' not found in config" ): run_dbt( [ "run", "--project-dir", str(project_root), "--profiles-dir", str(profiles_root), ], expect_pass=False, ) class TestSqlModelVarFromVarsFile: """SQL model with variable should be rendered properly when var is set through vars.yml""" @pytest.fixture(scope="class") def models(self): return {"model_with_var.sql": model_with_var_sql} @pytest.fixture(scope="class") def vars_yml_update(self, project_root): return {"vars": {"my_var": "sql_var_from_file"}} def test_sql_model_var_from_file(self, project): results = run_dbt(["run"]) assert len(results) == 1 relation = relation_from_name(project.adapter, "model_with_var") result = project.run_sql(f"select my_var_value from {relation}", fetch="one") assert result[0] == "sql_var_from_file" class TestSqlModelVarCliOverridesFile: """SQL model with variable should use CLI value when set in both vars.yml and cli""" @pytest.fixture(scope="class") def models(self): return {"model_with_var.sql": model_with_var_sql} @pytest.fixture(scope="class") def vars_yml_udpate(self, project_root): return {"vars": {"my_var": "from_file"}} def test_sql_model_cli_overrides_file(self, project): cli_vars = {"my_var": "from_cli"} results = run_dbt(["run", "--vars", yaml.safe_dump(cli_vars)]) assert len(results) == 1 relation = relation_from_name(project.adapter, "model_with_var") result = project.run_sql(f"select my_var_value from {relation}", fetch="one") assert result[0] == "from_cli" class TestSqlModelVarMissingFromVarsFile: """dbt should throw an error if sql model expects a var not set in vars.yml""" @pytest.fixture(scope="class") def models(self): # Model expects 'my_var' but it won't be provided return {"model_with_var.sql": model_with_var_sql} @pytest.fixture(scope="class") def vars_yml(self, project_root): return {"vars": {"other_var": "value"}} def test_error_when_model_var_missing(self, project): # run_dbt with expect_pass=False doesn't raise, it returns results with errors results = run_dbt(["run"], expect_pass=False) # The run should have an error for the missing var assert len(results) == 1 assert results[0].status == "error" class TestMacroVarFromVarsFile: """Macro using var should be rendered properly when var is set through vars.yml""" @pytest.fixture(scope="class") def models(self): return {"model_calling_macro.sql": model_calling_macro_sql} @pytest.fixture(scope="class") def macros(self): return {"my_macro.sql": macro_with_var_sql} @pytest.fixture(scope="class") def vars_yml_update(self, project_root): return {"vars": {"macro_var": "macro_var_from_file"}} def test_macro_var_from_file(self, project): results = run_dbt(["run"]) assert len(results) == 1 relation = relation_from_name(project.adapter, "model_calling_macro") result = project.run_sql(f"select macro_var_value from {relation}", fetch="one") assert result[0] == "macro_var_from_file" class TestMacroVarCliOverridesFile: """Macro using var should use CLI value when set in both vars.yml and cli""" @pytest.fixture(scope="class") def models(self): return {"model_calling_macro.sql": model_calling_macro_sql} @pytest.fixture(scope="class") def macros(self): return {"my_macro.sql": macro_with_var_sql} @pytest.fixture(scope="class") def vars_yml(self, project_root): return {"vars": {"macro_var": "from_file"}} def test_macro_var_cli_overrides_file(self, project): cli_vars = {"macro_var": "from_cli"} results = run_dbt(["run", "--vars", yaml.safe_dump(cli_vars)]) assert len(results) == 1 relation = relation_from_name(project.adapter, "model_calling_macro") result = project.run_sql(f"select macro_var_value from {relation}", fetch="one") assert result[0] == "from_cli" class TestMutualExclusivityError: """Project should throw error when variables are set in both vars.yml and dbt_project.yml""" @pytest.fixture(scope="class") def models(self): return {"model_with_var.sql": model_with_var_sql} @pytest.fixture(scope="class") def project_config_update(self): return { "name": "test", "vars": { "my_var": "from_project", }, } @pytest.fixture(scope="class") def vars_yml_update(self, project_root): return {"vars": {"my_var": "from_file"}} def test_error_when_both_have_vars( self, project_root, profiles_root, profiles_yml, dbt_project_yml ): # run_dbt catches exceptions, so we use run_dbt_and_capture to check output with pytest.raises( DbtProjectError, match="Variables cannot be defined in both vars.yml and dbt_project.yml.", ): run_dbt( [ "run", "--project-dir", str(project_root), "--profiles-dir", str(profiles_root), ], expect_pass=False, ) class TestEmptyVarsFileAllowsProjectVars: """If vars.yml file is empty, vars from dbt_project.yml should be used""" @pytest.fixture(scope="class") def models(self): return {"model_with_var.sql": model_with_var_sql} @pytest.fixture(scope="class") def project_config_update(self): return { "vars": { "my_var": "from_project", } } @pytest.fixture(scope="class") def vars_yml_update(self, project_root): return {} def test_empty_vars_file_uses_project_vars(self, project): results = run_dbt(["run"]) assert len(results) == 1 relation = relation_from_name(project.adapter, "model_with_var") result = project.run_sql(f"select my_var_value from {relation}", fetch="one") assert result[0] == "from_project" class TestVarsFileWithoutVarsKeyAllowsProjectVars: """Vars declared in vars.yml without a top level 'vars' key should use vars from dbt_project.yml""" @pytest.fixture(scope="class") def models(self): return {"model_with_var.sql": model_with_var_sql} @pytest.fixture(scope="class") def project_config_update(self): return { "vars": { "my_var": "from_project", } } @pytest.fixture(scope="class") def vars_yml_update(self, project_root): return {"other_key": "some_value"} def test_vars_file_without_vars_key_uses_project_vars(self, project): results = run_dbt(["run"]) assert len(results) == 1 relation = relation_from_name(project.adapter, "model_with_var") result = project.run_sql(f"select my_var_value from {relation}", fetch="one") assert result[0] == "from_project" class TestPartialCliOverride: """Variables from vars.yml and CLI should be merged""" @pytest.fixture(scope="class") def models(self): return {"model_with_multiple_vars.sql": model_with_multiple_vars_sql} @pytest.fixture(scope="class") def vars_yml_update(self, project_root): return { "vars": { "var_two": "var_two_from_file", } } def test_partial_cli_override(self, project): # Only override var_one, var_two should come from file cli_vars = {"var_one": "var_one_from_cli"} results = run_dbt(["run", "--vars", yaml.safe_dump(cli_vars)]) assert len(results) == 1 relation = relation_from_name(project.adapter, "model_with_multiple_vars") result = project.run_sql( f"select var_one_value, var_two_value from {relation}", fetch="one" ) assert result[0] == "var_one_from_cli" # From CLI assert result[1] == "var_two_from_file" # From file class TestComplexVarValues: """Complex var values like lists and dicts should work from vars.yml""" @pytest.fixture(scope="class") def models(self): return { "model_with_list_var.sql": """ {{ config(materialized='table') }} select '{{ var("list_var") | join(",") }}' as list_value """, "model_with_dict_var.sql": """ {{ config(materialized='table') }} select '{{ var("dict_var").key1 }}' as dict_value """, } @pytest.fixture(scope="class") def vars_yml_update(self, project_root): return { "vars": { "list_var": ["a", "b", "c"], "dict_var": {"key1": "value1", "key2": "value2"}, } } def test_complex_var_values(self, project): results = run_dbt(["run"]) assert len(results) == 2 list_relation = relation_from_name(project.adapter, "model_with_list_var") list_result = project.run_sql(f"select list_value from {list_relation}", fetch="one") assert list_result[0] == "a,b,c" dict_relation = relation_from_name(project.adapter, "model_with_dict_var") dict_result = project.run_sql(f"select dict_value from {dict_relation}", fetch="one") assert dict_result[0] == "value1" ================================================ FILE: tests/functional/configs/test_versioned_model_constraint.py ================================================ import pytest from dbt.exceptions import ParsingError from dbt.tests.util import get_manifest, rm_file, run_dbt, write_file schema_yml = """ models: - name: foo config: materialized: table contract: enforced: true constraints: - type: primary_key columns: [id, user_name] columns: - name: id data_type: int constraints: - type: not_null - name: user_name data_type: text """ foo_sql = """ select 1 as id, 'alice' as user_name """ foo_v2_sql = """ select 1 as id, 'alice' as user_name, 2 as another_pk """ versioned_schema_yml = """ models: - name: foo latest_version: 1 config: materialized: table contract: enforced: true constraints: - type: primary_key columns: [id, user_name] columns: - name: id data_type: int constraints: - type: not_null - name: user_name data_type: text versions: - v: 1 """ versioned_pk_model_column_schema_yml = """ models: - name: foo latest_version: 2 config: materialized: table contract: enforced: true constraints: - type: primary_key columns: [id] columns: - name: id data_type: int constraints: - type: not_null - name: user_name data_type: text versions: - v: 1 - v: 2 columns: - name: id data_type: int constraints: - type: not_null - type: primary_key - name: user_name data_type: text """ versioned_pk_mult_columns_schema_yml = """ models: - name: foo latest_version: 2 config: materialized: table contract: enforced: true columns: - name: id data_type: int constraints: - type: not_null - type: primary_key - name: user_name data_type: text versions: - v: 1 - v: 2 columns: - name: id data_type: int constraints: - type: not_null - type: primary_key - name: user_name data_type: text constraints: - type: primary_key """ class TestVersionedModelConstraints: @pytest.fixture(scope="class") def models(self): return { "foo.sql": foo_sql, "schema.yml": schema_yml, } def test_versioned_model_constraints(self, project): results = run_dbt(["run"]) assert len(results) == 1 manifest = get_manifest(project.project_root) model_node = manifest.nodes["model.test.foo"] assert len(model_node.constraints) == 1 # remove foo.sql and create foo_v1.sql rm_file(project.project_root, "models", "foo.sql") write_file(foo_sql, project.project_root, "models", "foo_v1.sql") write_file(versioned_schema_yml, project.project_root, "models", "schema.yml") results = run_dbt(["run"]) assert len(results) == 1 manifest = get_manifest(project.project_root) model_node = manifest.nodes["model.test.foo.v1"] assert model_node.contract.enforced is True assert len(model_node.constraints) == 1 # test primary key defined across model and column level constraints, expect error class TestPrimaryKeysModelAndColumnLevelConstraints: @pytest.fixture(scope="class") def models(self): return { "foo.sql": foo_sql, "schema.yml": schema_yml, } def test_model_column_pk_error(self, project): results = run_dbt(["run"]) assert len(results) == 1 manifest = get_manifest(project.project_root) model_node = manifest.nodes["model.test.foo"] assert len(model_node.constraints) == 1 # remove foo.sql and create foo_v1.sql rm_file(project.project_root, "models", "foo.sql") write_file(foo_sql, project.project_root, "models", "foo_v1.sql") write_file(versioned_schema_yml, project.project_root, "models", "schema.yml") results = run_dbt(["run"]) assert len(results) == 1 manifest = get_manifest(project.project_root) model_node = manifest.nodes["model.test.foo.v1"] assert model_node.contract.enforced is True assert len(model_node.constraints) == 1 # add foo_v2.sql write_file(foo_sql, project.project_root, "models", "foo_v2.sql") write_file( versioned_pk_model_column_schema_yml, project.project_root, "models", "schema.yml" ) expected_error = "Primary key constraints defined at the model level and the columns level" with pytest.raises(ParsingError) as exc_info: run_dbt(["run"]) assert expected_error in str(exc_info.value) # test primary key defined across multiple columns, expect error class TestPrimaryKeysMultipleColumns: @pytest.fixture(scope="class") def models(self): return { "foo.sql": foo_sql, "schema.yml": schema_yml, } def test_pk_multiple_columns(self, project): results = run_dbt(["run"]) assert len(results) == 1 manifest = get_manifest(project.project_root) model_node = manifest.nodes["model.test.foo"] assert len(model_node.constraints) == 1 # remove foo.sql and create foo_v1.sql rm_file(project.project_root, "models", "foo.sql") write_file(foo_sql, project.project_root, "models", "foo_v1.sql") write_file(versioned_schema_yml, project.project_root, "models", "schema.yml") results = run_dbt(["run"]) assert len(results) == 1 manifest = get_manifest(project.project_root) model_node = manifest.nodes["model.test.foo.v1"] assert model_node.contract.enforced is True assert len(model_node.constraints) == 1 # add foo_v2.sql write_file(foo_sql, project.project_root, "models", "foo_v2.sql") write_file( versioned_pk_mult_columns_schema_yml, project.project_root, "models", "schema.yml" ) expected_error = ( "Found 2 columns (['id', 'user_name']) with primary key constraints defined" ) with pytest.raises(ParsingError) as exc_info: run_dbt(["run"]) assert expected_error in str(exc_info.value) ================================================ FILE: tests/functional/configs/test_warn_error_options.py ================================================ from typing import Any, Dict, Union import pytest from dbt.cli.main import dbtRunner, dbtRunnerResult from dbt.events.types import ( DeprecatedModel, MainEncounteredError, MicrobatchModelNoEventTimeInputs, ) from dbt.flags import get_flags from dbt.tests.util import run_dbt, update_config_file from dbt_common.events.base_types import EventLevel from dbt_common.events.event_catcher import EventCatcher ModelsDictSpec = Dict[str, Union[str, "ModelsDictSpec"]] my_model_sql = """SELECT 1 AS id, 'cats are cute' AS description""" schema_yml = """ version: 2 models: - name: my_model deprecation_date: 2020-01-01 """ class BaseTestWarnErrorOptions: @pytest.fixture(scope="class") def models(self) -> ModelsDictSpec: return {"my_model.sql": my_model_sql, "schema.yml": schema_yml} @pytest.fixture(scope="function") def catcher(self) -> EventCatcher: return EventCatcher(event_to_catch=DeprecatedModel) @pytest.fixture(scope="function") def runner(self, catcher: EventCatcher) -> dbtRunner: return dbtRunner(callbacks=[catcher.catch]) def assert_deprecation_warning(self, result: dbtRunnerResult, catcher: EventCatcher) -> None: assert result.success assert result.exception is None assert len(catcher.caught_events) == 1 assert catcher.caught_events[0].info.level == EventLevel.WARN.value def assert_deprecation_error(self, result: dbtRunnerResult) -> None: assert not result.success assert result.exception is not None assert "Model my_model has passed its deprecation date of" in str(result.exception) class TestWarnErrorOptionsFromCLICanSilence(BaseTestWarnErrorOptions): def test_can_silence(self, project, catcher: EventCatcher, runner: dbtRunner) -> None: result = runner.invoke(["run"]) self.assert_deprecation_warning(result, catcher) catcher.flush() result = runner.invoke(["run", "--warn-error-options", "{'silence': ['DeprecatedModel']}"]) assert result.success assert len(catcher.caught_events) == 0 class TestWarnErrorOptionsFromCLICanRaiseWarningToError(BaseTestWarnErrorOptions): def test_can_raise_warning_to_error( self, project, catcher: EventCatcher, runner: dbtRunner ) -> None: result = runner.invoke(["run"]) self.assert_deprecation_warning(result, catcher) catcher.flush() result = runner.invoke(["run", "--warn-error-options", "{'include': ['DeprecatedModel']}"]) self.assert_deprecation_error(result) catcher.flush() result = runner.invoke( [ "run", "--warn-error-options", "{'include': 'all', 'warn': ['DeprecationsSummary', 'WEOIncludeExcludeDeprecation']}", ] ) self.assert_deprecation_error(result) catcher.flush() result = runner.invoke(["run", "--warn-error-options", "{'error': ['DeprecatedModel']}"]) self.assert_deprecation_error(result) catcher.flush() result = runner.invoke( ["run", "--warn-error-options", "{'error': 'all', 'warn': ['DeprecationsSummary']}"] ) self.assert_deprecation_error(result) class TestWarnErrorOptionsFromCLICanExcludeSpecificEvent(BaseTestWarnErrorOptions): def test_can_exclude_specific_event( self, project, catcher: EventCatcher, runner: dbtRunner ) -> None: result = runner.invoke( ["run", "--warn-error-options", "{'error': 'all', 'warn': ['DeprecationsSummary']}"] ) self.assert_deprecation_error(result) catcher.flush() result = runner.invoke( [ "run", "--warn-error-options", "{'error': 'all', 'exclude': ['DeprecatedModel', 'WEOIncludeExcludeDeprecation', 'DeprecationsSummary']}", ] ) self.assert_deprecation_warning(result, catcher) catcher.flush() result = runner.invoke( [ "run", "--warn-error-options", "{'error': 'all', 'warn': ['DeprecatedModel', 'DeprecationsSummary']}", ] ) self.assert_deprecation_warning(result, catcher) class TestWarnErrorOptionsFromCLICantSetBothIncludeAndError(BaseTestWarnErrorOptions): def test_cant_set_both_include_and_error(self, project, runner: dbtRunner) -> None: result = runner.invoke( ["run", "--warn-error-options", "{'include': 'all', 'error': 'all'}"] ) assert not result.success assert result.exception is not None assert "Only `include` or `error` can be specified" in str(result.exception) def test_cant_set_both_exclude_and_warn(self, project, runner: dbtRunner) -> None: result = runner.invoke( [ "run", "--warn-error-options", "{'include': 'all', 'exclude': ['DeprecatedModel'], 'warn': ['DeprecatedModel']}", ] ) assert not result.success assert result.exception is not None assert "Only `exclude` or `warn` can be specified" in str(result.exception) class BaseTestWarnErrorOptionsFromProject(BaseTestWarnErrorOptions): @pytest.fixture(scope="function") def clear_project_flags(self, project_root) -> None: # TODO: Is this still necessary now that the project based tests are broken into separate test classes? flags: Dict[str, Any] = {"flags": {}} update_config_file(flags, project_root, "dbt_project.yml") class TestWarnErrorOptionsFromProjectCanSilence(BaseTestWarnErrorOptionsFromProject): def test_can_silence( self, project, clear_project_flags, project_root, catcher: EventCatcher, runner: dbtRunner ) -> None: result = runner.invoke(["run"]) self.assert_deprecation_warning(result, catcher) silence_options = {"flags": {"warn_error_options": {"silence": ["DeprecatedModel"]}}} update_config_file(silence_options, project_root, "dbt_project.yml") catcher.flush() result = runner.invoke(["run"]) assert result.success assert len(catcher.caught_events) == 0 class TestWarnErrorOptionsFromProjectCanRaiseWarningToError(BaseTestWarnErrorOptionsFromProject): def test_can_raise_warning_to_error( self, project, clear_project_flags, project_root, catcher: EventCatcher, runner: dbtRunner ) -> None: result = runner.invoke(["run"]) self.assert_deprecation_warning(result, catcher) warn_error_options: Dict[str, Any] = { "flags": {"warn_error_options": {"error": ["DeprecatedModel"]}} } update_config_file(warn_error_options, project_root, "dbt_project.yml") catcher.flush() result = runner.invoke(["run"]) self.assert_deprecation_error(result) warn_error_options = { "flags": {"warn_error_options": {"error": "all", "warn": ["DeprecationsSummary"]}} } update_config_file(warn_error_options, project_root, "dbt_project.yml") catcher.flush() result = runner.invoke(["run"]) self.assert_deprecation_error(result) class TestWarnErrorOptionsFromProjectCanExcludeSpecificEvent(BaseTestWarnErrorOptionsFromProject): @pytest.mark.skip( reason="Flaky on structured logging tests, EventCatcher inexplicably picks up on 'include' usage across classes" ) def test_can_exclude_specific_event( self, project, clear_project_flags, project_root, catcher: EventCatcher, runner: dbtRunner ) -> None: warn_error_options: Dict[str, Any] = { "flags": {"warn_error_options": {"error": "all", "warn": ["DeprecationsSummary"]}} } update_config_file(warn_error_options, project_root, "dbt_project.yml") result = runner.invoke(["run"]) self.assert_deprecation_error(result) warn_error_options = { "flags": { "warn_error_options": { "error": "all", "warn": ["DeprecatedModel", "DeprecationsSummary"], } } } update_config_file(warn_error_options, project_root, "dbt_project.yml") catcher.flush() result = runner.invoke(["run"]) self.assert_deprecation_warning(result, catcher) class TestWarnErrorOptionsFromProjectCantSetBothIncludeAndError( BaseTestWarnErrorOptionsFromProject ): def test_cant_set_both_include_and_error( self, project, clear_project_flags, project_root, runner: dbtRunner ) -> None: warn_error_options = {"flags": {"warn_error_options": {"include": "all", "error": "all"}}} update_config_file(warn_error_options, project_root, "dbt_project.yml") result = runner.invoke(["run"]) assert not result.success assert result.exception is not None assert "Only `include` or `error` can be specified" in str(result.exception) class TestWarnErrorOptionsFromProjectCantSetBothExcludeAndWarn( BaseTestWarnErrorOptionsFromProject ): def test_cant_set_both_exclude_and_warn( self, project, clear_project_flags, project_root, runner: dbtRunner ) -> None: warn_error_options = { "flags": { "warn_error_options": { "error": "all", "exclude": ["DeprecatedModel"], "warn": ["DeprecatedModel"], } } } update_config_file(warn_error_options, project_root, "dbt_project.yml") result = runner.invoke(["run"]) assert not result.success assert result.exception is not None assert "Only `exclude` or `warn` can be specified" in str(result.exception) class TestEmptyWarnError: @pytest.fixture(scope="class") def models(self): return {"my_model.sql": my_model_sql, "schema.yml": schema_yml} # This tests for a bug in creating WarnErrorOptions when warn or # error are set to None (in yaml = warn:) def test_project_flags(self, project): project_flags = { "flags": { "send_anonymous_usage_stats": False, "warn_error_options": { "warn": None, "error": None, "silence": ["TestsConfigDeprecation"], }, } } update_config_file(project_flags, project.project_root, "dbt_project.yml") run_dbt(["run"]) flags = get_flags() assert flags.warn_error_options.silence == ["TestsConfigDeprecation"] input_model_without_event_time_sql = """ {{ config(materialized='table') }} select 1 as id, TIMESTAMP '2020-01-01 00:00:00-0' as event_time union all select 2 as id, TIMESTAMP '2020-01-02 00:00:00-0' as event_time union all select 3 as id, TIMESTAMP '2020-01-03 00:00:00-0' as event_time """ microbatch_model_sql = """ {{config(materialized='incremental', incremental_strategy='microbatch', unique_key='id', event_time='event_time', batch_size='day', begin=modules.datetime.datetime.now())}} SELECT id, event_time FROM {{ ref('input_model') }} """ class TestRequireAllWarningsHandledByWarnErrorBehaviorFlag: @pytest.fixture(scope="class") def models(self): return { "input_model.sql": input_model_without_event_time_sql, "microbatch_model.sql": microbatch_model_sql, } def test_require_all_warnings_handed_by_warn_error_behavior_flag(self, project): # Setup the event catchers microbatch_warning_catcher = EventCatcher(event_to_catch=MicrobatchModelNoEventTimeInputs) microbatch_error_catcher = EventCatcher(event_to_catch=MainEncounteredError) dbt_runner = dbtRunner( callbacks=[microbatch_warning_catcher.catch, microbatch_error_catcher.catch] ) # Run the command without the behavior flag off project_flags = { "flags": { "send_anonymous_usage_stats": False, "require_all_warnings_handled_by_warn_error": False, } } update_config_file(project_flags, project.project_root, "dbt_project.yml") dbt_runner.invoke(["run", "--warn-error"]) assert len(microbatch_warning_catcher.caught_events) == 1 assert len(microbatch_error_catcher.caught_events) == 0 # Reset the event catchers microbatch_warning_catcher.flush() microbatch_error_catcher.flush() # Run the command with the behavior flag on project_flags = { "flags": { "send_anonymous_usage_stats": False, "require_all_warnings_handled_by_warn_error": True, } } update_config_file(project_flags, project.project_root, "dbt_project.yml") dbt_runner.invoke(["run", "--warn-error", "--log-format", "json"]) assert len(microbatch_warning_catcher.caught_events) == 0 assert len(microbatch_error_catcher.caught_events) == 1 ================================================ FILE: tests/functional/conftest.py ================================================ import pytest from tests.functional.fixtures.happy_path_fixture import ( # noqa:D happy_path_project, happy_path_project_files, ) @pytest.fixture(scope="function", autouse=True) def clear_memoized_get_package_with_retries(): # This fixture is used to clear the memoized cache for _get_package_with_retries # in dbt.clients.registry. This is necessary because the cache is shared across # tests and can cause unexpected behavior if not cleared as some tests depend on # the deprecation warning that _get_package_with_retries fires yield from dbt.clients.registry import _get_cached _get_cached.cache = {} @pytest.fixture(autouse=True) def clear_buffered_deprecations(): # buffered_deprecations is a module-level list that can retain stale entries # across tests in the same process. This happens when Flags.__init__ buffers # a deprecation (via normalize_warn_error_options) but then raises before # fire_buffered_deprecations() is called to drain and clear the buffer. # The stale entry then fires in the next test's invocation, potentially # causing spurious failures (e.g. WEOIncludeExcludeDeprecation + --warn-error). from dbt.deprecations import buffered_deprecations buffered_deprecations.clear() ================================================ FILE: tests/functional/constraints/fixtures.py ================================================ model_foreign_key_model_schema_yml = """ models: - name: my_model constraints: - type: foreign_key columns: [id] to: ref('my_model_to') to_columns: [id] columns: - name: id data_type: integer """ model_foreign_key_source_schema_yml = """ sources: - name: test_source tables: - name: test_table models: - name: my_model constraints: - type: foreign_key columns: [id] to: source('test_source', 'test_table') to_columns: [id] columns: - name: id data_type: integer """ model_foreign_key_model_node_not_found_schema_yml = """ models: - name: my_model constraints: - type: foreign_key columns: [id] to: ref('doesnt_exist') to_columns: [id] columns: - name: id data_type: integer """ model_foreign_key_model_invalid_syntax_schema_yml = """ models: - name: my_model constraints: - type: foreign_key columns: [id] to: invalid to_columns: [id] columns: - name: id data_type: integer """ model_foreign_key_model_column_schema_yml = """ models: - name: my_model columns: - name: id data_type: integer constraints: - type: foreign_key to: ref('my_model_to') to_columns: [id] """ model_foreign_key_column_invalid_syntax_schema_yml = """ models: - name: my_model columns: - name: id data_type: integer constraints: - type: foreign_key to: invalid to_columns: [id] """ model_foreign_key_column_node_not_found_schema_yml = """ models: - name: my_model columns: - name: id data_type: integer constraints: - type: foreign_key to: ref('doesnt_exist') to_columns: [id] """ model_column_level_foreign_key_source_schema_yml = """ sources: - name: test_source tables: - name: test_table models: - name: my_model columns: - name: id data_type: integer constraints: - type: foreign_key to: source('test_source', 'test_table') to_columns: [id] """ stateful_generate_alias_name_macros_sql = """ {% macro generate_alias_name(custom_alias_name, node) -%} {{ node.name }}_{{ var("state", "dev") }} {%- endmacro %} """ ================================================ FILE: tests/functional/constraints/test_foreign_key_constraints.py ================================================ import os import shutil import pytest from dbt.artifacts.resources import RefArgs from dbt.exceptions import CompilationError, ParsingError from dbt.tests.util import get_artifact, run_dbt from dbt_common.contracts.constraints import ( ColumnLevelConstraint, ConstraintType, ModelLevelConstraint, ) from tests.functional.constraints.fixtures import ( model_column_level_foreign_key_source_schema_yml, model_foreign_key_column_invalid_syntax_schema_yml, model_foreign_key_column_node_not_found_schema_yml, model_foreign_key_model_column_schema_yml, model_foreign_key_model_invalid_syntax_schema_yml, model_foreign_key_model_node_not_found_schema_yml, model_foreign_key_model_schema_yml, model_foreign_key_source_schema_yml, stateful_generate_alias_name_macros_sql, ) class TestModelLevelForeignKeyConstraintToRef: @pytest.fixture(scope="class") def models(self): return { "constraints_schema.yml": model_foreign_key_model_schema_yml, "my_model.sql": "select 1 as id", "my_model_to.sql": "select 1 as id", } def test_model_level_fk_to(self, project, unique_schema): manifest = run_dbt(["parse"]) node_with_fk_constraint = manifest.nodes["model.test.my_model"] assert len(node_with_fk_constraint.constraints) == 1 parsed_constraint = node_with_fk_constraint.constraints[0] assert parsed_constraint == ModelLevelConstraint( type=ConstraintType.foreign_key, columns=["id"], to="ref('my_model_to')", to_columns=["id"], ) # Assert column-level constraint source included in node.depends_on assert node_with_fk_constraint.refs == [RefArgs("my_model_to")] assert node_with_fk_constraint.depends_on.nodes == ["model.test.my_model_to"] assert node_with_fk_constraint.sources == [] # Assert compilation renders to from 'ref' to relation identifer run_dbt(["compile"]) manifest = get_artifact(project.project_root, "target", "manifest.json") assert len(manifest["nodes"]["model.test.my_model"]["constraints"]) == 1 compiled_constraint = manifest["nodes"]["model.test.my_model"]["constraints"][0] assert compiled_constraint["to"] == f'"dbt"."{unique_schema}"."my_model_to"' # Other constraint fields should remain as parsed assert compiled_constraint["to_columns"] == parsed_constraint.to_columns assert compiled_constraint["columns"] == parsed_constraint.columns assert compiled_constraint["type"] == parsed_constraint.type class TestModelLevelForeignKeyConstraintToSource: @pytest.fixture(scope="class") def models(self): return { "constraints_schema.yml": model_foreign_key_source_schema_yml, "my_model.sql": "select 1 as id", "my_model_to.sql": "select 1 as id", } def test_model_level_fk_to(self, project, unique_schema): manifest = run_dbt(["parse"]) node_with_fk_constraint = manifest.nodes["model.test.my_model"] assert len(node_with_fk_constraint.constraints) == 1 parsed_constraint = node_with_fk_constraint.constraints[0] assert parsed_constraint == ModelLevelConstraint( type=ConstraintType.foreign_key, columns=["id"], to="source('test_source', 'test_table')", to_columns=["id"], ) # Assert column-level constraint source included in node.depends_on assert node_with_fk_constraint.refs == [] assert node_with_fk_constraint.depends_on.nodes == ["source.test.test_source.test_table"] assert node_with_fk_constraint.sources == [["test_source", "test_table"]] # Assert compilation renders to from 'ref' to relation identifer run_dbt(["compile"]) manifest = get_artifact(project.project_root, "target", "manifest.json") assert len(manifest["nodes"]["model.test.my_model"]["constraints"]) == 1 compiled_constraint = manifest["nodes"]["model.test.my_model"]["constraints"][0] assert compiled_constraint["to"] == '"dbt"."test_source"."test_table"' # Other constraint fields should remain as parsed assert compiled_constraint["to_columns"] == parsed_constraint.to_columns assert compiled_constraint["columns"] == parsed_constraint.columns assert compiled_constraint["type"] == parsed_constraint.type class TestModelLevelForeignKeyConstraintRefNotFoundError: @pytest.fixture(scope="class") def models(self): return { "constraints_schema.yml": model_foreign_key_model_node_not_found_schema_yml, "my_model.sql": "select 1 as id", "my_model_to.sql": "select 1 as id", } def test_model_level_fk_to_doesnt_exist(self, project): with pytest.raises( CompilationError, match="depends on a node named 'doesnt_exist' which was not found" ): run_dbt(["parse"]) class TestModelLevelForeignKeyConstraintRefSyntaxError: @pytest.fixture(scope="class") def models(self): return { "constraints_schema.yml": model_foreign_key_model_invalid_syntax_schema_yml, "my_model.sql": "select 1 as id", "my_model_to.sql": "select 1 as id", } def test_model_level_fk_to(self, project): with pytest.raises( ParsingError, match="Invalid 'ref' or 'source' syntax on foreign key constraint 'to' on model my_model: invalid", ): run_dbt(["parse"]) class TestColumnLevelForeignKeyConstraintToRef: @pytest.fixture(scope="class") def models(self): return { "constraints_schema.yml": model_foreign_key_model_column_schema_yml, "my_model.sql": "select 1 as id", "my_model_to.sql": "select 1 as id", } def test_column_level_fk_to(self, project, unique_schema): manifest = run_dbt(["parse"]) node_with_fk_constraint = manifest.nodes["model.test.my_model"] assert len(node_with_fk_constraint.columns["id"].constraints) == 1 parsed_constraint = node_with_fk_constraint.columns["id"].constraints[0] # Assert column-level constraint parsed assert parsed_constraint == ColumnLevelConstraint( type=ConstraintType.foreign_key, to="ref('my_model_to')", to_columns=["id"] ) # Assert column-level constraint ref included in node.depends_on assert node_with_fk_constraint.refs == [RefArgs(name="my_model_to")] assert node_with_fk_constraint.sources == [] assert node_with_fk_constraint.depends_on.nodes == ["model.test.my_model_to"] # Assert compilation renders to from 'ref' to relation identifer run_dbt(["compile"]) manifest = get_artifact(project.project_root, "target", "manifest.json") assert len(manifest["nodes"]["model.test.my_model"]["columns"]["id"]["constraints"]) == 1 compiled_constraint = manifest["nodes"]["model.test.my_model"]["columns"]["id"][ "constraints" ][0] assert compiled_constraint["to"] == f'"dbt"."{unique_schema}"."my_model_to"' # Other constraint fields should remain as parsed assert compiled_constraint["to_columns"] == parsed_constraint.to_columns assert compiled_constraint["type"] == parsed_constraint.type class TestColumnLevelForeignKeyConstraintToSource: @pytest.fixture(scope="class") def models(self): return { "constraints_schema.yml": model_column_level_foreign_key_source_schema_yml, "my_model.sql": "select 1 as id", "my_model_to.sql": "select 1 as id", } def test_model_level_fk_to(self, project, unique_schema): manifest = run_dbt(["parse"]) node_with_fk_constraint = manifest.nodes["model.test.my_model"] assert len(node_with_fk_constraint.columns["id"].constraints) == 1 parsed_constraint = node_with_fk_constraint.columns["id"].constraints[0] assert parsed_constraint == ColumnLevelConstraint( type=ConstraintType.foreign_key, to="source('test_source', 'test_table')", to_columns=["id"], ) # Assert column-level constraint source included in node.depends_on assert node_with_fk_constraint.refs == [] assert node_with_fk_constraint.depends_on.nodes == ["source.test.test_source.test_table"] assert node_with_fk_constraint.sources == [["test_source", "test_table"]] # Assert compilation renders to from 'ref' to relation identifer run_dbt(["compile"]) manifest = get_artifact(project.project_root, "target", "manifest.json") assert len(manifest["nodes"]["model.test.my_model"]["columns"]["id"]["constraints"]) == 1 compiled_constraint = manifest["nodes"]["model.test.my_model"]["columns"]["id"][ "constraints" ][0] assert compiled_constraint["to"] == '"dbt"."test_source"."test_table"' # # Other constraint fields should remain as parsed assert compiled_constraint["to_columns"] == parsed_constraint.to_columns assert compiled_constraint["type"] == parsed_constraint.type class TestColumnLevelForeignKeyConstraintRefNotFoundError: @pytest.fixture(scope="class") def models(self): return { "constraints_schema.yml": model_foreign_key_column_node_not_found_schema_yml, "my_model.sql": "select 1 as id", "my_model_to.sql": "select 1 as id", } def test_model_level_fk_to_doesnt_exist(self, project): with pytest.raises( CompilationError, match="depends on a node named 'doesnt_exist' which was not found" ): run_dbt(["parse"]) class TestColumnLevelForeignKeyConstraintRefSyntaxError: @pytest.fixture(scope="class") def models(self): return { "constraints_schema.yml": model_foreign_key_column_invalid_syntax_schema_yml, "my_model.sql": "select 1 as id", "my_model_to.sql": "select 1 as id", } def test_model_level_fk_to(self, project): with pytest.raises( ParsingError, match="Invalid 'ref' or 'source' syntax on foreign key constraint 'to' on model my_model: invalid.", ): run_dbt(["parse"]) class BaseForeignKeyDeferState: @pytest.fixture(scope="class") def macros(self): return { "generate_alias_name.sql": stateful_generate_alias_name_macros_sql, } def copy_state(self, project_root): state_path = os.path.join(project_root, "state") if not os.path.exists(state_path): os.makedirs(state_path) shutil.copyfile( f"{project_root}/target/manifest.json", f"{project_root}/state/manifest.json" ) class TestModelLevelForeignKeyConstraintRefToDeferRelation(BaseForeignKeyDeferState): """Test FK constraint uses deferred relation when FK target is NOT selected.""" @pytest.fixture(scope="class") def models(self): return { "constraints_schema.yml": model_foreign_key_model_schema_yml, "my_model.sql": "select 1 as id", "my_model_to.sql": "select 1 as id", } def test_model_level_fk_to_defer_relation_when_target_not_selected(self, project): """When FK target is NOT selected, use deferred (prod) relation.""" results = run_dbt(["run", "--vars", "state: prod"]) self.copy_state(project.project_root) # Only select my_model, not my_model_to - FK should use deferred relation results = run_dbt(["compile", "-s", "my_model", "--defer", "--state", "state"]) my_model_node = [r.node for r in results.results if r.node.name == "my_model"][0] assert my_model_node.constraints[0].to.split(".")[-1] == '"my_model_to_prod"' def test_model_level_fk_to_current_relation_when_target_selected(self, project): """When FK target IS selected (being built), use current relation, not deferred.""" results = run_dbt(["run", "--vars", "state: prod"]) self.copy_state(project.project_root) # Select both models - FK should use current (dev) relation since target is being built results = run_dbt(["compile", "--defer", "--state", "state"]) my_model_node = [r.node for r in results.results if r.node.name == "my_model"][0] # When both models are selected, FK should point to current (dev) relation assert my_model_node.constraints[0].to.split(".")[-1] == '"my_model_to_dev"' class TestColumnLevelForeignKeyConstraintToRefDeferRelation(BaseForeignKeyDeferState): """Test column-level FK constraint uses deferred relation when FK target is NOT selected.""" @pytest.fixture(scope="class") def models(self): return { "constraints_schema.yml": model_foreign_key_model_column_schema_yml, "my_model.sql": "select 1 as id", "my_model_to.sql": "select 1 as id", } def test_column_level_fk_to_defer_relation_when_target_not_selected(self, project): """When FK target is NOT selected, use deferred (prod) relation.""" results = run_dbt(["run", "--vars", "state: prod"]) self.copy_state(project.project_root) # Only select my_model, not my_model_to - FK should use deferred relation results = run_dbt(["compile", "-s", "my_model", "--defer", "--state", "state"]) my_model_node = [r.node for r in results.results if r.node.name == "my_model"][0] assert my_model_node.columns["id"].constraints[0].to.split(".")[-1] == '"my_model_to_prod"' def test_column_level_fk_to_current_relation_when_target_selected(self, project): """When FK target IS selected (being built), use current relation, not deferred.""" results = run_dbt(["run", "--vars", "state: prod"]) self.copy_state(project.project_root) # Select both models - FK should use current (dev) relation since target is being built results = run_dbt(["compile", "--defer", "--state", "state"]) my_model_node = [r.node for r in results.results if r.node.name == "my_model"][0] # When both models are selected, FK should point to current (dev) relation assert my_model_node.columns["id"].constraints[0].to.split(".")[-1] == '"my_model_to_dev"' ================================================ FILE: tests/functional/context_methods/first_dependency.py ================================================ import pytest from dbt.tests.fixtures.project import write_project_files first_dependency__dbt_project_yml = """ name: 'first_dep' version: '1.0' config-version: 2 profile: 'default' model-paths: ["models"] analysis-paths: ["analyses"] test-paths: ["tests"] seed-paths: ["seeds"] macro-paths: ["macros"] require-dbt-version: '>=0.1.0' target-path: "target" # directory which will store compiled SQL files clean-targets: # directories to be removed by `dbt clean` - "target" - "dbt_packages" vars: first_dep: first_dep_global: 'first_dep_global_value_overridden' test_config_root_override: 'configured_from_dependency' test_config_package: 'configured_from_dependency' seeds: quote_columns: True """ first_dependency__models__nested__first_dep_model_sql = """ select '{{ var("first_dep_global") }}' as first_dep_global, '{{ var("from_root_to_first") }}' as from_root """ first_dependency__seeds__first_dep_expected_csv = """first_dep_global,from_root first_dep_global_value_overridden,root_first_value """ first_dependency__models__nested__first_dep_model_var_expected_csv = """test_config_root_override,test_config_package configured_from_root,configured_from_dependency """ first_dependency__models__nested__first_dep_model_var_sql = """ select '{{ config.get("test_config_root_override") }}' as test_config_root_override, '{{ config.get("test_config_package") }}' as test_config_package """ first_dependency__model_var_in_config_schema = """ models: - name: first_dep_model config: test_config_root_override: "{{ var('test_config_root_override') }}" test_config_package: "{{ var('test_config_package') }}" """ class FirstDependencyProject: @pytest.fixture(scope="class") def first_dependency(self, project): first_dependency_files = { "dbt_project.yml": first_dependency__dbt_project_yml, "models": { "nested": { "first_dep_model.sql": first_dependency__models__nested__first_dep_model_sql } }, "seeds": {"first_dep_expected.csv": first_dependency__seeds__first_dep_expected_csv}, } write_project_files(project.project_root, "first_dependency", first_dependency_files) class FirstDependencyConfigProject: @pytest.fixture(scope="class") def first_dependency(self, project): first_dependency_files = { "dbt_project.yml": first_dependency__dbt_project_yml, "models": { "nested": { "first_dep_model.sql": first_dependency__models__nested__first_dep_model_var_sql, "schema.yml": first_dependency__model_var_in_config_schema, } }, "seeds": { "first_dep_expected.csv": first_dependency__models__nested__first_dep_model_var_expected_csv }, } write_project_files(project.project_root, "first_dependency", first_dependency_files) ================================================ FILE: tests/functional/context_methods/test_builtin_functions.py ================================================ import json import os import pytest from dbt.exceptions import CompilationError from dbt.tests.util import run_dbt, run_dbt_and_capture, write_file macros__validate_set_sql = """ {% macro validate_set() %} {% set set_result = set([1, 2, 2, 3, 'foo', False]) %} {{ log("set_result: " ~ set_result) }} {% set set_strict_result = set_strict([1, 2, 2, 3, 'foo', False]) %} {{ log("set_strict_result: " ~ set_strict_result) }} {% endmacro %} """ macros__validate_zip_sql = """ {% macro validate_zip() %} {% set list_a = [1, 2] %} {% set list_b = ['foo', 'bar'] %} {% set zip_result = zip(list_a, list_b) | list %} {{ log("zip_result: " ~ zip_result) }} {% set zip_strict_result = zip_strict(list_a, list_b) | list %} {{ log("zip_strict_result: " ~ zip_strict_result) }} {% endmacro %} """ macros__validate_invocation_sql = """ {% macro validate_invocation(my_variable) %} -- check a specific value {{ log("use_colors: "~ invocation_args_dict['use_colors']) }} -- whole dictionary (as string) {{ log("invocation_result: "~ invocation_args_dict) }} {% endmacro %} """ macros__validate_dbt_metadata_envs_sql = """ {% macro validate_dbt_metadata_envs() %} {{ log("dbt_metadata_envs_result:"~ dbt_metadata_envs) }} {% endmacro %} """ models__set_exception_sql = """ {% set set_strict_result = set_strict(1) %} """ models__zip_exception_sql = """ {% set zip_strict_result = zip_strict(1) %} """ def parse_json_logs(json_log_output): parsed_logs = [] for line in json_log_output.split("\n"): try: log = json.loads(line) except ValueError: continue parsed_logs.append(log) return parsed_logs def find_result_in_parsed_logs(parsed_logs, result_name): return next( ( item["data"]["msg"] for item in parsed_logs if result_name in item["data"].get("msg", "msg") ), False, ) class TestContextBuiltins: @pytest.fixture(scope="class") def macros(self): return { "validate_set.sql": macros__validate_set_sql, "validate_zip.sql": macros__validate_zip_sql, "validate_invocation.sql": macros__validate_invocation_sql, "validate_dbt_metadata_envs.sql": macros__validate_dbt_metadata_envs_sql, } def test_builtin_set_function(self, project): _, log_output = run_dbt_and_capture(["--debug", "run-operation", "validate_set"]) # The order of the set isn't guaranteed so we can't check for the actual set in the logs assert "set_result: " in log_output assert "False" in log_output assert "set_strict_result: " in log_output def test_builtin_zip_function(self, project): _, log_output = run_dbt_and_capture(["--debug", "run-operation", "validate_zip"]) expected_zip = [(1, "foo"), (2, "bar")] assert f"zip_result: {expected_zip}" in log_output assert f"zip_strict_result: {expected_zip}" in log_output def test_builtin_invocation_args_dict_function(self, project): _, log_output = run_dbt_and_capture( [ "--debug", "--log-format=json", "run-operation", "validate_invocation", "--args", "{my_variable: test_variable}", ] ) parsed_logs = parse_json_logs(log_output) use_colors = result = find_result_in_parsed_logs(parsed_logs, "use_colors") assert use_colors == "use_colors: True" invocation_dict = find_result_in_parsed_logs(parsed_logs, "invocation_result") assert result # The result should include a dictionary of all flags with values that aren't None expected = ( "'send_anonymous_usage_stats': False", "'quiet': False", "'print': True", "'cache_selected_only': False", "'macro': 'validate_invocation'", "'args': {'my_variable': 'test_variable'}", "'which': 'run-operation'", "'indirect_selection': 'eager'", ) assert all(element in invocation_dict for element in expected) def test_builtin_dbt_metadata_envs_function(self, project, monkeypatch): envs = { "DBT_ENV_CUSTOM_ENV_RUN_ID": "1234", "DBT_ENV_CUSTOM_ENV_JOB_ID": "5678", "DBT_ENV_RUN_ID": "91011", "RANDOM_ENV": "121314", } monkeypatch.setattr(os, "environ", envs) _, log_output = run_dbt_and_capture( ["--debug", "--log-format=json", "run-operation", "validate_dbt_metadata_envs"] ) parsed_logs = parse_json_logs(log_output) result = find_result_in_parsed_logs(parsed_logs, "dbt_metadata_envs_result") assert result expected = "dbt_metadata_envs_result:{'RUN_ID': '1234', 'JOB_ID': '5678'}" assert expected in str(result) class TestContextBuiltinExceptions: # Assert compilation errors are raised with _strict equivalents def test_builtin_function_exception(self, project): write_file(models__set_exception_sql, project.project_root, "models", "raise.sql") with pytest.raises(CompilationError): run_dbt(["compile"]) write_file(models__zip_exception_sql, project.project_root, "models", "raise.sql") with pytest.raises(CompilationError): run_dbt(["compile"]) ================================================ FILE: tests/functional/context_methods/test_cli_var_override.py ================================================ import pytest from dbt.tests.util import run_dbt models_override__schema_yml = """ version: 2 models: - name: test_vars columns: - name: field data_tests: - accepted_values: values: - override """ models_override__test_vars_sql = """ select '{{ var("required") }}'::varchar as field """ # Tests that cli vars override vars set in the project config class TestCLIVarOverride: @pytest.fixture(scope="class") def models(self): return { "schema.yml": models_override__schema_yml, "test_vars.sql": models_override__test_vars_sql, } @pytest.fixture(scope="class") def project_config_update(self): return { "vars": { "required": "present", }, } def test__override_vars_global(self, project): run_dbt(["run", "--vars", "{required: override}"]) run_dbt(["test"]) # This one switches to setting a var in 'test' class TestCLIVarOverridePorject: @pytest.fixture(scope="class") def models(self): return { "schema.yml": models_override__schema_yml, "test_vars.sql": models_override__test_vars_sql, } @pytest.fixture(scope="class") def project_config_update(self): return { "vars": { "test": { "required": "present", }, }, } def test__override_vars_project_level(self, project): # This should be "override" run_dbt(["run", "--vars", "{required: override}"]) run_dbt(["test"]) ================================================ FILE: tests/functional/context_methods/test_cli_vars.py ================================================ import pytest import yaml from dbt.exceptions import CompilationError, DbtRuntimeError from dbt.tests.fixtures.project import write_project_files from dbt.tests.util import ( get_artifact, get_logging_events, run_dbt, run_dbt_and_capture, write_config_file, ) from tests.fixtures.dbt_integration_project import dbt_integration_project # noqa: F401 models_complex__schema_yml = """ version: 2 models: - name: complex_model columns: - name: var_1 data_tests: - accepted_values: values: - abc - name: var_2 data_tests: - accepted_values: values: - def - name: var_3 data_tests: - accepted_values: values: - jkl """ models_complex__complex_model_sql = """ select '{{ var("variable_1") }}'::varchar as var_1, '{{ var("variable_2")[0] }}'::varchar as var_2, '{{ var("variable_3")["value"] }}'::varchar as var_3 """ models_simple__schema_yml = """ version: 2 models: - name: simple_model columns: - name: simple data_tests: - accepted_values: values: - abc """ models_simple__simple_model_sql = """ select '{{ var("simple") }}'::varchar as simple """ really_simple_model_sql = """ select 'abc' as simple """ class TestCLIVars: @pytest.fixture(scope="class") def models(self): return { "schema.yml": models_complex__schema_yml, "complex_model.sql": models_complex__complex_model_sql, } def test__cli_vars_longform(self, project): cli_vars = { "variable_1": "abc", "variable_2": ["def", "ghi"], "variable_3": {"value": "jkl"}, } results = run_dbt(["run", "--vars", yaml.dump(cli_vars)]) assert len(results) == 1 results = run_dbt(["test", "--vars", yaml.dump(cli_vars)]) assert len(results) == 3 class TestCLIVarsSimple: @pytest.fixture(scope="class") def models(self): return { "schema.yml": models_simple__schema_yml, "simple_model.sql": models_simple__simple_model_sql, } def test__cli_vars_shorthand(self, project): results = run_dbt(["run", "--vars", "simple: abc"]) assert len(results) == 1 results = run_dbt(["test", "--vars", "simple: abc"]) assert len(results) == 1 def test__cli_vars_longer(self, project): results = run_dbt(["run", "--vars", "{simple: abc, unused: def}"]) assert len(results) == 1 results = run_dbt(["test", "--vars", "{simple: abc, unused: def}"]) assert len(results) == 1 run_results = get_artifact(project.project_root, "target", "run_results.json") assert run_results["args"]["vars"] == {"simple": "abc", "unused": "def"} class TestCLIVarsProfile: @pytest.fixture(scope="class") def models(self): return { "schema.yml": models_simple__schema_yml, "simple_model.sql": really_simple_model_sql, } def test_cli_vars_in_profile(self, project, dbt_profile_data): profile = dbt_profile_data profile["test"]["outputs"]["default"]["host"] = "{{ var('db_host') }}" write_config_file(profile, project.profiles_dir, "profiles.yml") with pytest.raises(DbtRuntimeError): results = run_dbt(["run"]) results = run_dbt(["run", "--vars", "db_host: localhost"]) assert len(results) == 1 class TestCLIVarsPackages: @pytest.fixture(scope="class", autouse=True) def setUp(self, project_root, dbt_integration_project): # noqa: F811 write_project_files(project_root, "dbt_integration_project", dbt_integration_project) @pytest.fixture(scope="class") def models(self): return { "schema.yml": models_simple__schema_yml, "simple_model.sql": really_simple_model_sql, } @pytest.fixture(scope="class") def packages_config(self): return {"packages": [{"local": "dbt_integration_project"}]} def test_cli_vars_in_packages(self, project, packages_config): # Run working deps and run commands run_dbt(["deps"]) results = run_dbt(["run"]) assert len(results) == 1 # Change packages.yml to contain a var packages = packages_config packages["packages"][0]["local"] = "{{ var('path_to_project') }}" write_config_file(packages, project.project_root, "packages.yml") # Without vars args deps fails with pytest.raises(DbtRuntimeError): run_dbt(["deps"]) # With vars arg deps succeeds results = run_dbt(["deps", "--vars", "path_to_project: dbt_integration_project"]) assert results is None initial_selectors_yml = """ selectors: - name: dev_defer_snapshots default: "{{ target.name == 'dev' | as_bool }}" definition: method: fqn value: '*' exclude: - method: config.materialized value: snapshot """ var_selectors_yml = """ selectors: - name: dev_defer_snapshots default: "{{ var('snapshot_target') == 'dev' | as_bool }}" definition: method: fqn value: '*' exclude: - method: config.materialized value: snapshot """ class TestCLIVarsSelectors: @pytest.fixture(scope="class") def models(self): return { "schema.yml": models_simple__schema_yml, "simple_model.sql": really_simple_model_sql, } @pytest.fixture(scope="class") def selectors(self): return initial_selectors_yml def test_vars_in_selectors(self, project): # initially runs ok results = run_dbt(["run"]) assert len(results) == 1 # Update the selectors.yml file to have a var write_config_file(var_selectors_yml, project.project_root, "selectors.yml") with pytest.raises(CompilationError): run_dbt(["run"]) # Var in cli_vars works results = run_dbt(["run", "--vars", "snapshot_target: dev"]) assert len(results) == 1 models_scrubbing__schema_yml = """ version: 2 models: - name: simple_model columns: - name: simple data_tests: - accepted_values: values: - abc """ models_scrubbing__simple_model_sql = """ select '{{ var("DBT_ENV_SECRET_simple") }}'::varchar as simple """ class TestCLIVarsScrubbing: @pytest.fixture(scope="class") def models(self): return { "schema.yml": models_scrubbing__schema_yml, "simple_model.sql": models_scrubbing__simple_model_sql, } def test__run_results_scrubbing(self, project): results, output = run_dbt_and_capture( [ "--debug", "--log-format", "json", "run", "--vars", "{DBT_ENV_SECRET_simple: abc, unused: def}", ] ) assert len(results) == 1 run_results = get_artifact(project.project_root, "target", "run_results.json") assert run_results["args"]["vars"] == { "DBT_ENV_SECRET_simple": "*****", "unused": "def", } log_events = get_logging_events(log_output=output, event_name="StateCheckVarsHash") assert len(log_events) == 1 assert ( log_events[0]["data"]["vars"] == "{'DBT_ENV_SECRET_simple': '*****', 'unused': 'def'}" ) def test__exception_scrubbing(self, project): results, output = run_dbt_and_capture( [ "--debug", "--log-format", "json", "run", "--vars", "{DBT_ENV_SECRET_unused: abc, unused: def}", ], False, ) assert len(results) == 1 log_events = get_logging_events(log_output=output, event_name="CatchableExceptionOnRun") assert len(log_events) == 1 assert ( '{\n "DBT_ENV_SECRET_unused": "*****",\n "unused": "def"\n }' in log_events[0]["info"]["msg"] ) ================================================ FILE: tests/functional/context_methods/test_custom_env_vars.py ================================================ import json import os import pytest from dbt.tests.util import run_dbt_and_capture def parse_json_logs(json_log_output): parsed_logs = [] for line in json_log_output.split("\n"): try: log = json.loads(line) except ValueError: continue parsed_logs.append(log) return parsed_logs class TestCustomVarInLogs: @pytest.fixture(scope="class", autouse=True) def setup(self): # on windows, python uppercases env var names because windows is case insensitive os.environ["DBT_ENV_CUSTOM_ENV_SOME_VAR"] = "value" yield del os.environ["DBT_ENV_CUSTOM_ENV_SOME_VAR"] def test_extra_filled(self, project): _, log_output = run_dbt_and_capture( ["--log-format=json", "deps"], ) logs = parse_json_logs(log_output) for log in logs: assert log["info"].get("extra") == {"SOME_VAR": "value"} ================================================ FILE: tests/functional/context_methods/test_env_vars.py ================================================ import os import pytest from dbt.constants import DEFAULT_ENV_PLACEHOLDER from dbt.tests.util import get_manifest, run_dbt, run_dbt_and_capture from dbt_common.constants import SECRET_ENV_PREFIX context_sql = """ {{ config( materialized='table' ) }} select -- compile-time variables '{{ this }}' as "this", '{{ this.name }}' as "this.name", '{{ this.schema }}' as "this.schema", '{{ this.table }}' as "this.table", '{{ target.dbname }}' as "target.dbname", '{{ target.host }}' as "target.host", '{{ target.name }}' as "target.name", '{{ target.schema }}' as "target.schema", '{{ target.type }}' as "target.type", '{{ target.user }}' as "target.user", '{{ target.get("pass", "") }}' as "target.pass", -- not actually included, here to test that it is _not_ present! {{ target.port }} as "target.port", {{ target.threads }} as "target.threads", -- runtime variables '{{ run_started_at }}' as run_started_at, '{{ invocation_id }}' as invocation_id, '{{ thread_id }}' as thread_id, '{{ env_var("DBT_TEST_ENV_VAR") }}' as env_var, '{{ env_var("DBT_TEST_IGNORE_DEFAULT", "ignored_default_val") }}' as env_var_ignore_default, '{{ env_var("DBT_TEST_USE_DEFAULT", "use_my_default_val") }}' as env_var_use_default, 'secret_variable' as env_var_secret, -- make sure the value itself is scrubbed from the logs '{{ env_var("DBT_TEST_NOT_SECRET") }}' as env_var_not_secret """ class TestEnvVars: @pytest.fixture(scope="class") def models(self): return {"context.sql": context_sql} @pytest.fixture(scope="class", autouse=True) def setup(self): os.environ["DBT_TEST_ENV_VAR"] = "1" os.environ["DBT_TEST_USER"] = "root" os.environ["DBT_TEST_PASS"] = "password" os.environ[SECRET_ENV_PREFIX + "_SECRET"] = "secret_variable" os.environ["DBT_TEST_NOT_SECRET"] = "regular_variable" os.environ["DBT_TEST_IGNORE_DEFAULT"] = "ignored_default" yield del os.environ["DBT_TEST_ENV_VAR"] del os.environ["DBT_TEST_USER"] del os.environ[SECRET_ENV_PREFIX + "_SECRET"] del os.environ["DBT_TEST_NOT_SECRET"] del os.environ["DBT_TEST_IGNORE_DEFAULT"] @pytest.fixture(scope="class") def profiles_config_update(self, unique_schema): return { "test": { "outputs": { # don't use env_var's here so the integration tests can run # seed sql statements and the like. default target is used "dev": { "type": "postgres", "threads": 1, "host": "localhost", "port": 5432, "user": "root", "pass": "password", "dbname": "dbt", "schema": unique_schema, }, "prod": { "type": "postgres", "threads": 1, "host": "localhost", "port": 5432, # root/password "user": "{{ env_var('DBT_TEST_USER') }}", "pass": "{{ env_var('DBT_TEST_PASS') }}", "dbname": "dbt", "schema": unique_schema, }, }, "target": "dev", } } def get_ctx_vars(self, project): fields = [ "this", "this.name", "this.schema", "this.table", "target.dbname", "target.host", "target.name", "target.port", "target.schema", "target.threads", "target.type", "target.user", "target.pass", "run_started_at", "invocation_id", "thread_id", "env_var", ] field_list = ", ".join(['"{}"'.format(f) for f in fields]) query = "select {field_list} from {schema}.context".format( field_list=field_list, schema=project.test_schema ) vals = project.run_sql(query, fetch="all") ctx = dict([(k, v) for (k, v) in zip(fields, vals[0])]) return ctx def test_env_vars_dev( self, project, ): results = run_dbt(["run"]) assert len(results) == 1 ctx = self.get_ctx_vars(project) manifest = get_manifest(project.project_root) expected = { "DBT_TEST_ENV_VAR": "1", "DBT_TEST_NOT_SECRET": "regular_variable", "DBT_TEST_IGNORE_DEFAULT": "ignored_default", "DBT_TEST_USE_DEFAULT": DEFAULT_ENV_PLACEHOLDER, } assert manifest.env_vars == expected this = '"{}"."{}"."context"'.format(project.database, project.test_schema) assert ctx["this"] == this assert ctx["this.name"] == "context" assert ctx["this.schema"] == project.test_schema assert ctx["this.table"] == "context" assert ctx["target.dbname"] == "dbt" assert ctx["target.host"] == "localhost" assert ctx["target.name"] == "dev" assert ctx["target.port"] == 5432 assert ctx["target.schema"] == project.test_schema assert ctx["target.threads"] == 1 assert ctx["target.type"] == "postgres" assert ctx["target.user"] == "root" assert ctx["target.pass"] == "" assert ctx["env_var"] == "1" def test_env_vars_prod(self, project): results = run_dbt(["run", "--target", "prod"]) assert len(results) == 1 ctx = self.get_ctx_vars(project) this = '"{}"."{}"."context"'.format(project.database, project.test_schema) assert ctx["this"] == this assert ctx["this.name"] == "context" assert ctx["this.schema"] == project.test_schema assert ctx["this.table"] == "context" assert ctx["target.dbname"] == "dbt" assert ctx["target.host"] == "localhost" assert ctx["target.name"] == "prod" assert ctx["target.port"] == 5432 assert ctx["target.schema"] == project.test_schema assert ctx["target.threads"] == 1 assert ctx["target.type"] == "postgres" assert ctx["target.user"] == "root" assert ctx["target.pass"] == "" assert ctx["env_var"] == "1" def test_env_vars_secrets(self, project): os.environ["DBT_DEBUG"] = "True" _, log_output = run_dbt_and_capture(["run", "--target", "prod"]) assert not ("secret_variable" in log_output) assert "regular_variable" in log_output del os.environ["DBT_DEBUG"] class TestEnvVarInCreateSchema: """Test that the env_var() method works in overrides of the create_schema macro, which is called during a different phase of execution than most macros, causing problems.""" @pytest.fixture(scope="class", autouse=True) def setup(self): os.environ["DBT_TEST_ENV_VAR"] = "1" @pytest.fixture(scope="class") def macros(self): return { "macros.sql": """ {% macro create_schema(relation) %} {%- call statement('create_schema') -%} SELECT {{ env_var('DBT_TEST_ENV_VAR') }} as TEST {% endcall %} {% endmacro %}% """ } @pytest.fixture(scope="class") def models(self): return { "mymodel.sql": """ SELECT 1 as TEST -- {%- do adapter.create_schema(this) -%} """ } def test_env_var_in_create_schema(self, project): run_dbt(["run"]) ================================================ FILE: tests/functional/context_methods/test_secret_env_vars.py ================================================ import os import pytest from dbt.exceptions import DbtInternalError, ParsingError from dbt.tests.util import read_file, run_dbt, run_dbt_and_capture from dbt_common.constants import SECRET_ENV_PREFIX from tests.functional.context_methods.first_dependency import FirstDependencyProject secret_bad__context_sql = """ {{ config( materialized='table' ) }} select '{{ env_var("DBT_TEST_ENV_VAR") }}' as env_var, '{{ env_var("DBT_ENV_SECRET_SECRET") }}' as env_var_secret, -- this should raise an error! '{{ env_var("DBT_TEST_NOT_SECRET") }}' as env_var_not_secret """ class TestDisallowSecretModel: @pytest.fixture(scope="class") def models(self): return {"context.sql": secret_bad__context_sql} def test_disallow_secret(self, project): with pytest.raises(ParsingError): run_dbt(["compile"]) models__context_sql = """ {{ config( materialized='table' ) }} select -- compile-time variables '{{ this }}' as "this", '{{ this.name }}' as "this.name", '{{ this.schema }}' as "this.schema", '{{ this.table }}' as "this.table", '{{ target.dbname }}' as "target.dbname", '{{ target.host }}' as "target.host", '{{ target.name }}' as "target.name", '{{ target.schema }}' as "target.schema", '{{ target.type }}' as "target.type", '{{ target.user }}' as "target.user", '{{ target.get("pass", "") }}' as "target.pass", -- not actually included, here to test that it is _not_ present! {{ target.port }} as "target.port", {{ target.threads }} as "target.threads", -- runtime variables '{{ run_started_at }}' as run_started_at, '{{ invocation_id }}' as invocation_id, '{{ thread_id }}' as thread_id, '{{ env_var("DBT_TEST_ENV_VAR") }}' as env_var, 'secret_variable' as env_var_secret, -- make sure the value itself is scrubbed from the logs '{{ env_var("DBT_TEST_NOT_SECRET") }}' as env_var_not_secret """ class TestAllowSecretProfilePackage(FirstDependencyProject): @pytest.fixture(scope="class", autouse=True) def setup(self): os.environ[SECRET_ENV_PREFIX + "_USER"] = "root" os.environ[SECRET_ENV_PREFIX + "_PASS"] = "password" os.environ[SECRET_ENV_PREFIX + "_PACKAGE"] = "first_dependency" os.environ[SECRET_ENV_PREFIX + "_GIT_TOKEN"] = "abc123" yield del os.environ[SECRET_ENV_PREFIX + "_USER"] del os.environ[SECRET_ENV_PREFIX + "_PASS"] del os.environ[SECRET_ENV_PREFIX + "_PACKAGE"] del os.environ[SECRET_ENV_PREFIX + "_GIT_TOKEN"] @pytest.fixture(scope="class") def models(self): return {"context.sql": models__context_sql} @pytest.fixture(scope="class") def packages(self): return { "packages": [ { # the raw value of this secret *will* be written to lock file "local": "{{ env_var('DBT_ENV_SECRET_PACKAGE') }}" }, { # this secret env var will *not* be written to lock file "git": "https://{{ env_var('DBT_ENV_SECRET_GIT_TOKEN') }}@github.com/dbt-labs/dbt-external-tables.git" }, { # this secret env var will *not* be written to lock file "tarball": "https://{{ env_var('DBT_ENV_SECRET_GIT_TOKEN') }}@github.com/dbt-labs/dbt-utils/archive/refs/tags/1.1.1.tar.gz", "name": "dbt_utils", }, ] } @pytest.fixture(scope="class") def profile_target(self): return { "type": "postgres", "threads": 1, "host": "localhost", "port": 5432, # root/password "user": "{{ env_var('DBT_ENV_SECRET_USER') }}", "pass": "{{ env_var('DBT_ENV_SECRET_PASS') }}", "dbname": "dbt", } def test_allow_secrets(self, project, first_dependency): _, log_output = run_dbt_and_capture(["deps"]) lock_file_contents = read_file("package-lock.yml") # this will not be written to logs or lock file assert not ("abc123" in log_output) assert not ("abc123" in lock_file_contents) assert "{{ env_var('DBT_ENV_SECRET_GIT_TOKEN') }}" in lock_file_contents # this will be scrubbed from logs, but not from the lock file assert not ("first_dependency" in log_output) assert "first_dependency" in lock_file_contents class TestCloneFailSecretScrubbed: @pytest.fixture(scope="class", autouse=True) def setup(self): os.environ[SECRET_ENV_PREFIX + "_GIT_TOKEN"] = "abc123" @pytest.fixture(scope="class") def models(self): return {"context.sql": models__context_sql} @pytest.fixture(scope="class") def packages(self): return { "packages": [ { "git": "https://fakeuser:{{ env_var('DBT_ENV_SECRET_GIT_TOKEN') }}@github.com/dbt-labs/fake-repo.git" }, ] } def test_fail_clone_with_scrubbing(self, project): with pytest.raises(DbtInternalError) as excinfo: _, log_output = run_dbt_and_capture(["deps"]) assert "abc123" not in str(excinfo.value) class TestCloneFailSecretNotRendered(TestCloneFailSecretScrubbed): # as above, with some Jinja manipulation @pytest.fixture(scope="class") def packages(self): return { "packages": [ { "git": "https://fakeuser:{{ env_var('DBT_ENV_SECRET_GIT_TOKEN') | join(' ') }}@github.com/dbt-labs/fake-repo.git" }, ] } def test_fail_clone_with_scrubbing(self, project): with pytest.raises(DbtInternalError) as excinfo: _, log_output = run_dbt_and_capture(["deps"]) # we should not see any manipulated form of the secret value (abc123) here # we should see a manipulated form of the placeholder instead assert "a b c 1 2 3" not in str(excinfo.value) assert "D B T _ E N V _ S E C R E T _ G I T _ T O K E N" in str(excinfo.value) ================================================ FILE: tests/functional/context_methods/test_var_dependency.py ================================================ import pytest from dbt.tests.util import check_relations_equal, run_dbt from tests.functional.context_methods.first_dependency import ( FirstDependencyConfigProject, FirstDependencyProject, ) dependency_seeds__root_model_expected_csv = """first_dep_global,from_root dep_never_overridden,root_root_value """ dependency_models__inside__model_sql = """ select '{{ var("first_dep_override") }}' as first_dep_global, '{{ var("from_root_to_root") }}' as from_root """ class TestVarDependencyInheritance(FirstDependencyProject): @pytest.fixture(scope="class") def seeds(self): return {"root_model_expected.csv": dependency_seeds__root_model_expected_csv} @pytest.fixture(scope="class") def models(self): return {"inside": {"model.sql": dependency_models__inside__model_sql}} @pytest.fixture(scope="class") def packages(self): return { "packages": [ {"local": "first_dependency"}, ] } @pytest.fixture(scope="class") def project_config_update(self): return { "vars": { "first_dep_override": "dep_never_overridden", "test": { "from_root_to_root": "root_root_value", }, "first_dep": { "from_root_to_first": "root_first_value", }, }, } def test_var_mutual_overrides_v1_conversion(self, project, first_dependency): run_dbt(["deps"]) assert len(run_dbt(["seed"])) == 2 assert len(run_dbt(["run"])) == 2 check_relations_equal(project.adapter, ["root_model_expected", "model"]) check_relations_equal(project.adapter, ["first_dep_expected", "first_dep_model"]) class TestVarConfigDependencyInheritance(FirstDependencyConfigProject): @pytest.fixture(scope="class") def packages(self): return { "packages": [ {"local": "first_dependency"}, ] } @pytest.fixture(scope="class") def project_config_update(self): return { "vars": { "test_config_root_override": "configured_from_root", }, } def test_root_var_overrides_package_var(self, project, first_dependency): run_dbt(["deps"]) run_dbt(["seed"]) assert len(run_dbt(["run"])) == 1 check_relations_equal(project.adapter, ["first_dep_expected", "first_dep_model"]) ================================================ FILE: tests/functional/context_methods/test_var_in_generate_name.py ================================================ import pytest from dbt.exceptions import CompilationError from dbt.tests.util import run_dbt, update_config_file model_sql = """ select 1 as id """ bad_generate_macros__generate_names_sql = """ {% macro generate_schema_name(custom_schema_name, node) -%} {% do var('somevar') %} {% do return(dbt.generate_schema_name(custom_schema_name, node)) %} {%- endmacro %} """ class TestMissingVarGenerateNameMacro: @pytest.fixture(scope="class") def macros(self): return {"generate_names.sql": bad_generate_macros__generate_names_sql} @pytest.fixture(scope="class") def models(self): return {"model.sql": model_sql} def test_generate_schema_name_var(self, project): # var isn't set, so generate_name macro fails with pytest.raises(CompilationError) as excinfo: run_dbt(["compile"]) assert "Required var 'somevar' not found in config" in str(excinfo.value) # globally scoped -- var is set at top-level update_config_file({"vars": {"somevar": 1}}, project.project_root, "dbt_project.yml") run_dbt(["compile"]) # locally scoped -- var is set in 'test' scope update_config_file( {"vars": {"test": {"somevar": 1}}}, project.project_root, "dbt_project.yml" ) run_dbt(["compile"]) ================================================ FILE: tests/functional/context_methods/test_yaml_functions.py ================================================ import pytest from dbt.tests.util import run_dbt tests__from_yaml_sql = """ {% set simplest = (fromyaml('a: 1') == {'a': 1}) %} {% set nested_data %} a: b: - c: 1 d: 2 - c: 3 d: 4 {% endset %} {% set nested = (fromyaml(nested_data) == {'a': {'b': [{'c': 1, 'd': 2}, {'c': 3, 'd': 4}]}}) %} (select 'simplest' as name {% if simplest %}limit 0{% endif %}) union all (select 'nested' as name {% if nested %}limit 0{% endif %}) """ tests__to_yaml_sql = """ {% set simplest = (toyaml({'a': 1}) == 'a: 1\\n') %} {% set default_sort = (toyaml({'b': 2, 'a': 1}) == 'b: 2\\na: 1\\n') %} {% set unsorted = (toyaml({'b': 2, 'a': 1}, sort_keys=False) == 'b: 2\\na: 1\\n') %} {% set sorted = (toyaml({'b': 2, 'a': 1}, sort_keys=True) == 'a: 1\\nb: 2\\n') %} {% set default_results = (toyaml({'a': adapter}, 'failed') == 'failed') %} (select 'simplest' as name {% if simplest %}limit 0{% endif %}) union all (select 'default_sort' as name {% if default_sort %}limit 0{% endif %}) union all (select 'unsorted' as name {% if unsorted %}limit 0{% endif %}) union all (select 'sorted' as name {% if sorted %}limit 0{% endif %}) union all (select 'default_results' as name {% if default_results %}limit 0{% endif %}) """ class TestContextVars: # This test has no actual models @pytest.fixture(scope="class") def tests(self): return {"from_yaml.sql": tests__from_yaml_sql, "to_yaml.sql": tests__to_yaml_sql} def test_json_data_tests(self, project): assert len(run_dbt(["test"])) == 2 ================================================ FILE: tests/functional/contracts/test_contract_enforcement.py ================================================ import pytest from dbt.tests.util import run_dbt, write_file my_model_sql = """ select 'some string' as string_column """ my_model_int_sql = """ select 123 as int_column """ model_schema_yml = """ models: - name: my_model config: materialized: incremental on_schema_change: append_new_columns contract: {enforced: true} columns: - name: string_column data_type: text """ class TestIncrementalModelContractEnforcement: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_model_sql, "schema.yml": model_schema_yml, } def test_contracted_incremental(self, project): results = run_dbt() assert len(results) == 1 # now update the column type in the model to break the contract write_file(my_model_int_sql, project.project_root, "models", "my_model.sql") expected_msg = "This model has an enforced contract that failed." results = run_dbt(expect_pass=False) assert len(results) == 1 msg = results[0].message assert expected_msg in msg ================================================ FILE: tests/functional/contracts/test_contract_precision.py ================================================ import pytest from dbt.tests.util import run_dbt_and_capture my_numeric_model_sql = """ select 1.234 as non_integer """ model_schema_numerics_yml = """ version: 2 models: - name: my_numeric_model config: contract: enforced: true columns: - name: non_integer data_type: numeric """ model_schema_numerics_precision_yml = """ version: 2 models: - name: my_numeric_model config: contract: enforced: true columns: - name: non_integer data_type: numeric(38,3) """ class TestModelContractNumericNoPrecision: @pytest.fixture(scope="class") def models(self): return { "my_numeric_model.sql": my_numeric_model_sql, "schema.yml": model_schema_numerics_yml, } def test_contracted_numeric_without_precision(self, project): expected_msg = "Detected columns with numeric type and unspecified precision/scale, this can lead to unintended rounding: ['non_integer']" _, logs = run_dbt_and_capture(["run"], expect_pass=True) assert expected_msg in logs _, logs = run_dbt_and_capture( ["--warn-error-options", "{'error': 'all', 'warn': ['DeprecationsSummary']}", "run"], expect_pass=False, ) assert "Compilation Error in model my_numeric_model" in logs assert expected_msg in logs class TestModelContractNumericPrecision: @pytest.fixture(scope="class") def models(self): return { "my_numeric_model.sql": my_numeric_model_sql, "schema.yml": model_schema_numerics_precision_yml, } def test_contracted_numeric_with_precision(self, project): expected_msg = "Detected columns with numeric type and unspecified precision/scale, this can lead to unintended rounding: ['non_integer']" _, logs = run_dbt_and_capture(["run"], expect_pass=True) assert expected_msg not in logs ================================================ FILE: tests/functional/contracts/test_nonstandard_data_type.py ================================================ import pytest from dbt.tests.util import run_dbt, run_dbt_and_capture my_numeric_model_sql = """ select 12.34 as price """ my_money_model_sql = """ select cast('12.34' as money) as price """ model_schema_money_yml = """ models: - name: my_model config: contract: enforced: true columns: - name: price data_type: money """ model_schema_numeric_yml = """ models: - name: my_model config: contract: enforced: true columns: - name: price data_type: numeric """ class TestModelContractUnrecognizedTypeCode1: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_money_model_sql, "schema.yml": model_schema_money_yml, } def test_nonstandard_data_type(self, project): run_dbt(["run"], expect_pass=True) class TestModelContractUnrecognizedTypeCodeActualMismatch: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_money_model_sql, "schema.yml": model_schema_numeric_yml, } def test_nonstandard_data_type(self, project): expected_msg = "unknown type_code 790 | DECIMAL | data type mismatch" _, logs = run_dbt_and_capture(["run"], expect_pass=False) assert expected_msg in logs class TestModelContractUnrecognizedTypeCodeExpectedMismatch: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_numeric_model_sql, "schema.yml": model_schema_money_yml, } def test_nonstandard_data_type(self, project): expected_msg = "DECIMAL | unknown type_code 790 | data type mismatch" _, logs = run_dbt_and_capture(["run"], expect_pass=False) print(logs) assert expected_msg in logs ================================================ FILE: tests/functional/custom_aliases/fixtures.py ================================================ model1_sql = """ {{ config(materialized='table', alias='alias') }} select {{ string_literal(this.name) }} as model_name """ model2_sql = """ {{ config(materialized='table') }} select {{ string_literal(this.name) }} as model_name """ macros_sql = """ {% macro generate_alias_name(custom_alias_name, node) -%} {%- if custom_alias_name is none -%} {{ node.name }} {%- else -%} custom_{{ custom_alias_name | trim }} {%- endif -%} {%- endmacro %} {% macro string_literal(s) -%} {{ adapter.dispatch('string_literal', macro_namespace='test')(s) }} {%- endmacro %} {% macro default__string_literal(s) %} '{{ s }}'::text {% endmacro %} """ macros_config_sql = """ {#-- Verify that the config['alias'] key is present #} {% macro generate_alias_name(custom_alias_name, node) -%} {%- if custom_alias_name is none -%} {{ node.name }} {%- else -%} custom_{{ node.config['alias'] if 'alias' in node.config else '' | trim }} {%- endif -%} {%- endmacro %} {% macro string_literal(s) -%} {{ adapter.dispatch('string_literal', macro_namespace='test')(s) }} {%- endmacro %} {% macro default__string_literal(s) %} '{{ s }}'::text {% endmacro %} """ schema_yml = """ version: 2 models: - name: model1 columns: - name: model_name data_tests: - accepted_values: values: ['custom_alias'] - name: model2 columns: - name: model_name data_tests: - accepted_values: values: ['model2'] """ ================================================ FILE: tests/functional/custom_aliases/test_custom_aliases.py ================================================ import pytest from dbt.tests.util import run_dbt from tests.functional.custom_aliases.fixtures import ( macros_config_sql, macros_sql, model1_sql, model2_sql, schema_yml, ) class TestAliases: @pytest.fixture(scope="class") def models(self): return {"model1.sql": model1_sql, "model2.sql": model2_sql, "schema.yml": schema_yml} @pytest.fixture(scope="class") def macros(self): return { "macros.sql": macros_sql, } def test_customer_alias_name(self, project): results = run_dbt(["run"]) assert len(results) == 2 results = run_dbt(["test"]) assert len(results) == 2 class TestAliasesWithConfig: @pytest.fixture(scope="class") def models(self): return {"model1.sql": model1_sql, "model2.sql": model2_sql, "schema.yml": schema_yml} @pytest.fixture(scope="class") def macros(self): return { "macros.sql": macros_config_sql, } def test_customer_alias_name(self, project): results = run_dbt(["run"]) assert len(results) == 2 results = run_dbt(["test"]) assert len(results) == 2 ================================================ FILE: tests/functional/custom_schemas/test_custom_schemas.py ================================================ import pytest from dbt.exceptions import ParsingError from dbt.tests.util import run_dbt generate_schema_name_macro_sql = """ {% macro generate_schema_name(custom_schema_name, node) %} test_schema {% endmacro %} """ # this macro returns none when custom_schema_name (from config schema) is unset generate_schema_name_macro_null_return = """ {% macro generate_schema_name(custom_schema_name, node) %} {{ return(custom_schema_name) }} {% endmacro %} """ class TestCustomSchema: @pytest.fixture(scope="class") def models(self): return {"model.sql": "select 1 as id"} @pytest.fixture(scope="class") def macros(self): return { "generate_schema_name_null_return.sql": generate_schema_name_macro_sql, } def test_custom_schema(self, project): results = run_dbt(["run"]) assert len(results) == 1 assert results.results[0].node.schema == "test_schema" class TestCustomSchemaNullReturn: @pytest.fixture(scope="class") def project_config_update(self): return { "flags": { "require_valid_schema_from_generate_schema_name": True, }, } @pytest.fixture(scope="class") def models(self): return {"model.sql": "select 1 as id"} @pytest.fixture(scope="class") def macros(self): return { "generate_schema_name_null_return.sql": generate_schema_name_macro_null_return, } def test_custom_schema_null_return(self, project): with pytest.raises(ParsingError) as excinfo: run_dbt(["run"]) assert ( "Node 'model.test.model' has a schema set to None as a result of a generate_schema_name call." in str(excinfo.value) ) class TestCustomSchemaNullReturnLegacy: @pytest.fixture(scope="class") def project_config_update(self): return { "flags": { "require_valid_schema_from_generate_schema_name": False, }, } @pytest.fixture(scope="class") def models(self): return { "model.sql": "select 1 as id", } @pytest.fixture(scope="class") def macros(self): return { "generate_schema_name_null_return.sql": generate_schema_name_macro_null_return, } def test_custom_schema_null_return_legacy(self, project): manifest = run_dbt(["parse"], expect_pass=True) # This was buggy behavior (non-conformant to manifest schemas published in v12) but nonetheless legacy behavior assert manifest.nodes["model.test.model"].schema is None # Should be updated to TestCustomSchemaNullReturn instead of TestCustomSchemaNullReturnLegacy once # required_valid_schema_from_generate_schema_name flag is set to True by default class TestCustomSchemaNullReturnDefault(TestCustomSchemaNullReturnLegacy): pass ================================================ FILE: tests/functional/custom_singular_tests/data/seed_expected.sql ================================================ create table {schema}.seed ( favorite_color VARCHAR(10), id INTEGER, first_name VARCHAR(11), email VARCHAR(31), ip_address VARCHAR(15), updated_at TIMESTAMP WITHOUT TIME ZONE ); INSERT INTO {schema}.seed (favorite_color, id, first_name, email, ip_address, updated_at) VALUES ('blue', 1,'Larry','lking0@miitbeian.gov.cn','69.135.206.194','2008-09-12 19:08:31'), ('blue', 2,'Larry','lperkins1@toplist.cz','64.210.133.162','1978-05-09 04:15:14'), ('blue', 3,'Anna','amontgomery2@miitbeian.gov.cn','168.104.64.114','2011-10-16 04:07:57'), ('blue', 4,'Sandra','sgeorge3@livejournal.com','229.235.252.98','1973-07-19 10:52:43'), ('blue', 5,'Fred','fwoods4@google.cn','78.229.170.124','2012-09-30 16:38:29'), ('blue', 6,'Stephen','shanson5@livejournal.com','182.227.157.105','1995-11-07 21:40:50'), ('blue', 7,'William','wmartinez6@upenn.edu','135.139.249.50','1982-09-05 03:11:59'), ('blue', 8,'Jessica','jlong7@hao123.com','203.62.178.210','1991-10-16 11:03:15'), ('blue', 9,'Douglas','dwhite8@tamu.edu','178.187.247.1','1979-10-01 09:49:48'), ('blue', 10,'Lisa','lcoleman9@nydailynews.com','168.234.128.249','2011-05-26 07:45:49'), ('blue', 11,'Ralph','rfieldsa@home.pl','55.152.163.149','1972-11-18 19:06:11'), ('blue', 12,'Louise','lnicholsb@samsung.com','141.116.153.154','2014-11-25 20:56:14'), ('blue', 13,'Clarence','cduncanc@sfgate.com','81.171.31.133','2011-11-17 07:02:36'), ('blue', 14,'Daniel','dfranklind@omniture.com','8.204.211.37','1980-09-13 00:09:04'), ('blue', 15,'Katherine','klanee@auda.org.au','176.96.134.59','1997-08-22 19:36:56'), ('blue', 16,'Billy','bwardf@wikia.com','214.108.78.85','2003-10-19 02:14:47'), ('blue', 17,'Annie','agarzag@ocn.ne.jp','190.108.42.70','1988-10-28 15:12:35'), ('blue', 18,'Shirley','scolemanh@fastcompany.com','109.251.164.84','1988-08-24 10:50:57'), ('blue', 19,'Roger','rfrazieri@scribd.com','38.145.218.108','1985-12-31 15:17:15'), ('blue', 20,'Lillian','lstanleyj@goodreads.com','47.57.236.17','1970-06-08 02:09:05'), ('blue', 21,'Aaron','arodriguezk@nps.gov','205.245.118.221','1985-10-11 23:07:49'), ('blue', 22,'Patrick','pparkerl@techcrunch.com','19.8.100.182','2006-03-29 12:53:56'), ('blue', 23,'Phillip','pmorenom@intel.com','41.38.254.103','2011-11-07 15:35:43'), ('blue', 24,'Henry','hgarcian@newsvine.com','1.191.216.252','2008-08-28 08:30:44'), ('blue', 25,'Irene','iturnero@opera.com','50.17.60.190','1994-04-01 07:15:02'), ('blue', 26,'Andrew','adunnp@pen.io','123.52.253.176','2000-11-01 06:03:25'), ('blue', 27,'David','dgutierrezq@wp.com','238.23.203.42','1988-01-25 07:29:18'), ('blue', 28,'Henry','hsanchezr@cyberchimps.com','248.102.2.185','1983-01-01 13:36:37'), ('blue', 29,'Evelyn','epetersons@gizmodo.com','32.80.46.119','1979-07-16 17:24:12'), ('blue', 30,'Tammy','tmitchellt@purevolume.com','249.246.167.88','2001-04-03 10:00:23'), ('blue', 31,'Jacqueline','jlittleu@domainmarket.com','127.181.97.47','1986-02-11 21:35:50'), ('blue', 32,'Earl','eortizv@opera.com','166.47.248.240','1996-07-06 08:16:27'), ('blue', 33,'Juan','jgordonw@sciencedirect.com','71.77.2.200','1987-01-31 03:46:44'), ('blue', 34,'Diane','dhowellx@nyu.edu','140.94.133.12','1994-06-11 02:30:05'), ('blue', 35,'Randy','rkennedyy@microsoft.com','73.255.34.196','2005-05-26 20:28:39'), ('blue', 36,'Janice','jriveraz@time.com','22.214.227.32','1990-02-09 04:16:52'), ('blue', 37,'Laura','lperry10@diigo.com','159.148.145.73','2015-03-17 05:59:25'), ('blue', 38,'Gary','gray11@statcounter.com','40.193.124.56','1970-01-27 10:04:51'), ('blue', 39,'Jesse','jmcdonald12@typepad.com','31.7.86.103','2009-03-14 08:14:29'), ('blue', 40,'Sandra','sgonzalez13@goodreads.com','223.80.168.239','1993-05-21 14:08:54'), ('blue', 41,'Scott','smoore14@archive.org','38.238.46.83','1980-08-30 11:16:56'), ('blue', 42,'Phillip','pevans15@cisco.com','158.234.59.34','2011-12-15 23:26:31'), ('blue', 43,'Steven','sriley16@google.ca','90.247.57.68','2011-10-29 19:03:28'), ('blue', 44,'Deborah','dbrown17@hexun.com','179.125.143.240','1995-04-10 14:36:07'), ('blue', 45,'Lori','lross18@ow.ly','64.80.162.180','1980-12-27 16:49:15'), ('blue', 46,'Sean','sjackson19@tumblr.com','240.116.183.69','1988-06-12 21:24:45'), ('blue', 47,'Terry','tbarnes1a@163.com','118.38.213.137','1997-09-22 16:43:19'), ('blue', 48,'Dorothy','dross1b@ebay.com','116.81.76.49','2005-02-28 13:33:24'), ('blue', 49,'Samuel','swashington1c@house.gov','38.191.253.40','1989-01-19 21:15:48'), ('blue', 50,'Ralph','rcarter1d@tinyurl.com','104.84.60.174','2007-08-11 10:21:49'), ('green', 51,'Wayne','whudson1e@princeton.edu','90.61.24.102','1983-07-03 16:58:12'), ('green', 52,'Rose','rjames1f@plala.or.jp','240.83.81.10','1995-06-08 11:46:23'), ('green', 53,'Louise','lcox1g@theglobeandmail.com','105.11.82.145','2016-09-19 14:45:51'), ('green', 54,'Kenneth','kjohnson1h@independent.co.uk','139.5.45.94','1976-08-17 11:26:19'), ('green', 55,'Donna','dbrown1i@amazon.co.uk','19.45.169.45','2006-05-27 16:51:40'), ('green', 56,'Johnny','jvasquez1j@trellian.com','118.202.238.23','1975-11-17 08:42:32'), ('green', 57,'Patrick','pramirez1k@tamu.edu','231.25.153.198','1997-08-06 11:51:09'), ('green', 58,'Helen','hlarson1l@prweb.com','8.40.21.39','1993-08-04 19:53:40'), ('green', 59,'Patricia','pspencer1m@gmpg.org','212.198.40.15','1977-08-03 16:37:27'), ('green', 60,'Joseph','jspencer1n@marriott.com','13.15.63.238','2005-07-23 20:22:06'), ('green', 61,'Phillip','pschmidt1o@blogtalkradio.com','177.98.201.190','1976-05-19 21:47:44'), ('green', 62,'Joan','jwebb1p@google.ru','105.229.170.71','1972-09-07 17:53:47'), ('green', 63,'Phyllis','pkennedy1q@imgur.com','35.145.8.244','2000-01-01 22:33:37'), ('green', 64,'Katherine','khunter1r@smh.com.au','248.168.205.32','1991-01-09 06:40:24'), ('green', 65,'Laura','lvasquez1s@wiley.com','128.129.115.152','1997-10-23 12:04:56'), ('green', 66,'Juan','jdunn1t@state.gov','44.228.124.51','2004-11-10 05:07:35'), ('green', 67,'Judith','jholmes1u@wiley.com','40.227.179.115','1977-08-02 17:01:45'), ('green', 68,'Beverly','bbaker1v@wufoo.com','208.34.84.59','2016-03-06 20:07:23'), ('green', 69,'Lawrence','lcarr1w@flickr.com','59.158.212.223','1988-09-13 06:07:21'), ('green', 70,'Gloria','gwilliams1x@mtv.com','245.231.88.33','1995-03-18 22:32:46'), ('green', 71,'Steven','ssims1y@cbslocal.com','104.50.58.255','2001-08-05 21:26:20'), ('green', 72,'Betty','bmills1z@arstechnica.com','103.177.214.220','1981-12-14 21:26:54'), ('green', 73,'Mildred','mfuller20@prnewswire.com','151.158.8.130','2000-04-19 10:13:55'), ('green', 74,'Donald','dday21@icq.com','9.178.102.255','1972-12-03 00:58:24'), ('green', 75,'Eric','ethomas22@addtoany.com','85.2.241.227','1992-11-01 05:59:30'), ('green', 76,'Joyce','jarmstrong23@sitemeter.com','169.224.20.36','1985-10-24 06:50:01'), ('green', 77,'Maria','mmartinez24@amazonaws.com','143.189.167.135','2005-10-05 05:17:42'), ('green', 78,'Harry','hburton25@youtube.com','156.47.176.237','1978-03-26 05:53:33'), ('green', 79,'Kevin','klawrence26@hao123.com','79.136.183.83','1994-10-12 04:38:52'), ('green', 80,'David','dhall27@prweb.com','133.149.172.153','1976-12-15 16:24:24'), ('green', 81,'Kathy','kperry28@twitter.com','229.242.72.228','1979-03-04 02:58:56'), ('green', 82,'Adam','aprice29@elegantthemes.com','13.145.21.10','1982-11-07 11:46:59'), ('green', 83,'Brandon','bgriffin2a@va.gov','73.249.128.212','2013-10-30 05:30:36'), ('green', 84,'Henry','hnguyen2b@discovery.com','211.36.214.242','1985-01-09 06:37:27'), ('green', 85,'Eric','esanchez2c@edublogs.org','191.166.188.251','2004-05-01 23:21:42'), ('green', 86,'Jason','jlee2d@jimdo.com','193.92.16.182','1973-01-08 09:05:39'), ('green', 87,'Diana','drichards2e@istockphoto.com','19.130.175.245','1994-10-05 22:50:49'), ('green', 88,'Andrea','awelch2f@abc.net.au','94.155.233.96','2002-04-26 08:41:44'), ('green', 89,'Louis','lwagner2g@miitbeian.gov.cn','26.217.34.111','2003-08-25 07:56:39'), ('green', 90,'Jane','jsims2h@seesaa.net','43.4.220.135','1987-03-20 20:39:04'), ('green', 91,'Larry','lgrant2i@si.edu','97.126.79.34','2000-09-07 20:26:19'), ('green', 92,'Louis','ldean2j@prnewswire.com','37.148.40.127','2011-09-16 20:12:14'), ('green', 93,'Jennifer','jcampbell2k@xing.com','38.106.254.142','1988-07-15 05:06:49'), ('green', 94,'Wayne','wcunningham2l@google.com.hk','223.28.26.187','2009-12-15 06:16:54'), ('green', 95,'Lori','lstevens2m@icq.com','181.250.181.58','1984-10-28 03:29:19'), ('green', 96,'Judy','jsimpson2n@marriott.com','180.121.239.219','1986-02-07 15:18:10'), ('green', 97,'Phillip','phoward2o@usa.gov','255.247.0.175','2002-12-26 08:44:45'), ('green', 98,'Gloria','gwalker2p@usa.gov','156.140.7.128','1997-10-04 07:58:58'), ('green', 99,'Paul','pjohnson2q@umn.edu','183.59.198.197','1991-11-14 12:33:55'), ('green', 100,'Frank','fgreene2r@blogspot.com','150.143.68.121','2010-06-12 23:55:39'); ================================================ FILE: tests/functional/custom_singular_tests/test_custom_singular_tests.py ================================================ from pathlib import Path import pytest from dbt.tests.util import run_dbt # from `test/integration/009_data_test` # # Models # models__table_copy = """ {{ config( materialized='table' ) }} select * from {{ this.schema }}.seed """ # # Tests # tests__fail_email_is_always_null = """ select * from {{ ref('table_copy') }} where email is not null """ tests__fail_no_ref = """ select 1 """ tests__dotted_path_pass_id_not_null = """ {# Same as `pass_id_not_null` but with dots in its name #} select * from {{ ref('table_copy') }} where id is null """ tests__pass_id_not_null = """ select * from {{ ref('table_copy') }} where id is null """ tests__pass_no_ref = """ select 1 limit 0 """ class CustomSingularTestsBase(object): @pytest.fixture(scope="class", autouse=True) def setUp(self, project): """Create seed and downstream model tests are to be run on""" project.run_sql_file(project.test_data_dir / Path("seed_expected.sql")) results = run_dbt() assert len(results) == 1 @pytest.fixture(scope="class") def models(self): return {"table_copy.sql": models__table_copy} class TestPassingTests(CustomSingularTestsBase): @pytest.fixture(scope="class") def tests(self): return { "my_db.my_schema.table_copy.pass_id_not_null.sql": tests__dotted_path_pass_id_not_null, "tests__pass_id_not_null.sql": tests__pass_id_not_null, "tests__pass_no_ref.sql": tests__pass_no_ref, } def test_data_tests(self, project, tests): test_results = run_dbt(["test"]) assert len(test_results) == len(tests) for result in test_results: assert result.status == "pass" assert not result.skipped assert result.failures == 0 class TestFailingTests(CustomSingularTestsBase): @pytest.fixture(scope="class") def tests(self): return { "tests__fail_email_is_always_null.sql": tests__fail_email_is_always_null, "tests__fail_no_ref.sql": tests__fail_no_ref, } def test_data_tests(self, project, tests): """assert that all deliberately failing tests actually fail""" test_results = run_dbt(["test"], expect_pass=False) assert len(test_results) == len(tests) for result in test_results: assert result.status == "fail" assert not result.skipped assert result.failures > 0 assert result.adapter_response == { "_message": "SELECT 1", "code": "SELECT", "rows_affected": 1, } ================================================ FILE: tests/functional/custom_target_path/test_custom_target_path.py ================================================ from pathlib import Path import pytest from dbt.tests.util import run_dbt class TestTargetPathConfig: @pytest.fixture(scope="class") def project_config_update(self): return {"config-version": 2, "target-path": "project_target"} def test_target_path(self, project): run_dbt(["run"]) assert Path("project_target").is_dir() assert not Path("target").is_dir() class TestTargetPathEnvVar: def test_target_path(self, project, monkeypatch): monkeypatch.setenv("DBT_TARGET_PATH", "env_target") run_dbt(["run"]) assert Path("env_target").is_dir() assert not Path("project_target").is_dir() assert not Path("target").is_dir() class TestTargetPathCliArg: def test_target_path(self, project, monkeypatch): monkeypatch.setenv("DBT_TARGET_PATH", "env_target") run_dbt(["run", "--target-path", "cli_target"]) assert Path("cli_target").is_dir() assert not Path("env_target").is_dir() assert not Path("project_target").is_dir() assert not Path("target").is_dir() ================================================ FILE: tests/functional/cycles/test_cycles.py ================================================ import pytest from dbt.tests.util import run_dbt model_a_sql = """ select * from {{ ref('model_b') }} """ model_b_sql = """ select * from {{ ref('model_a') }} """ complex_cycle__model_a_sql = """ select 1 as id """ complex_cycle__model_b_sql = """ select * from {{ ref('model_a') }}s union all select * from {{ ref('model_e') }} """ complex_cycle__model_c_sql = """ select * from {{ ref('model_b') }} """ complex_cycle__model_d_sql = """ select * from {{ ref('model_c') }} """ complex_cycle__model_e_sql = """ select * from {{ ref('model_e') }} """ class TestSimpleCycle: @pytest.fixture(scope="class") def models(self): return {"model_a.sql": model_a_sql, "model_b.sql": model_b_sql} def test_simple_cycle(self, project): with pytest.raises(RuntimeError) as exc: run_dbt(["run"]) expected_msg = "Found a cycle" assert expected_msg in str(exc.value) class TestComplexCycle: @pytest.fixture(scope="class") def models(self): # The cycle in this graph looks like: # A -> B -> C -> D # ^ | # | | # +--- E <--+ return { "model_a.sql": complex_cycle__model_a_sql, "model_b.sql": complex_cycle__model_b_sql, "model_c.sql": complex_cycle__model_c_sql, "model_d.sql": complex_cycle__model_d_sql, "model_e.sql": complex_cycle__model_e_sql, } def test_complex_cycle(self, project): with pytest.raises(RuntimeError) as exc: run_dbt(["run"]) expected_msg = "Found a cycle" assert expected_msg in str(exc.value) ================================================ FILE: tests/functional/data_test_patch/fixtures.py ================================================ tests__my_singular_test_sql = """ with my_cte as ( select 1 as id, 'foo' as name union all select 2 as id, 'bar' as name ) select * from my_cte """ tests__schema_yml = """ data_tests: - name: my_singular_test description: "{{ doc('my_singular_test_documentation') }}" config: error_if: ">10" meta: some_key: some_val """ tests__schema_2_yml = """ data_tests: - name: my_singular_test description: "My singular test description" config: error_if: ">10" meta: some_key: another_val """ tests__doc_block_md = """ {% docs my_singular_test_documentation %} Some docs from a doc block {% enddocs %} """ tests__invalid_name_schema_yml = """ data_tests: - name: my_double_test description: documentation, but make it double """ tests__malformed_schema_yml = """ data_tests: ¬_null - not_null: where: some_condition """ ================================================ FILE: tests/functional/data_test_patch/test_singular_test_patch.py ================================================ from pathlib import Path import pytest from dbt.tests.util import get_manifest, run_dbt, run_dbt_and_capture, write_file from tests.functional.data_test_patch.fixtures import ( tests__doc_block_md, tests__invalid_name_schema_yml, tests__malformed_schema_yml, tests__my_singular_test_sql, tests__schema_2_yml, tests__schema_yml, ) class TestPatchSingularTest: @pytest.fixture(scope="class") def tests(self): return { "my_singular_test.sql": tests__my_singular_test_sql, "schema.yml": tests__schema_yml, "doc_block.md": tests__doc_block_md, } def test_compile(self, project): run_dbt(["compile"]) manifest = get_manifest(project.project_root) assert len(manifest.nodes) == 1 my_singular_test_node = manifest.nodes["test.test.my_singular_test"] assert my_singular_test_node.description == "Some docs from a doc block" assert my_singular_test_node.config.error_if == ">10" assert my_singular_test_node.config.meta == {"some_key": "some_val"} assert my_singular_test_node.meta == {"some_key": "some_val"} # partial parsing test write_file(tests__schema_2_yml, project.project_root, "tests", "schema.yml") manifest = run_dbt(["parse"]) test_node = manifest.nodes["test.test.my_singular_test"] assert test_node.description == "My singular test description" assert test_node.config.meta == {"some_key": "another_val"} assert test_node.meta == {"some_key": "another_val"} class TestPatchSingularTestInvalidName: @pytest.fixture(scope="class") def tests(self): return { "my_singular_test.sql": tests__my_singular_test_sql, "schema_with_invalid_name.yml": tests__invalid_name_schema_yml, } def test_compile(self, project): _, log_output = run_dbt_and_capture(["compile"]) file_path = Path("tests/schema_with_invalid_name.yml") assert ( f"Did not find matching node for patch with name 'my_double_test' in the 'data_tests' section of file '{file_path}'" in log_output ) class TestPatchSingularTestMalformedYaml: @pytest.fixture(scope="class") def tests(self): return { "my_singular_test.sql": tests__my_singular_test_sql, "schema.yml": tests__malformed_schema_yml, } def test_compile(self, project): _, log_output = run_dbt_and_capture(["compile"]) file_path = Path("tests/schema.yml") assert f"Unable to parse 'data_tests' section of file '{file_path}'" in log_output assert "Entry did not contain a name" in log_output ================================================ FILE: tests/functional/data_tests/test_hooks.py ================================================ from unittest import mock import pytest from dbt.tests.util import run_dbt, run_dbt_and_capture from dbt_common.exceptions import CompilationError orders_csv = """order_id,order_date,customer_id 1,2024-06-01,1001 2,2024-06-02,1002 3,2024-06-03,1003 4,2024-06-04,1004 """ orders_model_sql = """ with source as ( select order_id, order_date, customer_id from {{ ref('seed_orders') }} ), final as ( select order_id, order_date, customer_id from source ) select * from final """ orders_test_sql = """ select * from {{ ref('orders') }} where order_id is null """ class BaseSingularTestHooks: @pytest.fixture(scope="class") def seeds(self): return {"seed_orders.csv": orders_csv} @pytest.fixture(scope="class") def models(self): return {"orders.sql": orders_model_sql} @pytest.fixture(scope="class") def tests(self): return {"orders_test.sql": orders_test_sql} class TestSingularTestPreHook(BaseSingularTestHooks): def test_data_test_runs_adapter_pre_hook_pass(self, project): results = run_dbt(["seed"]) assert len(results) == 1 results = run_dbt(["run"]) assert len(results) == 1 mock_pre_model_hook = mock.Mock() with mock.patch.object(type(project.adapter), "pre_model_hook", mock_pre_model_hook): results = run_dbt(["test"], expect_pass=True) assert len(results) == 1 mock_pre_model_hook.assert_called_once() def test_data_test_runs_adapter_pre_hook_fails(self, project): results = run_dbt(["seed"]) assert len(results) == 1 results = run_dbt(["run"]) assert len(results) == 1 mock_pre_model_hook = mock.Mock() mock_pre_model_hook.side_effect = CompilationError("exception from adapter.pre_model_hook") with mock.patch.object(type(project.adapter), "pre_model_hook", mock_pre_model_hook): (_, log_output) = run_dbt_and_capture(["test"], expect_pass=False) assert "exception from adapter.pre_model_hook" in log_output class TestSingularTestPostHook(BaseSingularTestHooks): def test_data_test_runs_adapter_post_hook_pass(self, project): results = run_dbt(["seed"]) assert len(results) == 1 results = run_dbt(["run"]) assert len(results) == 1 mock_post_model_hook = mock.Mock() with mock.patch.object(type(project.adapter), "post_model_hook", mock_post_model_hook): results = run_dbt(["test"], expect_pass=True) assert len(results) == 1 mock_post_model_hook.assert_called_once() def test_data_test_runs_adapter_post_hook_fails(self, project): results = run_dbt(["seed"]) assert len(results) == 1 results = run_dbt(["run"]) assert len(results) == 1 mock_post_model_hook = mock.Mock() mock_post_model_hook.side_effect = CompilationError( "exception from adapter.post_model_hook" ) with mock.patch.object(type(project.adapter), "post_model_hook", mock_post_model_hook): (_, log_output) = run_dbt_and_capture(["test"], expect_pass=False) assert "exception from adapter.post_model_hook" in log_output ================================================ FILE: tests/functional/dbt_runner/test_dbt_runner.py ================================================ import os from unittest import mock import pytest from dbt.adapters.factory import FACTORY, reset_adapters from dbt.cli.exceptions import DbtUsageException from dbt.cli.main import dbtRunner from dbt.exceptions import DbtProjectError from dbt.tests.util import read_file, write_file from dbt.version import __version__ as dbt_version from dbt_common.events.contextvars import get_node_info class TestDbtRunner: @pytest.fixture def dbt(self) -> dbtRunner: return dbtRunner() @pytest.fixture(scope="class") def models(self): return { "models.sql": "select 1 as id", } def test_group_invalid_option(self, dbt: dbtRunner) -> None: res = dbt.invoke(["--invalid-option"]) assert type(res.exception) == DbtUsageException def test_command_invalid_option(self, dbt: dbtRunner) -> None: res = dbt.invoke(["deps", "--invalid-option"]) assert type(res.exception) == DbtUsageException def test_command_mutually_exclusive_option(self, dbt: dbtRunner) -> None: res = dbt.invoke(["--warn-error", "--warn-error-options", '{"error": "all"}', "deps"]) assert type(res.exception) == DbtUsageException res = dbt.invoke(["deps", "--warn-error", "--warn-error-options", '{"error": "all"}']) assert type(res.exception) == DbtUsageException res = dbt.invoke(["compile", "--select", "models", "--inline", "select 1 as id"]) assert type(res.exception) == DbtUsageException def test_invalid_command(self, dbt: dbtRunner) -> None: res = dbt.invoke(["invalid-command"]) assert type(res.exception) == DbtUsageException def test_invoke_version(self, dbt: dbtRunner) -> None: dbt.invoke(["--version"]) def test_callbacks(self) -> None: mock_callback = mock.MagicMock() dbt = dbtRunner(callbacks=[mock_callback]) # the `debug` command is one of the few commands wherein you don't need # to have a project to run it and it will emit events dbt.invoke(["debug"]) mock_callback.assert_called() def test_callback_node_finished_exceptions_are_raised(self, project): from dbt_common.events.base_types import EventMsg def callback_with_exception(event: EventMsg): if event.info.name == "NodeFinished": raise Exception("This should let continue the execution registering the failure") dbt = dbtRunner(callbacks=[callback_with_exception]) result = dbt.invoke(["run", "--select", "models"]) assert result is not None assert ( result.result.results[0].message == "Exception on worker thread. This should let continue the execution registering the failure" ) def test_invoke_kwargs(self, project, dbt): res = dbt.invoke( ["run"], log_format="json", log_path="some_random_path", version_check=False, profile_name="some_random_profile_name", target_dir="some_random_target_dir", ) assert res.result.args["log_format"] == "json" assert res.result.args["log_path"] == "some_random_path" assert res.result.args["version_check"] is False assert res.result.args["profile_name"] == "some_random_profile_name" assert res.result.args["target_dir"] == "some_random_target_dir" def test_invoke_kwargs_project_dir(self, project, dbt): res = dbt.invoke(["run"], project_dir="some_random_project_dir") assert type(res.exception) == DbtProjectError msg = "No dbt_project.yml found at expected path some_random_project_dir" assert msg in res.exception.msg def test_invoke_kwargs_profiles_dir(self, project, dbt): res = dbt.invoke(["run"], profiles_dir="some_random_profiles_dir") assert type(res.exception) == DbtProjectError msg = "Could not find profile named 'test'" assert msg in res.exception.msg def test_invoke_kwargs_and_flags(self, project, dbt): res = dbt.invoke(["--log-format=text", "run"], log_format="json") assert res.result.args["log_format"] == "json" def test_pass_in_manifest(self, project, dbt): result = dbt.invoke(["parse"]) manifest = result.result reset_adapters() assert len(FACTORY.adapters) == 0 result = dbtRunner(manifest=manifest).invoke(["run"]) # Check that the adapters are registered again. assert result.success assert len(FACTORY.adapters) == 1 def test_pass_in_args_variable(self, dbt): args = ["--log-format", "text"] args_before = args.copy() dbt.invoke(args) assert args == args_before def test_directory_does_not_change(self, project, dbt: dbtRunner) -> None: project_dir = os.getcwd() # The directory where dbt_project.yml exists. os.chdir("../") cmd_execution_dir = os.getcwd() # The directory where dbt command will be run commands = ["init", "deps", "clean"] for command in commands: args = [command, "--project-dir", project_dir] if command == "init": args.append("--skip-profile-setup") res = dbt.invoke(args) after_dir = os.getcwd() assert res.success is True assert cmd_execution_dir == after_dir class TestDbtRunnerQueryComments: @pytest.fixture(scope="class") def models(self): return { "models.sql": "select 1 as id", } @pytest.fixture(scope="class") def project_config_update(self): return { "query-comment": { "comment": f"comment: {dbt_version}", "append": True, } } def test_query_comment_saved_manifest(self, project, logs_dir): dbt = dbtRunner() dbt.invoke(["build", "--select", "models"]) result = dbt.invoke(["parse"]) write_file("", logs_dir, "dbt.log") # pass in manifest from parse command dbt = dbtRunner(result.result) dbt.invoke(["build", "--select", "models"]) log_file = read_file(logs_dir, "dbt.log") assert f"comment: {dbt_version}" in log_file class TestDbtRunnerHooks: @pytest.fixture(scope="class") def models(self): return { "models.sql": "select 1 as id", } @pytest.fixture(scope="class") def project_config_update(self): return {"on-run-end": ["select 1;"]} def test_node_info_non_persistence(self, project): dbt = dbtRunner() dbt.invoke(["run", "--select", "models"]) assert get_node_info() == {} ================================================ FILE: tests/functional/defer_state/data/manifest.json ================================================ { "metadata": { "dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v12.json", "dbt_version": "PREVIOUS", "generated_at": "2025-11-19T17:45:05.479140Z", "invocation_id": "e78f29e3-abf6-480a-a16c-b1f6cca0242e", "invocation_started_at": "2025-11-19T17:45:05.337658Z", "env": {}, "project_name": "test", "project_id": "098f6bcd4621d373cade4e832627b4f6", "user_id": null, "send_anonymous_usage_stats": false, "adapter_type": "postgres", "quoting": { "database": true, "schema": true, "identifier": true, "column": null }, "run_started_at": "2025-11-19T17:45:05.337822+00:00" }, "nodes": { "model.test.model_with_lots_of_schema_configs": { "database": "dbt", "schema": "test17635743055401889788_test_modified_state_schema_evolution_test", "name": "model_with_lots_of_schema_configs", "resource_type": "model", "package_name": "test", "path": "model_with_lots_of_schema_configs.sql", "original_file_path": "models/model_with_lots_of_schema_configs.sql", "unique_id": "model.test.model_with_lots_of_schema_configs", "fqn": [ "test", "model_with_lots_of_schema_configs" ], "alias": "outer_alias", "checksum": { "name": "sha256", "checksum": "0cb8913f9c7b25d64892882b5f85341353fd6862ae5b2b69df3da19a5bf474f2" }, "config": { "enabled": true, "alias": "outer_alias", "schema": "test", "database": "dbt", "tags": [ "string_tag" ], "meta": { "my_custom_property": "string_meta" }, "group": null, "materialized": "table", "incremental_strategy": null, "batch_size": "day", "lookback": 5, "begin": "2020-01-01", "persist_docs": { "columns": true, "relation": true }, "post-hook": [ { "sql": "SELECT 'string_post_hook' as my_post_hook;", "transaction": true, "index": null } ], "pre-hook": [ { "sql": "SELECT 'string_pre_hook' as my_pre_hook;", "transaction": true, "index": null } ], "quoting": {}, "column_types": {}, "full_refresh": false, "unique_key": "id", "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": { "select": [ "root" ] }, "packages": [], "docs": { "show": true, "node_color": "purple" }, "contract": { "enforced": false, "alias_types": true }, "event_time": null, "concurrent_batches": false, "access": "public", "freshness": null, "sql_header": "SELECT 1 as header;" }, "tags": [ "string_tag" ], "description": "A model with lots of configs", "columns": { "id": { "name": "id", "description": "The id value", "meta": { "column_meta": "column_meta_value" }, "data_type": null, "constraints": [], "quote": null, "config": { "meta": { "column_meta": "column_meta_value" }, "tags": [ "column_level_tag" ] }, "tags": [ "column_level_tag" ], "granularity": null, "doc_blocks": [] }, "created_at": { "name": "created_at", "description": "The date the row was created", "meta": {}, "data_type": null, "constraints": [], "quote": null, "config": { "meta": {}, "tags": [] }, "tags": [], "granularity": null, "doc_blocks": [] } }, "meta": { "my_custom_property": "string_meta" }, "group": null, "docs": { "show": true, "node_color": "purple" }, "patch_path": "test://models/schema.yml", "build_path": null, "unrendered_config": { "enabled": true, "access": "public", "alias": "outer_alias", "batch_size": "day", "begin": "2020-01-01", "concurrent_batches": false, "contract": { "alias_types": true, "enforce": true }, "docs": { "node_color": "purple", "show": true }, "database": "dbt", "full_refresh": false, "grants": { "select": [ "root", "root" ] }, "lookback": 5, "materialized": "table", "meta": { "my_custom_property": "string_meta" }, "on_configuration_change": "apply", "persist_docs": { "columns": true, "relation": true }, "post_hook": "SELECT 'string_post_hook' as my_post_hook;", "pre_hook": "SELECT 'string_pre_hook' as my_pre_hook;", "schema": "test", "sql_header": "SELECT 1 as header;", "tags": "string_tag", "unique_key": "id" }, "created_at": 1763574306.275303, "relation_name": "\"dbt\".\"test17635743055401889788_test_modified_state_schema_evolution_test\".\"outer_alias\"", "raw_code": "select * from {{ ref('ephemeral') }}", "doc_blocks": [], "language": "sql", "refs": [ { "name": "ephemeral", "package": null, "version": null } ], "sources": [ [ "my_source", "my_table" ] ], "metrics": [], "functions": [], "depends_on": { "macros": [], "nodes": [ "source.test.my_source.my_table", "model.test.ephemeral" ] }, "compiled_path": null, "contract": { "enforced": false, "alias_types": true, "checksum": null }, "access": "public", "constraints": [ { "type": "primary_key", "name": null, "expression": null, "warn_unenforced": true, "warn_unsupported": false, "to": null, "to_columns": [], "columns": [ "id" ] }, { "type": "foreign_key", "name": null, "expression": null, "warn_unenforced": true, "warn_unsupported": true, "to": "source('my_source', 'my_table')", "to_columns": [ "id" ], "columns": [ "id" ] }, { "type": "check", "name": "Check that id is greater than 0", "expression": "id > 0", "warn_unenforced": true, "warn_unsupported": true, "to": null, "to_columns": [], "columns": [ "id" ] } ], "version": null, "latest_version": null, "deprecation_date": "2052-05-01T00:00:00-04:00", "primary_key": [ "id" ], "time_spine": null }, "model.test.snapshot_source": { "database": "dbt", "schema": "test17635743055401889788_test_modified_state_schema_evolution", "name": "snapshot_source", "resource_type": "model", "package_name": "test", "path": "snapshot_source.sql", "original_file_path": "models/snapshot_source.sql", "unique_id": "model.test.snapshot_source", "fqn": [ "test", "snapshot_source" ], "alias": "snapshot_source", "checksum": { "name": "sha256", "checksum": "0e0757d1f14cdd418b570e1d0206e397b732254483eea28663600385370d3b74" }, "config": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "batch_size": null, "lookback": 1, "begin": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "contract": { "enforced": false, "alias_types": true }, "event_time": null, "concurrent_batches": null, "access": "protected", "freshness": null }, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": { "show": true, "node_color": null }, "patch_path": null, "build_path": null, "unrendered_config": { "enabled": true }, "created_at": 1763574305.7719748, "relation_name": "\"dbt\".\"test17635743055401889788_test_modified_state_schema_evolution\".\"snapshot_source\"", "raw_code": "select 0 as id, 1 as col1, 2 as col2, 3 as col3", "doc_blocks": [], "language": "sql", "refs": [], "sources": [], "metrics": [], "functions": [], "depends_on": { "macros": [], "nodes": [] }, "compiled_path": null, "contract": { "enforced": false, "alias_types": true, "checksum": null }, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null, "primary_key": [], "time_spine": null }, "model.test.ephemeral": { "database": "dbt", "schema": "test17635743055401889788_test_modified_state_schema_evolution", "name": "ephemeral", "resource_type": "model", "package_name": "test", "path": "ephemeral.sql", "original_file_path": "models/ephemeral.sql", "unique_id": "model.test.ephemeral", "fqn": [ "test", "ephemeral" ], "alias": "ephemeral", "checksum": { "name": "sha256", "checksum": "3cd361ea49139705c81325d9312d7a959b9afa7144d06fd04ea575ee3abd7467" }, "config": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "ephemeral", "incremental_strategy": null, "batch_size": null, "lookback": 1, "begin": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "contract": { "enforced": false, "alias_types": true }, "event_time": null, "concurrent_batches": null, "access": "protected", "freshness": null }, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": { "show": true, "node_color": null }, "patch_path": null, "build_path": null, "unrendered_config": { "enabled": true, "materialized": "Keyword(key='materialized', value=Const(value='ephemeral'))" }, "created_at": 1763574305.772938, "relation_name": null, "raw_code": "{{ config(materialized='ephemeral') }}\n\nselect\n 1 as id,\n {{ dbt.date_trunc('day', dbt.current_timestamp()) }} as created_at", "doc_blocks": [], "language": "sql", "refs": [], "sources": [], "metrics": [], "functions": [], "depends_on": { "macros": [ "macro.dbt.current_timestamp", "macro.dbt.date_trunc" ], "nodes": [] }, "compiled_path": null, "contract": { "enforced": false, "alias_types": true, "checksum": null }, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null, "primary_key": [], "time_spine": null }, "model.test.metricflow_time_spine": { "database": "dbt", "schema": "test17635743055401889788_test_modified_state_schema_evolution", "name": "metricflow_time_spine", "resource_type": "model", "package_name": "test", "path": "metricflow_time_spine.sql", "original_file_path": "models/metricflow_time_spine.sql", "unique_id": "model.test.metricflow_time_spine", "fqn": [ "test", "metricflow_time_spine" ], "alias": "metricflow_time_spine", "checksum": { "name": "sha256", "checksum": "dc614982c2a9f37c6a0db29f72c795e9b04b36edd4130b293c0237835c05c163" }, "config": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [ "list", "of", "tags" ], "meta": {}, "group": "finance", "materialized": "view", "incremental_strategy": null, "batch_size": null, "lookback": 1, "begin": null, "persist_docs": {}, "post-hook": [ { "sql": "SELECT 'string_post_hook' as my_post_hook;", "transaction": true, "index": null } ], "pre-hook": [ { "sql": "SELECT 'string_pre_hook' as my_pre_hook;", "transaction": true, "index": null } ], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "contract": { "enforced": false, "alias_types": true }, "event_time": null, "concurrent_batches": null, "access": "protected", "freshness": null }, "tags": [ "list", "of", "tags" ], "description": "Day time spine", "columns": { "date_day": { "name": "date_day", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "config": { "meta": {}, "tags": [] }, "tags": [], "granularity": "day", "doc_blocks": [] } }, "meta": {}, "group": "finance", "docs": { "show": true, "node_color": null }, "patch_path": "test://models/schema.yml", "build_path": null, "unrendered_config": { "enabled": true, "tags": [ "list", "of", "tags", "list", "of", "tags" ], "pre_hook": [ "SELECT 'string_pre_hook' as my_pre_hook;", "SELECT 'string_pre_hook' as my_pre_hook;" ], "post_hook": [ "SELECT 'string_post_hook' as my_post_hook;", "SELECT 'string_post_hook' as my_post_hook;" ], "group": "finance" }, "created_at": 1763574306.268285, "relation_name": "\"dbt\".\"test17635743055401889788_test_modified_state_schema_evolution\".\"metricflow_time_spine\"", "raw_code": "select\n {{ dbt.date_trunc('day', dbt.current_timestamp()) }} as date_day", "doc_blocks": [], "language": "sql", "refs": [], "sources": [], "metrics": [], "functions": [], "depends_on": { "macros": [ "macro.dbt.current_timestamp", "macro.dbt.date_trunc" ], "nodes": [] }, "compiled_path": null, "contract": { "enforced": false, "alias_types": true, "checksum": null }, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null, "primary_key": [], "time_spine": { "standard_granularity_column": "date_day", "custom_granularities": [] } }, "model.test.incremental": { "database": "dbt", "schema": "test17635743055401889788_test_modified_state_schema_evolution", "name": "incremental", "resource_type": "model", "package_name": "test", "path": "incremental.sql", "original_file_path": "models/incremental.sql", "unique_id": "model.test.incremental", "fqn": [ "test", "incremental" ], "alias": "incremental", "checksum": { "name": "sha256", "checksum": "cc19247f485dc040e2ab4b141f6e01b07e2d9a4e133f74e3cb19e235f8d73297" }, "config": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "incremental", "incremental_strategy": "delete+insert", "batch_size": null, "lookback": 1, "begin": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "contract": { "enforced": false, "alias_types": true }, "event_time": null, "concurrent_batches": null, "access": "protected", "freshness": null }, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": { "show": true, "node_color": null }, "patch_path": null, "build_path": null, "unrendered_config": { "enabled": true, "materialized": "Keyword(key='materialized', value=Const(value='incremental'))", "incremental_strategy": "Keyword(key='incremental_strategy', value=Const(value='delete+insert'))" }, "created_at": 1763574305.783703, "relation_name": "\"dbt\".\"test17635743055401889788_test_modified_state_schema_evolution\".\"incremental\"", "raw_code": "{{\n config(\n materialized = \"incremental\",\n incremental_strategy = \"delete+insert\",\n )\n}}\n\nselect * from {{ ref('seed') }}\n\n{% if is_incremental() %}\n where a > (select max(a) from {{this}})\n{% endif %}", "doc_blocks": [], "language": "sql", "refs": [ { "name": "seed", "package": null, "version": null } ], "sources": [], "metrics": [], "functions": [], "depends_on": { "macros": [ "macro.dbt.is_incremental" ], "nodes": [ "seed.test.seed" ] }, "compiled_path": null, "contract": { "enforced": false, "alias_types": true, "checksum": null }, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null, "primary_key": [], "time_spine": null }, "model.test.model_to_unit_test": { "database": "dbt", "schema": "test17635743055401889788_test_modified_state_schema_evolution", "name": "model_to_unit_test", "resource_type": "model", "package_name": "test", "path": "model_to_unit_test.sql", "original_file_path": "models/model_to_unit_test.sql", "unique_id": "model.test.model_to_unit_test", "fqn": [ "test", "model_to_unit_test" ], "alias": "model_to_unit_test", "checksum": { "name": "sha256", "checksum": "480d7b0969f63e01b8aa3355cd6b52043c81d0570f5f9a04bd4ffc124413cd43" }, "config": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "table", "incremental_strategy": null, "batch_size": null, "lookback": 1, "begin": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "contract": { "enforced": false, "alias_types": true }, "event_time": null, "concurrent_batches": null, "access": "protected", "freshness": null }, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": { "show": true, "node_color": null }, "patch_path": null, "build_path": null, "unrendered_config": { "enabled": true, "materialized": "table" }, "created_at": 1763574305.787908, "relation_name": "\"dbt\".\"test17635743055401889788_test_modified_state_schema_evolution\".\"model_to_unit_test\"", "raw_code": "{{ config(materialized='table') }}\n\nSELECT * FROM {{ ref('seed')}}", "doc_blocks": [], "language": "sql", "refs": [ { "name": "seed", "package": null, "version": null } ], "sources": [], "metrics": [], "functions": [], "depends_on": { "macros": [], "nodes": [ "seed.test.seed" ] }, "compiled_path": null, "contract": { "enforced": false, "alias_types": true, "checksum": null }, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null, "primary_key": [], "time_spine": null }, "model.test.outer": { "database": "dbt", "schema": "test17635743055401889788_test_modified_state_schema_evolution", "name": "outer", "resource_type": "model", "package_name": "test", "path": "outer.sql", "original_file_path": "models/outer.sql", "unique_id": "model.test.outer", "fqn": [ "test", "outer" ], "alias": "outer", "checksum": { "name": "sha256", "checksum": "0cb8913f9c7b25d64892882b5f85341353fd6862ae5b2b69df3da19a5bf474f2" }, "config": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "batch_size": null, "lookback": 1, "begin": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "contract": { "enforced": false, "alias_types": true }, "event_time": null, "concurrent_batches": null, "access": "protected", "freshness": null }, "tags": [], "description": "The outer table", "columns": { "id": { "name": "id", "description": "The id value", "meta": {}, "data_type": null, "constraints": [], "quote": null, "config": { "meta": {}, "tags": [] }, "tags": [], "granularity": null, "doc_blocks": [] } }, "meta": {}, "group": null, "docs": { "show": true, "node_color": null }, "patch_path": "test://models/schema.yml", "build_path": null, "unrendered_config": { "enabled": true }, "created_at": 1763574306.265461, "relation_name": "\"dbt\".\"test17635743055401889788_test_modified_state_schema_evolution\".\"outer\"", "raw_code": "select * from {{ ref('ephemeral') }}", "doc_blocks": [], "language": "sql", "refs": [ { "name": "ephemeral", "package": null, "version": null } ], "sources": [], "metrics": [], "functions": [], "depends_on": { "macros": [], "nodes": [ "model.test.ephemeral" ] }, "compiled_path": null, "contract": { "enforced": false, "alias_types": true, "checksum": null }, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null, "primary_key": [ "id" ], "time_spine": null }, "model.test.metricflow_time_spine_second": { "database": "dbt", "schema": "test17635743055401889788_test_modified_state_schema_evolution", "name": "metricflow_time_spine_second", "resource_type": "model", "package_name": "test", "path": "metricflow_time_spine_second.sql", "original_file_path": "models/metricflow_time_spine_second.sql", "unique_id": "model.test.metricflow_time_spine_second", "fqn": [ "test", "metricflow_time_spine_second" ], "alias": "metricflow_time_spine_second", "checksum": { "name": "sha256", "checksum": "7b2cca2e77552af42f8fd890f2c271bd8b462b9c04f138f39c59cb2d4aa6a2fb" }, "config": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "batch_size": null, "lookback": 1, "begin": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "contract": { "enforced": false, "alias_types": true }, "event_time": "ts_second", "concurrent_batches": null, "access": "protected", "freshness": null }, "tags": [], "description": "Second time spine", "columns": { "ts_second": { "name": "ts_second", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "config": { "meta": {}, "tags": [] }, "tags": [], "granularity": "second", "doc_blocks": [] } }, "meta": {}, "group": null, "docs": { "show": true, "node_color": null }, "patch_path": "test://models/schema.yml", "build_path": null, "unrendered_config": { "enabled": true, "event_time": "ts_second" }, "created_at": 1763574306.269429, "relation_name": "\"dbt\".\"test17635743055401889788_test_modified_state_schema_evolution\".\"metricflow_time_spine_second\"", "raw_code": "select\n {{ dbt.date_trunc('second', dbt.current_timestamp()) }} as ts_second", "doc_blocks": [], "language": "sql", "refs": [], "sources": [], "metrics": [], "functions": [], "depends_on": { "macros": [ "macro.dbt.current_timestamp", "macro.dbt.date_trunc" ], "nodes": [] }, "compiled_path": null, "contract": { "enforced": false, "alias_types": true, "checksum": null }, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null, "primary_key": [], "time_spine": null }, "model.test.inner": { "database": "dbt", "schema": "test17635743055401889788_test_modified_state_schema_evolution", "name": "inner", "resource_type": "model", "package_name": "test", "path": "sub/inner.sql", "original_file_path": "models/sub/inner.sql", "unique_id": "model.test.inner", "fqn": [ "test", "sub", "inner" ], "alias": "inner", "checksum": { "name": "sha256", "checksum": "61c1448ee04b0ef968040b9ff7724bd19419bb3af70761211289ee7f53993cb5" }, "config": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "batch_size": null, "lookback": 1, "begin": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "contract": { "enforced": false, "alias_types": true }, "event_time": null, "concurrent_batches": null, "access": "protected", "freshness": null }, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": { "show": true, "node_color": null }, "patch_path": null, "build_path": null, "unrendered_config": { "enabled": true }, "created_at": 1763574305.792904, "relation_name": "\"dbt\".\"test17635743055401889788_test_modified_state_schema_evolution\".\"inner\"", "raw_code": "select * from {{ ref('outer') }}", "doc_blocks": [], "language": "sql", "refs": [ { "name": "outer", "package": null, "version": null } ], "sources": [], "metrics": [], "functions": [], "depends_on": { "macros": [], "nodes": [ "model.test.outer" ] }, "compiled_path": null, "contract": { "enforced": false, "alias_types": true, "checksum": null }, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null, "primary_key": [], "time_spine": null }, "snapshot.test.my_snapshot": { "database": "dbt", "schema": "test17635743055401889788_test_modified_state_schema_evolution_test17635743055401889788_test_modified_state_schema_evolution", "name": "my_snapshot", "resource_type": "snapshot", "package_name": "test", "path": "snapshot.sql", "original_file_path": "snapshots/snapshot.sql", "unique_id": "snapshot.test.my_snapshot", "fqn": [ "test", "snapshot", "my_snapshot" ], "alias": "my_snapshot", "checksum": { "name": "sha256", "checksum": "a13f80cecda78dbbcada4125f66e98de708321d6ca0765f2023e046a8679ed0f" }, "config": { "enabled": true, "alias": null, "schema": "test17635743055401889788_test_modified_state_schema_evolution", "database": "dbt", "tags": [], "meta": {}, "group": null, "materialized": "snapshot", "incremental_strategy": null, "batch_size": null, "lookback": 1, "begin": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": "id", "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "contract": { "enforced": false, "alias_types": true }, "event_time": null, "concurrent_batches": null, "strategy": "timestamp", "target_schema": null, "target_database": null, "updated_at": "updated_at", "check_cols": null, "snapshot_meta_column_names": { "dbt_valid_to": null, "dbt_valid_from": null, "dbt_scd_id": null, "dbt_updated_at": null, "dbt_is_deleted": null }, "dbt_valid_to_current": null }, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": { "show": true, "node_color": null }, "patch_path": null, "build_path": null, "unrendered_config": { "database": "Keyword(key='database', value=Call(node=Name(name='var', ctx='load'), args=[Const(value='target_database'), Name(name='database', ctx='load')], kwargs=[], dyn_args=None, dyn_kwargs=None))", "schema": "Keyword(key='schema', value=Name(name='schema', ctx='load'))", "unique_key": "Keyword(key='unique_key', value=Const(value='id'))", "strategy": "Keyword(key='strategy', value=Const(value='timestamp'))", "updated_at": "Keyword(key='updated_at', value=Const(value='updated_at'))" }, "created_at": 1763574305.874798, "relation_name": "\"dbt\".\"test17635743055401889788_test_modified_state_schema_evolution_test17635743055401889788_test_modified_state_schema_evolution\".\"my_snapshot\"", "raw_code": "\n {{\n config(\n database=var('target_database', database),\n schema=schema,\n unique_key='id',\n strategy='timestamp',\n updated_at='updated_at',\n )\n }}\n select * from {{database}}.{{schema}}.seed\n", "doc_blocks": [], "language": "sql", "refs": [], "sources": [], "metrics": [], "functions": [], "depends_on": { "macros": [], "nodes": [] }, "compiled_path": null, "contract": { "enforced": false, "alias_types": true, "checksum": null } }, "analysis.test.a": { "database": "dbt", "schema": "test17635743055401889788_test_modified_state_schema_evolution", "name": "a", "resource_type": "analysis", "package_name": "test", "path": "analysis/a.sql", "original_file_path": "analyses/a.sql", "unique_id": "analysis.test.a", "fqn": [ "test", "analysis", "a" ], "alias": "a", "checksum": { "name": "sha256", "checksum": "a389c282f569f0bbdc2a8a4f174dea746c28582fdaf2048d31d9226af9feab23" }, "config": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [ "tag" ], "meta": { "test": 1 }, "group": "finance", "materialized": "view", "incremental_strategy": null, "batch_size": null, "lookback": 1, "begin": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": { "show": true, "node_color": "purple" }, "contract": { "enforced": false, "alias_types": true }, "event_time": null, "concurrent_batches": null }, "tags": [ "tag" ], "description": "description", "columns": { "id": { "name": "id", "description": "id description", "meta": {}, "data_type": null, "constraints": [], "quote": null, "config": { "meta": {}, "tags": [] }, "tags": [], "granularity": null, "doc_blocks": [] } }, "meta": { "test": 1 }, "group": "finance", "docs": { "show": true, "node_color": "purple" }, "patch_path": "test://analyses/a.yml", "build_path": null, "unrendered_config": { "enabled": true, "tags": [ "tag", "tag" ], "meta": { "test": 1 }, "docs": { "show": true, "node_color": "purple" }, "group": "finance" }, "created_at": 1763574306.459871, "relation_name": null, "raw_code": "select 4 as id", "doc_blocks": [], "language": "sql", "refs": [], "sources": [], "metrics": [], "functions": [], "depends_on": { "macros": [], "nodes": [] }, "compiled_path": null, "contract": { "enforced": false, "alias_types": true, "checksum": null } }, "test.test.t": { "database": "dbt", "schema": "test17635743055401889788_test_modified_state_schema_evolution_dbt_test__audit", "name": "t", "resource_type": "test", "package_name": "test", "path": "t.sql", "original_file_path": "tests/t.sql", "unique_id": "test.test.t", "fqn": [ "test", "t" ], "alias": "test_alias", "checksum": { "name": "sha256", "checksum": "26307df23d5d989e0b3bcc6c6079c1f2060a9b831945c7898678d18b0b6257c7" }, "config": { "enabled": true, "alias": "test_alias", "schema": "dbt_test__audit", "database": "dbt", "tags": [ "test_tag" ], "meta": { "my_custom_meta_key": "my_custom_meta_value" }, "group": "important_tests", "materialized": "test", "severity": "warn", "store_failures": true, "store_failures_as": "table", "where": "1 = 1", "limit": 10, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0" }, "tags": [ "test_tag" ], "description": "Single Data Test in happy path fixture", "columns": {}, "meta": { "my_custom_meta_key": "my_custom_meta_value" }, "group": "important_tests", "docs": { "show": true, "node_color": null }, "patch_path": "test://tests/schema.yml", "build_path": null, "unrendered_config": { "alias": "test_alias", "database": "dbt", "group": "important_tests", "enabled": true, "error_if": "!= 0", "fail_calc": "count(*)", "limit": 10, "meta": { "my_custom_meta_key": "my_custom_meta_value" }, "schema": "dbt_test__audit", "severity": "warn", "store_failures": true, "store_failures_as": "table", "tags": [ "test_tag" ], "warn_if": "!= 0", "where": "1 = 1" }, "created_at": 1763574306.435062, "relation_name": "\"dbt\".\"test17635743055401889788_test_modified_state_schema_evolution_dbt_test__audit\".\"test_alias\"", "raw_code": "select 1 as id WHERE 1 = 2", "doc_blocks": [], "language": "sql", "refs": [], "sources": [], "metrics": [], "functions": [], "depends_on": { "macros": [], "nodes": [] }, "compiled_path": null, "contract": { "enforced": false, "alias_types": true, "checksum": null } }, "seed.test.seed": { "database": "dbt", "schema": "test17635743055401889788_test_modified_state_schema_evolution_test", "name": "seed", "resource_type": "seed", "package_name": "test", "path": "seed.csv", "original_file_path": "seeds/seed.csv", "unique_id": "seed.test.seed", "fqn": [ "test", "seed" ], "alias": "test_alias", "checksum": { "name": "sha256", "checksum": "aeedab1ee7a1043753c9ab768594bc8420d7b85491d0be9421edc3813c237f4c" }, "config": { "enabled": true, "alias": "test_alias", "schema": "test", "database": "dbt", "tags": [ "tag" ], "meta": { "meta_key": "meta_value" }, "group": "finance", "materialized": "seed", "incremental_strategy": null, "batch_size": null, "lookback": 1, "begin": null, "persist_docs": {}, "post-hook": [ { "sql": "select 1", "transaction": true, "index": null } ], "pre-hook": [ { "sql": "select 1", "transaction": true, "index": null } ], "quoting": {}, "column_types": { "a": "BIGINT" }, "full_refresh": true, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": { "show": true, "node_color": "purple" }, "contract": { "enforced": false, "alias_types": true }, "event_time": "my_time_field", "concurrent_batches": null, "delimiter": ",", "quote_columns": false }, "tags": [ "tag" ], "description": "test_description", "columns": { "a": { "name": "a", "description": "a description", "meta": {}, "data_type": null, "constraints": [], "quote": true, "config": { "meta": {}, "tags": [ "tag" ] }, "tags": [ "tag" ], "granularity": null, "doc_blocks": [] }, "b": { "name": "b", "description": "b description", "meta": {}, "data_type": null, "constraints": [], "quote": true, "config": { "meta": {}, "tags": [ "tag" ] }, "tags": [ "tag" ], "granularity": null, "doc_blocks": [] } }, "meta": { "meta_key": "meta_value" }, "group": "finance", "docs": { "show": true, "node_color": "purple" }, "patch_path": "test://seeds/s.yml", "build_path": null, "unrendered_config": { "quote_columns": false, "docs": { "show": true, "node_color": "purple" }, "column_types": { "a": "BIGINT" }, "delimiter": ",", "enabled": true, "tags": "tag", "pre_hook": "select 1", "post_hook": "select 1", "database": "dbt", "schema": "test", "alias": "test_alias", "persist_docs": {}, "full_refresh": true, "meta": { "meta_key": "meta_value" }, "grants": {}, "event_time": "my_time_field", "group": "finance" }, "created_at": 1763574306.141397, "relation_name": "\"dbt\".\"test17635743055401889788_test_modified_state_schema_evolution_test\".\"test_alias\"", "raw_code": "", "doc_blocks": [], "root_path": "/private/var/folders/yj/7kyhyl8s3ql8bb1vjwkh_8pr0000gp/T/pytest-of-michelleark/pytest-20/project0", "depends_on": { "macros": [] } }, "test.test.not_null_seed__a_.6b59640cde": { "database": "dbt", "schema": "test17635743055401889788_test_modified_state_schema_evolution_dbt_test__audit", "name": "not_null_seed__a_", "resource_type": "test", "package_name": "test", "path": "not_null_seed__a_.sql", "original_file_path": "seeds/s.yml", "unique_id": "test.test.not_null_seed__a_.6b59640cde", "fqn": [ "test", "not_null_seed__a_" ], "alias": "not_null_seed__a_", "checksum": { "name": "none", "checksum": "" }, "config": { "enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0" }, "tags": [ "tag" ], "description": "", "columns": {}, "meta": {}, "group": "finance", "docs": { "show": true, "node_color": null }, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1763574306.232834, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "doc_blocks": [], "language": "sql", "refs": [ { "name": "seed", "package": null, "version": null } ], "sources": [], "metrics": [], "functions": [], "depends_on": { "macros": [ "macro.dbt.test_not_null" ], "nodes": [ "seed.test.seed" ] }, "compiled_path": null, "contract": { "enforced": false, "alias_types": true, "checksum": null }, "column_name": "\"a\"", "file_key_name": "seeds.seed", "attached_node": "seed.test.seed", "test_metadata": { "name": "not_null", "kwargs": { "column_name": "\"a\"", "model": "{{ get_where_subquery(ref('seed')) }}" }, "namespace": null } }, "test.test.not_null_seed__b_.a088b263cb": { "database": "dbt", "schema": "test17635743055401889788_test_modified_state_schema_evolution_dbt_test__audit", "name": "not_null_seed__b_", "resource_type": "test", "package_name": "test", "path": "not_null_seed__b_.sql", "original_file_path": "seeds/s.yml", "unique_id": "test.test.not_null_seed__b_.a088b263cb", "fqn": [ "test", "not_null_seed__b_" ], "alias": "not_null_seed__b_", "checksum": { "name": "none", "checksum": "" }, "config": { "enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0" }, "tags": [ "tag" ], "description": "", "columns": {}, "meta": {}, "group": "finance", "docs": { "show": true, "node_color": null }, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1763574306.2336462, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "doc_blocks": [], "language": "sql", "refs": [ { "name": "seed", "package": null, "version": null } ], "sources": [], "metrics": [], "functions": [], "depends_on": { "macros": [ "macro.dbt.test_not_null" ], "nodes": [ "seed.test.seed" ] }, "compiled_path": null, "contract": { "enforced": false, "alias_types": true, "checksum": null }, "column_name": "\"b\"", "file_key_name": "seeds.seed", "attached_node": "seed.test.seed", "test_metadata": { "name": "not_null", "kwargs": { "column_name": "\"b\"", "model": "{{ get_where_subquery(ref('seed')) }}" }, "namespace": null } }, "test.test.expression_is_true_seed_b_2.4e0babbea4": { "database": "dbt", "schema": "test17635743055401889788_test_modified_state_schema_evolution_dbt_test__audit", "name": "expression_is_true_seed_b_2", "resource_type": "test", "package_name": "test", "path": "expression_is_true_seed_b_2.sql", "original_file_path": "seeds/s.yml", "unique_id": "test.test.expression_is_true_seed_b_2.4e0babbea4", "fqn": [ "test", "expression_is_true_seed_b_2" ], "alias": "expression_is_true_seed_b_2", "checksum": { "name": "none", "checksum": "" }, "config": { "enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0" }, "tags": [], "description": "", "columns": {}, "meta": {}, "group": "finance", "docs": { "show": true, "node_color": null }, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1763574306.234266, "relation_name": null, "raw_code": "{{ test_expression_is_true(**_dbt_generic_test_kwargs) }}", "doc_blocks": [], "language": "sql", "refs": [ { "name": "seed", "package": null, "version": null } ], "sources": [], "metrics": [], "functions": [], "depends_on": { "macros": [ "macro.test.test_expression_is_true", "macro.dbt.get_where_subquery" ], "nodes": [ "seed.test.seed" ] }, "compiled_path": null, "contract": { "enforced": false, "alias_types": true, "checksum": null }, "column_name": null, "file_key_name": "seeds.seed", "attached_node": "seed.test.seed", "test_metadata": { "name": "expression_is_true", "kwargs": { "expression": "b = 2", "model": "{{ get_where_subquery(ref('seed')) }}" }, "namespace": null } }, "test.test.unique_outer_id.2195e332d3": { "database": "dbt", "schema": "test17635743055401889788_test_modified_state_schema_evolution_dbt_test__audit", "name": "unique_outer_id", "resource_type": "test", "package_name": "test", "path": "unique_outer_id.sql", "original_file_path": "models/schema.yml", "unique_id": "test.test.unique_outer_id.2195e332d3", "fqn": [ "test", "unique_outer_id" ], "alias": "unique_outer_id", "checksum": { "name": "none", "checksum": "" }, "config": { "enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0" }, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": { "show": true, "node_color": null }, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1763574306.276534, "relation_name": null, "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", "doc_blocks": [], "language": "sql", "refs": [ { "name": "outer", "package": null, "version": null } ], "sources": [], "metrics": [], "functions": [], "depends_on": { "macros": [ "macro.dbt.test_unique" ], "nodes": [ "model.test.outer" ] }, "compiled_path": null, "contract": { "enforced": false, "alias_types": true, "checksum": null }, "column_name": "id", "file_key_name": "models.outer", "attached_node": "model.test.outer", "test_metadata": { "name": "unique", "kwargs": { "column_name": "id", "model": "{{ get_where_subquery(ref('outer')) }}" }, "namespace": null } }, "test.test.not_null_outer_id.a226f4fb36": { "database": "dbt", "schema": "test17635743055401889788_test_modified_state_schema_evolution_dbt_test__audit", "name": "not_null_outer_id", "resource_type": "test", "package_name": "test", "path": "not_null_outer_id.sql", "original_file_path": "models/schema.yml", "unique_id": "test.test.not_null_outer_id.a226f4fb36", "fqn": [ "test", "not_null_outer_id" ], "alias": "not_null_outer_id", "checksum": { "name": "none", "checksum": "" }, "config": { "enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0" }, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": { "show": true, "node_color": null }, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1763574306.277153, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "doc_blocks": [], "language": "sql", "refs": [ { "name": "outer", "package": null, "version": null } ], "sources": [], "metrics": [], "functions": [], "depends_on": { "macros": [ "macro.dbt.test_not_null" ], "nodes": [ "model.test.outer" ] }, "compiled_path": null, "contract": { "enforced": false, "alias_types": true, "checksum": null }, "column_name": "id", "file_key_name": "models.outer", "attached_node": "model.test.outer", "test_metadata": { "name": "not_null", "kwargs": { "column_name": "id", "model": "{{ get_where_subquery(ref('outer')) }}" }, "namespace": null } }, "test.test.unique_model_with_lots_of_schema_configs_id.8328d84982": { "database": "dbt", "schema": "test17635743055401889788_test_modified_state_schema_evolution_dbt_test__audit", "name": "unique_model_with_lots_of_schema_configs_id", "resource_type": "test", "package_name": "test", "path": "unique_model_with_lots_of_schema_configs_id.sql", "original_file_path": "models/schema.yml", "unique_id": "test.test.unique_model_with_lots_of_schema_configs_id.8328d84982", "fqn": [ "test", "unique_model_with_lots_of_schema_configs_id" ], "alias": "unique_model_with_lots_of_schema_configs_id", "checksum": { "name": "none", "checksum": "" }, "config": { "enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0" }, "tags": [ "column_level_tag" ], "description": "", "columns": {}, "meta": {}, "group": null, "docs": { "show": true, "node_color": null }, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1763574306.277745, "relation_name": null, "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", "doc_blocks": [], "language": "sql", "refs": [ { "name": "model_with_lots_of_schema_configs", "package": null, "version": null } ], "sources": [], "metrics": [], "functions": [], "depends_on": { "macros": [ "macro.dbt.test_unique" ], "nodes": [ "model.test.model_with_lots_of_schema_configs" ] }, "compiled_path": null, "contract": { "enforced": false, "alias_types": true, "checksum": null }, "column_name": "id", "file_key_name": "models.model_with_lots_of_schema_configs", "attached_node": "model.test.model_with_lots_of_schema_configs", "test_metadata": { "name": "unique", "kwargs": { "column_name": "id", "model": "{{ get_where_subquery(ref('model_with_lots_of_schema_configs')) }}" }, "namespace": null } }, "test.test.my_favorite_test.b488d63233": { "database": "dbt", "schema": "test17635743055401889788_test_modified_state_schema_evolution_dbt_test__audit", "name": "my_favorite_test", "resource_type": "test", "package_name": "test", "path": "my_favorite_test.sql", "original_file_path": "models/schema.yml", "unique_id": "test.test.my_favorite_test.b488d63233", "fqn": [ "test", "my_favorite_test" ], "alias": "not_null__id__alias", "checksum": { "name": "none", "checksum": "" }, "config": { "enabled": true, "alias": "not_null__id__alias", "schema": "dbt_test__audit", "database": "dbt", "tags": [ "test_tag" ], "meta": { "my_custom_meta_key": "my_custom_meta_value" }, "group": "important_tests", "materialized": "test", "severity": "warn", "store_failures": true, "store_failures_as": "table", "where": "1 = 1", "limit": 10, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0" }, "tags": [ "column_level_tag", "test_tag" ], "description": "A test that should pass", "columns": {}, "meta": { "my_custom_meta_key": "my_custom_meta_value" }, "group": null, "docs": { "show": true, "node_color": null }, "patch_path": null, "build_path": null, "unrendered_config": { "severity": "warn", "tags": [ "test_tag" ], "enabled": true, "where": "1 = 1", "limit": 10, "warn_if": "!= 0", "error_if": "!= 0", "fail_calc": "count(*)", "store_failures": true, "store_failures_as": "table", "meta": { "my_custom_meta_key": "my_custom_meta_value" }, "database": "dbt", "schema": "dbt_test__audit", "alias": "not_null__id__alias", "group": "important_tests" }, "created_at": 1763574306.2793431, "relation_name": "\"dbt\".\"test17635743055401889788_test_modified_state_schema_evolution_dbt_test__audit\".\"not_null__id__alias\"", "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}{{ config(severity=\"warn\",tags=['test_tag'],enabled=True,where=\"1 = 1\",limit=10,warn_if=\"!= 0\",error_if=\"!= 0\",fail_calc=\"count(*)\",store_failures=True,store_failures_as=\"table\",meta={'my_custom_meta_key': 'my_custom_meta_value'},database=\"dbt\",schema=\"dbt_test__audit\",alias=\"not_null__id__alias\",group=\"important_tests\") }}", "doc_blocks": [], "language": "sql", "refs": [ { "name": "model_with_lots_of_schema_configs", "package": null, "version": null } ], "sources": [], "metrics": [], "functions": [], "depends_on": { "macros": [ "macro.dbt.test_not_null" ], "nodes": [ "model.test.model_with_lots_of_schema_configs" ] }, "compiled_path": null, "contract": { "enforced": false, "alias_types": true, "checksum": null }, "column_name": "id", "file_key_name": "models.model_with_lots_of_schema_configs", "attached_node": "model.test.model_with_lots_of_schema_configs", "test_metadata": { "name": "not_null", "kwargs": { "column_name": "id", "model": "{{ get_where_subquery(ref('model_with_lots_of_schema_configs')) }}" }, "namespace": null } }, "test.test.my_second_favorite_test.c8955109ad": { "database": "dbt", "schema": "test17635743055401889788_test_modified_state_schema_evolution_dbt_test__audit", "name": "my_second_favorite_test", "resource_type": "test", "package_name": "test", "path": "my_second_favorite_test.sql", "original_file_path": "models/schema.yml", "unique_id": "test.test.my_second_favorite_test.c8955109ad", "fqn": [ "test", "my_second_favorite_test" ], "alias": "my_generic_test__created_at__alias", "checksum": { "name": "none", "checksum": "" }, "config": { "enabled": true, "alias": "my_generic_test__created_at__alias", "schema": "dbt_test__audit", "database": "dbt", "tags": [ "test_tag", "test_tag" ], "meta": { "my_custom_meta_key": "my_custom_meta_value" }, "group": "important_tests", "materialized": "test", "severity": "warn", "store_failures": true, "store_failures_as": "table", "where": "1 = 1", "limit": 10, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0" }, "tags": [ "test_tag" ], "description": "A test that should pass", "columns": {}, "meta": { "my_custom_meta_key": "my_custom_meta_value" }, "group": null, "docs": { "show": true, "node_color": null }, "patch_path": null, "build_path": null, "unrendered_config": { "severity": "Keyword(key='severity', value=Const(value='warn'))", "tags": "Keyword(key='tags', value=List(items=[Const(value='test_tag')]))", "enabled": "Keyword(key='enabled', value=Const(value=True))", "where": "Keyword(key='where', value=Const(value='1 = 1'))", "limit": "Keyword(key='limit', value=Const(value=10))", "warn_if": "Keyword(key='warn_if', value=Const(value='!= 0'))", "error_if": "Keyword(key='error_if', value=Const(value='!= 0'))", "fail_calc": "Keyword(key='fail_calc', value=Const(value='count(*)'))", "store_failures": "Keyword(key='store_failures', value=Const(value=True))", "store_failures_as": "Keyword(key='store_failures_as', value=Const(value='table'))", "meta": "Keyword(key='meta', value=Dict(items=[Pair(key=Const(value='my_custom_meta_key'), value=Const(value='my_custom_meta_value'))]))", "database": "Keyword(key='database', value=Const(value='dbt'))", "schema": "Keyword(key='schema', value=Const(value='dbt_test__audit'))", "alias": "Keyword(key='alias', value=Const(value='my_generic_test__created_at__alias'))", "group": "Keyword(key='group', value=Const(value='important_tests'))" }, "created_at": 1763574306.2801669, "relation_name": "\"dbt\".\"test17635743055401889788_test_modified_state_schema_evolution_dbt_test__audit\".\"my_generic_test__created_at__alias\"", "raw_code": "{{ test_my_generic_test(**_dbt_generic_test_kwargs) }}{{ config(severity=\"warn\",tags=['test_tag'],enabled=True,where=\"1 = 1\",limit=10,warn_if=\"!= 0\",error_if=\"!= 0\",fail_calc=\"count(*)\",store_failures=True,store_failures_as=\"table\",meta={'my_custom_meta_key': 'my_custom_meta_value'},database=\"dbt\",schema=\"dbt_test__audit\",alias=\"my_generic_test__created_at__alias\",group=\"important_tests\") }}", "doc_blocks": [], "language": "sql", "refs": [ { "name": "model_with_lots_of_schema_configs", "package": null, "version": null } ], "sources": [], "metrics": [], "functions": [], "depends_on": { "macros": [ "macro.test.test_my_generic_test", "macro.dbt.get_where_subquery" ], "nodes": [ "model.test.model_with_lots_of_schema_configs" ] }, "compiled_path": null, "contract": { "enforced": false, "alias_types": true, "checksum": null }, "column_name": "created_at", "file_key_name": "models.model_with_lots_of_schema_configs", "attached_node": "model.test.model_with_lots_of_schema_configs", "test_metadata": { "name": "my_generic_test", "kwargs": { "column_name": "created_at", "model": "{{ get_where_subquery(ref('model_with_lots_of_schema_configs')) }}" }, "namespace": null } }, "snapshot.test.snapshot_2": { "database": "mydb", "schema": "test17635743055401889788_test_modified_state_schema_evolution_myschema", "name": "snapshot_2", "resource_type": "snapshot", "package_name": "test", "path": "snapshot_2.yml/snapshot_2.sql", "original_file_path": "snapshots/snapshot_2.yml", "unique_id": "snapshot.test.snapshot_2", "fqn": [ "test", "snapshot_2" ], "alias": "snapshot_2", "checksum": { "name": "sha256", "checksum": "a2b723667ee663b45eed04aeccb8accd9de12c66fa7e291cd053c67795093630" }, "config": { "enabled": true, "alias": null, "schema": "myschema", "database": "mydb", "tags": [], "meta": { "owner": "@alice", "maturity": "in dev" }, "group": null, "materialized": "snapshot", "incremental_strategy": null, "batch_size": null, "lookback": 1, "begin": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": "id", "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": { "show": true, "node_color": "purple" }, "contract": { "enforced": false, "alias_types": true }, "event_time": null, "concurrent_batches": null, "strategy": "check", "target_schema": null, "target_database": null, "updated_at": "updated_at", "check_cols": [ "alpha", "beta", "delta" ], "snapshot_meta_column_names": { "dbt_valid_to": "my_valid_to_col", "dbt_valid_from": "my_from_col", "dbt_scd_id": "my_scd_col", "dbt_updated_at": "my_updated_at_col", "dbt_is_deleted": "my_is_deleted_col" }, "dbt_valid_to_current": "valid_to_current", "hard_deletes": "new_record" }, "tags": [], "description": "Description of snapshot_2", "columns": {}, "meta": { "owner": "@alice", "maturity": "in dev" }, "group": null, "docs": { "show": true, "node_color": "purple" }, "patch_path": "test://snapshots/snapshot_2.yml", "build_path": null, "unrendered_config": { "database": "mydb", "schema": "myschema", "unique_key": "id", "strategy": "check", "updated_at": "updated_at", "check_cols": [ "alpha", "beta", "delta", "alpha", "beta", "delta" ], "snapshot_meta_column_names": { "dbt_valid_from": "my_from_col", "dbt_valid_to": "my_valid_to_col", "dbt_scd_id": "my_scd_col", "dbt_updated_at": "my_updated_at_col", "dbt_is_deleted": "my_is_deleted_col" }, "hard_deletes": "new_record", "dbt_valid_to_current": "valid_to_current", "meta": { "owner": "@alice", "maturity": "in dev" }, "docs": { "sfhow": true, "node_color": "purple" } }, "created_at": 1763574306.447405, "relation_name": "\"mydb\".\"test17635743055401889788_test_modified_state_schema_evolution_myschema\".\"snapshot_2\"", "raw_code": "select * from {{ ref(\"snapshot_source\") }}", "doc_blocks": [], "language": "sql", "refs": [ { "name": "snapshot_source", "package": null, "version": null } ], "sources": [], "metrics": [], "functions": [], "depends_on": { "macros": [], "nodes": [ "model.test.snapshot_source" ] }, "compiled_path": null, "contract": { "enforced": false, "alias_types": true, "checksum": null } }, "snapshot.test.snapshot_3": { "database": "mydb", "schema": "test17635743055401889788_test_modified_state_schema_evolution_myschema", "name": "snapshot_3", "resource_type": "snapshot", "package_name": "test", "path": "snapshot_3.yml/snapshot_3.sql", "original_file_path": "snapshots/snapshot_3.yml", "unique_id": "snapshot.test.snapshot_3", "fqn": [ "test", "snapshot_3" ], "alias": "snapshot_3", "checksum": { "name": "sha256", "checksum": "0310dbfcd03499e4cdc206b694a207008604fb45b7a7c6ee6849b55ecd64b1fe" }, "config": { "enabled": true, "alias": null, "schema": "myschema", "database": "mydb", "tags": [], "meta": { "owner": "@alice", "maturity": "in dev" }, "group": null, "materialized": "snapshot", "incremental_strategy": null, "batch_size": null, "lookback": 1, "begin": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": "id", "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": { "show": true, "node_color": "purple" }, "contract": { "enforced": false, "alias_types": true }, "event_time": null, "concurrent_batches": null, "strategy": "timestamp", "target_schema": null, "target_database": null, "updated_at": "updated_at", "check_cols": null, "snapshot_meta_column_names": { "dbt_valid_to": "my_valid_to_col", "dbt_valid_from": "my_from_col", "dbt_scd_id": "my_scd_col", "dbt_updated_at": "my_updated_at_col", "dbt_is_deleted": "my_is_deleted_col" }, "dbt_valid_to_current": "valid_to_current", "invalidate_hard_deletes": true }, "tags": [], "description": "Description of snapshot_3", "columns": {}, "meta": { "owner": "@alice", "maturity": "in dev" }, "group": null, "docs": { "show": true, "node_color": "purple" }, "patch_path": "test://snapshots/snapshot_3.yml", "build_path": null, "unrendered_config": { "database": "mydb", "schema": "myschema", "unique_key": "id", "strategy": "timestamp", "updated_at": "updated_at", "snapshot_meta_column_names": { "dbt_valid_from": "my_from_col", "dbt_valid_to": "my_valid_to_col", "dbt_scd_id": "my_scd_col", "dbt_updated_at": "my_updated_at_col", "dbt_is_deleted": "my_is_deleted_col" }, "invalidate_hard_deletes": true, "dbt_valid_to_current": "valid_to_current", "meta": { "owner": "@alice", "maturity": "in dev" }, "docs": { "sfhow": true, "node_color": "purple" } }, "created_at": 1763574306.4514718, "relation_name": "\"mydb\".\"test17635743055401889788_test_modified_state_schema_evolution_myschema\".\"snapshot_3\"", "raw_code": "select * from {{ ref(\"snapshot_source\") }}", "doc_blocks": [], "language": "sql", "refs": [ { "name": "snapshot_source", "package": null, "version": null } ], "sources": [], "metrics": [], "functions": [], "depends_on": { "macros": [], "nodes": [ "model.test.snapshot_source" ] }, "compiled_path": null, "contract": { "enforced": false, "alias_types": true, "checksum": null } } }, "sources": { "source.test.my_source.my_table": { "database": "raw", "schema": "jaffle_shop", "name": "my_table", "resource_type": "source", "package_name": "test", "path": "models/schema.yml", "original_file_path": "models/schema.yml", "unique_id": "source.test.my_source.my_table", "fqn": [ "test", "my_source", "my_table" ], "source_name": "my_source", "source_description": "description", "loader": "test", "identifier": "table_identifier", "quoting": { "database": true, "schema": true, "identifier": true, "column": null }, "loaded_at_field": "column_name", "loaded_at_query": null, "freshness": { "warn_after": { "count": 1, "period": "minute" }, "error_after": { "count": 2, "period": "hour" }, "filter": "column_name = 1" }, "external": { "location": "location", "file_format": "file_format", "row_format": "row_format", "tbl_properties": "tbl_properties", "partitions": [ { "name": "column_name", "data_type": "data_type", "description": "description", "meta": { "test": 1 } } ], "additional_property": "additional_value" }, "description": "description", "columns": { "column_name": { "name": "column_name", "description": "description", "meta": { "test": 1 }, "data_type": null, "constraints": [], "quote": true, "config": { "meta": { "test": 1 }, "tags": [ "column_tag" ] }, "tags": [ "column_tag" ], "granularity": null, "doc_blocks": [] } }, "meta": { "source_meta": 1, "table_meta": 1 }, "source_meta": { "source_meta": 1 }, "tags": [ "source_tag", "table_tag" ], "config": { "enabled": true, "event_time": "column_name", "freshness": { "warn_after": { "count": 1, "period": "minute" }, "error_after": { "count": 2, "period": "hour" }, "filter": "column_name = 1" }, "loaded_at_field": "column_name", "loaded_at_query": null, "meta": { "source_meta": 1, "table_meta": 1 }, "tags": [ "source_tag", "table_tag" ] }, "patch_path": null, "unrendered_config": { "enabled": true, "event_time": "column_name", "loaded_at_field": "column_name", "meta": { "source_meta": 1, "table_meta": 1 }, "tags": [ "source_tag", "table_tag" ], "freshness": { "warn_after": { "count": 1, "period": "minute" }, "error_after": { "count": 2, "period": "hour" }, "filter": "column_name = 1" }, "loaded_at_query": null }, "relation_name": "\"raw\".\"jaffle_shop\".\"table_identifier\"", "created_at": 1763574306.468715, "unrendered_database": "raw", "unrendered_schema": "jaffle_shop", "doc_blocks": [] } }, "macros": { "macro.test.test_expression_is_true": { "name": "test_expression_is_true", "resource_type": "macro", "package_name": "test", "path": "macros/expression_is_true.sql", "original_file_path": "macros/expression_is_true.sql", "unique_id": "macro.test.test_expression_is_true", "macro_sql": "{% test expression_is_true(model, expression, column_name=None) %}\n\n{% set column_list = '*' if should_store_failures() else \"1\" %}\n\nselect\n {{ column_list }}\nfrom {{ model }}\n{% if column_name is none %}\nwhere not({{ expression }})\n{%- else %}\nwhere not({{ column_name }} {{ expression }})\n{%- endif %}\n\n{% endtest %}", "depends_on": { "macros": [ "macro.dbt.should_store_failures" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.494549, "supported_languages": null }, "macro.test.cool_macro": { "name": "cool_macro", "resource_type": "macro", "package_name": "test", "path": "macros/macro_stuff.sql", "original_file_path": "macros/macro_stuff.sql", "unique_id": "macro.test.cool_macro", "macro_sql": "{% macro cool_macro() %}\n wow!\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4946592, "supported_languages": null }, "macro.test.other_cool_macro": { "name": "other_cool_macro", "resource_type": "macro", "package_name": "test", "path": "macros/macro_stuff.sql", "original_file_path": "macros/macro_stuff.sql", "unique_id": "macro.test.other_cool_macro", "macro_sql": "{% macro other_cool_macro(a, b) %}\n cool!\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.494739, "supported_languages": null }, "macro.test.test_my_generic_test": { "name": "test_my_generic_test", "resource_type": "macro", "package_name": "test", "path": "tests/generic/my_generic_test.sql", "original_file_path": "tests/generic/my_generic_test.sql", "unique_id": "macro.test.test_my_generic_test", "macro_sql": "{% test my_generic_test(model, column_name) %}\n SELECT 1 as a_column WHERE 1 = 2\n{% endtest %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4948502, "supported_languages": null }, "macro.dbt_postgres.postgres__current_timestamp": { "name": "postgres__current_timestamp", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/timestamps.sql", "original_file_path": "macros/timestamps.sql", "unique_id": "macro.dbt_postgres.postgres__current_timestamp", "macro_sql": "{% macro postgres__current_timestamp() -%}\n now()\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4948668, "supported_languages": null }, "macro.dbt_postgres.postgres__snapshot_string_as_time": { "name": "postgres__snapshot_string_as_time", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/timestamps.sql", "original_file_path": "macros/timestamps.sql", "unique_id": "macro.dbt_postgres.postgres__snapshot_string_as_time", "macro_sql": "{% macro postgres__snapshot_string_as_time(timestamp) -%}\n {%- set result = \"'\" ~ timestamp ~ \"'::timestamp without time zone\" -%}\n {{ return(result) }}\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4948711, "supported_languages": null }, "macro.dbt_postgres.postgres__snapshot_get_time": { "name": "postgres__snapshot_get_time", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/timestamps.sql", "original_file_path": "macros/timestamps.sql", "unique_id": "macro.dbt_postgres.postgres__snapshot_get_time", "macro_sql": "{% macro postgres__snapshot_get_time() -%}\n {{ current_timestamp() }}::timestamp without time zone\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.current_timestamp" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.494874, "supported_languages": null }, "macro.dbt_postgres.postgres__current_timestamp_backcompat": { "name": "postgres__current_timestamp_backcompat", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/timestamps.sql", "original_file_path": "macros/timestamps.sql", "unique_id": "macro.dbt_postgres.postgres__current_timestamp_backcompat", "macro_sql": "{% macro postgres__current_timestamp_backcompat() %}\n current_timestamp::{{ type_timestamp() }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.type_timestamp" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.49488, "supported_languages": null }, "macro.dbt_postgres.postgres__current_timestamp_in_utc_backcompat": { "name": "postgres__current_timestamp_in_utc_backcompat", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/timestamps.sql", "original_file_path": "macros/timestamps.sql", "unique_id": "macro.dbt_postgres.postgres__current_timestamp_in_utc_backcompat", "macro_sql": "{% macro postgres__current_timestamp_in_utc_backcompat() %}\n (current_timestamp at time zone 'utc')::{{ type_timestamp() }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.type_timestamp" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.494883, "supported_languages": null }, "macro.dbt_postgres.postgres__get_catalog_relations": { "name": "postgres__get_catalog_relations", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/catalog.sql", "original_file_path": "macros/catalog.sql", "unique_id": "macro.dbt_postgres.postgres__get_catalog_relations", "macro_sql": "{% macro postgres__get_catalog_relations(information_schema, relations) -%}\n {%- call statement('catalog', fetch_result=True) -%}\n\n {#\n If the user has multiple databases set and the first one is wrong, this will fail.\n But we won't fail in the case where there are multiple quoting-difference-only dbs, which is better.\n #}\n {% set database = information_schema.database %}\n {{ adapter.verify_database(database) }}\n\n select\n '{{ database }}' as table_database,\n sch.nspname as table_schema,\n tbl.relname as table_name,\n case tbl.relkind\n when 'v' then 'VIEW'\n when 'm' then 'MATERIALIZED VIEW'\n else 'BASE TABLE'\n end as table_type,\n tbl_desc.description as table_comment,\n col.attname as column_name,\n col.attnum as column_index,\n pg_catalog.format_type(col.atttypid, col.atttypmod) as column_type,\n col_desc.description as column_comment,\n pg_get_userbyid(tbl.relowner) as table_owner\n\n from pg_catalog.pg_namespace sch\n join pg_catalog.pg_class tbl on tbl.relnamespace = sch.oid\n join pg_catalog.pg_attribute col on col.attrelid = tbl.oid\n left outer join pg_catalog.pg_description tbl_desc on (tbl_desc.objoid = tbl.oid and tbl_desc.objsubid = 0)\n left outer join pg_catalog.pg_description col_desc on (col_desc.objoid = tbl.oid and col_desc.objsubid = col.attnum)\n where (\n {%- for relation in relations -%}\n {%- if relation.identifier -%}\n (upper(sch.nspname) = upper('{{ relation.schema }}') and\n upper(tbl.relname) = upper('{{ relation.identifier }}'))\n {%- else-%}\n upper(sch.nspname) = upper('{{ relation.schema }}')\n {%- endif -%}\n {%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n )\n and not pg_is_other_temp_schema(sch.oid) -- not a temporary schema belonging to another session\n and tbl.relpersistence in ('p', 'u') -- [p]ermanent table or [u]nlogged table. Exclude [t]emporary tables\n and tbl.relkind in ('r', 'v', 'f', 'p', 'm') -- o[r]dinary table, [v]iew, [f]oreign table, [p]artitioned table, [m]aterialized view. Other values are [i]ndex, [S]equence, [c]omposite type, [t]OAST table\n and col.attnum > 0 -- negative numbers are used for system columns such as oid\n and not col.attisdropped -- column as not been dropped\n\n order by\n sch.nspname,\n tbl.relname,\n col.attnum\n\n {%- endcall -%}\n\n {{ return(load_result('catalog').table) }}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.statement" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4948952, "supported_languages": null }, "macro.dbt_postgres.postgres__get_catalog": { "name": "postgres__get_catalog", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/catalog.sql", "original_file_path": "macros/catalog.sql", "unique_id": "macro.dbt_postgres.postgres__get_catalog", "macro_sql": "{% macro postgres__get_catalog(information_schema, schemas) -%}\n {%- set relations = [] -%}\n {%- for schema in schemas -%}\n {%- set dummy = relations.append({'schema': schema}) -%}\n {%- endfor -%}\n {{ return(postgres__get_catalog_relations(information_schema, relations)) }}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_catalog_relations" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.494899, "supported_languages": null }, "macro.dbt_postgres.postgres__get_relations": { "name": "postgres__get_relations", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations.sql", "original_file_path": "macros/relations.sql", "unique_id": "macro.dbt_postgres.postgres__get_relations", "macro_sql": "{% macro postgres__get_relations() -%}\n\n {#\n -- in pg_depend, objid is the dependent, refobjid is the referenced object\n -- > a pg_depend entry indicates that the referenced object cannot be\n -- > dropped without also dropping the dependent object.\n #}\n\n {%- call statement('relations', fetch_result=True) -%}\n select distinct\n dependent_namespace.nspname as dependent_schema,\n dependent_class.relname as dependent_name,\n referenced_namespace.nspname as referenced_schema,\n referenced_class.relname as referenced_name\n\n -- Query for views: views are entries in pg_class with an entry in pg_rewrite, but we avoid\n -- a seq scan on pg_rewrite by leveraging the fact there is an \"internal\" row in pg_depend for\n -- the view...\n from pg_class as dependent_class\n join pg_namespace as dependent_namespace on dependent_namespace.oid = dependent_class.relnamespace\n join pg_depend as dependent_depend on dependent_depend.refobjid = dependent_class.oid\n and dependent_depend.classid = 'pg_rewrite'::regclass\n and dependent_depend.refclassid = 'pg_class'::regclass\n and dependent_depend.deptype = 'i'\n\n -- ... and via pg_depend (that has a row per column, hence the need for \"distinct\" above, and\n -- making sure to exclude the internal row to avoid a view appearing to depend on itself)...\n join pg_depend as joining_depend on joining_depend.objid = dependent_depend.objid\n and joining_depend.classid = 'pg_rewrite'::regclass\n and joining_depend.refclassid = 'pg_class'::regclass\n and joining_depend.refobjid != dependent_depend.refobjid\n\n -- ... we can find the tables they query from in pg_class, but excluding system tables. Note we\n -- don't need need to exclude _dependent_ system tables, because they only query from other\n -- system tables, and so are automatically excluded by excluding _referenced_ system tables\n join pg_class as referenced_class on referenced_class.oid = joining_depend.refobjid\n join pg_namespace as referenced_namespace on referenced_namespace.oid = referenced_class.relnamespace\n and referenced_namespace.nspname != 'information_schema'\n and referenced_namespace.nspname not like 'pg\\_%'\n\n order by\n dependent_schema, dependent_name, referenced_schema, referenced_name;\n\n {%- endcall -%}\n\n {{ return(load_result('relations').table) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.statement" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4949079, "supported_languages": null }, "macro.dbt_postgres.postgres_get_relations": { "name": "postgres_get_relations", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations.sql", "original_file_path": "macros/relations.sql", "unique_id": "macro.dbt_postgres.postgres_get_relations", "macro_sql": "{% macro postgres_get_relations() %}\n {{ return(postgres__get_relations()) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_relations" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.494912, "supported_languages": null }, "macro.dbt_postgres.postgres__create_table_as": { "name": "postgres__create_table_as", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__create_table_as", "macro_sql": "{% macro postgres__create_table_as(temporary, relation, sql) -%}\n {%- set unlogged = config.get('unlogged', default=false) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary -%}\n temporary\n {%- elif unlogged -%}\n unlogged\n {%- endif %} table {{ relation }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {% endif -%}\n {% if contract_config.enforced and (not temporary) -%}\n {{ get_table_columns_and_constraints() }} ;\n insert into {{ relation }} (\n {{ adapter.dispatch('get_column_names', 'dbt')() }}\n )\n {%- set sql = get_select_subquery(sql) %}\n {% else %}\n as\n {% endif %}\n (\n {{ sql }}\n );\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.get_assert_columns_equivalent", "macro.dbt.get_table_columns_and_constraints", "macro.dbt.default__get_column_names", "macro.dbt.get_select_subquery" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.494923, "supported_languages": null }, "macro.dbt_postgres.postgres__get_create_index_sql": { "name": "postgres__get_create_index_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__get_create_index_sql", "macro_sql": "{% macro postgres__get_create_index_sql(relation, index_dict) -%}\n {%- set index_config = adapter.parse_index(index_dict) -%}\n {%- set comma_separated_columns = \", \".join(index_config.columns) -%}\n {%- set index_name = index_config.render(relation) -%}\n\n create {% if index_config.unique -%}\n unique\n {%- endif %} index if not exists\n \"{{ index_name }}\"\n on {{ relation }} {% if index_config.type -%}\n using {{ index_config.type }}\n {%- endif %}\n ({{ comma_separated_columns }})\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.494926, "supported_languages": null }, "macro.dbt_postgres.postgres__create_schema": { "name": "postgres__create_schema", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__create_schema", "macro_sql": "{% macro postgres__create_schema(relation) -%}\n {% if relation.database -%}\n {{ adapter.verify_database(relation.database) }}\n {%- endif -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier().include(database=False) }}\n {%- endcall -%}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.statement" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4949288, "supported_languages": null }, "macro.dbt_postgres.postgres__drop_schema": { "name": "postgres__drop_schema", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__drop_schema", "macro_sql": "{% macro postgres__drop_schema(relation) -%}\n {% if relation.database -%}\n {{ adapter.verify_database(relation.database) }}\n {%- endif -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier().include(database=False) }} cascade\n {%- endcall -%}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.statement" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.494932, "supported_languages": null }, "macro.dbt_postgres.postgres__get_columns_in_relation": { "name": "postgres__get_columns_in_relation", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__get_columns_in_relation", "macro_sql": "{% macro postgres__get_columns_in_relation(relation) -%}\n {% call statement('get_columns_in_relation', fetch_result=True) %}\n select\n column_name,\n data_type,\n character_maximum_length,\n numeric_precision,\n numeric_scale\n\n from {{ relation.information_schema('columns') }}\n where table_name = '{{ relation.identifier }}'\n {% if relation.schema %}\n and table_schema = '{{ relation.schema }}'\n {% endif %}\n order by ordinal_position\n\n {% endcall %}\n {% set table = load_result('get_columns_in_relation').table %}\n {{ return(sql_convert_columns_in_relation(table)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.statement", "macro.dbt.sql_convert_columns_in_relation" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4949381, "supported_languages": null }, "macro.dbt_postgres.postgres__list_relations_without_caching": { "name": "postgres__list_relations_without_caching", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__list_relations_without_caching", "macro_sql": "{% macro postgres__list_relations_without_caching(schema_relation) %}\n {% call statement('list_relations_without_caching', fetch_result=True) -%}\n select\n '{{ schema_relation.database }}' as database,\n tablename as name,\n schemaname as schema,\n 'table' as type\n from pg_tables\n where schemaname ilike '{{ schema_relation.schema }}'\n union all\n select\n '{{ schema_relation.database }}' as database,\n viewname as name,\n schemaname as schema,\n 'view' as type\n from pg_views\n where schemaname ilike '{{ schema_relation.schema }}'\n union all\n select\n '{{ schema_relation.database }}' as database,\n matviewname as name,\n schemaname as schema,\n 'materialized_view' as type\n from pg_matviews\n where schemaname ilike '{{ schema_relation.schema }}'\n {% endcall %}\n {{ return(load_result('list_relations_without_caching').table) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.statement" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.494941, "supported_languages": null }, "macro.dbt_postgres.postgres__information_schema_name": { "name": "postgres__information_schema_name", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__information_schema_name", "macro_sql": "{% macro postgres__information_schema_name(database) -%}\n {% if database_name -%}\n {{ adapter.verify_database(database_name) }}\n {%- endif -%}\n information_schema\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.494977, "supported_languages": null }, "macro.dbt_postgres.postgres__list_schemas": { "name": "postgres__list_schemas", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__list_schemas", "macro_sql": "{% macro postgres__list_schemas(database) %}\n {% if database -%}\n {{ adapter.verify_database(database) }}\n {%- endif -%}\n {% call statement('list_schemas', fetch_result=True, auto_begin=False) %}\n select distinct nspname from pg_namespace\n {% endcall %}\n {{ return(load_result('list_schemas').table) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.statement" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.494981, "supported_languages": null }, "macro.dbt_postgres.postgres__check_schema_exists": { "name": "postgres__check_schema_exists", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__check_schema_exists", "macro_sql": "{% macro postgres__check_schema_exists(information_schema, schema) -%}\n {% if information_schema.database -%}\n {{ adapter.verify_database(information_schema.database) }}\n {%- endif -%}\n {% call statement('check_schema_exists', fetch_result=True, auto_begin=False) %}\n select count(*) from pg_namespace where nspname = '{{ schema }}'\n {% endcall %}\n {{ return(load_result('check_schema_exists').table) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.statement" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.494984, "supported_languages": null }, "macro.dbt_postgres.postgres__make_relation_with_suffix": { "name": "postgres__make_relation_with_suffix", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__make_relation_with_suffix", "macro_sql": "{% macro postgres__make_relation_with_suffix(base_relation, suffix, dstring) %}\n {% if dstring %}\n {% set dt = modules.datetime.datetime.now() %}\n {% set dtstring = dt.strftime(\"%H%M%S%f\") %}\n {% set suffix = suffix ~ dtstring %}\n {% endif %}\n {% set suffix_length = suffix|length %}\n {% set relation_max_name_length = base_relation.relation_max_name_length() %}\n {% if suffix_length > relation_max_name_length %}\n {% do exceptions.raise_compiler_error('Relation suffix is too long (' ~ suffix_length ~ ' characters). Maximum length is ' ~ relation_max_name_length ~ ' characters.') %}\n {% endif %}\n {% set identifier = base_relation.identifier[:relation_max_name_length - suffix_length] ~ suffix %}\n\n {{ return(base_relation.incorporate(path={\"identifier\": identifier })) }}\n\n {% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4949892, "supported_languages": null }, "macro.dbt_postgres.postgres__make_intermediate_relation": { "name": "postgres__make_intermediate_relation", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__make_intermediate_relation", "macro_sql": "{% macro postgres__make_intermediate_relation(base_relation, suffix) %}\n {{ return(postgres__make_relation_with_suffix(base_relation, suffix, dstring=False)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__make_relation_with_suffix" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.494993, "supported_languages": null }, "macro.dbt_postgres.postgres__make_temp_relation": { "name": "postgres__make_temp_relation", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__make_temp_relation", "macro_sql": "{% macro postgres__make_temp_relation(base_relation, suffix) %}\n {% set temp_relation = postgres__make_relation_with_suffix(base_relation, suffix, dstring=True) %}\n {{ return(temp_relation.incorporate(path={\"schema\": none,\n \"database\": none})) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__make_relation_with_suffix" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4950001, "supported_languages": null }, "macro.dbt_postgres.postgres__make_backup_relation": { "name": "postgres__make_backup_relation", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__make_backup_relation", "macro_sql": "{% macro postgres__make_backup_relation(base_relation, backup_relation_type, suffix) %}\n {% set backup_relation = postgres__make_relation_with_suffix(base_relation, suffix, dstring=False) %}\n {{ return(backup_relation.incorporate(type=backup_relation_type)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__make_relation_with_suffix" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495003, "supported_languages": null }, "macro.dbt_postgres.postgres_escape_comment": { "name": "postgres_escape_comment", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres_escape_comment", "macro_sql": "{% macro postgres_escape_comment(comment) -%}\n {% if comment is not string %}\n {% do exceptions.raise_compiler_error('cannot escape a non-string: ' ~ comment) %}\n {% endif %}\n {%- set magic = '$dbt_comment_literal_block$' -%}\n {%- if magic in comment -%}\n {%- do exceptions.raise_compiler_error('The string ' ~ magic ~ ' is not allowed in comments.') -%}\n {%- endif -%}\n {{ magic }}{{ comment }}{{ magic }}\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495008, "supported_languages": null }, "macro.dbt_postgres.postgres__alter_relation_comment": { "name": "postgres__alter_relation_comment", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__alter_relation_comment", "macro_sql": "{% macro postgres__alter_relation_comment(relation, comment) %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n {% if relation.type == 'materialized_view' -%}\n {% set relation_type = \"materialized view\" %}\n {%- else -%}\n {%- set relation_type = relation.type -%}\n {%- endif -%}\n comment on {{ relation_type }} {{ relation }} is {{ escaped_comment }};\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres_escape_comment" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4950101, "supported_languages": null }, "macro.dbt_postgres.postgres__alter_column_comment": { "name": "postgres__alter_column_comment", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__alter_column_comment", "macro_sql": "{% macro postgres__alter_column_comment(relation, column_dict) %}\n {% set existing_columns = adapter.get_columns_in_relation(relation) | map(attribute=\"name\") | list %}\n {% for column_name in column_dict if (column_name in existing_columns) %}\n {% set comment = column_dict[column_name]['description'] %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n comment on column {{ relation }}.{{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }} is {{ escaped_comment }};\n {% endfor %}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres_escape_comment" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4950132, "supported_languages": null }, "macro.dbt_postgres.postgres__get_show_grant_sql": { "name": "postgres__get_show_grant_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__get_show_grant_sql", "macro_sql": "\n\n{%- macro postgres__get_show_grant_sql(relation) -%}\n select grantee, privilege_type\n from {{ relation.information_schema('role_table_grants') }}\n where grantor = current_role\n and grantee != current_role\n and table_schema = '{{ relation.schema }}'\n and table_name = '{{ relation.identifier }}'\n{%- endmacro -%}\n\n", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495018, "supported_languages": null }, "macro.dbt_postgres.postgres__copy_grants": { "name": "postgres__copy_grants", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__copy_grants", "macro_sql": "{% macro postgres__copy_grants() %}\n {{ return(False) }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495021, "supported_languages": null }, "macro.dbt_postgres.postgres__get_show_indexes_sql": { "name": "postgres__get_show_indexes_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__get_show_indexes_sql", "macro_sql": "{% macro postgres__get_show_indexes_sql(relation) %}\n select\n i.relname as name,\n m.amname as method,\n ix.indisunique as \"unique\",\n array_to_string(array_agg(a.attname), ',') as column_names\n from pg_index ix\n join pg_class i\n on i.oid = ix.indexrelid\n join pg_am m\n on m.oid=i.relam\n join pg_class t\n on t.oid = ix.indrelid\n join pg_namespace n\n on n.oid = t.relnamespace\n join pg_attribute a\n on a.attrelid = t.oid\n and a.attnum = ANY(ix.indkey)\n where t.relname = '{{ relation.identifier }}'\n and n.nspname = '{{ relation.schema }}'\n and t.relkind in ('r', 'm')\n group by 1, 2, 3\n order by 1, 2, 3\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4950242, "supported_languages": null }, "macro.dbt_postgres.postgres__get_drop_index_sql": { "name": "postgres__get_drop_index_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_postgres.postgres__get_drop_index_sql", "macro_sql": "\n\n\n{%- macro postgres__get_drop_index_sql(relation, index_name) -%}\n drop index if exists \"{{ relation.schema }}\".\"{{ index_name }}\"\n{%- endmacro -%}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4950268, "supported_languages": null }, "macro.dbt_postgres.postgres__get_incremental_default_sql": { "name": "postgres__get_incremental_default_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/materializations/incremental_strategies.sql", "original_file_path": "macros/materializations/incremental_strategies.sql", "unique_id": "macro.dbt_postgres.postgres__get_incremental_default_sql", "macro_sql": "{% macro postgres__get_incremental_default_sql(arg_dict) %}\n\n {% if arg_dict[\"unique_key\"] %}\n {% do return(get_incremental_delete_insert_sql(arg_dict)) %}\n {% else %}\n {% do return(get_incremental_append_sql(arg_dict)) %}\n {% endif %}\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.get_incremental_delete_insert_sql", "macro.dbt.get_incremental_append_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495036, "supported_languages": null }, "macro.dbt_postgres.postgres__get_incremental_microbatch_sql": { "name": "postgres__get_incremental_microbatch_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/materializations/incremental_strategies.sql", "original_file_path": "macros/materializations/incremental_strategies.sql", "unique_id": "macro.dbt_postgres.postgres__get_incremental_microbatch_sql", "macro_sql": "{% macro postgres__get_incremental_microbatch_sql(arg_dict) %}\n\n {% if arg_dict[\"unique_key\"] %}\n {% do return(adapter.dispatch('get_incremental_merge_sql', 'dbt')(arg_dict)) %}\n {% else %}\n {{ exceptions.raise_compiler_error(\"dbt-postgres 'microbatch' requires a `unique_key` config\") }}\n {% endif %}\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.get_incremental_merge_sql", "macro.dbt.default__get_incremental_merge_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495039, "supported_languages": null }, "macro.dbt_postgres.postgres__snapshot_merge_sql": { "name": "postgres__snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/materializations/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshot_merge.sql", "unique_id": "macro.dbt_postgres.postgres__snapshot_merge_sql", "macro_sql": "{% macro postgres__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n {%- set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() -%}\n\n update {{ target }}\n set {{ columns.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ columns.dbt_valid_to }}\n from {{ source }} as DBT_INTERNAL_SOURCE\n where DBT_INTERNAL_SOURCE.{{ columns.dbt_scd_id }}::text = {{ target }}.{{ columns.dbt_scd_id }}::text\n and DBT_INTERNAL_SOURCE.dbt_change_type::text in ('update'::text, 'delete'::text)\n {% if config.get(\"dbt_valid_to_current\") %}\n and ({{ target }}.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }} or {{ target }}.{{ columns.dbt_valid_to }} is null);\n {% else %}\n and {{ target }}.{{ columns.dbt_valid_to }} is null;\n {% endif %}\n\n\n insert into {{ target }} ({{ insert_cols_csv }})\n select {% for column in insert_cols -%}\n DBT_INTERNAL_SOURCE.{{ column }} {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n from {{ source }} as DBT_INTERNAL_SOURCE\n where DBT_INTERNAL_SOURCE.dbt_change_type::text = 'insert'::text;\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.get_snapshot_table_column_names" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495046, "supported_languages": null }, "macro.dbt_postgres.postgres__drop_materialized_view": { "name": "postgres__drop_materialized_view", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/materialized_view/drop.sql", "original_file_path": "macros/relations/materialized_view/drop.sql", "unique_id": "macro.dbt_postgres.postgres__drop_materialized_view", "macro_sql": "{% macro postgres__drop_materialized_view(relation) -%}\n drop materialized view if exists {{ relation }} cascade\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495053, "supported_languages": null }, "macro.dbt_postgres.postgres__describe_materialized_view": { "name": "postgres__describe_materialized_view", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/materialized_view/describe.sql", "original_file_path": "macros/relations/materialized_view/describe.sql", "unique_id": "macro.dbt_postgres.postgres__describe_materialized_view", "macro_sql": "{% macro postgres__describe_materialized_view(relation) %}\n -- for now just get the indexes, we don't need the name or the query yet\n {% set _indexes = run_query(get_show_indexes_sql(relation)) %}\n {% do return({'indexes': _indexes}) %}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.run_query", "macro.dbt.get_show_indexes_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495059, "supported_languages": null }, "macro.dbt_postgres.postgres__refresh_materialized_view": { "name": "postgres__refresh_materialized_view", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/materialized_view/refresh.sql", "original_file_path": "macros/relations/materialized_view/refresh.sql", "unique_id": "macro.dbt_postgres.postgres__refresh_materialized_view", "macro_sql": "{% macro postgres__refresh_materialized_view(relation) %}\n refresh materialized view {{ relation }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495064, "supported_languages": null }, "macro.dbt_postgres.postgres__get_rename_materialized_view_sql": { "name": "postgres__get_rename_materialized_view_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/materialized_view/rename.sql", "original_file_path": "macros/relations/materialized_view/rename.sql", "unique_id": "macro.dbt_postgres.postgres__get_rename_materialized_view_sql", "macro_sql": "{% macro postgres__get_rename_materialized_view_sql(relation, new_name) %}\n alter materialized view {{ relation }} rename to {{ new_name }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495074, "supported_languages": null }, "macro.dbt_postgres.postgres__get_alter_materialized_view_as_sql": { "name": "postgres__get_alter_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt_postgres.postgres__get_alter_materialized_view_as_sql", "macro_sql": "{% macro postgres__get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n\n -- apply a full refresh immediately if needed\n {% if configuration_changes.requires_full_refresh %}\n\n {{ get_replace_sql(existing_relation, relation, sql) }}\n\n -- otherwise apply individual changes as needed\n {% else %}\n\n {{ postgres__update_indexes_on_materialized_view(relation, configuration_changes.indexes) }}\n\n {%- endif -%}\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.get_replace_sql", "macro.dbt_postgres.postgres__update_indexes_on_materialized_view" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.49508, "supported_languages": null }, "macro.dbt_postgres.postgres__update_indexes_on_materialized_view": { "name": "postgres__update_indexes_on_materialized_view", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt_postgres.postgres__update_indexes_on_materialized_view", "macro_sql": "\n\n\n{%- macro postgres__update_indexes_on_materialized_view(relation, index_changes) -%}\n {{- log(\"Applying UPDATE INDEXES to: \" ~ relation) -}}\n\n {%- for _index_change in index_changes -%}\n {%- set _index = _index_change.context -%}\n\n {%- if _index_change.action == \"drop\" -%}\n\n {{ postgres__get_drop_index_sql(relation, _index.name) }}\n\n {%- elif _index_change.action == \"create\" -%}\n\n {{ postgres__get_create_index_sql(relation, _index.as_node_config) }}\n\n {%- endif -%}\n\t{{ ';' if not loop.last else \"\" }}\n\n {%- endfor -%}\n\n{%- endmacro -%}\n\n\n", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_drop_index_sql", "macro.dbt_postgres.postgres__get_create_index_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495084, "supported_languages": null }, "macro.dbt_postgres.postgres__get_materialized_view_configuration_changes": { "name": "postgres__get_materialized_view_configuration_changes", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt_postgres.postgres__get_materialized_view_configuration_changes", "macro_sql": "{% macro postgres__get_materialized_view_configuration_changes(existing_relation, new_config) %}\n {% set _existing_materialized_view = postgres__describe_materialized_view(existing_relation) %}\n {% set _configuration_changes = existing_relation.get_materialized_view_config_change_collection(_existing_materialized_view, new_config.model) %}\n {% do return(_configuration_changes) %}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__describe_materialized_view" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4950888, "supported_languages": null }, "macro.dbt_postgres.postgres__get_create_materialized_view_as_sql": { "name": "postgres__get_create_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/materialized_view/create.sql", "original_file_path": "macros/relations/materialized_view/create.sql", "unique_id": "macro.dbt_postgres.postgres__get_create_materialized_view_as_sql", "macro_sql": "{% macro postgres__get_create_materialized_view_as_sql(relation, sql) %}\n create materialized view if not exists {{ relation }} as {{ sql }};\n\n {% for _index_dict in config.get('indexes', []) -%}\n {{- get_create_index_sql(relation, _index_dict) -}}{{ ';' if not loop.last else \"\" }}\n {%- endfor -%}\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.get_create_index_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495095, "supported_languages": null }, "macro.dbt_postgres.postgres__drop_table": { "name": "postgres__drop_table", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/table/drop.sql", "original_file_path": "macros/relations/table/drop.sql", "unique_id": "macro.dbt_postgres.postgres__drop_table", "macro_sql": "{% macro postgres__drop_table(relation) -%}\n drop table if exists {{ relation }} cascade\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495102, "supported_languages": null }, "macro.dbt_postgres.postgres__get_replace_table_sql": { "name": "postgres__get_replace_table_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/table/replace.sql", "original_file_path": "macros/relations/table/replace.sql", "unique_id": "macro.dbt_postgres.postgres__get_replace_table_sql", "macro_sql": "{% macro postgres__get_replace_table_sql(relation, sql) -%}\n\n {%- set sql_header = config.get('sql_header', none) -%}\n {{ sql_header if sql_header is not none }}\n\n create or replace table {{ relation }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {{ get_table_columns_and_constraints() }}\n {%- set sql = get_select_subquery(sql) %}\n {% endif %}\n as (\n {{ sql }}\n );\n\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.get_assert_columns_equivalent", "macro.dbt.get_table_columns_and_constraints", "macro.dbt.get_select_subquery" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495109, "supported_languages": null }, "macro.dbt_postgres.postgres__get_rename_table_sql": { "name": "postgres__get_rename_table_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/table/rename.sql", "original_file_path": "macros/relations/table/rename.sql", "unique_id": "macro.dbt_postgres.postgres__get_rename_table_sql", "macro_sql": "{% macro postgres__get_rename_table_sql(relation, new_name) %}\n alter table {{ relation }} rename to {{ new_name }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495114, "supported_languages": null }, "macro.dbt_postgres.postgres__drop_view": { "name": "postgres__drop_view", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/view/drop.sql", "original_file_path": "macros/relations/view/drop.sql", "unique_id": "macro.dbt_postgres.postgres__drop_view", "macro_sql": "{% macro postgres__drop_view(relation) -%}\n drop view if exists {{ relation }} cascade\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4951198, "supported_languages": null }, "macro.dbt_postgres.postgres__get_replace_view_sql": { "name": "postgres__get_replace_view_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt_postgres.postgres__get_replace_view_sql", "macro_sql": "{% macro postgres__get_replace_view_sql(relation, sql) -%}\n\n {%- set sql_header = config.get('sql_header', none) -%}\n {{ sql_header if sql_header is not none }}\n\n create or replace view {{ relation }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {%- endif %}\n as (\n {{ sql }}\n );\n\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.get_assert_columns_equivalent" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495129, "supported_languages": null }, "macro.dbt_postgres.postgres__get_rename_view_sql": { "name": "postgres__get_rename_view_sql", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/relations/view/rename.sql", "original_file_path": "macros/relations/view/rename.sql", "unique_id": "macro.dbt_postgres.postgres__get_rename_view_sql", "macro_sql": "{% macro postgres__get_rename_view_sql(relation, new_name) %}\n alter view {{ relation }} rename to {{ new_name }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495135, "supported_languages": null }, "macro.dbt_postgres.postgres__dateadd": { "name": "postgres__dateadd", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt_postgres.postgres__dateadd", "macro_sql": "{% macro postgres__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n {{ from_date_or_timestamp }} + ((interval '1 {{ datepart }}') * ({{ interval }}))\n\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495144, "supported_languages": null }, "macro.dbt_postgres.postgres__listagg": { "name": "postgres__listagg", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "unique_id": "macro.dbt_postgres.postgres__listagg", "macro_sql": "{% macro postgres__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n {% if limit_num -%}\n array_to_string(\n (array_agg(\n {{ measure }}\n {% if order_by_clause -%}\n {{ order_by_clause }}\n {%- endif %}\n ))[1:{{ limit_num }}],\n {{ delimiter_text }}\n )\n {%- else %}\n string_agg(\n {{ measure }},\n {{ delimiter_text }}\n {% if order_by_clause -%}\n {{ order_by_clause }}\n {%- endif %}\n )\n {%- endif %}\n\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495154, "supported_languages": null }, "macro.dbt_postgres.postgres__datediff": { "name": "postgres__datediff", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt_postgres.postgres__datediff", "macro_sql": "{% macro postgres__datediff(first_date, second_date, datepart) -%}\n\n {% if datepart == 'year' %}\n (date_part('year', ({{second_date}})::date) - date_part('year', ({{first_date}})::date))\n {% elif datepart == 'quarter' %}\n ({{ datediff(first_date, second_date, 'year') }} * 4 + date_part('quarter', ({{second_date}})::date) - date_part('quarter', ({{first_date}})::date))\n {% elif datepart == 'month' %}\n ({{ datediff(first_date, second_date, 'year') }} * 12 + date_part('month', ({{second_date}})::date) - date_part('month', ({{first_date}})::date))\n {% elif datepart == 'day' %}\n (({{second_date}})::date - ({{first_date}})::date)\n {% elif datepart == 'week' %}\n ({{ datediff(first_date, second_date, 'day') }} / 7 + case\n when date_part('dow', ({{first_date}})::timestamp) <= date_part('dow', ({{second_date}})::timestamp) then\n case when {{first_date}} <= {{second_date}} then 0 else -1 end\n else\n case when {{first_date}} <= {{second_date}} then 1 else 0 end\n end)\n {% elif datepart == 'hour' %}\n ({{ datediff(first_date, second_date, 'day') }} * 24 + date_part('hour', ({{second_date}})::timestamp) - date_part('hour', ({{first_date}})::timestamp))\n {% elif datepart == 'minute' %}\n ({{ datediff(first_date, second_date, 'hour') }} * 60 + date_part('minute', ({{second_date}})::timestamp) - date_part('minute', ({{first_date}})::timestamp))\n {% elif datepart == 'second' %}\n ({{ datediff(first_date, second_date, 'minute') }} * 60 + floor(date_part('second', ({{second_date}})::timestamp)) - floor(date_part('second', ({{first_date}})::timestamp)))\n {% elif datepart == 'millisecond' %}\n ({{ datediff(first_date, second_date, 'minute') }} * 60000 + floor(date_part('millisecond', ({{second_date}})::timestamp)) - floor(date_part('millisecond', ({{first_date}})::timestamp)))\n {% elif datepart == 'microsecond' %}\n ({{ datediff(first_date, second_date, 'minute') }} * 60000000 + floor(date_part('microsecond', ({{second_date}})::timestamp)) - floor(date_part('microsecond', ({{first_date}})::timestamp)))\n {% else %}\n {{ exceptions.raise_compiler_error(\"Unsupported datepart for macro datediff in postgres: {!r}\".format(datepart)) }}\n {% endif %}\n\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.datediff" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495161, "supported_languages": null }, "macro.dbt_postgres.postgres__any_value": { "name": "postgres__any_value", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "unique_id": "macro.dbt_postgres.postgres__any_value", "macro_sql": "{% macro postgres__any_value(expression) -%}\n\n min({{ expression }})\n\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495168, "supported_languages": null }, "macro.dbt_postgres.postgres__last_day": { "name": "postgres__last_day", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt_postgres.postgres__last_day", "macro_sql": "{% macro postgres__last_day(date, datepart) -%}\n\n {%- if datepart == 'quarter' -%}\n -- postgres dateadd does not support quarter interval.\n cast(\n {{dbt.dateadd('day', '-1',\n dbt.dateadd('month', '3', dbt.date_trunc(datepart, date))\n )}}\n as date)\n {%- else -%}\n {{dbt.default_last_day(date, datepart)}}\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.dateadd", "macro.dbt.date_trunc", "macro.dbt.default_last_day" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495174, "supported_languages": null }, "macro.dbt_postgres.postgres__split_part": { "name": "postgres__split_part", "resource_type": "macro", "package_name": "dbt_postgres", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt_postgres.postgres__split_part", "macro_sql": "{% macro postgres__split_part(string_text, delimiter_text, part_number) %}\n\n {% if part_number >= 0 %}\n {{ dbt.default__split_part(string_text, delimiter_text, part_number) }}\n {% else %}\n {{ dbt._split_part_negative(string_text, delimiter_text, part_number) }}\n {% endif %}\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__split_part", "macro.dbt._split_part_negative" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4951801, "supported_languages": null }, "macro.dbt.run_hooks": { "name": "run_hooks", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.run_hooks", "macro_sql": "{% macro run_hooks(hooks, inside_transaction=True) %}\n {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %}\n {% if not inside_transaction and loop.first %}\n {% call statement(auto_begin=inside_transaction) %}\n commit;\n {% endcall %}\n {% endif %}\n {% set rendered = render(hook.get('sql')) | trim %}\n {% if (rendered | length) > 0 %}\n {% call statement(auto_begin=inside_transaction) %}\n {{ rendered }}\n {% endcall %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.statement" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495187, "supported_languages": null }, "macro.dbt.make_hook_config": { "name": "make_hook_config", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.make_hook_config", "macro_sql": "{% macro make_hook_config(sql, inside_transaction) %}\n {{ tojson({\"sql\": sql, \"transaction\": inside_transaction}) }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.49519, "supported_languages": null }, "macro.dbt.before_begin": { "name": "before_begin", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.before_begin", "macro_sql": "{% macro before_begin(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.make_hook_config" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495193, "supported_languages": null }, "macro.dbt.in_transaction": { "name": "in_transaction", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.in_transaction", "macro_sql": "{% macro in_transaction(sql) %}\n {{ make_hook_config(sql, inside_transaction=True) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.make_hook_config" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4951968, "supported_languages": null }, "macro.dbt.after_commit": { "name": "after_commit", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.after_commit", "macro_sql": "{% macro after_commit(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.make_hook_config" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4952, "supported_languages": null }, "macro.dbt.set_sql_header": { "name": "set_sql_header", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "unique_id": "macro.dbt.set_sql_header", "macro_sql": "{% macro set_sql_header(config) -%}\n {{ config.set('sql_header', caller()) }}\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4952059, "supported_languages": null }, "macro.dbt.should_full_refresh": { "name": "should_full_refresh", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "unique_id": "macro.dbt.should_full_refresh", "macro_sql": "{% macro should_full_refresh() %}\n {% set config_full_refresh = config.get('full_refresh') %}\n {% if config_full_refresh is none %}\n {% set config_full_refresh = flags.FULL_REFRESH %}\n {% endif %}\n {% do return(config_full_refresh) %}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495209, "supported_languages": null }, "macro.dbt.should_store_failures": { "name": "should_store_failures", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "unique_id": "macro.dbt.should_store_failures", "macro_sql": "{% macro should_store_failures() %}\n {% set config_store_failures = config.get('store_failures') %}\n {% if config_store_failures is none %}\n {% set config_store_failures = flags.STORE_FAILURES %}\n {% endif %}\n {% do return(config_store_failures) %}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495212, "supported_languages": null }, "macro.dbt.snapshot_merge_sql": { "name": "snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", "unique_id": "macro.dbt.snapshot_merge_sql", "macro_sql": "{% macro snapshot_merge_sql(target, source, insert_cols) -%}\n {{ adapter.dispatch('snapshot_merge_sql', 'dbt')(target, source, insert_cols) }}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__snapshot_merge_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495222, "supported_languages": null }, "macro.dbt.default__snapshot_merge_sql": { "name": "default__snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", "unique_id": "macro.dbt.default__snapshot_merge_sql", "macro_sql": "{% macro default__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n {%- set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() -%}\n\n merge into {{ target.render() }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on DBT_INTERNAL_SOURCE.{{ columns.dbt_scd_id }} = DBT_INTERNAL_DEST.{{ columns.dbt_scd_id }}\n\n when matched\n {% if config.get(\"dbt_valid_to_current\") %}\n\t{% set source_unique_key = (\"DBT_INTERNAL_DEST.\" ~ columns.dbt_valid_to) | trim %}\n\t{% set target_unique_key = config.get('dbt_valid_to_current') | trim %}\n\tand ({{ equals(source_unique_key, target_unique_key) }} or {{ source_unique_key }} is null)\n\n {% else %}\n and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null\n {% endif %}\n and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')\n then update\n set {{ columns.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ columns.dbt_valid_to }}\n\n when not matched\n and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'\n then insert ({{ insert_cols_csv }})\n values ({{ insert_cols_csv }})\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.get_snapshot_table_column_names", "macro.dbt.equals" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4952261, "supported_languages": null }, "macro.dbt.strategy_dispatch": { "name": "strategy_dispatch", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.strategy_dispatch", "macro_sql": "{% macro strategy_dispatch(name) -%}\n{% set original_name = name %}\n {% if '.' in name %}\n {% set package_name, name = name.split(\".\", 1) %}\n {% else %}\n {% set package_name = none %}\n {% endif %}\n\n {% if package_name is none %}\n {% set package_context = context %}\n {% elif package_name in context %}\n {% set package_context = context[package_name] %}\n {% else %}\n {% set error_msg %}\n Could not find package '{{package_name}}', called with '{{original_name}}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n\n {%- set search_name = 'snapshot_' ~ name ~ '_strategy' -%}\n\n {% if search_name not in package_context %}\n {% set error_msg %}\n The specified strategy macro '{{name}}' was not found in package '{{ package_name }}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n {{ return(package_context[search_name]) }}\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4952419, "supported_languages": null }, "macro.dbt.snapshot_hash_arguments": { "name": "snapshot_hash_arguments", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_hash_arguments", "macro_sql": "{% macro snapshot_hash_arguments(args) -%}\n {{ adapter.dispatch('snapshot_hash_arguments', 'dbt')(args) }}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__snapshot_hash_arguments" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495244, "supported_languages": null }, "macro.dbt.default__snapshot_hash_arguments": { "name": "default__snapshot_hash_arguments", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.default__snapshot_hash_arguments", "macro_sql": "{% macro default__snapshot_hash_arguments(args) -%}\n md5({%- for arg in args -%}\n coalesce(cast({{ arg }} as varchar ), '')\n {% if not loop.last %} || '|' || {% endif %}\n {%- endfor -%})\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4952478, "supported_languages": null }, "macro.dbt.snapshot_timestamp_strategy": { "name": "snapshot_timestamp_strategy", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_timestamp_strategy", "macro_sql": "{% macro snapshot_timestamp_strategy(node, snapshotted_rel, current_rel, model_config, target_exists) %}\n {# The model_config parameter is no longer used, but is passed in anyway for compatibility. #}\n {% set primary_key = config.get('unique_key') %}\n {% set updated_at = config.get('updated_at') %}\n {% set hard_deletes = adapter.get_hard_deletes_behavior(config) %}\n {% set invalidate_hard_deletes = hard_deletes == 'invalidate' %}\n {% set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() %}\n\n {#/*\n The snapshot relation might not have an {{ updated_at }} value if the\n snapshot strategy is changed from `check` to `timestamp`. We\n should use a dbt-created column for the comparison in the snapshot\n table instead of assuming that the user-supplied {{ updated_at }}\n will be present in the historical data.\n\n See https://github.com/dbt-labs/dbt-core/issues/2350\n */ #}\n {% set row_changed_expr -%}\n ({{ snapshotted_rel }}.{{ columns.dbt_valid_from }} < {{ current_rel }}.{{ updated_at }})\n {%- endset %}\n\n {% set scd_args = api.Relation.scd_args(primary_key, updated_at) %}\n {% set scd_id_expr = snapshot_hash_arguments(scd_args) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes,\n \"hard_deletes\": hard_deletes\n }) %}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.get_snapshot_table_column_names", "macro.dbt.snapshot_hash_arguments" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495252, "supported_languages": null }, "macro.dbt.snapshot_string_as_time": { "name": "snapshot_string_as_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_string_as_time", "macro_sql": "{% macro snapshot_string_as_time(timestamp) -%}\n {{ adapter.dispatch('snapshot_string_as_time', 'dbt')(timestamp) }}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__snapshot_string_as_time" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495255, "supported_languages": null }, "macro.dbt.default__snapshot_string_as_time": { "name": "default__snapshot_string_as_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.default__snapshot_string_as_time", "macro_sql": "{% macro default__snapshot_string_as_time(timestamp) %}\n {% do exceptions.raise_not_implemented(\n 'snapshot_string_as_time macro not implemented for adapter '+adapter.type()\n ) %}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4952588, "supported_languages": null }, "macro.dbt.snapshot_check_all_get_existing_columns": { "name": "snapshot_check_all_get_existing_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_check_all_get_existing_columns", "macro_sql": "{% macro snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) -%}\n {%- if not target_exists -%}\n {#-- no table yet -> return whatever the query does --#}\n {{ return((false, query_columns)) }}\n {%- endif -%}\n\n {#-- handle any schema changes --#}\n {%- set target_relation = adapter.get_relation(database=node.database, schema=node.schema, identifier=node.alias) -%}\n\n {% if check_cols_config == 'all' %}\n {%- set query_columns = get_columns_in_query(node['compiled_code']) -%}\n\n {% elif check_cols_config is iterable and (check_cols_config | length) > 0 %}\n {#-- query for proper casing/quoting, to support comparison below --#}\n {%- set select_check_cols_from_target -%}\n {#-- N.B. The whitespace below is necessary to avoid edge case issue with comments --#}\n {#-- See: https://github.com/dbt-labs/dbt-core/issues/6781 --#}\n select {{ check_cols_config | join(', ') }} from (\n {{ node['compiled_code'] }}\n ) subq\n {%- endset -%}\n {% set query_columns = get_columns_in_query(select_check_cols_from_target) %}\n\n {% else %}\n {% do exceptions.raise_compiler_error(\"Invalid value for 'check_cols': \" ~ check_cols_config) %}\n {% endif %}\n\n {%- set existing_cols = adapter.get_columns_in_relation(target_relation) | map(attribute = 'name') | list -%}\n {%- set ns = namespace() -%} {#-- handle for-loop scoping with a namespace --#}\n {%- set ns.column_added = false -%}\n\n {%- set intersection = [] -%}\n {%- for col in query_columns -%}\n {%- if col in existing_cols -%}\n {%- do intersection.append(adapter.quote(col)) -%}\n {%- else -%}\n {% set ns.column_added = true %}\n {%- endif -%}\n {%- endfor -%}\n {{ return((ns.column_added, intersection)) }}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.get_columns_in_query" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495264, "supported_languages": null }, "macro.dbt.snapshot_check_strategy": { "name": "snapshot_check_strategy", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_check_strategy", "macro_sql": "{% macro snapshot_check_strategy(node, snapshotted_rel, current_rel, model_config, target_exists) %}\n {# The model_config parameter is no longer used, but is passed in anyway for compatibility. #}\n {% set check_cols_config = config.get('check_cols') %}\n {% set primary_key = config.get('unique_key') %}\n {% set hard_deletes = adapter.get_hard_deletes_behavior(config) %}\n {% set invalidate_hard_deletes = hard_deletes == 'invalidate' %}\n {% set updated_at = config.get('updated_at') or snapshot_get_time() %}\n\n {% set column_added = false %}\n\n {% set column_added, check_cols = snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) %}\n\n {%- set row_changed_expr -%}\n (\n {%- if column_added -%}\n {{ get_true_sql() }}\n {%- else -%}\n {%- for col in check_cols -%}\n {{ snapshotted_rel }}.{{ col }} != {{ current_rel }}.{{ col }}\n or\n (\n (({{ snapshotted_rel }}.{{ col }} is null) and not ({{ current_rel }}.{{ col }} is null))\n or\n ((not {{ snapshotted_rel }}.{{ col }} is null) and ({{ current_rel }}.{{ col }} is null))\n )\n {%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n {%- endif -%}\n )\n {%- endset %}\n\n {% set scd_args = api.Relation.scd_args(primary_key, updated_at) %}\n {% set scd_id_expr = snapshot_hash_arguments(scd_args) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes,\n \"hard_deletes\": hard_deletes\n }) %}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.snapshot_get_time", "macro.dbt.snapshot_check_all_get_existing_columns", "macro.dbt.get_true_sql", "macro.dbt.snapshot_hash_arguments" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4952672, "supported_languages": null }, "macro.dbt.create_columns": { "name": "create_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.create_columns", "macro_sql": "{% macro create_columns(relation, columns) %}\n {{ adapter.dispatch('create_columns', 'dbt')(relation, columns) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__create_columns" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495279, "supported_languages": null }, "macro.dbt.default__create_columns": { "name": "default__create_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__create_columns", "macro_sql": "{% macro default__create_columns(relation, columns) %}\n {% for column in columns %}\n {% call statement() %}\n alter table {{ relation.render() }} add column {{ adapter.quote(column.name) }} {{ column.data_type }};\n {% endcall %}\n {% endfor %}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.statement" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495282, "supported_languages": null }, "macro.dbt.post_snapshot": { "name": "post_snapshot", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.post_snapshot", "macro_sql": "{% macro post_snapshot(staging_relation) %}\n {{ adapter.dispatch('post_snapshot', 'dbt')(staging_relation) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__post_snapshot" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495286, "supported_languages": null }, "macro.dbt.default__post_snapshot": { "name": "default__post_snapshot", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__post_snapshot", "macro_sql": "{% macro default__post_snapshot(staging_relation) %}\n {# no-op #}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4952881, "supported_languages": null }, "macro.dbt.get_true_sql": { "name": "get_true_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.get_true_sql", "macro_sql": "{% macro get_true_sql() %}\n {{ adapter.dispatch('get_true_sql', 'dbt')() }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__get_true_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495291, "supported_languages": null }, "macro.dbt.default__get_true_sql": { "name": "default__get_true_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__get_true_sql", "macro_sql": "{% macro default__get_true_sql() %}\n {{ return('TRUE') }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495295, "supported_languages": null }, "macro.dbt.snapshot_staging_table": { "name": "snapshot_staging_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.snapshot_staging_table", "macro_sql": "{% macro snapshot_staging_table(strategy, source_sql, target_relation) -%}\n {{ adapter.dispatch('snapshot_staging_table', 'dbt')(strategy, source_sql, target_relation) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__snapshot_staging_table" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495297, "supported_languages": null }, "macro.dbt.get_snapshot_table_column_names": { "name": "get_snapshot_table_column_names", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.get_snapshot_table_column_names", "macro_sql": "{% macro get_snapshot_table_column_names() %}\n {{ return({'dbt_valid_to': 'dbt_valid_to', 'dbt_valid_from': 'dbt_valid_from', 'dbt_scd_id': 'dbt_scd_id', 'dbt_updated_at': 'dbt_updated_at', 'dbt_is_deleted': 'dbt_is_deleted'}) }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495301, "supported_languages": null }, "macro.dbt.default__snapshot_staging_table": { "name": "default__snapshot_staging_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__snapshot_staging_table", "macro_sql": "{% macro default__snapshot_staging_table(strategy, source_sql, target_relation) -%}\n {% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %}\n {% if strategy.hard_deletes == 'new_record' %}\n {% set new_scd_id = snapshot_hash_arguments([columns.dbt_scd_id, snapshot_get_time()]) %}\n {% endif %}\n with snapshot_query as (\n\n {{ source_sql }}\n\n ),\n\n snapshotted_data as (\n\n select *, {{ unique_key_fields(strategy.unique_key) }}\n from {{ target_relation }}\n where\n {% if config.get('dbt_valid_to_current') %}\n\t\t{% set source_unique_key = columns.dbt_valid_to | trim %}\n\t\t{% set target_unique_key = config.get('dbt_valid_to_current') | trim %}\n\n\t\t{# The exact equals semantics between NULL values depends on the current behavior flag set. Also, update records if the source field is null #}\n ( {{ equals(source_unique_key, target_unique_key) }} or {{ source_unique_key }} is null )\n {% else %}\n {{ columns.dbt_valid_to }} is null\n {% endif %}\n\n ),\n\n insertions_source_data as (\n\n select *, {{ unique_key_fields(strategy.unique_key) }},\n {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},\n {{ get_dbt_valid_to_current(strategy, columns) }},\n {{ strategy.scd_id }} as {{ columns.dbt_scd_id }}\n\n from snapshot_query\n ),\n\n updates_source_data as (\n\n select *, {{ unique_key_fields(strategy.unique_key) }},\n {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_to }}\n\n from snapshot_query\n ),\n\n {%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' %}\n\n deletes_source_data as (\n\n select *, {{ unique_key_fields(strategy.unique_key) }}\n from snapshot_query\n ),\n {% endif %}\n\n insertions as (\n\n select\n 'insert' as dbt_change_type,\n source_data.*\n {%- if strategy.hard_deletes == 'new_record' -%}\n ,'False' as {{ columns.dbt_is_deleted }}\n {%- endif %}\n\n from insertions_source_data as source_data\n left outer join snapshotted_data\n on {{ unique_key_join_on(strategy.unique_key, \"snapshotted_data\", \"source_data\") }}\n where {{ unique_key_is_null(strategy.unique_key, \"snapshotted_data\") }}\n or ({{ unique_key_is_not_null(strategy.unique_key, \"snapshotted_data\") }} and (\n {{ strategy.row_changed }} {%- if strategy.hard_deletes == 'new_record' -%} or snapshotted_data.{{ columns.dbt_is_deleted }} = 'True' {% endif %}\n )\n\n )\n\n ),\n\n updates as (\n\n select\n 'update' as dbt_change_type,\n source_data.*,\n snapshotted_data.{{ columns.dbt_scd_id }}\n {%- if strategy.hard_deletes == 'new_record' -%}\n , snapshotted_data.{{ columns.dbt_is_deleted }}\n {%- endif %}\n\n from updates_source_data as source_data\n join snapshotted_data\n on {{ unique_key_join_on(strategy.unique_key, \"snapshotted_data\", \"source_data\") }}\n where (\n {{ strategy.row_changed }} {%- if strategy.hard_deletes == 'new_record' -%} or snapshotted_data.{{ columns.dbt_is_deleted }} = 'True' {% endif %}\n )\n )\n\n {%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' %}\n ,\n deletes as (\n\n select\n 'delete' as dbt_change_type,\n source_data.*,\n {{ snapshot_get_time() }} as {{ columns.dbt_valid_from }},\n {{ snapshot_get_time() }} as {{ columns.dbt_updated_at }},\n {{ snapshot_get_time() }} as {{ columns.dbt_valid_to }},\n snapshotted_data.{{ columns.dbt_scd_id }}\n {%- if strategy.hard_deletes == 'new_record' -%}\n , snapshotted_data.{{ columns.dbt_is_deleted }}\n {%- endif %}\n from snapshotted_data\n left join deletes_source_data as source_data\n on {{ unique_key_join_on(strategy.unique_key, \"snapshotted_data\", \"source_data\") }}\n where {{ unique_key_is_null(strategy.unique_key, \"source_data\") }}\n\n {%- if strategy.hard_deletes == 'new_record' %}\n and not (\n --avoid updating the record's valid_to if the latest entry is marked as deleted\n snapshotted_data.{{ columns.dbt_is_deleted }} = 'True'\n and\n {% if config.get('dbt_valid_to_current') -%}\n snapshotted_data.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }}\n {%- else -%}\n snapshotted_data.{{ columns.dbt_valid_to }} is null\n {%- endif %}\n )\n {%- endif %}\n )\n {%- endif %}\n\n {%- if strategy.hard_deletes == 'new_record' %}\n {% set snapshotted_cols = get_list_of_column_names(get_columns_in_relation(target_relation)) %}\n {% set source_sql_cols = get_column_schema_from_query(source_sql) %}\n ,\n deletion_records as (\n\n select\n 'insert' as dbt_change_type,\n {#/*\n If a column has been added to the source it won't yet exist in the\n snapshotted table so we insert a null value as a placeholder for the column.\n */#}\n {%- for col in source_sql_cols -%}\n {%- if col.name in snapshotted_cols -%}\n snapshotted_data.{{ adapter.quote(col.column) }},\n {%- else -%}\n NULL as {{ adapter.quote(col.column) }},\n {%- endif -%}\n {% endfor -%}\n {%- if strategy.unique_key | is_list -%}\n {%- for key in strategy.unique_key -%}\n snapshotted_data.{{ key }} as dbt_unique_key_{{ loop.index }},\n {% endfor -%}\n {%- else -%}\n snapshotted_data.dbt_unique_key as dbt_unique_key,\n {% endif -%}\n {{ snapshot_get_time() }} as {{ columns.dbt_valid_from }},\n {{ snapshot_get_time() }} as {{ columns.dbt_updated_at }},\n snapshotted_data.{{ columns.dbt_valid_to }} as {{ columns.dbt_valid_to }},\n {{ new_scd_id }} as {{ columns.dbt_scd_id }},\n 'True' as {{ columns.dbt_is_deleted }}\n from snapshotted_data\n left join deletes_source_data as source_data\n on {{ unique_key_join_on(strategy.unique_key, \"snapshotted_data\", \"source_data\") }}\n where {{ unique_key_is_null(strategy.unique_key, \"source_data\") }}\n and not (\n --avoid inserting a new record if the latest one is marked as deleted\n snapshotted_data.{{ columns.dbt_is_deleted }} = 'True'\n and\n {% if config.get('dbt_valid_to_current') -%}\n snapshotted_data.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }}\n {%- else -%}\n snapshotted_data.{{ columns.dbt_valid_to }} is null\n {%- endif %}\n )\n\n )\n {%- endif %}\n\n select * from insertions\n union all\n select * from updates\n {%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' %}\n union all\n select * from deletes\n {%- endif %}\n {%- if strategy.hard_deletes == 'new_record' %}\n union all\n select * from deletion_records\n {%- endif %}\n\n\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.get_snapshot_table_column_names", "macro.dbt.snapshot_hash_arguments", "macro.dbt.snapshot_get_time", "macro.dbt.unique_key_fields", "macro.dbt.equals", "macro.dbt.get_dbt_valid_to_current", "macro.dbt.unique_key_join_on", "macro.dbt.unique_key_is_null", "macro.dbt.unique_key_is_not_null", "macro.dbt.get_list_of_column_names", "macro.dbt.get_columns_in_relation", "macro.dbt.get_column_schema_from_query" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495305, "supported_languages": null }, "macro.dbt.build_snapshot_table": { "name": "build_snapshot_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.build_snapshot_table", "macro_sql": "{% macro build_snapshot_table(strategy, sql) -%}\n {{ adapter.dispatch('build_snapshot_table', 'dbt')(strategy, sql) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__build_snapshot_table" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495308, "supported_languages": null }, "macro.dbt.default__build_snapshot_table": { "name": "default__build_snapshot_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__build_snapshot_table", "macro_sql": "{% macro default__build_snapshot_table(strategy, sql) %}\n {% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %}\n\n select *,\n {{ strategy.scd_id }} as {{ columns.dbt_scd_id }},\n {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},\n {{ get_dbt_valid_to_current(strategy, columns) }}\n {%- if strategy.hard_deletes == 'new_record' -%}\n , 'False' as {{ columns.dbt_is_deleted }}\n {% endif -%}\n from (\n {{ sql }}\n ) sbq\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.get_snapshot_table_column_names", "macro.dbt.get_dbt_valid_to_current" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495311, "supported_languages": null }, "macro.dbt.build_snapshot_staging_table": { "name": "build_snapshot_staging_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.build_snapshot_staging_table", "macro_sql": "{% macro build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% set temp_relation = make_temp_relation(target_relation) %}\n\n {% set select = snapshot_staging_table(strategy, sql, target_relation) %}\n\n {% call statement('build_snapshot_staging_relation') %}\n {{ create_table_as(True, temp_relation, select) }}\n {% endcall %}\n\n {% do return(temp_relation) %}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.make_temp_relation", "macro.dbt.snapshot_staging_table", "macro.dbt.statement", "macro.dbt.create_table_as" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4953148, "supported_languages": null }, "macro.dbt.get_updated_at_column_data_type": { "name": "get_updated_at_column_data_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.get_updated_at_column_data_type", "macro_sql": "{% macro get_updated_at_column_data_type(snapshot_sql) %}\n {% set snapshot_sql_column_schema = get_column_schema_from_query(snapshot_sql) %}\n {% set dbt_updated_at_data_type = null %}\n {% set ns = namespace() -%} {#-- handle for-loop scoping with a namespace --#}\n {% set ns.dbt_updated_at_data_type = null -%}\n {% for column in snapshot_sql_column_schema %}\n {% if ((column.column == 'dbt_updated_at') or (column.column == 'DBT_UPDATED_AT')) %}\n {% set ns.dbt_updated_at_data_type = column.dtype %}\n {% endif %}\n {% endfor %}\n {{ return(ns.dbt_updated_at_data_type or none) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.get_column_schema_from_query" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495318, "supported_languages": null }, "macro.dbt.check_time_data_types": { "name": "check_time_data_types", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.check_time_data_types", "macro_sql": "{% macro check_time_data_types(sql) %}\n {% set dbt_updated_at_data_type = get_updated_at_column_data_type(sql) %}\n {% set snapshot_get_time_data_type = get_snapshot_get_time_data_type() %}\n {% if snapshot_get_time_data_type is not none and dbt_updated_at_data_type is not none and snapshot_get_time_data_type != dbt_updated_at_data_type %}\n {% if exceptions.warn_snapshot_timestamp_data_types %}\n {{ exceptions.warn_snapshot_timestamp_data_types(snapshot_get_time_data_type, dbt_updated_at_data_type) }}\n {% endif %}\n {% endif %}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.get_updated_at_column_data_type", "macro.dbt.get_snapshot_get_time_data_type" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.49532, "supported_languages": null }, "macro.dbt.get_dbt_valid_to_current": { "name": "get_dbt_valid_to_current", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.get_dbt_valid_to_current", "macro_sql": "{% macro get_dbt_valid_to_current(strategy, columns) %}\n {% set dbt_valid_to_current = config.get('dbt_valid_to_current') or \"null\" %}\n coalesce(nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}), {{dbt_valid_to_current}})\n as {{ columns.dbt_valid_to }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495323, "supported_languages": null }, "macro.dbt.unique_key_fields": { "name": "unique_key_fields", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.unique_key_fields", "macro_sql": "{% macro unique_key_fields(unique_key) %}\n {% if unique_key | is_list %}\n {% for key in unique_key %}\n {{ key }} as dbt_unique_key_{{ loop.index }}\n {%- if not loop.last %} , {%- endif %}\n {% endfor %}\n {% else %}\n {{ unique_key }} as dbt_unique_key\n {% endif %}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495326, "supported_languages": null }, "macro.dbt.unique_key_join_on": { "name": "unique_key_join_on", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.unique_key_join_on", "macro_sql": "{% macro unique_key_join_on(unique_key, identifier, from_identifier) %}\n {% if unique_key | is_list %}\n {% for key in unique_key %}\n\t {% set source_unique_key = (identifier ~ \".dbt_unique_key_\" ~ loop.index) | trim %}\n\t {% set target_unique_key = (from_identifier ~ \".dbt_unique_key_\" ~ loop.index) | trim %}\n\t {{ equals(source_unique_key, target_unique_key) }}\n {%- if not loop.last %} and {%- endif %}\n {% endfor %}\n {% else %}\n {{ identifier }}.dbt_unique_key = {{ from_identifier }}.dbt_unique_key\n {% endif %}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.equals" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4953291, "supported_languages": null }, "macro.dbt.unique_key_is_null": { "name": "unique_key_is_null", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.unique_key_is_null", "macro_sql": "{% macro unique_key_is_null(unique_key, identifier) %}\n {% if unique_key | is_list %}\n {{ identifier }}.dbt_unique_key_1 is null\n {% else %}\n {{ identifier }}.dbt_unique_key is null\n {% endif %}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495331, "supported_languages": null }, "macro.dbt.unique_key_is_not_null": { "name": "unique_key_is_not_null", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.unique_key_is_not_null", "macro_sql": "{% macro unique_key_is_not_null(unique_key, identifier) %}\n {% if unique_key | is_list %}\n {{ identifier }}.dbt_unique_key_1 is not null\n {% else %}\n {{ identifier }}.dbt_unique_key is not null\n {% endif %}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4953349, "supported_languages": null }, "macro.dbt.materialization_snapshot_default": { "name": "materialization_snapshot_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/snapshot.sql", "original_file_path": "macros/materializations/snapshots/snapshot.sql", "unique_id": "macro.dbt.materialization_snapshot_default", "macro_sql": "{% materialization snapshot, default %}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n -- grab current tables grants config for comparision later on\n {%- set grant_config = config.get('grants') -%}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=model.database,\n schema=model.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {# The model['config'] parameter below is no longer used, but passing anyway for compatibility #}\n {# It was a dictionary of config, instead of the config object from the context #}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", model['config'], target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_code']) %}\n {% set build_or_select_sql = build_sql %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% else %}\n\n {% set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() %}\n\n {{ adapter.assert_valid_snapshot_target_given_strategy(target_relation, columns, strategy) }}\n\n {% set build_or_select_sql = snapshot_staging_table(strategy, sql, target_relation) %}\n {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set remove_columns = ['dbt_change_type', 'DBT_CHANGE_TYPE', 'dbt_unique_key', 'DBT_UNIQUE_KEY'] %}\n {% if unique_key | is_list %}\n {% for key in strategy.unique_key %}\n {{ remove_columns.append('dbt_unique_key_' + loop.index|string) }}\n {{ remove_columns.append('DBT_UNIQUE_KEY_' + loop.index|string) }}\n {% endfor %}\n {% endif %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'in', remove_columns)\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'in', remove_columns)\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% endif %}\n\n\n {{ check_time_data_types(build_or_select_sql) }}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode=False) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if not target_relation_exists %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if staging_table is defined %}\n {% do post_snapshot(staging_table) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": { "macros": [ "macro.dbt.get_or_create_relation", "macro.dbt.run_hooks", "macro.dbt.strategy_dispatch", "macro.dbt.build_snapshot_table", "macro.dbt.create_table_as", "macro.dbt.get_snapshot_table_column_names", "macro.dbt.snapshot_staging_table", "macro.dbt.build_snapshot_staging_table", "macro.dbt.create_columns", "macro.dbt.snapshot_merge_sql", "macro.dbt.check_time_data_types", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes", "macro.dbt.post_snapshot" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4953442, "supported_languages": [ "sql" ] }, "macro.dbt.materialization_test_default": { "name": "materialization_test_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/test.sql", "original_file_path": "macros/materializations/tests/test.sql", "unique_id": "macro.dbt.materialization_test_default", "macro_sql": "{%- materialization test, default -%}\n\n {% set relations = [] %}\n {% set limit = config.get('limit') %}\n\n {% set sql_with_limit %}\n {{ get_limit_subquery_sql(sql, limit) }}\n {% endset %}\n\n {% if should_store_failures() %}\n\n {% set identifier = model['alias'] %}\n {% set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n\n {% set store_failures_as = config.get('store_failures_as') %}\n -- if `--store-failures` is invoked via command line and `store_failures_as` is not set,\n -- config.get('store_failures_as', 'table') returns None, not 'table'\n {% if store_failures_as == none %}{% set store_failures_as = 'table' %}{% endif %}\n {% if store_failures_as not in ['table', 'view'] %}\n {{ exceptions.raise_compiler_error(\n \"'\" ~ store_failures_as ~ \"' is not a valid value for `store_failures_as`. \"\n \"Accepted values are: ['ephemeral', 'table', 'view']\"\n ) }}\n {% endif %}\n\n {% set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database, type=store_failures_as) -%} %}\n\n {% if old_relation %}\n {% do adapter.drop_relation(old_relation) %}\n {% endif %}\n\n {% call statement(auto_begin=True) %}\n {{ get_create_sql(target_relation, sql_with_limit) }}\n {% endcall %}\n\n {% do relations.append(target_relation) %}\n\n {# Since the test failures have already been saved to the database, reuse that result rather than querying again #}\n {% set main_sql %}\n select *\n from {{ target_relation }}\n {% endset %}\n\n {{ adapter.commit() }}\n\n {% else %}\n\n {% set main_sql = sql_with_limit %}\n\n {% endif %}\n\n {% set fail_calc = config.get('fail_calc') %}\n {% set warn_if = config.get('warn_if') %}\n {% set error_if = config.get('error_if') %}\n\n {% call statement('main', fetch_result=True) -%}\n\n {# The limit has already been included above, and we do not want to duplicate it again. We also want to be safe for macro overrides treating `limit` as a required parameter. #}\n {{ get_test_sql(main_sql, fail_calc, warn_if, error_if, limit=none)}}\n\n {%- endcall %}\n\n {{ return({'relations': relations}) }}\n\n{%- endmaterialization -%}", "depends_on": { "macros": [ "macro.dbt.get_limit_subquery_sql", "macro.dbt.should_store_failures", "macro.dbt.statement", "macro.dbt.get_create_sql", "macro.dbt.get_test_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495357, "supported_languages": [ "sql" ] }, "macro.dbt.get_test_sql": { "name": "get_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.get_test_sql", "macro_sql": "{% macro get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n {{ adapter.dispatch('get_test_sql', 'dbt')(main_sql, fail_calc, warn_if, error_if, limit) }}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__get_test_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4953668, "supported_languages": null }, "macro.dbt.default__get_test_sql": { "name": "default__get_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.default__get_test_sql", "macro_sql": "{% macro default__get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n select\n {{ fail_calc }} as failures,\n {{ fail_calc }} {{ warn_if }} as should_warn,\n {{ fail_calc }} {{ error_if }} as should_error\n from (\n {{ main_sql }}\n {{ \"limit \" ~ limit if limit != none }}\n ) dbt_internal_test\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495369, "supported_languages": null }, "macro.dbt.get_unit_test_sql": { "name": "get_unit_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.get_unit_test_sql", "macro_sql": "{% macro get_unit_test_sql(main_sql, expected_fixture_sql, expected_column_names) -%}\n {{ adapter.dispatch('get_unit_test_sql', 'dbt')(main_sql, expected_fixture_sql, expected_column_names) }}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__get_unit_test_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495373, "supported_languages": null }, "macro.dbt.default__get_unit_test_sql": { "name": "default__get_unit_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.default__get_unit_test_sql", "macro_sql": "{% macro default__get_unit_test_sql(main_sql, expected_fixture_sql, expected_column_names) -%}\n-- Build actual result given inputs\nwith dbt_internal_unit_test_actual as (\n select\n {% for expected_column_name in expected_column_names %}{{expected_column_name}}{% if not loop.last -%},{% endif %}{%- endfor -%}, {{ dbt.string_literal(\"actual\") }} as {{ adapter.quote(\"actual_or_expected\") }}\n from (\n {{ main_sql }}\n ) _dbt_internal_unit_test_actual\n),\n-- Build expected result\ndbt_internal_unit_test_expected as (\n select\n {% for expected_column_name in expected_column_names %}{{expected_column_name}}{% if not loop.last -%}, {% endif %}{%- endfor -%}, {{ dbt.string_literal(\"expected\") }} as {{ adapter.quote(\"actual_or_expected\") }}\n from (\n {{ expected_fixture_sql }}\n ) _dbt_internal_unit_test_expected\n)\n-- Union actual and expected results\nselect * from dbt_internal_unit_test_actual\nunion all\nselect * from dbt_internal_unit_test_expected\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.string_literal" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.49538, "supported_languages": null }, "macro.dbt.get_where_subquery": { "name": "get_where_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/where_subquery.sql", "original_file_path": "macros/materializations/tests/where_subquery.sql", "unique_id": "macro.dbt.get_where_subquery", "macro_sql": "{% macro get_where_subquery(relation) -%}\n {% do return(adapter.dispatch('get_where_subquery', 'dbt')(relation)) %}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__get_where_subquery" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4953852, "supported_languages": null }, "macro.dbt.default__get_where_subquery": { "name": "default__get_where_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/where_subquery.sql", "original_file_path": "macros/materializations/tests/where_subquery.sql", "unique_id": "macro.dbt.default__get_where_subquery", "macro_sql": "{% macro default__get_where_subquery(relation) -%}\n {% set where = config.get('where', '') %}\n {% if where %}\n {%- set filtered -%}\n (select * from {{ relation }} where {{ where }}) dbt_subquery\n {%- endset -%}\n {% do return(filtered) %}\n {%- else -%}\n {% do return(relation) %}\n {%- endif -%}\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4953878, "supported_languages": null }, "macro.dbt.materialization_unit_default": { "name": "materialization_unit_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/unit.sql", "original_file_path": "macros/materializations/tests/unit.sql", "unique_id": "macro.dbt.materialization_unit_default", "macro_sql": "{%- materialization unit, default -%}\n\n {% set relations = [] %}\n\n {% set expected_rows = config.get('expected_rows') %}\n {% set expected_sql = config.get('expected_sql') %}\n {% set tested_expected_column_names = expected_rows[0].keys() if (expected_rows | length ) > 0 else get_columns_in_query(sql) %}\n\n {%- set target_relation = this.incorporate(type='table') -%}\n {%- set temp_relation = make_temp_relation(target_relation)-%}\n {% do run_query(get_create_table_as_sql(True, temp_relation, get_empty_subquery_sql(sql))) %}\n {%- set columns_in_relation = adapter.get_columns_in_relation(temp_relation) -%}\n {%- set column_name_to_data_types = {} -%}\n {%- set column_name_to_quoted = {} -%}\n {%- for column in columns_in_relation -%}\n {%- do column_name_to_data_types.update({column.name|lower: column.data_type}) -%}\n {%- do column_name_to_quoted.update({column.name|lower: column.quoted}) -%}\n {%- endfor -%}\n\n {%- set expected_column_names_quoted = [] -%}\n {%- for column_name in tested_expected_column_names -%}\n {%- do expected_column_names_quoted.append(column_name_to_quoted[column_name|lower]) -%}\n {%- endfor -%}\n\n {% if not expected_sql %}\n {% set expected_sql = get_expected_sql(expected_rows, column_name_to_data_types, column_name_to_quoted) %}\n {% endif %}\n {% set unit_test_sql = get_unit_test_sql(sql, expected_sql, expected_column_names_quoted) %}\n\n {% call statement('main', fetch_result=True) -%}\n\n {{ unit_test_sql }}\n\n {%- endcall %}\n\n {% do adapter.drop_relation(temp_relation) %}\n\n {{ return({'relations': relations}) }}\n\n{%- endmaterialization -%}", "depends_on": { "macros": [ "macro.dbt.get_columns_in_query", "macro.dbt.make_temp_relation", "macro.dbt.run_query", "macro.dbt.get_create_table_as_sql", "macro.dbt.get_empty_subquery_sql", "macro.dbt.get_expected_sql", "macro.dbt.get_unit_test_sql", "macro.dbt.statement" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495394, "supported_languages": [ "sql" ] }, "macro.dbt.materialization_materialized_view_default": { "name": "materialization_materialized_view_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialization_materialized_view_default", "macro_sql": "{% materialization materialized_view, default %}\n {% set existing_relation = load_cached_relation(this) %}\n {% set target_relation = this.incorporate(type=this.MaterializedView) %}\n {% set intermediate_relation = make_intermediate_relation(target_relation) %}\n {% set backup_relation_type = target_relation.MaterializedView if existing_relation is none else existing_relation.type %}\n {% set backup_relation = make_backup_relation(target_relation, backup_relation_type) %}\n\n {{ materialized_view_setup(backup_relation, intermediate_relation, pre_hooks) }}\n\n {% set build_sql = materialized_view_get_build_sql(existing_relation, target_relation, backup_relation, intermediate_relation) %}\n\n {% if build_sql == '' %}\n {{ materialized_view_execute_no_op(target_relation) }}\n {% else %}\n {{ materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) }}\n {% endif %}\n\n {{ materialized_view_teardown(backup_relation, intermediate_relation, post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": { "macros": [ "macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.materialized_view_setup", "macro.dbt.materialized_view_get_build_sql", "macro.dbt.materialized_view_execute_no_op", "macro.dbt.materialized_view_execute_build_sql", "macro.dbt.materialized_view_teardown" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495407, "supported_languages": [ "sql" ] }, "macro.dbt.materialized_view_setup": { "name": "materialized_view_setup", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_setup", "macro_sql": "{% macro materialized_view_setup(backup_relation, intermediate_relation, pre_hooks) %}\n\n -- backup_relation and intermediate_relation should not already exist in the database\n -- it's possible these exist because of a previous run that exited unexpectedly\n {% set preexisting_backup_relation = load_cached_relation(backup_relation) %}\n {% set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) %}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.load_cached_relation", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4954119, "supported_languages": null }, "macro.dbt.materialized_view_teardown": { "name": "materialized_view_teardown", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_teardown", "macro_sql": "{% macro materialized_view_teardown(backup_relation, intermediate_relation, post_hooks) %}\n\n -- drop the temp relations if they exist to leave the database clean for the next run\n {{ drop_relation_if_exists(backup_relation) }}\n {{ drop_relation_if_exists(intermediate_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495414, "supported_languages": null }, "macro.dbt.materialized_view_get_build_sql": { "name": "materialized_view_get_build_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_get_build_sql", "macro_sql": "{% macro materialized_view_get_build_sql(existing_relation, target_relation, backup_relation, intermediate_relation) %}\n\n {% set full_refresh_mode = should_full_refresh() %}\n\n -- determine the scenario we're in: create, full_refresh, alter, refresh data\n {% if existing_relation is none %}\n {% set build_sql = get_create_materialized_view_as_sql(target_relation, sql) %}\n {% elif full_refresh_mode or not existing_relation.is_materialized_view %}\n {% set build_sql = get_replace_sql(existing_relation, target_relation, sql) %}\n {% else %}\n\n -- get config options\n {% set on_configuration_change = config.get('on_configuration_change') %}\n {% set configuration_changes = get_materialized_view_configuration_changes(existing_relation, config) %}\n\n {% if configuration_changes is none %}\n {% set build_sql = refresh_materialized_view(target_relation) %}\n\n {% elif on_configuration_change == 'apply' %}\n {% set build_sql = get_alter_materialized_view_as_sql(target_relation, configuration_changes, sql, existing_relation, backup_relation, intermediate_relation) %}\n {% elif on_configuration_change == 'continue' %}\n {% set build_sql = '' %}\n {{ exceptions.warn(\"Configuration changes were identified and `on_configuration_change` was set to `continue` for `\" ~ target_relation.render() ~ \"`\") }}\n {% elif on_configuration_change == 'fail' %}\n {{ exceptions.raise_fail_fast_error(\"Configuration changes were identified and `on_configuration_change` was set to `fail` for `\" ~ target_relation.render() ~ \"`\") }}\n\n {% else %}\n -- this only happens if the user provides a value other than `apply`, 'skip', 'fail'\n {{ exceptions.raise_compiler_error(\"Unexpected configuration scenario\") }}\n\n {% endif %}\n\n {% endif %}\n\n {% do return(build_sql) %}\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.should_full_refresh", "macro.dbt.get_create_materialized_view_as_sql", "macro.dbt.get_replace_sql", "macro.dbt.get_materialized_view_configuration_changes", "macro.dbt.refresh_materialized_view", "macro.dbt.get_alter_materialized_view_as_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495417, "supported_languages": null }, "macro.dbt.materialized_view_execute_no_op": { "name": "materialized_view_execute_no_op", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_execute_no_op", "macro_sql": "{% macro materialized_view_execute_no_op(target_relation) %}\n {% do store_raw_result(\n name=\"main\",\n message=\"skip \" ~ target_relation,\n code=\"skip\",\n rows_affected=\"-1\"\n ) %}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.49542, "supported_languages": null }, "macro.dbt.materialized_view_execute_build_sql": { "name": "materialized_view_execute_build_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_execute_build_sql", "macro_sql": "{% macro materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) %}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set grant_config = config.get('grants') %}\n\n {% call statement(name=\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495422, "supported_languages": null }, "macro.dbt.materialization_view_default": { "name": "materialization_view_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/view.sql", "original_file_path": "macros/materializations/models/view.sql", "unique_id": "macro.dbt.materialization_view_default", "macro_sql": "{%- materialization view, default -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='view') -%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}\n\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}\n /*\n This relation (probably) doesn't exist yet. If it does exist, it's a leftover from\n a previous run, and we're going to try to drop it immediately. At the end of this\n materialization, we're going to rename the \"existing_relation\" to this identifier,\n and then we're going to drop it. In order to make sure we run the correct one of:\n - drop view ...\n - drop table ...\n\n We need to set the type of this relation to be the type of the existing_relation, if it exists,\n or else \"view\" as a sane default if it does not. Note that if the existing_relation does not\n exist, then there is nothing to move out of the way and subsequentally drop. In that case,\n this relation will be effectively unused.\n */\n {%- set backup_relation_type = 'view' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n -- move the existing view out of the way\n {% if existing_relation is not none %}\n /* Do the equivalent of rename_if_exists. 'existing_relation' could have been dropped\n since the variable was first set. */\n {% set existing_relation = load_cached_relation(existing_relation) %}\n {% if existing_relation is not none %}\n {{ adapter.rename_relation(existing_relation, backup_relation) }}\n {% endif %}\n {% endif %}\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization -%}", "depends_on": { "macros": [ "macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.run_hooks", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.get_create_view_as_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495431, "supported_languages": [ "sql" ] }, "macro.dbt.materialization_table_default": { "name": "materialization_table_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table.sql", "original_file_path": "macros/materializations/models/table.sql", "unique_id": "macro.dbt.materialization_table_default", "macro_sql": "{% materialization table, default %}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') %}\n {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}\n /*\n See ../view/view.sql for more information about this relation.\n */\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_table_as_sql(False, intermediate_relation, sql) }}\n {%- endcall %}\n\n {% do create_indexes(intermediate_relation) %}\n\n -- cleanup\n {% if existing_relation is not none %}\n /* Do the equivalent of rename_if_exists. 'existing_relation' could have been dropped\n since the variable was first set. */\n {% set existing_relation = load_cached_relation(existing_relation) %}\n {% if existing_relation is not none %}\n {{ adapter.rename_relation(existing_relation, backup_relation) }}\n {% endif %}\n {% endif %}\n\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n -- finally, drop the existing/backup relation after the commit\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n{% endmaterialization %}", "depends_on": { "macros": [ "macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.get_create_table_as_sql", "macro.dbt.create_indexes", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495441, "supported_languages": [ "sql" ] }, "macro.dbt.get_quoted_csv": { "name": "get_quoted_csv", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.get_quoted_csv", "macro_sql": "{% macro get_quoted_csv(column_names) %}\n\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote(col)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495455, "supported_languages": null }, "macro.dbt.diff_columns": { "name": "diff_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.diff_columns", "macro_sql": "{% macro diff_columns(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% set source_names = source_columns | map(attribute = 'column') | list %}\n {% set target_names = target_columns | map(attribute = 'column') | list %}\n\n {# --check whether the name attribute exists in the target - this does not perform a data type check #}\n {% for sc in source_columns %}\n {% if sc.name not in target_names %}\n {{ result.append(sc) }}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4954581, "supported_languages": null }, "macro.dbt.diff_column_data_types": { "name": "diff_column_data_types", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.diff_column_data_types", "macro_sql": "{% macro diff_column_data_types(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% for sc in source_columns %}\n {% set tc = target_columns | selectattr(\"name\", \"equalto\", sc.name) | list | first %}\n {% if tc %}\n {% if sc.data_type != tc.data_type and not sc.can_expand_to(other_column=tc) %}\n {{ result.append( { 'column_name': tc.name, 'new_type': sc.data_type } ) }}\n {% endif %}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4954612, "supported_languages": null }, "macro.dbt.get_merge_update_columns": { "name": "get_merge_update_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.get_merge_update_columns", "macro_sql": "{% macro get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {{ return(adapter.dispatch('get_merge_update_columns', 'dbt')(merge_update_columns, merge_exclude_columns, dest_columns)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__get_merge_update_columns" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495462, "supported_languages": null }, "macro.dbt.default__get_merge_update_columns": { "name": "default__get_merge_update_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.default__get_merge_update_columns", "macro_sql": "{% macro default__get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {%- set default_cols = dest_columns | map(attribute=\"quoted\") | list -%}\n\n {%- if merge_update_columns and merge_exclude_columns -%}\n {{ exceptions.raise_compiler_error(\n 'Model cannot specify merge_update_columns and merge_exclude_columns. Please update model to use only one config'\n )}}\n {%- elif merge_update_columns -%}\n {%- set update_columns = merge_update_columns -%}\n {%- elif merge_exclude_columns -%}\n {%- set update_columns = [] -%}\n {%- for column in dest_columns -%}\n {% if column.column | lower not in merge_exclude_columns | map(\"lower\") | list %}\n {%- do update_columns.append(column.quoted) -%}\n {% endif %}\n {%- endfor -%}\n {%- else -%}\n {%- set update_columns = default_cols -%}\n {%- endif -%}\n\n {{ return(update_columns) }}\n\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495465, "supported_languages": null }, "macro.dbt.get_merge_sql": { "name": "get_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.get_merge_sql", "macro_sql": "{% macro get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) -%}\n -- back compat for old kwarg name\n {% set incremental_predicates = kwargs.get('predicates', incremental_predicates) %}\n {{ adapter.dispatch('get_merge_sql', 'dbt')(target, source, unique_key, dest_columns, incremental_predicates) }}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__get_merge_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.49549, "supported_languages": null }, "macro.dbt.default__get_merge_sql": { "name": "default__get_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.default__get_merge_sql", "macro_sql": "{% macro default__get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) -%}\n {%- set predicates = [] if incremental_predicates is none else [] + incremental_predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set merge_update_columns = config.get('merge_update_columns') -%}\n {%- set merge_exclude_columns = config.get('merge_exclude_columns') -%}\n {%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}\n {% for key in unique_key %}\n {% set this_key_match %}\n DBT_INTERNAL_SOURCE.{{ key }} = DBT_INTERNAL_DEST.{{ key }}\n {% endset %}\n {% do predicates.append(this_key_match) %}\n {% endfor %}\n {% else %}\n {% set source_unique_key = (\"DBT_INTERNAL_SOURCE.\" ~ unique_key) | trim %}\n\t {% set target_unique_key = (\"DBT_INTERNAL_DEST.\" ~ unique_key) | trim %}\n\t {% set unique_key_match = equals(source_unique_key, target_unique_key) | trim %}\n {% do predicates.append(unique_key_match) %}\n {% endif %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n {{ sql_header if sql_header is not none }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{\"(\" ~ predicates | join(\") and (\") ~ \")\"}}\n\n {% if unique_key %}\n when matched then update set\n {% for column_name in update_columns -%}\n {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}\n {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n {% endif %}\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.get_quoted_csv", "macro.dbt.get_merge_update_columns", "macro.dbt.equals" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4954932, "supported_languages": null }, "macro.dbt.get_delete_insert_merge_sql": { "name": "get_delete_insert_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.get_delete_insert_merge_sql", "macro_sql": "{% macro get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}\n {{ adapter.dispatch('get_delete_insert_merge_sql', 'dbt')(target, source, unique_key, dest_columns, incremental_predicates) }}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__get_delete_insert_merge_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495501, "supported_languages": null }, "macro.dbt.default__get_delete_insert_merge_sql": { "name": "default__get_delete_insert_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.default__get_delete_insert_merge_sql", "macro_sql": "{% macro default__get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n {% if unique_key %}\n {% if unique_key is string %}\n {% set unique_key = [unique_key] %}\n {% endif %}\n\n {%- set unique_key_str = unique_key|join(', ') -%}\n\n delete from {{ target }} as DBT_INTERNAL_DEST\n where ({{ unique_key_str }}) in (\n select distinct {{ unique_key_str }}\n from {{ source }} as DBT_INTERNAL_SOURCE\n )\n {%- if incremental_predicates %}\n {% for predicate in incremental_predicates %}\n and {{ predicate }}\n {% endfor %}\n {%- endif -%};\n\n {% endif %}\n\n insert into {{ target }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ source }}\n )\n\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.get_quoted_csv" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4955049, "supported_languages": null }, "macro.dbt.get_insert_overwrite_merge_sql": { "name": "get_insert_overwrite_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.get_insert_overwrite_merge_sql", "macro_sql": "{% macro get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header=false) -%}\n {{ adapter.dispatch('get_insert_overwrite_merge_sql', 'dbt')(target, source, dest_columns, predicates, include_sql_header) }}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__get_insert_overwrite_merge_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495507, "supported_languages": null }, "macro.dbt.default__get_insert_overwrite_merge_sql": { "name": "default__get_insert_overwrite_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.default__get_insert_overwrite_merge_sql", "macro_sql": "{% macro default__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) -%}\n {#-- The only time include_sql_header is True: --#}\n {#-- BigQuery + insert_overwrite strategy + \"static\" partitions config --#}\n {#-- We should consider including the sql header at the materialization level instead --#}\n\n {%- set predicates = [] if predicates is none else [] + predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none and include_sql_header }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on FALSE\n\n when not matched by source\n {% if predicates %} and {{ predicates | join(' and ') }} {% endif %}\n then delete\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.get_quoted_csv" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495509, "supported_languages": null }, "macro.dbt.is_incremental": { "name": "is_incremental", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/is_incremental.sql", "original_file_path": "macros/materializations/models/incremental/is_incremental.sql", "unique_id": "macro.dbt.is_incremental", "macro_sql": "{% macro is_incremental() %}\n {#-- do not run introspective queries in parsing #}\n {% if not execute %}\n {{ return(False) }}\n {% else %}\n {% set relation = adapter.get_relation(this.database, this.schema, this.table) %}\n {{ return(relation is not none\n and relation.type == 'table'\n and model.config.materialized == 'incremental'\n and not should_full_refresh()) }}\n {% endif %}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.should_full_refresh" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495518, "supported_languages": null }, "macro.dbt.get_incremental_append_sql": { "name": "get_incremental_append_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_append_sql", "macro_sql": "{% macro get_incremental_append_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_append_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__get_incremental_append_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4955251, "supported_languages": null }, "macro.dbt.default__get_incremental_append_sql": { "name": "default__get_incremental_append_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_append_sql", "macro_sql": "{% macro default__get_incremental_append_sql(arg_dict) %}\n\n {% do return(get_insert_into_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"dest_columns\"])) %}\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.get_insert_into_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495527, "supported_languages": null }, "macro.dbt.get_incremental_delete_insert_sql": { "name": "get_incremental_delete_insert_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_delete_insert_sql", "macro_sql": "{% macro get_incremental_delete_insert_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_delete_insert_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__get_incremental_delete_insert_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.49553, "supported_languages": null }, "macro.dbt.default__get_incremental_delete_insert_sql": { "name": "default__get_incremental_delete_insert_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_delete_insert_sql", "macro_sql": "{% macro default__get_incremental_delete_insert_sql(arg_dict) %}\n\n {% do return(get_delete_insert_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"unique_key\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.get_delete_insert_merge_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495532, "supported_languages": null }, "macro.dbt.get_incremental_merge_sql": { "name": "get_incremental_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_merge_sql", "macro_sql": "{% macro get_incremental_merge_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_merge_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__get_incremental_merge_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4955342, "supported_languages": null }, "macro.dbt.default__get_incremental_merge_sql": { "name": "default__get_incremental_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_merge_sql", "macro_sql": "{% macro default__get_incremental_merge_sql(arg_dict) %}\n\n {% do return(get_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"unique_key\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.get_merge_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495536, "supported_languages": null }, "macro.dbt.get_incremental_insert_overwrite_sql": { "name": "get_incremental_insert_overwrite_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_insert_overwrite_sql", "macro_sql": "{% macro get_incremental_insert_overwrite_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_insert_overwrite_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__get_incremental_insert_overwrite_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495538, "supported_languages": null }, "macro.dbt.default__get_incremental_insert_overwrite_sql": { "name": "default__get_incremental_insert_overwrite_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_insert_overwrite_sql", "macro_sql": "{% macro default__get_incremental_insert_overwrite_sql(arg_dict) %}\n\n {% do return(get_insert_overwrite_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.get_insert_overwrite_merge_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.49554, "supported_languages": null }, "macro.dbt.get_incremental_default_sql": { "name": "get_incremental_default_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_default_sql", "macro_sql": "{% macro get_incremental_default_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_default_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_incremental_default_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495543, "supported_languages": null }, "macro.dbt.default__get_incremental_default_sql": { "name": "default__get_incremental_default_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_default_sql", "macro_sql": "{% macro default__get_incremental_default_sql(arg_dict) %}\n\n {% do return(get_incremental_append_sql(arg_dict)) %}\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.get_incremental_append_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495546, "supported_languages": null }, "macro.dbt.get_incremental_microbatch_sql": { "name": "get_incremental_microbatch_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_microbatch_sql", "macro_sql": "{% macro get_incremental_microbatch_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_microbatch_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_incremental_microbatch_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495548, "supported_languages": null }, "macro.dbt.default__get_incremental_microbatch_sql": { "name": "default__get_incremental_microbatch_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_microbatch_sql", "macro_sql": "{% macro default__get_incremental_microbatch_sql(arg_dict) %}\n\n {{ exceptions.raise_not_implemented('microbatch materialization strategy not implemented for adapter ' + adapter.type()) }}\n\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.49555, "supported_languages": null }, "macro.dbt.get_insert_into_sql": { "name": "get_insert_into_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_insert_into_sql", "macro_sql": "{% macro get_insert_into_sql(target_relation, temp_relation, dest_columns) %}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n insert into {{ target_relation }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ temp_relation }}\n )\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.get_quoted_csv" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495553, "supported_languages": null }, "macro.dbt.materialization_incremental_default": { "name": "materialization_incremental_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/incremental.sql", "original_file_path": "macros/materializations/models/incremental/incremental.sql", "unique_id": "macro.dbt.materialization_incremental_default", "macro_sql": "{% materialization incremental, default -%}\n\n -- relations\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') -%}\n {%- set temp_relation = make_temp_relation(target_relation)-%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation)-%}\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n\n -- configs\n {%- set unique_key = config.get('unique_key') -%}\n {%- set full_refresh_mode = (should_full_refresh() or existing_relation.is_view) -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n\n -- the temp_ and backup_ relations should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation. This has to happen before\n -- BEGIN, in a separate transaction\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation)-%}\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set to_drop = [] %}\n\n {% set incremental_strategy = config.get('incremental_strategy') or 'default' %}\n {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %}\n\n {% if existing_relation is none %}\n {% set build_sql = get_create_table_as_sql(False, target_relation, sql) %}\n {% set relation_for_indexes = target_relation %}\n {% elif full_refresh_mode %}\n {% set build_sql = get_create_table_as_sql(False, intermediate_relation, sql) %}\n {% set relation_for_indexes = intermediate_relation %}\n {% set need_swap = true %}\n {% else %}\n {% do run_query(get_create_table_as_sql(True, temp_relation, sql)) %}\n {% set relation_for_indexes = temp_relation %}\n {% set contract_config = config.get('contract') %}\n {% if not contract_config or not contract_config.enforced %}\n {% do adapter.expand_target_column_types(\n from_relation=temp_relation,\n to_relation=target_relation) %}\n {% endif %}\n {#-- Process schema changes. Returns dict of changes if successful. Use source columns for upserting/merging --#}\n {% set dest_columns = process_schema_changes(on_schema_change, temp_relation, existing_relation) %}\n {% if not dest_columns %}\n {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %}\n {% endif %}\n\n {#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#}\n {% set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) %}\n {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': temp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'incremental_predicates': incremental_predicates }) %}\n {% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %}\n\n {% endif %}\n\n {% call statement(\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% if existing_relation is none or existing_relation.is_view or should_full_refresh() %}\n {% do create_indexes(relation_for_indexes) %}\n {% endif %}\n\n {% if need_swap %}\n {% do adapter.rename_relation(target_relation, backup_relation) %}\n {% do adapter.rename_relation(intermediate_relation, target_relation) %}\n {% do to_drop.append(backup_relation) %}\n {% endif %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {% do adapter.commit() %}\n\n {% for rel in to_drop %}\n {% do adapter.drop_relation(rel) %}\n {% endfor %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "depends_on": { "macros": [ "macro.dbt.load_cached_relation", "macro.dbt.make_temp_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.should_full_refresh", "macro.dbt.incremental_validate_on_schema_change", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks", "macro.dbt.get_create_table_as_sql", "macro.dbt.run_query", "macro.dbt.process_schema_changes", "macro.dbt.statement", "macro.dbt.create_indexes", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495564, "supported_languages": [ "sql" ] }, "macro.dbt.incremental_validate_on_schema_change": { "name": "incremental_validate_on_schema_change", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.incremental_validate_on_schema_change", "macro_sql": "{% macro incremental_validate_on_schema_change(on_schema_change, default='ignore') %}\n\n {% if on_schema_change not in ['sync_all_columns', 'append_new_columns', 'fail', 'ignore'] %}\n\n {% set log_message = 'Invalid value for on_schema_change (%s) specified. Setting default value of %s.' % (on_schema_change, default) %}\n {% do log(log_message) %}\n\n {{ return(default) }}\n\n {% else %}\n\n {{ return(on_schema_change) }}\n\n {% endif %}\n\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495577, "supported_languages": null }, "macro.dbt.check_for_schema_changes": { "name": "check_for_schema_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.check_for_schema_changes", "macro_sql": "{% macro check_for_schema_changes(source_relation, target_relation) %}\n\n {% set schema_changed = False %}\n\n {%- set source_columns = adapter.get_columns_in_relation(source_relation) -%}\n {%- set target_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set source_not_in_target = diff_columns(source_columns, target_columns) -%}\n {%- set target_not_in_source = diff_columns(target_columns, source_columns) -%}\n\n {% set new_target_types = diff_column_data_types(source_columns, target_columns) %}\n\n {% if source_not_in_target != [] %}\n {% set schema_changed = True %}\n {% elif target_not_in_source != [] or new_target_types != [] %}\n {% set schema_changed = True %}\n {% elif new_target_types != [] %}\n {% set schema_changed = True %}\n {% endif %}\n\n {% set changes_dict = {\n 'schema_changed': schema_changed,\n 'source_not_in_target': source_not_in_target,\n 'target_not_in_source': target_not_in_source,\n 'source_columns': source_columns,\n 'target_columns': target_columns,\n 'new_target_types': new_target_types\n } %}\n\n {% set msg %}\n In {{ target_relation }}:\n Schema changed: {{ schema_changed }}\n Source columns not in target: {{ source_not_in_target }}\n Target columns not in source: {{ target_not_in_source }}\n New column types: {{ new_target_types }}\n {% endset %}\n\n {% do log(msg) %}\n\n {{ return(changes_dict) }}\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.diff_columns", "macro.dbt.diff_column_data_types" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4955788, "supported_languages": null }, "macro.dbt.sync_column_schemas": { "name": "sync_column_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.sync_column_schemas", "macro_sql": "{% macro sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {%- set add_to_target_arr = schema_changes_dict['source_not_in_target'] -%}\n\n {%- if on_schema_change == 'append_new_columns'-%}\n {%- if add_to_target_arr | length > 0 -%}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, none) -%}\n {%- endif -%}\n\n {% elif on_schema_change == 'sync_all_columns' %}\n {%- set remove_from_target_arr = schema_changes_dict['target_not_in_source'] -%}\n {%- set new_target_types = schema_changes_dict['new_target_types'] -%}\n\n {% if add_to_target_arr | length > 0 or remove_from_target_arr | length > 0 %}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, remove_from_target_arr) -%}\n {% endif %}\n\n {% if new_target_types != [] %}\n {% for ntt in new_target_types %}\n {% set column_name = ntt['column_name'] %}\n {% set new_type = ntt['new_type'] %}\n {% do alter_column_type(target_relation, column_name, new_type) %}\n {% endfor %}\n {% endif %}\n\n {% endif %}\n\n {% set schema_change_message %}\n In {{ target_relation }}:\n Schema change approach: {{ on_schema_change }}\n Columns added: {{ add_to_target_arr }}\n Columns removed: {{ remove_from_target_arr }}\n Data types changed: {{ new_target_types }}\n {% endset %}\n\n {% do log(schema_change_message) %}\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.alter_relation_add_remove_columns", "macro.dbt.alter_column_type" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495583, "supported_languages": null }, "macro.dbt.process_schema_changes": { "name": "process_schema_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.process_schema_changes", "macro_sql": "{% macro process_schema_changes(on_schema_change, source_relation, target_relation) %}\n\n {% if on_schema_change == 'ignore' %}\n\n {{ return({}) }}\n\n {% else %}\n\n {% set schema_changes_dict = check_for_schema_changes(source_relation, target_relation) %}\n\n {% if schema_changes_dict['schema_changed'] %}\n\n {% if on_schema_change == 'fail' %}\n\n {% set fail_msg %}\n The source and target schemas on this incremental model are out of sync!\n They can be reconciled in several ways:\n - set the `on_schema_change` config to either append_new_columns or sync_all_columns, depending on your situation.\n - Re-run the incremental model with `full_refresh: True` to update the target schema.\n - update the schema manually and re-run the process.\n\n Additional troubleshooting context:\n Source columns not in target: {{ schema_changes_dict['source_not_in_target'] }}\n Target columns not in source: {{ schema_changes_dict['target_not_in_source'] }}\n New column types: {{ schema_changes_dict['new_target_types'] }}\n {% endset %}\n\n {% do exceptions.raise_compiler_error(fail_msg) %}\n\n {# -- unless we ignore, run the sync operation per the config #}\n {% else %}\n\n {% do sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {% endif %}\n\n {% endif %}\n\n {{ return(schema_changes_dict['source_columns']) }}\n\n {% endif %}\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.check_for_schema_changes", "macro.dbt.sync_column_schemas" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4955862, "supported_languages": null }, "macro.dbt.can_clone_table": { "name": "can_clone_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/can_clone_table.sql", "original_file_path": "macros/materializations/models/clone/can_clone_table.sql", "unique_id": "macro.dbt.can_clone_table", "macro_sql": "{% macro can_clone_table() %}\n {{ return(adapter.dispatch('can_clone_table', 'dbt')()) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__can_clone_table" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495593, "supported_languages": null }, "macro.dbt.default__can_clone_table": { "name": "default__can_clone_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/can_clone_table.sql", "original_file_path": "macros/materializations/models/clone/can_clone_table.sql", "unique_id": "macro.dbt.default__can_clone_table", "macro_sql": "{% macro default__can_clone_table() %}\n {{ return(False) }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4955971, "supported_languages": null }, "macro.dbt.create_or_replace_clone": { "name": "create_or_replace_clone", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/create_or_replace_clone.sql", "original_file_path": "macros/materializations/models/clone/create_or_replace_clone.sql", "unique_id": "macro.dbt.create_or_replace_clone", "macro_sql": "{% macro create_or_replace_clone(this_relation, defer_relation) %}\n {{ return(adapter.dispatch('create_or_replace_clone', 'dbt')(this_relation, defer_relation)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__create_or_replace_clone" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495604, "supported_languages": null }, "macro.dbt.default__create_or_replace_clone": { "name": "default__create_or_replace_clone", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/create_or_replace_clone.sql", "original_file_path": "macros/materializations/models/clone/create_or_replace_clone.sql", "unique_id": "macro.dbt.default__create_or_replace_clone", "macro_sql": "{% macro default__create_or_replace_clone(this_relation, defer_relation) %}\n create or replace table {{ this_relation.render() }} clone {{ defer_relation.render() }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495606, "supported_languages": null }, "macro.dbt.materialization_clone_default": { "name": "materialization_clone_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/clone.sql", "original_file_path": "macros/materializations/models/clone/clone.sql", "unique_id": "macro.dbt.materialization_clone_default", "macro_sql": "{%- materialization clone, default -%}\n\n {%- set relations = {'relations': []} -%}\n\n {%- if not defer_relation -%}\n -- nothing to do\n {{ log(\"No relation found in state manifest for \" ~ model.unique_id, info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n\n {%- if existing_relation and not flags.FULL_REFRESH -%}\n -- noop!\n {{ log(\"Relation \" ~ existing_relation ~ \" already exists\", info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set other_existing_relation = load_cached_relation(defer_relation) -%}\n\n -- If this is a database that can do zero-copy cloning of tables, and the other relation is a table, then this will be a table\n -- Otherwise, this will be a view\n\n {% set can_clone_table = can_clone_table() %}\n\n {%- if other_existing_relation and other_existing_relation.type == 'table' and can_clone_table -%}\n\n {%- set target_relation = this.incorporate(type='table') -%}\n {% if existing_relation is not none and not existing_relation.is_table %}\n {{ log(\"Dropping relation \" ~ existing_relation.render() ~ \" because it is of type \" ~ existing_relation.type) }}\n {{ drop_relation_if_exists(existing_relation) }}\n {% endif %}\n\n -- as a general rule, data platforms that can clone tables can also do atomic 'create or replace'\n {% if target_relation.database == defer_relation.database and\n target_relation.schema == defer_relation.schema and\n target_relation.identifier == defer_relation.identifier %}\n {{ log(\"Target relation and defer relation are the same, skipping clone for relation: \" ~ target_relation.render()) }}\n {% else %}\n {% call statement('main') %}\n {{ create_or_replace_clone(target_relation, defer_relation) }}\n {% endcall %}\n {% endif %}\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n {% do persist_docs(target_relation, model) %}\n\n {{ return({'relations': [target_relation]}) }}\n\n {%- else -%}\n\n {%- set target_relation = this.incorporate(type='view') -%}\n\n -- reuse the view materialization\n -- TODO: support actual dispatch for materialization macros\n -- Tracking ticket: https://github.com/dbt-labs/dbt-core/issues/7799\n {% set search_name = \"materialization_view_\" ~ adapter.type() %}\n {% if not search_name in context %}\n {% set search_name = \"materialization_view_default\" %}\n {% endif %}\n {% set materialization_macro = context[search_name] %}\n {% set relations = materialization_macro() %}\n {{ return(relations) }}\n\n {%- endif -%}\n\n{%- endmaterialization -%}", "depends_on": { "macros": [ "macro.dbt.load_cached_relation", "macro.dbt.can_clone_table", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.create_or_replace_clone", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4956129, "supported_languages": [ "sql" ] }, "macro.dbt.materialization_seed_default": { "name": "materialization_seed_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/seed.sql", "original_file_path": "macros/materializations/seeds/seed.sql", "unique_id": "macro.dbt.materialization_seed_default", "macro_sql": "{% materialization seed, default %}\n\n {%- set identifier = model['alias'] -%}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set grant_config = config.get('grants') -%}\n {%- set agate_table = load_agate_table() -%}\n -- grab current tables grants config for comparison later on\n\n {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% set create_table_sql = \"\" %}\n {% if exists_as_view %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a view\".format(old_relation.render())) }}\n {% elif exists_as_table %}\n {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}\n {% else %}\n {% set create_table_sql = create_csv_table(model, agate_table) %}\n {% endif %}\n\n {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}\n {% set rows_affected = (agate_table.rows | length) %}\n {% set sql = load_csv_rows(model, agate_table) %}\n\n {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}\n {{ get_csv_sql(create_table_sql, sql) }};\n {% endcall %}\n\n {% set target_relation = this.incorporate(type='table') %}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if full_refresh_mode or not exists_as_table %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": { "macros": [ "macro.dbt.should_full_refresh", "macro.dbt.run_hooks", "macro.dbt.reset_csv_table", "macro.dbt.create_csv_table", "macro.dbt.load_csv_rows", "macro.dbt.noop_statement", "macro.dbt.get_csv_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495622, "supported_languages": [ "sql" ] }, "macro.dbt.create_csv_table": { "name": "create_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.create_csv_table", "macro_sql": "{% macro create_csv_table(model, agate_table) -%}\n {{ adapter.dispatch('create_csv_table', 'dbt')(model, agate_table) }}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__create_csv_table" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495633, "supported_languages": null }, "macro.dbt.default__create_csv_table": { "name": "default__create_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__create_csv_table", "macro_sql": "{% macro default__create_csv_table(model, agate_table) %}\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n\n {% set sql %}\n create table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.statement" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495637, "supported_languages": null }, "macro.dbt.reset_csv_table": { "name": "reset_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.reset_csv_table", "macro_sql": "{% macro reset_csv_table(model, full_refresh, old_relation, agate_table) -%}\n {{ adapter.dispatch('reset_csv_table', 'dbt')(model, full_refresh, old_relation, agate_table) }}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__reset_csv_table" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495639, "supported_languages": null }, "macro.dbt.default__reset_csv_table": { "name": "default__reset_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__reset_csv_table", "macro_sql": "{% macro default__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% set sql = \"\" %}\n {% if full_refresh %}\n {{ adapter.drop_relation(old_relation) }}\n {% set sql = create_csv_table(model, agate_table) %}\n {% else %}\n {{ adapter.truncate_relation(old_relation) }}\n {% set sql = \"truncate table \" ~ old_relation.render() %}\n {% endif %}\n\n {{ return(sql) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.create_csv_table" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495641, "supported_languages": null }, "macro.dbt.get_csv_sql": { "name": "get_csv_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_csv_sql", "macro_sql": "{% macro get_csv_sql(create_or_truncate_sql, insert_sql) %}\n {{ adapter.dispatch('get_csv_sql', 'dbt')(create_or_truncate_sql, insert_sql) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__get_csv_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4956439, "supported_languages": null }, "macro.dbt.default__get_csv_sql": { "name": "default__get_csv_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__get_csv_sql", "macro_sql": "{% macro default__get_csv_sql(create_or_truncate_sql, insert_sql) %}\n {{ create_or_truncate_sql }};\n -- dbt seed --\n {{ insert_sql }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495646, "supported_languages": null }, "macro.dbt.get_binding_char": { "name": "get_binding_char", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_binding_char", "macro_sql": "{% macro get_binding_char() -%}\n {{ adapter.dispatch('get_binding_char', 'dbt')() }}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__get_binding_char" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495656, "supported_languages": null }, "macro.dbt.default__get_binding_char": { "name": "default__get_binding_char", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__get_binding_char", "macro_sql": "{% macro default__get_binding_char() %}\n {{ return('%s') }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495659, "supported_languages": null }, "macro.dbt.get_batch_size": { "name": "get_batch_size", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_batch_size", "macro_sql": "{% macro get_batch_size() -%}\n {{ return(adapter.dispatch('get_batch_size', 'dbt')()) }}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__get_batch_size" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495661, "supported_languages": null }, "macro.dbt.default__get_batch_size": { "name": "default__get_batch_size", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__get_batch_size", "macro_sql": "{% macro default__get_batch_size() %}\n {{ return(10000) }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4956632, "supported_languages": null }, "macro.dbt.get_seed_column_quoted_csv": { "name": "get_seed_column_quoted_csv", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_seed_column_quoted_csv", "macro_sql": "{% macro get_seed_column_quoted_csv(model, column_names) %}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote_seed_column(col, quote_seed_column)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4956648, "supported_languages": null }, "macro.dbt.load_csv_rows": { "name": "load_csv_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.load_csv_rows", "macro_sql": "{% macro load_csv_rows(model, agate_table) -%}\n {{ adapter.dispatch('load_csv_rows', 'dbt')(model, agate_table) }}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__load_csv_rows" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495668, "supported_languages": null }, "macro.dbt.default__load_csv_rows": { "name": "default__load_csv_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__load_csv_rows", "macro_sql": "{% macro default__load_csv_rows(model, agate_table) %}\n\n {% set batch_size = get_batch_size() %}\n\n {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}\n {% set bindings = [] %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert into {{ this.render() }} ({{ cols_sql }}) values\n {% for row in chunk -%}\n ({%- for column in agate_table.column_names -%}\n {{ get_binding_char() }}\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.get_batch_size", "macro.dbt.get_seed_column_quoted_csv", "macro.dbt.get_binding_char" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495672, "supported_languages": null }, "macro.dbt.scalar_function_sql": { "name": "scalar_function_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.scalar_function_sql", "macro_sql": "{% macro scalar_function_sql(target_relation) %}\n {{ return(adapter.dispatch('scalar_function_sql', 'dbt')(target_relation)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__scalar_function_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495681, "supported_languages": null }, "macro.dbt.default__scalar_function_sql": { "name": "default__scalar_function_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.default__scalar_function_sql", "macro_sql": "{% macro default__scalar_function_sql(target_relation) %}\n {{ scalar_function_create_replace_signature_sql(target_relation) }}\n {{ scalar_function_body_sql() }};\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.scalar_function_create_replace_signature_sql", "macro.dbt.scalar_function_body_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4956841, "supported_languages": null }, "macro.dbt.scalar_function_create_replace_signature_sql": { "name": "scalar_function_create_replace_signature_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.scalar_function_create_replace_signature_sql", "macro_sql": "{% macro scalar_function_create_replace_signature_sql(target_relation) %}\n {{ return(adapter.dispatch('scalar_function_create_replace_signature_sql', 'dbt')(target_relation)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__scalar_function_create_replace_signature_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4956858, "supported_languages": null }, "macro.dbt.default__scalar_function_create_replace_signature_sql": { "name": "default__scalar_function_create_replace_signature_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.default__scalar_function_create_replace_signature_sql", "macro_sql": "{% macro default__scalar_function_create_replace_signature_sql(target_relation) %}\n CREATE OR REPLACE FUNCTION {{ target_relation.render() }} ({{ formatted_scalar_function_args_sql()}})\n RETURNS {{ model.returns.data_type }}\n {{ scalar_function_volatility_sql() }}\n AS\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.formatted_scalar_function_args_sql", "macro.dbt.scalar_function_volatility_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495688, "supported_languages": null }, "macro.dbt.formatted_scalar_function_args_sql": { "name": "formatted_scalar_function_args_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.formatted_scalar_function_args_sql", "macro_sql": "{% macro formatted_scalar_function_args_sql() %}\n {{ return(adapter.dispatch('formatted_scalar_function_args_sql', 'dbt')()) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__formatted_scalar_function_args_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495691, "supported_languages": null }, "macro.dbt.default__formatted_scalar_function_args_sql": { "name": "default__formatted_scalar_function_args_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.default__formatted_scalar_function_args_sql", "macro_sql": "{% macro default__formatted_scalar_function_args_sql() %}\n {% set args = [] %}\n {% for arg in model.arguments -%}\n {%- do args.append(arg.name ~ ' ' ~ arg.data_type) -%}\n {%- endfor %}\n {{ args | join(', ') }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495693, "supported_languages": null }, "macro.dbt.scalar_function_body_sql": { "name": "scalar_function_body_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.scalar_function_body_sql", "macro_sql": "{% macro scalar_function_body_sql() %}\n {{ return(adapter.dispatch('scalar_function_body_sql', 'dbt')()) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__scalar_function_body_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4956958, "supported_languages": null }, "macro.dbt.default__scalar_function_body_sql": { "name": "default__scalar_function_body_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.default__scalar_function_body_sql", "macro_sql": "{% macro default__scalar_function_body_sql() %}\n $$\n {{ model.compiled_code }}\n $$ LANGUAGE SQL\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495698, "supported_languages": null }, "macro.dbt.scalar_function_volatility_sql": { "name": "scalar_function_volatility_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.scalar_function_volatility_sql", "macro_sql": "{% macro scalar_function_volatility_sql() %}\n {{ return(adapter.dispatch('scalar_function_volatility_sql', 'dbt')()) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__scalar_function_volatility_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4957, "supported_languages": null }, "macro.dbt.default__scalar_function_volatility_sql": { "name": "default__scalar_function_volatility_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.default__scalar_function_volatility_sql", "macro_sql": "{% macro default__scalar_function_volatility_sql() %}\n {% set volatility = model.config.get('volatility') %}\n {% if volatility == 'deterministic' %}\n IMMUTABLE\n {% elif volatility == 'stable' %}\n STABLE\n {% elif volatility == 'non-deterministic' %}\n VOLATILE\n {% elif volatility != none %}\n {# This shouldn't happen unless a new volatility is invented #}\n {% do unsupported_volatility_warning(volatility) %}\n {% endif %}\n {# If no volatility is set, don't add anything and let the data warehouse default it #}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.unsupported_volatility_warning" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4957042, "supported_languages": null }, "macro.dbt.unsupported_volatility_warning": { "name": "unsupported_volatility_warning", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.unsupported_volatility_warning", "macro_sql": "{% macro unsupported_volatility_warning(volatility) %}\n {{ return(adapter.dispatch('unsupported_volatility_warning', 'dbt')(volatility)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__unsupported_volatility_warning" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495706, "supported_languages": null }, "macro.dbt.default__unsupported_volatility_warning": { "name": "default__unsupported_volatility_warning", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.default__unsupported_volatility_warning", "macro_sql": "{% macro default__unsupported_volatility_warning(volatility) %}\n {% set msg = \"Found `\" ~ volatility ~ \"` volatility specified on function `\" ~ model.name ~ \"`. This volatility is not supported by \" ~ adapter.type() ~ \", and will be ignored\" %}\n {% do exceptions.warn(msg) %}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495708, "supported_languages": null }, "macro.dbt.function_execute_build_sql": { "name": "function_execute_build_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/helpers.sql", "original_file_path": "macros/materializations/functions/helpers.sql", "unique_id": "macro.dbt.function_execute_build_sql", "macro_sql": "{% macro function_execute_build_sql(build_sql, existing_relation, target_relation) %}\n {{ return(adapter.dispatch('function_execute_build_sql', 'dbt')(build_sql, existing_relation, target_relation)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__function_execute_build_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495718, "supported_languages": null }, "macro.dbt.default__function_execute_build_sql": { "name": "default__function_execute_build_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/helpers.sql", "original_file_path": "macros/materializations/functions/helpers.sql", "unique_id": "macro.dbt.default__function_execute_build_sql", "macro_sql": "{% macro default__function_execute_build_sql(build_sql, existing_relation, target_relation) %}\n\n {% set grant_config = config.get('grants') %}\n\n {% call statement(name=\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ adapter.commit() }}\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4957209, "supported_languages": null }, "macro.dbt.get_aggregate_function_create_replace_signature": { "name": "get_aggregate_function_create_replace_signature", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/aggregate.sql", "original_file_path": "macros/materializations/functions/aggregate.sql", "unique_id": "macro.dbt.get_aggregate_function_create_replace_signature", "macro_sql": "{% macro get_aggregate_function_create_replace_signature(target_relation) %}\n {{ return(adapter.dispatch('get_aggregate_function_create_replace_signature', 'dbt')(target_relation)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__get_aggregate_function_create_replace_signature" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495728, "supported_languages": null }, "macro.dbt.default__get_aggregate_function_create_replace_signature": { "name": "default__get_aggregate_function_create_replace_signature", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/aggregate.sql", "original_file_path": "macros/materializations/functions/aggregate.sql", "unique_id": "macro.dbt.default__get_aggregate_function_create_replace_signature", "macro_sql": "{% macro default__get_aggregate_function_create_replace_signature(target_relation) %}\n CREATE OR REPLACE AGGREGATE FUNCTION {{ target_relation.render() }} ({{ get_formatted_aggregate_function_args()}})\n RETURNS {{ model.returns.data_type }}\n {{ get_function_language_specifier() }}\n {% if model.get('language') == 'python' %}\n {{ get_function_python_options() }}\n {% endif %}\n {{ scalar_function_volatility_sql() }}\n AS\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.get_formatted_aggregate_function_args", "macro.dbt.get_function_language_specifier", "macro.dbt.get_function_python_options", "macro.dbt.scalar_function_volatility_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4957309, "supported_languages": null }, "macro.dbt.get_formatted_aggregate_function_args": { "name": "get_formatted_aggregate_function_args", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/aggregate.sql", "original_file_path": "macros/materializations/functions/aggregate.sql", "unique_id": "macro.dbt.get_formatted_aggregate_function_args", "macro_sql": "{% macro get_formatted_aggregate_function_args() %}\n {{ return(adapter.dispatch('get_formatted_aggregate_function_args', 'dbt')()) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__get_formatted_aggregate_function_args" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495733, "supported_languages": null }, "macro.dbt.default__get_formatted_aggregate_function_args": { "name": "default__get_formatted_aggregate_function_args", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/aggregate.sql", "original_file_path": "macros/materializations/functions/aggregate.sql", "unique_id": "macro.dbt.default__get_formatted_aggregate_function_args", "macro_sql": "{% macro default__get_formatted_aggregate_function_args() %}\n {# conveniently we can reuse the sql scalar function args #}\n {{ formatted_scalar_function_args_sql() }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.formatted_scalar_function_args_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4957361, "supported_languages": null }, "macro.dbt.get_function_language_specifier": { "name": "get_function_language_specifier", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/aggregate.sql", "original_file_path": "macros/materializations/functions/aggregate.sql", "unique_id": "macro.dbt.get_function_language_specifier", "macro_sql": "{% macro get_function_language_specifier() %}\n {{ return(adapter.dispatch('get_function_language_specifier', 'dbt')()) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__get_function_language_specifier" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495738, "supported_languages": null }, "macro.dbt.default__get_function_language_specifier": { "name": "default__get_function_language_specifier", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/aggregate.sql", "original_file_path": "macros/materializations/functions/aggregate.sql", "unique_id": "macro.dbt.default__get_function_language_specifier", "macro_sql": "{% macro default__get_function_language_specifier() %}\n {% set language = model.get('language') %}\n {% if language == 'sql' %}\n {# generally you dont need to specify the language for sql functions #}\n {% elif language == 'python' %}\n LANGUAGE PYTHON\n {% else %}\n {{ 'LANGUAGE ' ~ language.upper() }}\n {% endif %}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495739, "supported_languages": null }, "macro.dbt.get_aggregate_function_volatility_specifier": { "name": "get_aggregate_function_volatility_specifier", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/aggregate.sql", "original_file_path": "macros/materializations/functions/aggregate.sql", "unique_id": "macro.dbt.get_aggregate_function_volatility_specifier", "macro_sql": "{% macro get_aggregate_function_volatility_specifier() %}\n {{ return(adapter.dispatch('get_aggregate_function_volatility_specifier', 'dbt')()) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__get_aggregate_function_volatility_specifier" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495743, "supported_languages": null }, "macro.dbt.default__get_aggregate_function_volatility_specifier": { "name": "default__get_aggregate_function_volatility_specifier", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/aggregate.sql", "original_file_path": "macros/materializations/functions/aggregate.sql", "unique_id": "macro.dbt.default__get_aggregate_function_volatility_specifier", "macro_sql": "{% macro default__get_aggregate_function_volatility_specifier() %}\n {{ scalar_function_volatility_sql() }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.scalar_function_volatility_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495745, "supported_languages": null }, "macro.dbt.get_function_python_options": { "name": "get_function_python_options", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/aggregate.sql", "original_file_path": "macros/materializations/functions/aggregate.sql", "unique_id": "macro.dbt.get_function_python_options", "macro_sql": "{% macro get_function_python_options() %}\n {{ return(adapter.dispatch('get_function_python_options', 'dbt')()) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__get_function_python_options" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4957519, "supported_languages": null }, "macro.dbt.default__get_function_python_options": { "name": "default__get_function_python_options", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/aggregate.sql", "original_file_path": "macros/materializations/functions/aggregate.sql", "unique_id": "macro.dbt.default__get_function_python_options", "macro_sql": "{% macro default__get_function_python_options() %}\n RUNTIME_VERSION = '{{ model.config.get('runtime_version') }}'\n HANDLER = '{{ model.config.get('entry_point') }}'\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495754, "supported_languages": null }, "macro.dbt.materialization_function_default": { "name": "materialization_function_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/function.sql", "original_file_path": "macros/materializations/functions/function.sql", "unique_id": "macro.dbt.materialization_function_default", "macro_sql": "{% materialization function, default, supported_languages=['sql', 'python'] %}\n {% set existing_relation = load_cached_relation(this) %}\n {% set target_relation = this.incorporate(type=this.Function) %}\n\n {{ run_hooks(pre_hooks) }}\n\n {% set function_config = this.get_function_config(model) %}\n {% set macro_name = this.get_function_macro_name(function_config) %}\n\n {# Doing this aliasing of adapter.dispatch is a hacky way to disable the static analysis of actually calling adapter.dispatch #}\n {# This is necessary because the static analysis breaks being able to dynamically pass a macro_name #}\n {% set _dispatch = adapter.dispatch %}\n\n {% set build_sql = _dispatch(macro_name, 'dbt')(target_relation) %}\n {{ function_execute_build_sql(build_sql, existing_relation, target_relation) }}\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": { "macros": [ "macro.dbt.load_cached_relation", "macro.dbt.run_hooks", "macro.dbt.function_execute_build_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.49576, "supported_languages": [ "sql", "python" ] }, "macro.dbt.generate_alias_name": { "name": "generate_alias_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_alias.sql", "original_file_path": "macros/get_custom_name/get_custom_alias.sql", "unique_id": "macro.dbt.generate_alias_name", "macro_sql": "{% macro generate_alias_name(custom_alias_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_alias_name', 'dbt')(custom_alias_name, node)) %}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__generate_alias_name" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495768, "supported_languages": null }, "macro.dbt.default__generate_alias_name": { "name": "default__generate_alias_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_alias.sql", "original_file_path": "macros/get_custom_name/get_custom_alias.sql", "unique_id": "macro.dbt.default__generate_alias_name", "macro_sql": "{% macro default__generate_alias_name(custom_alias_name=none, node=none) -%}\n\n {%- if custom_alias_name -%}\n\n {{ custom_alias_name | trim }}\n\n {%- elif node.version -%}\n\n {{ return(node.name ~ \"_v\" ~ (node.version | replace(\".\", \"_\"))) }}\n\n {%- else -%}\n\n {{ node.name }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495771, "supported_languages": null }, "macro.dbt.generate_schema_name": { "name": "generate_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "unique_id": "macro.dbt.generate_schema_name", "macro_sql": "{% macro generate_schema_name(custom_schema_name=none, node=none) -%}\n {{ return(adapter.dispatch('generate_schema_name', 'dbt')(custom_schema_name, node)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__generate_schema_name" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4957771, "supported_languages": null }, "macro.dbt.default__generate_schema_name": { "name": "default__generate_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "unique_id": "macro.dbt.default__generate_schema_name", "macro_sql": "{% macro default__generate_schema_name(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if custom_schema_name is none -%}\n\n {{ default_schema }}\n\n {%- else -%}\n\n {{ default_schema }}_{{ custom_schema_name | trim }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.49578, "supported_languages": null }, "macro.dbt.generate_schema_name_for_env": { "name": "generate_schema_name_for_env", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "unique_id": "macro.dbt.generate_schema_name_for_env", "macro_sql": "{% macro generate_schema_name_for_env(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if target.name == 'prod' and custom_schema_name is not none -%}\n\n {{ custom_schema_name | trim }}\n\n {%- else -%}\n\n {{ default_schema }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495782, "supported_languages": null }, "macro.dbt.generate_database_name": { "name": "generate_database_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "unique_id": "macro.dbt.generate_database_name", "macro_sql": "{% macro generate_database_name(custom_database_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_database_name', 'dbt')(custom_database_name, node)) %}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__generate_database_name" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495787, "supported_languages": null }, "macro.dbt.default__generate_database_name": { "name": "default__generate_database_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "unique_id": "macro.dbt.default__generate_database_name", "macro_sql": "{% macro default__generate_database_name(custom_database_name=none, node=none) -%}\n {%- set default_database = target.database -%}\n {%- if custom_database_name is none -%}\n\n {{ default_database }}\n\n {%- else -%}\n\n {{ custom_database_name }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495789, "supported_languages": null }, "macro.dbt.get_drop_sql": { "name": "get_drop_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.get_drop_sql", "macro_sql": "{%- macro get_drop_sql(relation) -%}\n {{- log('Applying DROP to: ' ~ relation) -}}\n {{- adapter.dispatch('get_drop_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "depends_on": { "macros": [ "macro.dbt.default__get_drop_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4957938, "supported_languages": null }, "macro.dbt.default__get_drop_sql": { "name": "default__get_drop_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.default__get_drop_sql", "macro_sql": "{%- macro default__get_drop_sql(relation) -%}\n\n {%- if relation.is_view -%}\n {{ drop_view(relation) }}\n\n {%- elif relation.is_table -%}\n {{ drop_table(relation) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ drop_materialized_view(relation) }}\n\n {%- else -%}\n drop {{ relation.type }} if exists {{ relation.render() }} cascade\n\n {%- endif -%}\n\n{%- endmacro -%}\n\n\n", "depends_on": { "macros": [ "macro.dbt.drop_view", "macro.dbt.drop_table", "macro.dbt.drop_materialized_view" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495797, "supported_languages": null }, "macro.dbt.drop_relation": { "name": "drop_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.drop_relation", "macro_sql": "{% macro drop_relation(relation) -%}\n {{ return(adapter.dispatch('drop_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__drop_relation" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495799, "supported_languages": null }, "macro.dbt.default__drop_relation": { "name": "default__drop_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.default__drop_relation", "macro_sql": "{% macro default__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n {{ get_drop_sql(relation) }}\n {%- endcall %}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.statement", "macro.dbt.get_drop_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495801, "supported_languages": null }, "macro.dbt.drop_relation_if_exists": { "name": "drop_relation_if_exists", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.drop_relation_if_exists", "macro_sql": "{% macro drop_relation_if_exists(relation) %}\n {% if relation is not none %}\n {{ adapter.drop_relation(relation) }}\n {% endif %}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4958038, "supported_languages": null }, "macro.dbt.get_replace_sql": { "name": "get_replace_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt.get_replace_sql", "macro_sql": "{% macro get_replace_sql(existing_relation, target_relation, sql) %}\n {{- log('Applying REPLACE to: ' ~ existing_relation) -}}\n {{- adapter.dispatch('get_replace_sql', 'dbt')(existing_relation, target_relation, sql) -}}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__get_replace_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495811, "supported_languages": null }, "macro.dbt.default__get_replace_sql": { "name": "default__get_replace_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt.default__get_replace_sql", "macro_sql": "{% macro default__get_replace_sql(existing_relation, target_relation, sql) %}\n\n {# /* use a create or replace statement if possible */ #}\n\n {% set is_replaceable = existing_relation.type == target_relation.type and existing_relation.can_be_replaced %}\n\n {% if is_replaceable and existing_relation.is_view %}\n {{ get_replace_view_sql(target_relation, sql) }}\n\n {% elif is_replaceable and existing_relation.is_table %}\n {{ get_replace_table_sql(target_relation, sql) }}\n\n {% elif is_replaceable and existing_relation.is_materialized_view %}\n {{ get_replace_materialized_view_sql(target_relation, sql) }}\n\n {# /* a create or replace statement is not possible, so try to stage and/or backup to be safe */ #}\n\n {# /* create target_relation as an intermediate relation, then swap it out with the existing one using a backup */ #}\n {%- elif target_relation.can_be_renamed and existing_relation.can_be_renamed -%}\n {{ get_create_intermediate_sql(target_relation, sql) }};\n {{ get_create_backup_sql(existing_relation) }};\n {{ get_rename_intermediate_sql(target_relation) }};\n {{ get_drop_backup_sql(existing_relation) }}\n\n {# /* create target_relation as an intermediate relation, then swap it out with the existing one without using a backup */ #}\n {%- elif target_relation.can_be_renamed -%}\n {{ get_create_intermediate_sql(target_relation, sql) }};\n {{ get_drop_sql(existing_relation) }};\n {{ get_rename_intermediate_sql(target_relation) }}\n\n {# /* create target_relation in place by first backing up the existing relation */ #}\n {%- elif existing_relation.can_be_renamed -%}\n {{ get_create_backup_sql(existing_relation) }};\n {{ get_create_sql(target_relation, sql) }};\n {{ get_drop_backup_sql(existing_relation) }}\n\n {# /* no renaming is allowed, so just drop and create */ #}\n {%- else -%}\n {{ get_drop_sql(existing_relation) }};\n {{ get_create_sql(target_relation, sql) }}\n\n {%- endif -%}\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.get_replace_view_sql", "macro.dbt.get_replace_table_sql", "macro.dbt.get_replace_materialized_view_sql", "macro.dbt.get_create_intermediate_sql", "macro.dbt.get_create_backup_sql", "macro.dbt.get_rename_intermediate_sql", "macro.dbt.get_drop_backup_sql", "macro.dbt.get_drop_sql", "macro.dbt.get_create_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4958138, "supported_languages": null }, "macro.dbt.get_create_intermediate_sql": { "name": "get_create_intermediate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create_intermediate.sql", "original_file_path": "macros/relations/create_intermediate.sql", "unique_id": "macro.dbt.get_create_intermediate_sql", "macro_sql": "{%- macro get_create_intermediate_sql(relation, sql) -%}\n {{- log('Applying CREATE INTERMEDIATE to: ' ~ relation) -}}\n {{- adapter.dispatch('get_create_intermediate_sql', 'dbt')(relation, sql) -}}\n{%- endmacro -%}\n\n\n", "depends_on": { "macros": [ "macro.dbt.default__get_create_intermediate_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.49582, "supported_languages": null }, "macro.dbt.default__get_create_intermediate_sql": { "name": "default__get_create_intermediate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create_intermediate.sql", "original_file_path": "macros/relations/create_intermediate.sql", "unique_id": "macro.dbt.default__get_create_intermediate_sql", "macro_sql": "{%- macro default__get_create_intermediate_sql(relation, sql) -%}\n\n -- get the standard intermediate name\n {% set intermediate_relation = make_intermediate_relation(relation) %}\n\n -- drop any pre-existing intermediate\n {{ get_drop_sql(intermediate_relation) }};\n\n {{ get_create_sql(intermediate_relation, sql) }}\n\n{%- endmacro -%}", "depends_on": { "macros": [ "macro.dbt.make_intermediate_relation", "macro.dbt.get_drop_sql", "macro.dbt.get_create_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4958231, "supported_languages": null }, "macro.dbt.drop_schema_named": { "name": "drop_schema_named", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/schema.sql", "original_file_path": "macros/relations/schema.sql", "unique_id": "macro.dbt.drop_schema_named", "macro_sql": "{% macro drop_schema_named(schema_name) %}\n {{ return(adapter.dispatch('drop_schema_named', 'dbt') (schema_name)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__drop_schema_named" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495828, "supported_languages": null }, "macro.dbt.default__drop_schema_named": { "name": "default__drop_schema_named", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/schema.sql", "original_file_path": "macros/relations/schema.sql", "unique_id": "macro.dbt.default__drop_schema_named", "macro_sql": "{% macro default__drop_schema_named(schema_name) %}\n {% set schema_relation = api.Relation.create(schema=schema_name) %}\n {{ adapter.drop_schema(schema_relation) }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.49583, "supported_languages": null }, "macro.dbt.get_drop_backup_sql": { "name": "get_drop_backup_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop_backup.sql", "original_file_path": "macros/relations/drop_backup.sql", "unique_id": "macro.dbt.get_drop_backup_sql", "macro_sql": "{%- macro get_drop_backup_sql(relation) -%}\n {{- log('Applying DROP BACKUP to: ' ~ relation) -}}\n {{- adapter.dispatch('get_drop_backup_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "depends_on": { "macros": [ "macro.dbt.default__get_drop_backup_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495837, "supported_languages": null }, "macro.dbt.default__get_drop_backup_sql": { "name": "default__get_drop_backup_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop_backup.sql", "original_file_path": "macros/relations/drop_backup.sql", "unique_id": "macro.dbt.default__get_drop_backup_sql", "macro_sql": "{%- macro default__get_drop_backup_sql(relation) -%}\n\n -- get the standard backup name\n {% set backup_relation = make_backup_relation(relation, relation.type) %}\n\n {{ get_drop_sql(backup_relation) }}\n\n{%- endmacro -%}", "depends_on": { "macros": [ "macro.dbt.make_backup_relation", "macro.dbt.get_drop_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4958389, "supported_languages": null }, "macro.dbt.get_rename_sql": { "name": "get_rename_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename.sql", "original_file_path": "macros/relations/rename.sql", "unique_id": "macro.dbt.get_rename_sql", "macro_sql": "{%- macro get_rename_sql(relation, new_name) -%}\n {{- log('Applying RENAME to: ' ~ relation) -}}\n {{- adapter.dispatch('get_rename_sql', 'dbt')(relation, new_name) -}}\n{%- endmacro -%}\n\n\n", "depends_on": { "macros": [ "macro.dbt.default__get_rename_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4958441, "supported_languages": null }, "macro.dbt.default__get_rename_sql": { "name": "default__get_rename_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename.sql", "original_file_path": "macros/relations/rename.sql", "unique_id": "macro.dbt.default__get_rename_sql", "macro_sql": "{%- macro default__get_rename_sql(relation, new_name) -%}\n\n {%- if relation.is_view -%}\n {{ get_rename_view_sql(relation, new_name) }}\n\n {%- elif relation.is_table -%}\n {{ get_rename_table_sql(relation, new_name) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ get_rename_materialized_view_sql(relation, new_name) }}\n\n {%- else -%}\n {{- exceptions.raise_compiler_error(\"`get_rename_sql` has not been implemented for: \" ~ relation.type ) -}}\n\n {%- endif -%}\n\n{%- endmacro -%}\n\n\n", "depends_on": { "macros": [ "macro.dbt.get_rename_view_sql", "macro.dbt.get_rename_table_sql", "macro.dbt.get_rename_materialized_view_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495848, "supported_languages": null }, "macro.dbt.rename_relation": { "name": "rename_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename.sql", "original_file_path": "macros/relations/rename.sql", "unique_id": "macro.dbt.rename_relation", "macro_sql": "{% macro rename_relation(from_relation, to_relation) -%}\n {{ return(adapter.dispatch('rename_relation', 'dbt')(from_relation, to_relation)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__rename_relation" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495851, "supported_languages": null }, "macro.dbt.default__rename_relation": { "name": "default__rename_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename.sql", "original_file_path": "macros/relations/rename.sql", "unique_id": "macro.dbt.default__rename_relation", "macro_sql": "{% macro default__rename_relation(from_relation, to_relation) -%}\n {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %}\n {% call statement('rename_relation') -%}\n alter table {{ from_relation.render() }} rename to {{ target_name }}\n {%- endcall %}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.statement" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495853, "supported_languages": null }, "macro.dbt.get_create_backup_sql": { "name": "get_create_backup_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create_backup.sql", "original_file_path": "macros/relations/create_backup.sql", "unique_id": "macro.dbt.get_create_backup_sql", "macro_sql": "{%- macro get_create_backup_sql(relation) -%}\n {{- log('Applying CREATE BACKUP to: ' ~ relation) -}}\n {{- adapter.dispatch('get_create_backup_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "depends_on": { "macros": [ "macro.dbt.default__get_create_backup_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495857, "supported_languages": null }, "macro.dbt.default__get_create_backup_sql": { "name": "default__get_create_backup_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create_backup.sql", "original_file_path": "macros/relations/create_backup.sql", "unique_id": "macro.dbt.default__get_create_backup_sql", "macro_sql": "{%- macro default__get_create_backup_sql(relation) -%}\n\n -- get the standard backup name\n {% set backup_relation = make_backup_relation(relation, relation.type) %}\n\n -- drop any pre-existing backup\n {{ get_drop_sql(backup_relation) }};\n\n {{ get_rename_sql(relation, backup_relation.identifier) }}\n\n{%- endmacro -%}", "depends_on": { "macros": [ "macro.dbt.make_backup_relation", "macro.dbt.get_drop_sql", "macro.dbt.get_rename_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495859, "supported_languages": null }, "macro.dbt.get_create_sql": { "name": "get_create_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create.sql", "original_file_path": "macros/relations/create.sql", "unique_id": "macro.dbt.get_create_sql", "macro_sql": "{%- macro get_create_sql(relation, sql) -%}\n {{- log('Applying CREATE to: ' ~ relation) -}}\n {{- adapter.dispatch('get_create_sql', 'dbt')(relation, sql) -}}\n{%- endmacro -%}\n\n\n", "depends_on": { "macros": [ "macro.dbt.default__get_create_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4958699, "supported_languages": null }, "macro.dbt.default__get_create_sql": { "name": "default__get_create_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create.sql", "original_file_path": "macros/relations/create.sql", "unique_id": "macro.dbt.default__get_create_sql", "macro_sql": "{%- macro default__get_create_sql(relation, sql) -%}\n\n {%- if relation.is_view -%}\n {{ get_create_view_as_sql(relation, sql) }}\n\n {%- elif relation.is_table -%}\n {{ get_create_table_as_sql(False, relation, sql) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ get_create_materialized_view_as_sql(relation, sql) }}\n\n {%- else -%}\n {{- exceptions.raise_compiler_error(\"`get_create_sql` has not been implemented for: \" ~ relation.type ) -}}\n\n {%- endif -%}\n\n{%- endmacro -%}", "depends_on": { "macros": [ "macro.dbt.get_create_view_as_sql", "macro.dbt.get_create_table_as_sql", "macro.dbt.get_create_materialized_view_as_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495872, "supported_languages": null }, "macro.dbt.get_rename_intermediate_sql": { "name": "get_rename_intermediate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename_intermediate.sql", "original_file_path": "macros/relations/rename_intermediate.sql", "unique_id": "macro.dbt.get_rename_intermediate_sql", "macro_sql": "{%- macro get_rename_intermediate_sql(relation) -%}\n {{- log('Applying RENAME INTERMEDIATE to: ' ~ relation) -}}\n {{- adapter.dispatch('get_rename_intermediate_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "depends_on": { "macros": [ "macro.dbt.default__get_rename_intermediate_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495897, "supported_languages": null }, "macro.dbt.default__get_rename_intermediate_sql": { "name": "default__get_rename_intermediate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename_intermediate.sql", "original_file_path": "macros/relations/rename_intermediate.sql", "unique_id": "macro.dbt.default__get_rename_intermediate_sql", "macro_sql": "{%- macro default__get_rename_intermediate_sql(relation) -%}\n\n -- get the standard intermediate name\n {% set intermediate_relation = make_intermediate_relation(relation) %}\n\n {{ get_rename_sql(intermediate_relation, relation.identifier) }}\n\n{%- endmacro -%}", "depends_on": { "macros": [ "macro.dbt.make_intermediate_relation", "macro.dbt.get_rename_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4959, "supported_languages": null }, "macro.dbt.drop_materialized_view": { "name": "drop_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/drop.sql", "original_file_path": "macros/relations/materialized_view/drop.sql", "unique_id": "macro.dbt.drop_materialized_view", "macro_sql": "{% macro drop_materialized_view(relation) -%}\n {{- adapter.dispatch('drop_materialized_view', 'dbt')(relation) -}}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__drop_materialized_view" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495905, "supported_languages": null }, "macro.dbt.default__drop_materialized_view": { "name": "default__drop_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/drop.sql", "original_file_path": "macros/relations/materialized_view/drop.sql", "unique_id": "macro.dbt.default__drop_materialized_view", "macro_sql": "{% macro default__drop_materialized_view(relation) -%}\n drop materialized view if exists {{ relation.render() }} cascade\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495907, "supported_languages": null }, "macro.dbt.get_replace_materialized_view_sql": { "name": "get_replace_materialized_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/replace.sql", "original_file_path": "macros/relations/materialized_view/replace.sql", "unique_id": "macro.dbt.get_replace_materialized_view_sql", "macro_sql": "{% macro get_replace_materialized_view_sql(relation, sql) %}\n {{- adapter.dispatch('get_replace_materialized_view_sql', 'dbt')(relation, sql) -}}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__get_replace_materialized_view_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4959118, "supported_languages": null }, "macro.dbt.default__get_replace_materialized_view_sql": { "name": "default__get_replace_materialized_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/replace.sql", "original_file_path": "macros/relations/materialized_view/replace.sql", "unique_id": "macro.dbt.default__get_replace_materialized_view_sql", "macro_sql": "{% macro default__get_replace_materialized_view_sql(relation, sql) %}\n {{ exceptions.raise_compiler_error(\n \"`get_replace_materialized_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495914, "supported_languages": null }, "macro.dbt.refresh_materialized_view": { "name": "refresh_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/refresh.sql", "original_file_path": "macros/relations/materialized_view/refresh.sql", "unique_id": "macro.dbt.refresh_materialized_view", "macro_sql": "{% macro refresh_materialized_view(relation) %}\n {{- log('Applying REFRESH to: ' ~ relation) -}}\n {{- adapter.dispatch('refresh_materialized_view', 'dbt')(relation) -}}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__refresh_materialized_view" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495919, "supported_languages": null }, "macro.dbt.default__refresh_materialized_view": { "name": "default__refresh_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/refresh.sql", "original_file_path": "macros/relations/materialized_view/refresh.sql", "unique_id": "macro.dbt.default__refresh_materialized_view", "macro_sql": "{% macro default__refresh_materialized_view(relation) %}\n {{ exceptions.raise_compiler_error(\"`refresh_materialized_view` has not been implemented for this adapter.\") }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495921, "supported_languages": null }, "macro.dbt.get_rename_materialized_view_sql": { "name": "get_rename_materialized_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/rename.sql", "original_file_path": "macros/relations/materialized_view/rename.sql", "unique_id": "macro.dbt.get_rename_materialized_view_sql", "macro_sql": "{% macro get_rename_materialized_view_sql(relation, new_name) %}\n {{- adapter.dispatch('get_rename_materialized_view_sql', 'dbt')(relation, new_name) -}}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_rename_materialized_view_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495926, "supported_languages": null }, "macro.dbt.default__get_rename_materialized_view_sql": { "name": "default__get_rename_materialized_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/rename.sql", "original_file_path": "macros/relations/materialized_view/rename.sql", "unique_id": "macro.dbt.default__get_rename_materialized_view_sql", "macro_sql": "{% macro default__get_rename_materialized_view_sql(relation, new_name) %}\n {{ exceptions.raise_compiler_error(\n \"`get_rename_materialized_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495928, "supported_languages": null }, "macro.dbt.get_alter_materialized_view_as_sql": { "name": "get_alter_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt.get_alter_materialized_view_as_sql", "macro_sql": "{% macro get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n {{- log('Applying ALTER to: ' ~ relation) -}}\n {{- adapter.dispatch('get_alter_materialized_view_as_sql', 'dbt')(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n ) -}}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_alter_materialized_view_as_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495934, "supported_languages": null }, "macro.dbt.default__get_alter_materialized_view_as_sql": { "name": "default__get_alter_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt.default__get_alter_materialized_view_as_sql", "macro_sql": "{% macro default__get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n {{ exceptions.raise_compiler_error(\"Materialized views have not been implemented for this adapter.\") }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495937, "supported_languages": null }, "macro.dbt.get_materialized_view_configuration_changes": { "name": "get_materialized_view_configuration_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt.get_materialized_view_configuration_changes", "macro_sql": "{% macro get_materialized_view_configuration_changes(existing_relation, new_config) %}\n /* {#\n It's recommended that configuration changes be formatted as follows:\n {\"\": [{\"action\": \"\", \"context\": ...}]}\n\n For example:\n {\n \"indexes\": [\n {\"action\": \"drop\", \"context\": \"index_abc\"},\n {\"action\": \"create\", \"context\": {\"columns\": [\"column_1\", \"column_2\"], \"type\": \"hash\", \"unique\": True}},\n ],\n }\n\n Either way, `get_materialized_view_configuration_changes` needs to align with `get_alter_materialized_view_as_sql`.\n #} */\n {{- log('Determining configuration changes on: ' ~ existing_relation) -}}\n {%- do return(adapter.dispatch('get_materialized_view_configuration_changes', 'dbt')(existing_relation, new_config)) -%}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_materialized_view_configuration_changes" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4959412, "supported_languages": null }, "macro.dbt.default__get_materialized_view_configuration_changes": { "name": "default__get_materialized_view_configuration_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt.default__get_materialized_view_configuration_changes", "macro_sql": "{% macro default__get_materialized_view_configuration_changes(existing_relation, new_config) %}\n {{ exceptions.raise_compiler_error(\"Materialized views have not been implemented for this adapter.\") }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4959428, "supported_languages": null }, "macro.dbt.get_create_materialized_view_as_sql": { "name": "get_create_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/create.sql", "original_file_path": "macros/relations/materialized_view/create.sql", "unique_id": "macro.dbt.get_create_materialized_view_as_sql", "macro_sql": "{% macro get_create_materialized_view_as_sql(relation, sql) -%}\n {{- adapter.dispatch('get_create_materialized_view_as_sql', 'dbt')(relation, sql) -}}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_create_materialized_view_as_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4959471, "supported_languages": null }, "macro.dbt.default__get_create_materialized_view_as_sql": { "name": "default__get_create_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/create.sql", "original_file_path": "macros/relations/materialized_view/create.sql", "unique_id": "macro.dbt.default__get_create_materialized_view_as_sql", "macro_sql": "{% macro default__get_create_materialized_view_as_sql(relation, sql) -%}\n {{ exceptions.raise_compiler_error(\n \"`get_create_materialized_view_as_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.49595, "supported_languages": null }, "macro.dbt.get_table_columns_and_constraints": { "name": "get_table_columns_and_constraints", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.get_table_columns_and_constraints", "macro_sql": "{%- macro get_table_columns_and_constraints() -%}\n {{ adapter.dispatch('get_table_columns_and_constraints', 'dbt')() }}\n{%- endmacro -%}\n\n", "depends_on": { "macros": [ "macro.dbt.default__get_table_columns_and_constraints" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495956, "supported_languages": null }, "macro.dbt.default__get_table_columns_and_constraints": { "name": "default__get_table_columns_and_constraints", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.default__get_table_columns_and_constraints", "macro_sql": "{% macro default__get_table_columns_and_constraints() -%}\n {{ return(table_columns_and_constraints()) }}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.table_columns_and_constraints" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495958, "supported_languages": null }, "macro.dbt.table_columns_and_constraints": { "name": "table_columns_and_constraints", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.table_columns_and_constraints", "macro_sql": "{% macro table_columns_and_constraints() %}\n {# loop through user_provided_columns to create DDL with data types and constraints #}\n {%- set raw_column_constraints = adapter.render_raw_columns_constraints(raw_columns=model['columns']) -%}\n {%- set raw_model_constraints = adapter.render_raw_model_constraints(raw_constraints=model['constraints']) -%}\n (\n {% for c in raw_column_constraints -%}\n {{ c }}{{ \",\" if not loop.last or raw_model_constraints }}\n {% endfor %}\n {% for c in raw_model_constraints -%}\n {{ c }}{{ \",\" if not loop.last }}\n {% endfor -%}\n )\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.49596, "supported_languages": null }, "macro.dbt.get_assert_columns_equivalent": { "name": "get_assert_columns_equivalent", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.get_assert_columns_equivalent", "macro_sql": "\n\n{%- macro get_assert_columns_equivalent(sql) -%}\n {{ adapter.dispatch('get_assert_columns_equivalent', 'dbt')(sql) }}\n{%- endmacro -%}\n\n", "depends_on": { "macros": [ "macro.dbt.default__get_assert_columns_equivalent" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4959621, "supported_languages": null }, "macro.dbt.default__get_assert_columns_equivalent": { "name": "default__get_assert_columns_equivalent", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.default__get_assert_columns_equivalent", "macro_sql": "{% macro default__get_assert_columns_equivalent(sql) -%}\n {{ return(assert_columns_equivalent(sql)) }}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.assert_columns_equivalent" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4959638, "supported_languages": null }, "macro.dbt.assert_columns_equivalent": { "name": "assert_columns_equivalent", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.assert_columns_equivalent", "macro_sql": "{% macro assert_columns_equivalent(sql) %}\n\n {#-- First ensure the user has defined 'columns' in yaml specification --#}\n {%- set user_defined_columns = model['columns'] -%}\n {%- if not user_defined_columns -%}\n {{ exceptions.raise_contract_error([], []) }}\n {%- endif -%}\n\n {#-- Obtain the column schema provided by sql file. #}\n {%- set sql_file_provided_columns = get_column_schema_from_query(sql, config.get('sql_header', none)) -%}\n {#--Obtain the column schema provided by the schema file by generating an 'empty schema' query from the model's columns. #}\n {%- set schema_file_provided_columns = get_column_schema_from_query(get_empty_schema_sql(user_defined_columns)) -%}\n\n {#-- create dictionaries with name and formatted data type and strings for exception #}\n {%- set sql_columns = format_columns(sql_file_provided_columns) -%}\n {%- set yaml_columns = format_columns(schema_file_provided_columns) -%}\n\n {%- if sql_columns|length != yaml_columns|length -%}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n\n {%- for sql_col in sql_columns -%}\n {%- set yaml_col = [] -%}\n {%- for this_col in yaml_columns -%}\n {%- if this_col['name'] == sql_col['name'] -%}\n {%- do yaml_col.append(this_col) -%}\n {%- break -%}\n {%- endif -%}\n {%- endfor -%}\n {%- if not yaml_col -%}\n {#-- Column with name not found in yaml #}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n {%- if sql_col['formatted'] != yaml_col[0]['formatted'] -%}\n {#-- Column data types don't match #}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n {%- endfor -%}\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.get_column_schema_from_query", "macro.dbt.get_empty_schema_sql", "macro.dbt.format_columns" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495967, "supported_languages": null }, "macro.dbt.format_columns": { "name": "format_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.format_columns", "macro_sql": "{% macro format_columns(columns) %}\n {% set formatted_columns = [] %}\n {% for column in columns %}\n {%- set formatted_column = adapter.dispatch('format_column', 'dbt')(column) -%}\n {%- do formatted_columns.append(formatted_column) -%}\n {% endfor %}\n {{ return(formatted_columns) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__format_column" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495973, "supported_languages": null }, "macro.dbt.default__format_column": { "name": "default__format_column", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.default__format_column", "macro_sql": "{% macro default__format_column(column) -%}\n {% set data_type = column.dtype %}\n {% set formatted = column.column.lower() ~ \" \" ~ data_type %}\n {{ return({'name': column.name, 'data_type': data_type, 'formatted': formatted}) }}\n{%- endmacro -%}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495976, "supported_languages": null }, "macro.dbt.drop_table": { "name": "drop_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/drop.sql", "original_file_path": "macros/relations/table/drop.sql", "unique_id": "macro.dbt.drop_table", "macro_sql": "{% macro drop_table(relation) -%}\n {{- adapter.dispatch('drop_table', 'dbt')(relation) -}}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__drop_table" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4959831, "supported_languages": null }, "macro.dbt.default__drop_table": { "name": "default__drop_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/drop.sql", "original_file_path": "macros/relations/table/drop.sql", "unique_id": "macro.dbt.default__drop_table", "macro_sql": "{% macro default__drop_table(relation) -%}\n drop table if exists {{ relation.render() }} cascade\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495986, "supported_languages": null }, "macro.dbt.get_replace_table_sql": { "name": "get_replace_table_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/replace.sql", "original_file_path": "macros/relations/table/replace.sql", "unique_id": "macro.dbt.get_replace_table_sql", "macro_sql": "{% macro get_replace_table_sql(relation, sql) %}\n {{- adapter.dispatch('get_replace_table_sql', 'dbt')(relation, sql) -}}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_replace_table_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.49599, "supported_languages": null }, "macro.dbt.default__get_replace_table_sql": { "name": "default__get_replace_table_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/replace.sql", "original_file_path": "macros/relations/table/replace.sql", "unique_id": "macro.dbt.default__get_replace_table_sql", "macro_sql": "{% macro default__get_replace_table_sql(relation, sql) %}\n {{ exceptions.raise_compiler_error(\n \"`get_replace_table_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495992, "supported_languages": null }, "macro.dbt.get_rename_table_sql": { "name": "get_rename_table_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/rename.sql", "original_file_path": "macros/relations/table/rename.sql", "unique_id": "macro.dbt.get_rename_table_sql", "macro_sql": "{% macro get_rename_table_sql(relation, new_name) %}\n {{- adapter.dispatch('get_rename_table_sql', 'dbt')(relation, new_name) -}}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_rename_table_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.495998, "supported_languages": null }, "macro.dbt.default__get_rename_table_sql": { "name": "default__get_rename_table_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/rename.sql", "original_file_path": "macros/relations/table/rename.sql", "unique_id": "macro.dbt.default__get_rename_table_sql", "macro_sql": "{% macro default__get_rename_table_sql(relation, new_name) %}\n {{ exceptions.raise_compiler_error(\n \"`get_rename_table_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496, "supported_languages": null }, "macro.dbt.get_create_table_as_sql": { "name": "get_create_table_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.get_create_table_as_sql", "macro_sql": "{% macro get_create_table_as_sql(temporary, relation, sql) -%}\n {{ adapter.dispatch('get_create_table_as_sql', 'dbt')(temporary, relation, sql) }}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__get_create_table_as_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4960058, "supported_languages": null }, "macro.dbt.default__get_create_table_as_sql": { "name": "default__get_create_table_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.default__get_create_table_as_sql", "macro_sql": "{% macro default__get_create_table_as_sql(temporary, relation, sql) -%}\n {{ return(create_table_as(temporary, relation, sql)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.create_table_as" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496008, "supported_languages": null }, "macro.dbt.create_table_as": { "name": "create_table_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.create_table_as", "macro_sql": "{% macro create_table_as(temporary, relation, compiled_code, language='sql') -%}\n {# backward compatibility for create_table_as that does not support language #}\n {% if language == \"sql\" %}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, compiled_code)}}\n {% else %}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, compiled_code, language) }}\n {% endif %}\n\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__create_table_as" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.49601, "supported_languages": null }, "macro.dbt.default__create_table_as": { "name": "default__create_table_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.default__create_table_as", "macro_sql": "{% macro default__create_table_as(temporary, relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary: -%}temporary{%- endif %} table\n {{ relation.include(database=(not temporary), schema=(not temporary)) }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced and (not temporary) %}\n {{ get_assert_columns_equivalent(sql) }}\n {{ get_table_columns_and_constraints() }}\n {%- set sql = get_select_subquery(sql) %}\n {% endif %}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.get_assert_columns_equivalent", "macro.dbt.get_table_columns_and_constraints", "macro.dbt.get_select_subquery" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496012, "supported_languages": null }, "macro.dbt.default__get_column_names": { "name": "default__get_column_names", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.default__get_column_names", "macro_sql": "{% macro default__get_column_names() %}\n {#- loop through user_provided_columns to get column names -#}\n {%- set user_provided_columns = model['columns'] -%}\n {%- for i in user_provided_columns %}\n {%- set col = user_provided_columns[i] -%}\n {%- set col_name = adapter.quote(col['name']) if col.get('quote') else col['name'] -%}\n {{ col_name }}{{ \", \" if not loop.last }}\n {%- endfor -%}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496014, "supported_languages": null }, "macro.dbt.get_select_subquery": { "name": "get_select_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.get_select_subquery", "macro_sql": "{% macro get_select_subquery(sql) %}\n {{ return(adapter.dispatch('get_select_subquery', 'dbt')(sql)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__get_select_subquery" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4960158, "supported_languages": null }, "macro.dbt.default__get_select_subquery": { "name": "default__get_select_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.default__get_select_subquery", "macro_sql": "{% macro default__get_select_subquery(sql) %}\n select {{ adapter.dispatch('get_column_names', 'dbt')() }}\n from (\n {{ sql }}\n ) as model_subq\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__get_column_names" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496019, "supported_languages": null }, "macro.dbt.drop_view": { "name": "drop_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/drop.sql", "original_file_path": "macros/relations/view/drop.sql", "unique_id": "macro.dbt.drop_view", "macro_sql": "{% macro drop_view(relation) -%}\n {{- adapter.dispatch('drop_view', 'dbt')(relation) -}}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__drop_view" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496028, "supported_languages": null }, "macro.dbt.default__drop_view": { "name": "default__drop_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/drop.sql", "original_file_path": "macros/relations/view/drop.sql", "unique_id": "macro.dbt.default__drop_view", "macro_sql": "{% macro default__drop_view(relation) -%}\n drop view if exists {{ relation.render() }} cascade\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496031, "supported_languages": null }, "macro.dbt.get_replace_view_sql": { "name": "get_replace_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.get_replace_view_sql", "macro_sql": "{% macro get_replace_view_sql(relation, sql) %}\n {{- adapter.dispatch('get_replace_view_sql', 'dbt')(relation, sql) -}}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_replace_view_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496036, "supported_languages": null }, "macro.dbt.default__get_replace_view_sql": { "name": "default__get_replace_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.default__get_replace_view_sql", "macro_sql": "{% macro default__get_replace_view_sql(relation, sql) %}\n {{ exceptions.raise_compiler_error(\n \"`get_replace_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496038, "supported_languages": null }, "macro.dbt.create_or_replace_view": { "name": "create_or_replace_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.create_or_replace_view", "macro_sql": "{% macro create_or_replace_view() %}\n {%- set identifier = model['alias'] -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database,\n type='view') -%}\n {% set grant_config = config.get('grants') %}\n\n {{ run_hooks(pre_hooks) }}\n\n -- If there's a table with the same name and we weren't told to full refresh,\n -- that's an error. If we were told to full refresh, drop it. This behavior differs\n -- for Snowflake and BigQuery, so multiple dispatch is used.\n {%- if old_relation is not none and old_relation.is_table -%}\n {{ handle_existing_table(should_full_refresh(), old_relation) }}\n {%- endif -%}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(target_relation, sql) }}\n {%- endcall %}\n\n {% set should_revoke = should_revoke(exists_as_view, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.run_hooks", "macro.dbt.handle_existing_table", "macro.dbt.should_full_refresh", "macro.dbt.statement", "macro.dbt.get_create_view_as_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496041, "supported_languages": null }, "macro.dbt.handle_existing_table": { "name": "handle_existing_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.handle_existing_table", "macro_sql": "{% macro handle_existing_table(full_refresh, old_relation) %}\n {{ adapter.dispatch('handle_existing_table', 'dbt')(full_refresh, old_relation) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__handle_existing_table" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496043, "supported_languages": null }, "macro.dbt.default__handle_existing_table": { "name": "default__handle_existing_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.default__handle_existing_table", "macro_sql": "{% macro default__handle_existing_table(full_refresh, old_relation) %}\n {{ log(\"Dropping relation \" ~ old_relation.render() ~ \" because it is of type \" ~ old_relation.type) }}\n {{ adapter.drop_relation(old_relation) }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496045, "supported_languages": null }, "macro.dbt.get_rename_view_sql": { "name": "get_rename_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/rename.sql", "original_file_path": "macros/relations/view/rename.sql", "unique_id": "macro.dbt.get_rename_view_sql", "macro_sql": "{% macro get_rename_view_sql(relation, new_name) %}\n {{- adapter.dispatch('get_rename_view_sql', 'dbt')(relation, new_name) -}}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_rename_view_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4960551, "supported_languages": null }, "macro.dbt.default__get_rename_view_sql": { "name": "default__get_rename_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/rename.sql", "original_file_path": "macros/relations/view/rename.sql", "unique_id": "macro.dbt.default__get_rename_view_sql", "macro_sql": "{% macro default__get_rename_view_sql(relation, new_name) %}\n {{ exceptions.raise_compiler_error(\n \"`get_rename_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4960582, "supported_languages": null }, "macro.dbt.get_create_view_as_sql": { "name": "get_create_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt.get_create_view_as_sql", "macro_sql": "{% macro get_create_view_as_sql(relation, sql) -%}\n {{ adapter.dispatch('get_create_view_as_sql', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__get_create_view_as_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496064, "supported_languages": null }, "macro.dbt.default__get_create_view_as_sql": { "name": "default__get_create_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt.default__get_create_view_as_sql", "macro_sql": "{% macro default__get_create_view_as_sql(relation, sql) -%}\n {{ return(create_view_as(relation, sql)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.create_view_as" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496066, "supported_languages": null }, "macro.dbt.create_view_as": { "name": "create_view_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt.create_view_as", "macro_sql": "{% macro create_view_as(relation, sql) -%}\n {{ adapter.dispatch('create_view_as', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__create_view_as" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496068, "supported_languages": null }, "macro.dbt.default__create_view_as": { "name": "default__create_view_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt.default__create_view_as", "macro_sql": "{% macro default__create_view_as(relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n create view {{ relation.render() }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {%- endif %}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.get_assert_columns_equivalent" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4960709, "supported_languages": null }, "macro.dbt.default__test_relationships": { "name": "default__test_relationships", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/relationships.sql", "original_file_path": "macros/generic_test_sql/relationships.sql", "unique_id": "macro.dbt.default__test_relationships", "macro_sql": "{% macro default__test_relationships(model, column_name, to, field) %}\n\nwith child as (\n select {{ column_name }} as from_field\n from {{ model }}\n where {{ column_name }} is not null\n),\n\nparent as (\n select {{ field }} as to_field\n from {{ to }}\n)\n\nselect\n from_field\n\nfrom child\nleft join parent\n on child.from_field = parent.to_field\n\nwhere parent.to_field is null\n\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496075, "supported_languages": null }, "macro.dbt.default__test_not_null": { "name": "default__test_not_null", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/not_null.sql", "original_file_path": "macros/generic_test_sql/not_null.sql", "unique_id": "macro.dbt.default__test_not_null", "macro_sql": "{% macro default__test_not_null(model, column_name) %}\n\n{% set column_list = '*' if should_store_failures() else column_name %}\n\nselect {{ column_list }}\nfrom {{ model }}\nwhere {{ column_name }} is null\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.should_store_failures" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4960809, "supported_languages": null }, "macro.dbt.default__test_unique": { "name": "default__test_unique", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/unique.sql", "original_file_path": "macros/generic_test_sql/unique.sql", "unique_id": "macro.dbt.default__test_unique", "macro_sql": "{% macro default__test_unique(model, column_name) %}\n\nselect\n {{ column_name }} as unique_field,\n count(*) as n_records\n\nfrom {{ model }}\nwhere {{ column_name }} is not null\ngroup by {{ column_name }}\nhaving count(*) > 1\n\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4960861, "supported_languages": null }, "macro.dbt.default__test_accepted_values": { "name": "default__test_accepted_values", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/accepted_values.sql", "original_file_path": "macros/generic_test_sql/accepted_values.sql", "unique_id": "macro.dbt.default__test_accepted_values", "macro_sql": "{% macro default__test_accepted_values(model, column_name, values, quote=True) %}\n\nwith all_values as (\n\n select\n {{ column_name }} as value_field,\n count(*) as n_records\n\n from {{ model }}\n group by {{ column_name }}\n\n)\n\nselect *\nfrom all_values\nwhere value_field not in (\n {% for value in values -%}\n {% if quote -%}\n '{{ value }}'\n {%- else -%}\n {{ value }}\n {%- endif -%}\n {%- if not loop.last -%},{%- endif %}\n {%- endfor %}\n)\n\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496091, "supported_languages": null }, "macro.dbt.statement": { "name": "statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt.statement", "macro_sql": "\n{%- macro statement(name=None, fetch_result=False, auto_begin=True, language='sql') -%}\n {%- if execute: -%}\n {%- set compiled_code = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime {} for node \"{}\"'.format(language, model['unique_id'])) }}\n {{ write(compiled_code) }}\n {%- endif -%}\n {%- if language == 'sql'-%}\n {%- set res, table = adapter.execute(compiled_code, auto_begin=auto_begin, fetch=fetch_result) -%}\n {%- elif language == 'python' -%}\n {%- set res = submit_python_job(model, compiled_code) -%}\n {#-- TODO: What should table be for python models? --#}\n {%- set table = None -%}\n {%- else -%}\n {% do exceptions.raise_compiler_error(\"statement macro didn't get supported language\") %}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_result(name, response=res, agate_table=table) }}\n {%- endif -%}\n\n {%- endif -%}\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496097, "supported_languages": null }, "macro.dbt.noop_statement": { "name": "noop_statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt.noop_statement", "macro_sql": "{% macro noop_statement(name=None, message=None, code=None, rows_affected=None, res=None) -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_raw_result(name, message=message, code=code, rows_affected=rows_affected, agate_table=res) }}\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4961002, "supported_languages": null }, "macro.dbt.run_query": { "name": "run_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt.run_query", "macro_sql": "{% macro run_query(sql) %}\n {% call statement(\"run_query_statement\", fetch_result=true, auto_begin=false) %}\n {{ sql }}\n {% endcall %}\n\n {% do return(load_result(\"run_query_statement\").table) %}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.statement" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4961028, "supported_languages": null }, "macro.dbt.convert_datetime": { "name": "convert_datetime", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.convert_datetime", "macro_sql": "{% macro convert_datetime(date_str, date_fmt) %}\n\n {% set error_msg -%}\n The provided partition date '{{ date_str }}' does not match the expected format '{{ date_fmt }}'\n {%- endset %}\n\n {% set res = try_or_compiler_error(error_msg, modules.datetime.datetime.strptime, date_str.strip(), date_fmt) %}\n {{ return(res) }}\n\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.49611, "supported_languages": null }, "macro.dbt.dates_in_range": { "name": "dates_in_range", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.dates_in_range", "macro_sql": "{% macro dates_in_range(start_date_str, end_date_str=none, in_fmt=\"%Y%m%d\", out_fmt=\"%Y%m%d\") %}\n {% set end_date_str = start_date_str if end_date_str is none else end_date_str %}\n\n {% set start_date = convert_datetime(start_date_str, in_fmt) %}\n {% set end_date = convert_datetime(end_date_str, in_fmt) %}\n\n {% set day_count = (end_date - start_date).days %}\n {% if day_count < 0 %}\n {% set msg -%}\n Partition start date is after the end date ({{ start_date }}, {{ end_date }})\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg, model) }}\n {% endif %}\n\n {% set date_list = [] %}\n {% for i in range(0, day_count + 1) %}\n {% set the_date = (modules.datetime.timedelta(days=i) + start_date) %}\n {% if not out_fmt %}\n {% set _ = date_list.append(the_date) %}\n {% else %}\n {% set _ = date_list.append(the_date.strftime(out_fmt)) %}\n {% endif %}\n {% endfor %}\n\n {{ return(date_list) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.convert_datetime" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4961128, "supported_languages": null }, "macro.dbt.partition_range": { "name": "partition_range", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.partition_range", "macro_sql": "{% macro partition_range(raw_partition_date, date_fmt='%Y%m%d') %}\n {% set partition_range = (raw_partition_date | string).split(\",\") %}\n\n {% if (partition_range | length) == 1 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = none %}\n {% elif (partition_range | length) == 2 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = partition_range[1] %}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Invalid partition time. Expected format: {Start Date}[,{End Date}]. Got: \" ~ raw_partition_date) }}\n {% endif %}\n\n {{ return(dates_in_range(start_date, end_date, in_fmt=date_fmt)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.dates_in_range" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496115, "supported_languages": null }, "macro.dbt.py_current_timestring": { "name": "py_current_timestring", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.py_current_timestring", "macro_sql": "{% macro py_current_timestring() %}\n {% set dt = modules.datetime.datetime.now() %}\n {% do return(dt.strftime(\"%Y%m%d%H%M%S%f\")) %}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4961169, "supported_languages": null }, "macro.dbt.except": { "name": "except", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/except.sql", "original_file_path": "macros/utils/except.sql", "unique_id": "macro.dbt.except", "macro_sql": "{% macro except() %}\n {{ return(adapter.dispatch('except', 'dbt')()) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__except" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4961221, "supported_languages": null }, "macro.dbt.default__except": { "name": "default__except", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/except.sql", "original_file_path": "macros/utils/except.sql", "unique_id": "macro.dbt.default__except", "macro_sql": "{% macro default__except() %}\n\n except\n\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4961238, "supported_languages": null }, "macro.dbt.get_intervals_between": { "name": "get_intervals_between", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_spine.sql", "original_file_path": "macros/utils/date_spine.sql", "unique_id": "macro.dbt.get_intervals_between", "macro_sql": "{% macro get_intervals_between(start_date, end_date, datepart) -%}\n {{ return(adapter.dispatch('get_intervals_between', 'dbt')(start_date, end_date, datepart)) }}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__get_intervals_between" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4961321, "supported_languages": null }, "macro.dbt.default__get_intervals_between": { "name": "default__get_intervals_between", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_spine.sql", "original_file_path": "macros/utils/date_spine.sql", "unique_id": "macro.dbt.default__get_intervals_between", "macro_sql": "{% macro default__get_intervals_between(start_date, end_date, datepart) -%}\n {%- call statement('get_intervals_between', fetch_result=True) %}\n\n select {{ dbt.datediff(start_date, end_date, datepart) }}\n\n {%- endcall -%}\n\n {%- set value_list = load_result('get_intervals_between') -%}\n\n {%- if value_list and value_list['data'] -%}\n {%- set values = value_list['data'] | map(attribute=0) | list %}\n {{ return(values[0]) }}\n {%- else -%}\n {{ return(1) }}\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.statement", "macro.dbt.datediff" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496139, "supported_languages": null }, "macro.dbt.date_spine": { "name": "date_spine", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_spine.sql", "original_file_path": "macros/utils/date_spine.sql", "unique_id": "macro.dbt.date_spine", "macro_sql": "{% macro date_spine(datepart, start_date, end_date) %}\n {{ return(adapter.dispatch('date_spine', 'dbt')(datepart, start_date, end_date)) }}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__date_spine" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496141, "supported_languages": null }, "macro.dbt.default__date_spine": { "name": "default__date_spine", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_spine.sql", "original_file_path": "macros/utils/date_spine.sql", "unique_id": "macro.dbt.default__date_spine", "macro_sql": "{% macro default__date_spine(datepart, start_date, end_date) %}\n\n\n {# call as follows:\n\n date_spine(\n \"day\",\n \"to_date('01/01/2016', 'mm/dd/yyyy')\",\n \"dbt.dateadd(week, 1, current_date)\"\n ) #}\n\n\n with rawdata as (\n\n {{dbt.generate_series(\n dbt.get_intervals_between(start_date, end_date, datepart)\n )}}\n\n ),\n\n all_periods as (\n\n select (\n {{\n dbt.dateadd(\n datepart,\n \"row_number() over (order by 1) - 1\",\n start_date\n )\n }}\n ) as date_{{datepart}}\n from rawdata\n\n ),\n\n filtered as (\n\n select *\n from all_periods\n where date_{{datepart}} <= {{ end_date }}\n\n )\n\n select * from filtered\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.generate_series", "macro.dbt.get_intervals_between", "macro.dbt.dateadd" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4961438, "supported_languages": null }, "macro.dbt.date": { "name": "date", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date.sql", "original_file_path": "macros/utils/date.sql", "unique_id": "macro.dbt.date", "macro_sql": "{% macro date(year, month, day) %}\n {{ return(adapter.dispatch('date', 'dbt') (year, month, day)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__date" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496149, "supported_languages": null }, "macro.dbt.default__date": { "name": "default__date", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date.sql", "original_file_path": "macros/utils/date.sql", "unique_id": "macro.dbt.default__date", "macro_sql": "{% macro default__date(year, month, day) -%}\n {%- set dt = modules.datetime.date(year, month, day) -%}\n {%- set iso_8601_formatted_date = dt.strftime('%Y-%m-%d') -%}\n to_date('{{ iso_8601_formatted_date }}', 'YYYY-MM-DD')\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496151, "supported_languages": null }, "macro.dbt.replace": { "name": "replace", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/replace.sql", "original_file_path": "macros/utils/replace.sql", "unique_id": "macro.dbt.replace", "macro_sql": "{% macro replace(field, old_chars, new_chars) -%}\n {{ return(adapter.dispatch('replace', 'dbt') (field, old_chars, new_chars)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__replace" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496158, "supported_languages": null }, "macro.dbt.default__replace": { "name": "default__replace", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/replace.sql", "original_file_path": "macros/utils/replace.sql", "unique_id": "macro.dbt.default__replace", "macro_sql": "{% macro default__replace(field, old_chars, new_chars) %}\n\n replace(\n {{ field }},\n {{ old_chars }},\n {{ new_chars }}\n )\n\n\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496161, "supported_languages": null }, "macro.dbt.concat": { "name": "concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/concat.sql", "original_file_path": "macros/utils/concat.sql", "unique_id": "macro.dbt.concat", "macro_sql": "{% macro concat(fields) -%}\n {{ return(adapter.dispatch('concat', 'dbt')(fields)) }}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__concat" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496165, "supported_languages": null }, "macro.dbt.default__concat": { "name": "default__concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/concat.sql", "original_file_path": "macros/utils/concat.sql", "unique_id": "macro.dbt.default__concat", "macro_sql": "{% macro default__concat(fields) -%}\n {{ fields|join(' || ') }}\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496167, "supported_languages": null }, "macro.dbt.get_powers_of_two": { "name": "get_powers_of_two", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/generate_series.sql", "original_file_path": "macros/utils/generate_series.sql", "unique_id": "macro.dbt.get_powers_of_two", "macro_sql": "{% macro get_powers_of_two(upper_bound) %}\n {{ return(adapter.dispatch('get_powers_of_two', 'dbt')(upper_bound)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__get_powers_of_two" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496172, "supported_languages": null }, "macro.dbt.default__get_powers_of_two": { "name": "default__get_powers_of_two", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/generate_series.sql", "original_file_path": "macros/utils/generate_series.sql", "unique_id": "macro.dbt.default__get_powers_of_two", "macro_sql": "{% macro default__get_powers_of_two(upper_bound) %}\n\n {% if upper_bound <= 0 %}\n {{ exceptions.raise_compiler_error(\"upper bound must be positive\") }}\n {% endif %}\n\n {% for _ in range(1, 100) %}\n {% if upper_bound <= 2 ** loop.index %}{{ return(loop.index) }}{% endif %}\n {% endfor %}\n\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496174, "supported_languages": null }, "macro.dbt.generate_series": { "name": "generate_series", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/generate_series.sql", "original_file_path": "macros/utils/generate_series.sql", "unique_id": "macro.dbt.generate_series", "macro_sql": "{% macro generate_series(upper_bound) %}\n {{ return(adapter.dispatch('generate_series', 'dbt')(upper_bound)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__generate_series" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.49618, "supported_languages": null }, "macro.dbt.default__generate_series": { "name": "default__generate_series", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/generate_series.sql", "original_file_path": "macros/utils/generate_series.sql", "unique_id": "macro.dbt.default__generate_series", "macro_sql": "{% macro default__generate_series(upper_bound) %}\n\n {% set n = dbt.get_powers_of_two(upper_bound) %}\n\n with p as (\n select 0 as generated_number union all select 1\n ), unioned as (\n\n select\n\n {% for i in range(n) %}\n p{{i}}.generated_number * power(2, {{i}})\n {% if not loop.last %} + {% endif %}\n {% endfor %}\n + 1\n as generated_number\n\n from\n\n {% for i in range(n) %}\n p as p{{i}}\n {% if not loop.last %} cross join {% endif %}\n {% endfor %}\n\n )\n\n select *\n from unioned\n where generated_number <= {{upper_bound}}\n order by generated_number\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.get_powers_of_two" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496184, "supported_languages": null }, "macro.dbt.length": { "name": "length", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/length.sql", "original_file_path": "macros/utils/length.sql", "unique_id": "macro.dbt.length", "macro_sql": "{% macro length(expression) -%}\n {{ return(adapter.dispatch('length', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__length" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496188, "supported_languages": null }, "macro.dbt.default__length": { "name": "default__length", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/length.sql", "original_file_path": "macros/utils/length.sql", "unique_id": "macro.dbt.default__length", "macro_sql": "{% macro default__length(expression) %}\n\n length(\n {{ expression }}\n )\n\n{%- endmacro -%}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.49619, "supported_languages": null }, "macro.dbt.dateadd": { "name": "dateadd", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt.dateadd", "macro_sql": "{% macro dateadd(datepart, interval, from_date_or_timestamp) %}\n {{ return(adapter.dispatch('dateadd', 'dbt')(datepart, interval, from_date_or_timestamp)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__dateadd" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496195, "supported_languages": null }, "macro.dbt.default__dateadd": { "name": "default__dateadd", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt.default__dateadd", "macro_sql": "{% macro default__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n dateadd(\n {{ datepart }},\n {{ interval }},\n {{ from_date_or_timestamp }}\n )\n\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4961972, "supported_languages": null }, "macro.dbt.intersect": { "name": "intersect", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/intersect.sql", "original_file_path": "macros/utils/intersect.sql", "unique_id": "macro.dbt.intersect", "macro_sql": "{% macro intersect() %}\n {{ return(adapter.dispatch('intersect', 'dbt')()) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__intersect" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496201, "supported_languages": null }, "macro.dbt.default__intersect": { "name": "default__intersect", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/intersect.sql", "original_file_path": "macros/utils/intersect.sql", "unique_id": "macro.dbt.default__intersect", "macro_sql": "{% macro default__intersect() %}\n\n intersect\n\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496203, "supported_languages": null }, "macro.dbt.escape_single_quotes": { "name": "escape_single_quotes", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/escape_single_quotes.sql", "original_file_path": "macros/utils/escape_single_quotes.sql", "unique_id": "macro.dbt.escape_single_quotes", "macro_sql": "{% macro escape_single_quotes(expression) %}\n {{ return(adapter.dispatch('escape_single_quotes', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__escape_single_quotes" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496208, "supported_languages": null }, "macro.dbt.default__escape_single_quotes": { "name": "default__escape_single_quotes", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/escape_single_quotes.sql", "original_file_path": "macros/utils/escape_single_quotes.sql", "unique_id": "macro.dbt.default__escape_single_quotes", "macro_sql": "{% macro default__escape_single_quotes(expression) -%}\n{{ expression | replace(\"'\",\"''\") }}\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4962099, "supported_languages": null }, "macro.dbt.right": { "name": "right", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/right.sql", "original_file_path": "macros/utils/right.sql", "unique_id": "macro.dbt.right", "macro_sql": "{% macro right(string_text, length_expression) -%}\n {{ return(adapter.dispatch('right', 'dbt') (string_text, length_expression)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__right" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496214, "supported_languages": null }, "macro.dbt.default__right": { "name": "default__right", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/right.sql", "original_file_path": "macros/utils/right.sql", "unique_id": "macro.dbt.default__right", "macro_sql": "{% macro default__right(string_text, length_expression) %}\n\n right(\n {{ string_text }},\n {{ length_expression }}\n )\n\n{%- endmacro -%}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496217, "supported_languages": null }, "macro.dbt.listagg": { "name": "listagg", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "unique_id": "macro.dbt.listagg", "macro_sql": "{% macro listagg(measure, delimiter_text=\"','\", order_by_clause=none, limit_num=none) -%}\n {{ return(adapter.dispatch('listagg', 'dbt') (measure, delimiter_text, order_by_clause, limit_num)) }}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__listagg" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496222, "supported_languages": null }, "macro.dbt.default__listagg": { "name": "default__listagg", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "unique_id": "macro.dbt.default__listagg", "macro_sql": "{% macro default__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n {% if limit_num -%}\n array_to_string(\n array_slice(\n array_agg(\n {{ measure }}\n ){% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n ,0\n ,{{ limit_num }}\n ),\n {{ delimiter_text }}\n )\n {%- else %}\n listagg(\n {{ measure }},\n {{ delimiter_text }}\n )\n {% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n {%- endif %}\n\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496226, "supported_languages": null }, "macro.dbt.datediff": { "name": "datediff", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt.datediff", "macro_sql": "{% macro datediff(first_date, second_date, datepart) %}\n {{ return(adapter.dispatch('datediff', 'dbt')(first_date, second_date, datepart)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__datediff" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496232, "supported_languages": null }, "macro.dbt.default__datediff": { "name": "default__datediff", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt.default__datediff", "macro_sql": "{% macro default__datediff(first_date, second_date, datepart) -%}\n\n datediff(\n {{ datepart }},\n {{ first_date }},\n {{ second_date }}\n )\n\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496234, "supported_languages": null }, "macro.dbt.safe_cast": { "name": "safe_cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "unique_id": "macro.dbt.safe_cast", "macro_sql": "{% macro safe_cast(field, type) %}\n {{ return(adapter.dispatch('safe_cast', 'dbt') (field, type)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__safe_cast" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4962418, "supported_languages": null }, "macro.dbt.default__safe_cast": { "name": "default__safe_cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "unique_id": "macro.dbt.default__safe_cast", "macro_sql": "{% macro default__safe_cast(field, type) %}\n {# most databases don't support this function yet\n so we just need to use cast #}\n cast({{field}} as {{type}})\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496245, "supported_languages": null }, "macro.dbt.equals": { "name": "equals", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/equals.sql", "original_file_path": "macros/utils/equals.sql", "unique_id": "macro.dbt.equals", "macro_sql": "{% macro equals(expr1, expr2) %}\n {{ return(adapter.dispatch('equals', 'dbt') (expr1, expr2)) }}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__equals" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4962492, "supported_languages": null }, "macro.dbt.default__equals": { "name": "default__equals", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/equals.sql", "original_file_path": "macros/utils/equals.sql", "unique_id": "macro.dbt.default__equals", "macro_sql": "{% macro default__equals(expr1, expr2) -%}\n{%- if adapter.behavior.enable_truthy_nulls_equals_macro.no_warn %}\n case when (({{ expr1 }} = {{ expr2 }}) or ({{ expr1 }} is null and {{ expr2 }} is null))\n then 0\n else 1\n end = 0\n{%- else -%}\n ({{ expr1 }} = {{ expr2 }})\n{%- endif %}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4962518, "supported_languages": null }, "macro.dbt.hash": { "name": "hash", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/hash.sql", "original_file_path": "macros/utils/hash.sql", "unique_id": "macro.dbt.hash", "macro_sql": "{% macro hash(field) -%}\n {{ return(adapter.dispatch('hash', 'dbt') (field)) }}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__hash" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496257, "supported_languages": null }, "macro.dbt.default__hash": { "name": "default__hash", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/hash.sql", "original_file_path": "macros/utils/hash.sql", "unique_id": "macro.dbt.default__hash", "macro_sql": "{% macro default__hash(field) -%}\n md5(cast({{ field }} as {{ api.Column.translate_type('string') }}))\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496259, "supported_languages": null }, "macro.dbt.cast_bool_to_text": { "name": "cast_bool_to_text", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast_bool_to_text.sql", "original_file_path": "macros/utils/cast_bool_to_text.sql", "unique_id": "macro.dbt.cast_bool_to_text", "macro_sql": "{% macro cast_bool_to_text(field) %}\n {{ adapter.dispatch('cast_bool_to_text', 'dbt') (field) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__cast_bool_to_text" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496264, "supported_languages": null }, "macro.dbt.default__cast_bool_to_text": { "name": "default__cast_bool_to_text", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast_bool_to_text.sql", "original_file_path": "macros/utils/cast_bool_to_text.sql", "unique_id": "macro.dbt.default__cast_bool_to_text", "macro_sql": "{% macro default__cast_bool_to_text(field) %}\n cast({{ field }} as {{ api.Column.translate_type('string') }})\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496267, "supported_languages": null }, "macro.dbt.cast": { "name": "cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast.sql", "original_file_path": "macros/utils/cast.sql", "unique_id": "macro.dbt.cast", "macro_sql": "{% macro cast(field, type) %}\n {{ return(adapter.dispatch('cast', 'dbt') (field, type)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__cast" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.49629, "supported_languages": null }, "macro.dbt.default__cast": { "name": "default__cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast.sql", "original_file_path": "macros/utils/cast.sql", "unique_id": "macro.dbt.default__cast", "macro_sql": "{% macro default__cast(field, type) %}\n cast({{field}} as {{type}})\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496292, "supported_languages": null }, "macro.dbt.any_value": { "name": "any_value", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "unique_id": "macro.dbt.any_value", "macro_sql": "{% macro any_value(expression) -%}\n {{ return(adapter.dispatch('any_value', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__any_value" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496297, "supported_languages": null }, "macro.dbt.default__any_value": { "name": "default__any_value", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "unique_id": "macro.dbt.default__any_value", "macro_sql": "{% macro default__any_value(expression) -%}\n\n any_value({{ expression }})\n\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496299, "supported_languages": null }, "macro.dbt.position": { "name": "position", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/position.sql", "original_file_path": "macros/utils/position.sql", "unique_id": "macro.dbt.position", "macro_sql": "{% macro position(substring_text, string_text) -%}\n {{ return(adapter.dispatch('position', 'dbt') (substring_text, string_text)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__position" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496305, "supported_languages": null }, "macro.dbt.default__position": { "name": "default__position", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/position.sql", "original_file_path": "macros/utils/position.sql", "unique_id": "macro.dbt.default__position", "macro_sql": "{% macro default__position(substring_text, string_text) %}\n\n position(\n {{ substring_text }} in {{ string_text }}\n )\n\n{%- endmacro -%}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496307, "supported_languages": null }, "macro.dbt.string_literal": { "name": "string_literal", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/literal.sql", "original_file_path": "macros/utils/literal.sql", "unique_id": "macro.dbt.string_literal", "macro_sql": "{%- macro string_literal(value) -%}\n {{ return(adapter.dispatch('string_literal', 'dbt') (value)) }}\n{%- endmacro -%}\n\n", "depends_on": { "macros": [ "macro.dbt.default__string_literal" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496311, "supported_languages": null }, "macro.dbt.default__string_literal": { "name": "default__string_literal", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/literal.sql", "original_file_path": "macros/utils/literal.sql", "unique_id": "macro.dbt.default__string_literal", "macro_sql": "{% macro default__string_literal(value) -%}\n '{{ value }}'\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4963148, "supported_languages": null }, "macro.dbt.type_string": { "name": "type_string", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_string", "macro_sql": "\n\n{%- macro type_string() -%}\n {{ return(adapter.dispatch('type_string', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": { "macros": [ "macro.dbt.default__type_string" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4963222, "supported_languages": null }, "macro.dbt.default__type_string": { "name": "default__type_string", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_string", "macro_sql": "{% macro default__type_string() %}\n {{ return(api.Column.translate_type(\"string\")) }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496324, "supported_languages": null }, "macro.dbt.type_timestamp": { "name": "type_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_timestamp", "macro_sql": "\n\n{%- macro type_timestamp() -%}\n {{ return(adapter.dispatch('type_timestamp', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": { "macros": [ "macro.dbt.default__type_timestamp" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496326, "supported_languages": null }, "macro.dbt.default__type_timestamp": { "name": "default__type_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_timestamp", "macro_sql": "{% macro default__type_timestamp() %}\n {{ return(api.Column.translate_type(\"timestamp\")) }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496331, "supported_languages": null }, "macro.dbt.type_float": { "name": "type_float", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_float", "macro_sql": "\n\n{%- macro type_float() -%}\n {{ return(adapter.dispatch('type_float', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": { "macros": [ "macro.dbt.default__type_float" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4963331, "supported_languages": null }, "macro.dbt.default__type_float": { "name": "default__type_float", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_float", "macro_sql": "{% macro default__type_float() %}\n {{ return(api.Column.translate_type(\"float\")) }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496335, "supported_languages": null }, "macro.dbt.type_numeric": { "name": "type_numeric", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_numeric", "macro_sql": "\n\n{%- macro type_numeric() -%}\n {{ return(adapter.dispatch('type_numeric', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": { "macros": [ "macro.dbt.default__type_numeric" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496342, "supported_languages": null }, "macro.dbt.default__type_numeric": { "name": "default__type_numeric", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_numeric", "macro_sql": "{% macro default__type_numeric() %}\n {{ return(api.Column.numeric_type(\"numeric\", 28, 6)) }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496344, "supported_languages": null }, "macro.dbt.type_bigint": { "name": "type_bigint", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_bigint", "macro_sql": "\n\n{%- macro type_bigint() -%}\n {{ return(adapter.dispatch('type_bigint', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": { "macros": [ "macro.dbt.default__type_bigint" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496346, "supported_languages": null }, "macro.dbt.default__type_bigint": { "name": "default__type_bigint", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_bigint", "macro_sql": "{% macro default__type_bigint() %}\n {{ return(api.Column.translate_type(\"bigint\")) }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4963489, "supported_languages": null }, "macro.dbt.type_int": { "name": "type_int", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_int", "macro_sql": "\n\n{%- macro type_int() -%}\n {{ return(adapter.dispatch('type_int', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": { "macros": [ "macro.dbt.default__type_int" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496351, "supported_languages": null }, "macro.dbt.default__type_int": { "name": "default__type_int", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_int", "macro_sql": "{%- macro default__type_int() -%}\n {{ return(api.Column.translate_type(\"integer\")) }}\n{%- endmacro -%}\n\n", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496354, "supported_languages": null }, "macro.dbt.type_boolean": { "name": "type_boolean", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_boolean", "macro_sql": "\n\n{%- macro type_boolean() -%}\n {{ return(adapter.dispatch('type_boolean', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": { "macros": [ "macro.dbt.default__type_boolean" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496356, "supported_languages": null }, "macro.dbt.default__type_boolean": { "name": "default__type_boolean", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_boolean", "macro_sql": "{%- macro default__type_boolean() -%}\n {{ return(api.Column.translate_type(\"boolean\")) }}\n{%- endmacro -%}\n\n", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4963589, "supported_languages": null }, "macro.dbt.array_concat": { "name": "array_concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_concat.sql", "original_file_path": "macros/utils/array_concat.sql", "unique_id": "macro.dbt.array_concat", "macro_sql": "{% macro array_concat(array_1, array_2) -%}\n {{ return(adapter.dispatch('array_concat', 'dbt')(array_1, array_2)) }}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__array_concat" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496364, "supported_languages": null }, "macro.dbt.default__array_concat": { "name": "default__array_concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_concat.sql", "original_file_path": "macros/utils/array_concat.sql", "unique_id": "macro.dbt.default__array_concat", "macro_sql": "{% macro default__array_concat(array_1, array_2) -%}\n array_cat({{ array_1 }}, {{ array_2 }})\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496366, "supported_languages": null }, "macro.dbt.bool_or": { "name": "bool_or", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/bool_or.sql", "original_file_path": "macros/utils/bool_or.sql", "unique_id": "macro.dbt.bool_or", "macro_sql": "{% macro bool_or(expression) -%}\n {{ return(adapter.dispatch('bool_or', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__bool_or" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496372, "supported_languages": null }, "macro.dbt.default__bool_or": { "name": "default__bool_or", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/bool_or.sql", "original_file_path": "macros/utils/bool_or.sql", "unique_id": "macro.dbt.default__bool_or", "macro_sql": "{% macro default__bool_or(expression) -%}\n\n bool_or({{ expression }})\n\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4963741, "supported_languages": null }, "macro.dbt.last_day": { "name": "last_day", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt.last_day", "macro_sql": "{% macro last_day(date, datepart) %}\n {{ return(adapter.dispatch('last_day', 'dbt') (date, datepart)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__last_day" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496379, "supported_languages": null }, "macro.dbt.default_last_day": { "name": "default_last_day", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt.default_last_day", "macro_sql": "\n\n{%- macro default_last_day(date, datepart) -%}\n cast(\n {{dbt.dateadd('day', '-1',\n dbt.dateadd(datepart, '1', dbt.date_trunc(datepart, date))\n )}}\n as date)\n{%- endmacro -%}\n\n", "depends_on": { "macros": [ "macro.dbt.dateadd", "macro.dbt.date_trunc" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496383, "supported_languages": null }, "macro.dbt.default__last_day": { "name": "default__last_day", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt.default__last_day", "macro_sql": "{% macro default__last_day(date, datepart) -%}\n {{dbt.default_last_day(date, datepart)}}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.default_last_day" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496385, "supported_languages": null }, "macro.dbt.split_part": { "name": "split_part", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt.split_part", "macro_sql": "{% macro split_part(string_text, delimiter_text, part_number) %}\n {{ return(adapter.dispatch('split_part', 'dbt') (string_text, delimiter_text, part_number)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__split_part" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4963892, "supported_languages": null }, "macro.dbt.default__split_part": { "name": "default__split_part", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt.default__split_part", "macro_sql": "{% macro default__split_part(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n {{ part_number }}\n )\n\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4963908, "supported_languages": null }, "macro.dbt._split_part_negative": { "name": "_split_part_negative", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt._split_part_negative", "macro_sql": "{% macro _split_part_negative(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n length({{ string_text }})\n - length(\n replace({{ string_text }}, {{ delimiter_text }}, '')\n ) + 2 + {{ part_number }}\n )\n\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496397, "supported_languages": null }, "macro.dbt.date_trunc": { "name": "date_trunc", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_trunc.sql", "original_file_path": "macros/utils/date_trunc.sql", "unique_id": "macro.dbt.date_trunc", "macro_sql": "{% macro date_trunc(datepart, date) -%}\n {{ return(adapter.dispatch('date_trunc', 'dbt') (datepart, date)) }}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__date_trunc" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496402, "supported_languages": null }, "macro.dbt.default__date_trunc": { "name": "default__date_trunc", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_trunc.sql", "original_file_path": "macros/utils/date_trunc.sql", "unique_id": "macro.dbt.default__date_trunc", "macro_sql": "{% macro default__date_trunc(datepart, date) -%}\n date_trunc('{{datepart}}', {{date}})\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496404, "supported_languages": null }, "macro.dbt.array_construct": { "name": "array_construct", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_construct.sql", "original_file_path": "macros/utils/array_construct.sql", "unique_id": "macro.dbt.array_construct", "macro_sql": "{% macro array_construct(inputs=[], data_type=api.Column.translate_type('integer')) -%}\n {{ return(adapter.dispatch('array_construct', 'dbt')(inputs, data_type)) }}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__array_construct" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4964101, "supported_languages": null }, "macro.dbt.default__array_construct": { "name": "default__array_construct", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_construct.sql", "original_file_path": "macros/utils/array_construct.sql", "unique_id": "macro.dbt.default__array_construct", "macro_sql": "{% macro default__array_construct(inputs, data_type) -%}\n {% if inputs|length > 0 %}\n array[ {{ inputs|join(' , ') }} ]\n {% else %}\n array[]::{{data_type}}[]\n {% endif %}\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4964118, "supported_languages": null }, "macro.dbt.array_append": { "name": "array_append", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_append.sql", "original_file_path": "macros/utils/array_append.sql", "unique_id": "macro.dbt.array_append", "macro_sql": "{% macro array_append(array, new_element) -%}\n {{ return(adapter.dispatch('array_append', 'dbt')(array, new_element)) }}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__array_append" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496416, "supported_languages": null }, "macro.dbt.default__array_append": { "name": "default__array_append", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_append.sql", "original_file_path": "macros/utils/array_append.sql", "unique_id": "macro.dbt.default__array_append", "macro_sql": "{% macro default__array_append(array, new_element) -%}\n array_append({{ array }}, {{ new_element }})\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496418, "supported_languages": null }, "macro.dbt.create_schema": { "name": "create_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.create_schema", "macro_sql": "{% macro create_schema(relation) -%}\n {{ adapter.dispatch('create_schema', 'dbt')(relation) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__create_schema" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496423, "supported_languages": null }, "macro.dbt.default__create_schema": { "name": "default__create_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.default__create_schema", "macro_sql": "{% macro default__create_schema(relation) -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier() }}\n {% endcall %}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.statement" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496425, "supported_languages": null }, "macro.dbt.drop_schema": { "name": "drop_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.drop_schema", "macro_sql": "{% macro drop_schema(relation) -%}\n {{ adapter.dispatch('drop_schema', 'dbt')(relation) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__drop_schema" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496435, "supported_languages": null }, "macro.dbt.default__drop_schema": { "name": "default__drop_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.default__drop_schema", "macro_sql": "{% macro default__drop_schema(relation) -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier() }} cascade\n {% endcall %}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.statement" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496437, "supported_languages": null }, "macro.dbt.current_timestamp": { "name": "current_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.current_timestamp", "macro_sql": "{%- macro current_timestamp() -%}\n {{ adapter.dispatch('current_timestamp', 'dbt')() }}\n{%- endmacro -%}\n\n", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__current_timestamp" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496442, "supported_languages": null }, "macro.dbt.default__current_timestamp": { "name": "default__current_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__current_timestamp", "macro_sql": "{% macro default__current_timestamp() -%}\n {{ exceptions.raise_not_implemented(\n 'current_timestamp macro not implemented for adapter ' + adapter.type()) }}\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496445, "supported_languages": null }, "macro.dbt.snapshot_get_time": { "name": "snapshot_get_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.snapshot_get_time", "macro_sql": "\n\n{%- macro snapshot_get_time() -%}\n {{ adapter.dispatch('snapshot_get_time', 'dbt')() }}\n{%- endmacro -%}\n\n", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__snapshot_get_time" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496447, "supported_languages": null }, "macro.dbt.default__snapshot_get_time": { "name": "default__snapshot_get_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__snapshot_get_time", "macro_sql": "{% macro default__snapshot_get_time() %}\n {{ current_timestamp() }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.current_timestamp" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496449, "supported_languages": null }, "macro.dbt.get_snapshot_get_time_data_type": { "name": "get_snapshot_get_time_data_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.get_snapshot_get_time_data_type", "macro_sql": "{% macro get_snapshot_get_time_data_type() %}\n {% set snapshot_time = adapter.dispatch('snapshot_get_time', 'dbt')() %}\n {% set time_data_type_sql = 'select ' ~ snapshot_time ~ ' as dbt_snapshot_time' %}\n {% set snapshot_time_column_schema = get_column_schema_from_query(time_data_type_sql) %}\n {% set time_data_type = snapshot_time_column_schema[0].dtype %}\n {{ return(time_data_type or none) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.snapshot_get_time", "macro.dbt_postgres.postgres__snapshot_get_time", "macro.dbt.get_column_schema_from_query" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4964511, "supported_languages": null }, "macro.dbt.current_timestamp_backcompat": { "name": "current_timestamp_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.current_timestamp_backcompat", "macro_sql": "{% macro current_timestamp_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_backcompat', 'dbt')()) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__current_timestamp_backcompat" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496452, "supported_languages": null }, "macro.dbt.default__current_timestamp_backcompat": { "name": "default__current_timestamp_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__current_timestamp_backcompat", "macro_sql": "{% macro default__current_timestamp_backcompat() %}\n current_timestamp::timestamp\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496454, "supported_languages": null }, "macro.dbt.current_timestamp_in_utc_backcompat": { "name": "current_timestamp_in_utc_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.current_timestamp_in_utc_backcompat", "macro_sql": "{% macro current_timestamp_in_utc_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_in_utc_backcompat', 'dbt')()) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__current_timestamp_in_utc_backcompat" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496458, "supported_languages": null }, "macro.dbt.default__current_timestamp_in_utc_backcompat": { "name": "default__current_timestamp_in_utc_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__current_timestamp_in_utc_backcompat", "macro_sql": "{% macro default__current_timestamp_in_utc_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_backcompat', 'dbt')()) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.current_timestamp_backcompat", "macro.dbt_postgres.postgres__current_timestamp_backcompat" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.49646, "supported_languages": null }, "macro.dbt.get_create_index_sql": { "name": "get_create_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.get_create_index_sql", "macro_sql": "{% macro get_create_index_sql(relation, index_dict) -%}\n {{ return(adapter.dispatch('get_create_index_sql', 'dbt')(relation, index_dict)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_create_index_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4964669, "supported_languages": null }, "macro.dbt.default__get_create_index_sql": { "name": "default__get_create_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__get_create_index_sql", "macro_sql": "{% macro default__get_create_index_sql(relation, index_dict) -%}\n {% do return(None) %}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496469, "supported_languages": null }, "macro.dbt.create_indexes": { "name": "create_indexes", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.create_indexes", "macro_sql": "{% macro create_indexes(relation) -%}\n {{ adapter.dispatch('create_indexes', 'dbt')(relation) }}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__create_indexes" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496471, "supported_languages": null }, "macro.dbt.default__create_indexes": { "name": "default__create_indexes", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__create_indexes", "macro_sql": "{% macro default__create_indexes(relation) -%}\n {%- set _indexes = config.get('indexes', default=[]) -%}\n\n {% for _index_dict in _indexes %}\n {% set create_index_sql = get_create_index_sql(relation, _index_dict) %}\n {% if create_index_sql %}\n {% do run_query(create_index_sql) %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.get_create_index_sql", "macro.dbt.run_query" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496473, "supported_languages": null }, "macro.dbt.get_drop_index_sql": { "name": "get_drop_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.get_drop_index_sql", "macro_sql": "{% macro get_drop_index_sql(relation, index_name) -%}\n {{ adapter.dispatch('get_drop_index_sql', 'dbt')(relation, index_name) }}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_drop_index_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496475, "supported_languages": null }, "macro.dbt.default__get_drop_index_sql": { "name": "default__get_drop_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__get_drop_index_sql", "macro_sql": "{% macro default__get_drop_index_sql(relation, index_name) -%}\n {{ exceptions.raise_compiler_error(\"`get_drop_index_sql has not been implemented for this adapter.\") }}\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496477, "supported_languages": null }, "macro.dbt.get_show_indexes_sql": { "name": "get_show_indexes_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.get_show_indexes_sql", "macro_sql": "{% macro get_show_indexes_sql(relation) -%}\n {{ adapter.dispatch('get_show_indexes_sql', 'dbt')(relation) }}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_show_indexes_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496479, "supported_languages": null }, "macro.dbt.default__get_show_indexes_sql": { "name": "default__get_show_indexes_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__get_show_indexes_sql", "macro_sql": "{% macro default__get_show_indexes_sql(relation) -%}\n {{ exceptions.raise_compiler_error(\"`get_show_indexes_sql has not been implemented for this adapter.\") }}\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496481, "supported_languages": null }, "macro.dbt.make_intermediate_relation": { "name": "make_intermediate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.make_intermediate_relation", "macro_sql": "{% macro make_intermediate_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_intermediate_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__make_intermediate_relation" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496489, "supported_languages": null }, "macro.dbt.default__make_intermediate_relation": { "name": "default__make_intermediate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__make_intermediate_relation", "macro_sql": "{% macro default__make_intermediate_relation(base_relation, suffix) %}\n {{ return(default__make_temp_relation(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__make_temp_relation" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496491, "supported_languages": null }, "macro.dbt.make_temp_relation": { "name": "make_temp_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.make_temp_relation", "macro_sql": "{% macro make_temp_relation(base_relation, suffix='__dbt_tmp') %}\n {#-- This ensures microbatch batches get unique temp relations to avoid clobbering --#}\n {% if suffix == '__dbt_tmp' and model.batch %}\n {% set suffix = suffix ~ '_' ~ model.batch.id %}\n {% endif %}\n\n {{ return(adapter.dispatch('make_temp_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__make_temp_relation" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496495, "supported_languages": null }, "macro.dbt.default__make_temp_relation": { "name": "default__make_temp_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__make_temp_relation", "macro_sql": "{% macro default__make_temp_relation(base_relation, suffix) %}\n {%- set temp_identifier = base_relation.identifier ~ suffix -%}\n {%- set temp_relation = base_relation.incorporate(\n path={\"identifier\": temp_identifier}) -%}\n\n {{ return(temp_relation) }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496497, "supported_languages": null }, "macro.dbt.make_backup_relation": { "name": "make_backup_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.make_backup_relation", "macro_sql": "{% macro make_backup_relation(base_relation, backup_relation_type, suffix='__dbt_backup') %}\n {{ return(adapter.dispatch('make_backup_relation', 'dbt')(base_relation, backup_relation_type, suffix)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__make_backup_relation" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496499, "supported_languages": null }, "macro.dbt.default__make_backup_relation": { "name": "default__make_backup_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__make_backup_relation", "macro_sql": "{% macro default__make_backup_relation(base_relation, backup_relation_type, suffix) %}\n {%- set backup_identifier = base_relation.identifier ~ suffix -%}\n {%- set backup_relation = base_relation.incorporate(\n path={\"identifier\": backup_identifier},\n type=backup_relation_type\n ) -%}\n {{ return(backup_relation) }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496501, "supported_languages": null }, "macro.dbt.truncate_relation": { "name": "truncate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.truncate_relation", "macro_sql": "{% macro truncate_relation(relation) -%}\n {{ return(adapter.dispatch('truncate_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__truncate_relation" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496503, "supported_languages": null }, "macro.dbt.default__truncate_relation": { "name": "default__truncate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__truncate_relation", "macro_sql": "{% macro default__truncate_relation(relation) -%}\n {% call statement('truncate_relation') -%}\n truncate table {{ relation.render() }}\n {%- endcall %}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.statement" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496505, "supported_languages": null }, "macro.dbt.get_or_create_relation": { "name": "get_or_create_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.get_or_create_relation", "macro_sql": "{% macro get_or_create_relation(database, schema, identifier, type) -%}\n {{ return(adapter.dispatch('get_or_create_relation', 'dbt')(database, schema, identifier, type)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__get_or_create_relation" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496507, "supported_languages": null }, "macro.dbt.default__get_or_create_relation": { "name": "default__get_or_create_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__get_or_create_relation", "macro_sql": "{% macro default__get_or_create_relation(database, schema, identifier, type) %}\n {%- set target_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n\n {% if target_relation %}\n {% do return([true, target_relation]) %}\n {% endif %}\n\n {%- set new_relation = api.Relation.create(\n database=database,\n schema=schema,\n identifier=identifier,\n type=type\n ) -%}\n {% do return([false, new_relation]) %}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4965088, "supported_languages": null }, "macro.dbt.load_cached_relation": { "name": "load_cached_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.load_cached_relation", "macro_sql": "{% macro load_cached_relation(relation) %}\n {% do return(adapter.get_relation(\n database=relation.database,\n schema=relation.schema,\n identifier=relation.identifier\n )) -%}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496511, "supported_languages": null }, "macro.dbt.load_relation": { "name": "load_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.load_relation", "macro_sql": "{% macro load_relation(relation) %}\n {{ return(load_cached_relation(relation)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.load_cached_relation" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496512, "supported_languages": null }, "macro.dbt.collect_freshness": { "name": "collect_freshness", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "unique_id": "macro.dbt.collect_freshness", "macro_sql": "{% macro collect_freshness(source, loaded_at_field, filter) %}\n {{ return(adapter.dispatch('collect_freshness', 'dbt')(source, loaded_at_field, filter))}}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__collect_freshness" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4965189, "supported_languages": null }, "macro.dbt.default__collect_freshness": { "name": "default__collect_freshness", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "unique_id": "macro.dbt.default__collect_freshness", "macro_sql": "{% macro default__collect_freshness(source, loaded_at_field, filter) %}\n {% call statement('collect_freshness', fetch_result=True, auto_begin=False) -%}\n select\n max({{ loaded_at_field }}) as max_loaded_at,\n {{ current_timestamp() }} as snapshotted_at\n from {{ source }}\n {% if filter %}\n where {{ filter }}\n {% endif %}\n {% endcall %}\n {{ return(load_result('collect_freshness')) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.statement", "macro.dbt.current_timestamp" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496521, "supported_languages": null }, "macro.dbt.collect_freshness_custom_sql": { "name": "collect_freshness_custom_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "unique_id": "macro.dbt.collect_freshness_custom_sql", "macro_sql": "{% macro collect_freshness_custom_sql(source, loaded_at_query) %}\n {{ return(adapter.dispatch('collect_freshness_custom_sql', 'dbt')(source, loaded_at_query))}}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__collect_freshness_custom_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496523, "supported_languages": null }, "macro.dbt.default__collect_freshness_custom_sql": { "name": "default__collect_freshness_custom_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "unique_id": "macro.dbt.default__collect_freshness_custom_sql", "macro_sql": "{% macro default__collect_freshness_custom_sql(source, loaded_at_query) %}\n {% call statement('collect_freshness_custom_sql', fetch_result=True, auto_begin=False) -%}\n with source_query as (\n {{ loaded_at_query }}\n )\n select\n (select * from source_query) as max_loaded_at,\n {{ current_timestamp() }} as snapshotted_at\n {% endcall %}\n {{ return(load_result('collect_freshness_custom_sql')) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.statement", "macro.dbt.current_timestamp" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496525, "supported_languages": null }, "macro.dbt.validate_sql": { "name": "validate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/validate_sql.sql", "original_file_path": "macros/adapters/validate_sql.sql", "unique_id": "macro.dbt.validate_sql", "macro_sql": "{% macro validate_sql(sql) -%}\n {{ return(adapter.dispatch('validate_sql', 'dbt')(sql)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__validate_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4965298, "supported_languages": null }, "macro.dbt.default__validate_sql": { "name": "default__validate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/validate_sql.sql", "original_file_path": "macros/adapters/validate_sql.sql", "unique_id": "macro.dbt.default__validate_sql", "macro_sql": "{% macro default__validate_sql(sql) -%}\n {% call statement('validate_sql') -%}\n explain {{ sql }}\n {% endcall %}\n {{ return(load_result('validate_sql')) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.statement" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496532, "supported_languages": null }, "macro.dbt.copy_grants": { "name": "copy_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.copy_grants", "macro_sql": "{% macro copy_grants() %}\n {{ return(adapter.dispatch('copy_grants', 'dbt')()) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__copy_grants" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4965398, "supported_languages": null }, "macro.dbt.default__copy_grants": { "name": "default__copy_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__copy_grants", "macro_sql": "{% macro default__copy_grants() %}\n {{ return(True) }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496543, "supported_languages": null }, "macro.dbt.support_multiple_grantees_per_dcl_statement": { "name": "support_multiple_grantees_per_dcl_statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.support_multiple_grantees_per_dcl_statement", "macro_sql": "{% macro support_multiple_grantees_per_dcl_statement() %}\n {{ return(adapter.dispatch('support_multiple_grantees_per_dcl_statement', 'dbt')()) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__support_multiple_grantees_per_dcl_statement" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496545, "supported_languages": null }, "macro.dbt.default__support_multiple_grantees_per_dcl_statement": { "name": "default__support_multiple_grantees_per_dcl_statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__support_multiple_grantees_per_dcl_statement", "macro_sql": "\n\n{%- macro default__support_multiple_grantees_per_dcl_statement() -%}\n {{ return(True) }}\n{%- endmacro -%}\n\n\n", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496547, "supported_languages": null }, "macro.dbt.should_revoke": { "name": "should_revoke", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.should_revoke", "macro_sql": "{% macro should_revoke(existing_relation, full_refresh_mode=True) %}\n\n {% if not existing_relation %}\n {#-- The table doesn't already exist, so no grants to copy over --#}\n {{ return(False) }}\n {% elif full_refresh_mode %}\n {#-- The object is being REPLACED -- whether grants are copied over depends on the value of user config --#}\n {{ return(copy_grants()) }}\n {% else %}\n {#-- The table is being merged/upserted/inserted -- grants will be carried over --#}\n {{ return(True) }}\n {% endif %}\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.copy_grants" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4965498, "supported_languages": null }, "macro.dbt.get_show_grant_sql": { "name": "get_show_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_show_grant_sql", "macro_sql": "{% macro get_show_grant_sql(relation) %}\n {{ return(adapter.dispatch(\"get_show_grant_sql\", \"dbt\")(relation)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_show_grant_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496552, "supported_languages": null }, "macro.dbt.default__get_show_grant_sql": { "name": "default__get_show_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_show_grant_sql", "macro_sql": "{% macro default__get_show_grant_sql(relation) %}\n show grants on {{ relation.render() }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496554, "supported_languages": null }, "macro.dbt.get_grant_sql": { "name": "get_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_grant_sql", "macro_sql": "{% macro get_grant_sql(relation, privilege, grantees) %}\n {{ return(adapter.dispatch('get_grant_sql', 'dbt')(relation, privilege, grantees)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__get_grant_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496556, "supported_languages": null }, "macro.dbt.default__get_grant_sql": { "name": "default__get_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_grant_sql", "macro_sql": "\n\n{%- macro default__get_grant_sql(relation, privilege, grantees) -%}\n grant {{ privilege }} on {{ relation.render() }} to {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4965582, "supported_languages": null }, "macro.dbt.get_revoke_sql": { "name": "get_revoke_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_revoke_sql", "macro_sql": "{% macro get_revoke_sql(relation, privilege, grantees) %}\n {{ return(adapter.dispatch('get_revoke_sql', 'dbt')(relation, privilege, grantees)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__get_revoke_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.49656, "supported_languages": null }, "macro.dbt.default__get_revoke_sql": { "name": "default__get_revoke_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_revoke_sql", "macro_sql": "\n\n{%- macro default__get_revoke_sql(relation, privilege, grantees) -%}\n revoke {{ privilege }} on {{ relation.render() }} from {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496568, "supported_languages": null }, "macro.dbt.get_dcl_statement_list": { "name": "get_dcl_statement_list", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_dcl_statement_list", "macro_sql": "{% macro get_dcl_statement_list(relation, grant_config, get_dcl_macro) %}\n {{ return(adapter.dispatch('get_dcl_statement_list', 'dbt')(relation, grant_config, get_dcl_macro)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__get_dcl_statement_list" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.49657, "supported_languages": null }, "macro.dbt.default__get_dcl_statement_list": { "name": "default__get_dcl_statement_list", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_dcl_statement_list", "macro_sql": "\n\n{%- macro default__get_dcl_statement_list(relation, grant_config, get_dcl_macro) -%}\n {#\n -- Unpack grant_config into specific privileges and the set of users who need them granted/revoked.\n -- Depending on whether this database supports multiple grantees per statement, pass in the list of\n -- all grantees per privilege, or (if not) template one statement per privilege-grantee pair.\n -- `get_dcl_macro` will be either `get_grant_sql` or `get_revoke_sql`\n #}\n {%- set dcl_statements = [] -%}\n {%- for privilege, grantees in grant_config.items() %}\n {%- if support_multiple_grantees_per_dcl_statement() and grantees -%}\n {%- set dcl = get_dcl_macro(relation, privilege, grantees) -%}\n {%- do dcl_statements.append(dcl) -%}\n {%- else -%}\n {%- for grantee in grantees -%}\n {% set dcl = get_dcl_macro(relation, privilege, [grantee]) %}\n {%- do dcl_statements.append(dcl) -%}\n {% endfor -%}\n {%- endif -%}\n {%- endfor -%}\n {{ return(dcl_statements) }}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt.support_multiple_grantees_per_dcl_statement" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496572, "supported_languages": null }, "macro.dbt.call_dcl_statements": { "name": "call_dcl_statements", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.call_dcl_statements", "macro_sql": "{% macro call_dcl_statements(dcl_statement_list) %}\n {{ return(adapter.dispatch(\"call_dcl_statements\", \"dbt\")(dcl_statement_list)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__call_dcl_statements" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4965749, "supported_languages": null }, "macro.dbt.default__call_dcl_statements": { "name": "default__call_dcl_statements", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__call_dcl_statements", "macro_sql": "{% macro default__call_dcl_statements(dcl_statement_list) %}\n {#\n -- By default, supply all grant + revoke statements in a single semicolon-separated block,\n -- so that they're all processed together.\n\n -- Some databases do not support this. Those adapters will need to override this macro\n -- to run each statement individually.\n #}\n {% call statement('grants') %}\n {% for dcl_statement in dcl_statement_list %}\n {{ dcl_statement }};\n {% endfor %}\n {% endcall %}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.statement" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496577, "supported_languages": null }, "macro.dbt.apply_grants": { "name": "apply_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.apply_grants", "macro_sql": "{% macro apply_grants(relation, grant_config, should_revoke) %}\n {{ return(adapter.dispatch(\"apply_grants\", \"dbt\")(relation, grant_config, should_revoke)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__apply_grants" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4965792, "supported_languages": null }, "macro.dbt.default__apply_grants": { "name": "default__apply_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__apply_grants", "macro_sql": "{% macro default__apply_grants(relation, grant_config, should_revoke=True) %}\n {#-- If grant_config is {} or None, this is a no-op --#}\n {% if grant_config %}\n {% if should_revoke %}\n {#-- We think previous grants may have carried over --#}\n {#-- Show current grants and calculate diffs --#}\n {% set current_grants_table = run_query(get_show_grant_sql(relation)) %}\n {% set current_grants_dict = adapter.standardize_grants_dict(current_grants_table) %}\n {% set needs_granting = diff_of_two_dicts(grant_config, current_grants_dict) %}\n {% set needs_revoking = diff_of_two_dicts(current_grants_dict, grant_config) %}\n {% if not (needs_granting or needs_revoking) %}\n {{ log('On ' ~ relation.render() ~': All grants are in place, no revocation or granting needed.')}}\n {% endif %}\n {% else %}\n {#-- We don't think there's any chance of previous grants having carried over. --#}\n {#-- Jump straight to granting what the user has configured. --#}\n {% set needs_revoking = {} %}\n {% set needs_granting = grant_config %}\n {% endif %}\n {% if needs_granting or needs_revoking %}\n {% set revoke_statement_list = get_dcl_statement_list(relation, needs_revoking, get_revoke_sql) %}\n {% set grant_statement_list = get_dcl_statement_list(relation, needs_granting, get_grant_sql) %}\n {% set dcl_statement_list = revoke_statement_list + grant_statement_list %}\n {% if dcl_statement_list %}\n {{ call_dcl_statements(dcl_statement_list) }}\n {% endif %}\n {% endif %}\n {% endif %}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.run_query", "macro.dbt.get_show_grant_sql", "macro.dbt.get_dcl_statement_list", "macro.dbt.call_dcl_statements" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496586, "supported_languages": null }, "macro.dbt.get_show_sql": { "name": "get_show_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/show.sql", "original_file_path": "macros/adapters/show.sql", "unique_id": "macro.dbt.get_show_sql", "macro_sql": "{% macro get_show_sql(compiled_code, sql_header, limit) -%}\n {%- if sql_header is not none -%}\n {{ sql_header }}\n {%- endif %}\n {{ get_limit_subquery_sql(compiled_code, limit) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.get_limit_subquery_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4965918, "supported_languages": null }, "macro.dbt.get_limit_subquery_sql": { "name": "get_limit_subquery_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/show.sql", "original_file_path": "macros/adapters/show.sql", "unique_id": "macro.dbt.get_limit_subquery_sql", "macro_sql": "\n{%- macro get_limit_subquery_sql(sql, limit) -%}\n {{ adapter.dispatch('get_limit_sql', 'dbt')(sql, limit) }}\n{%- endmacro -%}\n\n", "depends_on": { "macros": [ "macro.dbt.default__get_limit_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496594, "supported_languages": null }, "macro.dbt.default__get_limit_sql": { "name": "default__get_limit_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/show.sql", "original_file_path": "macros/adapters/show.sql", "unique_id": "macro.dbt.default__get_limit_sql", "macro_sql": "{% macro default__get_limit_sql(sql, limit) %}\n {{ sql }}\n {% if limit is not none %}\n limit {{ limit }}\n {%- endif -%}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496597, "supported_languages": null }, "macro.dbt.alter_column_comment": { "name": "alter_column_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.alter_column_comment", "macro_sql": "{% macro alter_column_comment(relation, column_dict) -%}\n {{ return(adapter.dispatch('alter_column_comment', 'dbt')(relation, column_dict)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__alter_column_comment" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496609, "supported_languages": null }, "macro.dbt.default__alter_column_comment": { "name": "default__alter_column_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.default__alter_column_comment", "macro_sql": "{% macro default__alter_column_comment(relation, column_dict) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_column_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4966102, "supported_languages": null }, "macro.dbt.alter_relation_comment": { "name": "alter_relation_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.alter_relation_comment", "macro_sql": "{% macro alter_relation_comment(relation, relation_comment) -%}\n {{ return(adapter.dispatch('alter_relation_comment', 'dbt')(relation, relation_comment)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__alter_relation_comment" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496612, "supported_languages": null }, "macro.dbt.default__alter_relation_comment": { "name": "default__alter_relation_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.default__alter_relation_comment", "macro_sql": "{% macro default__alter_relation_comment(relation, relation_comment) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_relation_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496615, "supported_languages": null }, "macro.dbt.persist_docs": { "name": "persist_docs", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.persist_docs", "macro_sql": "{% macro persist_docs(relation, model, for_relation=true, for_columns=true) -%}\n {{ return(adapter.dispatch('persist_docs', 'dbt')(relation, model, for_relation, for_columns)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__persist_docs" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.49684, "supported_languages": null }, "macro.dbt.default__persist_docs": { "name": "default__persist_docs", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.default__persist_docs", "macro_sql": "{% macro default__persist_docs(relation, model, for_relation, for_columns) -%}\n {% if for_relation and config.persist_relation_docs() and model.description %}\n {% do run_query(alter_relation_comment(relation, model.description)) %}\n {% endif %}\n\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {% do run_query(alter_column_comment(relation, model.columns)) %}\n {% endif %}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.run_query", "macro.dbt.alter_relation_comment", "macro.dbt.alter_column_comment" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496843, "supported_languages": null }, "macro.dbt.get_catalog_relations": { "name": "get_catalog_relations", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_catalog_relations", "macro_sql": "{% macro get_catalog_relations(information_schema, relations) -%}\n {{ return(adapter.dispatch('get_catalog_relations', 'dbt')(information_schema, relations)) }}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_catalog_relations" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4968488, "supported_languages": null }, "macro.dbt.default__get_catalog_relations": { "name": "default__get_catalog_relations", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_catalog_relations", "macro_sql": "{% macro default__get_catalog_relations(information_schema, relations) -%}\n {% set typename = adapter.type() %}\n {% set msg -%}\n get_catalog_relations not implemented for {{ typename }}\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg) }}\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496851, "supported_languages": null }, "macro.dbt.get_catalog": { "name": "get_catalog", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_catalog", "macro_sql": "{% macro get_catalog(information_schema, schemas) -%}\n {{ return(adapter.dispatch('get_catalog', 'dbt')(information_schema, schemas)) }}\n{%- endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_catalog" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4968529, "supported_languages": null }, "macro.dbt.default__get_catalog": { "name": "default__get_catalog", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_catalog", "macro_sql": "{% macro default__get_catalog(information_schema, schemas) -%}\n\n {% set typename = adapter.type() %}\n {% set msg -%}\n get_catalog not implemented for {{ typename }}\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg) }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496855, "supported_languages": null }, "macro.dbt.information_schema_name": { "name": "information_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.information_schema_name", "macro_sql": "{% macro information_schema_name(database) %}\n {{ return(adapter.dispatch('information_schema_name', 'dbt')(database)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__information_schema_name" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4968572, "supported_languages": null }, "macro.dbt.default__information_schema_name": { "name": "default__information_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__information_schema_name", "macro_sql": "{% macro default__information_schema_name(database) -%}\n {%- if database -%}\n {{ database }}.INFORMATION_SCHEMA\n {%- else -%}\n INFORMATION_SCHEMA\n {%- endif -%}\n{%- endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496861, "supported_languages": null }, "macro.dbt.list_schemas": { "name": "list_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.list_schemas", "macro_sql": "{% macro list_schemas(database) -%}\n {{ return(adapter.dispatch('list_schemas', 'dbt')(database)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__list_schemas" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4968681, "supported_languages": null }, "macro.dbt.default__list_schemas": { "name": "default__list_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__list_schemas", "macro_sql": "{% macro default__list_schemas(database) -%}\n {% set sql %}\n select distinct schema_name\n from {{ information_schema_name(database) }}.SCHEMATA\n where catalog_name ilike '{{ database }}'\n {% endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.information_schema_name", "macro.dbt.run_query" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4968698, "supported_languages": null }, "macro.dbt.check_schema_exists": { "name": "check_schema_exists", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.check_schema_exists", "macro_sql": "{% macro check_schema_exists(information_schema, schema) -%}\n {{ return(adapter.dispatch('check_schema_exists', 'dbt')(information_schema, schema)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__check_schema_exists" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496872, "supported_languages": null }, "macro.dbt.default__check_schema_exists": { "name": "default__check_schema_exists", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__check_schema_exists", "macro_sql": "{% macro default__check_schema_exists(information_schema, schema) -%}\n {% set sql -%}\n select count(*)\n from {{ information_schema.replace(information_schema_view='SCHEMATA') }}\n where catalog_name='{{ information_schema.database }}'\n and schema_name='{{ schema }}'\n {%- endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.replace", "macro.dbt.run_query" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496874, "supported_languages": null }, "macro.dbt.list_relations_without_caching": { "name": "list_relations_without_caching", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.list_relations_without_caching", "macro_sql": "{% macro list_relations_without_caching(schema_relation) %}\n {{ return(adapter.dispatch('list_relations_without_caching', 'dbt')(schema_relation)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__list_relations_without_caching" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496876, "supported_languages": null }, "macro.dbt.default__list_relations_without_caching": { "name": "default__list_relations_without_caching", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__list_relations_without_caching", "macro_sql": "{% macro default__list_relations_without_caching(schema_relation) %}\n {{ exceptions.raise_not_implemented(\n 'list_relations_without_caching macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4968781, "supported_languages": null }, "macro.dbt.get_catalog_for_single_relation": { "name": "get_catalog_for_single_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_catalog_for_single_relation", "macro_sql": "{% macro get_catalog_for_single_relation(relation) %}\n {{ return(adapter.dispatch('get_catalog_for_single_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__get_catalog_for_single_relation" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.49688, "supported_languages": null }, "macro.dbt.default__get_catalog_for_single_relation": { "name": "default__get_catalog_for_single_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_catalog_for_single_relation", "macro_sql": "{% macro default__get_catalog_for_single_relation(relation) %}\n {{ exceptions.raise_not_implemented(\n 'get_catalog_for_single_relation macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496882, "supported_languages": null }, "macro.dbt.get_relations": { "name": "get_relations", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_relations", "macro_sql": "{% macro get_relations() %}\n {{ return(adapter.dispatch('get_relations', 'dbt')()) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_relations" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4968882, "supported_languages": null }, "macro.dbt.default__get_relations": { "name": "default__get_relations", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_relations", "macro_sql": "{% macro default__get_relations() %}\n {{ exceptions.raise_not_implemented(\n 'get_relations macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4968908, "supported_languages": null }, "macro.dbt.get_relation_last_modified": { "name": "get_relation_last_modified", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_relation_last_modified", "macro_sql": "{% macro get_relation_last_modified(information_schema, relations) %}\n {{ return(adapter.dispatch('get_relation_last_modified', 'dbt')(information_schema, relations)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__get_relation_last_modified" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496893, "supported_languages": null }, "macro.dbt.default__get_relation_last_modified": { "name": "default__get_relation_last_modified", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_relation_last_modified", "macro_sql": "{% macro default__get_relation_last_modified(information_schema, relations) %}\n {{ exceptions.raise_not_implemented(\n 'get_relation_last_modified macro not implemented for adapter ' + adapter.type()) }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496895, "supported_languages": null }, "macro.dbt.get_columns_in_relation": { "name": "get_columns_in_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_columns_in_relation", "macro_sql": "{% macro get_columns_in_relation(relation) -%}\n {{ return(adapter.dispatch('get_columns_in_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt_postgres.postgres__get_columns_in_relation" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496903, "supported_languages": null }, "macro.dbt.default__get_columns_in_relation": { "name": "default__get_columns_in_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_columns_in_relation", "macro_sql": "{% macro default__get_columns_in_relation(relation) -%}\n {{ exceptions.raise_not_implemented(\n 'get_columns_in_relation macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4969049, "supported_languages": null }, "macro.dbt.sql_convert_columns_in_relation": { "name": "sql_convert_columns_in_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.sql_convert_columns_in_relation", "macro_sql": "{% macro sql_convert_columns_in_relation(table) -%}\n {% set columns = [] %}\n {% for row in table %}\n {% do columns.append(api.Column(*row)) %}\n {% endfor %}\n {{ return(columns) }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496907, "supported_languages": null }, "macro.dbt.get_list_of_column_names": { "name": "get_list_of_column_names", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_list_of_column_names", "macro_sql": "\n\n{%- macro get_list_of_column_names(columns) -%}\n {% set col_names = [] %}\n {% for col in columns %}\n {% do col_names.append(col.name) %}\n {% endfor %}\n {{ return(col_names) }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.49691, "supported_languages": null }, "macro.dbt.get_empty_subquery_sql": { "name": "get_empty_subquery_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_empty_subquery_sql", "macro_sql": "{% macro get_empty_subquery_sql(select_sql, select_sql_header=none) -%}\n {{ return(adapter.dispatch('get_empty_subquery_sql', 'dbt')(select_sql, select_sql_header)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__get_empty_subquery_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4969118, "supported_languages": null }, "macro.dbt.default__get_empty_subquery_sql": { "name": "default__get_empty_subquery_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_empty_subquery_sql", "macro_sql": "{% macro default__get_empty_subquery_sql(select_sql, select_sql_header=none) %}\n {%- if select_sql_header is not none -%}\n {{ select_sql_header }}\n {%- endif -%}\n select * from (\n {{ select_sql }}\n ) as __dbt_sbq\n where false\n limit 0\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496913, "supported_languages": null }, "macro.dbt.get_empty_schema_sql": { "name": "get_empty_schema_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_empty_schema_sql", "macro_sql": "{% macro get_empty_schema_sql(columns) -%}\n {{ return(adapter.dispatch('get_empty_schema_sql', 'dbt')(columns)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__get_empty_schema_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4969149, "supported_languages": null }, "macro.dbt.default__get_empty_schema_sql": { "name": "default__get_empty_schema_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_empty_schema_sql", "macro_sql": "{% macro default__get_empty_schema_sql(columns) %}\n {%- set col_err = [] -%}\n {%- set col_naked_numeric = [] -%}\n select\n {% for i in columns %}\n {%- set col = columns[i] -%}\n {%- if col['data_type'] is not defined -%}\n {%- do col_err.append(col['name']) -%}\n {#-- If this column's type is just 'numeric' then it is missing precision/scale, raise a warning --#}\n {%- elif col['data_type'].strip().lower() in ('numeric', 'decimal', 'number') -%}\n {%- do col_naked_numeric.append(col['name']) -%}\n {%- endif -%}\n {% set col_name = adapter.quote(col['name']) if col.get('quote') else col['name'] %}\n {{ cast('null', col['data_type']) }} as {{ col_name }}{{ \", \" if not loop.last }}\n {%- endfor -%}\n {%- if (col_err | length) > 0 -%}\n {{ exceptions.column_type_missing(column_names=col_err) }}\n {%- elif (col_naked_numeric | length) > 0 -%}\n {{ exceptions.warn(\"Detected columns with numeric type and unspecified precision/scale, this can lead to unintended rounding: \" ~ col_naked_numeric ~ \"`\") }}\n {%- endif -%}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.cast" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4969249, "supported_languages": null }, "macro.dbt.get_column_schema_from_query": { "name": "get_column_schema_from_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_column_schema_from_query", "macro_sql": "{% macro get_column_schema_from_query(select_sql, select_sql_header=none) -%}\n {% set columns = [] %}\n {# -- Using an 'empty subquery' here to get the same schema as the given select_sql statement, without necessitating a data scan.#}\n {% set sql = get_empty_subquery_sql(select_sql, select_sql_header) %}\n {% set column_schema = adapter.get_column_schema_from_query(sql) %}\n {{ return(column_schema) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.get_empty_subquery_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496927, "supported_languages": null }, "macro.dbt.get_columns_in_query": { "name": "get_columns_in_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_columns_in_query", "macro_sql": "{% macro get_columns_in_query(select_sql) -%}\n {{ return(adapter.dispatch('get_columns_in_query', 'dbt')(select_sql)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__get_columns_in_query" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496929, "supported_languages": null }, "macro.dbt.default__get_columns_in_query": { "name": "default__get_columns_in_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_columns_in_query", "macro_sql": "{% macro default__get_columns_in_query(select_sql) %}\n {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%}\n {{ get_empty_subquery_sql(select_sql) }}\n {% endcall %}\n {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.statement", "macro.dbt.get_empty_subquery_sql" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496932, "supported_languages": null }, "macro.dbt.alter_column_type": { "name": "alter_column_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.alter_column_type", "macro_sql": "{% macro alter_column_type(relation, column_name, new_column_type) -%}\n {{ return(adapter.dispatch('alter_column_type', 'dbt')(relation, column_name, new_column_type)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__alter_column_type" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496935, "supported_languages": null }, "macro.dbt.default__alter_column_type": { "name": "default__alter_column_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__alter_column_type", "macro_sql": "{% macro default__alter_column_type(relation, column_name, new_column_type) -%}\n {#\n 1. Create a new column (w/ temp name and correct type)\n 2. Copy data over to it\n 3. Drop the existing column (cascade!)\n 4. Rename the new column to existing column\n #}\n {%- set tmp_column = column_name + \"__dbt_alter\" -%}\n\n {% call statement('alter_column_type') %}\n alter table {{ relation.render() }} add column {{ adapter.quote(tmp_column) }} {{ new_column_type }};\n update {{ relation.render() }} set {{ adapter.quote(tmp_column) }} = {{ adapter.quote(column_name) }};\n alter table {{ relation.render() }} drop column {{ adapter.quote(column_name) }} cascade;\n alter table {{ relation.render() }} rename column {{ adapter.quote(tmp_column) }} to {{ adapter.quote(column_name) }}\n {% endcall %}\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.statement" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496942, "supported_languages": null }, "macro.dbt.alter_relation_add_remove_columns": { "name": "alter_relation_add_remove_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.alter_relation_add_remove_columns", "macro_sql": "{% macro alter_relation_add_remove_columns(relation, add_columns = none, remove_columns = none) -%}\n {{ return(adapter.dispatch('alter_relation_add_remove_columns', 'dbt')(relation, add_columns, remove_columns)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__alter_relation_add_remove_columns" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4969442, "supported_languages": null }, "macro.dbt.default__alter_relation_add_remove_columns": { "name": "default__alter_relation_add_remove_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__alter_relation_add_remove_columns", "macro_sql": "{% macro default__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}\n\n {% if add_columns is none %}\n {% set add_columns = [] %}\n {% endif %}\n {% if remove_columns is none %}\n {% set remove_columns = [] %}\n {% endif %}\n\n {% set sql -%}\n\n alter {{ relation.type }} {{ relation.render() }}\n\n {% for column in add_columns %}\n add column {{ column.quoted }} {{ column.data_type }}{{ ',' if not loop.last }}\n {% endfor %}{{ ',' if add_columns and remove_columns }}\n\n {% for column in remove_columns %}\n drop column {{ column.quoted }}{{ ',' if not loop.last }}\n {% endfor %}\n\n {%- endset -%}\n\n {% do run_query(sql) %}\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.run_query" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4969459, "supported_languages": null }, "macro.dbt.get_fixture_sql": { "name": "get_fixture_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.get_fixture_sql", "macro_sql": "{% macro get_fixture_sql(rows, column_name_to_data_types) %}\n-- Fixture for {{ model.name }}\n{% set default_row = {} %}\n\n{%- if not column_name_to_data_types -%}\n{#-- Use defer_relation IFF it is available in the manifest and 'this' is missing from the database --#}\n{%- set this_or_defer_relation = defer_relation if (defer_relation and not load_relation(this)) else this -%}\n{%- set columns_in_relation = adapter.get_columns_in_relation(this_or_defer_relation) -%}\n\n{%- set column_name_to_data_types = {} -%}\n{%- set column_name_to_quoted = {} -%}\n{%- for column in columns_in_relation -%}\n\n{#-- This needs to be a case-insensitive comparison --#}\n{%- do column_name_to_data_types.update({column.name|lower: column.data_type}) -%}\n{%- do column_name_to_quoted.update({column.name|lower: column.quoted}) -%}\n{%- endfor -%}\n{%- endif -%}\n\n{%- if not column_name_to_data_types -%}\n {{ exceptions.raise_compiler_error(\"Not able to get columns for unit test '\" ~ model.name ~ \"' from relation \" ~ this ~ \" because the relation doesn't exist\") }}\n{%- endif -%}\n\n{%- for column_name, column_type in column_name_to_data_types.items() -%}\n {%- do default_row.update({column_name: (safe_cast(\"null\", column_type) | trim )}) -%}\n{%- endfor -%}\n\n{{ validate_fixture_rows(rows, row_number) }}\n\n{%- for row in rows -%}\n{%- set formatted_row = format_row(row, column_name_to_data_types) -%}\n{%- set default_row_copy = default_row.copy() -%}\n{%- do default_row_copy.update(formatted_row) -%}\nselect\n{%- for column_name, column_value in default_row_copy.items() %} {{ column_value }} as {{ column_name_to_quoted[column_name] }}{% if not loop.last -%}, {%- endif %}\n{%- endfor %}\n{%- if not loop.last %}\nunion all\n{% endif %}\n{%- endfor -%}\n\n{%- if (rows | length) == 0 -%}\n select\n {%- for column_name, column_value in default_row.items() %} {{ column_value }} as {{ column_name_to_quoted[column_name] }}{% if not loop.last -%},{%- endif %}\n {%- endfor %}\n limit 0\n{%- endif -%}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.load_relation", "macro.dbt.safe_cast", "macro.dbt.validate_fixture_rows", "macro.dbt.format_row" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496955, "supported_languages": null }, "macro.dbt.get_expected_sql": { "name": "get_expected_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.get_expected_sql", "macro_sql": "{% macro get_expected_sql(rows, column_name_to_data_types, column_name_to_quoted) %}\n\n{%- if (rows | length) == 0 -%}\n select * from dbt_internal_unit_test_actual\n limit 0\n{%- else -%}\n{%- for row in rows -%}\n{%- set formatted_row = format_row(row, column_name_to_data_types) -%}\nselect\n{%- for column_name, column_value in formatted_row.items() %} {{ column_value }} as {{ column_name_to_quoted[column_name] }}{% if not loop.last -%}, {%- endif %}\n{%- endfor %}\n{%- if not loop.last %}\nunion all\n{% endif %}\n{%- endfor -%}\n{%- endif -%}\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.format_row" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4969568, "supported_languages": null }, "macro.dbt.format_row": { "name": "format_row", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.format_row", "macro_sql": "\n\n{%- macro format_row(row, column_name_to_data_types) -%}\n {#-- generate case-insensitive formatted row --#}\n {% set formatted_row = {} %}\n {%- for column_name, column_value in row.items() -%}\n {% set column_name = column_name|lower %}\n\n {%- if column_name not in column_name_to_data_types %}\n {#-- if user-provided row contains column name that relation does not contain, raise an error --#}\n {% set fixture_name = \"expected output\" if model.resource_type == 'unit_test' else (\"'\" ~ model.name ~ \"'\") %}\n {{ exceptions.raise_compiler_error(\n \"Invalid column name: '\" ~ column_name ~ \"' in unit test fixture for \" ~ fixture_name ~ \".\"\n \"\\nAccepted columns for \" ~ fixture_name ~ \" are: \" ~ (column_name_to_data_types.keys()|list)\n ) }}\n {%- endif -%}\n\n {%- set column_type = column_name_to_data_types[column_name] %}\n\n {#-- sanitize column_value: wrap yaml strings in quotes, apply cast --#}\n {%- set column_value_clean = column_value -%}\n {%- if column_value is string -%}\n {%- set column_value_clean = dbt.string_literal(dbt.escape_single_quotes(column_value)) -%}\n {%- elif column_value is none -%}\n {%- set column_value_clean = 'null' -%}\n {%- endif -%}\n\n {%- set row_update = {column_name: safe_cast(column_value_clean, column_type) } -%}\n {%- do formatted_row.update(row_update) -%}\n {%- endfor -%}\n {{ return(formatted_row) }}\n{%- endmacro -%}\n\n", "depends_on": { "macros": [ "macro.dbt.string_literal", "macro.dbt.escape_single_quotes", "macro.dbt.safe_cast" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496959, "supported_languages": null }, "macro.dbt.validate_fixture_rows": { "name": "validate_fixture_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.validate_fixture_rows", "macro_sql": "{%- macro validate_fixture_rows(rows, row_number) -%}\n {{ return(adapter.dispatch('validate_fixture_rows', 'dbt')(rows, row_number)) }}\n{%- endmacro -%}\n\n", "depends_on": { "macros": [ "macro.dbt.default__validate_fixture_rows" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496963, "supported_languages": null }, "macro.dbt.default__validate_fixture_rows": { "name": "default__validate_fixture_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.default__validate_fixture_rows", "macro_sql": "{%- macro default__validate_fixture_rows(rows, row_number) -%}\n {# This is an abstract method for adapter overrides as needed #}\n{%- endmacro -%}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4969652, "supported_languages": null }, "macro.dbt.resolve_model_name": { "name": "resolve_model_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.resolve_model_name", "macro_sql": "{% macro resolve_model_name(input_model_name) %}\n {{ return(adapter.dispatch('resolve_model_name', 'dbt')(input_model_name)) }}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.default__resolve_model_name" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496973, "supported_languages": null }, "macro.dbt.default__resolve_model_name": { "name": "default__resolve_model_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.default__resolve_model_name", "macro_sql": "\n\n{%- macro default__resolve_model_name(input_model_name) -%}\n {{ input_model_name | string | replace('\"', '\\\"') }}\n{%- endmacro -%}\n\n", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4969752, "supported_languages": null }, "macro.dbt.build_ref_function": { "name": "build_ref_function", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.build_ref_function", "macro_sql": "{% macro build_ref_function(model) %}\n\n {%- set ref_dict = {} -%}\n {%- for _ref in model.refs -%}\n {% set _ref_args = [_ref.get('package'), _ref['name']] if _ref.get('package') else [_ref['name'],] %}\n {%- set resolved = ref(*_ref_args, v=_ref.get('version')) -%}\n\n {#\n We want to get the string of the returned relation by calling .render() in order to skip sample/empty\n mode rendering logic. However, people override the default ref macro, and often return a string instead\n of a relation (like the ref macro does by default). Thus, to make sure we dont blow things up, we have\n to ensure the resolved relation has a .render() method.\n #}\n {%- if resolved.render is defined and resolved.render is callable -%}\n {%- set resolved = resolved.render() -%}\n {%- endif -%}\n\n {%- if _ref.get('version') -%}\n {% do _ref_args.extend([\"v\" ~ _ref['version']]) %}\n {%- endif -%}\n {%- do ref_dict.update({_ref_args | join('.'): resolve_model_name(resolved)}) -%}\n {%- endfor -%}\n\ndef ref(*args, **kwargs):\n refs = {{ ref_dict | tojson }}\n key = '.'.join(args)\n version = kwargs.get(\"v\") or kwargs.get(\"version\")\n if version:\n key += f\".v{version}\"\n dbt_load_df_function = kwargs.get(\"dbt_load_df_function\")\n return dbt_load_df_function(refs[key])\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.resolve_model_name" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4969778, "supported_languages": null }, "macro.dbt.build_source_function": { "name": "build_source_function", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.build_source_function", "macro_sql": "{% macro build_source_function(model) %}\n\n {%- set source_dict = {} -%}\n {%- for _source in model.sources -%}\n {%- set resolved = source(*_source) -%}\n {%- do source_dict.update({_source | join('.'): resolve_model_name(resolved)}) -%}\n {%- endfor -%}\n\ndef source(*args, dbt_load_df_function):\n sources = {{ source_dict | tojson }}\n key = '.'.join(args)\n return dbt_load_df_function(sources[key])\n\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.resolve_model_name" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.49698, "supported_languages": null }, "macro.dbt.build_config_dict": { "name": "build_config_dict", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.build_config_dict", "macro_sql": "{% macro build_config_dict(model) %}\n {%- set config_dict = {} -%}\n {% set config_dbt_used = zip(model.config.config_keys_used, model.config.config_keys_defaults) | list %}\n {%- for key, default in config_dbt_used -%}\n {# weird type testing with enum, would be much easier to write this logic in Python! #}\n {%- if key == \"language\" -%}\n {%- set value = \"python\" -%}\n {%- endif -%}\n {%- set value = model.config.get(key, default) -%}\n {%- do config_dict.update({key: value}) -%}\n {%- endfor -%}\nconfig_dict = {{ config_dict }}\n{% endmacro %}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496982, "supported_languages": null }, "macro.dbt.py_script_postfix": { "name": "py_script_postfix", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.py_script_postfix", "macro_sql": "{% macro py_script_postfix(model) %}\n# This part is user provided model code\n# you will need to copy the next section to run the code\n# COMMAND ----------\n# this part is dbt logic for get ref work, do not modify\n\n{{ build_ref_function(model ) }}\n{{ build_source_function(model ) }}\n{{ build_config_dict(model) }}\n\nclass config:\n def __init__(self, *args, **kwargs):\n pass\n\n @staticmethod\n def get(key, default=None):\n return config_dict.get(key, default)\n\nclass this:\n \"\"\"dbt.this() or dbt.this.identifier\"\"\"\n database = \"{{ this.database }}\"\n schema = \"{{ this.schema }}\"\n identifier = \"{{ this.identifier }}\"\n {% set this_relation_name = resolve_model_name(this) %}\n def __repr__(self):\n return '{{ this_relation_name }}'\n\n\nclass dbtObj:\n def __init__(self, load_df_function) -> None:\n self.source = lambda *args: source(*args, dbt_load_df_function=load_df_function)\n self.ref = lambda *args, **kwargs: ref(*args, **kwargs, dbt_load_df_function=load_df_function)\n self.config = config\n self.this = this()\n self.is_incremental = {{ is_incremental() }}\n\n# COMMAND ----------\n{{py_script_comment()}}\n{% endmacro %}", "depends_on": { "macros": [ "macro.dbt.build_ref_function", "macro.dbt.build_source_function", "macro.dbt.build_config_dict", "macro.dbt.resolve_model_name", "macro.dbt.is_incremental", "macro.dbt.py_script_comment" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.496984, "supported_languages": null }, "macro.dbt.py_script_comment": { "name": "py_script_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.py_script_comment", "macro_sql": "{%macro py_script_comment()%}\n{%endmacro%}", "depends_on": { "macros": [] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4969862, "supported_languages": null }, "macro.dbt.test_unique": { "name": "test_unique", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_unique", "macro_sql": "{% test unique(model, column_name) %}\n {% set macro = adapter.dispatch('test_unique', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", "depends_on": { "macros": [ "macro.dbt.default__test_unique" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.4973178, "supported_languages": null }, "macro.dbt.test_not_null": { "name": "test_not_null", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_not_null", "macro_sql": "{% test not_null(model, column_name) %}\n {% set macro = adapter.dispatch('test_not_null', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", "depends_on": { "macros": [ "macro.dbt.default__test_not_null" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.497498, "supported_languages": null }, "macro.dbt.test_accepted_values": { "name": "test_accepted_values", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_accepted_values", "macro_sql": "{% test accepted_values(model, column_name, values, quote=True) %}\n {% set macro = adapter.dispatch('test_accepted_values', 'dbt') %}\n {{ macro(model, column_name, values, quote) }}\n{% endtest %}", "depends_on": { "macros": [ "macro.dbt.default__test_accepted_values" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.497705, "supported_languages": null }, "macro.dbt.test_relationships": { "name": "test_relationships", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_relationships", "macro_sql": "{% test relationships(model, column_name, to, field) %}\n {% set macro = adapter.dispatch('test_relationships', 'dbt') %}\n {{ macro(model, column_name, to, field) }}\n{% endtest %}", "depends_on": { "macros": [ "macro.dbt.default__test_relationships" ] }, "description": "", "meta": {}, "docs": { "show": true, "node_color": null }, "patch_path": null, "arguments": [], "created_at": 1763574305.497903, "supported_languages": null } }, "docs": { "doc.test.my_docs": { "name": "my_docs", "resource_type": "doc", "package_name": "test", "path": "docs.md", "original_file_path": "models/docs.md", "unique_id": "doc.test.my_docs", "block_contents": "some docs" }, "doc.dbt.__overview__": { "name": "__overview__", "resource_type": "doc", "package_name": "dbt", "path": "overview.md", "original_file_path": "docs/overview.md", "unique_id": "doc.dbt.__overview__", "block_contents": "### Welcome!\n\nWelcome to the auto-generated documentation for your dbt project!\n\n### Navigation\n\nYou can use the `Project` and `Database` navigation tabs on the left side of the window to explore the models\nin your project.\n\n#### Project Tab\nThe `Project` tab mirrors the directory structure of your dbt project. In this tab, you can see all of the\nmodels defined in your dbt project, as well as models imported from dbt packages.\n\n#### Database Tab\nThe `Database` tab also exposes your models, but in a format that looks more like a database explorer. This view\nshows relations (tables and views) grouped into database schemas. Note that ephemeral models are _not_ shown\nin this interface, as they do not exist in the database.\n\n### Graph Exploration\nYou can click the blue icon on the bottom-right corner of the page to view the lineage graph of your models.\n\nOn model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the `Expand`\nbutton at the top-right of this lineage pane, you'll be able to see all of the models that are used to build,\nor are built from, the model you're exploring.\n\nOnce expanded, you'll be able to use the `--select` and `--exclude` model selection syntax to filter the\nmodels in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax).\n\nNote that you can also right-click on models to interactively filter and explore the graph.\n\n---\n\n### More information\n\n- [What is dbt](https://docs.getdbt.com/docs/introduction)?\n- Read the [dbt viewpoint](https://docs.getdbt.com/docs/viewpoint)\n- [Installation](https://docs.getdbt.com/docs/installation)\n- Join the [dbt Community](https://www.getdbt.com/community/) for questions and discussion" } }, "exposures": { "exposure.test.weekly_jaffle_metrics": { "name": "weekly_jaffle_metrics", "resource_type": "exposure", "package_name": "test", "path": "e.yml", "original_file_path": "models/e.yml", "unique_id": "exposure.test.weekly_jaffle_metrics", "fqn": [ "test", "weekly_jaffle_metrics" ], "type": "dashboard", "owner": { "email": "data@jaffleshop.com", "name": "Callum McData" }, "description": "Did someone say \"exponential growth\"?\n", "label": "Jaffles by the Week", "maturity": "high", "meta": {}, "tags": [ "tag" ], "config": { "enabled": true, "tags": [ "tag" ], "meta": {} }, "unrendered_config": { "enabled": true, "meta": {}, "tags": [ "tag" ] }, "url": "https://bi.tool/dashboards/1", "depends_on": { "macros": [], "nodes": [ "model.test.incremental" ] }, "refs": [ { "name": "incremental", "package": null, "version": null } ], "sources": [], "metrics": [], "created_at": 1763574306.332556 } }, "metrics": { "metric.test.discrete_order_value_p99": { "name": "discrete_order_value_p99", "resource_type": "metric", "package_name": "test", "path": "sm.yml", "original_file_path": "models/sm.yml", "unique_id": "metric.test.discrete_order_value_p99", "fqn": [ "test", "discrete_order_value_p99" ], "description": "Metric created from measure discrete_order_value_p99", "label": "discrete_order_value_p99", "type": "simple", "type_params": { "measure": { "name": "discrete_order_value_p99", "filter": null, "alias": null, "join_to_timespine": false, "fill_nulls_with": null }, "input_measures": [ { "name": "discrete_order_value_p99", "filter": null, "alias": null, "join_to_timespine": false, "fill_nulls_with": null } ], "numerator": null, "denominator": null, "expr": "order_total", "window": null, "grain_to_date": null, "metrics": [], "conversion_type_params": null, "cumulative_type_params": null, "metric_aggregation_params": null }, "filter": null, "metadata": null, "time_granularity": null, "meta": {}, "tags": [], "config": { "enabled": true, "group": null, "meta": {} }, "unrendered_config": { "enabled": true, "meta": {} }, "sources": [], "depends_on": { "macros": [], "nodes": [ "semantic_model.test.my_sm" ] }, "refs": [], "metrics": [], "created_at": 1763574306.37896, "group": null }, "metric.test.total_outer": { "name": "total_outer", "resource_type": "metric", "package_name": "test", "path": "m.yml", "original_file_path": "models/m.yml", "unique_id": "metric.test.total_outer", "fqn": [ "test", "total_outer" ], "description": "The total count of outer", "label": "Total Outer", "type": "simple", "type_params": { "measure": { "name": "total_outer_count", "filter": null, "alias": null, "join_to_timespine": false, "fill_nulls_with": null }, "input_measures": [ { "name": "total_outer_count", "filter": null, "alias": null, "join_to_timespine": false, "fill_nulls_with": null } ], "numerator": null, "denominator": null, "expr": null, "window": null, "grain_to_date": null, "metrics": [], "conversion_type_params": null, "cumulative_type_params": null, "metric_aggregation_params": null }, "filter": null, "metadata": null, "time_granularity": "month", "meta": {}, "tags": [], "config": { "enabled": true, "group": null, "meta": {} }, "unrendered_config": {}, "sources": [], "depends_on": { "macros": [], "nodes": [ "semantic_model.test.my_sm" ] }, "refs": [], "metrics": [], "created_at": 1763574306.423567, "group": null }, "metric.test.simple_ratio_metric": { "name": "simple_ratio_metric", "resource_type": "metric", "package_name": "test", "path": "m.yml", "original_file_path": "models/m.yml", "unique_id": "metric.test.simple_ratio_metric", "fqn": [ "test", "simple_ratio_metric" ], "description": "a simple ratio metric", "label": "Simple Ratio Metric", "type": "ratio", "type_params": { "measure": null, "input_measures": [ { "name": "total_outer_count", "filter": null, "alias": null, "join_to_timespine": false, "fill_nulls_with": null } ], "numerator": { "name": "total_outer", "filter": null, "alias": null, "offset_window": null, "offset_to_grain": null }, "denominator": { "name": "total_outer", "filter": null, "alias": "filtered_total_outer", "offset_window": null, "offset_to_grain": null }, "expr": null, "window": null, "grain_to_date": null, "metrics": [], "conversion_type_params": null, "cumulative_type_params": null, "metric_aggregation_params": null }, "filter": null, "metadata": null, "time_granularity": null, "meta": {}, "tags": [], "config": { "enabled": true, "group": null, "meta": {} }, "unrendered_config": {}, "sources": [], "depends_on": { "macros": [], "nodes": [ "metric.test.total_outer" ] }, "refs": [], "metrics": [], "created_at": 1763574306.42508, "group": null }, "metric.test.filtered_ratio_metric": { "name": "filtered_ratio_metric", "resource_type": "metric", "package_name": "test", "path": "m.yml", "original_file_path": "models/m.yml", "unique_id": "metric.test.filtered_ratio_metric", "fqn": [ "test", "filtered_ratio_metric" ], "description": "a ratio metric", "label": "Ratio Metric 2", "type": "ratio", "type_params": { "measure": null, "input_measures": [ { "name": "total_outer_count", "filter": null, "alias": null, "join_to_timespine": false, "fill_nulls_with": null } ], "numerator": { "name": "total_outer", "filter": { "where_filters": [ { "where_sql_template": "1 = 1" } ] }, "alias": null, "offset_window": null, "offset_to_grain": null }, "denominator": { "name": "total_outer", "filter": { "where_filters": [ { "where_sql_template": "1 = 1" }, { "where_sql_template": "2 = 2" } ] }, "alias": "filtered_total_outer_again", "offset_window": null, "offset_to_grain": null }, "expr": null, "window": null, "grain_to_date": null, "metrics": [], "conversion_type_params": null, "cumulative_type_params": null, "metric_aggregation_params": null }, "filter": null, "metadata": null, "time_granularity": null, "meta": {}, "tags": [], "config": { "enabled": true, "group": null, "meta": {} }, "unrendered_config": {}, "sources": [], "depends_on": { "macros": [], "nodes": [ "metric.test.total_outer" ] }, "refs": [], "metrics": [], "created_at": 1763574306.425653, "group": null }, "metric.test.cumulative_metric": { "name": "cumulative_metric", "resource_type": "metric", "package_name": "test", "path": "m.yml", "original_file_path": "models/m.yml", "unique_id": "metric.test.cumulative_metric", "fqn": [ "test", "cumulative_metric" ], "description": "a cumulative metric", "label": "Cumulative Metric", "type": "cumulative", "type_params": { "measure": { "name": "total_outer_count", "filter": { "where_filters": [ { "where_sql_template": "1 = 1" } ] }, "alias": "filtered_total_outer_count", "join_to_timespine": false, "fill_nulls_with": 0 }, "input_measures": [ { "name": "total_outer_count", "filter": { "where_filters": [ { "where_sql_template": "1 = 1" } ] }, "alias": "filtered_total_outer_count", "join_to_timespine": false, "fill_nulls_with": 0 } ], "numerator": null, "denominator": null, "expr": null, "window": null, "grain_to_date": null, "metrics": [], "conversion_type_params": null, "cumulative_type_params": { "window": null, "grain_to_date": "day", "period_agg": "first", "metric": null }, "metric_aggregation_params": null }, "filter": null, "metadata": null, "time_granularity": null, "meta": {}, "tags": [], "config": { "enabled": true, "group": null, "meta": {} }, "unrendered_config": {}, "sources": [], "depends_on": { "macros": [], "nodes": [ "semantic_model.test.my_sm" ] }, "refs": [], "metrics": [], "created_at": 1763574306.426938, "group": null }, "metric.test.cumulative_metric_2": { "name": "cumulative_metric_2", "resource_type": "metric", "package_name": "test", "path": "m.yml", "original_file_path": "models/m.yml", "unique_id": "metric.test.cumulative_metric_2", "fqn": [ "test", "cumulative_metric_2" ], "description": "a cumulative metric", "label": "Cumulative Metric 2", "type": "cumulative", "type_params": { "measure": { "name": "total_outer_count", "filter": { "where_filters": [ { "where_sql_template": "1 = 1" } ] }, "alias": "filtered_total_outer_count_2", "join_to_timespine": false, "fill_nulls_with": 0 }, "input_measures": [ { "name": "total_outer_count", "filter": { "where_filters": [ { "where_sql_template": "1 = 1" } ] }, "alias": "filtered_total_outer_count_2", "join_to_timespine": false, "fill_nulls_with": 0 } ], "numerator": null, "denominator": null, "expr": null, "window": null, "grain_to_date": null, "metrics": [], "conversion_type_params": null, "cumulative_type_params": { "window": { "count": 1, "granularity": "day" }, "grain_to_date": null, "period_agg": "first", "metric": null }, "metric_aggregation_params": null }, "filter": null, "metadata": null, "time_granularity": null, "meta": {}, "tags": [], "config": { "enabled": true, "group": null, "meta": {} }, "unrendered_config": {}, "sources": [], "depends_on": { "macros": [], "nodes": [ "semantic_model.test.my_sm" ] }, "refs": [], "metrics": [], "created_at": 1763574306.4276102, "group": null }, "metric.test.conversion_metric": { "name": "conversion_metric", "resource_type": "metric", "package_name": "test", "path": "m.yml", "original_file_path": "models/m.yml", "unique_id": "metric.test.conversion_metric", "fqn": [ "test", "conversion_metric" ], "description": "a conversion metric", "label": "Conversion Metric", "type": "conversion", "type_params": { "measure": null, "input_measures": [ { "name": "total_outer_count", "filter": null, "alias": null, "join_to_timespine": false, "fill_nulls_with": null }, { "name": "total_outer_count", "filter": null, "alias": null, "join_to_timespine": false, "fill_nulls_with": 0 } ], "numerator": null, "denominator": null, "expr": null, "window": null, "grain_to_date": null, "metrics": [], "conversion_type_params": { "base_measure": { "name": "total_outer_count", "filter": null, "alias": null, "join_to_timespine": false, "fill_nulls_with": null }, "conversion_measure": { "name": "total_outer_count", "filter": null, "alias": null, "join_to_timespine": false, "fill_nulls_with": 0 }, "entity": "my_entity", "calculation": "conversion_rate", "window": { "count": 1, "granularity": "day" }, "constant_properties": [ { "base_property": "my_entity", "conversion_property": "created_at" } ] }, "cumulative_type_params": null, "metric_aggregation_params": null }, "filter": null, "metadata": null, "time_granularity": null, "meta": {}, "tags": [], "config": { "enabled": true, "group": null, "meta": {} }, "unrendered_config": {}, "sources": [], "depends_on": { "macros": [], "nodes": [ "semantic_model.test.my_sm" ] }, "refs": [], "metrics": [], "created_at": 1763574306.429516, "group": null }, "metric.test.derived_metric": { "name": "derived_metric", "resource_type": "metric", "package_name": "test", "path": "m.yml", "original_file_path": "models/m.yml", "unique_id": "metric.test.derived_metric", "fqn": [ "test", "derived_metric" ], "description": "a derived metric", "label": "Derived Metric", "type": "derived", "type_params": { "measure": null, "input_measures": [ { "name": "total_outer_count", "filter": null, "alias": null, "join_to_timespine": false, "fill_nulls_with": null }, { "name": "total_outer_count", "filter": null, "alias": null, "join_to_timespine": false, "fill_nulls_with": 0 } ], "numerator": null, "denominator": null, "expr": "srm - cm + filtered_ratio_metric", "window": null, "grain_to_date": null, "metrics": [ { "name": "simple_ratio_metric", "filter": { "where_filters": [ { "where_sql_template": "1 = 1" } ] }, "alias": "srm", "offset_window": { "count": 1, "granularity": "month" }, "offset_to_grain": null }, { "name": "conversion_metric", "filter": null, "alias": "cm", "offset_window": null, "offset_to_grain": "month" }, { "name": "filtered_ratio_metric", "filter": null, "alias": null, "offset_window": null, "offset_to_grain": null } ], "conversion_type_params": null, "cumulative_type_params": null, "metric_aggregation_params": null }, "filter": null, "metadata": null, "time_granularity": null, "meta": {}, "tags": [], "config": { "enabled": true, "group": null, "meta": {} }, "unrendered_config": {}, "sources": [], "depends_on": { "macros": [], "nodes": [ "metric.test.simple_ratio_metric", "metric.test.conversion_metric", "metric.test.filtered_ratio_metric" ] }, "refs": [], "metrics": [], "created_at": 1763574306.430549, "group": null } }, "groups": { "group.test.finance": { "name": "finance", "resource_type": "group", "package_name": "test", "path": "g.yml", "original_file_path": "models/g.yml", "unique_id": "group.test.finance", "owner": { "email": "finance@jaffleshop.com", "name": null }, "description": "description", "config": { "meta": { "custom_field": "value" }, "enabled": true } }, "group.test.important_tests": { "name": "important_tests", "resource_type": "group", "package_name": "test", "path": "schema.yml", "original_file_path": "tests/schema.yml", "unique_id": "group.test.important_tests", "owner": { "email": "data@example.com", "name": null }, "description": null, "config": { "meta": {}, "enabled": true } } }, "selectors": {}, "disabled": {}, "parent_map": { "model.test.model_with_lots_of_schema_configs": [ "model.test.ephemeral", "source.test.my_source.my_table" ], "model.test.snapshot_source": [], "model.test.ephemeral": [], "model.test.metricflow_time_spine": [], "model.test.incremental": [ "seed.test.seed" ], "model.test.model_to_unit_test": [ "seed.test.seed" ], "model.test.outer": [ "model.test.ephemeral" ], "model.test.metricflow_time_spine_second": [], "model.test.inner": [ "model.test.outer" ], "snapshot.test.my_snapshot": [], "analysis.test.a": [], "test.test.t": [], "seed.test.seed": [], "test.test.not_null_seed__a_.6b59640cde": [ "seed.test.seed" ], "test.test.not_null_seed__b_.a088b263cb": [ "seed.test.seed" ], "test.test.expression_is_true_seed_b_2.4e0babbea4": [ "seed.test.seed" ], "test.test.unique_outer_id.2195e332d3": [ "model.test.outer" ], "test.test.not_null_outer_id.a226f4fb36": [ "model.test.outer" ], "test.test.unique_model_with_lots_of_schema_configs_id.8328d84982": [ "model.test.model_with_lots_of_schema_configs" ], "test.test.my_favorite_test.b488d63233": [ "model.test.model_with_lots_of_schema_configs" ], "test.test.my_second_favorite_test.c8955109ad": [ "model.test.model_with_lots_of_schema_configs" ], "snapshot.test.snapshot_2": [ "model.test.snapshot_source" ], "snapshot.test.snapshot_3": [ "model.test.snapshot_source" ], "source.test.my_source.my_table": [], "exposure.test.weekly_jaffle_metrics": [ "model.test.incremental" ], "function.test.area_of_circle": [], "metric.test.discrete_order_value_p99": [ "semantic_model.test.my_sm" ], "metric.test.total_outer": [ "semantic_model.test.my_sm" ], "metric.test.simple_ratio_metric": [ "metric.test.total_outer" ], "metric.test.filtered_ratio_metric": [ "metric.test.total_outer" ], "metric.test.cumulative_metric": [ "semantic_model.test.my_sm" ], "metric.test.cumulative_metric_2": [ "semantic_model.test.my_sm" ], "metric.test.conversion_metric": [ "semantic_model.test.my_sm" ], "metric.test.derived_metric": [ "metric.test.conversion_metric", "metric.test.filtered_ratio_metric", "metric.test.simple_ratio_metric" ], "semantic_model.test.my_sm": [ "model.test.outer" ], "saved_query.test.my_saved_query": [ "metric.test.total_outer" ], "unit_test.test.model_to_unit_test.test_model_to_unit_test": [ "model.test.model_to_unit_test" ], "unit_test.test.model_to_unit_test.test_model_to_unit_test_2": [ "model.test.model_to_unit_test" ] }, "child_map": { "model.test.model_with_lots_of_schema_configs": [ "test.test.my_favorite_test.b488d63233", "test.test.my_second_favorite_test.c8955109ad", "test.test.unique_model_with_lots_of_schema_configs_id.8328d84982" ], "model.test.snapshot_source": [ "snapshot.test.snapshot_2", "snapshot.test.snapshot_3" ], "model.test.ephemeral": [ "model.test.model_with_lots_of_schema_configs", "model.test.outer" ], "model.test.metricflow_time_spine": [], "model.test.incremental": [ "exposure.test.weekly_jaffle_metrics" ], "model.test.model_to_unit_test": [ "unit_test.test.model_to_unit_test.test_model_to_unit_test", "unit_test.test.model_to_unit_test.test_model_to_unit_test_2" ], "model.test.outer": [ "model.test.inner", "semantic_model.test.my_sm", "test.test.not_null_outer_id.a226f4fb36", "test.test.unique_outer_id.2195e332d3" ], "model.test.metricflow_time_spine_second": [], "model.test.inner": [], "snapshot.test.my_snapshot": [], "analysis.test.a": [], "test.test.t": [], "seed.test.seed": [ "model.test.incremental", "model.test.model_to_unit_test", "test.test.expression_is_true_seed_b_2.4e0babbea4", "test.test.not_null_seed__a_.6b59640cde", "test.test.not_null_seed__b_.a088b263cb" ], "test.test.not_null_seed__a_.6b59640cde": [], "test.test.not_null_seed__b_.a088b263cb": [], "test.test.expression_is_true_seed_b_2.4e0babbea4": [], "test.test.unique_outer_id.2195e332d3": [], "test.test.not_null_outer_id.a226f4fb36": [], "test.test.unique_model_with_lots_of_schema_configs_id.8328d84982": [], "test.test.my_favorite_test.b488d63233": [], "test.test.my_second_favorite_test.c8955109ad": [], "snapshot.test.snapshot_2": [], "snapshot.test.snapshot_3": [], "source.test.my_source.my_table": [ "model.test.model_with_lots_of_schema_configs" ], "exposure.test.weekly_jaffle_metrics": [], "function.test.area_of_circle": [], "metric.test.discrete_order_value_p99": [], "metric.test.total_outer": [ "metric.test.filtered_ratio_metric", "metric.test.simple_ratio_metric", "saved_query.test.my_saved_query" ], "metric.test.simple_ratio_metric": [ "metric.test.derived_metric" ], "metric.test.filtered_ratio_metric": [ "metric.test.derived_metric" ], "metric.test.cumulative_metric": [], "metric.test.cumulative_metric_2": [], "metric.test.conversion_metric": [ "metric.test.derived_metric" ], "metric.test.derived_metric": [], "semantic_model.test.my_sm": [ "metric.test.conversion_metric", "metric.test.cumulative_metric", "metric.test.cumulative_metric_2", "metric.test.discrete_order_value_p99", "metric.test.total_outer" ], "saved_query.test.my_saved_query": [], "unit_test.test.model_to_unit_test.test_model_to_unit_test": [], "unit_test.test.model_to_unit_test.test_model_to_unit_test_2": [] }, "group_map": { "finance": [ "model.test.metricflow_time_spine", "analysis.test.a", "seed.test.seed", "test.test.not_null_seed__a_.6b59640cde", "test.test.not_null_seed__b_.a088b263cb", "test.test.expression_is_true_seed_b_2.4e0babbea4" ], "important_tests": [ "test.test.t" ] }, "saved_queries": { "saved_query.test.my_saved_query": { "name": "my_saved_query", "resource_type": "saved_query", "package_name": "test", "path": "sq.yml", "original_file_path": "models/sq.yml", "unique_id": "saved_query.test.my_saved_query", "fqn": [ "test", "my_saved_query" ], "query_params": { "metrics": [ "total_outer" ], "group_by": [ "Dimension('my_entity__created_at')" ], "where": null, "order_by": [], "limit": null }, "exports": [ { "name": "my_export", "config": { "export_as": "table", "schema_name": "my_export_schema_name", "alias": "my_export_alias", "database": "dbt" }, "unrendered_config": { "alias": "my_export_alias", "export_as": "table", "schema": "my_export_schema_name" } } ], "description": null, "label": "My Saved Query", "metadata": null, "config": { "enabled": true, "group": null, "meta": {}, "export_as": null, "schema": null, "cache": { "enabled": false } }, "unrendered_config": {}, "group": null, "depends_on": { "macros": [], "nodes": [ "metric.test.total_outer" ] }, "created_at": 1763574306.391684, "refs": [], "tags": [] } }, "semantic_models": { "semantic_model.test.my_sm": { "name": "my_sm", "resource_type": "semantic_model", "package_name": "test", "path": "sm.yml", "original_file_path": "models/sm.yml", "unique_id": "semantic_model.test.my_sm", "fqn": [ "test", "my_sm" ], "model": "ref('outer')", "node_relation": { "alias": "outer", "schema_name": "test17635743055401889788_test_modified_state_schema_evolution", "database": "dbt", "relation_name": "\"dbt\".\"test17635743055401889788_test_modified_state_schema_evolution\".\"outer\"" }, "description": null, "label": null, "defaults": { "agg_time_dimension": "created_at" }, "entities": [ { "name": "my_entity", "type": "primary", "description": null, "label": null, "role": null, "expr": "id", "config": { "meta": {} } }, { "name": "user", "type": "foreign", "description": null, "label": null, "role": null, "expr": "user_id", "config": { "meta": {} } } ], "measures": [ { "name": "total_outer_count", "agg": "count", "description": null, "label": null, "create_metric": false, "expr": "1", "agg_params": null, "non_additive_dimension": null, "agg_time_dimension": null, "config": { "meta": {} } }, { "name": "discrete_order_value_p99", "agg": "percentile", "description": null, "label": null, "create_metric": false, "expr": "order_total", "agg_params": { "percentile": 0.99, "use_discrete_percentile": true, "use_approximate_percentile": false }, "non_additive_dimension": null, "agg_time_dimension": "created_at", "config": { "meta": {} } }, { "name": "sum_of_things", "agg": "sum", "description": null, "label": null, "create_metric": false, "expr": "2", "agg_params": null, "non_additive_dimension": null, "agg_time_dimension": "created_at", "config": { "meta": {} } }, { "name": "has_revenue", "agg": "sum_boolean", "description": null, "label": null, "create_metric": false, "expr": "True", "agg_params": null, "non_additive_dimension": null, "agg_time_dimension": "created_at", "config": { "meta": {} } }, { "name": "test_non_additive", "agg": "sum", "description": null, "label": null, "create_metric": false, "expr": "txn_revenue", "agg_params": null, "non_additive_dimension": { "name": "created_at", "window_choice": "max", "window_groupings": [] }, "agg_time_dimension": null, "config": { "meta": {} } } ], "dimensions": [ { "name": "created_at", "type": "time", "description": null, "label": null, "is_partition": false, "type_params": { "time_granularity": "day", "validity_params": null }, "expr": null, "metadata": null, "config": { "meta": {} } } ], "metadata": null, "depends_on": { "macros": [], "nodes": [ "model.test.outer" ] }, "refs": [ { "name": "outer", "package": null, "version": null } ], "created_at": 1763574306.3763149, "config": { "enabled": true, "group": null, "meta": {} }, "unrendered_config": {}, "primary_entity": null, "group": null } }, "unit_tests": { "unit_test.test.model_to_unit_test.test_model_to_unit_test": { "model": "model_to_unit_test", "given": [ { "input": "ref('seed')", "rows": [ { "a": "1", "b": "2" }, { "a": "2", "b": "3" } ], "format": "csv", "fixture": "test_incremental_fixture" } ], "expect": { "rows": [ { "a": "1", "b": "2" }, { "a": "2", "b": "3" } ], "format": "csv", "fixture": "test_incremental_fixture" }, "name": "test_model_to_unit_test", "resource_type": "unit_test", "package_name": "test", "path": "unit_tests.yml", "original_file_path": "models/unit_tests.yml", "unique_id": "unit_test.test.model_to_unit_test.test_model_to_unit_test", "fqn": [ "test", "model_to_unit_test", "test_model_to_unit_test" ], "description": "A simple test of the outer model", "overrides": null, "depends_on": { "macros": [], "nodes": [ "model.test.model_to_unit_test" ] }, "config": { "tags": [], "meta": {}, "enabled": true }, "checksum": "e7987dd2500df4fa4db5715bd41e0998f25587ddd344be97dfe038f1535b0230", "schema": "test17635743055401889788_test_modified_state_schema_evolution", "created_at": 1763574306.34929, "versions": null, "version": null }, "unit_test.test.model_to_unit_test.test_model_to_unit_test_2": { "model": "model_to_unit_test", "given": [ { "input": "ref('seed')", "rows": "SELECT 2 as a, 3 as b", "format": "sql", "fixture": null } ], "expect": { "rows": [ { "a": 2, "b": 3 } ], "format": "dict", "fixture": null }, "name": "test_model_to_unit_test_2", "resource_type": "unit_test", "package_name": "test", "path": "unit_tests.yml", "original_file_path": "models/unit_tests.yml", "unique_id": "unit_test.test.model_to_unit_test.test_model_to_unit_test_2", "fqn": [ "test", "model_to_unit_test", "test_model_to_unit_test_2" ], "description": "A simple test of the outer model", "overrides": { "macros": { "is_incremental": false }, "vars": {}, "env_vars": {} }, "depends_on": { "macros": [], "nodes": [ "model.test.model_to_unit_test" ] }, "config": { "tags": [ "test_tag" ], "meta": { "my_custom_meta_key": "my_custom_meta_value" }, "enabled": true }, "checksum": "635a3c7a84af38b2f3c52a526773bef3df93278099e2ccefc011986cf26060c2", "schema": "test17635743055401889788_test_modified_state_schema_evolution", "created_at": 1763574306.3518012, "versions": { "include": null, "exclude": null }, "version": null } }, "functions": { "function.test.area_of_circle": { "returns": { "data_type": "float", "description": null }, "database": "dbt", "schema": "test17635743055401889788_test_modified_state_schema_evolution", "name": "area_of_circle", "resource_type": "function", "package_name": "test", "path": "area_of_circle.sql", "original_file_path": "functions/area_of_circle.sql", "unique_id": "function.test.area_of_circle", "fqn": [ "test", "area_of_circle" ], "alias": "area_of_circle", "checksum": { "name": "sha256", "checksum": "5f20c7f9790d05c41d31e0e4032104143372a588e9ca7ca04a337f4b652a6e84" }, "config": { "enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "function", "incremental_strategy": null, "batch_size": null, "lookback": 1, "begin": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": { "show": true, "node_color": null }, "contract": { "enforced": false, "alias_types": true }, "event_time": null, "concurrent_batches": null, "type": "scalar", "volatility": null, "runtime_version": null, "entry_point": null }, "tags": [], "description": "Calculates the area of a circle for a given radius", "columns": {}, "meta": {}, "group": null, "docs": { "show": true, "node_color": null }, "patch_path": "test://functions/area_of_circle.yml", "build_path": null, "unrendered_config": { "enabled": true }, "created_at": 1763574306.4420989, "relation_name": null, "raw_code": "SELECT pi() * radius * radius", "doc_blocks": [], "language": "sql", "refs": [], "sources": [], "metrics": [], "functions": [], "depends_on": { "macros": [], "nodes": [] }, "compiled_path": null, "contract": { "enforced": false, "alias_types": true, "checksum": null }, "arguments": [ { "name": "radius", "data_type": "float", "description": "A floating point number representing the radius of the circle" } ] } } } ================================================ FILE: tests/functional/defer_state/fixtures.py ================================================ seed_csv = """id,name 1,Alice 2,Bob """ table_model_sql = """ {{ config(materialized='table') }} select * from {{ ref('ephemeral_model') }} -- establish a macro dependency to trigger state:modified.macros -- depends on: {{ my_macro() }} """ table_model_now_view_sql = """ {{ config(materialized='view') }} select * from {{ ref('ephemeral_model') }} -- establish a macro dependency to trigger state:modified.macros -- depends on: {{ my_macro() }} """ table_model_now_incremental_sql = """ {{ config(materialized='incremental', on_schema_change='append_new_columns') }} select * from {{ ref('ephemeral_model') }} -- establish a macro dependency to trigger state:modified.macros -- depends on: {{ my_macro() }} """ changed_table_model_sql = """ {{ config(materialized='table') }} select 1 as fun """ view_model_sql = """ select * from {{ ref('seed') }} -- establish a macro dependency that trips infinite recursion if not handled -- depends on: {{ my_infinitely_recursive_macro() }} """ view_model_now_table_sql = """ {{ config(materialized='table') }} select * from {{ ref('seed') }} -- establish a macro dependency that trips infinite recursion if not handled -- depends on: {{ my_infinitely_recursive_macro() }} """ changed_view_model_sql = """ select * from no.such.table """ ephemeral_model_sql = """ {{ config(materialized='ephemeral') }} select * from {{ ref('view_model') }} """ changed_ephemeral_model_sql = """ {{ config(materialized='ephemeral') }} select * from no.such.table """ schema_yml = """ version: 2 models: - name: view_model columns: - name: id data_tests: - unique: config: severity: error - not_null - name: name """ no_contract_schema_yml = """ version: 2 models: - name: table_model config: {} columns: - name: id data_type: integer data_tests: - unique: severity: error - not_null - name: name data_type: text """ contract_schema_yml = """ version: 2 models: - name: table_model config: contract: enforced: True columns: - name: id data_type: integer data_tests: - unique: severity: error - not_null - name: name data_type: text """ disabled_contract_schema_yml = """ version: 2 models: - name: table_model config: contract: enforced: True enabled: False columns: - name: id data_type: integer data_tests: - unique: severity: error - not_null - name: name data_type: text """ modified_contract_schema_yml = """ version: 2 models: - name: table_model config: contract: enforced: True columns: - name: id data_type: integer data_tests: - unique: severity: error - not_null - name: user_name data_type: text """ unenforced_contract_schema_yml = """ version: 2 models: - name: table_model config: contract: enforced: False columns: - name: id data_type: integer data_tests: - unique: severity: error - not_null - name: name data_type: text """ disabled_unenforced_contract_schema_yml = """ version: 2 models: - name: table_model config: contract: enforced: False enabled: False columns: - name: id data_type: integer data_tests: - unique: severity: error - not_null - name: name data_type: text """ versioned_no_contract_schema_yml = """ version: 2 models: - name: table_model config: {} versions: - v: 1 columns: - name: id data_type: integer data_tests: - unique: severity: error - not_null - name: name data_type: text """ versioned_contract_schema_yml = """ version: 2 models: - name: table_model config: contract: enforced: True versions: - v: 1 columns: - name: id data_type: integer data_tests: - unique: severity: error - not_null - name: name data_type: text """ disabled_versioned_contract_schema_yml = """ version: 2 models: - name: table_model config: contract: enforced: True enabled: False versions: - v: 1 columns: - name: id data_type: integer data_tests: - unique: severity: error - not_null - name: name data_type: text """ versioned_modified_contract_schema_yml = """ version: 2 models: - name: table_model config: contract: enforced: True versions: - v: 1 columns: - name: id data_type: integer data_tests: - unique: severity: error - not_null - name: user_name data_type: text """ disabled_versioned_unenforced_contract_schema_yml = """ version: 2 models: - name: table_model config: contract: enforced: False enabled: False versions: - v: 1 columns: - name: id data_type: integer data_tests: - unique: severity: error - not_null - name: name data_type: text """ versioned_unenforced_contract_schema_yml = """ version: 2 models: - name: table_model config: contract: enforced: False versions: - v: 1 columns: - name: id data_type: integer data_tests: - unique: severity: error - not_null - name: name data_type: text """ constraint_schema_yml = """ version: 2 models: - name: view_model columns: - name: id data_tests: - unique: severity: error - not_null - name: name - name: table_model config: contract: enforced: True constraints: - type: primary_key columns: [id] columns: - name: id constraints: - type: not_null data_type: integer data_tests: - unique: severity: error - not_null - name: name data_type: text """ modified_column_constraint_schema_yml = """ version: 2 models: - name: view_model columns: - name: id data_tests: - unique: severity: error - not_null - name: name - name: table_model config: contract: enforced: True constraints: - type: primary_key columns: [id] columns: - name: id data_type: integer data_tests: - unique: severity: error - not_null - name: name data_type: text """ modified_model_constraint_schema_yml = """ version: 2 models: - name: view_model columns: - name: id data_tests: - unique: severity: error - not_null - name: name - name: table_model config: contract: enforced: True columns: - name: id constraints: - type: not_null data_type: integer data_tests: - unique: severity: error - not_null - name: name data_type: text """ exposures_yml = """ version: 2 exposures: - name: my_exposure type: application depends_on: - ref('view_model') owner: email: test@example.com """ macros_sql = """ {% macro my_macro() %} {% do log('in a macro' ) %} {% endmacro %} """ infinite_macros_sql = """ {# trigger infinite recursion if not handled #} {% macro my_infinitely_recursive_macro() %} {{ return(adapter.dispatch('my_infinitely_recursive_macro')()) }} {% endmacro %} {% macro default__my_infinitely_recursive_macro() %} {% if unmet_condition %} {{ my_infinitely_recursive_macro() }} {% else %} {{ return('') }} {% endif %} {% endmacro %} """ snapshot_sql = """ {% snapshot my_cool_snapshot %} {{ config( target_database=database, target_schema=schema, unique_key='id', strategy='check', check_cols=['id'], ) }} select * from {{ ref('view_model') }} {% endsnapshot %} """ semantic_model_schema_yml = """ models: - name: view_model columns: - name: id data_tests: - unique: severity: error - not_null - name: name semantic_models: - name: my_sm model: ref('view_model') """ modified_semantic_model_schema_yml = """ models: - name: view_model columns: - name: id data_tests: - unique: severity: error - not_null - name: name semantic_models: - name: my_sm model: ref('view_model') description: modified description """ model_1_sql = """ select * from {{ ref('seed') }} """ modified_model_1_sql = """ select * from {{ ref('seed') }} order by 1 """ model_2_sql = """ select id from {{ ref('model_1') }} """ modified_model_2_sql = """ select * from {{ ref('model_1') }} order by 1 """ group_schema_yml = """ groups: - name: finance owner: email: finance@jaffleshop.com models: - name: model_1 config: group: finance - name: model_2 config: group: finance """ group_modified_schema_yml = """ groups: - name: accounting owner: email: finance@jaffleshop.com models: - name: model_1 config: group: accounting - name: model_2 config: group: accounting """ group_modified_fail_schema_yml = """ groups: - name: finance owner: email: finance@jaffleshop.com models: - name: model_1 config: group: accounting - name: model_2 config: group: finance """ metricflow_time_spine_sql = """ SELECT to_date('02/20/2023', 'mm/dd/yyyy') as date_day """ model_with_env_var_in_config_sql = """ {{ config(materialized=env_var('DBT_TEST_STATE_MODIFIED')) }} select 1 as id """ model_with_no_in_config_sql = """ select 1 as id """ schema_model_with_env_var_in_config_yml = """ models: - name: model config: materialized: "{{ env_var('DBT_TEST_STATE_MODIFIED') }}" """ model_with_var_in_config_sql = """ {{ config(materialized=var('DBT_TEST_STATE_MODIFIED')) }} select 1 as id """ model_with_jinja_in_config_sql = """ {{ config( materialized = ('table' if execute else 'view') ) }} select 1 as id """ model_with_updated_jinja_in_config_sql = """ {{ config( materialized = ('view' if execute else 'table') ) }} select 1 as id """ schema_model_with_jinja_in_config_yml = """ models: - name: model config: materialized: "{{ ('table' if execute else 'view') }}" """ schema_model_with_updated_jinja_in_config_yml = """ models: - name: model config: materialized: "{{ ('view' if execute else 'table') }}" """ schema_source_with_env_var_as_database_property_yml = """ sources: - name: jaffle_shop database: "{{ env_var('DBT_TEST_STATE_MODIFIED') }}" tables: - name: customers """ schema_source_with_env_var_as_schema_property_yml = """ sources: - name: jaffle_shop database: "test" schema: "{{ env_var('DBT_TEST_STATE_MODIFIED') }}" tables: - name: customers """ schema_source_with_updated_env_var_as_schema_property_yml = """ sources: - name: jaffle_shop database: "test" schema: "updated" tables: - name: customers """ schema_source_with_jinja_as_database_property_yml = """ sources: - name: jaffle_shop database: "{{ ('foo' if execute else 'bar') }}" tables: - name: customers """ schema_source_with_updated_jinja_as_database_property_yml = """ sources: - name: jaffle_shop database: "{{ ('bar' if execute else 'foo') }}" tables: - name: customers """ schema_source_with_jinja_as_schema_property_yml = """ sources: - name: jaffle_shop database: "test" schema: "{{ ('foo' if execute else 'bar') }}" tables: - name: customers """ schema_source_with_updated_jinja_as_schema_property_yml = """ sources: - name: jaffle_shop database: "test" schema: "{{ ('bar' if execute else 'foo') }}" tables: - name: customers """ # Fixtures for test_removed_test_state.py sample_test_sql = """ {% test sample_test(model, column_name) %} select * from {{ model }} where {{ column_name }} is null {% endtest %} """ removed_test_model_sql = """ select 1 as id """ removed_test_schema_yml = """ version: 2 models: - name: model_a columns: - name: id data_tests: - sample_test """ # Fixtures for test_modified_state.py - varchar/numeric size changes varchar_size_contract_schema_yml = """ version: 2 models: - name: table_model config: contract: enforced: true versions: - v: 1 columns: - name: id data_type: integer - name: name data_type: varchar(5) """ varchar_size_increased_contract_schema_yml = """ version: 2 models: - name: table_model config: contract: enforced: true versions: - v: 1 columns: - name: id data_type: integer - name: name data_type: varchar(20) """ numeric_precision_contract_schema_yml = """ version: 2 models: - name: table_model config: contract: enforced: true versions: - v: 1 columns: - name: id data_type: integer - name: amount data_type: numeric(10,2) """ numeric_precision_increased_contract_schema_yml = """ version: 2 models: - name: table_model config: contract: enforced: true versions: - v: 1 columns: - name: id data_type: integer - name: amount data_type: numeric(12,4) """ # Case sensitivity test fixtures varchar_size_uppercase_contract_schema_yml = """ version: 2 models: - name: table_model config: contract: enforced: true versions: - v: 1 columns: - name: id data_type: integer - name: name data_type: VARCHAR(5) """ varchar_size_lowercase_increased_contract_schema_yml = """ version: 2 models: - name: table_model config: contract: enforced: true versions: - v: 1 columns: - name: id data_type: integer - name: name data_type: varchar(20) """ # Unversioned model fixtures varchar_size_unversioned_contract_schema_yml = """ version: 2 models: - name: table_model config: contract: enforced: true columns: - name: id data_type: integer - name: name data_type: varchar(5) """ varchar_size_unversioned_increased_contract_schema_yml = """ version: 2 models: - name: table_model config: contract: enforced: true columns: - name: id data_type: integer - name: name data_type: varchar(20) """ double_it_sql = """ SELECT value * 2 """ double_it_yml = """ functions: - name: double_it description: Doubles whatever number is passed in arguments: - name: value data_type: float description: A number to be doubled returns: data_type: float """ ================================================ FILE: tests/functional/defer_state/test_defer_state.py ================================================ import os import shutil from copy import deepcopy from typing import Dict import pytest from dbt.contracts.results import RunStatus from dbt.exceptions import DbtRuntimeError from dbt.tests.util import rm_file, run_dbt, write_file from tests.functional.defer_state.fixtures import ( changed_ephemeral_model_sql, changed_table_model_sql, changed_view_model_sql, double_it_sql, double_it_yml, ephemeral_model_sql, exposures_yml, infinite_macros_sql, macros_sql, schema_yml, seed_csv, snapshot_sql, table_model_sql, view_model_now_table_sql, view_model_sql, ) class BaseDeferState: _models = { "table_model.sql": table_model_sql, "view_model.sql": view_model_sql, "ephemeral_model.sql": ephemeral_model_sql, "schema.yml": schema_yml, "exposures.yml": exposures_yml, } @pytest.fixture(scope="class") def models(self): return self._models @pytest.fixture(scope="class") def macros(self): return { "macros.sql": macros_sql, "infinite_macros.sql": infinite_macros_sql, } @pytest.fixture(scope="class") def seeds(self): return { "seed.csv": seed_csv, } @pytest.fixture(scope="class") def snapshots(self): return { "snapshot.sql": snapshot_sql, } @pytest.fixture(scope="class") def other_schema(self, unique_schema): return unique_schema + "_other" @property def project_config_update(self): return { "seeds": { "test": { "quote_columns": False, } } } @pytest.fixture(scope="class") def profiles_config_update(self, dbt_profile_target, unique_schema, other_schema): outputs = {"default": dbt_profile_target, "otherschema": deepcopy(dbt_profile_target)} outputs["default"]["schema"] = unique_schema outputs["otherschema"]["schema"] = other_schema return {"test": {"outputs": outputs, "target": "default"}} def copy_state(self, project_root): state_path = os.path.join(project_root, "state") if not os.path.exists(state_path): os.makedirs(state_path) shutil.copyfile( f"{project_root}/target/manifest.json", f"{project_root}/state/manifest.json" ) def run_and_save_state(self, project_root, with_snapshot=False): results = run_dbt(["seed"]) assert len(results) == 1 results = run_dbt(["run"]) assert len(results) == 2 results = run_dbt(["test"]) assert len(results) == 2 if with_snapshot: results = run_dbt(["snapshot"]) assert len(results) == 1 # copy files self.copy_state(project_root) class TestDeferStateUnsupportedCommands(BaseDeferState): def test_no_state(self, project): # no "state" files present, snapshot fails with pytest.raises(DbtRuntimeError): run_dbt(["snapshot", "--state", "state", "--defer"]) class TestRunCompileState(BaseDeferState): def test_run_and_compile_defer(self, project): self.run_and_save_state(project.project_root) # defer test, it succeeds # Change directory to ensure that state directory is underneath # project directory. os.chdir(project.profiles_dir) results = run_dbt(["compile", "--state", "state", "--defer"]) assert len(results.results) == 6 assert results.results[0].node.name == "seed" class TestSnapshotState(BaseDeferState): def test_snapshot_state_defer(self, project): self.run_and_save_state(project.project_root) # snapshot succeeds without --defer run_dbt(["snapshot"]) # copy files self.copy_state(project.project_root) # defer test, it succeeds run_dbt(["snapshot", "--state", "state", "--defer"]) # favor_state test, it succeeds run_dbt(["snapshot", "--state", "state", "--defer", "--favor-state"]) class TestRunDeferState(BaseDeferState): def test_run_and_defer(self, project, unique_schema, other_schema): project.create_test_schema(other_schema) self.run_and_save_state(project.project_root) # test tests first, because run will change things # no state, wrong schema, failure. run_dbt(["test", "--target", "otherschema"], expect_pass=False) # test generate docs # no state, wrong schema, empty nodes catalog = run_dbt(["docs", "generate", "--target", "otherschema"]) assert not catalog.nodes # no state, run also fails run_dbt(["run", "--target", "otherschema"], expect_pass=False) # defer test, it succeeds results = run_dbt( ["test", "-m", "view_model+", "--state", "state", "--defer", "--target", "otherschema"] ) # defer docs generate with state, catalog refers schema from the happy times catalog = run_dbt( [ "docs", "generate", "-m", "view_model+", "--state", "state", "--defer", "--target", "otherschema", ] ) assert "seed.test.seed" not in catalog.nodes # with state it should work though results = run_dbt( ["run", "-m", "view_model", "--state", "state", "--defer", "--target", "otherschema"] ) assert other_schema not in results[0].node.compiled_code assert unique_schema in results[0].node.compiled_code assert len(results) == 1 class TestRunDeferStateChangedModel(BaseDeferState): def test_run_defer_state_changed_model(self, project): self.run_and_save_state(project.project_root) # change "view_model" write_file(changed_view_model_sql, "models", "view_model.sql") # the sql here is just wrong, so it should fail run_dbt( ["run", "-m", "view_model", "--state", "state", "--defer", "--target", "otherschema"], expect_pass=False, ) # but this should work since we just use the old happy model run_dbt( ["run", "-m", "table_model", "--state", "state", "--defer", "--target", "otherschema"], expect_pass=True, ) # change "ephemeral_model" write_file(changed_ephemeral_model_sql, "models", "ephemeral_model.sql") # this should fail because the table model refs a broken ephemeral # model, which it should see run_dbt( ["run", "-m", "table_model", "--state", "state", "--defer", "--target", "otherschema"], expect_pass=False, ) class TestRunDeferStateIFFNotExists(BaseDeferState): def test_run_defer_iff_not_exists(self, project, unique_schema, other_schema): project.create_test_schema(other_schema) self.run_and_save_state(project.project_root) results = run_dbt(["seed", "--target", "otherschema"]) assert len(results) == 1 results = run_dbt(["run", "--state", "state", "--defer", "--target", "otherschema"]) assert len(results) == 2 # because the seed now exists in our "other" schema, we should prefer it over the one # available from state assert other_schema in results[0].node.compiled_code # this time with --favor-state: even though the seed now exists in our "other" schema, # we should still favor the one available from state results = run_dbt( ["run", "--state", "state", "--defer", "--favor-state", "--target", "otherschema"] ) assert len(results) == 2 assert other_schema not in results[0].node.compiled_code # again with --favor-state, but this time select both the seed and the view # because the seed is also selected, the view should select from the seed in our schema ('other_schema') results = run_dbt( [ "build", "--state", "state", "--select", "seed view_model", "--resource-type", "seed model", "--defer", "--favor-state", "--target", "otherschema", ] ) assert len(results) == 2 assert other_schema in results[1].node.compiled_code class TestDeferStateDeletedUpstream(BaseDeferState): def test_run_defer_deleted_upstream(self, project, unique_schema, other_schema): project.create_test_schema(other_schema) self.run_and_save_state(project.project_root) # remove "ephemeral_model" + change "table_model" rm_file("models", "ephemeral_model.sql") write_file(changed_table_model_sql, "models", "table_model.sql") # ephemeral_model is now gone. previously this caused a # keyerror (dbt#2875), now it should pass run_dbt( ["run", "-m", "view_model", "--state", "state", "--defer", "--target", "otherschema"], expect_pass=True, ) # despite deferral, we should use models just created in our schema results = run_dbt(["test", "--state", "state", "--defer", "--target", "otherschema"]) assert other_schema in results[0].node.compiled_code # this time with --favor-state: prefer the models in the "other" schema, even though they exist in ours run_dbt( [ "run", "-m", "view_model", "--state", "state", "--defer", "--favor-state", "--target", "otherschema", ], expect_pass=True, ) results = run_dbt(["test", "--state", "state", "--defer", "--favor-state"]) assert other_schema not in results[0].node.compiled_code class TestDeferStateFlag(BaseDeferState): def test_defer_state_flag(self, project, unique_schema, other_schema): project.create_test_schema(other_schema) # test that state deferral works correctly run_dbt(["compile", "--target-path", "target_compile"]) write_file(view_model_now_table_sql, "models", "table_model.sql") results = run_dbt(["ls", "--select", "state:modified", "--state", "target_compile"]) assert results == ["test.table_model"] run_dbt(["seed", "--target", "otherschema", "--target-path", "target_otherschema"]) # this will fail because we haven't loaded the seed in the default schema run_dbt( [ "run", "--select", "state:modified", "--defer", "--state", "target_compile", "--favor-state", ], expect_pass=False, ) # Test that retry of a defer command works run_dbt(["retry"], expect_pass=False) # this will fail because we haven't passed in --state with pytest.raises( DbtRuntimeError, match="Got a state selector method, but no comparison manifest" ): run_dbt( [ "run", "--select", "state:modified", "--defer", "--defer-state", "target_otherschema", "--favor-state", ], expect_pass=False, ) # this will succeed because we've loaded the seed in other schema and are successfully deferring to it instead results = run_dbt( [ "run", "--select", "state:modified", "--defer", "--state", "target_compile", "--defer-state", "target_otherschema", "--favor-state", ] ) assert len(results.results) == 1 assert results.results[0].status == RunStatus.Success assert results.results[0].node.name == "table_model" assert results.results[0].adapter_response["rows_affected"] == 2 class TestFunctionDeferral(BaseDeferState): @pytest.fixture(scope="class") def functions(self) -> Dict[str, str]: return { "double_it.sql": double_it_sql, "double_it.yml": double_it_yml, } @pytest.fixture(scope="class") def models(self): return { **self._models, "double_it_model.sql": "select {{ function('double_it') }}(1) as double_it", } def test_build_with_other_schema_defer(self, project, other_schema): project.create_test_schema(other_schema) run_dbt(["build"]) self.copy_state(project.project_root) result = run_dbt( [ "run", "-s", "double_it_model", "--state", "state", "--defer", "--target", "otherschema", ] ) assert len(result.results) == 1 assert result.results[0].node.name == "double_it_model" ================================================ FILE: tests/functional/defer_state/test_group_updates.py ================================================ import os import pytest from dbt.exceptions import ParsingError from dbt.tests.util import copy_file, run_dbt, write_file from tests.functional.defer_state.fixtures import ( group_modified_fail_schema_yml, group_modified_schema_yml, group_schema_yml, model_1_sql, model_2_sql, modified_model_1_sql, modified_model_2_sql, seed_csv, ) class GroupSetup: @pytest.fixture(scope="class") def models(self): return { "model_1.sql": model_1_sql, "model_2.sql": model_2_sql, "schema.yml": group_schema_yml, } @pytest.fixture(scope="class") def seeds(self): return { "seed.csv": seed_csv, } def group_setup(self): # save initial state run_dbt(["seed"]) results = run_dbt(["compile"]) # add sanity checks for first result assert len(results) == 3 seed_result = results[0].node assert seed_result.unique_id == "seed.test.seed" model_1_result = results[1].node assert model_1_result.unique_id == "model.test.model_1" assert model_1_result.group == "finance" model_2_result = results[2].node assert model_2_result.unique_id == "model.test.model_2" assert model_2_result.group == "finance" class TestFullyModifiedGroups(GroupSetup): def test_changed_groups(self, project): self.group_setup() # copy manifest.json to "state" directory os.makedirs("state") target_path = os.path.join(project.project_root, "target") copy_file(target_path, "manifest.json", project.project_root, ["state", "manifest.json"]) # update group name, modify model so it gets picked up write_file(modified_model_1_sql, "models", "model_1.sql") write_file(modified_model_2_sql, "models", "model_2.sql") write_file(group_modified_schema_yml, "models", "schema.yml") # this test is flaky if you don't clean first before the build run_dbt(["clean"]) # only thing in results should be model_1 results = run_dbt(["build", "-s", "state:modified", "--defer", "--state", "./state"]) assert len(results) == 2 model_1_result = results[0].node assert model_1_result.unique_id == "model.test.model_1" assert model_1_result.group == "accounting" # new group name! model_2_result = results[1].node assert model_2_result.unique_id == "model.test.model_2" assert model_2_result.group == "accounting" # new group name! class TestPartiallyModifiedGroups(GroupSetup): def test_changed_groups(self, project): self.group_setup() # copy manifest.json to "state" directory os.makedirs("state") target_path = os.path.join(project.project_root, "target") copy_file(target_path, "manifest.json", project.project_root, ["state", "manifest.json"]) # update group name, modify model so it gets picked up write_file(modified_model_1_sql, "models", "model_1.sql") write_file(group_modified_schema_yml, "models", "schema.yml") # this test is flaky if you don't clean first before the build run_dbt(["clean"]) # only thing in results should be model_1 results = run_dbt(["build", "-s", "state:modified", "--defer", "--state", "./state"]) assert len(results) == 1 model_1_result = results[0].node assert model_1_result.unique_id == "model.test.model_1" assert model_1_result.group == "accounting" # new group name! class TestBadGroups(GroupSetup): def test_changed_groups(self, project): self.group_setup() # copy manifest.json to "state" directory os.makedirs("state") target_path = os.path.join(project.project_root, "target") copy_file(target_path, "manifest.json", project.project_root, ["state", "manifest.json"]) # update group with invalid name, modify model so it gets picked up write_file(modified_model_1_sql, "models", "model_1.sql") write_file(group_modified_fail_schema_yml, "models", "schema.yml") # this test is flaky if you don't clean first before the build run_dbt(["clean"]) with pytest.raises(ParsingError, match="Invalid group 'accounting'"): run_dbt(["build", "-s", "state:modified", "--defer", "--state", "./state"]) ================================================ FILE: tests/functional/defer_state/test_modified_state.py ================================================ import os import random import shutil import string import pytest from dbt.exceptions import CompilationError, ContractBreakingChangeError from dbt.tests.util import ( get_manifest, rm_file, run_dbt, run_dbt_and_capture, update_config_file, write_file, ) from tests.functional.defer_state.fixtures import ( constraint_schema_yml, contract_schema_yml, disabled_contract_schema_yml, disabled_unenforced_contract_schema_yml, disabled_versioned_contract_schema_yml, disabled_versioned_unenforced_contract_schema_yml, ephemeral_model_sql, exposures_yml, infinite_macros_sql, macros_sql, metricflow_time_spine_sql, modified_column_constraint_schema_yml, modified_contract_schema_yml, modified_model_constraint_schema_yml, modified_semantic_model_schema_yml, no_contract_schema_yml, numeric_precision_contract_schema_yml, numeric_precision_increased_contract_schema_yml, schema_yml, seed_csv, semantic_model_schema_yml, table_model_now_incremental_sql, table_model_now_view_sql, table_model_sql, unenforced_contract_schema_yml, varchar_size_contract_schema_yml, varchar_size_increased_contract_schema_yml, varchar_size_lowercase_increased_contract_schema_yml, varchar_size_unversioned_contract_schema_yml, varchar_size_unversioned_increased_contract_schema_yml, varchar_size_uppercase_contract_schema_yml, versioned_contract_schema_yml, versioned_modified_contract_schema_yml, versioned_no_contract_schema_yml, versioned_unenforced_contract_schema_yml, view_model_now_table_sql, view_model_sql, ) class BaseModifiedState: @pytest.fixture(scope="class") def models(self): return { "table_model.sql": table_model_sql, "view_model.sql": view_model_sql, "ephemeral_model.sql": ephemeral_model_sql, "schema.yml": schema_yml, "exposures.yml": exposures_yml, } @pytest.fixture(scope="class") def macros(self): return { "macros.sql": macros_sql, "infinite_macros.sql": infinite_macros_sql, } @pytest.fixture(scope="class") def seeds(self): return { "seed.csv": seed_csv, } @property def project_config_update(self): return { "seeds": { "test": { "quote_columns": False, } } } def copy_state(self): if not os.path.exists("state"): os.makedirs("state") shutil.copyfile("target/manifest.json", "state/manifest.json") def run_and_save_state(self): run_dbt(["seed"]) run_dbt(["run"]) self.copy_state() class TestChangedSeedContents(BaseModifiedState): def test_changed_seed_contents_state(self, project): self.run_and_save_state() results = run_dbt( ["ls", "--resource-type", "seed", "--select", "state:modified", "--state", "./state"], expect_pass=True, ) assert len(results) == 0 results = run_dbt( [ "ls", "--resource-type", "seed", "--exclude", "state:unmodified", "--state", "./state", ], expect_pass=True, ) assert len(results) == 0 results = run_dbt( [ "ls", "--resource-type", "seed", "--select", "state:unmodified", "--state", "./state", ], expect_pass=True, ) assert len(results) == 1 # add a new row to the seed changed_seed_contents = seed_csv + "\n" + "3,carl" write_file(changed_seed_contents, "seeds", "seed.csv") results = run_dbt( ["ls", "--resource-type", "seed", "--select", "state:modified", "--state", "./state"] ) assert len(results) == 1 assert results[0] == "test.seed" results = run_dbt( [ "ls", "--resource-type", "seed", "--exclude", "state:unmodified", "--state", "./state", ] ) assert len(results) == 1 assert results[0] == "test.seed" results = run_dbt( ["ls", "--resource-type", "seed", "--select", "state:unmodified", "--state", "./state"] ) assert len(results) == 0 results = run_dbt(["ls", "--select", "state:modified", "--state", "./state"]) assert len(results) == 1 assert results[0] == "test.seed" results = run_dbt(["ls", "--exclude", "state:unmodified", "--state", "./state"]) assert len(results) == 1 assert results[0] == "test.seed" results = run_dbt(["ls", "--select", "state:unmodified", "--state", "./state"]) assert len(results) == 6 results = run_dbt(["ls", "--select", "state:modified+", "--state", "./state"]) assert len(results) == 7 assert set(results) == { "test.seed", "test.table_model", "test.view_model", "test.ephemeral_model", "test.not_null_view_model_id", "test.unique_view_model_id", "exposure:test.my_exposure", } results = run_dbt(["ls", "--select", "state:unmodified+", "--state", "./state"]) assert len(results) == 6 assert set(results) == { "test.table_model", "test.view_model", "test.ephemeral_model", "test.not_null_view_model_id", "test.unique_view_model_id", "exposure:test.my_exposure", } shutil.rmtree("./state") self.copy_state() # make a very big seed # assume each line is ~2 bytes + len(name) target_size = 1 * 1024 * 1024 line_size = 64 num_lines = target_size // line_size maxlines = num_lines + 4 seed_lines = [seed_csv] for idx in range(4, maxlines): value = "".join(random.choices(string.ascii_letters, k=62)) seed_lines.append(f"{idx},{value}") seed_contents = "\n".join(seed_lines) write_file(seed_contents, "seeds", "seed.csv") # now if we run again, we should get a warning results = run_dbt( ["ls", "--resource-type", "seed", "--select", "state:modified", "--state", "./state"] ) assert len(results) == 1 assert results[0] == "test.seed" with pytest.raises(CompilationError) as exc: run_dbt( [ "--warn-error", "ls", "--resource-type", "seed", "--select", "state:modified", "--state", "./state", ] ) assert ">1MB" in str(exc.value) # now check if unmodified returns none results = run_dbt( ["ls", "--resource-type", "seed", "--select", "state:unmodified", "--state", "./state"] ) assert len(results) == 0 shutil.rmtree("./state") self.copy_state() # once it"s in path mode, we don"t mark it as modified if it changes write_file(seed_contents + "\n1,test", "seeds", "seed.csv") results = run_dbt( ["ls", "--resource-type", "seed", "--select", "state:modified", "--state", "./state"], expect_pass=True, ) assert len(results) == 0 results = run_dbt( [ "ls", "--resource-type", "seed", "--exclude", "state:unmodified", "--state", "./state", ], expect_pass=True, ) assert len(results) == 0 results = run_dbt( [ "ls", "--resource-type", "seed", "--select", "state:unmodified", "--state", "./state", ], expect_pass=True, ) assert len(results) == 1 class TestChangedSeedConfig(BaseModifiedState): def test_changed_seed_config(self, project): self.run_and_save_state() results = run_dbt( ["ls", "--resource-type", "seed", "--select", "state:modified", "--state", "./state"], expect_pass=True, ) assert len(results) == 0 results = run_dbt( [ "ls", "--resource-type", "seed", "--exclude", "state:unmodified", "--state", "./state", ], expect_pass=True, ) assert len(results) == 0 results = run_dbt( [ "ls", "--resource-type", "seed", "--select", "state:unmodified", "--state", "./state", ], expect_pass=True, ) assert len(results) == 1 update_config_file({"seeds": {"test": {"quote_columns": False}}}, "dbt_project.yml") # quoting change -> seed changed results = run_dbt( ["ls", "--resource-type", "seed", "--select", "state:modified", "--state", "./state"] ) assert len(results) == 1 assert results[0] == "test.seed" results = run_dbt( [ "ls", "--resource-type", "seed", "--exclude", "state:unmodified", "--state", "./state", ] ) assert len(results) == 1 assert results[0] == "test.seed" results = run_dbt( ["ls", "--resource-type", "seed", "--select", "state:unmodified", "--state", "./state"] ) assert len(results) == 0 class TestUnrenderedConfigSame(BaseModifiedState): def test_unrendered_config_same(self, project): self.run_and_save_state() results = run_dbt( ["ls", "--resource-type", "model", "--select", "state:modified", "--state", "./state"], expect_pass=True, ) assert len(results) == 0 results = run_dbt( [ "ls", "--resource-type", "model", "--exclude", "state:unmodified", "--state", "./state", ], expect_pass=True, ) assert len(results) == 0 results = run_dbt( [ "ls", "--resource-type", "model", "--select", "state:unmodified", "--state", "./state", ], expect_pass=True, ) assert len(results) == 3 # although this is the default value, dbt will recognize it as a change # for previously-unconfigured models, because it"s been explicitly set update_config_file({"models": {"test": {"materialized": "view"}}}, "dbt_project.yml") results = run_dbt( ["ls", "--resource-type", "model", "--select", "state:modified", "--state", "./state"] ) assert len(results) == 1 assert results[0] == "test.view_model" # converse of above statement results = run_dbt( [ "ls", "--resource-type", "model", "--exclude", "state:unmodified", "--state", "./state", ] ) assert len(results) == 1 assert results[0] == "test.view_model" results = run_dbt( [ "ls", "--resource-type", "model", "--select", "state:unmodified", "--state", "./state", ] ) assert len(results) == 2 assert set(results) == { "test.table_model", "test.ephemeral_model", } class TestChangedModelContents(BaseModifiedState): def test_changed_model_contents(self, project): self.run_and_save_state() results = run_dbt(["run", "--models", "state:modified", "--state", "./state"]) assert len(results) == 0 table_model_update = """ {{ config(materialized="table") }} select * from {{ ref("seed") }} """ write_file(table_model_update, "models", "table_model.sql") results = run_dbt(["run", "--models", "state:modified", "--state", "./state"]) assert len(results) == 1 assert results[0].node.name == "table_model" results = run_dbt(["run", "--exclude", "state:unmodified", "--state", "./state"]) assert len(results) == 1 assert results[0].node.name == "table_model" class TestNewMacro(BaseModifiedState): def test_new_macro(self, project): self.run_and_save_state() new_macro = """ {% macro my_other_macro() %} {% endmacro %} """ # add a new macro to a new file write_file(new_macro, "macros", "second_macro.sql") results = run_dbt(["run", "--models", "state:modified", "--state", "./state"]) assert len(results) == 0 os.remove("macros/second_macro.sql") # add a new macro to the existing file with open("macros/macros.sql", "a") as fp: fp.write(new_macro) results = run_dbt(["run", "--models", "state:modified", "--state", "./state"]) assert len(results) == 0 results = run_dbt(["run", "--exclude", "state:unmodified", "--state", "./state"]) assert len(results) == 0 class TestChangedMacroContents(BaseModifiedState): def test_changed_macro_contents(self, project): self.run_and_save_state() # modify an existing macro updated_macro = """ {% macro my_macro() %} {% do log("in a macro", info=True) %} {% endmacro %} """ write_file(updated_macro, "macros", "macros.sql") # table_model calls this macro results = run_dbt(["run", "--models", "state:modified", "--state", "./state"]) assert len(results) == 1 results = run_dbt(["run", "--exclude", "state:unmodified", "--state", "./state"]) assert len(results) == 1 class TestChangedExposure(BaseModifiedState): def test_changed_exposure(self, project): self.run_and_save_state() # add an "owner.name" to existing exposure updated_exposure = exposures_yml + "\n name: John Doe\n" write_file(updated_exposure, "models", "exposures.yml") results = run_dbt(["run", "--models", "+state:modified", "--state", "./state"]) assert len(results) == 1 assert results[0].node.name == "view_model" results = run_dbt(["run", "--exclude", "state:unmodified", "--state", "./state"]) assert len(results) == 0 class TestChangedContractUnversioned(BaseModifiedState): MODEL_UNIQUE_ID = "model.test.table_model" CONTRACT_SCHEMA_YML = contract_schema_yml MODIFIED_SCHEMA_YML = modified_contract_schema_yml UNENFORCED_SCHEMA_YML = unenforced_contract_schema_yml NO_CONTRACT_SCHEMA_YML = no_contract_schema_yml def test_changed_contract(self, project): self.run_and_save_state() # update contract for table_model write_file(self.CONTRACT_SCHEMA_YML, "models", "schema.yml") # This will find the table_model node modified both through a config change # and by a non-breaking change to contract: true results = run_dbt(["run", "--models", "state:modified", "--state", "./state"]) assert len(results) == 1 assert results[0].node.name == "table_model" results = run_dbt(["run", "--exclude", "state:unmodified", "--state", "./state"]) assert len(results) == 1 assert results[0].node.name == "table_model" manifest = get_manifest(project.project_root) model_unique_id = self.MODEL_UNIQUE_ID model = manifest.nodes[model_unique_id] expected_unrendered_config = {"contract": {"enforced": True}, "materialized": "table"} assert model.unrendered_config == expected_unrendered_config # Run it again with "state:modified:contract", still finds modified due to contract: true results = run_dbt(["run", "--models", "state:modified.contract", "--state", "./state"]) assert len(results) == 1 manifest = get_manifest(project.project_root) model = manifest.nodes[model_unique_id] first_contract_checksum = model.contract.checksum assert first_contract_checksum # save a new state self.copy_state() # This should raise because a column name has changed write_file(self.MODIFIED_SCHEMA_YML, "models", "schema.yml") results = run_dbt(["run"], expect_pass=False) assert len(results) == 2 manifest = get_manifest(project.project_root) model = manifest.nodes[model_unique_id] second_contract_checksum = model.contract.checksum # double check different contract_checksums assert first_contract_checksum != second_contract_checksum _, logs = run_dbt_and_capture( ["run", "--models", "state:modified.contract", "--state", "./state"], expect_pass=False ) expected_error = "This model has an enforced contract that failed." expected_warning = "While comparing to previous project state, dbt detected a breaking change to an unversioned model" expected_change = "Please ensure the name, data_type, and number of columns in your contract match the columns in your model's definition" assert expected_error in logs assert expected_warning in logs assert expected_change in logs # Go back to schema file without contract. Should throw a warning. write_file(self.NO_CONTRACT_SCHEMA_YML, "models", "schema.yml") _, logs = run_dbt_and_capture( ["run", "--models", "state:modified.contract", "--state", "./state"] ) expected_warning = "While comparing to previous project state, dbt detected a breaking change to an unversioned model" expected_change = "Contract enforcement was removed" # Now unenforce the contract. Should throw a warning - force warning into an error. write_file(self.UNENFORCED_SCHEMA_YML, "models", "schema.yml") with pytest.raises(CompilationError): _, logs = run_dbt_and_capture( [ "--warn-error", "run", "--models", "state:modified.contract", "--state", "./state", ] ) expected_warning = "While comparing to previous project state, dbt detected a breaking change to an unversioned model" expected_change = "Contract enforcement was removed" class TestChangedContractVersioned(BaseModifiedState): MODEL_UNIQUE_ID = "model.test.table_model.v1" CONTRACT_SCHEMA_YML = versioned_contract_schema_yml MODIFIED_SCHEMA_YML = versioned_modified_contract_schema_yml UNENFORCED_SCHEMA_YML = versioned_unenforced_contract_schema_yml NO_CONTRACT_SCHEMA_YML = versioned_no_contract_schema_yml def test_changed_contract_versioned(self, project): self.run_and_save_state() # update contract for table_model write_file(self.CONTRACT_SCHEMA_YML, "models", "schema.yml") # This will find the table_model node modified both through a config change # and by a non-breaking change to contract: true results = run_dbt(["run", "--models", "state:modified", "--state", "./state"]) assert len(results) == 1 assert results[0].node.name == "table_model" results = run_dbt(["run", "--exclude", "state:unmodified", "--state", "./state"]) assert len(results) == 1 assert results[0].node.name == "table_model" manifest = get_manifest(project.project_root) model_unique_id = self.MODEL_UNIQUE_ID model = manifest.nodes[model_unique_id] expected_unrendered_config = {"contract": {"enforced": True}, "materialized": "table"} assert model.unrendered_config == expected_unrendered_config # Run it again with "state:modified:contract", still finds modified due to contract: true results = run_dbt(["run", "--models", "state:modified.contract", "--state", "./state"]) assert len(results) == 1 manifest = get_manifest(project.project_root) model = manifest.nodes[model_unique_id] first_contract_checksum = model.contract.checksum assert first_contract_checksum # save a new state self.copy_state() # This should raise because a column name has changed write_file(self.MODIFIED_SCHEMA_YML, "models", "schema.yml") results = run_dbt(["run"], expect_pass=False) assert len(results) == 2 manifest = get_manifest(project.project_root) model = manifest.nodes[model_unique_id] second_contract_checksum = model.contract.checksum # double check different contract_checksums assert first_contract_checksum != second_contract_checksum with pytest.raises(ContractBreakingChangeError): results = run_dbt(["run", "--models", "state:modified.contract", "--state", "./state"]) # Go back to schema file without contract. Should raise an error. write_file(self.NO_CONTRACT_SCHEMA_YML, "models", "schema.yml") with pytest.raises(ContractBreakingChangeError): results = run_dbt(["run", "--models", "state:modified.contract", "--state", "./state"]) # Now unenforce the contract. Should raise an error. write_file(self.UNENFORCED_SCHEMA_YML, "models", "schema.yml") with pytest.raises(ContractBreakingChangeError): results = run_dbt(["run", "--models", "state:modified.contract", "--state", "./state"]) class TestDeleteUnversionedContractedModel(BaseModifiedState): MODEL_UNIQUE_ID = "model.test.table_model" CONTRACT_SCHEMA_YML = contract_schema_yml def test_delete_unversioned_contracted_model(self, project): # ensure table_model is contracted write_file(self.CONTRACT_SCHEMA_YML, "models", "schema.yml") self.run_and_save_state() # delete versioned contracted model rm_file(project.project_root, "models", "table_model.sql") # since the models are unversioned, they raise a warning but not an error _, logs = run_dbt_and_capture( ["run", "--models", "state:modified.contract", "--state", "./state"] ) expected_warning = "While comparing to previous project state, dbt detected a breaking change to an unversioned model" expected_change = "Contracted model 'model.test.table_model' was deleted or renamed" assert expected_warning in logs assert expected_change in logs # the same but for general-purpose state:modified _, logs = run_dbt_and_capture( ["run", "--models", "state:modified.contract", "--state", "./state"] ) expected_warning = "While comparing to previous project state, dbt detected a breaking change to an unversioned model" expected_change = "Contracted model 'model.test.table_model' was deleted or renamed" assert expected_warning in logs assert expected_change in logs class TestDeleteVersionedContractedModel(BaseModifiedState): MODEL_UNIQUE_ID = "model.test.table_model.v1" CONTRACT_SCHEMA_YML = versioned_contract_schema_yml def test_delete_versioned_contracted_model(self, project): # ensure table_model is versioned + contracted write_file(self.CONTRACT_SCHEMA_YML, "models", "schema.yml") self.run_and_save_state() # delete versioned contracted model rm_file(project.project_root, "models", "table_model.sql") # since the models are versioned, they raise an error with pytest.raises(ContractBreakingChangeError) as e: run_dbt(["run", "--models", "state:modified.contract", "--state", "./state"]) assert "Contracted model 'model.test.table_model.v1' was deleted or renamed." in str( e.value ) # the same but for general-purpose state:modified with pytest.raises(ContractBreakingChangeError) as e: run_dbt(["run", "--models", "state:modified", "--state", "./state"]) assert "Contracted model 'model.test.table_model.v1' was deleted or renamed." in str( e.value ) class TestDisableUnversionedContractedModel(BaseModifiedState): MODEL_UNIQUE_ID = "model.test.table_model" CONTRACT_SCHEMA_YML = contract_schema_yml DISABLED_CONTRACT_SCHEMA_YML = disabled_contract_schema_yml def test_disable_unversioned_contracted_model(self, project): # ensure table_model is contracted and enabled write_file(self.CONTRACT_SCHEMA_YML, "models", "schema.yml") self.run_and_save_state() # disable unversioned + contracted model write_file(self.DISABLED_CONTRACT_SCHEMA_YML, "models", "schema.yml") # since the models are unversioned, they raise a warning but not an error _, logs = run_dbt_and_capture( ["run", "--models", "state:modified.contract", "--state", "./state"] ) expected_warning = "While comparing to previous project state, dbt detected a breaking change to an unversioned model" expected_change = "Contracted model 'model.test.table_model' was disabled" assert expected_warning in logs assert expected_change in logs class TestDisableVersionedContractedModel(BaseModifiedState): MODEL_UNIQUE_ID = "model.test.table_model.v1" CONTRACT_SCHEMA_YML = versioned_contract_schema_yml DISABLED_CONTRACT_SCHEMA_YML = disabled_versioned_contract_schema_yml def test_disable_versioned_contracted_model(self, project): # ensure table_model is versioned + contracted write_file(self.CONTRACT_SCHEMA_YML, "models", "schema.yml") self.run_and_save_state() # disable versioned + contracted model write_file(self.DISABLED_CONTRACT_SCHEMA_YML, "models", "schema.yml") # since the models are versioned, they raise an error with pytest.raises(ContractBreakingChangeError) as e: run_dbt(["run", "--models", "state:modified.contract", "--state", "./state"]) assert "Contracted model 'model.test.table_model.v1' was disabled." in str(e.value) class TestDisableUnversionedUncontractedModel(BaseModifiedState): MODEL_UNIQUE_ID = "model.test.table_model" NO_CONTRACT_SCHEMA_YML = unenforced_contract_schema_yml DISABLED_NO_CONTRACT_SCHEMA_YML = disabled_unenforced_contract_schema_yml def test_delete_versioned_contracted_model(self, project): # ensure table_model is not contracted write_file(self.NO_CONTRACT_SCHEMA_YML, "models", "schema.yml") self.run_and_save_state() # disable uncontracted model write_file(self.DISABLED_NO_CONTRACT_SCHEMA_YML, "models", "schema.yml") # since the models are unversioned, no warning or error is raised _, logs = run_dbt_and_capture( ["run", "--models", "state:modified.contract", "--state", "./state"] ) assert "breaking change" not in logs.lower() class TestDisableVersionedUncontractedModel(BaseModifiedState): MODEL_UNIQUE_ID = "model.test.table_model.v1" NO_CONTRACT_SCHEMA_YML = versioned_unenforced_contract_schema_yml DISABLED_NO_CONTRACT_SCHEMA_YML = disabled_versioned_unenforced_contract_schema_yml def test_delete_versioned_contracted_model(self, project): # ensure table_model is not contracted write_file(self.NO_CONTRACT_SCHEMA_YML, "models", "schema.yml") self.run_and_save_state() # disable uncontracted model write_file(self.DISABLED_NO_CONTRACT_SCHEMA_YML, "models", "schema.yml") # since the models are unversioned, no warning or error is raised run_dbt_and_capture(["run", "--models", "state:modified.contract", "--state", "./state"]) class TestChangedConstraintUnversioned(BaseModifiedState): def test_changed_constraint(self, project): self.run_and_save_state() # update constraint for table_model write_file(constraint_schema_yml, "models", "schema.yml") # This will find the table_model node modified both through adding constraint # and by a non-breaking change to contract: true results = run_dbt(["run", "--models", "state:modified", "--state", "./state"]) assert len(results) == 1 assert results[0].node.name == "table_model" results = run_dbt(["run", "--exclude", "state:unmodified", "--state", "./state"]) assert len(results) == 1 assert results[0].node.name == "table_model" manifest = get_manifest(project.project_root) model_unique_id = "model.test.table_model" model = manifest.nodes[model_unique_id] expected_unrendered_config = {"contract": {"enforced": True}, "materialized": "table"} assert model.unrendered_config == expected_unrendered_config # Run it again with "state:modified:contract", still finds modified due to contract: true results = run_dbt(["run", "--models", "state:modified.contract", "--state", "./state"]) assert len(results) == 1 manifest = get_manifest(project.project_root) model = manifest.nodes[model_unique_id] first_contract_checksum = model.contract.checksum assert first_contract_checksum # save a new state self.copy_state() # This should raise because a column level constraint was removed write_file(modified_column_constraint_schema_yml, "models", "schema.yml") # we don't have a way to know this failed unless we have a previous state to refer to, so the run succeeds results = run_dbt(["run"]) assert len(results) == 2 manifest = get_manifest(project.project_root) model = manifest.nodes[model_unique_id] second_contract_checksum = model.contract.checksum # double check different contract_checksums assert first_contract_checksum != second_contract_checksum # since the models are unversioned, they raise a warning but not an error _, logs = run_dbt_and_capture( ["run", "--models", "state:modified.contract", "--state", "./state"] ) expected_warning = "While comparing to previous project state, dbt detected a breaking change to an unversioned model" expected_change = "Enforced column level constraints were removed" assert expected_warning in logs assert expected_change in logs # This should raise because a model level constraint was removed (primary_key on id) write_file(modified_model_constraint_schema_yml, "models", "schema.yml") # we don't have a way to know this failed unless we have a previous state to refer to, so the run succeeds results = run_dbt(["run"]) assert len(results) == 2 manifest = get_manifest(project.project_root) model = manifest.nodes[model_unique_id] second_contract_checksum = model.contract.checksum # double check different contract_checksums assert first_contract_checksum != second_contract_checksum _, logs = run_dbt_and_capture( ["run", "--models", "state:modified.contract", "--state", "./state"] ) expected_warning = "While comparing to previous project state, dbt detected a breaking change to an unversioned model" expected_change = "Enforced model level constraints were removed" assert expected_warning in logs assert expected_change in logs class TestChangedMaterializationConstraint(BaseModifiedState): def test_changed_materialization(self, project): self.run_and_save_state() # update constraint for table_model write_file(constraint_schema_yml, "models", "schema.yml") # This will find the table_model node modified both through adding constraint # and by a non-breaking change to contract: true results = run_dbt(["run", "--models", "state:modified", "--state", "./state"]) assert len(results) == 1 assert results[0].node.name == "table_model" results = run_dbt(["run", "--exclude", "state:unmodified", "--state", "./state"]) assert len(results) == 1 assert results[0].node.name == "table_model" manifest = get_manifest(project.project_root) model_unique_id = "model.test.table_model" model = manifest.nodes[model_unique_id] expected_unrendered_config = {"contract": {"enforced": True}, "materialized": "table"} assert model.unrendered_config == expected_unrendered_config # Run it again with "state:modified:contract", still finds modified due to contract: true results = run_dbt(["run", "--models", "state:modified.contract", "--state", "./state"]) assert len(results) == 1 manifest = get_manifest(project.project_root) model = manifest.nodes[model_unique_id] first_contract_checksum = model.contract.checksum assert first_contract_checksum # save a new state self.copy_state() # This should raise because materialization changed from table to view write_file(table_model_now_view_sql, "models", "table_model.sql") # we don't have a way to know this failed unless we have a previous state to refer to, so the run succeeds results = run_dbt(["run"]) assert len(results) == 2 manifest = get_manifest(project.project_root) model = manifest.nodes[model_unique_id] second_contract_checksum = model.contract.checksum # double check different contract_checksums assert first_contract_checksum != second_contract_checksum _, logs = run_dbt_and_capture( ["run", "--models", "state:modified.contract", "--state", "./state"] ) expected_warning = "While comparing to previous project state, dbt detected a breaking change to an unversioned model" expected_change = "Materialization changed with enforced constraints" assert expected_warning in logs assert expected_change in logs # This should not raise because materialization changed from table to incremental, both enforce constraints write_file(table_model_now_incremental_sql, "models", "table_model.sql") # we don't have a way to know this failed unless we have a previous state to refer to, so the run succeeds results = run_dbt(["run"]) assert len(results) == 2 # This should pass because materialization changed from view to table which is the same as just adding new constraint, not breaking write_file(view_model_now_table_sql, "models", "view_model.sql") write_file(table_model_sql, "models", "table_model.sql") results = run_dbt(["run"]) assert len(results) == 2 manifest = get_manifest(project.project_root) model = manifest.nodes[model_unique_id] second_contract_checksum = model.contract.checksum # contract_checksums should be equal because we only save constraint related changes if the materialization is table/incremental assert first_contract_checksum == second_contract_checksum run_dbt(["run", "--models", "state:modified.contract", "--state", "./state"]) assert len(results) == 2 my_model_sql = """ select 1 as id """ modified_my_model_sql = """ -- a comment select 1 as id """ modified_my_model_non_breaking_sql = """ -- a comment select 1 as id, 'blue' as color """ my_model_yml = """ models: - name: my_model latest_version: 1 config: contract: enforced: true columns: - name: id data_type: int versions: - v: 1 """ modified_my_model_yml = """ models: - name: my_model latest_version: 1 config: contract: enforced: true columns: - name: id data_type: text versions: - v: 1 """ modified_my_model_non_breaking_yml = """ models: - name: my_model latest_version: 1 config: contract: enforced: true columns: - name: id data_type: int - name: color data_type: text versions: - v: 1 """ class TestModifiedBodyAndContract: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_model_sql, "my_model.yml": my_model_yml, } def copy_state(self): if not os.path.exists("state"): os.makedirs("state") shutil.copyfile("target/manifest.json", "state/manifest.json") def test_modified_body_and_contract(self, project): results = run_dbt(["run"]) assert len(results) == 1 self.copy_state() # Change both body and contract in a *breaking* way (= changing data_type of existing column) write_file(modified_my_model_yml, "models", "my_model.yml") write_file(modified_my_model_sql, "models", "my_model.sql") # Should raise even without specifying state:modified.contract with pytest.raises(ContractBreakingChangeError): results = run_dbt(["run", "-s", "state:modified", "--state", "./state"]) with pytest.raises(ContractBreakingChangeError): results = run_dbt(["run", "--exclude", "state:unmodified", "--state", "./state"]) # Change both body and contract in a *non-breaking* way (= adding a new column) write_file(modified_my_model_non_breaking_yml, "models", "my_model.yml") write_file(modified_my_model_non_breaking_sql, "models", "my_model.sql") # Should pass run_dbt(["run", "-s", "state:modified", "--state", "./state"]) # The model's contract has changed, even if non-breaking, so it should be selected by 'state:modified.contract' results = run_dbt(["list", "-s", "state:modified.contract", "--state", "./state"]) assert results == ["test.my_model.v1"] modified_table_model_access_yml = """ version: 2 models: - name: table_model access: public """ class TestModifiedAccess(BaseModifiedState): def test_changed_access(self, project): self.run_and_save_state() # No access change assert not run_dbt(["list", "-s", "state:modified", "--state", "./state"]) # Modify access (protected -> public) write_file(modified_table_model_access_yml, "models", "schema.yml") assert run_dbt(["list", "-s", "state:modified", "--state", "./state"]) results = run_dbt(["list", "-s", "state:modified", "--state", "./state"]) assert results == ["test.table_model"] modified_table_model_access_yml = """ version: 2 models: - name: table_model deprecation_date: 2020-01-01 """ class TestModifiedDeprecationDate(BaseModifiedState): def test_changed_access(self, project): self.run_and_save_state() # No access change assert not run_dbt(["list", "-s", "state:modified", "--state", "./state"]) # Modify deprecation_date (None -> 2020-01-01) write_file(modified_table_model_access_yml, "models", "schema.yml") assert run_dbt(["list", "-s", "state:modified", "--state", "./state"]) results = run_dbt(["list", "-s", "state:modified", "--state", "./state"]) assert results == ["test.table_model"] modified_table_model_version_yml = """ version: 2 models: - name: table_model versions: - v: 1 defined_in: table_model """ class TestModifiedVersion(BaseModifiedState): def test_changed_access(self, project): self.run_and_save_state() # Change version (null -> v1) write_file(modified_table_model_version_yml, "models", "schema.yml") results = run_dbt(["list", "-s", "state:modified", "--state", "./state"]) assert results == ["test.table_model.v1"] table_model_latest_version_yml = """ version: 2 models: - name: table_model latest_version: 1 versions: - v: 1 defined_in: table_model """ modified_table_model_latest_version_yml = """ version: 2 models: - name: table_model latest_version: 2 versions: - v: 1 defined_in: table_model - v: 2 """ class TestModifiedLatestVersion(BaseModifiedState): def test_changed_access(self, project): # Setup initial latest_version: 1 write_file(table_model_latest_version_yml, "models", "schema.yml") self.run_and_save_state() # Bump latest version write_file(table_model_sql, "models", "table_model_v2.sql") write_file(modified_table_model_latest_version_yml, "models", "schema.yml") results = run_dbt(["list", "-s", "state:modified", "--state", "./state"]) assert results == ["test.table_model.v1", "test.table_model.v2"] class TestChangedSemanticModelContents(BaseModifiedState): @pytest.fixture(scope="class") def models(self): return { "view_model.sql": view_model_sql, "schema.yml": semantic_model_schema_yml, "metricflow_time_spine.sql": metricflow_time_spine_sql, } def test_changed_semantic_model_contents(self, project): self.run_and_save_state() results = run_dbt(["list", "-s", "state:modified", "--state", "./state"]) assert len(results) == 0 write_file(modified_semantic_model_schema_yml, "models", "schema.yml") results = run_dbt(["list", "-s", "state:modified", "--state", "./state"]) assert len(results) == 1 class TestVersionedContractVarcharSizeChange(BaseModifiedState): """ Test that changing varchar size (e.g., varchar(5) to varchar(20)) does not trigger a breaking change error for versioned models. Per dbt docs, size/precision/scale changes should NOT be breaking changes. Reproduces issue: https://github.com/dbt-labs/dbt-core/issues/11186 """ MODEL_UNIQUE_ID = "model.test.table_model.v1" def test_varchar_size_increase_not_breaking(self, project): # Start with varchar(5) write_file(varchar_size_contract_schema_yml, "models", "schema.yml") self.run_and_save_state() # Change to varchar(20) - should NOT be a breaking change write_file(varchar_size_increased_contract_schema_yml, "models", "schema.yml") # This should PASS without errors or breaking change warnings _, logs = run_dbt_and_capture( ["run", "--models", "state:modified.contract", "--state", "./state"] ) # Verify no breaking change warning/error assert "breaking change" not in logs.lower() assert "ContractBreakingChangeError" not in logs def test_varchar_case_sensitivity_not_breaking(self, project): # Start with VARCHAR(5) - uppercase write_file(varchar_size_uppercase_contract_schema_yml, "models", "schema.yml") self.run_and_save_state() # Change to varchar(20) - lowercase with different size # Should NOT be a breaking change (case-insensitive comparison) write_file(varchar_size_lowercase_increased_contract_schema_yml, "models", "schema.yml") # This should PASS without errors or breaking change warnings _, logs = run_dbt_and_capture( ["run", "--models", "state:modified.contract", "--state", "./state"] ) # Verify no breaking change warning/error assert "breaking change" not in logs.lower() assert "ContractBreakingChangeError" not in logs def test_numeric_precision_increase_not_breaking(self, project): # Start with numeric(10,2) write_file(numeric_precision_contract_schema_yml, "models", "schema.yml") # Need to modify the table_model.sql to have an amount column instead of name modified_table_model = """ select 1 as id, 100.50 as amount """ write_file(modified_table_model, "models", "table_model.sql") self.run_and_save_state() # Change to numeric(12,4) - should NOT be a breaking change write_file(numeric_precision_increased_contract_schema_yml, "models", "schema.yml") # This should PASS without errors or breaking change warnings _, logs = run_dbt_and_capture( ["run", "--models", "state:modified.contract", "--state", "./state"] ) # Verify no breaking change warning/error assert "breaking change" not in logs.lower() assert "ContractBreakingChangeError" not in logs class TestUnversionedContractVarcharSizeChange(BaseModifiedState): """ Test that changing varchar size for UNVERSIONED models behaves correctly. Unversioned models should also NOT issue warnings for size-only changes. This ensures versioned and unversioned models handle size changes consistently. """ MODEL_UNIQUE_ID = "model.test.table_model" def test_varchar_size_increase_not_breaking_unversioned(self, project): # Start with varchar(5) - no version write_file(varchar_size_unversioned_contract_schema_yml, "models", "schema.yml") self.run_and_save_state() # Change to varchar(20) - should NOT be a breaking change write_file(varchar_size_unversioned_increased_contract_schema_yml, "models", "schema.yml") # For unversioned models, should also have no breaking change warnings _, logs = run_dbt_and_capture( ["run", "--models", "state:modified.contract", "--state", "./state"] ) # Verify no breaking change warning (consistent with versioned behavior) assert "breaking change" not in logs.lower() # Model should still be detected as modified and run successfully assert "Completed successfully" in logs ================================================ FILE: tests/functional/defer_state/test_modified_state_environment_vars.py ================================================ import os import pytest from dbt.tests.util import run_dbt from tests.functional.defer_state.fixtures import ( model_with_env_var_in_config_sql, model_with_no_in_config_sql, schema_model_with_env_var_in_config_yml, ) from tests.functional.defer_state.test_modified_state import BaseModifiedState class BaseTestStateSelectionEnvVarConfig(BaseModifiedState): @pytest.fixture(scope="class", autouse=True) def setup(self): os.environ["DBT_TEST_STATE_MODIFIED"] = "table" yield del os.environ["DBT_TEST_STATE_MODIFIED"] @pytest.fixture(scope="class") def project_config_update(self): return { "flags": { "state_modified_compare_more_unrendered_values": True, } } def test_change_env_var(self, project): # Generate ./state without changing environment variable value run_dbt(["run"]) self.copy_state() # Assert no false positive results = run_dbt(["list", "-s", "state:modified", "--state", "./state"]) assert len(results) == 0 # Change environment variable and assert no false positive # Environment variables do not have an effect on state:modified os.environ["DBT_TEST_STATE_MODIFIED"] = "view" results = run_dbt(["list", "-s", "state:modified", "--state", "./state"]) assert len(results) == 0 class TestModelNodeWithEnvVarConfigInSqlFile(BaseTestStateSelectionEnvVarConfig): @pytest.fixture(scope="class") def models(self): return { "model.sql": model_with_env_var_in_config_sql, } class TestModelNodeWithEnvVarConfigInSchemaYml(BaseTestStateSelectionEnvVarConfig): @pytest.fixture(scope="class") def models(self): return { "model.sql": model_with_no_in_config_sql, "schema.yml": schema_model_with_env_var_in_config_yml, } class TestModelNodeWithEnvVarConfigInProjectYml(BaseTestStateSelectionEnvVarConfig): @pytest.fixture(scope="class") def models(self): return { "model.sql": model_with_no_in_config_sql, } @pytest.fixture(scope="class") def project_config_update(self): return { "models": { "test": { "+materialized": "{{ env_var('DBT_TEST_STATE_MODIFIED') }}", } } } class TestModelNodeWithEnvVarConfigInProjectYmlAndSchemaYml(BaseTestStateSelectionEnvVarConfig): @pytest.fixture(scope="class") def models(self): return { "model.sql": model_with_no_in_config_sql, "schema.yml": schema_model_with_env_var_in_config_yml, } @pytest.fixture(scope="class") def project_config_update(self): return { "flags": { "state_modified_compare_more_unrendered_values": True, }, "models": { "test": { "+materialized": "{{ env_var('DBT_TEST_STATE_MODIFIED') }}", } }, } class TestModelNodeWithEnvVarConfigInSqlAndSchemaYml(BaseTestStateSelectionEnvVarConfig): @pytest.fixture(scope="class") def models(self): return { "model.sql": model_with_env_var_in_config_sql, "schema.yml": schema_model_with_env_var_in_config_yml, } ================================================ FILE: tests/functional/defer_state/test_modified_state_jinja.py ================================================ import pytest from dbt.tests.util import ( get_artifact, get_project_config, run_dbt, update_config_file, write_file, ) from tests.functional.defer_state.fixtures import ( model_with_jinja_in_config_sql, model_with_no_in_config_sql, model_with_updated_jinja_in_config_sql, schema_model_with_jinja_in_config_yml, schema_model_with_updated_jinja_in_config_yml, ) from tests.functional.defer_state.test_modified_state import BaseModifiedState class BaseTestStateSelectionJinjaInConfig(BaseModifiedState): @pytest.fixture(scope="class") def project_config_update(self): return { "flags": { "state_modified_compare_more_unrendered_values": True, } } def update_jinja_expression_in_config(self, project): pass def test_change_jinja_if(self, project): run_dbt(["run"]) self.copy_state() # Model is table when execute = True manifest_json = get_artifact(project.project_root, "target", "manifest.json") assert manifest_json["nodes"]["model.test.model"]["config"]["materialized"] == "view" # Assert no false positive (execute = False) results = run_dbt(["list", "-s", "state:modified", "--state", "./state"]) assert len(results) == 0 # Update unrendered config (change jinja expression) self.update_jinja_expression_in_config(project) # Assert no false negatives (jinja expression has changed) results = run_dbt(["list", "-s", "state:modified", "--state", "./state"]) assert len(results) == 1 class TestModelNodeWithJinjaConfigInSqlFile(BaseTestStateSelectionJinjaInConfig): def update_jinja_expression_in_config(self, project): write_file( model_with_updated_jinja_in_config_sql, project.project_root, "models", "model.sql" ) @pytest.fixture(scope="class") def models(self): return { "model.sql": model_with_jinja_in_config_sql, } class TestModelNodeWithEnvVarConfigInSchemaYml(BaseTestStateSelectionJinjaInConfig): def update_jinja_expression_in_config(self, project): write_file( schema_model_with_updated_jinja_in_config_yml, project.project_root, "models", "schema.yml", ) @pytest.fixture(scope="class") def models(self): return { "model.sql": model_with_no_in_config_sql, "schema.yml": schema_model_with_jinja_in_config_yml, } class TestModelNodeWithJinjaConfigInProjectYml(BaseTestStateSelectionJinjaInConfig): def update_jinja_expression_in_config(self, project): config = get_project_config(project) config["models"]["test"]["+materialized"] = "{{ ('view' if execute else 'table') }}" update_config_file(config, "dbt_project.yml") @pytest.fixture(scope="class") def models(self): return { "model.sql": model_with_no_in_config_sql, } @pytest.fixture(scope="class") def project_config_update(self): return { "models": { "test": { "+materialized": "{{ ('table' if execute else 'view') }}", } } } class TestModelNodeWithJinjaConfigInProjectYmlAndSchemaYml(BaseTestStateSelectionJinjaInConfig): def update_jinja_expression_in_config(self, project): write_file( schema_model_with_updated_jinja_in_config_yml, project.project_root, "models", "schema.yml", ) @pytest.fixture(scope="class") def models(self): return { "model.sql": model_with_no_in_config_sql, "schema.yml": schema_model_with_jinja_in_config_yml, } @pytest.fixture(scope="class") def project_config_update(self): return { "flags": { "state_modified_compare_more_unrendered_values": True, }, "models": { "test": { "+materialized": "{{ ('view' if execute else 'table') }}", } }, } class TestModelNodeWithJinjaConfigInSqlAndSchemaYml(BaseTestStateSelectionJinjaInConfig): def update_jinja_expression_in_config(self, project): write_file( model_with_updated_jinja_in_config_sql, project.project_root, "models", "model.sql" ) @pytest.fixture(scope="class") def models(self): return { "model.sql": model_with_jinja_in_config_sql, "schema.yml": schema_model_with_jinja_in_config_yml, } ================================================ FILE: tests/functional/defer_state/test_modified_state_schema_evolution.py ================================================ import json import os import shutil import sys import pytest from dbt.tests.util import run_dbt class TestModifiedStateSchemaEvolution: def update_state(self, test_data_dir): run_dbt(["parse"]) shutil.copyfile("target/manifest.json", os.path.join(test_data_dir, "manifest.json")) # Set dbt_version to PREVIOUS to trigger manifest upgrades on state:modified state_manifest_path = os.path.join(test_data_dir, "manifest.json") with open(state_manifest_path, "r") as f: manifest = json.load(f) manifest["metadata"]["dbt_version"] = "PREVIOUS" with open(state_manifest_path, "w") as f: json.dump(manifest, f, indent=2) @pytest.mark.skipif(sys.platform == "win32", reason="Flaky on Windows") def test_modified_state_schema_evolution(self, happy_path_project): # Uncomment this line when happy_path_project is updated # If the happy_path_project needs to be updated in order to # test schema evolutions not introducing state:modified false positives, # make sure to update state off of main so that the functional changes of the # schema evolution branch do not get reflected in the 'previous' state. # self.update_state(happy_path_project.test_data_dir) results = run_dbt( ["ls", "--select", "state:modified", "--state", happy_path_project.test_data_dir] ) # No false positives when no project changes are made assert len(results) == 0 ================================================ FILE: tests/functional/defer_state/test_modified_state_sources_unrendered.py ================================================ import os import pytest from dbt.tests.util import get_artifact, run_dbt, write_file from tests.functional.defer_state.fixtures import ( schema_source_with_env_var_as_database_property_yml, schema_source_with_env_var_as_schema_property_yml, schema_source_with_jinja_as_database_property_yml, schema_source_with_jinja_as_schema_property_yml, schema_source_with_updated_env_var_as_schema_property_yml, schema_source_with_updated_jinja_as_database_property_yml, schema_source_with_updated_jinja_as_schema_property_yml, ) from tests.functional.defer_state.test_modified_state import BaseModifiedState from tests.functional.defer_state.test_modified_state_environment_vars import ( BaseTestStateSelectionEnvVarConfig, ) class TestSourceNodeWithEnvVarConfigInDatabase(BaseTestStateSelectionEnvVarConfig): @pytest.fixture(scope="class") def models(self): return { "schema.yml": schema_source_with_env_var_as_database_property_yml, } class TestSourceNodeWithEnvVarConfigInSchema(BaseTestStateSelectionEnvVarConfig): @pytest.fixture(scope="class") def models(self): return { "schema.yml": schema_source_with_env_var_as_schema_property_yml, } def test_change_env_var(self, project): # Generate ./state without changing environment variable value run_dbt(["run"]) self.copy_state() manifest_json = get_artifact(project.project_root, "target", "manifest.json") assert manifest_json["sources"]["source.test.jaffle_shop.customers"]["schema"] == "table" assert ( manifest_json["sources"]["source.test.jaffle_shop.customers"]["unrendered_schema"] == "{{ env_var('DBT_TEST_STATE_MODIFIED') }}" ) # Assert no false positive results = run_dbt(["list", "-s", "state:modified", "--state", "./state"]) assert len(results) == 0 # Change environment variable and assert no false positive # Environment variables do not have an effect on state:modified os.environ["DBT_TEST_STATE_MODIFIED"] = "view" results = run_dbt(["list", "-s", "state:modified", "--state", "./state"]) assert len(results) == 0 # Confirm env change changed schema, but not unrendered_schema manifest_json = get_artifact(project.project_root, "target", "manifest.json") assert manifest_json["sources"]["source.test.jaffle_shop.customers"]["schema"] == "view" assert ( manifest_json["sources"]["source.test.jaffle_shop.customers"]["unrendered_schema"] == "{{ env_var('DBT_TEST_STATE_MODIFIED') }}" ) # Assert no false negative after actual change to schema write_file( schema_source_with_updated_env_var_as_schema_property_yml, project.project_root, "models", "schema.yml", ) results = run_dbt(["list", "-s", "state:modified", "--state", "./state"]) assert len(results) == 1 manifest_json = get_artifact(project.project_root, "target", "manifest.json") assert manifest_json["sources"]["source.test.jaffle_shop.customers"]["schema"] == "updated" assert ( manifest_json["sources"]["source.test.jaffle_shop.customers"]["unrendered_schema"] == "updated" ) class TestSourceNodeWithJinjaInDatabase(BaseModifiedState): @pytest.fixture(scope="class") def project_config_update(self): return { "flags": { "state_modified_compare_more_unrendered_values": True, } } def update_jinja_expression_in_config(self, project): write_file( schema_source_with_updated_jinja_as_database_property_yml, project.project_root, "models", "schema.yml", ) @pytest.fixture(scope="class") def models(self): return { "schema.yml": schema_source_with_jinja_as_database_property_yml, } def test_change_jinja_if(self, project): run_dbt(["run"]) self.copy_state() # source database is 'bar' when execute = False manifest_json = get_artifact(project.project_root, "target", "manifest.json") assert manifest_json["sources"]["source.test.jaffle_shop.customers"]["database"] == "bar" assert ( manifest_json["sources"]["source.test.jaffle_shop.customers"]["unrendered_database"] == "{{ ('foo' if execute else 'bar') }}" ) # Assert no false positive (execute = False) results = run_dbt(["list", "-s", "state:modified", "--state", "./state"]) assert len(results) == 0 # Update unrendered config (change jinja expression) self.update_jinja_expression_in_config(project) # Assert no false negatives (jinja expression has changed) results = run_dbt(["list", "-s", "state:modified", "--state", "./state"]) assert len(results) == 1 class TestSourceNodeWithJinjaInSchema(BaseModifiedState): def update_jinja_expression_in_config(self, project): write_file( schema_source_with_updated_jinja_as_schema_property_yml, project.project_root, "models", "schema.yml", ) @pytest.fixture(scope="class") def models(self): return { "schema.yml": schema_source_with_jinja_as_schema_property_yml, } def test_change_jinja_if(self, project): run_dbt(["run"]) self.copy_state() # source database is 'bar' when execute = False manifest_json = get_artifact(project.project_root, "target", "manifest.json") assert manifest_json["sources"]["source.test.jaffle_shop.customers"]["schema"] == "bar" assert ( manifest_json["sources"]["source.test.jaffle_shop.customers"]["unrendered_schema"] == "{{ ('foo' if execute else 'bar') }}" ) # Assert no false positive (execute = False) results = run_dbt(["list", "-s", "state:modified", "--state", "./state"]) assert len(results) == 0 # Update unrendered config (change jinja expression) self.update_jinja_expression_in_config(project) # Assert no false negatives (jinja expression has changed) results = run_dbt(["list", "-s", "state:modified", "--state", "./state"]) assert len(results) == 1 ================================================ FILE: tests/functional/defer_state/test_modified_state_vars.py ================================================ import pytest from dbt.tests.util import run_dbt from tests.functional.defer_state.fixtures import model_with_var_in_config_sql from tests.functional.defer_state.test_modified_state import BaseModifiedState class TestStateSelectionVarConfigLegacy(BaseModifiedState): @pytest.fixture(scope="class") def models(self): return { "model.sql": model_with_var_in_config_sql, } @pytest.fixture(scope="class") def project_config_update(self): return { "flags": { "state_modified_compare_more_unrendered_values": False, } } def test_change_var(self, project): # Generate ./state without changing variable value run_dbt(["run", "--vars", "DBT_TEST_STATE_MODIFIED: view"]) self.copy_state() # Assert no false positive results = run_dbt( [ "list", "-s", "state:modified", "--state", "./state", "--vars", "DBT_TEST_STATE_MODIFIED: view", ] ) assert len(results) == 0 # Change var and assert no false negative - legacy behaviour results = run_dbt( [ "list", "-s", "state:modified", "--state", "./state", "--vars", "DBT_TEST_STATE_MODIFIED: table", ] ) assert len(results) == 1 class TestStateSelectionVarConfig(BaseModifiedState): @pytest.fixture(scope="class") def models(self): return { "model.sql": model_with_var_in_config_sql, } @pytest.fixture(scope="class") def project_config_update(self): return { "flags": { "state_modified_compare_more_unrendered_values": True, } } def test_change_var(self, project): # Generate ./state without changing variable value run_dbt(["run", "--vars", "DBT_TEST_STATE_MODIFIED: view"]) self.copy_state() # Assert no false positive results = run_dbt( [ "list", "-s", "state:modified", "--state", "./state", "--vars", "DBT_TEST_STATE_MODIFIED: view", ] ) assert len(results) == 0 # Change var and assert no sensitivity to var changes -- new behaviour until state:modified.vars included in state:modified by default results = run_dbt( [ "list", "-s", "state:modified", "--state", "./state", "--vars", "DBT_TEST_STATE_MODIFIED: table", ] ) assert len(results) == 0 ================================================ FILE: tests/functional/defer_state/test_removed_test_state.py ================================================ import pytest from dbt.exceptions import CompilationError from dbt.tests.util import run_dbt from tests.functional.defer_state.fixtures import ( removed_test_model_sql, removed_test_schema_yml, sample_test_sql, ) class TestRemovedGenericTest: """Test that removing a generic test while it's still referenced gives a clear error message.""" @pytest.fixture(scope="class") def models(self): return { "model_a.sql": removed_test_model_sql, "schema.yml": removed_test_schema_yml, } @pytest.fixture(scope="class") def tests(self): return { "generic": { "sample_test.sql": sample_test_sql, } } def copy_state(self, project): import os import shutil if not os.path.exists(f"{project.project_root}/state"): os.makedirs(f"{project.project_root}/state") shutil.copyfile( f"{project.project_root}/target/manifest.json", f"{project.project_root}/state/manifest.json", ) def test_removed_generic_test_with_state_modified(self, project): """ Test that state:modified selector handles missing test macros gracefully. Issue #10630: When a generic test is removed but still referenced, using --select state:modified would crash with KeyError: None. Solution: We check for None macro_uid in the state selector and raise a clear error. """ # Initial run - everything works results = run_dbt(["run"]) assert len(results) == 1 # Save state self.copy_state(project) # Remove the generic test file but keep the reference in schema.yml import os test_file_path = os.path.join(project.project_root, "tests", "generic", "sample_test.sql") if os.path.exists(test_file_path): os.remove(test_file_path) # The key bug fix: dbt run --select state:modified used to crash with KeyError: None # After fix: it should give a clear compilation error during the selection phase with pytest.raises(CompilationError, match="does not exist|macro or test"): run_dbt(["run", "--select", "state:modified", "--state", "state"]) class TestRemovedGenericTestStateModifiedGracefulError: """Test that state:modified selector handles missing test macros gracefully.""" @pytest.fixture(scope="class") def models(self): return { "model_a.sql": removed_test_model_sql, "schema.yml": removed_test_schema_yml, } @pytest.fixture(scope="class") def tests(self): return { "generic": { "sample_test.sql": sample_test_sql, } } def copy_state(self, project): import os import shutil if not os.path.exists(f"{project.project_root}/state"): os.makedirs(f"{project.project_root}/state") shutil.copyfile( f"{project.project_root}/target/manifest.json", f"{project.project_root}/state/manifest.json", ) def test_list_with_state_modified_after_test_removal(self, project): """ Test that state:modified selector handles missing test macros gracefully. This exercises the selector_methods.py code path that was failing with KeyError: None. """ # Initial run - everything works results = run_dbt(["run"]) assert len(results) == 1 # Save state self.copy_state(project) # Remove the generic test file but keep the reference in schema.yml import os test_file_path = os.path.join(project.project_root, "tests", "generic", "sample_test.sql") if os.path.exists(test_file_path): os.remove(test_file_path) # dbt run with state:modified should not crash with KeyError: None # After the fix, it should give a clear CompilationError about the missing test # Previously this crashed with KeyError: None in recursively_check_macros_modified with pytest.raises( CompilationError, match="sample_test|does not exist|macro or generic test" ): run_dbt(["run", "--select", "state:modified", "--state", "state"]) ================================================ FILE: tests/functional/defer_state/test_run_results_state.py ================================================ import os import shutil import pytest from dbt.tests.util import run_dbt, write_file from tests.functional.defer_state.fixtures import ( ephemeral_model_sql, exposures_yml, infinite_macros_sql, macros_sql, schema_yml, seed_csv, table_model_sql, view_model_sql, ) class BaseRunResultsState: @pytest.fixture(scope="class") def models(self): return { "table_model.sql": table_model_sql, "view_model.sql": view_model_sql, "ephemeral_model.sql": ephemeral_model_sql, "schema.yml": schema_yml, "exposures.yml": exposures_yml, } @pytest.fixture(scope="class") def macros(self): return { "macros.sql": macros_sql, "infinite_macros.sql": infinite_macros_sql, } @pytest.fixture(scope="class") def seeds(self): return { "seed.csv": seed_csv, } @property def project_config_update(self): return { "seeds": { "test": { "quote_columns": False, } } } def clear_state(self): shutil.rmtree("./state") def copy_state(self): if not os.path.exists("state"): os.makedirs("state") shutil.copyfile("target/manifest.json", "state/manifest.json") shutil.copyfile("target/run_results.json", "state/run_results.json") def run_and_save_state(self): run_dbt(["build"]) self.copy_state() def rebuild_run_dbt(self, expect_pass=True): self.clear_state() run_dbt(["build"], expect_pass=expect_pass) self.copy_state() def update_view_model_bad_sql(self): # update view model to generate a failure case not_unique_sql = "select * from forced_error" write_file(not_unique_sql, "models", "view_model.sql") def update_view_model_failing_tests(self, with_dupes=True, with_nulls=False): # test failure on build tests # fail the unique test select_1 = "select 1 as id" select_stmts = [select_1] if with_dupes: select_stmts.append(select_1) if with_nulls: select_stmts.append("select null as id") failing_tests_sql = " union all ".join(select_stmts) write_file(failing_tests_sql, "models", "view_model.sql") def update_unique_test_severity_warn(self): # change the unique test severity from error to warn and reuse the same view_model.sql changes above new_config = schema_yml.replace("error", "warn") write_file(new_config, "models", "schema.yml") class TestSeedRunResultsState(BaseRunResultsState): def test_seed_run_results_state(self, project): self.run_and_save_state() self.clear_state() run_dbt(["seed"]) self.copy_state() results = run_dbt( ["ls", "--resource-type", "seed", "--select", "result:success", "--state", "./state"], expect_pass=True, ) assert len(results) == 1 assert results[0] == "test.seed" results = run_dbt(["ls", "--select", "result:success", "--state", "./state"]) assert len(results) == 1 assert results[0] == "test.seed" results = run_dbt(["ls", "--select", "result:success+", "--state", "./state"]) assert len(results) == 7 assert set(results) == { "test.seed", "test.table_model", "test.view_model", "test.ephemeral_model", "test.not_null_view_model_id", "test.unique_view_model_id", "exposure:test.my_exposure", } # add a new faulty row to the seed changed_seed_contents = seed_csv + "\n" + "\\\3,carl" write_file(changed_seed_contents, "seeds", "seed.csv") self.clear_state() run_dbt(["seed"], expect_pass=False) self.copy_state() results = run_dbt( ["ls", "--resource-type", "seed", "--select", "result:error", "--state", "./state"], expect_pass=True, ) assert len(results) == 1 assert results[0] == "test.seed" results = run_dbt(["ls", "--select", "result:error", "--state", "./state"]) assert len(results) == 1 assert results[0] == "test.seed" results = run_dbt(["ls", "--select", "result:error+", "--state", "./state"]) assert len(results) == 7 assert set(results) == { "test.seed", "test.table_model", "test.view_model", "test.ephemeral_model", "test.not_null_view_model_id", "test.unique_view_model_id", "exposure:test.my_exposure", } class TestBuildRunResultsState(BaseRunResultsState): def test_build_run_results_state(self, project): self.run_and_save_state() results = run_dbt(["build", "--select", "result:error", "--state", "./state"]) assert len(results) == 0 self.update_view_model_bad_sql() self.rebuild_run_dbt(expect_pass=False) results = run_dbt( ["build", "--select", "result:error", "--state", "./state"], expect_pass=False ) assert len(results) == 3 nodes = set([elem.node.name for elem in results]) assert nodes == {"view_model", "not_null_view_model_id", "unique_view_model_id"} results = run_dbt(["ls", "--select", "result:error", "--state", "./state"]) assert len(results) == 3 assert set(results) == { "test.view_model", "test.not_null_view_model_id", "test.unique_view_model_id", } results = run_dbt( ["build", "--select", "result:error+", "--state", "./state"], expect_pass=False ) assert len(results) == 5 nodes = set([elem.node.name for elem in results]) assert nodes == { "table_model", "view_model", "not_null_view_model_id", "unique_view_model_id", "my_exposure", } results = run_dbt(["ls", "--select", "result:error+", "--state", "./state"]) assert len(results) == 6 # includes exposure assert set(results) == { "test.table_model", "test.view_model", "test.ephemeral_model", "test.not_null_view_model_id", "test.unique_view_model_id", "exposure:test.my_exposure", } self.update_view_model_failing_tests() self.rebuild_run_dbt(expect_pass=False) results = run_dbt( ["build", "--select", "result:fail", "--state", "./state"], expect_pass=False ) assert len(results) == 1 assert results[0].node.name == "unique_view_model_id" results = run_dbt(["ls", "--select", "result:fail", "--state", "./state"]) assert len(results) == 1 assert results[0] == "test.unique_view_model_id" results = run_dbt( ["build", "--select", "result:fail+", "--state", "./state"], expect_pass=False ) assert len(results) == 1 nodes = set([elem.node.name for elem in results]) assert nodes == {"unique_view_model_id"} results = run_dbt(["ls", "--select", "result:fail+", "--state", "./state"]) assert len(results) == 1 assert set(results) == {"test.unique_view_model_id"} self.update_unique_test_severity_warn() self.rebuild_run_dbt(expect_pass=True) results = run_dbt( ["build", "--select", "result:warn", "--state", "./state"], expect_pass=True ) assert len(results) == 1 assert results[0].node.name == "unique_view_model_id" results = run_dbt(["ls", "--select", "result:warn", "--state", "./state"]) assert len(results) == 1 assert results[0] == "test.unique_view_model_id" results = run_dbt( ["build", "--select", "result:warn+", "--state", "./state"], expect_pass=True ) assert len(results) == 1 nodes = set([elem.node.name for elem in results]) assert nodes == {"unique_view_model_id"} results = run_dbt(["ls", "--select", "result:warn+", "--state", "./state"]) assert len(results) == 1 assert set(results) == {"test.unique_view_model_id"} class TestRunRunResultsState(BaseRunResultsState): def test_run_run_results_state(self, project): self.run_and_save_state() results = run_dbt( ["run", "--select", "result:success", "--state", "./state"], expect_pass=True ) assert len(results) == 2 assert results[0].node.name == "view_model" assert results[1].node.name == "table_model" # clear state and rerun upstream view model to test + operator self.clear_state() run_dbt(["run", "--select", "view_model"], expect_pass=True) self.copy_state() results = run_dbt( ["run", "--select", "result:success+", "--state", "./state"], expect_pass=True ) assert len(results) == 2 assert results[0].node.name == "view_model" assert results[1].node.name == "table_model" # check we are starting from a place with 0 errors results = run_dbt(["run", "--select", "result:error", "--state", "./state"]) assert len(results) == 0 self.update_view_model_bad_sql() self.clear_state() run_dbt(["run"], expect_pass=False) self.copy_state() # test single result selector on error results = run_dbt( ["run", "--select", "result:error", "--state", "./state"], expect_pass=False ) assert len(results) == 1 assert results[0].node.name == "view_model" # test + operator selection on error results = run_dbt( ["run", "--select", "result:error+", "--state", "./state"], expect_pass=False ) assert len(results) == 2 assert results[0].node.name == "view_model" assert results[1].node.name == "table_model" # single result selector on skipped. Expect this to pass becase underlying view already defined above results = run_dbt( ["run", "--select", "result:skipped", "--state", "./state"], expect_pass=True ) assert len(results) == 1 assert results[0].node.name == "table_model" # add a downstream model that depends on table_model for skipped+ selector downstream_model_sql = "select * from {{ref('table_model')}}" write_file(downstream_model_sql, "models", "table_model_downstream.sql") self.clear_state() run_dbt(["run"], expect_pass=False) self.copy_state() results = run_dbt( ["run", "--select", "result:skipped+", "--state", "./state"], expect_pass=True ) assert len(results) == 2 assert results[0].node.name == "table_model" assert results[1].node.name == "table_model_downstream" class TestTestRunResultsState(BaseRunResultsState): def test_test_run_results_state(self, project): self.run_and_save_state() # run passed nodes results = run_dbt( ["test", "--select", "result:pass", "--state", "./state"], expect_pass=True ) assert len(results) == 2 nodes = set([elem.node.name for elem in results]) assert nodes == {"unique_view_model_id", "not_null_view_model_id"} # run passed nodes with + operator results = run_dbt( ["test", "--select", "result:pass+", "--state", "./state"], expect_pass=True ) assert len(results) == 2 nodes = set([elem.node.name for elem in results]) assert nodes == {"unique_view_model_id", "not_null_view_model_id"} self.update_view_model_failing_tests() self.rebuild_run_dbt(expect_pass=False) # test with failure selector results = run_dbt( ["test", "--select", "result:fail", "--state", "./state"], expect_pass=False ) assert len(results) == 1 assert results[0].node.name == "unique_view_model_id" # test with failure selector and + operator results = run_dbt( ["test", "--select", "result:fail+", "--state", "./state"], expect_pass=False ) assert len(results) == 1 assert results[0].node.name == "unique_view_model_id" self.update_unique_test_severity_warn() # rebuild - expect_pass = True because we changed the error to a warning this time around self.rebuild_run_dbt(expect_pass=True) # test with warn selector results = run_dbt( ["test", "--select", "result:warn", "--state", "./state"], expect_pass=True ) assert len(results) == 1 assert results[0].node.name == "unique_view_model_id" # test with warn selector and + operator results = run_dbt( ["test", "--select", "result:warn+", "--state", "./state"], expect_pass=True ) assert len(results) == 1 assert results[0].node.name == "unique_view_model_id" class TestConcurrentSelectionRunResultsState(BaseRunResultsState): def test_concurrent_selection_run_run_results_state(self, project): self.run_and_save_state() results = run_dbt( ["run", "--select", "state:modified+", "result:error+", "--state", "./state"] ) assert len(results) == 0 self.update_view_model_bad_sql() self.clear_state() run_dbt(["run"], expect_pass=False) self.copy_state() # add a new failing dbt model bad_sql = "select * from forced_error" write_file(bad_sql, "models", "table_model_modified_example.sql") results = run_dbt( ["run", "--select", "state:modified+", "result:error+", "--state", "./state"], expect_pass=False, ) assert len(results) == 3 nodes = set([elem.node.name for elem in results]) assert nodes == {"view_model", "table_model_modified_example", "table_model"} class TestConcurrentSelectionTestRunResultsState(BaseRunResultsState): def test_concurrent_selection_test_run_results_state(self, project): self.run_and_save_state() # create failure test case for result:fail selector self.update_view_model_failing_tests(with_nulls=True) # run dbt build again to trigger test errors self.rebuild_run_dbt(expect_pass=False) # get the failures from results = run_dbt( [ "test", "--select", "result:fail", "--exclude", "not_null_view_model_id", "--state", "./state", ], expect_pass=False, ) assert len(results) == 1 nodes = set([elem.node.name for elem in results]) assert nodes == {"unique_view_model_id"} class TestConcurrentSelectionBuildRunResultsState(BaseRunResultsState): def test_concurrent_selectors_build_run_results_state(self, project): self.run_and_save_state() results = run_dbt( ["build", "--select", "state:modified+", "result:error+", "--state", "./state"] ) assert len(results) == 0 self.update_view_model_bad_sql() self.rebuild_run_dbt(expect_pass=False) # add a new failing dbt model bad_sql = "select * from forced_error" write_file(bad_sql, "models", "table_model_modified_example.sql") results = run_dbt( ["build", "--select", "state:modified+", "result:error+", "--state", "./state"], expect_pass=False, ) assert len(results) == 6 nodes = set([elem.node.name for elem in results]) assert nodes == { "table_model_modified_example", "view_model", "table_model", "not_null_view_model_id", "unique_view_model_id", "my_exposure", } self.update_view_model_failing_tests() # create error model case for result:error selector more_bad_sql = "select 1 as id from not_exists" write_file(more_bad_sql, "models", "error_model.sql") # create something downstream from the error model to rerun downstream_model_sql = "select * from {{ ref('error_model') }} )" write_file(downstream_model_sql, "models", "downstream_of_error_model.sql") # regenerate build state self.rebuild_run_dbt(expect_pass=False) # modify model again to trigger the state:modified selector bad_again_sql = "select * from forced_anothererror" write_file(bad_again_sql, "models", "table_model_modified_example.sql") results = run_dbt( [ "build", "--select", "state:modified+", "result:error+", "result:fail+", "--state", "./state", ], expect_pass=False, ) assert len(results) == 4 nodes = set([elem.node.name for elem in results]) assert nodes == { "error_model", "downstream_of_error_model", "table_model_modified_example", "unique_view_model_id", } ================================================ FILE: tests/functional/defer_state/test_unrendered_config.py ================================================ import pytest from dbt.tests.util import run_dbt dbt_project_update = """ models: my_dbt_project: +materialized: table tests: +store_failures: true """ foo_sql = """ select 1 as id """ schema_yml = """ models: - name: foo columns: - name: id tests: - unique """ class TestGenericTestUnrenderedConfig: @pytest.fixture(scope="class") def project_config_update(self): return dbt_project_update @pytest.fixture(scope="class") def models(self): return { "foo.sql": foo_sql, "schema.yml": schema_yml, } def test_unrendered_config(self, project): manifest = run_dbt(["parse"]) assert manifest test_node_id = "test.test.unique_foo_id.fa8c520a2e" test_node = manifest.nodes[test_node_id] assert test_node.unrendered_config == {"store_failures": True} ================================================ FILE: tests/functional/dependencies/data/seed.sql ================================================ create table {schema}.seed ( id INTEGER, first_name VARCHAR(11), email VARCHAR(31), ip_address VARCHAR(15), updated_at TIMESTAMP WITHOUT TIME ZONE ); INSERT INTO {schema}.seed ("id","first_name","email","ip_address","updated_at") VALUES (1,'Larry','lking0@miitbeian.gov.cn','69.135.206.194','2008-09-12 19:08:31'), (2,'Larry','lperkins1@toplist.cz','64.210.133.162','1978-05-09 04:15:14'), (3,'Anna','amontgomery2@miitbeian.gov.cn','168.104.64.114','2011-10-16 04:07:57'), (4,'Sandra','sgeorge3@livejournal.com','229.235.252.98','1973-07-19 10:52:43'), (5,'Fred','fwoods4@google.cn','78.229.170.124','2012-09-30 16:38:29'), (6,'Stephen','shanson5@livejournal.com','182.227.157.105','1995-11-07 21:40:50'), (7,'William','wmartinez6@upenn.edu','135.139.249.50','1982-09-05 03:11:59'), (8,'Jessica','jlong7@hao123.com','203.62.178.210','1991-10-16 11:03:15'), (9,'Douglas','dwhite8@tamu.edu','178.187.247.1','1979-10-01 09:49:48'), (10,'Lisa','lcoleman9@nydailynews.com','168.234.128.249','2011-05-26 07:45:49'), (11,'Ralph','rfieldsa@home.pl','55.152.163.149','1972-11-18 19:06:11'), (12,'Louise','lnicholsb@samsung.com','141.116.153.154','2014-11-25 20:56:14'), (13,'Clarence','cduncanc@sfgate.com','81.171.31.133','2011-11-17 07:02:36'), (14,'Daniel','dfranklind@omniture.com','8.204.211.37','1980-09-13 00:09:04'), (15,'Katherine','klanee@auda.org.au','176.96.134.59','1997-08-22 19:36:56'), (16,'Billy','bwardf@wikia.com','214.108.78.85','2003-10-19 02:14:47'), (17,'Annie','agarzag@ocn.ne.jp','190.108.42.70','1988-10-28 15:12:35'), (18,'Shirley','scolemanh@fastcompany.com','109.251.164.84','1988-08-24 10:50:57'), (19,'Roger','rfrazieri@scribd.com','38.145.218.108','1985-12-31 15:17:15'), (20,'Lillian','lstanleyj@goodreads.com','47.57.236.17','1970-06-08 02:09:05'), (21,'Aaron','arodriguezk@nps.gov','205.245.118.221','1985-10-11 23:07:49'), (22,'Patrick','pparkerl@techcrunch.com','19.8.100.182','2006-03-29 12:53:56'), (23,'Phillip','pmorenom@intel.com','41.38.254.103','2011-11-07 15:35:43'), (24,'Henry','hgarcian@newsvine.com','1.191.216.252','2008-08-28 08:30:44'), (25,'Irene','iturnero@opera.com','50.17.60.190','1994-04-01 07:15:02'), (26,'Andrew','adunnp@pen.io','123.52.253.176','2000-11-01 06:03:25'), (27,'David','dgutierrezq@wp.com','238.23.203.42','1988-01-25 07:29:18'), (28,'Henry','hsanchezr@cyberchimps.com','248.102.2.185','1983-01-01 13:36:37'), (29,'Evelyn','epetersons@gizmodo.com','32.80.46.119','1979-07-16 17:24:12'), (30,'Tammy','tmitchellt@purevolume.com','249.246.167.88','2001-04-03 10:00:23'), (31,'Jacqueline','jlittleu@domainmarket.com','127.181.97.47','1986-02-11 21:35:50'), (32,'Earl','eortizv@opera.com','166.47.248.240','1996-07-06 08:16:27'), (33,'Juan','jgordonw@sciencedirect.com','71.77.2.200','1987-01-31 03:46:44'), (34,'Diane','dhowellx@nyu.edu','140.94.133.12','1994-06-11 02:30:05'), (35,'Randy','rkennedyy@microsoft.com','73.255.34.196','2005-05-26 20:28:39'), (36,'Janice','jriveraz@time.com','22.214.227.32','1990-02-09 04:16:52'), (37,'Laura','lperry10@diigo.com','159.148.145.73','2015-03-17 05:59:25'), (38,'Gary','gray11@statcounter.com','40.193.124.56','1970-01-27 10:04:51'), (39,'Jesse','jmcdonald12@typepad.com','31.7.86.103','2009-03-14 08:14:29'), (40,'Sandra','sgonzalez13@goodreads.com','223.80.168.239','1993-05-21 14:08:54'), (41,'Scott','smoore14@archive.org','38.238.46.83','1980-08-30 11:16:56'), (42,'Phillip','pevans15@cisco.com','158.234.59.34','2011-12-15 23:26:31'), (43,'Steven','sriley16@google.ca','90.247.57.68','2011-10-29 19:03:28'), (44,'Deborah','dbrown17@hexun.com','179.125.143.240','1995-04-10 14:36:07'), (45,'Lori','lross18@ow.ly','64.80.162.180','1980-12-27 16:49:15'), (46,'Sean','sjackson19@tumblr.com','240.116.183.69','1988-06-12 21:24:45'), (47,'Terry','tbarnes1a@163.com','118.38.213.137','1997-09-22 16:43:19'), (48,'Dorothy','dross1b@ebay.com','116.81.76.49','2005-02-28 13:33:24'), (49,'Samuel','swashington1c@house.gov','38.191.253.40','1989-01-19 21:15:48'), (50,'Ralph','rcarter1d@tinyurl.com','104.84.60.174','2007-08-11 10:21:49'), (51,'Wayne','whudson1e@princeton.edu','90.61.24.102','1983-07-03 16:58:12'), (52,'Rose','rjames1f@plala.or.jp','240.83.81.10','1995-06-08 11:46:23'), (53,'Louise','lcox1g@theglobeandmail.com','105.11.82.145','2016-09-19 14:45:51'), (54,'Kenneth','kjohnson1h@independent.co.uk','139.5.45.94','1976-08-17 11:26:19'), (55,'Donna','dbrown1i@amazon.co.uk','19.45.169.45','2006-05-27 16:51:40'), (56,'Johnny','jvasquez1j@trellian.com','118.202.238.23','1975-11-17 08:42:32'), (57,'Patrick','pramirez1k@tamu.edu','231.25.153.198','1997-08-06 11:51:09'), (58,'Helen','hlarson1l@prweb.com','8.40.21.39','1993-08-04 19:53:40'), (59,'Patricia','pspencer1m@gmpg.org','212.198.40.15','1977-08-03 16:37:27'), (60,'Joseph','jspencer1n@marriott.com','13.15.63.238','2005-07-23 20:22:06'), (61,'Phillip','pschmidt1o@blogtalkradio.com','177.98.201.190','1976-05-19 21:47:44'), (62,'Joan','jwebb1p@google.ru','105.229.170.71','1972-09-07 17:53:47'), (63,'Phyllis','pkennedy1q@imgur.com','35.145.8.244','2000-01-01 22:33:37'), (64,'Katherine','khunter1r@smh.com.au','248.168.205.32','1991-01-09 06:40:24'), (65,'Laura','lvasquez1s@wiley.com','128.129.115.152','1997-10-23 12:04:56'), (66,'Juan','jdunn1t@state.gov','44.228.124.51','2004-11-10 05:07:35'), (67,'Judith','jholmes1u@wiley.com','40.227.179.115','1977-08-02 17:01:45'), (68,'Beverly','bbaker1v@wufoo.com','208.34.84.59','2016-03-06 20:07:23'), (69,'Lawrence','lcarr1w@flickr.com','59.158.212.223','1988-09-13 06:07:21'), (70,'Gloria','gwilliams1x@mtv.com','245.231.88.33','1995-03-18 22:32:46'), (71,'Steven','ssims1y@cbslocal.com','104.50.58.255','2001-08-05 21:26:20'), (72,'Betty','bmills1z@arstechnica.com','103.177.214.220','1981-12-14 21:26:54'), (73,'Mildred','mfuller20@prnewswire.com','151.158.8.130','2000-04-19 10:13:55'), (74,'Donald','dday21@icq.com','9.178.102.255','1972-12-03 00:58:24'), (75,'Eric','ethomas22@addtoany.com','85.2.241.227','1992-11-01 05:59:30'), (76,'Joyce','jarmstrong23@sitemeter.com','169.224.20.36','1985-10-24 06:50:01'), (77,'Maria','mmartinez24@amazonaws.com','143.189.167.135','2005-10-05 05:17:42'), (78,'Harry','hburton25@youtube.com','156.47.176.237','1978-03-26 05:53:33'), (79,'Kevin','klawrence26@hao123.com','79.136.183.83','1994-10-12 04:38:52'), (80,'David','dhall27@prweb.com','133.149.172.153','1976-12-15 16:24:24'), (81,'Kathy','kperry28@twitter.com','229.242.72.228','1979-03-04 02:58:56'), (82,'Adam','aprice29@elegantthemes.com','13.145.21.10','1982-11-07 11:46:59'), (83,'Brandon','bgriffin2a@va.gov','73.249.128.212','2013-10-30 05:30:36'), (84,'Henry','hnguyen2b@discovery.com','211.36.214.242','1985-01-09 06:37:27'), (85,'Eric','esanchez2c@edublogs.org','191.166.188.251','2004-05-01 23:21:42'), (86,'Jason','jlee2d@jimdo.com','193.92.16.182','1973-01-08 09:05:39'), (87,'Diana','drichards2e@istockphoto.com','19.130.175.245','1994-10-05 22:50:49'), (88,'Andrea','awelch2f@abc.net.au','94.155.233.96','2002-04-26 08:41:44'), (89,'Louis','lwagner2g@miitbeian.gov.cn','26.217.34.111','2003-08-25 07:56:39'), (90,'Jane','jsims2h@seesaa.net','43.4.220.135','1987-03-20 20:39:04'), (91,'Larry','lgrant2i@si.edu','97.126.79.34','2000-09-07 20:26:19'), (92,'Louis','ldean2j@prnewswire.com','37.148.40.127','2011-09-16 20:12:14'), (93,'Jennifer','jcampbell2k@xing.com','38.106.254.142','1988-07-15 05:06:49'), (94,'Wayne','wcunningham2l@google.com.hk','223.28.26.187','2009-12-15 06:16:54'), (95,'Lori','lstevens2m@icq.com','181.250.181.58','1984-10-28 03:29:19'), (96,'Judy','jsimpson2n@marriott.com','180.121.239.219','1986-02-07 15:18:10'), (97,'Phillip','phoward2o@usa.gov','255.247.0.175','2002-12-26 08:44:45'), (98,'Gloria','gwalker2p@usa.gov','156.140.7.128','1997-10-04 07:58:58'), (99,'Paul','pjohnson2q@umn.edu','183.59.198.197','1991-11-14 12:33:55'), (100,'Frank','fgreene2r@blogspot.com','150.143.68.121','2010-06-12 23:55:39'), (101,'Deborah','dknight2s@reverbnation.com','222.131.211.191','1970-07-08 08:54:23'), (102,'Sandra','sblack2t@tripadvisor.com','254.183.128.254','2000-04-12 02:39:36'), (103,'Edward','eburns2u@dailymotion.com','253.89.118.18','1993-10-10 10:54:01'), (104,'Anthony','ayoung2v@ustream.tv','118.4.193.176','1978-08-26 17:07:29'), (105,'Donald','dlawrence2w@wp.com','139.200.159.227','2007-07-21 20:56:20'), (106,'Matthew','mfreeman2x@google.fr','205.26.239.92','2014-12-05 17:05:39'), (107,'Sean','ssanders2y@trellian.com','143.89.82.108','1993-07-14 21:45:02'), (108,'Sharon','srobinson2z@soundcloud.com','66.234.247.54','1977-04-06 19:07:03'), (109,'Jennifer','jwatson30@t-online.de','196.102.127.7','1998-03-07 05:12:23'), (110,'Clarence','cbrooks31@si.edu','218.93.234.73','2002-11-06 17:22:25'), (111,'Jose','jflores32@goo.gl','185.105.244.231','1995-01-05 06:32:21'), (112,'George','glee33@adobe.com','173.82.249.196','2015-01-04 02:47:46'), (113,'Larry','lhill34@linkedin.com','66.5.206.195','2010-11-02 10:21:17'), (114,'Marie','mmeyer35@mysql.com','151.152.88.107','1990-05-22 20:52:51'), (115,'Clarence','cwebb36@skype.com','130.198.55.217','1972-10-27 07:38:54'), (116,'Sarah','scarter37@answers.com','80.89.18.153','1971-08-24 19:29:30'), (117,'Henry','hhughes38@webeden.co.uk','152.60.114.174','1973-01-27 09:00:42'), (118,'Teresa','thenry39@hao123.com','32.187.239.106','2015-11-06 01:48:44'), (119,'Billy','bgutierrez3a@sun.com','52.37.70.134','2002-03-19 03:20:19'), (120,'Anthony','agibson3b@github.io','154.251.232.213','1991-04-19 01:08:15'), (121,'Sandra','sromero3c@wikia.com','44.124.171.2','1998-09-06 20:30:34'), (122,'Paula','pandrews3d@blogs.com','153.142.118.226','2003-06-24 16:31:24'), (123,'Terry','tbaker3e@csmonitor.com','99.120.45.219','1970-12-09 23:57:21'), (124,'Lois','lwilson3f@reuters.com','147.44.171.83','1971-01-09 22:28:51'), (125,'Sara','smorgan3g@nature.com','197.67.192.230','1992-01-28 20:33:24'), (126,'Charles','ctorres3h@china.com.cn','156.115.216.2','1993-10-02 19:36:34'), (127,'Richard','ralexander3i@marriott.com','248.235.180.59','1999-02-03 18:40:55'), (128,'Christina','charper3j@cocolog-nifty.com','152.114.116.129','1978-09-13 00:37:32'), (129,'Steve','sadams3k@economist.com','112.248.91.98','2004-03-21 09:07:43'), (130,'Katherine','krobertson3l@ow.ly','37.220.107.28','1977-03-18 19:28:50'), (131,'Donna','dgibson3m@state.gov','222.218.76.221','1999-02-01 06:46:16'), (132,'Christina','cwest3n@mlb.com','152.114.6.160','1979-12-24 15:30:35'), (133,'Sandra','swillis3o@meetup.com','180.71.49.34','1984-09-27 08:05:54'), (134,'Clarence','cedwards3p@smugmug.com','10.64.180.186','1979-04-16 16:52:10'), (135,'Ruby','rjames3q@wp.com','98.61.54.20','2007-01-13 14:25:52'), (136,'Sarah','smontgomery3r@tripod.com','91.45.164.172','2009-07-25 04:34:30'), (137,'Sarah','soliver3s@eventbrite.com','30.106.39.146','2012-05-09 22:12:33'), (138,'Deborah','dwheeler3t@biblegateway.com','59.105.213.173','1999-11-09 08:08:44'), (139,'Deborah','dray3u@i2i.jp','11.108.186.217','2014-02-04 03:15:19'), (140,'Paul','parmstrong3v@alexa.com','6.250.59.43','2009-12-21 10:08:53'), (141,'Aaron','abishop3w@opera.com','207.145.249.62','1996-04-25 23:20:23'), (142,'Henry','hsanders3x@google.ru','140.215.203.171','2012-01-29 11:52:32'), (143,'Anne','aanderson3y@1688.com','74.150.102.118','1982-04-03 13:46:17'), (144,'Victor','vmurphy3z@hugedomains.com','222.155.99.152','1987-11-03 19:58:41'), (145,'Evelyn','ereid40@pbs.org','249.122.33.117','1977-12-14 17:09:57'), (146,'Brian','bgonzalez41@wikia.com','246.254.235.141','1991-02-24 00:45:58'), (147,'Sandra','sgray42@squarespace.com','150.73.28.159','1972-07-28 17:26:32'), (148,'Alice','ajones43@a8.net','78.253.12.177','2002-12-05 16:57:46'), (149,'Jessica','jhanson44@mapquest.com','87.229.30.160','1994-01-30 11:40:04'), (150,'Louise','lbailey45@reuters.com','191.219.31.101','2011-09-07 21:11:45'), (151,'Christopher','cgonzalez46@printfriendly.com','83.137.213.239','1984-10-24 14:58:04'), (152,'Gregory','gcollins47@yandex.ru','28.176.10.115','1998-07-25 17:17:10'), (153,'Jane','jperkins48@usnews.com','46.53.164.159','1979-08-19 15:25:00'), (154,'Phyllis','plong49@yahoo.co.jp','208.140.88.2','1985-07-06 02:16:36'), (155,'Adam','acarter4a@scribd.com','78.48.148.204','2005-07-20 03:31:09'), (156,'Frank','fweaver4b@angelfire.com','199.180.255.224','2011-03-04 23:07:54'), (157,'Ronald','rmurphy4c@cloudflare.com','73.42.97.231','1991-01-11 10:39:41'), (158,'Richard','rmorris4d@e-recht24.de','91.9.97.223','2009-01-17 21:05:15'), (159,'Rose','rfoster4e@woothemes.com','203.169.53.16','1991-04-21 02:09:38'), (160,'George','ggarrett4f@uiuc.edu','186.61.5.167','1989-11-11 11:29:42'), (161,'Victor','vhamilton4g@biblegateway.com','121.229.138.38','2012-06-22 18:01:23'), (162,'Mark','mbennett4h@businessinsider.com','209.184.29.203','1980-04-16 15:26:34'), (163,'Martin','mwells4i@ifeng.com','97.223.55.105','2010-05-26 14:08:18'), (164,'Diana','dstone4j@google.ru','90.155.52.47','2013-02-11 00:14:54'), (165,'Walter','wferguson4k@blogger.com','30.63.212.44','1986-02-20 17:46:46'), (166,'Denise','dcoleman4l@vistaprint.com','10.209.153.77','1992-05-13 20:14:14'), (167,'Philip','pknight4m@xing.com','15.28.135.167','2000-09-11 18:41:13'), (168,'Russell','rcarr4n@youtube.com','113.55.165.50','2008-07-10 17:49:27'), (169,'Donna','dburke4o@dion.ne.jp','70.0.105.111','1992-02-10 17:24:58'), (170,'Anne','along4p@squidoo.com','36.154.58.107','2012-08-19 23:35:31'), (171,'Clarence','cbanks4q@webeden.co.uk','94.57.53.114','1972-03-11 21:46:44'), (172,'Betty','bbowman4r@cyberchimps.com','178.115.209.69','2013-01-13 21:34:51'), (173,'Andrew','ahudson4s@nytimes.com','84.32.252.144','1998-09-15 14:20:04'), (174,'Keith','kgordon4t@cam.ac.uk','189.237.211.102','2009-01-22 05:34:38'), (175,'Patrick','pwheeler4u@mysql.com','47.22.117.226','1984-09-05 22:33:15'), (176,'Jesse','jfoster4v@mapquest.com','229.95.131.46','1990-01-20 12:19:15'), (177,'Arthur','afisher4w@jugem.jp','107.255.244.98','1983-10-13 11:08:46'), (178,'Nicole','nryan4x@wsj.com','243.211.33.221','1974-05-30 23:19:14'), (179,'Bruce','bjohnson4y@sfgate.com','17.41.200.101','1992-09-23 02:02:19'), (180,'Terry','tcox4z@reference.com','20.189.120.106','1982-02-13 12:43:14'), (181,'Ashley','astanley50@kickstarter.com','86.3.56.98','1976-05-09 01:27:16'), (182,'Michael','mrivera51@about.me','72.118.249.0','1971-11-11 17:28:37'), (183,'Steven','sgonzalez52@mozilla.org','169.112.247.47','2002-08-24 14:59:25'), (184,'Kathleen','kfuller53@bloglovin.com','80.93.59.30','2002-03-11 13:41:29'), (185,'Nicole','nhenderson54@usda.gov','39.253.60.30','1995-04-24 05:55:07'), (186,'Ralph','rharper55@purevolume.com','167.147.142.189','1980-02-10 18:35:45'), (187,'Heather','hcunningham56@photobucket.com','96.222.196.229','2007-06-15 05:37:50'), (188,'Nancy','nlittle57@cbc.ca','241.53.255.175','2007-07-12 23:42:48'), (189,'Juan','jramirez58@pinterest.com','190.128.84.27','1978-11-07 23:37:37'), (190,'Beverly','bfowler59@chronoengine.com','54.144.230.49','1979-03-31 23:27:28'), (191,'Shirley','sstevens5a@prlog.org','200.97.231.248','2011-12-06 07:08:50'), (192,'Annie','areyes5b@squidoo.com','223.32.182.101','2011-05-28 02:42:09'), (193,'Jack','jkelley5c@tiny.cc','47.34.118.150','1981-12-05 17:31:40'), (194,'Keith','krobinson5d@1und1.de','170.210.209.31','1999-03-09 11:05:43'), (195,'Joseph','jmiller5e@google.com.au','136.74.212.139','1984-10-08 13:18:20'), (196,'Annie','aday5f@blogspot.com','71.99.186.69','1986-02-18 12:27:34'), (197,'Nancy','nperez5g@liveinternet.ru','28.160.6.107','1983-10-20 17:51:20'), (198,'Tammy','tward5h@ucoz.ru','141.43.164.70','1980-03-31 04:45:29'), (199,'Doris','dryan5i@ted.com','239.117.202.188','1985-07-03 03:17:53'), (200,'Rose','rmendoza5j@photobucket.com','150.200.206.79','1973-04-21 21:36:40'), (201,'Cynthia','cbutler5k@hubpages.com','80.153.174.161','2001-01-20 01:42:26'), (202,'Samuel','soliver5l@people.com.cn','86.127.246.140','1970-09-02 02:19:00'), (203,'Carl','csanchez5m@mysql.com','50.149.237.107','1993-12-01 07:02:09'), (204,'Kathryn','kowens5n@geocities.jp','145.166.205.201','2004-07-06 18:39:33'), (205,'Nicholas','nnichols5o@parallels.com','190.240.66.170','2014-11-11 18:52:19'), (206,'Keith','kwillis5p@youtube.com','181.43.206.100','1998-06-13 06:30:51'), (207,'Justin','jwebb5q@intel.com','211.54.245.74','2000-11-04 16:58:26'), (208,'Gary','ghicks5r@wikipedia.org','196.154.213.104','1992-12-01 19:48:28'), (209,'Martin','mpowell5s@flickr.com','153.67.12.241','1983-06-30 06:24:32'), (210,'Brenda','bkelley5t@xinhuanet.com','113.100.5.172','2005-01-08 20:50:22'), (211,'Edward','eray5u@a8.net','205.187.246.65','2011-09-26 08:04:44'), (212,'Steven','slawson5v@senate.gov','238.150.250.36','1978-11-22 02:48:09'), (213,'Robert','rthompson5w@furl.net','70.7.89.236','2001-09-12 08:52:07'), (214,'Jack','jporter5x@diigo.com','220.172.29.99','1976-07-26 14:29:21'), (215,'Lisa','ljenkins5y@oakley.com','150.151.170.180','2010-03-20 19:21:16'), (216,'Theresa','tbell5z@mayoclinic.com','247.25.53.173','2001-03-11 05:36:40'), (217,'Jimmy','jstephens60@weather.com','145.101.93.235','1983-04-12 09:35:30'), (218,'Louis','lhunt61@amazon.co.jp','78.137.6.253','1997-08-29 19:34:34'), (219,'Lawrence','lgilbert62@ted.com','243.132.8.78','2015-04-08 22:06:56'), (220,'David','dgardner63@4shared.com','204.40.46.136','1971-07-09 03:29:11'), (221,'Charles','ckennedy64@gmpg.org','211.83.233.2','2011-02-26 11:55:04'), (222,'Lillian','lbanks65@msu.edu','124.233.12.80','2010-05-16 20:29:02'), (223,'Ernest','enguyen66@baidu.com','82.45.128.148','1996-07-04 10:07:04'), (224,'Ryan','rrussell67@cloudflare.com','202.53.240.223','1983-08-05 12:36:29'), (225,'Donald','ddavis68@ustream.tv','47.39.218.137','1989-05-27 02:30:56'), (226,'Joe','jscott69@blogspot.com','140.23.131.75','1973-03-16 12:21:31'), (227,'Anne','amarshall6a@google.ca','113.162.200.197','1988-12-09 03:38:29'), (228,'Willie','wturner6b@constantcontact.com','85.83.182.249','1991-10-06 01:51:10'), (229,'Nicole','nwilson6c@sogou.com','30.223.51.135','1977-05-29 19:54:56'), (230,'Janet','jwheeler6d@stumbleupon.com','153.194.27.144','2011-03-13 12:48:47'), (231,'Lois','lcarr6e@statcounter.com','0.41.36.53','1993-02-06 04:52:01'), (232,'Shirley','scruz6f@tmall.com','37.156.39.223','2007-02-18 17:47:01'), (233,'Patrick','pford6g@reverbnation.com','36.198.200.89','1977-03-06 15:47:24'), (234,'Lisa','lhudson6h@usatoday.com','134.213.58.137','2014-10-28 01:56:56'), (235,'Pamela','pmartinez6i@opensource.org','5.151.127.202','1987-11-30 16:44:47'), (236,'Larry','lperez6j@infoseek.co.jp','235.122.96.148','1979-01-18 06:33:45'), (237,'Pamela','pramirez6k@census.gov','138.233.34.163','2012-01-29 10:35:20'), (238,'Daniel','dcarr6l@php.net','146.21.152.242','1984-11-17 08:22:59'), (239,'Patrick','psmith6m@indiegogo.com','136.222.199.36','2001-05-30 22:16:44'), (240,'Raymond','rhenderson6n@hc360.com','116.31.112.38','2000-01-05 20:35:41'), (241,'Teresa','treynolds6o@miitbeian.gov.cn','198.126.205.220','1996-11-08 01:27:31'), (242,'Johnny','jmason6p@flickr.com','192.8.232.114','2013-05-14 05:35:50'), (243,'Angela','akelly6q@guardian.co.uk','234.116.60.197','1977-08-20 02:05:17'), (244,'Douglas','dcole6r@cmu.edu','128.135.212.69','2016-10-26 17:40:36'), (245,'Frances','fcampbell6s@twitpic.com','94.22.243.235','1987-04-26 07:07:13'), (246,'Donna','dgreen6t@chron.com','227.116.46.107','2011-07-25 12:59:54'), (247,'Benjamin','bfranklin6u@redcross.org','89.141.142.89','1974-05-03 20:28:18'), (248,'Randy','rpalmer6v@rambler.ru','70.173.63.178','2011-12-20 17:40:18'), (249,'Melissa','mmurray6w@bbb.org','114.234.118.137','1991-02-26 12:45:44'), (250,'Jean','jlittle6x@epa.gov','141.21.163.254','1991-08-16 04:57:09'), (251,'Daniel','dolson6y@nature.com','125.75.104.97','2010-04-23 06:25:54'), (252,'Kathryn','kwells6z@eventbrite.com','225.104.28.249','2015-01-31 02:21:50'), (253,'Theresa','tgonzalez70@ox.ac.uk','91.93.156.26','1971-12-11 10:31:31'), (254,'Beverly','broberts71@bluehost.com','244.40.158.89','2013-09-21 13:02:31'), (255,'Pamela','pmurray72@netscape.com','218.54.95.216','1985-04-16 00:34:00'), (256,'Timothy','trichardson73@amazonaws.com','235.49.24.229','2000-11-11 09:48:28'), (257,'Mildred','mpalmer74@is.gd','234.125.95.132','1992-05-25 02:25:02'), (258,'Jessica','jcampbell75@google.it','55.98.30.140','2014-08-26 00:26:34'), (259,'Beverly','bthomas76@cpanel.net','48.78.228.176','1970-08-18 10:40:05'), (260,'Eugene','eward77@cargocollective.com','139.226.204.2','1996-12-04 23:17:00'), (261,'Andrea','aallen78@webnode.com','160.31.214.38','2009-07-06 07:22:37'), (262,'Justin','jruiz79@merriam-webster.com','150.149.246.122','2005-06-06 11:44:19'), (263,'Kenneth','kedwards7a@networksolutions.com','98.82.193.128','2001-07-03 02:00:10'), (264,'Rachel','rday7b@miibeian.gov.cn','114.15.247.221','1994-08-18 19:45:40'), (265,'Russell','rmiller7c@instagram.com','184.130.152.253','1977-11-06 01:58:12'), (266,'Bonnie','bhudson7d@cornell.edu','235.180.186.206','1990-12-03 22:45:24'), (267,'Raymond','rknight7e@yandex.ru','161.2.44.252','1995-08-25 04:31:19'), (268,'Bonnie','brussell7f@elpais.com','199.237.57.207','1991-03-29 08:32:06'), (269,'Marie','mhenderson7g@elpais.com','52.203.131.144','2004-06-04 21:50:28'), (270,'Alan','acarr7h@trellian.com','147.51.205.72','2005-03-03 10:51:31'), (271,'Barbara','bturner7i@hugedomains.com','103.160.110.226','2004-08-04 13:42:40'), (272,'Christina','cdaniels7j@census.gov','0.238.61.251','1972-10-18 12:47:33'), (273,'Jeremy','jgomez7k@reuters.com','111.26.65.56','2013-01-13 10:41:35'), (274,'Laura','lwood7l@icio.us','149.153.38.205','2011-06-25 09:33:59'), (275,'Matthew','mbowman7m@auda.org.au','182.138.206.172','1999-03-05 03:25:36'), (276,'Denise','dparker7n@icq.com','0.213.88.138','2011-11-04 09:43:06'), (277,'Phillip','pparker7o@discuz.net','219.242.165.240','1973-10-19 04:22:29'), (278,'Joan','jpierce7p@salon.com','63.31.213.202','1989-04-09 22:06:24'), (279,'Irene','ibaker7q@cbc.ca','102.33.235.114','1992-09-04 13:00:57'), (280,'Betty','bbowman7r@ted.com','170.91.249.242','2015-09-28 08:14:22'), (281,'Teresa','truiz7s@boston.com','82.108.158.207','1999-07-18 05:17:09'), (282,'Helen','hbrooks7t@slideshare.net','102.87.162.187','2003-01-06 15:45:29'), (283,'Karen','kgriffin7u@wunderground.com','43.82.44.184','2010-05-28 01:56:37'), (284,'Lisa','lfernandez7v@mtv.com','200.238.218.220','1993-04-03 20:33:51'), (285,'Jesse','jlawrence7w@timesonline.co.uk','95.122.105.78','1990-01-05 17:28:43'), (286,'Terry','tross7x@macromedia.com','29.112.114.133','2009-08-29 21:32:17'), (287,'Angela','abradley7y@icq.com','177.44.27.72','1989-10-04 21:46:06'), (288,'Maria','mhart7z@dailymotion.com','55.27.55.202','1975-01-21 01:22:57'), (289,'Raymond','randrews80@pinterest.com','88.90.78.67','1992-03-16 21:37:40'), (290,'Kathy','krice81@bluehost.com','212.63.196.102','2000-12-14 03:06:44'), (291,'Cynthia','cramos82@nymag.com','107.89.190.6','2005-06-28 02:02:33'), (292,'Kimberly','kjones83@mysql.com','86.169.101.101','2007-06-13 22:56:49'), (293,'Timothy','thansen84@microsoft.com','108.100.254.90','2003-04-04 10:31:57'), (294,'Carol','cspencer85@berkeley.edu','75.118.144.187','1999-03-30 14:53:21'), (295,'Louis','lmedina86@latimes.com','141.147.163.24','1991-04-11 17:53:13'), (296,'Margaret','mcole87@google.fr','53.184.26.83','1991-12-19 01:54:10'), (297,'Mary','mgomez88@yellowpages.com','208.56.57.99','1976-05-21 18:05:08'), (298,'Amanda','aanderson89@geocities.com','147.73.15.252','1987-08-22 15:05:28'), (299,'Kathryn','kgarrett8a@nature.com','27.29.177.220','1976-07-15 04:25:04'), (300,'Dorothy','dmason8b@shareasale.com','106.210.99.193','1990-09-03 21:39:31'), (301,'Lois','lkennedy8c@amazon.de','194.169.29.187','2007-07-29 14:09:31'), (302,'Irene','iburton8d@washingtonpost.com','196.143.110.249','2013-09-05 11:32:46'), (303,'Betty','belliott8e@wired.com','183.105.222.199','1979-09-19 19:29:13'), (304,'Bobby','bmeyer8f@census.gov','36.13.161.145','2014-05-24 14:34:39'), (305,'Ann','amorrison8g@sfgate.com','72.154.54.137','1978-10-05 14:22:34'), (306,'Daniel','djackson8h@wunderground.com','144.95.32.34','1990-07-27 13:23:05'), (307,'Joe','jboyd8i@alibaba.com','187.105.86.178','2011-09-28 16:46:32'), (308,'Ralph','rdunn8j@fc2.com','3.19.87.255','1984-10-18 08:00:40'), (309,'Craig','ccarter8k@gizmodo.com','235.152.76.215','1998-07-04 12:15:21'), (310,'Paula','pdean8l@hhs.gov','161.100.173.197','1973-02-13 09:38:55'), (311,'Andrew','agarrett8m@behance.net','199.253.123.218','1991-02-14 13:36:32'), (312,'Janet','jhowell8n@alexa.com','39.189.139.79','2012-11-24 20:17:33'), (313,'Keith','khansen8o@godaddy.com','116.186.223.196','1987-08-23 21:22:05'), (314,'Nicholas','nedwards8p@state.gov','142.175.142.11','1977-03-28 18:27:27'), (315,'Jacqueline','jallen8q@oaic.gov.au','189.66.135.192','1994-10-26 11:44:26'), (316,'Frank','fgardner8r@mapy.cz','154.77.119.169','1983-01-29 19:19:51'), (317,'Eric','eharrison8s@google.cn','245.139.65.123','1984-02-04 09:54:36'), (318,'Gregory','gcooper8t@go.com','171.147.0.221','2004-06-14 05:22:08'), (319,'Jean','jfreeman8u@rakuten.co.jp','67.243.121.5','1977-01-07 18:23:43'), (320,'Juan','jlewis8v@shinystat.com','216.181.171.189','2001-08-23 17:32:43'), (321,'Randy','rwilliams8w@shinystat.com','105.152.146.28','1983-02-17 00:05:50'), (322,'Stephen','shart8x@sciencedirect.com','196.131.205.148','2004-02-15 10:12:03'), (323,'Annie','ahunter8y@example.com','63.36.34.103','2003-07-23 21:15:25'), (324,'Melissa','mflores8z@cbc.ca','151.230.217.90','1983-11-02 14:53:56'), (325,'Jane','jweaver90@about.me','0.167.235.217','1987-07-29 00:13:44'), (326,'Anthony','asmith91@oracle.com','97.87.48.41','2001-05-31 18:44:11'), (327,'Terry','tdavis92@buzzfeed.com','46.20.12.51','2015-09-12 23:13:55'), (328,'Brandon','bmontgomery93@gravatar.com','252.101.48.186','2010-10-28 08:26:27'), (329,'Chris','cmurray94@bluehost.com','25.158.167.97','2004-05-05 16:10:31'), (330,'Denise','dfuller95@hugedomains.com','216.210.149.28','1979-04-20 08:57:24'), (331,'Arthur','amcdonald96@sakura.ne.jp','206.42.36.213','2009-08-15 03:26:16'), (332,'Jesse','jhoward97@google.cn','46.181.118.30','1974-04-18 14:08:41'), (333,'Frank','fsimpson98@domainmarket.com','163.220.211.87','2006-06-30 14:46:52'), (334,'Janice','jwoods99@pen.io','229.245.237.182','1988-04-06 11:52:58'), (335,'Rebecca','rroberts9a@huffingtonpost.com','148.96.15.80','1976-10-05 08:44:16'), (336,'Joshua','jray9b@opensource.org','192.253.12.198','1971-12-25 22:27:07'), (337,'Joyce','jcarpenter9c@statcounter.com','125.171.46.215','2001-12-31 22:08:13'), (338,'Andrea','awest9d@privacy.gov.au','79.101.180.201','1983-02-18 20:07:47'), (339,'Christine','chudson9e@yelp.com','64.198.43.56','1997-09-08 08:03:43'), (340,'Joe','jparker9f@earthlink.net','251.215.148.153','1973-11-04 05:08:18'), (341,'Thomas','tkim9g@answers.com','49.187.34.47','1991-08-07 21:13:48'), (342,'Janice','jdean9h@scientificamerican.com','4.197.117.16','2009-12-08 02:35:49'), (343,'James','jmitchell9i@umich.edu','43.121.18.147','2011-04-28 17:04:09'), (344,'Charles','cgardner9j@purevolume.com','197.78.240.240','1998-02-11 06:47:07'), (345,'Robert','rhenderson9k@friendfeed.com','215.84.180.88','2002-05-10 15:33:14'), (346,'Chris','cgray9l@4shared.com','249.70.192.240','1998-10-03 16:43:42'), (347,'Gloria','ghayes9m@hibu.com','81.103.138.26','1999-12-26 11:23:13'), (348,'Edward','eramirez9n@shareasale.com','38.136.90.136','2010-08-19 08:01:06'), (349,'Cheryl','cbutler9o@google.ca','172.180.78.172','1995-05-27 20:03:52'), (350,'Margaret','mwatkins9p@sfgate.com','3.20.198.6','2014-10-21 01:42:58'), (351,'Rebecca','rwelch9q@examiner.com','45.81.42.208','2001-02-08 12:19:06'), (352,'Joe','jpalmer9r@phpbb.com','163.202.92.190','1970-01-05 11:29:12'), (353,'Sandra','slewis9s@dyndns.org','77.215.201.236','1974-01-05 07:04:04'), (354,'Todd','tfranklin9t@g.co','167.125.181.82','2009-09-28 10:13:58'), (355,'Joseph','jlewis9u@webmd.com','244.204.6.11','1990-10-21 15:49:57'), (356,'Alan','aknight9v@nydailynews.com','152.197.95.83','1996-03-08 08:43:17'), (357,'Sharon','sdean9w@123-reg.co.uk','237.46.40.26','1985-11-30 12:09:24'), (358,'Annie','awright9x@cafepress.com','190.45.231.111','2000-08-24 11:56:06'), (359,'Diane','dhamilton9y@youtube.com','85.146.171.196','2015-02-24 02:03:57'), (360,'Antonio','alane9z@auda.org.au','61.63.146.203','2001-05-13 03:43:34'), (361,'Matthew','mallena0@hhs.gov','29.97.32.19','1973-02-19 23:43:32'), (362,'Bonnie','bfowlera1@soup.io','251.216.99.53','2013-08-01 15:35:41'), (363,'Margaret','mgraya2@examiner.com','69.255.151.79','1998-01-23 22:24:59'), (364,'Joan','jwagnera3@printfriendly.com','192.166.120.61','1973-07-13 00:30:22'), (365,'Catherine','cperkinsa4@nytimes.com','58.21.24.214','2006-11-19 11:52:26'), (366,'Mark','mcartera5@cpanel.net','220.33.102.142','2007-09-09 09:43:27'), (367,'Paula','ppricea6@msn.com','36.182.238.124','2009-11-11 09:13:05'), (368,'Catherine','cgreena7@army.mil','228.203.58.19','2005-08-09 16:52:15'), (369,'Helen','hhamiltona8@symantec.com','155.56.194.99','2005-02-01 05:40:36'), (370,'Jane','jmeyera9@ezinearticles.com','133.244.113.213','2013-11-06 22:10:23'), (371,'Wanda','wevansaa@bloglovin.com','233.125.192.48','1994-12-26 23:43:42'), (372,'Mark','mmarshallab@tumblr.com','114.74.60.47','2016-09-29 18:03:01'), (373,'Andrew','amartinezac@google.cn','182.54.37.130','1976-06-06 17:04:17'), (374,'Helen','hmoralesad@e-recht24.de','42.45.4.123','1977-03-28 19:06:59'), (375,'Bonnie','bstoneae@php.net','196.149.79.137','1970-02-05 17:05:58'), (376,'Douglas','dfreemanaf@nasa.gov','215.65.124.218','2008-11-20 21:51:55'), (377,'Willie','wwestag@army.mil','35.189.92.118','1992-07-24 05:08:08'), (378,'Cheryl','cwagnerah@upenn.edu','228.239.222.141','2010-01-25 06:29:01'), (379,'Sandra','swardai@baidu.com','63.11.113.240','1985-05-23 08:07:37'), (380,'Julie','jrobinsonaj@jugem.jp','110.58.202.50','2015-03-05 09:42:07'), (381,'Larry','lwagnerak@shop-pro.jp','98.234.25.24','1975-07-22 22:22:02'), (382,'Juan','jcastilloal@yelp.com','24.174.74.202','2007-01-17 09:32:43'), (383,'Donna','dfrazieram@artisteer.com','205.26.147.45','1990-02-11 20:55:46'), (384,'Rachel','rfloresan@w3.org','109.60.216.162','1983-05-22 22:42:18'), (385,'Robert','rreynoldsao@theguardian.com','122.65.209.130','2009-05-01 18:02:51'), (386,'Donald','dbradleyap@etsy.com','42.54.35.126','1997-01-16 16:31:52'), (387,'Rachel','rfisheraq@nih.gov','160.243.250.45','2006-02-17 22:05:49'), (388,'Nicholas','nhamiltonar@princeton.edu','156.211.37.111','1976-06-21 03:36:29'), (389,'Timothy','twhiteas@ca.gov','36.128.23.70','1975-09-24 03:51:18'), (390,'Diana','dbradleyat@odnoklassniki.ru','44.102.120.184','1983-04-27 09:02:50'), (391,'Billy','bfowlerau@jimdo.com','91.200.68.196','1995-01-29 06:57:35'), (392,'Bruce','bandrewsav@ucoz.com','48.12.101.125','1992-10-27 04:31:39'), (393,'Linda','lromeroaw@usa.gov','100.71.233.19','1992-06-08 15:13:18'), (394,'Debra','dwatkinsax@ucoz.ru','52.160.233.193','2001-11-11 06:51:01'), (395,'Katherine','kburkeay@wix.com','151.156.242.141','2010-06-14 19:54:28'), (396,'Martha','mharrisonaz@youku.com','21.222.10.199','1989-10-16 14:17:55'), (397,'Dennis','dwellsb0@youtu.be','103.16.29.3','1985-12-21 06:05:51'), (398,'Gloria','grichardsb1@bloglines.com','90.147.120.234','1982-08-27 01:04:43'), (399,'Brenda','bfullerb2@t.co','33.253.63.90','2011-04-20 05:00:35'), (400,'Larry','lhendersonb3@disqus.com','88.95.132.128','1982-08-31 02:15:12'), (401,'Richard','rlarsonb4@wisc.edu','13.48.231.150','1979-04-15 14:08:09'), (402,'Terry','thuntb5@usa.gov','65.91.103.240','1998-05-15 11:50:49'), (403,'Harry','hburnsb6@nasa.gov','33.38.21.244','1981-04-12 14:02:20'), (404,'Diana','dellisb7@mlb.com','218.229.81.135','1997-01-29 00:17:25'), (405,'Jack','jburkeb8@tripadvisor.com','210.227.182.216','1984-03-09 17:24:03'), (406,'Julia','jlongb9@fotki.com','10.210.12.104','2005-10-26 03:54:13'), (407,'Lois','lscottba@msu.edu','188.79.136.138','1973-02-02 18:40:39'), (408,'Sandra','shendersonbb@shareasale.com','114.171.220.108','2012-06-09 18:22:26'), (409,'Irene','isanchezbc@cdbaby.com','109.255.50.119','1983-09-28 21:11:27'), (410,'Emily','ebrooksbd@bandcamp.com','227.81.93.79','1970-08-31 21:08:01'), (411,'Michelle','mdiazbe@businessweek.com','236.249.6.226','1993-05-22 08:07:07'), (412,'Tammy','tbennettbf@wisc.edu','145.253.239.152','1978-12-31 20:24:51'), (413,'Christine','cgreenebg@flickr.com','97.25.140.118','1978-07-17 12:55:30'), (414,'Patricia','pgarzabh@tuttocitta.it','139.246.192.211','1984-02-27 13:40:08'), (415,'Kimberly','kromerobi@aol.com','73.56.88.247','1976-09-16 14:22:04'), (416,'George','gjohnstonbj@fda.gov','240.36.245.185','1979-07-24 14:36:02'), (417,'Eugene','efullerbk@sciencedaily.com','42.38.105.140','2012-09-12 01:56:41'), (418,'Andrea','astevensbl@goo.gl','31.152.207.204','1979-05-24 11:06:21'), (419,'Shirley','sreidbm@scientificamerican.com','103.60.31.241','1984-02-23 04:07:41'), (420,'Terry','tmorenobn@blinklist.com','92.161.34.42','1994-06-25 14:01:35'), (421,'Christopher','cmorenobo@go.com','158.86.176.82','1973-09-05 09:18:47'), (422,'Dennis','dhansonbp@ning.com','40.160.81.75','1982-01-20 10:19:41'), (423,'Beverly','brussellbq@de.vu','138.32.56.204','1997-11-06 07:20:19'), (424,'Howard','hparkerbr@163.com','103.171.134.171','2015-06-24 15:37:10'), (425,'Helen','hmccoybs@fema.gov','61.200.4.71','1995-06-20 08:59:10'), (426,'Ann','ahudsonbt@cafepress.com','239.187.71.125','1977-04-11 07:59:28'), (427,'Tina','twestbu@nhs.uk','80.213.117.74','1992-08-19 05:54:44'), (428,'Terry','tnguyenbv@noaa.gov','21.93.118.95','1991-09-19 23:22:55'), (429,'Ashley','aburtonbw@wix.com','233.176.205.109','2009-11-10 05:01:20'), (430,'Eric','emyersbx@1und1.de','168.91.212.67','1987-08-10 07:16:20'), (431,'Barbara','blittleby@lycos.com','242.14.189.239','2008-08-02 12:13:04'), (432,'Sean','sevansbz@instagram.com','14.39.177.13','2007-04-16 17:28:49'), (433,'Shirley','sburtonc0@newsvine.com','34.107.138.76','1980-12-10 02:19:29'), (434,'Patricia','pfreemanc1@so-net.ne.jp','219.213.142.117','1987-03-01 02:25:45'), (435,'Paula','pfosterc2@vkontakte.ru','227.14.138.141','1972-09-22 12:59:34'), (436,'Nicole','nstewartc3@1688.com','8.164.23.115','1998-10-27 00:10:17'), (437,'Earl','ekimc4@ovh.net','100.26.244.177','2013-01-22 10:05:46'), (438,'Beverly','breedc5@reuters.com','174.12.226.27','1974-09-22 07:29:36'), (439,'Lawrence','lbutlerc6@a8.net','105.164.42.164','1992-06-05 00:43:40'), (440,'Charles','cmoorec7@ucoz.com','252.197.131.69','1990-04-09 02:34:05'), (441,'Alice','alawsonc8@live.com','183.73.220.232','1989-02-28 09:11:04'), (442,'Dorothy','dcarpenterc9@arstechnica.com','241.47.200.14','2005-05-02 19:57:21'), (443,'Carolyn','cfowlerca@go.com','213.109.55.202','1978-09-10 20:18:20'), (444,'Anthony','alongcb@free.fr','169.221.158.204','1984-09-13 01:59:23'), (445,'Annie','amoorecc@e-recht24.de','50.34.148.61','2009-03-26 03:41:07'), (446,'Carlos','candrewscd@ihg.com','236.69.59.212','1972-03-29 22:42:48'), (447,'Beverly','bramosce@google.ca','164.250.184.49','1982-11-10 04:34:01'), (448,'Teresa','tlongcf@umich.edu','174.88.53.223','1987-05-17 12:48:00'), (449,'Roy','rboydcg@uol.com.br','91.58.243.215','1974-06-16 17:59:54'), (450,'Ashley','afieldsch@tamu.edu','130.138.11.126','1983-09-15 05:52:36'), (451,'Judith','jhawkinsci@cmu.edu','200.187.103.245','2003-10-22 12:24:03'), (452,'Rebecca','rwestcj@ocn.ne.jp','72.85.3.103','1980-11-13 11:01:26'), (453,'Raymond','rporterck@infoseek.co.jp','146.33.216.151','1982-05-17 23:58:03'), (454,'Janet','jmarshallcl@odnoklassniki.ru','52.46.193.166','1998-10-04 00:02:21'), (455,'Shirley','speterscm@salon.com','248.126.31.15','1987-01-30 06:04:59'), (456,'Annie','abowmancn@economist.com','222.213.248.59','2006-03-14 23:52:59'), (457,'Jean','jlarsonco@blogspot.com','71.41.25.195','2007-09-08 23:49:45'), (458,'Phillip','pmoralescp@stanford.edu','74.119.87.28','2011-03-14 20:25:40'), (459,'Norma','nrobinsoncq@economist.com','28.225.21.54','1989-10-21 01:22:43'), (460,'Kimberly','kclarkcr@dion.ne.jp','149.171.132.153','2008-06-27 02:27:30'), (461,'Ruby','rmorriscs@ucla.edu','177.85.163.249','2016-01-28 16:43:44'), (462,'Jonathan','jcastilloct@tripod.com','78.4.28.77','2000-05-24 17:33:06'), (463,'Edward','ebryantcu@jigsy.com','140.31.98.193','1992-12-17 08:32:47'), (464,'Chris','chamiltoncv@eepurl.com','195.171.234.206','1970-12-05 03:42:19'), (465,'Michael','mweavercw@reference.com','7.233.133.213','1987-03-29 02:30:54'), (466,'Howard','hlawrencecx@businessweek.com','113.225.124.224','1990-07-30 07:20:57'), (467,'Philip','phowardcy@comsenz.com','159.170.247.249','2010-10-15 10:18:37'), (468,'Mary','mmarshallcz@xing.com','125.132.189.70','2007-07-19 13:48:47'), (469,'Scott','salvarezd0@theguardian.com','78.49.103.230','1987-10-31 06:10:44'), (470,'Wayne','wcarrolld1@blog.com','238.1.120.204','1980-11-19 03:26:10'), (471,'Jennifer','jwoodsd2@multiply.com','92.20.224.49','2010-05-06 22:17:04'), (472,'Raymond','rwelchd3@toplist.cz','176.158.35.240','2007-12-12 19:02:51'), (473,'Steven','sdixond4@wisc.edu','167.55.237.52','1984-05-05 11:44:37'), (474,'Ralph','rjamesd5@ameblo.jp','241.190.50.133','2000-07-06 08:44:37'), (475,'Jason','jrobinsond6@hexun.com','138.119.139.56','2006-02-03 05:27:45'), (476,'Doris','dwoodd7@fema.gov','180.220.156.190','1978-05-11 20:14:20'), (477,'Elizabeth','eberryd8@youtu.be','74.188.53.229','2006-11-18 08:29:06'), (478,'Irene','igilbertd9@privacy.gov.au','194.152.218.1','1985-09-17 02:46:52'), (479,'Jessica','jdeanda@ameblo.jp','178.103.93.118','1974-06-07 19:04:05'), (480,'Rachel','ralvarezdb@phoca.cz','17.22.223.174','1999-03-08 02:43:25'), (481,'Kenneth','kthompsondc@shinystat.com','229.119.91.234','2007-05-15 13:17:32'), (482,'Harold','hmurraydd@parallels.com','133.26.188.80','1993-11-15 03:42:07'), (483,'Paula','phowellde@samsung.com','34.215.28.216','1993-11-29 15:55:00'), (484,'Ruth','rpiercedf@tripadvisor.com','111.30.130.123','1986-08-17 10:19:38'), (485,'Phyllis','paustindg@vk.com','50.84.34.178','1994-04-13 03:05:24'), (486,'Laura','lfosterdh@usnews.com','37.8.101.33','2001-06-30 08:58:59'), (487,'Eric','etaylordi@com.com','103.183.253.45','2006-09-15 20:18:46'), (488,'Doris','driveradj@prweb.com','247.16.2.199','1989-05-08 09:27:09'), (489,'Ryan','rhughesdk@elegantthemes.com','103.234.153.232','1989-08-01 18:36:06'), (490,'Steve','smoralesdl@jigsy.com','3.76.84.207','2011-03-13 17:01:05'), (491,'Louis','lsullivandm@who.int','78.135.44.208','1975-11-26 16:01:23'), (492,'Catherine','ctuckerdn@seattletimes.com','93.137.106.21','1990-03-13 16:14:56'), (493,'Ann','adixondo@gmpg.org','191.136.222.111','2002-06-05 14:22:18'), (494,'Johnny','jhartdp@amazon.com','103.252.198.39','1988-07-30 23:54:49'), (495,'Susan','srichardsdq@skype.com','126.247.192.11','2005-01-09 12:08:14'), (496,'Brenda','bparkerdr@skype.com','63.232.216.86','1974-05-18 05:58:29'), (497,'Tammy','tmurphyds@constantcontact.com','56.56.37.112','2014-08-05 18:22:25'), (498,'Larry','lhayesdt@wordpress.com','162.146.13.46','1997-02-26 14:01:53'), (499,'Evelyn','ethomasdu@hhs.gov','6.241.88.250','2007-09-14 13:03:34'), (500,'Paula','pshawdv@networksolutions.com','123.27.47.249','2003-10-30 21:19:20'); create table {schema}.seed_config_expected_1 as ( select *, 'default'::text as c1, 'default'::text as c2, 'was true'::text as some_bool from {schema}.seed ); create table {schema}.seed_config_expected_2 as ( select *, 'abc'::text as c1, 'def'::text as c2, 'was true'::text as some_bool from {schema}.seed ); create table {schema}.seed_config_expected_3 as ( select *, 'ghi'::text as c1, 'jkl'::text as c2, 'was true'::text as some_bool from {schema}.seed ); create table {schema}.seed_summary ( year timestamp without time zone, count bigint ); INSERT INTO {schema}.seed_summary ("year","count") VALUES ('1970-01-01 00:00:00',10), ('1971-01-01 00:00:00',6), ('1972-01-01 00:00:00',9), ('1973-01-01 00:00:00',12), ('1974-01-01 00:00:00',8), ('1975-01-01 00:00:00',5), ('1976-01-01 00:00:00',11), ('1977-01-01 00:00:00',13), ('1978-01-01 00:00:00',11), ('1979-01-01 00:00:00',13), ('1980-01-01 00:00:00',9), ('1981-01-01 00:00:00',3), ('1982-01-01 00:00:00',9), ('1983-01-01 00:00:00',15), ('1984-01-01 00:00:00',13), ('1985-01-01 00:00:00',11), ('1986-01-01 00:00:00',5), ('1987-01-01 00:00:00',14), ('1988-01-01 00:00:00',9), ('1989-01-01 00:00:00',10), ('1990-01-01 00:00:00',12), ('1991-01-01 00:00:00',16), ('1992-01-01 00:00:00',15), ('1993-01-01 00:00:00',11), ('1994-01-01 00:00:00',10), ('1995-01-01 00:00:00',10), ('1996-01-01 00:00:00',6), ('1997-01-01 00:00:00',11), ('1998-01-01 00:00:00',12), ('1999-01-01 00:00:00',9), ('2000-01-01 00:00:00',13), ('2001-01-01 00:00:00',14), ('2002-01-01 00:00:00',9), ('2003-01-01 00:00:00',8), ('2004-01-01 00:00:00',9), ('2005-01-01 00:00:00',14), ('2006-01-01 00:00:00',9), ('2007-01-01 00:00:00',16), ('2008-01-01 00:00:00',6), ('2009-01-01 00:00:00',15), ('2010-01-01 00:00:00',13), ('2011-01-01 00:00:00',23), ('2012-01-01 00:00:00',9), ('2013-01-01 00:00:00',10), ('2014-01-01 00:00:00',9), ('2015-01-01 00:00:00',10), ('2016-01-01 00:00:00',5); ================================================ FILE: tests/functional/dependencies/data/update.sql ================================================ UPDATE {schema}.seed set first_name = 'Paul', updated_at = now() where id = 500; INSERT INTO {schema}.seed ("id","first_name","email","ip_address","updated_at") VALUES (501, 'Steve', 'sthomas@hhs.gov', '6.241.88.251', now()); ================================================ FILE: tests/functional/dependencies/duplicate_dependency/dbt_project.yml ================================================ name: 'test' version: '1.0' config-version: 2 profile: 'default' ================================================ FILE: tests/functional/dependencies/early_hook_dependency/dbt_project.yml ================================================ name: early_hooks version: '1.0' config-version: 2 on-run-start: - create table {{ var('test_create_table') }} as (select 1 as id) - create table {{ var('test_create_second_table') }} as (select 3 as id) ================================================ FILE: tests/functional/dependencies/inverted_ref_dependency/dbt_project.yml ================================================ name: 'inverted_ref_dependency' version: '1.0' config-version: 2 profile: 'default' model-paths: ["models"] analysis-paths: ["analyses"] test-paths: ["tests"] seed-paths: ["seeds"] macro-paths: ["macros"] require-dbt-version: '>=0.1.0' target-path: "target" # directory which will store compiled SQL files clean-targets: # directories to be removed by `dbt clean` - "target" - "dbt_packages" seeds: quote_columns: False ================================================ FILE: tests/functional/dependencies/inverted_ref_dependency/models/a.sql ================================================ {{ config(alias='package_a')}} select 1 as id ================================================ FILE: tests/functional/dependencies/inverted_ref_dependency/models/b.sql ================================================ select * from {{ ref('a') }} ================================================ FILE: tests/functional/dependencies/inverted_ref_dependency/models/b_root_package_in_ref.sql ================================================ select * from {{ ref('test', 'a') }} ================================================ FILE: tests/functional/dependencies/late_hook_dependency/dbt_project.yml ================================================ name: late_hooks version: '1.0' config-version: 2 on-run-start: - insert into {{ var('test_create_table') }} values (2) - insert into {{ var('test_create_second_table') }} values (4) ================================================ FILE: tests/functional/dependencies/local_dependency/dbt_project.yml ================================================ name: 'local_dep' version: '1.0' config-version: 2 profile: 'default' model-paths: ["models"] analysis-paths: ["analyses"] test-paths: ["tests"] seed-paths: ["seeds"] macro-paths: ["macros"] require-dbt-version: '>=0.1.0' target-path: "target" # directory which will store compiled SQL files clean-targets: # directories to be removed by `dbt clean` - "target" - "dbt_packages" seeds: quote_columns: False ================================================ FILE: tests/functional/dependencies/local_dependency/macros/dep_macro.sql ================================================ {% macro some_overridden_macro() -%} 100 {%- endmacro %} ================================================ FILE: tests/functional/dependencies/local_dependency/macros/generate_schema_name.sql ================================================ {# This should not be ignored, even as it's in a subpackage #} {% macro generate_schema_name(custom_schema_name=none, node=none) -%} {{ var('schema_override', target.schema) }} {%- endmacro %} {# This should not be ignored, even as it's in a subpackage #} {% macro generate_database_name(custom_database_name=none, node=none) -%} {{ 'dbt' }} {%- endmacro %} {# This should not be ignored, even as it's in a subpackage #} {% macro generate_alias_name(custom_alias_name=none, node=none) -%} {{ node.name ~ '_subpackage_generate_alias_name' }} {%- endmacro %} ================================================ FILE: tests/functional/dependencies/local_dependency/models/model_to_import.sql ================================================ select * from {{ ref('seed') }} ================================================ FILE: tests/functional/dependencies/local_dependency/models/schema.yml ================================================ version: 2 sources: - name: my_source schema: invalid_schema tables: - name: my_table - name: seed_source schema: "{{ var('schema_override', target.schema) }}" tables: - name: "seed" identifier: "seed_subpackage_generate_alias_name" ================================================ FILE: tests/functional/dependencies/local_dependency/seeds/seed.csv ================================================ id 1 ================================================ FILE: tests/functional/dependencies/models_local/dep_source_model.sql ================================================ {# If our dependency source didn't exist, this would be an errror #} select * from {{ source('seed_source', 'seed') }} ================================================ FILE: tests/functional/dependencies/models_local/my_configured_model.sql ================================================ {{ config(schema='configured') }} select * from {{ ref('model_to_import') }} ================================================ FILE: tests/functional/dependencies/models_local/my_model.sql ================================================ select * from {{ ref('model_to_import') }} ================================================ FILE: tests/functional/dependencies/models_local/schema.yml ================================================ version: 2 sources: - name: my_source schema: "{{ var('schema_override', target.schema) }}" tables: - name: my_table identifier: seed ================================================ FILE: tests/functional/dependencies/models_local/source_override_model.sql ================================================ {# If our source override didn't take, this would be an errror #} select * from {{ source('my_source', 'my_table') }} ================================================ FILE: tests/functional/dependencies/nested_dependency/dbt_project.yml ================================================ name: 'nested_dependency' version: '1.0' config-version: 2 model-paths: ["models"] ================================================ FILE: tests/functional/dependencies/nested_dependency/models/model.sql ================================================ select 1 as id ================================================ FILE: tests/functional/dependencies/nested_dependency/packages.yml ================================================ packages: - package: godatadriven/dbt_date version: ">=0.10.0" ================================================ FILE: tests/functional/dependencies/test_add_package_edge_cases.py ================================================ import os import shutil import pytest from dbt.tests.util import run_dbt class TestAddPackageWithWarnUnpinnedInYaml: """Functional test: Adding packages works even with warn-unpinned in packages.yml. This is a regression test for issue #9104. The bug occurred when packages.yml contained warn-unpinned: false and dbt deps --add-package was run. The code would fail with "TypeError: argument of type 'bool' is not iterable". """ @pytest.fixture(scope="class") def packages(self): # Start with a git package that has warn-unpinned (matching the bug report) return { "packages": [ { "git": "https://github.com/fivetran/dbt_amplitude", "warn-unpinned": False, # This is the config that caused the bug }, ] } @pytest.fixture def clean_start(self, project): if os.path.exists("dbt_packages"): shutil.rmtree("dbt_packages") if os.path.exists("package-lock.yml"): os.remove("package-lock.yml") def test_add_package_with_warn_unpinned_in_yaml(self, clean_start): """Test that adding a package works when packages.yml contains warn-unpinned: false""" # Before the fix, this would raise: TypeError: argument of type 'bool' is not iterable # This matches the exact scenario from issue #9104 run_dbt(["deps", "--add-package", "dbt-labs/dbt_utils@1.0.0"]) with open("packages.yml") as fp: contents = fp.read() # Verify both packages are present assert "dbt_amplitude" in contents or "fivetran/dbt_amplitude" in contents assert "dbt-labs/dbt_utils" in contents or "dbt_utils" in contents # The warn-unpinned should still be there assert "warn-unpinned:" in contents or "warn_unpinned:" in contents ================================================ FILE: tests/functional/dependencies/test_dependency_inverted_ref.py ================================================ import shutil from pathlib import Path import pytest from dbt.events.types import PackageNodeDependsOnRootProjectNode from dbt.tests.util import run_dbt from dbt_common.events.event_catcher import EventCatcher class BaseInvertedRefDependencyTest(object): @pytest.fixture(scope="class") def models(self): return { "a.sql": "select 1 as id", } @pytest.fixture(scope="class", autouse=True) def setUp(self, project): shutil.copytree( project.test_dir / Path("inverted_ref_dependency"), project.project_root / Path("inverted_ref_dependency"), ) @pytest.fixture(scope="class") def packages(self): return {"packages": [{"local": "inverted_ref_dependency"}]} class TestInvertedRefDependency(BaseInvertedRefDependencyTest): @pytest.fixture(scope="class") def project_config_update(self): return { "flags": { "require_ref_searches_node_package_before_root": True, } } def test_inverted_ref_dependency(self, project): event_catcher = EventCatcher(PackageNodeDependsOnRootProjectNode) run_dbt(["deps"]) manifest = run_dbt(["parse"], callbacks=[event_catcher.catch]) assert len(manifest.nodes) == 4 # Correct behavior - package node depends on node from same package assert manifest.nodes["model.inverted_ref_dependency.b"].depends_on.nodes == [ "model.inverted_ref_dependency.a" ] # If a package explicitly references a root project node, it still resolves to root project manifest.nodes["model.inverted_ref_dependency.b_root_package_in_ref"].depends_on.nodes == [ "model.test.a" ] # No inverted ref warning raised assert len(event_catcher.caught_events) == 0 class TestInvertedRefDependencyLegacy(BaseInvertedRefDependencyTest): def test_inverted_ref_dependency(self, project): event_catcher = EventCatcher(PackageNodeDependsOnRootProjectNode) run_dbt(["deps"]) manifest = run_dbt(["parse"], callbacks=[event_catcher.catch]) assert len(manifest.nodes) == 4 # Legacy behavior - package node depends on node from root project assert manifest.nodes["model.inverted_ref_dependency.b"].depends_on.nodes == [ "model.test.a" ] assert manifest.nodes[ "model.inverted_ref_dependency.b_root_package_in_ref" ].depends_on.nodes == ["model.test.a"] # Inverted ref warning raised - only for b, not b_root_package_in_ref assert len(event_catcher.caught_events) == 1 assert event_catcher.caught_events[0].data.node_name == "b" assert event_catcher.caught_events[0].data.package_name == "inverted_ref_dependency" ================================================ FILE: tests/functional/dependencies/test_dependency_options.py ================================================ import os import shutil import pytest from dbt.tests.util import run_dbt class TestDepsOptions(object): # this revision of dbt-integration-project requires dbt-utils.git@0.5.0, which the # package config handling should detect @pytest.fixture(scope="class") def packages(self): return { "packages": [ { "package": "fivetran/fivetran_utils", "version": "0.4.7", }, ] } @pytest.fixture def clean_start(self, project): if os.path.exists("dbt_packages"): shutil.rmtree("dbt_packages") if os.path.exists("package-lock.yml"): os.remove("package-lock.yml") def test_deps_lock(self, clean_start): run_dbt(["deps", "--lock"]) assert not os.path.exists("dbt_packages") assert os.path.exists("package-lock.yml") with open("package-lock.yml") as fp: contents = fp.read() fivetran_package = """ - name: fivetran_utils package: fivetran/fivetran_utils version: 0.4.7 """ # dbt-utils is a dep in fivetran so we can't check for a specific version or this test fails everytime a new dbt-utils version comes out dbt_labs_package = """ - name: dbt_utils package: dbt-labs/dbt_utils """ package_sha = "sha1_hash: 71304bca2138cf8004070b3573a1e17183c0c1a8" assert fivetran_package in contents assert dbt_labs_package in contents assert package_sha in contents def test_deps_default(self, clean_start): run_dbt(["deps"]) assert len(os.listdir("dbt_packages")) == 2 assert os.path.exists("package-lock.yml") with open("package-lock.yml") as fp: contents = fp.read() fivetran_package = """ - name: fivetran_utils package: fivetran/fivetran_utils version: 0.4.7 """ # dbt-utils is a dep in fivetran so we can't check for a specific version or this test fails everytime a new dbt-utils version comes out dbt_labs_package = """ - name: dbt_utils package: dbt-labs/dbt_utils """ package_sha = "sha1_hash: 71304bca2138cf8004070b3573a1e17183c0c1a8" assert fivetran_package in contents assert dbt_labs_package in contents assert package_sha in contents def test_deps_add(self, clean_start): run_dbt(["deps", "--add-package", "dbt-labs/audit_helper@0.9.0"]) with open("packages.yml") as fp: contents = fp.read() assert ( contents == """packages: - package: fivetran/fivetran_utils version: 0.4.7 - package: dbt-labs/audit_helper version: 0.9.0 """ ) assert len(os.listdir("dbt_packages")) == 3 def test_deps_add_without_install(self, clean_start): os.rename("packages.yml", "dependencies.yml") run_dbt( [ "deps", "--add-package", "dbt-labs/audit_helper@0.9.0", "--lock", ] ) assert not os.path.exists("dbt_packages") assert not os.path.exists("packages.yml") with open("dependencies.yml") as fp: contents = fp.read() assert ( contents == """packages: - package: fivetran/fivetran_utils version: 0.4.7 - package: dbt-labs/audit_helper version: 0.9.0 """ ) def test_deps_upgrade(self, clean_start, mocker): run_dbt(["deps", "--lock"]) patched_lock = mocker.patch("dbt.task.deps.DepsTask.lock") run_dbt(["deps", "--upgrade"]) assert patched_lock.call_count == 1 ================================================ FILE: tests/functional/dependencies/test_dependency_secrets.py ================================================ import os import pytest from dbt.tests.util import run_dbt_and_capture from dbt_common.constants import SECRET_ENV_PREFIX class TestSecretInPackage: @pytest.fixture(scope="class", autouse=True) def setUp(self): os.environ[SECRET_ENV_PREFIX + "_FOR_LOGGING"] = "super secret" yield del os.environ[SECRET_ENV_PREFIX + "_FOR_LOGGING"] @pytest.fixture(scope="class") def packages(self): return { "packages": [ { "package": "dbt-labs/dbt_utils{{ log(env_var('DBT_ENV_SECRET_FOR_LOGGING'), info = true) }}", "version": "1.0.0", } ] } def test_mask_secrets(self, project): _, log_output = run_dbt_and_capture(["deps"]) # this will not be written to logs assert not ("super secret" in log_output) assert "*****" in log_output assert not ("DBT_ENV_SECRET_FOR_LOGGING" in log_output) ================================================ FILE: tests/functional/dependencies/test_local_dependency.py ================================================ import json import os import shutil from pathlib import Path from unittest import mock import pytest import yaml import dbt.config import dbt.exceptions import dbt_common.exceptions import dbt_common.semver as semver from dbt import deprecations from dbt.tests.util import ( check_relations_equal, run_dbt, run_dbt_and_capture, write_file, ) from tests.functional.utils import up_one # todo: make self.unique_schema to fixture models__dep_source = """ {# If our dependency source didn't exist, this would be an errror #} select * from {{ source('seed_source', 'seed') }} """ models__my_configured_model = """ {{ config(schema='configured') }} select * from {{ ref('model_to_import') }} """ models__my_model = """ select * from {{ ref('model_to_import') }} """ models__source_override_model = """ {# If our source override didn't take, this would be an errror #} select * from {{ source('my_source', 'my_table') }} """ models__iterate = """ {% for x in no_such_dependency.no_such_method() %} {% endfor %} """ models__hooks_actual = """ select * from {{ var('test_create_table') }} union all select * from {{ var('test_create_second_table') }} """ models__hooks_expected = """ {# surely there is a better way to do this! #} {% for _ in range(1, 5) %} select {{ loop.index }} as id {% if not loop.last %}union all{% endif %} {% endfor %} """ properties__schema_yml = """ version: 2 sources: - name: my_source schema: "{{ var('schema_override', target.schema) }}" tables: - name: my_table identifier: seed_subpackage_generate_alias_name """ macros__macro_sql = """ {# This macro also exists in the dependency -dbt should be fine with that #} {% macro some_overridden_macro() -%} 999 {%- endmacro %} """ macros__macro_override_schema_sql = """ {% macro generate_schema_name(schema_name, node) -%} {{ schema_name }}_{{ node.schema }}_macro {%- endmacro %} """ class BaseDependencyTest(object): @pytest.fixture(scope="class") def macros(self): return {"macro.sql": macros__macro_sql} @pytest.fixture(scope="class") def models(self): return { "dep_source_model.sql": models__dep_source, "my_configured_model.sql": models__my_configured_model, "my_model.sql": models__my_model, "source_override_model.sql": models__source_override_model, } @pytest.fixture(scope="class") def properties(self): return { "schema.yml": properties__schema_yml, } @pytest.fixture(scope="class", autouse=True) def modify_schema_fqn(self, project): schema_fqn = "{}.{}".format( project.database, project.test_schema, ) schema_fqn_configured = "{}.{}".format( project.database, project.test_schema + "_configured", ) project.created_schemas.append(schema_fqn) project.created_schemas.append(schema_fqn_configured) @pytest.fixture(scope="class", autouse=True) def setUp(self, project, modify_schema_fqn): shutil.copytree( project.test_dir / Path("local_dependency"), project.project_root / Path("local_dependency"), ) @pytest.fixture(scope="class") def packages(self): return {"packages": [{"local": "local_dependency"}]} class TestSimpleDependency(BaseDependencyTest): def test_local_dependency(self, project): run_dbt(["deps"]) run_dbt(["seed"]) results = run_dbt() assert len(results) == 5 assert {r.node.schema for r in results} == { project.test_schema, project.test_schema + "_configured", } base_schema_nodes = [r.node for r in results if r.node.schema == project.test_schema] assert len(base_schema_nodes) == 4 check_relations_equal( project.adapter, [ f"{project.test_schema}.source_override_model", f"{project.test_schema}.seed_subpackage_generate_alias_name", ], ) check_relations_equal( project.adapter, [ f"{project.test_schema}.dep_source_model", f"{project.test_schema}.seed_subpackage_generate_alias_name", ], ) def test_no_dependency_paths(self, project): run_dbt(["deps"]) run_dbt(["seed"]) # prove dependency does not exist as model in project dep_path = os.path.join("models_local", "model_to_import.sql") results = run_dbt( ["run", "--models", f"+{dep_path}"], ) assert len(results) == 0 # prove model can run when importing that dependency local_path = Path("models") / "my_model.sql" results = run_dbt( ["run", "--models", f"+{local_path}"], ) assert len(results) == 2 class TestSimpleDependencyRelativePath(BaseDependencyTest): def test_local_dependency_relative_path(self, project): last_dir = Path(project.project_root).name with up_one(): _, stdout = run_dbt_and_capture(["deps", "--project-dir", last_dir]) assert ( "Installed from " in stdout ), "Test output didn't contain expected string" class TestMissingDependency(object): @pytest.fixture(scope="class") def models(self): return { "iterate.sql": models__iterate, } def test_missing_dependency(self, project): # dbt should raise a runtime exception with pytest.raises(dbt_common.exceptions.DbtRuntimeError): run_dbt(["compile"]) class TestSimpleDependencyWithSchema(BaseDependencyTest): def dbt_vargs(self, schema): # we can't add this to the config because Sources don't respect dbt_project.yml vars_arg = yaml.safe_dump({"schema_override": "dbt_test_{}_macro".format(schema)}) return ["--vars", vars_arg] def project_config(self): return { "models": { "schema": "dbt_test", }, "seeds": { "schema": "dbt_test", }, } @mock.patch("dbt.config.project.get_installed_version") def test_local_dependency_out_of_date(self, mock_get, project): mock_get.return_value = semver.VersionSpecifier.from_version_string("0.0.1") run_dbt(["deps"] + self.dbt_vargs(project.test_schema)) # check seed with pytest.raises(dbt.exceptions.DbtProjectError) as exc: run_dbt(["seed"] + self.dbt_vargs(project.test_schema)) assert "--no-version-check" in str(exc.value) # check run too with pytest.raises(dbt.exceptions.DbtProjectError) as exc: run_dbt(["run"] + self.dbt_vargs(project.test_schema)) assert "--no-version-check" in str(exc.value) @mock.patch("dbt.config.project.get_installed_version") def test_local_dependency_out_of_date_no_check(self, mock_get): mock_get.return_value = semver.VersionSpecifier.from_version_string("0.0.1") run_dbt(["deps"]) run_dbt(["seed", "--no-version-check"]) results = run_dbt(["run", "--no-version-check"]) assert len(results) == 5 class TestSimpleDependencyNoVersionCheckConfig(BaseDependencyTest): @pytest.fixture(scope="class") def project_config_update(self): return { "flags": { "send_anonymous_usage_stats": False, "version_check": False, }, "models": { "schema": "dbt_test", }, "seeds": { "schema": "dbt_test", }, } @pytest.fixture(scope="class") def macros(self): return {"macro.sql": macros__macro_override_schema_sql} @mock.patch("dbt.config.project.get_installed_version") def test_local_dependency_out_of_date_no_check(self, mock_get, project): # we can't add this to the config because Sources don't respect dbt_project.yml base_schema = "dbt_test_{}_macro".format(project.test_schema) vars_arg = yaml.safe_dump( { "schema_override": base_schema, } ) mock_get.return_value = semver.VersionSpecifier.from_version_string("0.0.1") run_dbt(["deps", "--vars", vars_arg]) run_dbt(["seed", "--vars", vars_arg]) results = run_dbt(["run", "--vars", vars_arg]) len(results) == 5 class TestSimpleDependencyHooks(BaseDependencyTest): @pytest.fixture(scope="class") def models(self): return { "actual.sql": models__hooks_actual, "expected.sql": models__hooks_expected, } @pytest.fixture(scope="class") def project_config_update(self): # these hooks should run first, so nothing to drop return { "on-run-start": [ "drop table if exists {{ var('test_create_table') }}", "drop table if exists {{ var('test_create_second_table') }}", ] } @pytest.fixture(scope="class") def packages(self): return { "packages": [{"local": "early_hook_dependency"}, {"local": "late_hook_dependency"}] } @pytest.fixture(scope="class") def prepare_dependencies(self, project): shutil.copytree( project.test_dir / Path("early_hook_dependency"), project.project_root / Path("early_hook_dependency"), ) shutil.copytree( project.test_dir / Path("late_hook_dependency"), project.project_root / Path("late_hook_dependency"), ) def test_hook_dependency(self, prepare_dependencies, project): cli_vars = json.dumps( { "test_create_table": '"{}"."hook_test"'.format(project.test_schema), "test_create_second_table": '"{}"."hook_test_2"'.format(project.test_schema), } ) run_dbt(["deps", "--vars", cli_vars]) results = run_dbt(["run", "--vars", cli_vars]) assert len(results) == 8 check_relations_equal(project.adapter, ["actual", "expected"]) class TestSimpleDependencyDuplicateName(BaseDependencyTest): @pytest.fixture(scope="class", autouse=True) def setUp(self): pass # do not copy local dependency automatically @pytest.fixture(scope="class") def packages(self): return {"packages": [{"local": "duplicate_dependency"}]} @pytest.fixture(scope="class") def prepare_dependencies(self, project): shutil.copytree( project.test_dir / Path("duplicate_dependency"), project.project_root / Path("duplicate_dependency"), ) def test_local_dependency_same_name(self, prepare_dependencies, project): with pytest.raises(dbt.exceptions.DependencyError): run_dbt(["deps"], expect_pass=False) def test_local_dependency_same_name_sneaky(self, prepare_dependencies, project): shutil.copytree("duplicate_dependency", "./dbt_packages/duplicate_dependency") with pytest.raises(dbt_common.exceptions.CompilationError): run_dbt(["compile"]) # needed to avoid compilation errors from duplicate package names in test autocleanup run_dbt(["clean"]) source_with_tests = """ sources: - name: my_source schema: invalid_schema tables: - name: my_table - name: seed_source schema: "{{ var('schema_override', target.schema) }}" tables: - name: "seed" identifier: "seed_subpackage_generate_alias_name" columns: - name: id tests: - unique - not_null """ class TestDependencyTestsConfig(BaseDependencyTest): def test_dependency_tests_config(self, project): run_dbt(["deps"]) # Write a file to local_dependency with a "tests" config write_file( source_with_tests, project.project_root, "local_dependency", "models", "schema.yml" ) run_dbt(["parse"]) # Check that project-test-config is NOT in active deprecations, since "tests" is only # in a dependent project. assert "project-test-config" not in deprecations.active_deprecations ================================================ FILE: tests/functional/dependencies/test_simple_dependency.py ================================================ import os import tempfile from pathlib import Path import pytest from dbt.exceptions import DbtProjectError from dbt.tests.util import check_relations_equal, run_dbt, write_config_file models__disabled_one = """ {{config(enabled=False)}} select 1 """ models__disabled_two = """ {{config(enabled=False)}} select * from {{ref('disabled_one')}} """ models__empty = """ """ models__view_summary = """ {{ config( materialized='view' ) }} with t as ( select * from {{ ref('view_model') }} ) select date_trunc('year', updated_at) as year, count(*) from t group by 1 """ class SimpleDependencyBase(object): @pytest.fixture(scope="class", autouse=True) def setUp(self, project): project.run_sql_file(project.test_data_dir / Path("seed.sql")) @pytest.fixture(scope="class") def models(self): return { "empty.sql": models__empty, "view_summary.sql": models__view_summary, "view_summary.sql": models__view_summary, } @pytest.fixture(scope="class") def packages(self): return { "packages": [ { "git": "https://github.com/dbt-labs/dbt-integration-project", "revision": "1.1", } ] } # These two functions included to enable override in ...NoProfile derived test class @pytest.fixture(scope="class") def run_deps(self, project): return run_dbt(["deps"]) @pytest.fixture(scope="function") def run_clean(self, project): yield # clear test schema assert os.path.exists("target") run_dbt(["clean"]) assert not os.path.exists("target") class TestSimpleDependency(SimpleDependencyBase): def test_simple_dependency(self, run_deps, project, run_clean): """dependencies should draw from a changing base table""" results = run_dbt() assert len(results) == 4 check_relations_equal(project.adapter, ["seed", "table_model"]) check_relations_equal(project.adapter, ["seed", "view_model"]) check_relations_equal(project.adapter, ["seed", "incremental"]) check_relations_equal(project.adapter, ["seed_summary", "view_summary"]) project.run_sql_file(project.test_data_dir / Path("update.sql")) results = run_dbt() assert len(results) == 4 check_relations_equal(project.adapter, ["seed", "table_model"]) check_relations_equal(project.adapter, ["seed", "view_model"]) check_relations_equal(project.adapter, ["seed", "incremental"]) class TestSimpleDependencyWithDependenciesFile(SimpleDependencyBase): @pytest.fixture(scope="class") def packages(self): return {} @pytest.fixture(scope="class") def dependencies(self): return { "packages": [ { "git": "https://github.com/dbt-labs/dbt-integration-project", "warn-unpinned": True, } ] } def test_dependency_with_dependencies_file(self, run_deps, project): # Tests that "packages" defined in a dependencies.yml file works run_dbt(["deps"]) results = run_dbt() assert len(results) == 4 class TestSimpleDependencyWithEmptyPackagesFile(SimpleDependencyBase): @pytest.fixture(scope="class") def packages(self): return " " def test_dependency_with_empty_packages_file(self, run_deps, project): # Tests that an empty packages file doesn't fail with a Python error run_dbt(["deps"]) class TestSimpleDependencyNoProfile(SimpleDependencyBase): """dbt deps and clean commands should not require a profile.""" @pytest.fixture(scope="class") def run_deps(self, project): with tempfile.TemporaryDirectory() as tmpdir: result = run_dbt(["deps", "--profiles-dir", tmpdir]) return result @pytest.fixture(scope="class") def run_clean(self, project): with tempfile.TemporaryDirectory() as tmpdir: result = run_dbt(["clean", "--profiles-dir", tmpdir]) return result def test_simple_dependency_no_profile(self, project, run_deps, run_clean): """only need fixtures as opposed to any model assertions since those are irrelevant and won't occur within the same runtime as a dbt run -s ...""" pass class TestSimpleDependencyWithModels(SimpleDependencyBase): def test_simple_dependency_with_models(self, run_deps, project, run_clean): results = run_dbt(["run", "--models", "view_model+"]) len(results) == 2 check_relations_equal(project.adapter, ["seed", "view_model"]) check_relations_equal(project.adapter, ["seed_summary", "view_summary"]) created_models = project.get_tables_in_schema() assert "table_model" not in created_models assert "incremental" not in created_models assert created_models["view_model"] == "view" assert created_models["view_summary"] == "view" class TestSimpleDependencyUnpinned(object): @pytest.fixture(scope="class", autouse=True) def setUp(self, project): project.run_sql_file(project.test_data_dir / Path("seed.sql")) @pytest.fixture(scope="class") def packages(self): return { "packages": [ { "git": "https://github.com/dbt-labs/dbt-integration-project", "warn-unpinned": True, } ] } def test_simple_dependency(self, project): run_dbt(["deps"]) class TestSimpleDependencyWithDuplicates(object): # dbt should convert these into a single dependency internally @pytest.fixture(scope="class") def packages(self): return { "packages": [ { "git": "https://github.com/dbt-labs/dbt-integration-project", "revision": "dbt/1.0.0", }, { "git": "https://github.com/dbt-labs/dbt-integration-project.git", "revision": "dbt/1.0.0", }, ] } def test_simple_dependency_deps(self, project): run_dbt(["deps"]) class TestSimpleDependencyWithSubdirs(object): # dbt should convert these into a single dependency internally @pytest.fixture(scope="class") def packages(self): return { "packages": [ { "git": "https://github.com/dbt-labs/dbt-multipe-packages.git", "subdirectory": "dbt-utils-main", "revision": "v0.1.0", }, { "git": "https://github.com/dbt-labs/dbt-multipe-packages.git", "subdirectory": "dbt-date-main", "revision": "v0.1.0", }, ] } def test_git_with_multiple_subdir(self, project): run_dbt(["deps"]) assert os.path.exists("package-lock.yml") expected = """packages: - git: https://github.com/dbt-labs/dbt-multipe-packages.git name: dbt_utils revision: 53782f3ede8fdf307ee1d8e418aa65733a4b72fa subdirectory: dbt-utils-main - git: https://github.com/dbt-labs/dbt-multipe-packages.git name: dbt_date revision: 53782f3ede8fdf307ee1d8e418aa65733a4b72fa subdirectory: dbt-date-main sha1_hash: b9c8042f29446c55a33f9f211737f445a640c7a1 """ with open("package-lock.yml") as fp: contents = fp.read() assert contents == expected assert len(os.listdir("dbt_packages")) == 2 class TestRekeyedDependencyWithSubduplicates(object): # this revision of dbt-integration-project requires dbt-utils.git@0.5.0, which the # package config handling should detect @pytest.fixture(scope="class") def packages(self): return { "packages": [ { "git": "https://github.com/dbt-labs/dbt-integration-project", "revision": "config-1.0.0-deps", }, { "git": "https://github.com/dbt-labs/dbt-utils", "revision": "0.5.0", }, ] } def test_simple_dependency_deps(self, project): run_dbt(["deps"]) assert len(os.listdir("dbt_packages")) == 2 class TestTarballNestedDependencies(object): # this version of dbt_expectations has a dependency on dbt_date, which the # package config handling should detect @pytest.fixture(scope="class") def packages(self): return { "packages": [ { "tarball": "https://github.com/calogica/dbt-expectations/archive/refs/tags/0.9.0.tar.gz", "name": "dbt_expectations", }, ] } def test_simple_dependency_deps(self, project): run_dbt(["deps"]) assert set(os.listdir("dbt_packages")) == set(["dbt_expectations", "dbt_date"]) class DependencyBranchBase(object): @pytest.fixture(scope="class", autouse=True) def setUp(self, project): project.run_sql_file(project.test_data_dir / Path("seed.sql")) @pytest.fixture(scope="class") def packages(self): return { "packages": [ { "git": "https://github.com/dbt-labs/dbt-integration-project", "revision": "dbt/1.0.0", }, ] } def deps_run_assert_equality(self, project): run_dbt(["deps"]) results = run_dbt() assert len(results) == 4 check_relations_equal(project.adapter, ["seed", "table_model"]) check_relations_equal(project.adapter, ["seed", "view_model"]) check_relations_equal(project.adapter, ["seed", "incremental"]) created_models = project.get_tables_in_schema() assert created_models["table_model"] == "table" assert created_models["view_model"] == "view" assert created_models["view_summary"] == "view" assert created_models["incremental"] == "table" class TestSimpleDependencyBranch(DependencyBranchBase): @pytest.fixture(scope="class") def models(self): return { "view_summary.sql": models__view_summary, } def test_simple_dependency(self, project): self.deps_run_assert_equality(project) check_relations_equal(project.adapter, ["seed_summary", "view_summary"]) project.run_sql_file(project.test_data_dir / Path("update.sql")) self.deps_run_assert_equality(project) class TestSimpleDependencyBranchWithEmpty(DependencyBranchBase): @pytest.fixture(scope="class") def models(self): """extra models included""" return { "disabled_one.sql": models__disabled_one, "disabled_two.sql": models__disabled_two, "view_summary.sql": models__view_summary, "empty.sql": models__empty, } def test_empty_models_not_compiled_in_dependencies(self, project): self.deps_run_assert_equality(project) models = project.get_tables_in_schema() assert "empty" not in models.keys() class TestSimpleDependencyBadProfile(object): @pytest.fixture(scope="class") def project_config_update(self): return { "config-version": 2, "models": { "+any_config": "{{ target.name }}", "+enabled": "{{ target.name in ['redshift', 'postgres'] | as_bool }}", }, } # Write out the profile data as a yaml file @pytest.fixture(scope="class", autouse=True) def dbt_profile_target(self): # Need to set the environment variable here initially because # the unittest setup does a load_config. os.environ["PROFILE_TEST_HOST"] = "localhost" return { "type": "postgres", "threads": 4, "host": "{{ env_var('PROFILE_TEST_HOST') }}", "port": 5432, "user": "root", "pass": "password", "dbname": "dbt", } def test_deps_bad_profile(self, project): del os.environ["PROFILE_TEST_HOST"] run_dbt(["deps"]) run_dbt(["clean"]) class TestSimpleDependcyTarball(object): @pytest.fixture(scope="class") def packages(self): return { "packages": [ { "tarball": "https://codeload.github.com/dbt-labs/dbt-utils/tar.gz/0.9.6", "name": "dbt_utils", } ] } def test_deps_simple_tarball_doesnt_error_out(self, project): run_dbt(["deps"]) assert len(os.listdir("dbt_packages")) == 1 class TestBadTarballDependency(object): def test_malformed_tarball_package_causes_exception(self, project): # We have to specify the bad formatted package here because if we do it # in a `packages` fixture, the test will blow up in the setup phase, meaning # we can't appropriately catch it with a `pytest.raises` bad_tarball_package_spec = { "packages": [ { "tarball": "https://codeload.github.com/dbt-labs/dbt-utils/tar.gz/0.9.6", "version": "dbt_utils", } ] } write_config_file(bad_tarball_package_spec, "packages.yml") with pytest.raises( DbtProjectError, match=r"The packages.yml file in this project is malformed" ) as e: run_dbt(["deps"]) assert e is not None class TestEmptyDependency: def test_empty_package(self, project): # We have to specify the bad formatted package here because if we do it # in a `packages` fixture, the test will blow up in the setup phase, meaning # we can't appropriately catch it with a `pytest.raises` empty_hub_package = { "packages": [ { "package": "", "version": "1.0.0", } ] } write_config_file(empty_hub_package, "packages.yml") with pytest.raises(DbtProjectError, match="A hub package is missing the value"): run_dbt(["deps"]) empty_git_package = { "packages": [ { "git": "", "revision": "1.0.0", } ] } write_config_file(empty_git_package, "packages.yml") with pytest.raises(DbtProjectError, match="A git package is missing the value"): run_dbt(["deps"]) empty_local_package = { "packages": [ { "local": "", } ] } write_config_file(empty_local_package, "packages.yml") with pytest.raises(DbtProjectError, match="A local package is missing the value"): run_dbt(["deps"]) ================================================ FILE: tests/functional/dependencies/test_simple_dependency_with_configs.py ================================================ from pathlib import Path import pytest from dbt.tests.util import check_relations_equal, run_dbt models__view_summary = """ {{ config( materialized='view' ) }} with t as ( select * from {{ ref('view_model') }} ) select date_trunc('year', updated_at) as year, count(*) from t group by 1 """ class BaseTestSimpleDependencyWithConfigs(object): @pytest.fixture(scope="class", autouse=True) def setUp(self, project): project.run_sql_file(project.test_data_dir / Path("seed.sql")) @pytest.fixture(scope="class") def models(self): return { "view_summary.sql": models__view_summary, } class TestSimpleDependencyWithConfigs(BaseTestSimpleDependencyWithConfigs): @pytest.fixture(scope="class") def packages(self): return { "packages": [ { "git": "https://github.com/dbt-labs/dbt-integration-project", "revision": "with-configs-1.0.0", }, ] } @pytest.fixture(scope="class") def project_config_update(self): return { "config-version": 2, "vars": { "dbt_integration_project": {"bool_config": True}, }, } def test_simple_dependency(self, project): run_dbt(["deps"]) results = run_dbt() assert len(results) == 5 check_relations_equal(project.adapter, ["seed_config_expected_1", "config"]) check_relations_equal(project.adapter, ["seed", "table_model"]) check_relations_equal(project.adapter, ["seed", "view_model"]) check_relations_equal(project.adapter, ["seed", "incremental"]) class TestSimpleDependencyWithOverriddenConfigs(BaseTestSimpleDependencyWithConfigs): @pytest.fixture(scope="class") def packages(self): return { "packages": [ { "git": "https://github.com/dbt-labs/dbt-integration-project", "revision": "with-configs-1.0.0", }, ] } @pytest.fixture(scope="class") def project_config_update(self): return { "config-version": 2, "vars": { # project-level configs "dbt_integration_project": { "config_1": "abc", "config_2": "def", "bool_config": True, }, }, } def test_simple_dependency(self, project): run_dbt(["deps"]) results = run_dbt(["run"]) len(results) == 5 check_relations_equal(project.adapter, ["seed_config_expected_2", "config"]) check_relations_equal(project.adapter, ["seed", "table_model"]) check_relations_equal(project.adapter, ["seed", "view_model"]) check_relations_equal(project.adapter, ["seed", "incremental"]) ================================================ FILE: tests/functional/dependencies/test_uninstalled_package_found_error.py ================================================ import shutil from pathlib import Path import pytest from dbt.exceptions import UninstalledPackagesFoundError from dbt.tests.util import run_dbt class TestUninstalledPackageWithNestedDependency: """When package_a and package_b are specified, package_b has a recursive dependency on package_c, and package_a is uninstalled (missing from dbt_packages), UninstalledPackagesFoundError should be raised. """ @pytest.fixture(scope="class", autouse=True) def setUp(self, project): shutil.copytree( project.test_dir / Path("nested_dependency"), project.project_root / Path("nested_dependency"), ) @pytest.fixture(scope="class") def packages(self): return { "packages": [ {"package": "dbt-labs/dbt_utils", "version": "1.1.1"}, {"local": "nested_dependency"}, ] } def test_uninstalled_package_with_nested_dependency(self, project): run_dbt(["deps"]) # Remove the local package by removing the symlink nested_dep_pkg = Path(project.project_root) / "dbt_packages" / "nested_dependency" nested_dep_pkg.unlink() with pytest.raises(UninstalledPackagesFoundError) as exc_info: run_dbt(["parse"]) assert exc_info.value.count_packages_specified == 3 assert exc_info.value.count_packages_installed == 2 assert "nested_dependency" in exc_info.value.uninstalled_packages class TestUninstalledPackagesErrorRaisedIfPackageLockDoesNotExist: """When package_lock.yml does not exist, error should be raised if packages.yml is non empty.""" @pytest.fixture(scope="class") def packages(self): return { "packages": [ {"package": "dbt-labs/dbt_utils", "version": "1.1.1"}, ] } def test_error_raised_if_package_lock_does_not_exist(self, project): package_lock_path = Path(project.project_root) / "package-lock.yml" # ensure that package lock does not exist package_lock_path.unlink(missing_ok=True) with pytest.raises(UninstalledPackagesFoundError) as exc_info: run_dbt(["parse"]) assert len(exc_info.value.uninstalled_packages) == 0 assert exc_info.value.count_packages_specified == 1 assert exc_info.value.count_packages_installed == 0 class TestNoErrorIfPackageYamlDoesNotExist: """When packages.yml does not exist, no error should be raised.""" def test_no_error_raised_if_package_yml_does_not_exist(self, project): packages_yml_path = Path(project.project_root) / "packages.yml" packages_yml_path.unlink(missing_ok=True) run_dbt(["parse"]) ================================================ FILE: tests/functional/deprecations/fixtures.py ================================================ models__already_exists_sql = """ select 1 as id {% if adapter.already_exists(this.schema, this.identifier) and not should_full_refresh() %} where id > (select max(id) from {{this}}) {% endif %} """ models_trivial__model_sql = """ select 1 as id """ models_custom_key_in_config_sql = """ {{ config(my_custom_key="my_custom_value") }} select 1 as id """ models_custom_key_in_config_non_static_parser_sql = """ {{ config(my_custom_key="my_custom_value") }} select {{ dbt.current_timestamp() }} as my_timestamp """ macros__custom_test_sql = """ {% test custom(model) %} select * from {{ model }} limit 0 {% endtest %} """ models_pre_post_hook_in_config_sql = """ {{ config(post_hook="select 1", pre_hook="select 2") }} select 1 as id """ bad_name_yaml = """ version: 2 exposures: - name: simple exposure spaced!! type: dashboard depends_on: - ref('model') owner: email: something@example.com """ deprecated_model_exposure_yaml = """ version: 2 models: - name: model deprecation_date: 1999-01-01 00:00:00.00+00:00 exposures: - name: simple_exposure type: dashboard depends_on: - ref('model') owner: email: something@example.com """ # deprecated test config fixtures data_tests_yaml = """ models: - name: model columns: - name: id data_tests: - not_null """ test_type_mixed_yaml = """ models: - name: model columns: - name: id data_tests: - not_null tests: - unique """ old_tests_yml = """ models: - name: model tests: - custom columns: - name: id tests: - not_null - name: versioned_model tests: - custom versions: - v: 1 tests: columns: - name: id tests: - not_null """ sources_old_tests_yaml = """ sources: - name: seed_source schema: "{{ var('schema_override', target.schema) }}" tables: - name: "seed" tests: - custom columns: - name: id tests: - unique """ seed_csv = """id,name 1,Mary 2,Sam 3,John """ local_dependency__dbt_project_yml = """ name: 'local_dep' version: '1.0' seeds: quote_columns: False """ local_dependency__schema_yml = """ sources: - name: seed_source schema: "{{ var('schema_override', target.schema) }}" tables: - name: "seed" columns: - name: id tests: - unique """ local_dependency__seed_csv = """id,name 1,Mary 2,Sam 3,John """ invalid_deprecation_date_yaml = """ models: - name: models_trivial description: "This is a test model" deprecation_date: 1 """ duplicate_keys_yaml = """ models: - name: models_trivial description: "This is a test model" deprecation_date: 1999-01-01 00:00:00.00+00:00 models: - name: models_trivial description: "This is a test model" deprecation_date: 1999-01-01 00:00:00.00+00:00 """ custom_key_in_config_yaml = """ models: - name: models_trivial description: "This is a test model" deprecation_date: 1999-01-01 00:00:00.00+00:00 config: my_custom_key: "my_custom_value" """ multiple_custom_keys_in_config_yaml = """ models: - name: models_trivial description: "This is a test model" deprecation_date: 1999-01-01 00:00:00.00+00:00 config: my_custom_key: "my_custom_value" my_custom_key2: "my_custom_value2" """ custom_key_in_object_yaml = """ models: - name: models_trivial description: "This is a test model" deprecation_date: 1999-01-01 00:00:00.00+00:00 my_custom_property: "It's over, I have the high ground" """ pre_post_hook_in_config_yaml = """ models: - name: model_with_hook_configs config: post_hook: "select 1" pre_hook: "select 2" """ property_moved_to_config_yaml = """ models: - name: models_trivial description: "This is a test model" access: public # deprecated - should be in config columns: - name: id tags: ["test"] # deprecated - should be in config sources: - name: seed_source schema: "{{ var('schema_override', target.schema) }}" tags: ["test"] # deprecated - should be in config freshness: # deprecated - should be in config warn_after: count: 1 period: day tables: - name: "seed" tags: ["test"] # deprecated - should be in config freshness: # deprecated - should be in config warn_after: count: 1 period: day columns: - name: id tags: ["test"] # deprecated - should be in config - name: "another_table" """ python_model_py = """ def model(dbt, session): finalized_model = dbt.ref("trivial_model") dbt.config.get("materialized") return finalized_model """ python_model_yml = """ models: - name: python_model config: materialized: table """ test_with_arguments_yaml = """ models: - name: models_trivial tests: - test_name: unique arguments: custom: arg - custom_test: arguments: custom: arg - unique - not_null: where: "1=1" """ test_with_arguments_yaml = """ models: - name: models_trivial tests: - test_name: unique arguments: custom: arg - custom_test: arguments: custom: arg - unique - not_null: where: "1=1" columns: - name: column tests: - test_name: unique arguments: custom: arg - custom_test: arguments: custom: arg - custom_test2_valid: column_name: id config: where: "1=1" custom: arg """ test_missing_arguments_property_yaml = """ models: - name: models_trivial tests: - test_name: unique custom: arg - custom_test: custom: arg - custom_test2_valid: column_name: id config: where: "1=1" arguments: custom: arg columns: - name: column tests: - test_name: unique custom: arg - custom_test: custom: arg - custom_test2_valid: column_name: id config: where: "1=1" arguments: custom: arg """ generate_schema_name_null_return_macro_sql = """ {% macro generate_schema_name(custom_schema_name, node) %} {{ return(custom_schema_name) }} {% endmacro %} """ ================================================ FILE: tests/functional/deprecations/test_config_deprecations.py ================================================ import os from collections import defaultdict from typing import Dict import pytest from pytest_mock import MockerFixture from dbt import deprecations from dbt.exceptions import CompilationError, ProjectContractError, YamlParseDictError from dbt.tests.fixtures.project import write_project_files from dbt.tests.util import run_dbt, update_config_file from tests.functional.deprecations.fixtures import ( data_tests_yaml, local_dependency__dbt_project_yml, local_dependency__schema_yml, local_dependency__seed_csv, macros__custom_test_sql, models_trivial__model_sql, old_tests_yml, seed_csv, sources_old_tests_yaml, test_type_mixed_yaml, ) # TODO: This is now defined 4 times throughout the test suite. We should move it to a utils file. def normalize(path): """On windows, neither is enough on its own: >>> normcase('C:\\documents/ALL CAPS/subdir\\..') 'c:\\documents\\all caps\\subdir\\..' >>> normpath('C:\\documents/ALL CAPS/subdir\\..') 'C:\\documents\\ALL CAPS' >>> normpath(normcase('C:\\documents/ALL CAPS/subdir\\..')) 'c:\\documents\\all caps' """ return os.path.normcase(os.path.normpath(path)) # test deprecation messages class TestTestsConfigDeprecation: @pytest.fixture(scope="class") def models(self): return {"model.sql": models_trivial__model_sql} @pytest.fixture(scope="class") def project_config_update(self, unique_schema): return {"tests": {"enabled": "true"}} def test_project_tests_config(self, project): deprecations.reset_deprecations() assert deprecations.active_deprecations == defaultdict(int) run_dbt(["parse"]) assert deprecations.active_deprecations == defaultdict(int) def test_project_tests_config_fail(self, project): deprecations.reset_deprecations() assert deprecations.active_deprecations == defaultdict(int) with pytest.raises(CompilationError) as exc: run_dbt(["--warn-error", "--no-partial-parse", "parse"]) exc_str = " ".join(str(exc.value).split()) # flatten all whitespace expected_msg = "Configuration paths exist in your dbt_project.yml file which do not apply to any resources. There are 1 unused configuration paths: - data_tests" assert expected_msg in exc_str class TestSchemaTestDeprecation: @pytest.fixture(scope="class") def models(self): return { "model.sql": models_trivial__model_sql, "versioned_model.sql": models_trivial__model_sql, "schema.yml": old_tests_yml, } @pytest.fixture(scope="class") def macros(self): return {"custom.sql": macros__custom_test_sql} def test_generic_tests_config(self, project): deprecations.reset_deprecations() assert deprecations.active_deprecations == defaultdict(int) run_dbt(["parse"]) assert deprecations.active_deprecations == defaultdict(int) def test_generic_tests_fail(self, project): deprecations.reset_deprecations() assert deprecations.active_deprecations == defaultdict(int) run_dbt(["--warn-error", "--no-partial-parse", "parse"]) def test_generic_data_test_parsing(self, project): results = run_dbt(["list", "--resource-type", "test"]) assert len(results) == 4 class TestSourceSchemaTestDeprecation: @pytest.fixture(scope="class") def models(self): return {"schema.yml": sources_old_tests_yaml} @pytest.fixture(scope="class") def macros(self): return {"custom.sql": macros__custom_test_sql} @pytest.fixture(scope="class") def seeds(self): return { "seed.csv": seed_csv, } def test_source_tests_config(self, project): deprecations.reset_deprecations() assert deprecations.active_deprecations == defaultdict(int) run_dbt(["parse"]) assert deprecations.active_deprecations == defaultdict(int) def test_generic_data_tests(self, project): run_dbt(["seed"]) results = run_dbt(["test"]) assert len(results) == 2 # test for failure with test and data_tests in the same file class TestBothSchemaTestDeprecation: @pytest.fixture(scope="class") def models(self): return {"model.sql": models_trivial__model_sql, "schema.yml": test_type_mixed_yaml} def test_schema(self, project): expected_msg = "Invalid test config: cannot have both 'tests' and 'data_tests' defined" with pytest.raises(YamlParseDictError) as excinfo: run_dbt(["parse"]) assert expected_msg in str(excinfo.value) # test for failure with test and data_tests in the same dbt_project.yml class TestBothProjectTestDeprecation: @pytest.fixture(scope="class") def models(self): return {"model.sql": models_trivial__model_sql} def test_tests_config(self, project): config_patch = {"tests": {"+enabled": "true"}, "data_tests": {"+tags": "super"}} update_config_file(config_patch, project.project_root, "dbt_project.yml") expected_msg = "Invalid project config: cannot have both 'tests' and 'data_tests' defined" with pytest.raises(ProjectContractError) as excinfo: run_dbt(["parse"]) assert expected_msg in str(excinfo.value) # test a local dependency can have tests while the rest of the project uses data_tests class TestTestConfigInDependency: @pytest.fixture(scope="class", autouse=True) def setUp(self, project_root): local_dependency_files = { "dbt_project.yml": local_dependency__dbt_project_yml, "models": { "schema.yml": local_dependency__schema_yml, }, "seeds": {"seed.csv": local_dependency__seed_csv}, } write_project_files(project_root, "local_dependency", local_dependency_files) @pytest.fixture(scope="class") def packages(self): return {"packages": [{"local": "local_dependency"}]} @pytest.fixture(scope="class") def models(self): return { "model.sql": models_trivial__model_sql, "schema.yml": data_tests_yaml, } def test_test_dep(self, project): run_dbt(["deps"]) run_dbt(["seed"]) run_dbt(["run"]) results = run_dbt(["test"]) # 1 data_test in the dep and 1 in the project assert len(results) == 2 class TestValidateModelConfigOnlyCalledOncePerModel: @pytest.fixture(scope="class") def models(self): return { "model.sql": models_trivial__model_sql, "versioned_model.sql": models_trivial__model_sql, "schema.yml": old_tests_yml, } @pytest.fixture(scope="class") def macros(self): return {"custom.sql": macros__custom_test_sql} def test_validate_model_config_only_called_once_per_model( self, project, models: Dict[str, str], mocker: MockerFixture ): mocked_vmc = mocker.patch("dbt.parser.base.validate_model_config") run_dbt( ["parse", "--no-partial-parse"], ) # this is a list of all the files that validate_model_config was called on called_on_files = [call.args[1] for call in mocked_vmc.call_args_list] # this entire test is only useful if we have validated more than one file assert len(called_on_files) > 1 # Each model sql file should have been validated once assert len(called_on_files) == len(set(called_on_files)) # Only model sql files should have been validated sql_model_files = [file for file in models.keys() if file.endswith(".sql")] assert len(called_on_files) == len(sql_model_files) for file in sql_model_files: assert normalize(f"models/{file}") in called_on_files ================================================ FILE: tests/functional/deprecations/test_deprecations.py ================================================ import os import sys from collections import defaultdict from unittest import mock import pytest import yaml from pytest_mock import MockerFixture import dbt_common from dbt import deprecations from dbt.cli.main import dbtRunner from dbt.clients.registry import _get_cached from dbt.events.types import ( ArgumentsPropertyInGenericTestDeprecation, CustomKeyInConfigDeprecation, CustomKeyInObjectDeprecation, CustomOutputPathInSourceFreshnessDeprecation, DeprecationsSummary, DuplicateYAMLKeysDeprecation, EnvironmentVariableNamespaceDeprecation, GenerateSchemaNameNullValueDeprecation, GenericJSONSchemaValidationDeprecation, MissingArgumentsPropertyInGenericTestDeprecation, ModelParamUsageDeprecation, ModulesItertoolsUsageDeprecation, PackageRedirectDeprecation, PropertyMovedToConfigDeprecation, WEOIncludeExcludeDeprecation, ) from dbt.tests.util import read_file, run_dbt, run_dbt_and_capture, write_file from dbt_common.events.event_catcher import EventCatcher from dbt_common.events.types import Note from dbt_common.exceptions import EventCompilationError from tests.functional.deprecations.fixtures import ( bad_name_yaml, custom_key_in_config_yaml, custom_key_in_object_yaml, deprecated_model_exposure_yaml, duplicate_keys_yaml, generate_schema_name_null_return_macro_sql, invalid_deprecation_date_yaml, models_custom_key_in_config_non_static_parser_sql, models_custom_key_in_config_sql, models_pre_post_hook_in_config_sql, models_trivial__model_sql, multiple_custom_keys_in_config_yaml, pre_post_hook_in_config_yaml, property_moved_to_config_yaml, python_model_py, python_model_yml, test_missing_arguments_property_yaml, test_with_arguments_yaml, ) class TestConfigPathDeprecation: @pytest.fixture(scope="class") def models(self): return {"already_exists.sql": models_trivial__model_sql} @pytest.fixture(scope="class") def project_config_update(self): return { "config-version": 2, "data-paths": ["data"], "log-path": "customlogs", "target-path": "customtarget", } def test_data_path(self, project): deprecations.reset_deprecations() assert deprecations.active_deprecations == defaultdict(int) run_dbt(["debug"]) expected = { "project-config-data-paths", "project-config-log-path", "project-config-target-path", } for deprecation in expected: assert deprecation in deprecations.active_deprecations def test_data_path_fail(self, project): deprecations.reset_deprecations() assert deprecations.active_deprecations == defaultdict(int) with pytest.raises(dbt_common.exceptions.CompilationError) as exc: run_dbt(["--warn-error", "debug"]) exc_str = " ".join(str(exc.value).split()) # flatten all whitespace expected_msg = "The `data-paths` config has been renamed" assert expected_msg in exc_str class TestPackageInstallPathDeprecation: @pytest.fixture(scope="class") def models_trivial(self): return {"model.sql": models_trivial__model_sql} @pytest.fixture(scope="class") def project_config_update(self): return {"config-version": 2, "clean-targets": ["dbt_modules"]} def test_package_path(self, project): deprecations.reset_deprecations() assert deprecations.active_deprecations == defaultdict(int) run_dbt(["clean"]) assert "install-packages-path" in deprecations.active_deprecations def test_package_path_not_set(self, project): deprecations.reset_deprecations() assert deprecations.active_deprecations == defaultdict(int) with pytest.raises(dbt_common.exceptions.CompilationError) as exc: run_dbt(["--warn-error", "clean"]) exc_str = " ".join(str(exc.value).split()) # flatten all whitespace expected_msg = "path has changed from `dbt_modules` to `dbt_packages`." assert expected_msg in exc_str class TestPackageRedirectDeprecation: @pytest.fixture(scope="class") def models(self): return {"already_exists.sql": models_trivial__model_sql} @pytest.fixture(scope="class") def packages(self): return {"packages": [{"package": "fishtown-analytics/dbt_utils", "version": "0.7.0"}]} def test_package_redirect(self, project): deprecations.reset_deprecations() assert deprecations.active_deprecations == defaultdict(int) run_dbt(["deps"]) assert "package-redirect" in deprecations.active_deprecations # if this test comes before test_package_redirect it will raise an exception as expected def test_package_redirect_fail(self, project): deprecations.reset_deprecations() assert deprecations.active_deprecations == defaultdict(int) with pytest.raises(dbt_common.exceptions.CompilationError) as exc: run_dbt(["--warn-error", "deps"]) exc_str = " ".join(str(exc.value).split()) # flatten all whitespace expected_msg = "The `fishtown-analytics/dbt_utils` package is deprecated in favor of `dbt-labs/dbt_utils`" assert expected_msg in exc_str class TestDeprecatedModelExposure: @pytest.fixture(scope="class") def models(self): return { "model.sql": models_trivial__model_sql, "exposure.yml": deprecated_model_exposure_yaml, } def test_exposure_with_deprecated_model(self, project): run_dbt(["parse"]) class TestExposureNameDeprecation: @pytest.fixture(scope="class") def models(self): return {"model.sql": models_trivial__model_sql, "bad_name.yml": bad_name_yaml} def test_exposure_name(self, project): deprecations.reset_deprecations() assert deprecations.active_deprecations == defaultdict(int) run_dbt(["parse"]) assert "exposure-name" in deprecations.active_deprecations def test_exposure_name_fail(self, project): deprecations.reset_deprecations() assert deprecations.active_deprecations == defaultdict(int) with pytest.raises(dbt_common.exceptions.CompilationError) as exc: run_dbt(["--warn-error", "--no-partial-parse", "parse"]) exc_str = " ".join(str(exc.value).split()) # flatten all whitespace expected_msg = "Starting in v1.3, the 'name' of an exposure should contain only letters, numbers, and underscores." assert expected_msg in exc_str class TestProjectFlagsMovedDeprecation: @pytest.fixture(scope="class") def profiles_config_update(self): return { "config": {"send_anonymous_usage_stats": False}, } @pytest.fixture(scope="class") def dbt_project_yml(self, project_root, project_config_update): project_config = { "name": "test", "profile": "test", } write_file(yaml.safe_dump(project_config), project_root, "dbt_project.yml") return project_config @pytest.fixture(scope="class") def models(self): return {"my_model.sql": "select 1 as fun"} def test_profile_config_deprecation(self, project): deprecations.reset_deprecations() assert deprecations.active_deprecations == defaultdict(int) _, logs = run_dbt_and_capture(["parse"]) assert ( "User config should be moved from the 'config' key in profiles.yml to the 'flags' key in dbt_project.yml." in logs ) assert "project-flags-moved" in deprecations.active_deprecations class TestProjectFlagsMovedDeprecationQuiet(TestProjectFlagsMovedDeprecation): def test_profile_config_deprecation(self, project): deprecations.reset_deprecations() assert deprecations.active_deprecations == defaultdict(int) _, logs = run_dbt_and_capture(["--quiet", "parse"]) assert ( "User config should be moved from the 'config' key in profiles.yml to the 'flags' key in dbt_project.yml." not in logs ) assert "project-flags-moved" in deprecations.active_deprecations class TestProjectFlagsMovedDeprecationWarnErrorOptions(TestProjectFlagsMovedDeprecation): def test_profile_config_deprecation(self, project): deprecations.reset_deprecations() with pytest.raises(EventCompilationError): run_dbt(["--warn-error-options", "{'error': 'all'}", "parse"]) with pytest.raises(EventCompilationError): run_dbt( ["--warn-error-options", "{'error': ['ProjectFlagsMovedDeprecation']}", "parse"] ) _, logs = run_dbt_and_capture( ["--warn-error-options", "{'silence': ['ProjectFlagsMovedDeprecation']}", "parse"] ) assert ( "User config should be moved from the 'config' key in profiles.yml to the 'flags' key in dbt_project.yml." not in logs ) class TestShowAllDeprecationsFlag: @pytest.fixture(scope="class") def models(self): return {"already_exists.sql": models_trivial__model_sql} @pytest.fixture(scope="class") def packages(self): return { "packages": [ {"package": "fishtown-analytics/dbt_utils", "version": "0.7.0"}, {"package": "calogica/dbt_date", "version": "0.10.0"}, ] } @pytest.fixture(scope="class") def event_catcher(self) -> EventCatcher: return EventCatcher(event_to_catch=PackageRedirectDeprecation) def test_package_redirect(self, project, event_catcher: EventCatcher): deprecations.reset_deprecations() assert deprecations.active_deprecations == defaultdict(int) run_dbt(["deps"], callbacks=[event_catcher.catch]) assert "package-redirect" in deprecations.active_deprecations assert deprecations.active_deprecations["package-redirect"] == 2 assert len(event_catcher.caught_events) == 1 deprecations.reset_deprecations() _get_cached.cache = {} event_catcher.flush() run_dbt(["deps", "--show-all-deprecations"], callbacks=[event_catcher.catch]) assert "package-redirect" in deprecations.active_deprecations assert deprecations.active_deprecations["package-redirect"] == 2 assert len(event_catcher.caught_events) == 2 class TestDeprecationSummary: @pytest.fixture(scope="class") def models(self): return {"already_exists.sql": models_trivial__model_sql} @pytest.fixture(scope="class") def packages(self): return { "packages": [ {"package": "fishtown-analytics/dbt_utils", "version": "0.7.0"}, {"package": "calogica/dbt_date", "version": "0.10.0"}, ] } @pytest.fixture(scope="class") def event_catcher(self) -> EventCatcher: return EventCatcher(event_to_catch=DeprecationsSummary) def test_package_redirect(self, project, event_catcher: EventCatcher): deprecations.reset_deprecations() assert deprecations.active_deprecations == defaultdict(int) run_dbt(["deps"], callbacks=[event_catcher.catch]) assert "package-redirect" in deprecations.active_deprecations assert deprecations.active_deprecations["package-redirect"] == 2 assert len(event_catcher.caught_events) == 1 for summary in event_catcher.caught_events[0].data.summaries: # type: ignore found_summary = False if summary.event_name == "PackageRedirectDeprecation": assert ( summary.occurrences == 2 ), f"Expected 2 occurrences of PackageRedirectDeprecation, got {summary.occurrences}" found_summary = True assert found_summary, "Expected to find PackageRedirectDeprecation in deprecations summary" @mock.patch("dbt.jsonschemas.jsonschemas._JSONSCHEMA_SUPPORTED_ADAPTERS", {"postgres"}) class TestDeprecatedInvalidDeprecationDate: @pytest.fixture(scope="class") def models(self): return { "models_trivial.sql": models_trivial__model_sql, "models.yml": invalid_deprecation_date_yaml, } def test_deprecated_invalid_deprecation_date(self, project): event_catcher = EventCatcher(GenericJSONSchemaValidationDeprecation) note_catcher = EventCatcher(Note) try: run_dbt( ["parse", "--no-partial-parse"], callbacks=[event_catcher.catch, note_catcher.catch], ) except: # noqa assert ( True ), "Expected an exception to be raised, because a model object can't be created with a deprecation_date as an int" # type-based jsonschema validation is not enabled, so no deprecations are raised even though deprecation_date is an int assert len(event_catcher.caught_events) == 0 assert len(note_catcher.caught_events) == 0 class TestDuplicateYAMLKeysInSchemaFiles: @pytest.fixture(scope="class") def models(self): return { "models_trivial.sql": models_trivial__model_sql, "models.yml": duplicate_keys_yaml, } def test_duplicate_yaml_keys_in_schema_files(self, project): event_catcher = EventCatcher(DuplicateYAMLKeysDeprecation) run_dbt(["parse", "--no-partial-parse"], callbacks=[event_catcher.catch]) assert len(event_catcher.caught_events) == 1 assert ( "Duplicate key 'models' in \"\", line 6, column 1 in file" in event_catcher.caught_events[0].info.msg ) class TestCustomKeyInConfigDeprecation: @pytest.fixture(scope="class") def models(self): return { "models_trivial.sql": models_trivial__model_sql, "models.yml": custom_key_in_config_yaml, } @mock.patch("dbt.jsonschemas.jsonschemas._JSONSCHEMA_SUPPORTED_ADAPTERS", {"postgres"}) def test_custom_key_in_config_deprecation(self, project): event_catcher = EventCatcher(CustomKeyInConfigDeprecation) run_dbt( ["parse", "--no-partial-parse", "--show-all-deprecations"], callbacks=[event_catcher.catch], ) assert len(event_catcher.caught_events) == 1 assert ( "Custom key `my_custom_key` found in `config` at path `models[0].config`" in event_catcher.caught_events[0].info.msg ) class TestCustomKeyInConfigSQLDeprecation: @pytest.fixture(scope="class") def models(self): return { "model_custom_key_in_config.sql": models_custom_key_in_config_sql, } @mock.patch("dbt.jsonschemas.jsonschemas._JSONSCHEMA_SUPPORTED_ADAPTERS", {"postgres"}) def test_custom_key_in_config_sql_deprecation(self, project): event_catcher = EventCatcher(CustomKeyInConfigDeprecation) run_dbt( ["parse", "--no-partial-parse", "--show-all-deprecations"], callbacks=[event_catcher.catch], ) assert len(event_catcher.caught_events) == 1 assert ( "Custom key `my_custom_key` found in `config`" in event_catcher.caught_events[0].info.msg ) @mock.patch("dbt.jsonschemas.jsonschemas._JSONSCHEMA_SUPPORTED_ADAPTERS", {"postgres"}) @mock.patch( "dbt.jsonschemas.jsonschemas._get_allowed_config_key_aliases", return_value=["my_custom_key"], ) def test_custom_key_in_config_sql_deprecation_adapter_specific_config_key_aliases( self, mock_get_aliases, project ): event_catcher = EventCatcher(CustomKeyInConfigDeprecation) run_dbt( ["parse", "--no-partial-parse", "--show-all-deprecations"], callbacks=[event_catcher.catch], ) assert len(event_catcher.caught_events) == 0 class TestCustomKeyInConfigComplexSQLDeprecation(TestCustomKeyInConfigSQLDeprecation): @pytest.fixture(scope="class") def models(self): return { "model_custom_key_in_config.sql": models_custom_key_in_config_non_static_parser_sql, } class TestMultipleCustomKeysInConfigDeprecation: @pytest.fixture(scope="class") def models(self): return { "models_trivial.sql": models_trivial__model_sql, "models.yml": multiple_custom_keys_in_config_yaml, } @mock.patch("dbt.jsonschemas.jsonschemas._JSONSCHEMA_SUPPORTED_ADAPTERS", {"postgres"}) def test_multiple_custom_keys_in_config_deprecation(self, project): event_catcher = EventCatcher(CustomKeyInConfigDeprecation) run_dbt( ["parse", "--no-partial-parse", "--show-all-deprecations"], callbacks=[event_catcher.catch], ) assert len(event_catcher.caught_events) == 2 assert ( "Custom key `my_custom_key` found in `config` at path `models[0].config`" in event_catcher.caught_events[0].info.msg ) assert ( "Custom key `my_custom_key2` found in `config` at path `models[0].config`" in event_catcher.caught_events[1].info.msg ) class TestCustomKeyInObjectDeprecation: @pytest.fixture(scope="class") def models(self): return { "models_trivial.sql": models_trivial__model_sql, "models.yml": custom_key_in_object_yaml, } @mock.patch("dbt.jsonschemas.jsonschemas._JSONSCHEMA_SUPPORTED_ADAPTERS", {"postgres"}) def test_custom_key_in_object_deprecation(self, project): event_catcher = EventCatcher(CustomKeyInObjectDeprecation) run_dbt(["parse", "--no-partial-parse"], callbacks=[event_catcher.catch]) assert len(event_catcher.caught_events) == 1 assert ( "Custom key `my_custom_property` found at `models[0]` in file" in event_catcher.caught_events[0].info.msg ) class TestJsonschemaValidationDeprecationsArentRunWithoutEnvVar: @pytest.fixture(scope="class") def models(self): return { "models_trivial.sql": models_trivial__model_sql, "models.yml": custom_key_in_object_yaml, } def test_jsonschema_validation_deprecations_arent_run_without_env_var(self, project): event_catcher = EventCatcher(CustomKeyInObjectDeprecation) run_dbt(["parse", "--no-partial-parse"], callbacks=[event_catcher.catch]) assert len(event_catcher.caught_events) == 0 class TestCustomOutputPathInSourceFreshnessDeprecation: @pytest.fixture(scope="class") def models(self): return {} def test_jsonschema_validation_deprecations_arent_run_without_env_var( self, project, project_root ): event_catcher = EventCatcher(CustomOutputPathInSourceFreshnessDeprecation) write_file(yaml.safe_dump({}), project_root, "custom_output.json") run_dbt( ["source", "freshness", "--output", "custom_output.json"], callbacks=[event_catcher.catch], ) assert len(event_catcher.caught_events) == 1 class TestHappyPathProjectHasNoDeprecations: @mock.patch("dbt.jsonschemas.jsonschemas._JSONSCHEMA_SUPPORTED_ADAPTERS", {"postgres"}) def test_happy_path_project_has_no_deprecations(self, happy_path_project): event_cathcer = EventCatcher(DeprecationsSummary) run_dbt( ["parse", "--no-partial-parse", "--show-all-deprecations"], callbacks=[event_cathcer.catch], ) assert len(event_cathcer.caught_events) == 0 class TestBaseProjectHasNoDeprecations: @mock.patch("dbt.jsonschemas.jsonschemas._JSONSCHEMA_SUPPORTED_ADAPTERS", {"postgres"}) def test_base_project_has_no_deprecations(self, project): event_cathcer = EventCatcher(DeprecationsSummary) run_dbt( ["parse", "--no-partial-parse", "--show-all-deprecations"], callbacks=[event_cathcer.catch], ) assert len(event_cathcer.caught_events) == 0 class TestWEOIncludeExcludeDeprecation: @pytest.mark.parametrize( "include_error,exclude_warn,expect_deprecation", [ ("include", "exclude", 1), ("include", "warn", 1), ("error", "exclude", 1), ("error", "warn", 0), ], ) def test_weo_include_exclude_deprecation( self, project, include_error: str, exclude_warn: str, expect_deprecation: int, ): event_catcher = EventCatcher(WEOIncludeExcludeDeprecation) warn_error_options = f"{{'{include_error}': 'all', '{exclude_warn}': ['Deprecations']}}" run_dbt( ["parse", "--show-all-deprecations", "--warn-error-options", warn_error_options], callbacks=[event_catcher.catch], ) assert len(event_catcher.caught_events) == expect_deprecation if expect_deprecation > 0: if include_error == "include": assert "include" in event_catcher.caught_events[0].info.msg else: assert "include" not in event_catcher.caught_events[0].info.msg if exclude_warn == "exclude": assert "exclude" in event_catcher.caught_events[0].info.msg else: assert "exclude" not in event_catcher.caught_events[0].info.msg class TestModulesItertoolsDeprecation: @pytest.fixture(scope="class") def models(self): return { "models_itertools.sql": """ {%- set A = [1] -%} {%- set B = ['x'] -%} {%- set AB_cartesian = modules.itertools.product(A, B) -%} {%- for item in AB_cartesian %} select {{ item[0] }} {%- endfor -%} """, } def test_models_itertools(self, project): event_catcher = EventCatcher(ModulesItertoolsUsageDeprecation) run_dbt(["run", "--no-partial-parse"], callbacks=[event_catcher.catch]) assert len(event_catcher.caught_events) == 1 assert ( "Usage of itertools modules is deprecated" in event_catcher.caught_events[0].info.msg ) assert ( read_file("target/compiled/test/models/models_itertools.sql").strip() == "select 1".strip() ) class TestNoModulesItertoolsDeprecation: @pytest.fixture(scope="class") def models(self): return { "models_itertools.sql": "select {{ modules.datetime.datetime.now() }}", } def test_models_itertools(self, project): event_catcher = EventCatcher(ModulesItertoolsUsageDeprecation) run_dbt(["parse", "--no-partial-parse"], callbacks=[event_catcher.catch]) assert len(event_catcher.caught_events) == 0 class TestModelsParamUsageDeprecation: @mock.patch.object(sys, "argv", ["dbt", "ls", "--models", "some_model"]) def test_models_usage(self, project): event_catcher = EventCatcher(ModelParamUsageDeprecation) assert len(event_catcher.caught_events) == 0 run_dbt( ["ls", "--models", "some_model"], callbacks=[event_catcher.catch], ) assert len(event_catcher.caught_events) == 1 class TestModelsParamUsageRunnerDeprecation: def test_models_usage(self, project): event_catcher = EventCatcher(ModelParamUsageDeprecation) assert len(event_catcher.caught_events) == 0 dbtRunner(callbacks=[event_catcher.catch]).invoke(["ls", "--models", "some_model"]) assert len(event_catcher.caught_events) == 1 class TestModelParamUsageDeprecation: @mock.patch.object(sys, "argv", ["dbt", "ls", "--model", "some_model"]) def test_model_usage(self, project): event_catcher = EventCatcher(ModelParamUsageDeprecation) assert len(event_catcher.caught_events) == 0 run_dbt( ["ls", "--model", "some_model"], callbacks=[event_catcher.catch], ) assert len(event_catcher.caught_events) == 1 class TestModelParamUsageRunnerDeprecation: def test_model_usage(self, project): event_catcher = EventCatcher(ModelParamUsageDeprecation) assert len(event_catcher.caught_events) == 0 dbtRunner(callbacks=[event_catcher.catch]).invoke(["ls", "--model", "some_model"]) assert len(event_catcher.caught_events) == 1 class TestMParamUsageDeprecation: @mock.patch.object(sys, "argv", ["dbt", "ls", "-m", "some_model"]) def test_m_usage(self, project): event_catcher = EventCatcher(ModelParamUsageDeprecation) assert len(event_catcher.caught_events) == 0 run_dbt( ["ls", "-m", "some_model"], callbacks=[event_catcher.catch], ) assert len(event_catcher.caught_events) == 1 class TestMParamUsageRunnerDeprecation: def test_m_usage(self, project): event_catcher = EventCatcher(ModelParamUsageDeprecation) assert len(event_catcher.caught_events) == 0 dbtRunner(callbacks=[event_catcher.catch]).invoke(["ls", "-m", "some_model"]) assert len(event_catcher.caught_events) == 1 class TestSelectParamNoModelUsageDeprecation: @mock.patch.object(sys, "argv", ["dbt", "ls", "--select", "some_model"]) def test_select_usage(self, project): event_catcher = EventCatcher(ModelParamUsageDeprecation) assert len(event_catcher.caught_events) == 0 run_dbt( ["ls", "--select", "some_model"], callbacks=[event_catcher.catch], ) assert len(event_catcher.caught_events) == 0 class TestSelectParamNoModelUsageRunnerDeprecation: def test_select_usage(self, project): event_catcher = EventCatcher(ModelParamUsageDeprecation) assert len(event_catcher.caught_events) == 0 dbtRunner(callbacks=[event_catcher.catch]).invoke(["ls", "--select", "some_model"]) assert len(event_catcher.caught_events) == 0 class TestEnvironmentVariableNamespaceDeprecation: @mock.patch.dict( os.environ, { "DBT_ENGINE_PARTIAL_PARSE": "False", "DBT_ENGINE_MY_CUSTOM_ENV_VAR_FOR_TESTING": "True", }, ) def test_environment_variable_namespace_deprecation(self, project): event_catcher = EventCatcher(event_to_catch=EnvironmentVariableNamespaceDeprecation) run_dbt(["parse", "--show-all-deprecations"], callbacks=[event_catcher.catch]) assert len(event_catcher.caught_events) == 1 assert ( "DBT_ENGINE_MY_CUSTOM_ENV_VAR_FOR_TESTING" == event_catcher.caught_events[0].data.env_var ) class TestCustomConfigInDbtProjectYmlNoDeprecation: @pytest.fixture(scope="class") def project_config_update(self): return {"seeds": {"path": {"+custom_config": True}}} @mock.patch("dbt.jsonschemas.jsonschemas._JSONSCHEMA_SUPPORTED_ADAPTERS", {"postgres"}) def test_missing_plus_prefix_deprecation_sub_path(self, project): note_catcher = EventCatcher(Note) run_dbt(["parse", "--no-partial-parse"], callbacks=[note_catcher.catch]) assert len(note_catcher.caught_events) == 0 class TestJsonSchemaValidationGating: @pytest.fixture(scope="class") def models(self): return { "models_trivial.sql": models_trivial__model_sql, "models.yml": custom_key_in_config_yaml, } @pytest.mark.parametrize( "postgres_is_valid,dbt_private_run_jsonschema_validations,expected_events", [ (True, "True", 1), (False, "True", 0), (False, "False", 0), (False, "False", 0), ], ) def test_jsonschema_validation_gating( self, project, mocker: MockerFixture, postgres_is_valid: bool, dbt_private_run_jsonschema_validations: bool, expected_events: int, ) -> None: if postgres_is_valid: supported_adapters_with_postgres = { "postgres", "bigquery", "databricks", "redshift", "snowflake", } mocker.patch( "dbt.jsonschemas.jsonschemas._JSONSCHEMA_SUPPORTED_ADAPTERS", supported_adapters_with_postgres, ) event_catcher = EventCatcher(CustomKeyInConfigDeprecation) run_dbt( ["parse", "--no-partial-parse", "--show-all-deprecations"], callbacks=[event_catcher.catch], ) assert len(event_catcher.caught_events) == expected_events class TestArgumentsPropertyInGenericTestDeprecationFalse: @pytest.fixture(scope="class") def project_config_update(self): return { "config-version": 2, "flags": { "require_generic_test_arguments_property": False, }, } @pytest.fixture(scope="class") def models(self): return { "models_trivial.sql": models_trivial__model_sql, "models.yml": test_with_arguments_yaml, } def test_arguments_property_in_generic_test_deprecation(self, project): event_catcher = EventCatcher(ArgumentsPropertyInGenericTestDeprecation) run_dbt( ["parse", "--no-partial-parse", "--show-all-deprecations"], callbacks=[event_catcher.catch], ) assert len(event_catcher.caught_events) == 4 class TestArgumentsPropertyInGenericTestDeprecationBehaviorChangeDefault: @pytest.fixture(scope="class") def models(self): return { "models_trivial.sql": models_trivial__model_sql, "models.yml": test_with_arguments_yaml, } def test_arguments_property_in_generic_test_deprecation(self, project): event_catcher = EventCatcher(ArgumentsPropertyInGenericTestDeprecation) run_dbt( ["parse", "--no-partial-parse", "--show-all-deprecations"], callbacks=[event_catcher.catch], ) assert len(event_catcher.caught_events) == 0 class TestArgumentsPropertyInGenericTestDeprecationTrue: @pytest.fixture(scope="class") def project_config_update(self): return { "config-version": 2, "flags": { "require_generic_test_arguments_property": True, }, } @pytest.fixture(scope="class") def models(self): return { "models_trivial.sql": models_trivial__model_sql, "models.yml": test_with_arguments_yaml, } def test_arguments_property_in_generic_test_deprecation(self, project): event_catcher = EventCatcher(ArgumentsPropertyInGenericTestDeprecation) run_dbt( ["parse", "--no-partial-parse", "--show-all-deprecations"], callbacks=[event_catcher.catch], ) assert len(event_catcher.caught_events) == 0 class TestMissingArgumentsPropertyInGenericTestDeprecation: @pytest.fixture(scope="class") def project_config_update(self): return { "config-version": 2, "flags": { "require_generic_test_arguments_property": True, }, } @pytest.fixture(scope="class") def models(self): return { "models_trivial.sql": models_trivial__model_sql, "models.yml": test_missing_arguments_property_yaml, } def test_missing_arguments_property_in_generic_test_deprecation(self, project): event_catcher = EventCatcher(MissingArgumentsPropertyInGenericTestDeprecation) run_dbt( ["parse", "--no-partial-parse", "--show-all-deprecations"], callbacks=[event_catcher.catch], ) assert len(event_catcher.caught_events) == 4 class TestPropertyMovedToConfigDeprecation: @pytest.fixture(scope="class") def models(self): return { "models_trivial.sql": models_trivial__model_sql, "models.yml": property_moved_to_config_yaml, } @mock.patch("dbt.jsonschemas.jsonschemas._JSONSCHEMA_SUPPORTED_ADAPTERS", {"postgres"}) def test_property_moved_to_config_deprecation(self, project): event_catcher = EventCatcher(PropertyMovedToConfigDeprecation) run_dbt( ["parse", "--no-partial-parse", "--show-all-deprecations"], callbacks=[event_catcher.catch], ) assert len(event_catcher.caught_events) == 7 class TestPrePostHookNoFalsePositiveDeprecation: @pytest.fixture(scope="class") def models(self): return { "model_hook_configs.sql": models_pre_post_hook_in_config_sql, "schema.yml": pre_post_hook_in_config_yaml, } @mock.patch("dbt.jsonschemas.jsonschemas._JSONSCHEMA_SUPPORTED_ADAPTERS", {"postgres"}) def test_pre_post_hook_no_false_positive_deprecation(self, project): event_catcher = EventCatcher(CustomKeyInConfigDeprecation) run_dbt( ["parse", "--no-partial-parse", "--show-all-deprecations"], callbacks=[event_catcher.catch], ) assert len(event_catcher.caught_events) == 0 class TestGenerateSchemaNameNullValueDeprecation: @pytest.fixture(scope="class") def models(self): return { "models_trivial.sql": models_trivial__model_sql, } @pytest.fixture(scope="class") def macros(self): return { "macros.sql": generate_schema_name_null_return_macro_sql, } def test_generate_schema_name_null_value_deprecation(self, project): event_catcher = EventCatcher(GenerateSchemaNameNullValueDeprecation) run_dbt( ["parse", "--no-partial-parse", "--show-all-deprecations"], callbacks=[event_catcher.catch], ) assert len(event_catcher.caught_events) == 1 assert "Node 'model.test.models_trivial' has a schema set to None as a result of a generate_schema_name call." in event_catcher.caught_events[ 0 ].info.msg.replace( "\n", " " ) class TestPythonModelConfigAdditionsDontRaiseDeprecations: @pytest.fixture(scope="class") def models(self): return { "trivial_model.sql": models_trivial__model_sql, "python_model.py": python_model_py, "python_model.yml": python_model_yml, } @mock.patch("dbt.jsonschemas.jsonschemas._JSONSCHEMA_SUPPORTED_ADAPTERS", {"postgres"}) def test_python_model_config_additions_dont_raise_deprecations(self, project): event_catcher = EventCatcher(CustomKeyInConfigDeprecation) run_dbt( ["parse", "--no-partial-parse", "--show-all-deprecations"], callbacks=[event_catcher.catch], ) assert len(event_catcher.caught_events) == 0 ================================================ FILE: tests/functional/deprecations/test_missing_plus_in_config_deprecations.py ================================================ from unittest import mock import pytest from dbt.events.types import MissingPlusPrefixDeprecation from dbt.tests.util import run_dbt from dbt_common.events.event_catcher import EventCatcher @mock.patch("dbt.jsonschemas.jsonschemas._JSONSCHEMA_SUPPORTED_ADAPTERS", {"postgres"}) class TestEmptyConfig: @pytest.fixture(scope="class") def project_config_update(self): return {"models": None} def test_no_warning(self, project): event_catcher = EventCatcher(MissingPlusPrefixDeprecation) run_dbt(["parse", "--no-partial-parse"], callbacks=[event_catcher.catch]) assert len(event_catcher.caught_events) == 0 @mock.patch("dbt.jsonschemas.jsonschemas._JSONSCHEMA_SUPPORTED_ADAPTERS", {"postgres"}) class TestEmptyNestedDir: @pytest.fixture(scope="class") def project_config_update(self): return {"models": {"nested_dir": None}} def test_no_warning(self, project): event_catcher = EventCatcher(MissingPlusPrefixDeprecation) run_dbt(["parse", "--no-partial-parse"], callbacks=[event_catcher.catch]) assert len(event_catcher.caught_events) == 0 @mock.patch("dbt.jsonschemas.jsonschemas._JSONSCHEMA_SUPPORTED_ADAPTERS", {"postgres"}) class TestValidConfigKey: @pytest.fixture(scope="class") def project_config_update(self): return {"models": {"docs": None}} def test_raises_warning(self, project): event_catcher = EventCatcher(MissingPlusPrefixDeprecation) run_dbt(["parse", "--no-partial-parse"], callbacks=[event_catcher.catch]) assert len(event_catcher.caught_events) == 1 assert "Missing '+' prefix on `docs`" in event_catcher.caught_events[0].info.msg @mock.patch("dbt.jsonschemas.jsonschemas._JSONSCHEMA_SUPPORTED_ADAPTERS", {"postgres"}) class TestValidPlusPrefixConfigKeyWithNonPlusPrefixProperty: @pytest.fixture(scope="class") def project_config_update(self): return {"models": {"+docs": {"show": True}}} def test_no_warning(self, project): event_catcher = EventCatcher(MissingPlusPrefixDeprecation) run_dbt(["parse", "--no-partial-parse"], callbacks=[event_catcher.catch]) assert len(event_catcher.caught_events) == 0 @mock.patch("dbt.jsonschemas.jsonschemas._JSONSCHEMA_SUPPORTED_ADAPTERS", {"postgres"}) class TestValidPlusPrefixConfigKeyWithPlusPrefixInNestedDir: @pytest.fixture(scope="class") def project_config_update(self): return { "models": { "nested_dir_l1": { "nested_dir_l2": {"+enabled": True}, }, }, } def test_no_warning(self, project): event_catcher = EventCatcher(MissingPlusPrefixDeprecation) run_dbt(["parse", "--no-partial-parse"], callbacks=[event_catcher.catch]) assert len(event_catcher.caught_events) == 0 @mock.patch("dbt.jsonschemas.jsonschemas._JSONSCHEMA_SUPPORTED_ADAPTERS", {"postgres"}) class TestValidConfigKeyWithNonPlusPrefixInNestedDir: @pytest.fixture(scope="class") def project_config_update(self): return { "models": { "nested_dir_l1": { "nested_dir_l2": {"enabled": True}, }, }, } def test_raises_warning(self, project): event_catcher = EventCatcher(MissingPlusPrefixDeprecation) run_dbt(["parse", "--no-partial-parse"], callbacks=[event_catcher.catch]) assert len(event_catcher.caught_events) == 1 assert "Missing '+' prefix on `enabled`" in event_catcher.caught_events[0].info.msg class TestMissingPlusPrefixDeprecation: @pytest.fixture(scope="class") def project_config_update(self): return {"seeds": {"path": {"enabled": True}}} @mock.patch("dbt.jsonschemas.jsonschemas._JSONSCHEMA_SUPPORTED_ADAPTERS", {"postgres"}) def test_missing_plus_prefix_deprecation(self, project): event_catcher = EventCatcher(MissingPlusPrefixDeprecation) run_dbt(["parse", "--no-partial-parse"], callbacks=[event_catcher.catch]) assert len(event_catcher.caught_events) == 1 assert "Missing '+' prefix on `enabled`" in event_catcher.caught_events[0].info.msg class TestMissingPlusPrefixDeprecationSubPath: @pytest.fixture(scope="class") def project_config_update(self): return {"seeds": {"path": {"+enabled": True, "sub_path": {"enabled": True}}}} @mock.patch("dbt.jsonschemas.jsonschemas._JSONSCHEMA_SUPPORTED_ADAPTERS", {"postgres"}) def test_missing_plus_prefix_deprecation_sub_path(self, project): event_catcher = EventCatcher(MissingPlusPrefixDeprecation) run_dbt(["parse", "--no-partial-parse"], callbacks=[event_catcher.catch]) assert len(event_catcher.caught_events) == 1 assert "Missing '+' prefix on `enabled`" in event_catcher.caught_events[0].info.msg ================================================ FILE: tests/functional/deprecations/test_model_deprecations.py ================================================ import pytest from dbt.cli.main import dbtRunner from dbt.tests.util import run_dbt from dbt_common.exceptions import EventCompilationError deprecated_model__yml = """ version: 2 models: - name: my_model description: deprecated deprecation_date: 1999-01-01 """ deprecating_model__yml = """ version: 2 models: - name: my_model description: deprecating in the future deprecation_date: 2999-01-01 """ model__sql = """ select 1 as Id """ dependant_model__sql = """ select * from {{ ref("my_model") }} """ class TestModelDeprecationWarning: @pytest.fixture(scope="class") def models(self): return {"my_model.sql": model__sql, "my_schema.yml": deprecated_model__yml} def test_deprecation_warning(self, project): events = [] dbtRunner(callbacks=[events.append]).invoke(["parse"]) matches = list([e for e in events if e.info.name == "DeprecatedModel"]) assert len(matches) == 1 assert matches[0].data.model_name == "my_model" def test_deprecation_warning_error(self, project): with pytest.raises(EventCompilationError): run_dbt(["--warn-error", "parse"]) def test_deprecation_warning_error_options(self, project): with pytest.raises(EventCompilationError): run_dbt(["--warn-error-options", '{"error": ["DeprecatedModel"]}', "parse"]) class TestUpcomingReferenceDeprecationWarning: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": model__sql, "my_dependant_model.sql": dependant_model__sql, "my_schema.yml": deprecating_model__yml, } def test_deprecation_warning(self, project): events = [] dbtRunner(callbacks=[events.append]).invoke(["parse"]) matches = list([e for e in events if e.info.name == "UpcomingReferenceDeprecation"]) assert len(matches) == 1 assert matches[0].data.model_name == "my_dependant_model" assert matches[0].data.ref_model_name == "my_model" def test_deprecation_warning_error(self, project): with pytest.raises(EventCompilationError): run_dbt(["--warn-error", "parse"]) def test_deprecation_warning_error_options(self, project): with pytest.raises(EventCompilationError): run_dbt( ["--warn-error-options", '{"error": ["UpcomingReferenceDeprecation"]}', "parse"] ) class TestDeprecatedReferenceWarning: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": model__sql, "my_dependant_model.sql": dependant_model__sql, "my_schema.yml": deprecated_model__yml, } def test_deprecation_warning(self, project): events = [] dbtRunner(callbacks=[events.append]).invoke(["parse"]) matches = list([e for e in events if e.info.name == "DeprecatedReference"]) assert len(matches) == 1 assert matches[0].data.model_name == "my_dependant_model" assert matches[0].data.ref_model_name == "my_model" def test_deprecation_warning_error(self, project): with pytest.raises(EventCompilationError): run_dbt(["--warn-error", "parse"]) def test_deprecation_warning_error_options(self, project): with pytest.raises(EventCompilationError): run_dbt(["--warn-error-options", '{"error": ["DeprecatedReference"]}', "parse"]) ================================================ FILE: tests/functional/deps/test_deps_with_vars.py ================================================ """Test that dbt deps works when vars are used in dbt_project.yml without defaults. The key behavior being tested: - dbt deps uses lenient mode (require_vars=False) and succeeds even with missing vars - dbt run/compile/build/debug use strict mode (require_vars=True) and show the right error messages Expected behavior from reviewer's scenario: 1. dbt deps succeeds (doesn't need vars) 2. dbt run fails with error "Required var 'X' not found" 3. dbt run --vars succeeds when vars provided """ import pytest from dbt.tests.util import run_dbt, update_config_file from dbt_common.exceptions import CompilationError # Simple model for testing model_sql = """ select 1 as id """ # Base class with common fixtures class VarTestingBase: """Base class for var testing with common fixtures""" @pytest.fixture(scope="class") def models(self): return {"test_model.sql": model_sql} @pytest.fixture(scope="class") def project_config_update(self): return { "models": {"test_project": {"+materialized": "{{ var('materialized_var', 'view') }}"}} } # Test 1: Happy path - deps with defaults class TestDepsSucceedsWithVarDefaults(VarTestingBase): """Test that dbt deps succeeds when vars have default values""" @pytest.fixture(scope="class") def project_config_update(self): # config: +dataset: "{{ var('my_dataset', 'default') }}" return {"models": {"test_project": {"+dataset": "dqm_{{ var('my_dataset', 'default') }}"}}} def test_deps_succeeds(self, project): # run: dbt deps # assert: succeeds results = run_dbt(["deps"]) assert results is None or results == [] # Test 2: Happy path - run with defaults class TestRunSucceedsWithVarDefaults(VarTestingBase): """Test that dbt run succeeds when vars have default values""" def test_run_succeeds(self, project): # run: dbt run # assert: succeeds results = run_dbt(["run"]) assert len(results) == 1 # Test 3: Happy path - run with explicit vars class TestRunSucceedsWithExplicitVars(VarTestingBase): """Test that dbt run succeeds when vars provided via --vars""" def test_run_succeeds_with_vars(self, project): # run: dbt run --vars '{"my_var": "table"}' # assert: succeeds results = run_dbt(["run", "--vars", '{"materialized_var": "table"}']) assert len(results) == 1 # Test 4: Run fails with the right error message class TestRunFailsWithMissingVar(VarTestingBase): """Test dbt run fails with right error""" def test_run_fails_with_error(self, project): # IN TEST: dynamically remove default update_config_file( {"models": {"test_project": {"+materialized": "{{ var('materialized_var') }}"}}}, project.project_root, "dbt_project.yml", ) # run: dbt run # assert: fails with "Required var 'X' not found" try: run_dbt(["run"], expect_pass=False) assert False, "Expected run to fail with missing required var" except CompilationError as e: error_msg = str(e) # ✅ Verify error message assert "materialized_var" in error_msg, "Error should mention var name" assert ( "Required var" in error_msg or "not found" in error_msg ), "Error should say 'Required var' or 'not found'" # Test 5: compile also fails with the correct error class TestCompileFailsWithMissingVar(VarTestingBase): """Test dbt compile fails with error for missing vars""" @pytest.fixture(scope="class") def project_config_update(self): # config: start with simple hardcoded value (no var) return {"models": {"test_project": {"+materialized": "view"}}} def test_compile_fails_with_error(self, project): # IN TEST: dynamically add var without default update_config_file( {"models": {"test_project": {"+materialized": "{{ var('compile_var_no_default') }}"}}}, project.project_root, "dbt_project.yml", ) # run: dbt compile # assert: fails with "Required var 'X' not found" try: run_dbt(["compile"], expect_pass=False) assert False, "Expected compile to fail with missing var" except CompilationError as e: error_msg = str(e) assert "compile_var_no_default" in error_msg assert "Required var" in error_msg or "not found" in error_msg # Test 6: deps succeeds even when var missing class TestDepsSucceedsEvenWhenVarMissing(VarTestingBase): """Test dbt deps succeeds even when var has no default""" def test_deps_still_succeeds(self, project): # run: dbt deps (succeeds) results = run_dbt(["deps"]) assert results is None or results == [] # IN TEST: modify config to remove var default update_config_file( {"models": {"test_project": {"+materialized": "{{ var('materialized_var') }}"}}}, project.project_root, "dbt_project.yml", ) # run: dbt deps again (still succeeds - lenient mode) results = run_dbt(["deps"]) assert results is None or results == [] # run: dbt run (fails - strict mode) try: run_dbt(["run"], expect_pass=False) assert False, "Expected run to fail with missing var" except CompilationError as e: error_msg = str(e) assert "materialized_var" in error_msg assert "Required var" in error_msg or "not found" in error_msg # Test 7: build also fails class TestBuildFailsWithMissingVar(VarTestingBase): """Test dbt build fails with error for missing vars""" @pytest.fixture(scope="class") def project_config_update(self): # config: start with simple hardcoded value (no var) return {"models": {"test_project": {"+materialized": "view"}}} def test_build_fails_with_error(self, project): # IN TEST: dynamically add var without default update_config_file( {"models": {"test_project": {"+materialized": "{{ var('build_var_no_default') }}"}}}, project.project_root, "dbt_project.yml", ) # run: dbt build # assert: fails with "Required var 'X' not found" try: run_dbt(["build"], expect_pass=False) assert False, "Expected build to fail with missing var" except CompilationError as e: error_msg = str(e) assert "build_var_no_default" in error_msg assert "Required var" in error_msg or "not found" in error_msg # Test 8: debug with defaults class TestDebugSucceedsWithVarDefaults(VarTestingBase): """Test dbt debug succeeds when vars have defaults (no regression)""" def test_debug_succeeds(self, project): # run: dbt debug # assert: succeeds (no regression) run_dbt(["debug"]) # Test 9: debug fails like run/compile (strict mode) class TestDebugFailsWithMissingVar(VarTestingBase): """Test dbt debug fails with error (strict mode like run/compile)""" def test_debug_fails_with_error(self, project): # First verify debug works with default run_dbt(["debug"]) # IN TEST: dynamically remove default update_config_file( {"models": {"test_project": {"+materialized": "{{ var('materialized_var') }}"}}}, project.project_root, "dbt_project.yml", ) # run: dbt debug # assert: fails with "Required var 'X' not found" try: run_dbt(["debug"], expect_pass=False) assert False, "Expected debug to fail with missing var" except CompilationError as e: error_msg = str(e) assert "materialized_var" in error_msg assert "Required var" in error_msg or "not found" in error_msg ================================================ FILE: tests/functional/docs/test_doc_blocks_backcompat.py ================================================ import json import os import pytest from dbt.tests.util import run_dbt schema_yml = """ models: - name: my_colors doc_blocks: 2 columns: - name: id doc_blocks: 2 - name: color doc_blocks: ["hello", 2, "world"] """ class TestDocBlocksBackCompat: @pytest.fixture(scope="class") def models(self): return { "my_colors.sql": "select 1 as id, 'blue' as color", "schema.yml": schema_yml, } def test_doc_blocks_back_compat(self, project): run_dbt(["parse"]) assert os.path.exists("./target/manifest.json") with open("./target/manifest.json") as fp: manifest = json.load(fp) model_data = manifest["nodes"]["model.test.my_colors"] assert model_data["doc_blocks"] == [] assert all(column["doc_blocks"] == [] for column in model_data["columns"].values()) ================================================ FILE: tests/functional/docs/test_doc_blocks_formatting.py ================================================ import json import os import pytest from dbt.tests.util import run_dbt docs_md = """{% docs test_doc %} This is a test for column {test_name} {% enddocs %} """ schema_yml = """ models: - name: my_colors columns: - name: id description: "{{ doc('test_doc').format(test_name = 'id') }}" - name: color description: "{{ 'This is a test for column {test_name}'.format(test_name = 'color') }}" """ class TestDocBlocksBackCompat: @pytest.fixture(scope="class") def models(self): return { "my_colors.sql": "select 1 as id, 'blue' as color", "schema.yml": schema_yml, "docs.md": docs_md, } def test_doc_blocks_back_compat(self, project): run_dbt(["parse"]) assert os.path.exists("./target/manifest.json") with open("./target/manifest.json") as fp: manifest = json.load(fp) model_data = manifest["nodes"]["model.test.my_colors"] for column_name, column in model_data["columns"].items(): assert column["description"] == f"This is a test for column {column_name}" assert column["doc_blocks"] == [] ================================================ FILE: tests/functional/docs/test_doc_concat_arg.py ================================================ import json import pytest from dbt.tests.util import run_dbt docs_md = """{% docs test_doc %} this is a docs block {% enddocs %} """ schema_yml = """ models: - name: my_model description: "{{ doc('test_' ~ 'doc') }}" columns: - name: id description: "{{ doc('test_' ~ 'doc') }}" """ class TestDocConcatArg: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": "select 1 as id", "schema.yml": schema_yml, "docs.md": docs_md, } def test_concat_arg_succeeds(self, project): run_dbt(["parse"]) with open("./target/manifest.json") as fp: manifest = json.load(fp) model_data = manifest["nodes"]["model.test.my_model"] assert model_data["description"] == "this is a docs block" # Ideally, this would be able to track the doc block in lineage. # However, Const jinja nodes are not handled statically for this resolution. assert model_data["doc_blocks"] == [] column_data = model_data["columns"]["id"] assert column_data["description"] == "this is a docs block" # Ideally, this would be able to track the doc block in lineage. # However, Const jinja nodes are not handled statically for this resolution. assert column_data["doc_blocks"] == [] ================================================ FILE: tests/functional/docs/test_doc_variable_arg.py ================================================ import pytest from dbt.exceptions import DocTargetNotFoundError from dbt.tests.util import run_dbt schema_yml = """ models: - name: my_model description: "{{ doc(my_variable) }}" columns: - name: id description: "{{ doc(some_var) }}" """ class TestDocVariableArg: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": "select 1 as id", "schema.yml": schema_yml, } def test_variable_arg_raises_doc_not_found_not_attribute_error(self, project): with pytest.raises(DocTargetNotFoundError): run_dbt(["parse"]) ================================================ FILE: tests/functional/docs/test_duplicate_docs_block.py ================================================ import pytest import dbt_common.exceptions from dbt.tests.util import run_dbt duplicate_doc_blocks_model_sql = "select 1 as id, 'joe' as first_name" duplicate_doc_blocks_docs_md = """{% docs my_model_doc %} a doc string {% enddocs %} {% docs my_model_doc %} duplicate doc string {% enddocs %}""" duplicate_doc_blocks_schema_yml = """version: 2 models: - name: model description: "{{ doc('my_model_doc') }}" """ class TestDuplicateDocsBlock: @pytest.fixture(scope="class") def models(self): return { "model.sql": duplicate_doc_blocks_model_sql, "schema.yml": duplicate_doc_blocks_schema_yml, } def test_duplicate_doc_ref(self, project): with pytest.raises(dbt_common.exceptions.CompilationError): run_dbt(expect_pass=False) ================================================ FILE: tests/functional/docs/test_generate.py ================================================ from unittest import mock import pytest from dbt.plugins.manifest import ModelNodeArgs, PluginNodes from dbt.tests.util import get_manifest, run_dbt sample_seed = """sample_num,sample_bool 1,true 2,false 3,true """ second_seed = """sample_num,sample_bool 4,true 5,false 6,true """ sample_config = """ sources: - name: my_source_schema schema: "{{ target.schema }}" tables: - name: sample_source - name: second_source - name: non_existent_source - name: source_from_seed """ class TestBaseGenerate: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": "select 1 as fun", "alt_model.sql": "select 1 as notfun", "sample_config.yml": sample_config, } @pytest.fixture(scope="class") def seeds(self): return { "sample_seed.csv": sample_seed, "second_seed.csv": sample_seed, } class TestGenerateManifestNotCompiled(TestBaseGenerate): def test_manifest_not_compiled(self, project): run_dbt(["docs", "generate", "--no-compile"]) # manifest.json is written out in parsing now, but it # shouldn't be compiled because of the --no-compile flag manifest = get_manifest(project.project_root) model_id = "model.test.my_model" assert model_id in manifest.nodes assert manifest.nodes[model_id].compiled is False class TestGenerateEmptyCatalog(TestBaseGenerate): def test_generate_empty_catalog(self, project): catalog = run_dbt(["docs", "generate", "--empty-catalog"]) assert catalog.nodes == {}, "nodes should be empty" assert catalog.sources == {}, "sources should be empty" assert catalog.errors is None, "errors should be null" class TestGenerateSelectLimitsCatalog(TestBaseGenerate): def test_select_limits_catalog(self, project): run_dbt(["run"]) catalog = run_dbt(["docs", "generate", "--select", "my_model"]) assert len(catalog.nodes) == 1 assert "model.test.my_model" in catalog.nodes class TestGenerateSelectLimitsNoMatch(TestBaseGenerate): def test_select_limits_no_match(self, project): run_dbt(["run"]) catalog = run_dbt(["docs", "generate", "--select", "my_missing_model"]) assert len(catalog.nodes) == 0 assert len(catalog.sources) == 0 class TestGenerateCatalogWithSources(TestBaseGenerate): def test_catalog_with_sources(self, project): # populate sources other than non_existent_source project.run_sql("create table {}.sample_source (id int)".format(project.test_schema)) project.run_sql("create table {}.second_source (id int)".format(project.test_schema)) # build nodes run_dbt(["build"]) catalog = run_dbt(["docs", "generate"]) # 2 seeds + 2 models assert len(catalog.nodes) == 4 # 2 sources (only ones that exist) assert len(catalog.sources) == 2 class TestGenerateCatalogWithExternalNodes(TestBaseGenerate): @mock.patch("dbt.plugins.get_plugin_manager") def test_catalog_with_external_node(self, get_plugin_manager, project): project.run_sql("create table {}.external_model (id int)".format(project.test_schema)) run_dbt(["build"]) external_nodes = PluginNodes() external_model_node = ModelNodeArgs( name="external_model", package_name="external_package", identifier="external_model", schema=project.test_schema, database="dbt", ) external_nodes.add_model(external_model_node) get_plugin_manager.return_value.get_nodes.return_value = external_nodes catalog = run_dbt(["docs", "generate"]) assert "model.external_package.external_model" in catalog.nodes class TestGenerateSelectSource(TestBaseGenerate): @pytest.fixture(scope="class") def seeds(self): return { "sample_seed.csv": sample_seed, "second_seed.csv": sample_seed, "source_from_seed.csv": sample_seed, } def test_select_source(self, project): run_dbt(["build"]) project.run_sql("create table {}.sample_source (id int)".format(project.test_schema)) project.run_sql("create table {}.second_source (id int)".format(project.test_schema)) # 2 existing sources, 1 selected catalog = run_dbt( ["docs", "generate", "--select", "source:test.my_source_schema.sample_source"] ) assert len(catalog.sources) == 1 assert "source.test.my_source_schema.sample_source" in catalog.sources # no nodes selected assert len(catalog.nodes) == 0 # 2 existing sources sources, 1 selected that has relation as a seed catalog = run_dbt( ["docs", "generate", "--select", "source:test.my_source_schema.source_from_seed"] ) assert len(catalog.sources) == 1 assert "source.test.my_source_schema.source_from_seed" in catalog.sources # seed with same relation that was not selected not in catalog assert len(catalog.nodes) == 0 class TestGenerateSelectOverMaxSchemaMetadataRelations(TestBaseGenerate): @pytest.fixture(scope="class") def seeds(self): return { "sample_seed.csv": sample_seed, "second_seed.csv": sample_seed, "source_from_seed.csv": sample_seed, } def test_select_source(self, project): run_dbt(["build"]) project.run_sql("create table {}.sample_source (id int)".format(project.test_schema)) project.run_sql("create table {}.second_source (id int)".format(project.test_schema)) with mock.patch.object(type(project.adapter), "MAX_SCHEMA_METADATA_RELATIONS", 1): # more relations than MAX_SCHEMA_METADATA_RELATIONS -> all sources and nodes correctly returned catalog = run_dbt(["docs", "generate"]) assert len(catalog.sources) == 3 assert len(catalog.nodes) == 5 # full source selection respected catalog = run_dbt(["docs", "generate", "--select", "source:*"]) assert len(catalog.sources) == 3 assert len(catalog.nodes) == 0 # full node selection respected catalog = run_dbt(["docs", "generate", "--exclude", "source:*"]) assert len(catalog.sources) == 0 assert len(catalog.nodes) == 5 # granular source selection respected (> MAX_SCHEMA_METADATA_RELATIONS selected sources) catalog = run_dbt( [ "docs", "generate", "--select", "source:test.my_source_schema.sample_source", "source:test.my_source_schema.second_source", ] ) assert len(catalog.sources) == 2 assert len(catalog.nodes) == 0 # granular node selection respected (> MAX_SCHEMA_METADATA_RELATIONS selected nodes) catalog = run_dbt(["docs", "generate", "--select", "my_model", "alt_model"]) assert len(catalog.sources) == 0 assert len(catalog.nodes) == 2 class TestGenerateSelectSeed(TestBaseGenerate): @pytest.fixture(scope="class") def seeds(self): return { "sample_seed.csv": sample_seed, "second_seed.csv": sample_seed, "source_from_seed.csv": sample_seed, } def test_select_seed(self, project): run_dbt(["build"]) # 3 seeds, 1 selected catalog = run_dbt(["docs", "generate", "--select", "sample_seed"]) assert len(catalog.nodes) == 1 assert "seed.test.sample_seed" in catalog.nodes # no sources selected assert len(catalog.sources) == 0 # 3 seeds, 1 selected that has same relation as a source catalog = run_dbt(["docs", "generate", "--select", "source_from_seed"]) assert len(catalog.nodes) == 1 assert "seed.test.source_from_seed" in catalog.nodes # source with same relation that was not selected not in catalog assert len(catalog.sources) == 0 ================================================ FILE: tests/functional/docs/test_good_docs_blocks.py ================================================ import json import os from pathlib import Path import pytest from dbt.tests.util import run_dbt, update_config_file, write_file good_docs_blocks_model_sql = "select 1 as id, 'joe' as first_name" good_docs_blocks_docs_md = """{% docs my_model_doc %} My model is just a copy of the seed {% enddocs %} {% docs my_model_doc__id %} The user ID number {% enddocs %} The following doc is never used, which should be fine. {% docs my_model_doc__first_name %} The user's first name (should not be shown!) {% enddocs %} This doc is referenced by its full name {% docs my_model_doc__last_name %} The user's last name {% enddocs %} """ good_doc_blocks_alt_docs_md = """{% docs my_model_doc %} Alt text about the model {% enddocs %} {% docs my_model_doc__id %} The user ID number with alternative text {% enddocs %} The following doc is never used, which should be fine. {% docs my_model_doc__first_name %} The user's first name - don't show this text! {% enddocs %} This doc is referenced by its full name {% docs my_model_doc__last_name %} The user's last name in this other file {% enddocs %} """ good_docs_blocks_schema_yml = """version: 2 models: - name: model description: "{{ doc('my_model_doc') }}" columns: - name: id description: "{{ doc('my_model_doc__id') }}" - name: first_name description: The user's first name - name: last_name description: "{{ doc('test', 'my_model_doc__last_name') }}" - name: tricky description: "{{ doc('my_model_doc__id') }} The user's first name {{ doc('test', 'my_model_doc__last_name') }}" """ class TestGoodDocsBlocks: @pytest.fixture(scope="class") def models(self): return { "model.sql": good_docs_blocks_model_sql, "schema.yml": good_docs_blocks_schema_yml, "docs.md": good_docs_blocks_docs_md, } def test_valid_doc_ref(self, project): result = run_dbt() assert len(result.results) == 1 assert os.path.exists("./target/manifest.json") with open("./target/manifest.json") as fp: manifest = json.load(fp) model_data = manifest["nodes"]["model.test.model"] assert model_data["description"] == "My model is just a copy of the seed" assert model_data["doc_blocks"] == ["doc.test.my_model_doc"] assert { "name": "id", "description": "The user ID number", "dimension": None, "entity": None, "data_type": None, "constraints": [], "meta": {}, "quote": None, "tags": [], "granularity": None, "doc_blocks": ["doc.test.my_model_doc__id"], "config": {"meta": {}, "tags": []}, } == model_data["columns"]["id"] assert { "name": "first_name", "description": "The user's first name", "dimension": None, "entity": None, "data_type": None, "constraints": [], "meta": {}, "quote": None, "tags": [], "granularity": None, "doc_blocks": [], "config": {"meta": {}, "tags": []}, } == model_data["columns"]["first_name"] assert { "name": "last_name", "description": "The user's last name", "dimension": None, "entity": None, "data_type": None, "constraints": [], "meta": {}, "quote": None, "tags": [], "granularity": None, "doc_blocks": ["doc.test.my_model_doc__last_name"], "config": {"meta": {}, "tags": []}, } == model_data["columns"]["last_name"] assert { "name": "tricky", "description": "The user ID number The user's first name The user's last name", "dimension": None, "entity": None, "data_type": None, "constraints": [], "meta": {}, "quote": None, "tags": [], "granularity": None, "doc_blocks": [ "doc.test.my_model_doc__id", "doc.test.my_model_doc__last_name", ], "config": {"meta": {}, "tags": []}, } == model_data["columns"]["tricky"] assert len(model_data["columns"]) == 4 class TestGoodDocsBlocksAltPath: @pytest.fixture(scope="class") def models(self): return {"model.sql": good_docs_blocks_model_sql, "schema.yml": good_docs_blocks_schema_yml} def test_alternative_docs_path(self, project): # self.use_default_project({"docs-paths": [self.dir("docs")]}) docs_path = Path(project.project_root, "alt-docs") docs_path.mkdir() write_file(good_doc_blocks_alt_docs_md, project.project_root, "alt-docs", "docs.md") update_config_file( {"docs-paths": [str(docs_path)]}, project.project_root, "dbt_project.yml" ) result = run_dbt() assert len(result.results) == 1 assert os.path.exists("./target/manifest.json") with open("./target/manifest.json") as fp: manifest = json.load(fp) model_data = manifest["nodes"]["model.test.model"] assert model_data["description"] == "Alt text about the model" assert model_data["doc_blocks"] == ["doc.test.my_model_doc"] assert { "name": "id", "description": "The user ID number with alternative text", "dimension": None, "entity": None, "data_type": None, "constraints": [], "meta": {}, "quote": None, "tags": [], "granularity": None, "doc_blocks": ["doc.test.my_model_doc__id"], "config": {"meta": {}, "tags": []}, } == model_data["columns"]["id"] assert { "name": "first_name", "description": "The user's first name", "dimension": None, "entity": None, "data_type": None, "constraints": [], "meta": {}, "quote": None, "tags": [], "granularity": None, "doc_blocks": [], "config": {"meta": {}, "tags": []}, } == model_data["columns"]["first_name"] assert { "name": "last_name", "description": "The user's last name in this other file", "dimension": None, "entity": None, "data_type": None, "constraints": [], "meta": {}, "quote": None, "tags": [], "granularity": None, "doc_blocks": ["doc.test.my_model_doc__last_name"], "config": {"meta": {}, "tags": []}, } == model_data["columns"]["last_name"] assert { "name": "tricky", "description": "The user ID number with alternative text The user's first name The user's last name in this other file", "dimension": None, "entity": None, "data_type": None, "constraints": [], "meta": {}, "quote": None, "tags": [], "granularity": None, "doc_blocks": [ "doc.test.my_model_doc__id", "doc.test.my_model_doc__last_name", ], "config": {"meta": {}, "tags": []}, } == model_data["columns"]["tricky"] assert len(model_data["columns"]) == 4 ================================================ FILE: tests/functional/docs/test_invalid_doc_ref.py ================================================ import pytest import dbt_common.exceptions from dbt.tests.util import run_dbt invalid_doc_ref_model_sql = "select 1 as id, 'joe' as first_name" invalid_doc_ref_docs_md = """{% docs my_model_doc %} My model is just a copy of the seed {% enddocs %} {% docs my_model_doc__id %} The user ID number {% enddocs %} The following doc is never used, which should be fine. {% docs my_model_doc__first_name %} The user's first name {% enddocs %}""" invalid_doc_ref_schema_yml = """version: 2 models: - name: model description: "{{ doc('my_model_doc') }}" columns: - name: id description: "{{ doc('my_model_doc__id') }}" - name: first_name description: "{{ doc('foo.bar.my_model_doc__id') }}" """ class TestInvalidDocRef: @pytest.fixture(scope="class") def models(self): return { "model.sql": invalid_doc_ref_model_sql, "docs.md": invalid_doc_ref_docs_md, "schema.yml": invalid_doc_ref_schema_yml, } def test_invalid_doc_ref(self, project): # The run should fail since we could not find the docs reference. with pytest.raises(dbt_common.exceptions.CompilationError): run_dbt(expect_pass=False) ================================================ FILE: tests/functional/docs/test_missing_docs_blocks.py ================================================ import pytest import dbt_common.exceptions from dbt.tests.util import run_dbt missing_docs_blocks_model_sql = "select 1 as id, 'joe' as first_name" missing_docs_blocks_docs_md = """{% docs my_model_doc %} My model is just a copy of the seed {% enddocs %} {% docs my_model_doc__id %} The user ID number {% enddocs %}""" missing_docs_blocks_schema_yml = """version: 2 models: - name: model description: "{{ doc('my_model_doc') }}" columns: - name: id description: "{{ doc('my_model_doc__id') }}" - name: first_name # invalid reference description: "{{ doc('my_model_doc__first_name') }}" """ class TestMissingDocsBlocks: @pytest.fixture(scope="class") def models(self): return { "model.sql": missing_docs_blocks_model_sql, "schema.yml": missing_docs_blocks_schema_yml, "docs.md": missing_docs_blocks_docs_md, } def test_missing_doc_ref(self, project): # The run should fail since we could not find the docs reference. with pytest.raises(dbt_common.exceptions.CompilationError): run_dbt() ================================================ FILE: tests/functional/docs/test_model_version_docs_blocks.py ================================================ import pytest from dbt.tests.util import run_dbt model_1 = """ select 1 as id, 'joe' as first_name """ model_versioned = """ select 1 as id, 'joe' as first_name """ docs_md = """ {% docs model_description %} unversioned model {% enddocs %} {% docs column_id_doc %} column id for some thing {% enddocs %} {% docs versioned_model_description %} versioned model {% enddocs %} """ schema_yml = """ models: - name: model_1 description: '{{ doc("model_description") }}' columns: - name: id description: '{{ doc("column_id_doc") }}' - name: model_versioned description: '{{ doc("versioned_model_description") }}' latest_version: 1 versions: - v: 1 config: alias: my_alias columns: - name: id description: '{{ doc("column_id_doc") }}' - name: first_name description: 'plain text' - v: 2 columns: - name: other_id """ class TestVersionedModelDocsBlock: @pytest.fixture(scope="class") def models(self): return { "model_1.sql": model_1, "model_versioned.sql": model_versioned, "schema.yml": schema_yml, "docs.md": docs_md, } def test_versioned_doc_ref(self, project): manifest = run_dbt(["parse"]) model_1 = manifest.nodes["model.test.model_1"] model_v1 = manifest.nodes["model.test.model_versioned.v1"] assert model_1.description == "unversioned model" assert model_v1.description == "versioned model" assert model_1.columns["id"].description == "column id for some thing" assert model_v1.columns["id"].description == "column id for some thing" assert model_v1.columns["first_name"].description == "plain text" ================================================ FILE: tests/functional/docs/test_static.py ================================================ import os import pytest from dbt.task.docs import DOCS_INDEX_FILE_PATH from dbt.tests.util import run_dbt from dbt_common.clients.system import load_file_contents class TestStaticGenerate: @pytest.fixture(scope="class") def models(self): return {"my_model.sql": "select 1 as fun"} def test_static_generated(self, project): run_dbt(["docs", "generate", "--static"]) source_index_html = load_file_contents(DOCS_INDEX_FILE_PATH) target_index_html = load_file_contents( os.path.join(project.project_root, "target", "index.html") ) # Validate index.html was copied correctly assert len(target_index_html) == len(source_index_html) assert hash(target_index_html) == hash(source_index_html) manifest_data = load_file_contents( os.path.join(project.project_root, "target", "manifest.json") ) catalog_data = load_file_contents( os.path.join(project.project_root, "target", "catalog.json") ) static_index_html = load_file_contents( os.path.join(project.project_root, "target", "static_index.html") ) # Calculate expected static_index.html expected_static_index_html = source_index_html expected_static_index_html = expected_static_index_html.replace( '"MANIFEST.JSON INLINE DATA"', manifest_data ) expected_static_index_html = expected_static_index_html.replace( '"CATALOG.JSON INLINE DATA"', catalog_data ) # Validate static_index.html was generated correctly assert len(expected_static_index_html) == len(static_index_html) assert hash(expected_static_index_html) == hash(static_index_html) ================================================ FILE: tests/functional/duplicates/test_duplicate_analysis.py ================================================ import pytest from dbt.exceptions import CompilationError from dbt.tests.util import run_dbt my_model_sql = """ select 1 as id """ my_analysis_sql = """ select * from {{ ref('my_model') }} """ class TestDuplicateAnalysis: @pytest.fixture(scope="class") def models(self): return {"my_model.sql": my_model_sql} @pytest.fixture(scope="class") def analyses(self): return { "anlysis-1": {"model.sql": my_analysis_sql}, "anlysis-2": {"model.sql": my_analysis_sql}, } def test_duplicate_model_enabled(self, project): message = "dbt found two analyses with the name" with pytest.raises(CompilationError) as exc: run_dbt(["compile"]) exc_str = " ".join(str(exc.value).split()) # flatten all whitespace assert message in exc_str ================================================ FILE: tests/functional/duplicates/test_duplicate_exposure.py ================================================ import pytest from dbt.exceptions import CompilationError from dbt.tests.util import run_dbt exposure_dupes_schema_yml = """ version: 2 exposures: - name: something type: dashboard owner: email: test@example.com - name: something type: dashboard owner: email: test@example.com """ class TestDuplicateExposure: @pytest.fixture(scope="class") def models(self): return {"schema.yml": exposure_dupes_schema_yml} def test_duplicate_exposure(self, project): message = "dbt found two exposures with the name" with pytest.raises(CompilationError) as exc: run_dbt(["compile"]) assert message in str(exc.value) ================================================ FILE: tests/functional/duplicates/test_duplicate_macro.py ================================================ import pytest from dbt.exceptions import CompilationError from dbt.tests.util import run_dbt bad_same_macros_sql = """ {% macro some_macro() %} {% endmacro %} {% macro some_macro() %} {% endmacro %} """ bad_separate_one_sql = """ {% macro some_macro() %} {% endmacro %} """ bad_separate_two_sql = """ {% macro some_macro() %} {% endmacro %} """ model_sql = """ select 1 as value """ class TestDuplicateMacroEnabledSameFile: @pytest.fixture(scope="class") def models(self): return {"model.sql": model_sql} @pytest.fixture(scope="class") def macros(self): return { "macro.sql": bad_same_macros_sql, } def test_duplicate_macros(self, project): message = 'dbt found multiple macros named "some_macro" in the project' with pytest.raises(CompilationError) as exc: run_dbt(["parse"]) exc_str = " ".join(str(exc.value).split()) # flatten all whitespace assert message in exc_str assert "macro.sql" in exc_str class TestDuplicateMacroEnabledDifferentFiles: @pytest.fixture(scope="class") def models(self): return {"model.sql": model_sql} @pytest.fixture(scope="class") def macros(self): return { "one.sql": bad_separate_one_sql, "two.sql": bad_separate_two_sql, } def test_duplicate_macros(self, project): message = 'dbt found multiple macros named "some_macro" in the project' with pytest.raises(CompilationError) as exc: run_dbt(["compile"]) exc_str = " ".join(str(exc.value).split()) # flatten all whitespace assert message in exc_str assert "one.sql" in exc_str assert "two.sql" in exc_str ================================================ FILE: tests/functional/duplicates/test_duplicate_metric.py ================================================ import pytest from dbt.exceptions import CompilationError from dbt.tests.util import run_dbt metric_dupes_schema_yml = """ version: 2 metrics: - name: number_of_people label: "Number of people" description: Total count of people type: simple type_params: measure: "people" meta: my_meta: 'testing' - name: number_of_people label: "Collective tenure" description: Total number of years of team experience type: simple type_params: measure: name: "years_tenure" filter: "{{ Dimension('people_entity__loves_dbt') }} is true" """ class TestDuplicateMetric: @pytest.fixture(scope="class") def models(self): return {"schema.yml": metric_dupes_schema_yml} def test_duplicate_metric(self, project): message = "dbt found two metrics with the name" with pytest.raises(CompilationError) as exc: run_dbt(["compile"]) assert message in str(exc.value) ================================================ FILE: tests/functional/duplicates/test_duplicate_model.py ================================================ import pytest from dbt.exceptions import AmbiguousAliasError, CompilationError from dbt.tests.fixtures.project import write_project_files from dbt.tests.util import get_manifest, run_dbt disabled_model_sql = """ {{ config( enabled=False, materialized="table", ) }} select 1 """ enabled_model_sql = """ {{ config( enabled=True, materialized="table", ) }} select 1 as value """ dbt_project_yml = """ name: 'local_dep' version: '1.0' config-version: 2 profile: 'default' model-paths: ["models"] seeds: quote_columns: False """ local_dep_schema_yml = """ models: - name: table_model config: alias: table_model_local_dep columns: - name: id data_tests: - unique """ local_dep_versions_schema_yml = """ models: - name: table_model config: alias: table_model_local_dep versions: - v: 1 """ class TestDuplicateModelEnabled: @pytest.fixture(scope="class") def models(self): return { "model-enabled-1": {"model.sql": enabled_model_sql}, "model-enabled-2": {"model.sql": enabled_model_sql}, } def test_duplicate_model_enabled(self, project): message = "dbt found two models with the name" with pytest.raises(CompilationError) as exc: run_dbt(["compile"]) exc_str = " ".join(str(exc.value).split()) # flatten all whitespace assert message in exc_str class TestDuplicateModelDisabled: @pytest.fixture(scope="class") def models(self): return { "model-disabled": {"model.sql": disabled_model_sql}, "model-enabled": {"model.sql": enabled_model_sql}, } def test_duplicate_model_disabled(self, project): results = run_dbt(["compile"]) assert len(results) == 1 manifest = get_manifest(project.project_root) model_id = "model.test.model" assert model_id in manifest.nodes assert model_id in manifest.disabled def test_duplicate_model_disabled_partial_parsing(self, project): run_dbt(["clean"]) results = run_dbt(["--partial-parse", "compile"]) assert len(results) == 1 results = run_dbt(["--partial-parse", "compile"]) assert len(results) == 1 results = run_dbt(["--partial-parse", "compile"]) assert len(results) == 1 class TestDuplicateModelAliasEnabledAcrossPackages: @pytest.fixture(scope="class") def models(self): return {"table_model.sql": enabled_model_sql} @pytest.fixture(scope="class", autouse=True) def setUp(self, project_root): local_dependency_files = { "dbt_project.yml": dbt_project_yml, "models": {"table_model.sql": enabled_model_sql}, } write_project_files(project_root, "local_dependency", local_dependency_files) @pytest.fixture(scope="class") def packages(self): return {"packages": [{"local": "local_dependency"}]} def test_duplicate_model_alias_enabled_across_packages(self, project): run_dbt(["deps"]) message = "dbt found two resources with the database representation" with pytest.raises(AmbiguousAliasError) as exc: run_dbt(["run"]) assert message in str(exc.value) class TestDuplicateModelDisabledAcrossPackages: @pytest.fixture(scope="class", autouse=True) def setUp(self, project_root): local_dependency_files = { "dbt_project.yml": dbt_project_yml, "models": {"table_model.sql": enabled_model_sql}, } write_project_files(project_root, "local_dependency", local_dependency_files) @pytest.fixture(scope="class") def models(self): return {"table_model.sql": disabled_model_sql} @pytest.fixture(scope="class") def packages(self): return {"packages": [{"local": "local_dependency"}]} def test_duplicate_model_disabled_across_packages(self, project): run_dbt(["deps"]) results = run_dbt(["compile"]) assert len(results) == 1 manifest = get_manifest(project.project_root) local_dep_model_id = "model.local_dep.table_model" model_id = "model.test.table_model" assert local_dep_model_id in manifest.nodes assert model_id in manifest.disabled class TestDuplicateModelNameWithTestAcrossPackages: @pytest.fixture(scope="class", autouse=True) def setUp(self, project_root): local_dependency_files = { "dbt_project.yml": dbt_project_yml, "models": {"table_model.sql": enabled_model_sql, "schema.yml": local_dep_schema_yml}, } write_project_files(project_root, "local_dependency", local_dependency_files) @pytest.fixture(scope="class") def models(self): return {"table_model.sql": enabled_model_sql} @pytest.fixture(scope="class") def packages(self): return {"packages": [{"local": "local_dependency"}]} def test_duplicate_model_name_with_test_across_packages(self, project): run_dbt(["deps"]) manifest = run_dbt(["parse"]) assert len(manifest.nodes) == 3 # model nodes with duplicate names exist local_dep_model_node_id = "model.local_dep.table_model" root_model_node_id = "model.test.table_model" assert local_dep_model_node_id in manifest.nodes assert root_model_node_id in manifest.nodes # test node exists and is attached to correct node test_node_id = "test.local_dep.unique_table_model_id.1da9e464d9" assert test_node_id in manifest.nodes assert manifest.nodes[test_node_id].attached_node == local_dep_model_node_id class TestDuplicateModelNameWithVersionAcrossPackages: @pytest.fixture(scope="class", autouse=True) def setUp(self, project_root): local_dependency_files = { "dbt_project.yml": dbt_project_yml, "models": { "table_model.sql": enabled_model_sql, "schema.yml": local_dep_versions_schema_yml, }, } write_project_files(project_root, "local_dependency", local_dependency_files) @pytest.fixture(scope="class") def models(self): return {"table_model.sql": enabled_model_sql} @pytest.fixture(scope="class") def packages(self): return {"packages": [{"local": "local_dependency"}]} def test_duplicate_model_name_with_test_across_packages(self, project): run_dbt(["deps"]) manifest = run_dbt(["parse"]) assert len(manifest.nodes) == 2 # model nodes with duplicate names exist local_dep_model_node_id = "model.local_dep.table_model.v1" root_model_node_id = "model.test.table_model" assert local_dep_model_node_id in manifest.nodes assert root_model_node_id in manifest.nodes class TestModelTestOverlap: @pytest.fixture(scope="class") def models(self): return {"table_model.sql": enabled_model_sql} @property def project_config(self): return { "config-version": 2, "test-paths": ["models"], } def test_duplicate_test_model_paths(self, project): # this should be ok: test/model overlap is fine run_dbt(["compile"]) run_dbt(["--partial-parse", "compile"]) run_dbt(["--partial-parse", "compile"]) class TestMultipleDisabledModels: @pytest.fixture(scope="class") def models(self): return { "subdir3": {"model_alt.sql": disabled_model_sql}, "subdir2": {"model_alt.sql": disabled_model_sql}, "subdir1": {"model_alt.sql": enabled_model_sql}, } def test_multiple_disabled_models(self, project): run_dbt(["compile"]) manifest = get_manifest(project.project_root) model_id = "model.test.model_alt" assert model_id in manifest.nodes ================================================ FILE: tests/functional/duplicates/test_duplicate_resource.py ================================================ import pytest from dbt.tests.util import run_dbt models_naming_dupes_schema_yml = """ version: 2 models: - name: something description: This table has basic information about orders, as well as some derived facts based on payments exposure: - name: something """ something_model_sql = """ select 1 as item """ class TestDuplicateSchemaResource: @pytest.fixture(scope="class") def models(self): return { "schema.yml": models_naming_dupes_schema_yml, "something.sql": something_model_sql, } # a model and an exposure can share the same name def test_duplicate_model_and_exposure(self, project): result = run_dbt(["compile"]) assert len(result) == 1 ================================================ FILE: tests/functional/duplicates/test_duplicate_resource_names.py ================================================ from collections import defaultdict import pytest import dbt.deprecations as deprecations from dbt.exceptions import DuplicateResourceNameError from dbt.tests.fixtures.project import write_project_files from dbt.tests.util import run_dbt # Test resources with duplicate names but different database aliases model_sql = """ select 1 as id, 'test' as name """ seed_csv = """ id,value 1,test 2,another """ macros_sql = """ {% macro generate_alias_name(custom_alias_name, node) -%} {{ node.name }}_{{ node.resource_type }} {%- endmacro %} """ versioned_model_yml = """ models: - name: same_name versions: - v: 1 """ local_dependency__dbt_project_yml = """ name: 'local_dep' version: '1.0' """ @pytest.fixture(scope="class") def set_up_deprecations(): deprecations.reset_deprecations() assert deprecations.active_deprecations == defaultdict(int) class BaseTestDuplicateNames: @pytest.fixture(scope="class") def models(self): return { "same_name.sql": model_sql, } @pytest.fixture(scope="class") def seeds(self): return { "same_name.csv": seed_csv, } @pytest.fixture(scope="class") def macros(self): return { "generate_alias_name.sql": macros_sql, } class TestDuplicateNamesRequireUniqueResourceNamesTrue(BaseTestDuplicateNames): @pytest.fixture(scope="class") def project_config_update(self): return { "flags": { "require_unique_project_resource_names": True, } } def test_duplicate_names_with_flag_enabled(self, project): """When require_unique_project_resource_names is True, duplicate unversioned names should raise DuplicateResourceNameError""" with pytest.raises(DuplicateResourceNameError): run_dbt(["parse"]) class TestDuplicateNamesRequireUniqueResourceNamesTrueDifferentPackages(BaseTestDuplicateNames): @pytest.fixture(scope="class") def project_config_update(self): return { "flags": { "require_unique_project_resource_names": True, } } @pytest.fixture(scope="class") def seeds(self): # Seed in local dep instead return {} @pytest.fixture(scope="class") def packages(self): return {"packages": [{"local": "local_dependency"}]} @pytest.fixture(scope="class", autouse=True) def setUp(self, project_root): local_dependency_files = { "dbt_project.yml": local_dependency__dbt_project_yml, "seeds": {"same_name.csv": seed_csv}, } write_project_files(project_root, "local_dependency", local_dependency_files) def test_duplicate_names_with_flag_enabled_different_packages(self, project): run_dbt(["deps"]) # Behavior flag is true, however, the seed is in a different package. # No error is raised when encountering duplicate names between different packages. manifest = run_dbt(["parse"]) assert len(manifest.nodes) == 2 assert "model.test.same_name" in manifest.nodes assert "seed.local_dep.same_name" in manifest.nodes class TestDuplicateNamesRequireUniqueResourceNamesFalse(BaseTestDuplicateNames): @pytest.fixture(scope="class") def project_config_update(self): return { "flags": { "require_unique_project_resource_names": False, } } def test_duplicate_names_with_flag_disabled(self, project, set_up_deprecations): """When require_unique_project_resource_names is False, duplicate unversioned names should be allowed (continue behavior)""" manifest = run_dbt(["parse"]) assert ( manifest.nodes["model.test.same_name"].name == manifest.nodes["seed.test.same_name"].name ) assert "duplicate-name-distinct-node-types-deprecation" in deprecations.active_deprecations class TestDuplicateNamesDefaultBehavior(TestDuplicateNamesRequireUniqueResourceNamesFalse): @pytest.fixture(scope="class") def project_config_update(self): return {} class TestDuplicateNamesDifferentResourceTypesVersionedUnversioned(BaseTestDuplicateNames): @pytest.fixture(scope="class") def models(self): return { "same_name.sql": model_sql, "schema.yml": versioned_model_yml, } def test_duplicate_names_versioned_unversioned(self, project): # DuplicateVersionedUnversionedError is not raised because parsing fails upstream. # However, parsing still fails with an AssertionError because versioning is attempted on a non-model node. with pytest.raises(AssertionError): run_dbt(["parse"]) ================================================ FILE: tests/functional/duplicates/test_duplicate_source.py ================================================ import pytest from dbt.exceptions import CompilationError from dbt.tests.util import run_dbt source_dupes_schema_yml = """ version: 2 sources: - name: something tables: - name: dupe - name: dupe """ class TestDuplicateSourceEnabled: @pytest.fixture(scope="class") def models(self): return {"schema.yml": source_dupes_schema_yml} def test_duplicate_source_enabled(self, project): message = "dbt found two sources with the name" with pytest.raises(CompilationError) as exc: run_dbt(["compile"]) assert message in str(exc.value) ================================================ FILE: tests/functional/events/events.py ================================================ import os from dbt.cli.main import dbtRunner from dbt_common.events.base_types import EventLevel def test_performance_report(project): resource_report_level = None def check_for_report(e): # If we see a ResourceReport event, record its level if e.info.name == "ResourceReport": nonlocal resource_report_level resource_report_level = e.info.level runner = dbtRunner(callbacks=[check_for_report]) runner.invoke(["run"]) # With not cli flag or env var set, ResourceReport should be debug level. assert resource_report_level == EventLevel.DEBUG try: os.environ["DBT_SHOW_RESOURCE_REPORT"] = "1" runner.invoke(["run"]) # With the appropriate env var set, ResourceReport should be info level. # This allows this fairly technical log line to be omitted by default # but still available in production scenarios. assert resource_report_level == EventLevel.INFO finally: del os.environ["DBT_SHOW_RESOURCE_REPORT"] ================================================ FILE: tests/functional/exit_codes/fixtures.py ================================================ import pytest bad_sql = """ select bad sql here """ dupe_sql = """ select 1 as id, current_date as updated_at union all select 2 as id, current_date as updated_at union all select 3 as id, current_date as updated_at union all select 4 as id, current_date as updated_at """ good_sql = """ select 1 as id, current_date as updated_at union all select 2 as id, current_date as updated_at union all select 3 as id, current_date as updated_at union all select 4 as id, current_date as updated_at """ snapshots_good_sql = """ {% snapshot good_snapshot %} {{ config(target_schema=schema, target_database=database, strategy='timestamp', unique_key='id', updated_at='updated_at')}} select * from {{ schema }}.good {% endsnapshot %} """ snapshots_bad_sql = """ {% snapshot good_snapshot %} {{ config(target_schema=schema, target_database=database, strategy='timestamp', unique_key='id', updated_at='updated_at_not_real')}} select * from {{ schema }}.good {% endsnapshot %} """ schema_yml = """ version: 2 models: - name: good columns: - name: updated_at data_tests: - not_null - name: bad columns: - name: updated_at data_tests: - not_null - name: dupe columns: - name: updated_at data_tests: - unique """ data_seed_good_csv = """a,b,c 1,2,3 """ data_seed_bad_csv = """a,b,c 1,\2,3,a,a,a """ class BaseConfigProject: @pytest.fixture(scope="class") def models(self): return { "bad.sql": bad_sql, "dupe.sql": dupe_sql, "good.sql": good_sql, "schema.yml": schema_yml, } ================================================ FILE: tests/functional/exit_codes/test_exit_codes.py ================================================ import pytest import dbt.exceptions from dbt.tests.util import check_table_does_exist, check_table_does_not_exist, run_dbt from tests.functional.exit_codes.fixtures import ( BaseConfigProject, data_seed_bad_csv, data_seed_good_csv, snapshots_bad_sql, snapshots_good_sql, ) class TestExitCodes(BaseConfigProject): @pytest.fixture(scope="class") def snapshots(self): return {"g.sql": snapshots_good_sql} def test_exit_code_run_succeed(self, project): results = run_dbt(["run", "--model", "good"]) assert len(results) == 1 check_table_does_exist(project.adapter, "good") def test_exit_code_run_fail(self, project): results = run_dbt(["run", "--model", "bad"], expect_pass=False) assert len(results) == 1 check_table_does_not_exist(project.adapter, "bad") def test_schema_test_pass(self, project): results = run_dbt(["run", "--model", "good"]) assert len(results) == 1 results = run_dbt(["test", "--model", "good"]) assert len(results) == 1 def test_schema_test_fail(self, project): results = run_dbt(["run", "--model", "dupe"]) assert len(results) == 1 results = run_dbt(["test", "--model", "dupe"], expect_pass=False) assert len(results) == 1 def test_compile(self, project): results = run_dbt(["compile"]) assert len(results) == 7 def test_snapshot_pass(self, project): run_dbt(["run", "--model", "good"]) results = run_dbt(["snapshot"]) assert len(results) == 1 check_table_does_exist(project.adapter, "good_snapshot") class TestExitCodesSnapshotFail(BaseConfigProject): @pytest.fixture(scope="class") def snapshots(self): return {"b.sql": snapshots_bad_sql} def test_snapshot_fail(self, project): results = run_dbt(["run", "--model", "good"]) assert len(results) == 1 results = run_dbt(["snapshot"], expect_pass=False) assert len(results) == 1 check_table_does_not_exist(project.adapter, "good_snapshot") class TestExitCodesDeps: @pytest.fixture(scope="class") def packages(self): return { "packages": [ { "git": "https://github.com/dbt-labs/dbt-integration-project", "revision": "dbt/1.0.0", } ] } def test_deps(self, project): results = run_dbt(["deps"]) assert results is None class TestExitCodesDepsFail: @pytest.fixture(scope="class") def packages(self): return { "packages": [ { "git": "https://github.com/dbt-labs/dbt-integration-project", "revision": "bad-branch", }, ] } def test_deps_fail(self, project): with pytest.raises(dbt.exceptions.GitCheckoutError) as exc: run_dbt(["deps"]) expected_msg = "Error checking out spec='bad-branch'" assert expected_msg in str(exc.value) class TestExitCodesSeed: @pytest.fixture(scope="class") def seeds(self): return {"good.csv": data_seed_good_csv} def test_seed(self, project): results = run_dbt(["seed"]) assert len(results) == 1 class TestExitCodesSeedFail: @pytest.fixture(scope="class") def seeds(self): return {"bad.csv": data_seed_bad_csv} def test_seed(self, project): run_dbt(["seed"], expect_pass=False) ================================================ FILE: tests/functional/experimental_parser/test_all_experimental_parser.py ================================================ import os import pytest from dbt.artifacts.resources import RefArgs from dbt.contracts.graph.manifest import Manifest from dbt.tests.util import run_dbt def get_manifest(): path = "./target/partial_parse.msgpack" if os.path.exists(path): with open(path, "rb") as fp: manifest_mp = fp.read() manifest: Manifest = Manifest.from_msgpack(manifest_mp) return manifest else: return None basic__schema_yml = """ version: 2 sources: - name: my_src schema: "{{ target.schema }}" tables: - name: my_tbl models: - name: model_a columns: - name: fun """ basic__model_a_sql = """ {{ config(tags='hello', x=False) }} {{ config(tags='world', x=True) }} select * from {{ ref('model_b') }} cross join {{ source('my_src', 'my_tbl') }} where false as boop """ basic__model_b_sql = """ select 1 as fun """ class BasicExperimentalParser: @pytest.fixture(scope="class") def models(self): return { "model_a.sql": basic__model_a_sql, "model_b.sql": basic__model_b_sql, "schema.yml": basic__schema_yml, } class TestBasicExperimentalParser(BasicExperimentalParser): # test that the experimental parser extracts some basic ref, source, and config calls. def test_experimental_parser_basic( self, project, ): run_dbt(["--use-experimental-parser", "parse"]) manifest = get_manifest() node = manifest.nodes["model.test.model_a"] assert node.refs == [RefArgs(name="model_b")] assert node.sources == [["my_src", "my_tbl"]] assert node.config._extra == {"x": True} assert node.config.tags == ["hello", "world"] class TestBasicStaticParser(BasicExperimentalParser): # test that the static parser extracts some basic ref, source, and config calls by default # without the experimental flag and without rendering jinja def test_static_parser_basic(self, project): run_dbt(["--debug", "parse"]) manifest = get_manifest() node = manifest.nodes["model.test.model_a"] assert node.refs == [RefArgs(name="model_b")] assert node.sources == [["my_src", "my_tbl"]] assert node.config._extra == {"x": True} assert node.config.tags == ["hello", "world"] class TestBasicNoStaticParser(BasicExperimentalParser): # test that the static parser doesn't run when the flag is set def test_static_parser_is_disabled(self, project): run_dbt(["--debug", "--no-static-parser", "parse"]) manifest = get_manifest() node = manifest.nodes["model.test.model_a"] assert node.refs == [RefArgs(name="model_b")] assert node.sources == [["my_src", "my_tbl"]] assert node.config._extra == {"x": True} assert node.config.tags == ["hello", "world"] ================================================ FILE: tests/functional/exposures/fixtures.py ================================================ models_sql = """ select 1 as id """ second_model_sql = """ select 1 as id """ metricflow_time_spine_sql = """ SELECT to_date('02/20/2023', 'mm/dd/yyyy') as date_day """ source_schema_yml = """version: 2 sources: - name: test_source tables: - name: test_table """ semantic_models_schema_yml = """version: 2 semantic_models: - name: semantic_model model: ref('model') dimensions: - name: created_at type: time measures: - name: distinct_metrics agg: count_distinct expr: id entities: - name: model type: primary expr: id defaults: agg_time_dimension: created_at """ metrics_schema_yml = """version: 2 metrics: - name: metric label: "label" type: simple type_params: measure: "distinct_metrics" """ simple_exposure_yml = """ version: 2 exposures: - name: simple_exposure label: simple exposure label type: dashboard depends_on: - ref('model') - source('test_source', 'test_table') - metric('metric') owner: email: something@example.com - name: notebook_exposure type: notebook depends_on: - ref('model') - ref('second_model') owner: email: something@example.com name: Some name description: > A description of the complex exposure maturity: medium meta: tool: 'my_tool' languages: - python tags: ['my_department'] url: http://example.com/notebook/1 """ disabled_models_exposure_yml = """ version: 2 exposures: - name: simple_exposure type: dashboard config: enabled: False depends_on: - ref('model') owner: email: something@example.com - name: notebook_exposure type: notebook depends_on: - ref('model') - ref('second_model') owner: email: something@example.com name: Some name description: > A description of the complex exposure maturity: medium meta: tool: 'my_tool' languages: - python tags: ['my_department'] url: http://example.com/notebook/1 """ enabled_yaml_level_exposure_yml = """ version: 2 exposures: - name: simple_exposure type: dashboard config: enabled: True tags: ['local_tag', 'common_tag'] meta: some_key: 'some_value' type_change: 123 depends_on: - ref('model') owner: email: something@example.com - name: notebook_exposure type: notebook depends_on: - ref('model') - ref('second_model') owner: email: something@example.com name: Some name description: > A description of the complex exposure maturity: medium meta: tool: 'my_tool' languages: - python tags: ['my_department'] url: http://example.com/notebook/1 """ invalid_config_exposure_yml = """ version: 2 exposures: - name: simple_exposure type: dashboard config: enabled: True and False depends_on: - ref('model') owner: email: something@example.com """ ================================================ FILE: tests/functional/exposures/test_exposure_configs.py ================================================ import pytest from dbt.artifacts.resources import ExposureConfig from dbt.tests.util import get_manifest, run_dbt, update_config_file from dbt_common.dataclass_schema import ValidationError from tests.functional.exposures.fixtures import ( disabled_models_exposure_yml, enabled_yaml_level_exposure_yml, invalid_config_exposure_yml, metricflow_time_spine_sql, metrics_schema_yml, models_sql, second_model_sql, semantic_models_schema_yml, simple_exposure_yml, source_schema_yml, ) # Test enabled config for exposure in dbt_project.yml class TestExposureEnabledConfigProjectLevel: @pytest.fixture(scope="class") def models(self): return { "model.sql": models_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "second_model.sql": second_model_sql, "exposure.yml": simple_exposure_yml, "schema.yml": source_schema_yml, "semantic_models.yml": semantic_models_schema_yml, "metrics.yml": metrics_schema_yml, } @pytest.fixture(scope="class") def project_config_update(self): return { "exposures": { "simple_exposure": { "enabled": True, }, } } def test_enabled_exposure_config_dbt_project(self, project): run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "exposure.test.simple_exposure" in manifest.exposures new_enabled_config = { "exposures": { "test": { "simple_exposure": { "enabled": False, }, } } } update_config_file(new_enabled_config, project.project_root, "dbt_project.yml") run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "exposure.test.simple_exposure" not in manifest.exposures assert "exposure.test.notebook_exposure" in manifest.exposures # Test disabled config at exposure level in yml file class TestConfigYamlLevel: @pytest.fixture(scope="class") def models(self): return { "model.sql": models_sql, "second_model.sql": second_model_sql, "schema.yml": disabled_models_exposure_yml, } def test_exposure_config_yaml_level(self, project): run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "exposure.test.simple_exposure" not in manifest.exposures assert "exposure.test.notebook_exposure" in manifest.exposures # Test inheritence - set configs at project and exposure level - expect exposure level to win class TestExposureConfigsInheritence: @pytest.fixture(scope="class") def models(self): return { "model.sql": models_sql, "second_model.sql": second_model_sql, "schema.yml": enabled_yaml_level_exposure_yml, } @pytest.fixture(scope="class") def project_config_update(self): return { "exposures": { "enabled": False, "tags": ["global_tag", "common_tag"], "meta": { "some_key": "should_be_overridden", "another_key": "should_stay", "type_change": ["foo", "bar"], }, } } @pytest.fixture(scope="class") def expected_config(self): return ExposureConfig( enabled=True, tags=["common_tag", "global_tag", "local_tag"], meta={"some_key": "some_value", "another_key": "should_stay", "type_change": 123}, ) def test_exposure_all_configs(self, project, expected_config): run_dbt(["parse"]) manifest = get_manifest(project.project_root) # This should be overridden assert "exposure.test.simple_exposure" in manifest.exposures # This should stay disabled assert "exposure.test.notebook_exposure" not in manifest.exposures exposure = manifest.exposures.get("exposure.test.simple_exposure") assert exposure.tags == expected_config.tags assert exposure.meta == expected_config.meta assert isinstance(exposure.config, ExposureConfig) assert exposure.config.enabled == expected_config.enabled assert exposure.config.tags == expected_config.tags assert exposure.config.meta == expected_config.meta # Test invalid config triggers error class TestInvalidConfig: @pytest.fixture(scope="class") def models(self): return { "model.sql": models_sql, "second_model.sql": second_model_sql, "schema.yml": invalid_config_exposure_yml, } def test_exposure_config_yaml_level(self, project): with pytest.raises(ValidationError) as excinfo: run_dbt(["parse"]) expected_msg = "'True and False' is not of type 'boolean'" assert expected_msg in str(excinfo.value) ================================================ FILE: tests/functional/exposures/test_exposures.py ================================================ import pytest from dbt.artifacts.schemas.results import RunStatus from dbt.contracts.graph.nodes import Exposure from dbt.tests.util import get_manifest, run_dbt from tests.functional.exposures.fixtures import ( metricflow_time_spine_sql, metrics_schema_yml, models_sql, second_model_sql, semantic_models_schema_yml, simple_exposure_yml, source_schema_yml, ) class TestBasicExposures: @pytest.fixture(scope="class") def models(self): return { "exposure.yml": simple_exposure_yml, "model.sql": models_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "second_model.sql": second_model_sql, "schema.yml": source_schema_yml, "semantic_models.yml": semantic_models_schema_yml, "metrics.yml": metrics_schema_yml, } def test_compilation_names_with_spaces(self, project): run_dbt(["compile"]) manifest = get_manifest(project.project_root) exposure_ids = list(manifest.exposures.keys()) expected_exposure_ids = [ "exposure.test.simple_exposure", "exposure.test.notebook_exposure", ] assert exposure_ids == expected_exposure_ids assert manifest.exposures["exposure.test.simple_exposure"].label == "simple exposure label" def test_compilation_depends_on(self, project): run_dbt(["compile"]) manifest = get_manifest(project.project_root) exposure_depends_on = manifest.exposures["exposure.test.simple_exposure"].depends_on.nodes expected_exposure_depends_on = [ "source.test.test_source.test_table", "model.test.model", "metric.test.metric", ] assert sorted(exposure_depends_on) == sorted(expected_exposure_depends_on) def test_execution_default(self, project): results = run_dbt(["build"]) exposure_results = ( result for result in results.results if isinstance(result.node, Exposure) ) assert {result.node.name for result in exposure_results} == { "simple_exposure", "notebook_exposure", } assert all(result.status == RunStatus.NoOp for result in exposure_results) assert all("NO-OP" in result.message for result in exposure_results) def test_execution_exclude(self, project): results = run_dbt(["build", "--exclude", "simple_exposure"]) exposure_results = ( result for result in results.results if isinstance(result.node, Exposure) ) assert {result.node.name for result in exposure_results} == {"notebook_exposure"} def test_execution_select(self, project): results = run_dbt(["build", "--select", "simple_exposure"]) exposure_results = ( result for result in results.results if isinstance(result.node, Exposure) ) assert {result.node.name for result in exposure_results} == {"simple_exposure"} ================================================ FILE: tests/functional/external_reference/test_external_reference.py ================================================ import pytest from dbt.tests.util import run_dbt external_model_sql = """ {{ config( materialized = "view" ) }} select * from "{{ this.schema + 'z' }}"."external" """ model_sql = """ select 1 as id """ class TestExternalReference: @pytest.fixture(scope="class") def models(self): return {"model.sql": external_model_sql} def test_external_reference(self, project, unique_schema): external_schema = unique_schema + "z" project.run_sql(f'create schema "{external_schema}"') project.run_sql(f'create table "{external_schema}"."external" (id integer)') project.run_sql(f'insert into "{external_schema}"."external" values (1), (2)') results = run_dbt(["run"]) assert len(results) == 1 # running it again should succeed results = run_dbt(["run"]) assert len(results) == 1 # The opposite of the test above -- check that external relations that # depend on a dbt model do not create issues with caching class TestExternalDependency: @pytest.fixture(scope="class") def models(self): return {"model.sql": model_sql} def test_external_reference(self, project, unique_schema): results = run_dbt(["run"]) assert len(results) == 1 external_schema = unique_schema + "z" project.run_sql(f'create schema "{external_schema}"') project.run_sql( f'create view "{external_schema}"."external" as (select * from {unique_schema}.model)' ) # running it again should succeed results = run_dbt(["run"]) assert len(results) == 1 ================================================ FILE: tests/functional/fail_fast/test_fail_fast_run.py ================================================ import json from pathlib import Path import pytest from dbt.tests.util import run_dbt models__one_sql = """ select 1 """ models__two_sql = """ -- depends_on: {{ ref('one') }} select 1 /failed """ class FailFastBase: @pytest.fixture(scope="class") def models(self): return {"one.sql": models__one_sql, "two.sql": models__two_sql} class TestFastFailingDuringRun(FailFastBase): def test_fail_fast_run( self, project, models, # noqa: F811 ): res = run_dbt(["run", "--fail-fast", "--threads", "1"], expect_pass=False) assert {r.node.unique_id: r.status for r in res.results} == { "model.test.one": "success", "model.test.two": "error", } run_results_file = Path(project.project_root) / "target/run_results.json" assert run_results_file.is_file() with run_results_file.open() as run_results_str: run_results = json.loads(run_results_str.read()) assert len(run_results["results"]) == 2 assert run_results["results"][0]["status"] == "success" assert run_results["results"][1]["status"] == "error" class TestFailFastFromConfig(FailFastBase): @pytest.fixture(scope="class") def project_config_update(self): return { "flags": { "send_anonymous_usage_stats": False, "fail_fast": True, } } def test_fail_fast_run_project_flags( self, project, models, # noqa: F811 ): res = run_dbt(["run", "--threads", "1"], expect_pass=False) assert {r.node.unique_id: r.status for r in res.results} == { "model.test.one": "success", "model.test.two": "error", } ================================================ FILE: tests/functional/fixtures/__init__.py ================================================ ================================================ FILE: tests/functional/fixtures/happy_path_fixture.py ================================================ import os from distutils.dir_util import copy_tree import pytest def delete_files_in_directory(directory_path): try: with os.scandir(directory_path) as entries: for entry in entries: if entry.is_file(): os.unlink(entry.path) print("All files deleted successfully.") except OSError: print("Error occurred while deleting files.") @pytest.fixture(scope="class") def happy_path_project_files(project_root): # copy fixture files to the project root delete_files_in_directory(project_root) copy_tree( os.path.dirname(os.path.realpath(__file__)) + "/happy_path_project", str(project_root) ) # We do project_setup first because it will write out a dbt_project.yml. # This file will be overwritten by the files in happy_path_project later on. @pytest.fixture(scope="class") def happy_path_project(project_setup, happy_path_project_files): # A fixture that gives functional test the project living in happy_path_project return project_setup ================================================ FILE: tests/functional/fixtures/happy_path_project/analyses/a.sql ================================================ select 4 as id ================================================ FILE: tests/functional/fixtures/happy_path_project/analyses/a.yml ================================================ analyses: - name: a description: description config: tags: ["tag"] enabled: true meta: { "test": 1 } docs: show: true node_color: purple group: "finance" columns: - name: id description: id description ================================================ FILE: tests/functional/fixtures/happy_path_project/dbt_project.yml ================================================ analysis-paths: - analyses config-version: 2 flags: send_anonymous_usage_stats: false state_modified_compare_more_unrendered_values: true macro-paths: - macros name: test profile: test seed-paths: - seeds models: +enabled: true test: +enabled: true sub: +enabled: true inner: +enabled: true seeds: +quote_columns: true test: +quote_columns: false snapshot-paths: - snapshots test-paths: - tests ================================================ FILE: tests/functional/fixtures/happy_path_project/functions/area_of_circle.sql ================================================ SELECT pi() * radius * radius ================================================ FILE: tests/functional/fixtures/happy_path_project/functions/area_of_circle.yml ================================================ functions: - name: area_of_circle description: Calculates the area of a circle for a given radius arguments: - name: radius data_type: float description: A floating point number representing the radius of the circle returns: data_type: float ================================================ FILE: tests/functional/fixtures/happy_path_project/macros/expression_is_true.sql ================================================ {% test expression_is_true(model, expression, column_name=None) %} {% set column_list = '*' if should_store_failures() else "1" %} select {{ column_list }} from {{ model }} {% if column_name is none %} where not({{ expression }}) {%- else %} where not({{ column_name }} {{ expression }}) {%- endif %} {% endtest %} ================================================ FILE: tests/functional/fixtures/happy_path_project/macros/macro_stuff.sql ================================================ {% macro cool_macro() %} wow! {% endmacro %} {% macro other_cool_macro(a, b) %} cool! {% endmacro %} ================================================ FILE: tests/functional/fixtures/happy_path_project/models/docs.md ================================================ {% docs my_docs %} some docs {% enddocs %} ================================================ FILE: tests/functional/fixtures/happy_path_project/models/e.yml ================================================ exposures: - name: weekly_jaffle_metrics label: Jaffles by the Week type: dashboard maturity: high url: https://bi.tool/dashboards/1 description: > Did someone say "exponential growth"? depends_on: - ref('incremental') owner: name: Callum McData email: data@jaffleshop.com config: enabled: true meta: {} tags: ["tag"] ================================================ FILE: tests/functional/fixtures/happy_path_project/models/ephemeral.sql ================================================ {{ config(materialized='ephemeral') }} select 1 as id, {{ dbt.date_trunc('day', dbt.current_timestamp()) }} as created_at ================================================ FILE: tests/functional/fixtures/happy_path_project/models/g.yml ================================================ groups: - name: finance description: description config: meta: custom_field: value owner: # one of 'name' or 'email' is required; email: finance@jaffleshop.com ================================================ FILE: tests/functional/fixtures/happy_path_project/models/incremental.sql ================================================ {{ config( materialized = "incremental", incremental_strategy = "delete+insert", ) }} select * from {{ ref('seed') }} {% if is_incremental() %} where a > (select max(a) from {{this}}) {% endif %} ================================================ FILE: tests/functional/fixtures/happy_path_project/models/m.yml ================================================ metrics: - name: total_outer type: simple description: The total count of outer label: Total Outer type_params: measure: total_outer_count time_granularity: month - name: simple_ratio_metric description: a simple ratio metric type: ratio label: Simple Ratio Metric type_params: numerator: total_outer denominator: name: total_outer alias: filtered_total_outer - name: filtered_ratio_metric description: a ratio metric type: ratio label: Ratio Metric 2 type_params: numerator: name: total_outer filter: 1 = 1 denominator: name: total_outer filter: - 1 = 1 - 2 = 2 alias: filtered_total_outer_again - name: cumulative_metric description: a cumulative metric type: cumulative label: Cumulative Metric type_params: measure: name: total_outer_count fill_nulls_with: 0 join_to_timespine: false filter: 1 = 1 alias: filtered_total_outer_count cumulative_type_params: grain_to_date: day period_agg: first - name: cumulative_metric_2 description: a cumulative metric type: cumulative label: Cumulative Metric 2 type_params: measure: name: total_outer_count fill_nulls_with: 0 join_to_timespine: false filter: 1 = 1 alias: filtered_total_outer_count_2 cumulative_type_params: period_agg: first window: 1 day - name: conversion_metric description: a conversion metric type: conversion label: Conversion Metric type_params: conversion_type_params: entity: my_entity calculation: conversion_rate base_measure: total_outer_count conversion_measure: name: total_outer_count fill_nulls_with: 0 join_to_timespine: false window: 1 day constant_properties: - base_property: my_entity conversion_property: created_at - name: derived_metric description: a derived metric type: derived label: Derived Metric type_params: expr: srm - cm + filtered_ratio_metric metrics: - name: simple_ratio_metric alias: srm filter: 1 = 1 offset_window: 1 month - name: conversion_metric alias: cm offset_to_grain: month - filtered_ratio_metric ================================================ FILE: tests/functional/fixtures/happy_path_project/models/macros.yml ================================================ macros: - name: some_macro description: a description config: meta: owner: "person@gmail.com" ================================================ FILE: tests/functional/fixtures/happy_path_project/models/metricflow_time_spine.sql ================================================ select {{ dbt.date_trunc('day', dbt.current_timestamp()) }} as date_day ================================================ FILE: tests/functional/fixtures/happy_path_project/models/metricflow_time_spine_second.sql ================================================ select {{ dbt.date_trunc('second', dbt.current_timestamp()) }} as ts_second ================================================ FILE: tests/functional/fixtures/happy_path_project/models/model_to_unit_test.sql ================================================ {{ config(materialized='table') }} SELECT * FROM {{ ref('seed')}} ================================================ FILE: tests/functional/fixtures/happy_path_project/models/model_with_lots_of_schema_configs.sql ================================================ select * from {{ ref('ephemeral') }} ================================================ FILE: tests/functional/fixtures/happy_path_project/models/outer.sql ================================================ select * from {{ ref('ephemeral') }} ================================================ FILE: tests/functional/fixtures/happy_path_project/models/schema.yml ================================================ version: 2 models: - name: outer description: The outer table columns: - name: id description: The id value data_tests: - unique - not_null - name: metricflow_time_spine description: Day time spine time_spine: standard_granularity_column: date_day columns: - name: date_day granularity: day config: tags: - "list" - "of" - "tags" pre_hook: - "SELECT 'string_pre_hook' as my_pre_hook;" post_hook: - "SELECT 'string_post_hook' as my_post_hook;" group: "finance" - name: metricflow_time_spine_second description: Second time spine config: event_time: ts_second columns: - name: ts_second granularity: second - name: model_with_lots_of_schema_configs columns: - name: id description: The id value config: tags: - "column_level_tag" meta: column_meta: "column_meta_value" data_tests: - unique - not_null: name: "my_favorite_test" description: "A test that should pass" config: alias: "not_null__id__alias" database: "dbt" group: "important_tests" enabled: true error_if: "!= 0" fail_calc: "count(*)" limit: 10 meta: my_custom_meta_key: "my_custom_meta_value" schema: "dbt_test__audit" severity: "warn" store_failures: true store_failures_as: "table" tags: - "test_tag" warn_if: "!= 0" where: "1 = 1" - name: created_at description: The date the row was created data_tests: - my_generic_test: name: "my_second_favorite_test" description: "A test that should pass" config: alias: "my_generic_test__created_at__alias" database: "dbt" group: "important_tests" enabled: true error_if: "!= 0" fail_calc: "count(*)" limit: 10 meta: my_custom_meta_key: "my_custom_meta_value" schema: "dbt_test__audit" severity: "warn" store_failures: true store_failures_as: "table" tags: - "test_tag" warn_if: "!= 0" where: "1 = 1" config: access: public alias: "outer_alias" batch_size: day begin: "2020-01-01" concurrent_batches: false contract: alias_types: true enforce: true docs: node_color: purple show: true database: "dbt" enabled: true full_refresh: false grants: select: ["root"] lookback: 5 materialized: table meta: my_custom_property: "string_meta" on_configuration_change: apply persist_docs: columns: true relation: true post_hook: "SELECT 'string_post_hook' as my_post_hook;" pre_hook: "SELECT 'string_pre_hook' as my_pre_hook;" schema: "test" sql_header: "SELECT 1 as header;" tags: "string_tag" unique_key: id constraints: - type: primary_key columns: ["id"] warn_unsupported: false - type: foreign_key columns: ["id"] to: source('my_source', 'my_table') to_columns: ["id"] - type: check columns: ["id"] expression: "id > 0" name: "Check that id is greater than 0" deprecation_date: "2052-05-01 00:00:00-04:00" description: A model with lots of configs sources: - name: my_source description: description database: raw schema: jaffle_shop loader: test config: enabled: true event_time: column_name loaded_at_field: column_name meta: source_meta: 1 tags: ["source_tag"] freshness: warn_after: count: 1 period: minute error_after: count: 2 period: hour filter: "column_name = 1" # # overrides: test quoting: database: true schema: true identifier: true tables: - name: my_table description: description identifier: table_identifier tests: [] quoting: database: true schema: true identifier: true external: location: location file_format: file_format row_format: row_format tbl_properties: tbl_properties partitions: - name: column_name data_type: data_type description: description meta: test: 1 additional_property: additional_value columns: - name: column_name description: description quote: true tests: [] config: meta: test: 1 tags: ["column_tag"] config: enabled: true event_time: column_name loaded_at_field: column_name meta: table_meta: 1 tags: ["table_tag"] freshness: warn_after: count: 1 period: minute error_after: count: 2 period: hour filter: "column_name = 1" ================================================ FILE: tests/functional/fixtures/happy_path_project/models/sm.yml ================================================ semantic_models: - name: my_sm model: ref('outer') defaults: agg_time_dimension: created_at entities: - name: my_entity type: primary expr: id - name: user type: foreign expr: user_id dimensions: - name: created_at type: time type_params: time_granularity: day measures: - name: total_outer_count agg: count expr: 1 - name: discrete_order_value_p99 expr: order_total agg: percentile agg_time_dimension: created_at agg_params: percentile: 0.99 use_discrete_percentile: True use_approximate_percentile: False create_metric: true - name: sum_of_things expr: 2 agg: sum agg_time_dimension: created_at - name: has_revenue expr: true agg: sum_boolean agg_time_dimension: created_at - name: test_non_additive expr: txn_revenue agg: sum non_additive_dimension: name: created_at window_choice: max ================================================ FILE: tests/functional/fixtures/happy_path_project/models/snapshot_source.sql ================================================ select 0 as id, 1 as col1, 2 as col2, 3 as col3 ================================================ FILE: tests/functional/fixtures/happy_path_project/models/sq.yml ================================================ saved_queries: - name: my_saved_query label: My Saved Query query_params: metrics: - total_outer group_by: - "Dimension('my_entity__created_at')" exports: - name: my_export config: alias: my_export_alias export_as: table schema: my_export_schema_name ================================================ FILE: tests/functional/fixtures/happy_path_project/models/sub/inner.sql ================================================ select * from {{ ref('outer') }} ================================================ FILE: tests/functional/fixtures/happy_path_project/models/unit_tests.yml ================================================ unit_tests: - name: test_model_to_unit_test description: "A simple test of the outer model" model: model_to_unit_test given: - input: ref('seed') format: csv fixture: test_incremental_fixture expect: format: csv fixture: test_incremental_fixture - name: test_model_to_unit_test_2 description: "A simple test of the outer model" model: model_to_unit_test versions: include: exclude: config: enabled: true meta: my_custom_meta_key: "my_custom_meta_value" tags: - "test_tag" given: - input: ref('seed') format: sql rows: | SELECT 2 as a, 3 as b expect: format: dict rows: - {a: 2, b: 3} overrides: macros: is_incremental: false vars: {} env_vars: {} ================================================ FILE: tests/functional/fixtures/happy_path_project/seeds/s.yml ================================================ seeds: - name: seed description: 'test_description' data_tests: - expression_is_true: arguments: expression: "b = 2" columns: - name: a description: a description quote: true tests: - not_null config: tags: ["tag"] meta: {} - name: b description: b description quote: true tests: - not_null config: meta: {} tags: ["tag"] config: docs: show: true node_color: purple quote_columns: false column_types: a: BIGINT delimiter: "," enabled: true tags: "tag" pre_hook: "select 1" post_hook: "select 1" database: "dbt" schema: "test" alias: test_alias persist_docs: {} full_refresh: true meta: {'meta_key': 'meta_value'} grants: {} event_time: my_time_field group: finance ================================================ FILE: tests/functional/fixtures/happy_path_project/seeds/seed.csv ================================================ a,b 1,2 ================================================ FILE: tests/functional/fixtures/happy_path_project/snapshots/snapshot.sql ================================================ {% snapshot my_snapshot %} {{ config( database=var('target_database', database), schema=schema, unique_key='id', strategy='timestamp', updated_at='updated_at', ) }} select * from {{database}}.{{schema}}.seed {% endsnapshot %} ================================================ FILE: tests/functional/fixtures/happy_path_project/snapshots/snapshot_2.yml ================================================ version: 2 snapshots: - name: snapshot_2 relation: ref("snapshot_source") description: Description of snapshot_2 config: database: mydb schema: myschema unique_key: id strategy: check updated_at: updated_at check_cols: [ "alpha", "beta", "delta" ] snapshot_meta_column_names: dbt_valid_from: my_from_col dbt_valid_to: my_valid_to_col dbt_scd_id: my_scd_col dbt_updated_at: my_updated_at_col dbt_is_deleted: my_is_deleted_col hard_deletes: new_record dbt_valid_to_current: valid_to_current meta: owner: "@alice" maturity: in dev docs: sfhow: true node_color: purple ================================================ FILE: tests/functional/fixtures/happy_path_project/snapshots/snapshot_3.yml ================================================ version: 2 snapshots: - name: snapshot_3 relation: ref("snapshot_source") description: Description of snapshot_3 config: database: mydb schema: myschema unique_key: id strategy: timestamp updated_at: updated_at snapshot_meta_column_names: dbt_valid_from: my_from_col dbt_valid_to: my_valid_to_col dbt_scd_id: my_scd_col dbt_updated_at: my_updated_at_col dbt_is_deleted: my_is_deleted_col invalidate_hard_deletes: true dbt_valid_to_current: valid_to_current meta: owner: "@alice" maturity: in dev docs: sfhow: true node_color: purple ================================================ FILE: tests/functional/fixtures/happy_path_project/tests/fixtures/test_incremental_fixture.csv ================================================ a,b 1,2 2,3 ================================================ FILE: tests/functional/fixtures/happy_path_project/tests/generic/my_generic_test.sql ================================================ {% test my_generic_test(model, column_name) %} SELECT 1 as a_column WHERE 1 = 2 {% endtest %} ================================================ FILE: tests/functional/fixtures/happy_path_project/tests/schema.yml ================================================ version: 2 data_tests: - name: t description: "Single Data Test in happy path fixture" config: alias: "test_alias" database: "dbt" group: "important_tests" enabled: true error_if: "!= 0" fail_calc: "count(*)" limit: 10 meta: my_custom_meta_key: "my_custom_meta_value" schema: "dbt_test__audit" severity: "warn" store_failures: true store_failures_as: "table" tags: - "test_tag" warn_if: "!= 0" where: "1 = 1" groups: - name: important_tests owner: email: data@example.com ================================================ FILE: tests/functional/fixtures/happy_path_project/tests/t.sql ================================================ select 1 as id WHERE 1 = 2 ================================================ FILE: tests/functional/fixtures.py ================================================ models__my_model_sql = """ with my_cte as ( select 1 as id, 'blue' as color union all select 2 as id, 'green' as red union all select 3 as id, 'red' as red ) select * from my_cte """ models__schema_yml = """ models: - name: my_model columns: - name: id tests: - unique: description: "id must be unique" - not_null - name: color tests: - accepted_values: values: ['blue', 'green', 'red'] description: "{{ doc('color_accepted_values') }}" """ models__doc_block_md = """ {% docs color_accepted_values %} The `color` column must be one of 'blue', 'green', or 'red'. {% enddocs %} """ ================================================ FILE: tests/functional/functions/test_udafs.py ================================================ from typing import Dict import pytest from dbt.artifacts.resources import FunctionReturns from dbt.artifacts.resources.types import FunctionType from dbt.contracts.graph.nodes import FunctionNode from dbt.tests.util import run_dbt double_total_sql = """ SELECT SUM(values) * 2 """ double_total_yml = """ functions: - name: double_total description: Sums the sequence of numbers and then doubles the result config: type: aggregate arguments: - name: values data_type: float description: A sequence of numbers returns: data_type: float """ class BasicUDAFSetup: @pytest.fixture(scope="class") def functions(self) -> Dict[str, str]: return { "double_total.sql": double_total_sql, "double_total.yml": double_total_yml, } class TestBasicSQLUDAF(BasicUDAFSetup): def test_basic_sql_udaf_parsing(self, project): manifest = run_dbt(["parse"]) assert len(manifest.functions) == 1 assert "function.test.double_total" in manifest.functions function_node = manifest.functions["function.test.double_total"] assert isinstance(function_node, FunctionNode) assert ( function_node.description == "Sums the sequence of numbers and then doubles the result" ) assert function_node.config.type == FunctionType.Aggregate assert len(function_node.arguments) == 1 argument = function_node.arguments[0] assert argument.name == "values" assert argument.data_type == "float" assert argument.description == "A sequence of numbers" assert function_node.returns == FunctionReturns(data_type="float") ================================================ FILE: tests/functional/functions/test_udfs.py ================================================ from typing import Dict import agate import pytest from dbt.artifacts.resources import FunctionReturns from dbt.artifacts.resources.types import FunctionType, FunctionVolatility from dbt.contracts.graph.nodes import FunctionNode from dbt.exceptions import ParsingError from dbt.tests.util import run_dbt, write_file double_it_sql = """ SELECT value * 2 """ double_it_py = """ def entry(value): return value * 2 """ double_it_py_with_jinja = """ def entry(value): {% if 1 == 1 %} return value * 2 {% else %} {# this should never happen #} return value * 3 {% endif %} """ double_it_deterministic_sql = """ {{ config(volatility='deterministic') }} SELECT value * 2 """ double_it_deterministic_py = """ {{ config(volatility='deterministic') }} def entry(value): return value * 2 """ double_it_yml = """ functions: - name: double_it description: Doubles whatever number is passed in arguments: - name: value data_type: float description: A number to be doubled returns: data_type: float """ double_it_python_yml = """ functions: - name: double_it description: Doubles whatever number is passed in config: runtime_version: "3.11" entry_point: entry arguments: - name: value data_type: float description: A number to be doubled returns: data_type: float """ double_it_non_deterministic_yml = """ functions: - name: double_it description: Doubles whatever number is passed in config: volatility: non-deterministic arguments: - name: value data_type: float description: A number to be doubled returns: data_type: float """ double_it_non_deterministic_python_yml = """ functions: - name: double_it description: Doubles whatever number is passed in config: volatility: non-deterministic language_version: "3.11" entry_point: entry arguments: - name: value data_type: float description: A number to be doubled returns: data_type: float """ numbers_model_sql = """ SELECT 1 as number UNION ALL SELECT 2 as number UNION ALL SELECT 3 as number """ sum_numbers_function_sql = """ SELECT sum(number) as sum_numbers FROM {{ ref('numbers_model') }} """ sum_numbers_function_yml = """ functions: - name: sum_numbers_function description: Sums the numbers in the numbers_model returns: data_type: integer """ numbers_seed_csv = """number 1 2 3 """ numbers_source_yml = """ sources: - name: test_source schema: "{{ target.schema }}" tables: - name: numbers_seed """ sum_numbers_function_from_source_sql = """ SELECT sum(number) as sum_numbers FROM {{ source('test_source', 'numbers_seed') }} """ class BasicUDFSetup: @pytest.fixture(scope="class") def functions(self) -> Dict[str, str]: return { "double_it.sql": double_it_sql, "double_it.yml": double_it_yml, } scalar_function_python_macro = """ {% macro postgres__scalar_function_python(target_relation) %} SELECT 1; {% endmacro %} """ sum_2_values_sql = """ SELECT val1 + val2 as sum_2_values """ sum_2_values_yml = """ functions: - name: sum_2_values description: Add two values together arguments: - name: val1 data_type: integer description: The first value - name: val2 data_type: integer description: The second value default_value: 0 returns: data_type: integer """ sum_2_values_bad_default_arg_order_yml = """ functions: - name: sum_2_values description: Add two values together arguments: - name: val1 data_type: integer description: The first value default_value: 0 - name: val2 data_type: integer description: The second value returns: data_type: integer """ class TestBasicSQLUDF(BasicUDFSetup): def test_basic_parsing(self, project): # Simple parsing manifest = run_dbt(["parse"]) assert len(manifest.functions) == 1 assert "function.test.double_it" in manifest.functions function_node = manifest.functions["function.test.double_it"] assert isinstance(function_node, FunctionNode) assert function_node.description == "Doubles whatever number is passed in" assert function_node.language == "sql" assert function_node.config.type == FunctionType.Scalar assert function_node.config.volatility is None assert len(function_node.arguments) == 1 argument = function_node.arguments[0] assert argument.name == "value" assert argument.data_type == "float" assert argument.description == "A number to be doubled" assert function_node.returns == FunctionReturns(data_type="float") # Update with volatility specified in sql write_file(double_it_deterministic_sql, project.project_root, "functions", "double_it.sql") manifest = run_dbt(["parse", "--no-partial-parse"]) assert len(manifest.functions) == 1 assert "function.test.double_it" in manifest.functions function_node = manifest.functions["function.test.double_it"] assert function_node.config.volatility == FunctionVolatility.Deterministic # Update with volatility specified in yml write_file( double_it_non_deterministic_yml, project.project_root, "functions", "double_it.yml" ) write_file(double_it_sql, project.project_root, "functions", "double_it.sql") manifest = run_dbt(["parse", "--no-partial-parse"]) assert len(manifest.functions) == 1 assert "function.test.double_it" in manifest.functions function_node = manifest.functions["function.test.double_it"] assert function_node.config.volatility == FunctionVolatility.NonDeterministic class TestBasicPythonUDF(BasicUDFSetup): @pytest.fixture(scope="class") def functions(self) -> Dict[str, str]: return { "double_it.py": double_it_py, "double_it.yml": double_it_python_yml, } def test_basic_parsing(self, project): # Simple parsing manifest = run_dbt(["parse"]) assert len(manifest.functions) == 1 assert "function.test.double_it" in manifest.functions function_node = manifest.functions["function.test.double_it"] assert isinstance(function_node, FunctionNode) assert function_node.description == "Doubles whatever number is passed in" assert function_node.language == "python" assert function_node.config.type == FunctionType.Scalar assert function_node.config.volatility is None assert function_node.config.runtime_version == "3.11" assert function_node.config.entry_point == "entry" assert len(function_node.arguments) == 1 argument = function_node.arguments[0] assert argument.name == "value" assert argument.data_type == "float" assert argument.description == "A number to be doubled" assert function_node.returns == FunctionReturns(data_type="float") # Update with volatility specified in sql write_file(double_it_deterministic_py, project.project_root, "functions", "double_it.py") manifest = run_dbt(["parse", "--no-partial-parse"]) assert len(manifest.functions) == 1 assert "function.test.double_it" in manifest.functions function_node = manifest.functions["function.test.double_it"] assert function_node.config.volatility == FunctionVolatility.Deterministic # Update with volatility specified in yml write_file( double_it_non_deterministic_python_yml, project.project_root, "functions", "double_it.yml", ) write_file(double_it_py, project.project_root, "functions", "double_it.py") manifest = run_dbt(["parse", "--no-partial-parse"]) assert len(manifest.functions) == 1 assert "function.test.double_it" in manifest.functions function_node = manifest.functions["function.test.double_it"] assert function_node.config.volatility == FunctionVolatility.NonDeterministic class TestCreationOfUDFs(BasicUDFSetup): def test_can_create_udf(self, project): results = run_dbt(["build"]) assert len(results) == 1 function_node = results[0].node assert isinstance(function_node, FunctionNode) assert function_node.name == "double_it" assert function_node.description == "Doubles whatever number is passed in" argument = function_node.arguments[0] assert argument.name == "value" assert argument.data_type == "float" assert results[0].node.returns == FunctionReturns(data_type="float") class TestCanInlineShowUDF(BasicUDFSetup): def test_can_inline_show_udf(self, project): run_dbt(["build"]) result = run_dbt(["show", "--inline", "select {{ function('double_it') }}(1)"]) assert len(result.results) == 1 agate_table = result.results[0].agate_table assert isinstance(agate_table, agate.Table) assert agate_table.column_names == ("double_it",) assert agate_table.rows == [(2.0,)] class TestCanCallUDFInModel(BasicUDFSetup): @pytest.fixture(scope="class") def models(self): return { "double_it_model.sql": "select {{ function('double_it') }}(1) as double_it", } def test_can_call_udf_in_model(self, project): run_dbt(["build"]) result = run_dbt(["show", "--select", "double_it_model"]) assert len(result.results) == 1 agate_table = result.results[0].agate_table assert isinstance(agate_table, agate.Table) assert agate_table.column_names == ("double_it",) assert agate_table.rows == [(2.0,)] class TestCanUseWithEmptyMode(BasicUDFSetup): @pytest.fixture(scope="class") def models(self): return { "double_it_model.sql": "select {{ function('double_it') }}(1) as double_it", } def test_can_use_with_empty_model(self, project): run_dbt(["build", "--empty"]) result = run_dbt(["show", "--select", "double_it_model"]) assert len(result.results) == 1 agate_table = result.results[0].agate_table assert isinstance(agate_table, agate.Table) assert agate_table.column_names == ("double_it",) assert agate_table.rows == [(2.0,)] class TestCanUseRefInUDF: @pytest.fixture(scope="class") def functions(self) -> Dict[str, str]: return { "sum_numbers_function.sql": sum_numbers_function_sql, "sum_numbers_function.yml": sum_numbers_function_yml, } @pytest.fixture(scope="class") def models(self) -> Dict[str, str]: return { "numbers_model.sql": numbers_model_sql, } def test_can_use_ref_in_udf(self, project): run_dbt(["build"]) result = run_dbt( [ "show", "--inline", "select {{ function('sum_numbers_function') }}() as summed_numbers", ] ) assert len(result.results) == 1 agate_table = result.results[0].agate_table assert isinstance(agate_table, agate.Table) assert agate_table.column_names == ("summed_numbers",) assert agate_table.rows == [(6,)] class TestCanUseSourceInUDF: @pytest.fixture(scope="class") def functions(self) -> Dict[str, str]: return { "sum_numbers_function.sql": sum_numbers_function_from_source_sql, "sum_numbers_function.yml": sum_numbers_function_yml, } @pytest.fixture(scope="class") def models(self) -> Dict[str, str]: return { "numbers_source.yml": numbers_source_yml, } @pytest.fixture(scope="class") def seeds(self) -> Dict[str, str]: return { "numbers_seed.csv": numbers_seed_csv, } def test_can_use_ref_in_udf(self, project): run_dbt(["seed"]) run_dbt(["build"]) result = run_dbt( [ "show", "--inline", "select {{ function('sum_numbers_function') }}() as summed_numbers", ] ) assert len(result.results) == 1 agate_table = result.results[0].agate_table assert isinstance(agate_table, agate.Table) assert agate_table.column_names == ("summed_numbers",) assert agate_table.rows == [(6,)] class TestCanConfigFunctionsFromProjectConfig: @pytest.fixture(scope="class") def functions(self) -> Dict[str, str]: return { "double_it.sql": double_it_sql, "double_it.yml": double_it_yml, } @pytest.fixture(scope="class") def project_config_update(self): return { "functions": {"+volatility": "stable"}, } def test_can_config_functions_from_project_config(self, project): manifest = run_dbt(["parse"]) assert len(manifest.functions) == 1 assert "function.test.double_it" in manifest.functions function_node = manifest.functions["function.test.double_it"] assert function_node.config.volatility == FunctionVolatility.Stable # Update with volatility specified in sql write_file(double_it_deterministic_sql, project.project_root, "functions", "double_it.sql") manifest = run_dbt(["parse", "--no-partial-parse"]) assert len(manifest.functions) == 1 assert "function.test.double_it" in manifest.functions function_node = manifest.functions["function.test.double_it"] # Volatility from sql should take precedence over the project config assert function_node.config.volatility == FunctionVolatility.Deterministic class TestPythonFunctionWithoutJinjaHasEquivalentRawCodeAndCompiledCode: @pytest.fixture(scope="class") def functions(self) -> Dict[str, str]: return { "double_it.py": double_it_py, "double_it.yml": double_it_python_yml, } @pytest.fixture(scope="class") def macros(self) -> Dict[str, str]: return { "postgres__scalar_function_python.sql": scalar_function_python_macro, } def test_udfs(self, project): run_dbt(["build"]) result = run_dbt(["compile"]) assert len(result.results) == 1 node = result.results[0].node assert isinstance(node, FunctionNode) assert node.raw_code == node.compiled_code class TestPythonFunctionWithJinjaHasCorrectCompiledCode: @pytest.fixture(scope="class") def functions(self) -> Dict[str, str]: return { "double_it.py": double_it_py_with_jinja, "double_it.yml": double_it_python_yml, } @pytest.fixture(scope="class") def macros(self) -> Dict[str, str]: return { "postgres__scalar_function_python.sql": scalar_function_python_macro, } def test_udfs(self, project): result = run_dbt(["compile"]) assert len(result.results) == 1 node = result.results[0].node assert isinstance(node, FunctionNode) assert node.compiled_code == "def entry(value):\n \n return value * 2\n " class TestDefaultArgumentsBasic: @pytest.fixture(scope="class") def functions(self) -> Dict[str, str]: return { "sum_2_values.py": sum_2_values_sql, "sum_2_values.yml": sum_2_values_yml, } def test_udfs(self, project): manifest = run_dbt(["parse"]) assert len(manifest.functions) == 1 function_node = manifest.functions["function.test.sum_2_values"] assert isinstance(function_node, FunctionNode) assert len(function_node.arguments) == 2 assert function_node.arguments[0].default_value is None assert function_node.arguments[1].default_value == 0 class TestDefaultArgumentsMustComeLast: @pytest.fixture(scope="class") def functions(self) -> Dict[str, str]: return { "sum_2_values.py": sum_2_values_sql, "sum_2_values.yml": sum_2_values_bad_default_arg_order_yml, } def test_udfs(self, project): with pytest.raises(ParsingError) as excinfo: run_dbt(["parse"]) assert ( "Non-defaulted argument 'val2' of function 'sum_2_values' comes after a defaulted argument. Non-defaulted arguments cannot come after defaulted arguments. " in str(excinfo.value) ) class TestFunctionsIncludeAndExcludeByResourceType: @pytest.fixture(scope="class") def functions(self) -> Dict[str, str]: return { "double_it.sql": double_it_sql, "double_it.yml": double_it_yml, } def test_udfs(self, project): result = run_dbt(["build", "--resource-type", "function"]) assert len(result.results) == 1 function_node = result.results[0].node assert isinstance(function_node, FunctionNode) result = run_dbt(["build", "--exclude-resource-type", "function"]) assert len(result.results) == 0 double_it_py_with_packages = """ {{ config(packages=['scikit-learn', 'pandas==1.5.0']) }} def entry(value): return value * 2 """ double_it_python_with_packages_yml = """ functions: - name: double_it description: Doubles whatever number is passed in config: runtime_version: "3.11" entry_point: entry packages: - scikit-learn - pandas==1.5.0 arguments: - name: value data_type: float description: A number to be doubled returns: data_type: float """ class TestPythonFunctionPackagesViaJinjaConfig: @pytest.fixture(scope="class") def functions(self) -> Dict[str, str]: return { "double_it.py": double_it_py_with_packages, "double_it.yml": double_it_python_yml, } def test_packages_set_via_jinja_config(self, project): manifest = run_dbt(["parse"]) assert len(manifest.functions) == 1 function_node = manifest.functions["function.test.double_it"] assert isinstance(function_node, FunctionNode) assert function_node.language == "python" assert function_node.config.packages == ["scikit-learn", "pandas==1.5.0"] class TestPythonFunctionPackagesViaYamlConfig: @pytest.fixture(scope="class") def functions(self) -> Dict[str, str]: return { "double_it.py": double_it_py, "double_it.yml": double_it_python_with_packages_yml, } def test_packages_set_via_yaml_config(self, project): manifest = run_dbt(["parse"]) assert len(manifest.functions) == 1 function_node = manifest.functions["function.test.double_it"] assert isinstance(function_node, FunctionNode) assert function_node.language == "python" assert function_node.config.packages == ["scikit-learn", "pandas==1.5.0"] class TestPythonFunctionPackagesViaProjectConfig: @pytest.fixture(scope="class") def functions(self) -> Dict[str, str]: return { "double_it.py": double_it_py, "double_it.yml": double_it_python_yml, } @pytest.fixture(scope="class") def project_config_update(self): return { "functions": {"+packages": ["scikit-learn"]}, } def test_packages_set_via_project_config(self, project): manifest = run_dbt(["parse"]) assert len(manifest.functions) == 1 function_node = manifest.functions["function.test.double_it"] assert isinstance(function_node, FunctionNode) assert function_node.language == "python" assert function_node.config.packages == ["scikit-learn"] class TestFunctionsGetSchemaCreatedIfNecessary: @pytest.fixture(scope="class") def functions(self) -> Dict[str, str]: return { "double_it.sql": double_it_sql, "double_it.yml": double_it_yml, } @pytest.fixture(scope="class") def project_config_update(self): return { "functions": {"+schema": "{{ 'alt_function_schema_' ~ target.schema }}"}, } def test_functions_get_schema_created_if_necessary(self, project): # previously an error occurred because the function schema wasn't guaranteed to be created. # so it just not erroring out is a good indicator that that's been fixed result = run_dbt(["build", "--debug"]) # some additional saqnity validations assert len(result.results) == 1 function_node = result.results[0].node assert isinstance(function_node, FunctionNode) assert "alt_function_schema_" in function_node.schema ================================================ FILE: tests/functional/generic_test_description/fixtures.py ================================================ models__my_model_sql = """ with my_cte as ( select 1 as id, 'blue' as color union all select 2 as id, 'green' as red union all select 3 as id, 'red' as red ) select * from my_cte """ models__schema_yml = """ models: - name: my_model columns: - name: id tests: - unique: description: "id must be unique" - not_null - name: color tests: - accepted_values: values: ['blue', 'green', 'red'] description: "{{ doc('color_accepted_values') }}" """ models__doc_block_md = """ {% docs color_accepted_values %} The `color` column must be one of 'blue', 'green', or 'red'. {% enddocs %} """ ================================================ FILE: tests/functional/generic_test_description/test_generic_test_description.py ================================================ import pytest from dbt.tests.util import get_artifact, run_dbt from tests.functional.generic_test_description.fixtures import ( models__doc_block_md, models__my_model_sql, models__schema_yml, ) class TestBuiltinGenericTestDescription: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": models__my_model_sql, "schema.yml": models__schema_yml, "doc_block.md": models__doc_block_md, } def test_compile(self, project): run_dbt(["compile"]) manifest = get_artifact(project.project_root, "target", "manifest.json") assert len(manifest["nodes"]) == 4 nodes = {node["alias"]: node for node in manifest["nodes"].values()} assert nodes["unique_my_model_id"]["description"] == "id must be unique" assert nodes["not_null_my_model_id"]["description"] == "" assert ( nodes["accepted_values_my_model_color__blue__green__red"]["description"] == "The `color` column must be one of 'blue', 'green', or 'red'." ) ================================================ FILE: tests/functional/graph_selection/data/seed.csv ================================================ id,first_name,last_name,email,gender,ip_address,updated_at 1,Jack,Hunter,jhunter0@pbs.org,Male,59.80.20.168,2015-12-24 12:19:28 2,Kathryn,Walker,kwalker1@ezinearticles.com,Female,194.121.179.35,2015-12-24 12:19:28 3,Gerald,Ryan,gryan2@com.com,Male,11.3.212.243,2015-12-24 12:19:28 4,Bonnie,Spencer,bspencer3@ameblo.jp,Female,216.32.196.175,2015-12-24 12:19:28 5,Harold,Taylor,htaylor4@people.com.cn,Male,253.10.246.136,2015-12-24 12:19:28 6,Jacqueline,Griffin,jgriffin5@t.co,Female,16.13.192.220,2015-12-24 12:19:28 7,Wanda,Arnold,warnold6@google.nl,Female,232.116.150.64,2015-12-24 12:19:28 8,Craig,Ortiz,cortiz7@sciencedaily.com,Male,199.126.106.13,2015-12-24 12:19:28 9,Gary,Day,gday8@nih.gov,Male,35.81.68.186,2015-12-24 12:19:28 10,Rose,Wright,rwright9@yahoo.co.jp,Female,236.82.178.100,2015-12-24 12:19:28 11,Raymond,Kelley,rkelleya@fc2.com,Male,213.65.166.67,2015-12-24 12:19:28 12,Gerald,Robinson,grobinsonb@disqus.com,Male,72.232.194.193,2015-12-24 12:19:28 13,Mildred,Martinez,mmartinezc@samsung.com,Female,198.29.112.5,2015-12-24 12:19:28 14,Dennis,Arnold,darnoldd@google.com,Male,86.96.3.250,2015-12-24 12:19:28 15,Judy,Gray,jgraye@opensource.org,Female,79.218.162.245,2015-12-24 12:19:28 16,Theresa,Garza,tgarzaf@epa.gov,Female,21.59.100.54,2015-12-24 12:19:28 17,Gerald,Robertson,grobertsong@csmonitor.com,Male,131.134.82.96,2015-12-24 12:19:28 18,Philip,Hernandez,phernandezh@adobe.com,Male,254.196.137.72,2015-12-24 12:19:28 19,Julia,Gonzalez,jgonzalezi@cam.ac.uk,Female,84.240.227.174,2015-12-24 12:19:28 20,Andrew,Davis,adavisj@patch.com,Male,9.255.67.25,2015-12-24 12:19:28 21,Kimberly,Harper,kharperk@foxnews.com,Female,198.208.120.253,2015-12-24 12:19:28 22,Mark,Martin,mmartinl@marketwatch.com,Male,233.138.182.153,2015-12-24 12:19:28 23,Cynthia,Ruiz,cruizm@google.fr,Female,18.178.187.201,2015-12-24 12:19:28 24,Samuel,Carroll,scarrolln@youtu.be,Male,128.113.96.122,2015-12-24 12:19:28 25,Jennifer,Larson,jlarsono@vinaora.com,Female,98.234.85.95,2015-12-24 12:19:28 26,Ashley,Perry,aperryp@rakuten.co.jp,Female,247.173.114.52,2015-12-24 12:19:28 27,Howard,Rodriguez,hrodriguezq@shutterfly.com,Male,231.188.95.26,2015-12-24 12:19:28 28,Amy,Brooks,abrooksr@theatlantic.com,Female,141.199.174.118,2015-12-24 12:19:28 29,Louise,Warren,lwarrens@adobe.com,Female,96.105.158.28,2015-12-24 12:19:28 30,Tina,Watson,twatsont@myspace.com,Female,251.142.118.177,2015-12-24 12:19:28 31,Janice,Kelley,jkelleyu@creativecommons.org,Female,239.167.34.233,2015-12-24 12:19:28 32,Terry,Mccoy,tmccoyv@bravesites.com,Male,117.201.183.203,2015-12-24 12:19:28 33,Jeffrey,Morgan,jmorganw@surveymonkey.com,Male,78.101.78.149,2015-12-24 12:19:28 34,Louis,Harvey,lharveyx@sina.com.cn,Male,51.50.0.167,2015-12-24 12:19:28 35,Philip,Miller,pmillery@samsung.com,Male,103.255.222.110,2015-12-24 12:19:28 36,Willie,Marshall,wmarshallz@ow.ly,Male,149.219.91.68,2015-12-24 12:19:28 37,Patrick,Lopez,plopez10@redcross.org,Male,250.136.229.89,2015-12-24 12:19:28 38,Adam,Jenkins,ajenkins11@harvard.edu,Male,7.36.112.81,2015-12-24 12:19:28 39,Benjamin,Cruz,bcruz12@linkedin.com,Male,32.38.98.15,2015-12-24 12:19:28 40,Ruby,Hawkins,rhawkins13@gmpg.org,Female,135.171.129.255,2015-12-24 12:19:28 41,Carlos,Barnes,cbarnes14@a8.net,Male,240.197.85.140,2015-12-24 12:19:28 42,Ruby,Griffin,rgriffin15@bravesites.com,Female,19.29.135.24,2015-12-24 12:19:28 43,Sean,Mason,smason16@icq.com,Male,159.219.155.249,2015-12-24 12:19:28 44,Anthony,Payne,apayne17@utexas.edu,Male,235.168.199.218,2015-12-24 12:19:28 45,Steve,Cruz,scruz18@pcworld.com,Male,238.201.81.198,2015-12-24 12:19:28 46,Anthony,Garcia,agarcia19@flavors.me,Male,25.85.10.18,2015-12-24 12:19:28 47,Doris,Lopez,dlopez1a@sphinn.com,Female,245.218.51.238,2015-12-24 12:19:28 48,Susan,Nichols,snichols1b@freewebs.com,Female,199.99.9.61,2015-12-24 12:19:28 49,Wanda,Ferguson,wferguson1c@yahoo.co.jp,Female,236.241.135.21,2015-12-24 12:19:28 50,Andrea,Pierce,apierce1d@google.co.uk,Female,132.40.10.209,2015-12-24 12:19:28 51,Lawrence,Phillips,lphillips1e@jugem.jp,Male,72.226.82.87,2015-12-24 12:19:28 52,Judy,Gilbert,jgilbert1f@multiply.com,Female,196.250.15.142,2015-12-24 12:19:28 53,Eric,Williams,ewilliams1g@joomla.org,Male,222.202.73.126,2015-12-24 12:19:28 54,Ralph,Romero,rromero1h@sogou.com,Male,123.184.125.212,2015-12-24 12:19:28 55,Jean,Wilson,jwilson1i@ocn.ne.jp,Female,176.106.32.194,2015-12-24 12:19:28 56,Lori,Reynolds,lreynolds1j@illinois.edu,Female,114.181.203.22,2015-12-24 12:19:28 57,Donald,Moreno,dmoreno1k@bbc.co.uk,Male,233.249.97.60,2015-12-24 12:19:28 58,Steven,Berry,sberry1l@eepurl.com,Male,186.193.50.50,2015-12-24 12:19:28 59,Theresa,Shaw,tshaw1m@people.com.cn,Female,120.37.71.222,2015-12-24 12:19:28 60,John,Stephens,jstephens1n@nationalgeographic.com,Male,191.87.127.115,2015-12-24 12:19:28 61,Richard,Jacobs,rjacobs1o@state.tx.us,Male,66.210.83.155,2015-12-24 12:19:28 62,Andrew,Lawson,alawson1p@over-blog.com,Male,54.98.36.94,2015-12-24 12:19:28 63,Peter,Morgan,pmorgan1q@rambler.ru,Male,14.77.29.106,2015-12-24 12:19:28 64,Nicole,Garrett,ngarrett1r@zimbio.com,Female,21.127.74.68,2015-12-24 12:19:28 65,Joshua,Kim,jkim1s@edublogs.org,Male,57.255.207.41,2015-12-24 12:19:28 66,Ralph,Roberts,rroberts1t@people.com.cn,Male,222.143.131.109,2015-12-24 12:19:28 67,George,Montgomery,gmontgomery1u@smugmug.com,Male,76.75.111.77,2015-12-24 12:19:28 68,Gerald,Alvarez,galvarez1v@flavors.me,Male,58.157.186.194,2015-12-24 12:19:28 69,Donald,Olson,dolson1w@whitehouse.gov,Male,69.65.74.135,2015-12-24 12:19:28 70,Carlos,Morgan,cmorgan1x@pbs.org,Male,96.20.140.87,2015-12-24 12:19:28 71,Aaron,Stanley,astanley1y@webnode.com,Male,163.119.217.44,2015-12-24 12:19:28 72,Virginia,Long,vlong1z@spiegel.de,Female,204.150.194.182,2015-12-24 12:19:28 73,Robert,Berry,rberry20@tripadvisor.com,Male,104.19.48.241,2015-12-24 12:19:28 74,Antonio,Brooks,abrooks21@unesco.org,Male,210.31.7.24,2015-12-24 12:19:28 75,Ruby,Garcia,rgarcia22@ovh.net,Female,233.218.162.214,2015-12-24 12:19:28 76,Jack,Hanson,jhanson23@blogtalkradio.com,Male,31.55.46.199,2015-12-24 12:19:28 77,Kathryn,Nelson,knelson24@walmart.com,Female,14.189.146.41,2015-12-24 12:19:28 78,Jason,Reed,jreed25@printfriendly.com,Male,141.189.89.255,2015-12-24 12:19:28 79,George,Coleman,gcoleman26@people.com.cn,Male,81.189.221.144,2015-12-24 12:19:28 80,Rose,King,rking27@ucoz.com,Female,212.123.168.231,2015-12-24 12:19:28 81,Johnny,Holmes,jholmes28@boston.com,Male,177.3.93.188,2015-12-24 12:19:28 82,Katherine,Gilbert,kgilbert29@altervista.org,Female,199.215.169.61,2015-12-24 12:19:28 83,Joshua,Thomas,jthomas2a@ustream.tv,Male,0.8.205.30,2015-12-24 12:19:28 84,Julie,Perry,jperry2b@opensource.org,Female,60.116.114.192,2015-12-24 12:19:28 85,Richard,Perry,rperry2c@oracle.com,Male,181.125.70.232,2015-12-24 12:19:28 86,Kenneth,Ruiz,kruiz2d@wikimedia.org,Male,189.105.137.109,2015-12-24 12:19:28 87,Jose,Morgan,jmorgan2e@webnode.com,Male,101.134.215.156,2015-12-24 12:19:28 88,Donald,Campbell,dcampbell2f@goo.ne.jp,Male,102.120.215.84,2015-12-24 12:19:28 89,Debra,Collins,dcollins2g@uol.com.br,Female,90.13.153.235,2015-12-24 12:19:28 90,Jesse,Johnson,jjohnson2h@stumbleupon.com,Male,225.178.125.53,2015-12-24 12:19:28 91,Elizabeth,Stone,estone2i@histats.com,Female,123.184.126.221,2015-12-24 12:19:28 92,Angela,Rogers,arogers2j@goodreads.com,Female,98.104.132.187,2015-12-24 12:19:28 93,Emily,Dixon,edixon2k@mlb.com,Female,39.190.75.57,2015-12-24 12:19:28 94,Albert,Scott,ascott2l@tinypic.com,Male,40.209.13.189,2015-12-24 12:19:28 95,Barbara,Peterson,bpeterson2m@ow.ly,Female,75.249.136.180,2015-12-24 12:19:28 96,Adam,Greene,agreene2n@fastcompany.com,Male,184.173.109.144,2015-12-24 12:19:28 97,Earl,Sanders,esanders2o@hc360.com,Male,247.34.90.117,2015-12-24 12:19:28 98,Angela,Brooks,abrooks2p@mtv.com,Female,10.63.249.126,2015-12-24 12:19:28 99,Harold,Foster,hfoster2q@privacy.gov.au,Male,139.214.40.244,2015-12-24 12:19:28 100,Carl,Meyer,cmeyer2r@disqus.com,Male,204.117.7.88,2015-12-24 12:19:28 ================================================ FILE: tests/functional/graph_selection/data/summary_expected.csv ================================================ gender,ct Female,40 Male,60 ================================================ FILE: tests/functional/graph_selection/fixtures.py ================================================ import pytest from dbt.tests.util import read_file schema_yml = """ version: 2 groups: - name: emails_group owner: name: Jeremy email: data@jer.co slack: talk-jerco-memes github: jtcohen6 whatever: you want - name: users_group owner: name: Jeremy email: data@jer.co slack: talk-jerco-memes github: jtcohen6 whatever: you want - name: users_rollup_group owner: name: Jeremy email: data@jer.co slack: talk-jerco-memes github: jtcohen6 whatever: you want models: - name: emails group: emails_group columns: - name: email data_tests: - not_null: severity: warn - name: users group: users_group columns: - name: id data_tests: - unique - name: users_rollup group: users_rollup_group columns: - name: gender data_tests: - unique - name: versioned latest_version: 2 versions: - v: 0 - v: 1 - v: 2 - v: 3 - v: 4.5 - v: "5.0" - v: 21 - v: "test" sources: - name: raw schema: '{{ target.schema }}' tables: - name: seed exposures: - name: user_exposure type: dashboard depends_on: - ref('users') - ref('users_rollup') - ref('versioned', v=3) owner: email: nope@example.com - name: seed_ml_exposure type: ml depends_on: - source('raw', 'seed') owner: email: nope@example.com """ patch_path_selection_schema_yml = """ version: 2 models: - name: subdir description: submarine sandwich directory """ base_users_sql = """ {{ config( materialized = 'ephemeral', tags = ['base'] ) }} select * from {{ source('raw', 'seed') }} """ users_sql = """ {{ config( materialized = 'table', tags=['bi', 'users'] ) }} select * from {{ ref('base_users') }} """ users_rollup_sql = """ {{ config( materialized = 'view', tags = 'bi' ) }} with users as ( select * from {{ ref('users') }} ) select gender, count(*) as ct from users group by 1 """ users_rollup_dependency_sql = """ {{ config(materialized='table') }} select * from {{ ref('users_rollup') }} """ emails_sql = """ {{ config(materialized='ephemeral', tags=['base']) }} select distinct email from {{ ref('base_users') }} """ emails_alt_sql = """ select distinct email from {{ ref('users') }} """ alternative_users_sql = """ {# Same as ´users´ model, but with dots in the model name #} {{ config( materialized = 'table', tags=['dots'] ) }} select * from {{ ref('base_users') }} """ never_selected_sql = """ {{ config(schema='_and_then') }} select * from {{ this.schema }}.seed """ subdir_sql = """ select 1 as id """ nested_users_sql = """ select 1 as id """ properties_yml = """ version: 2 seeds: - name: summary_expected config: column_types: ct: BIGINT gender: text """ class SelectionFixtures: @pytest.fixture(scope="class") def models(self): return { "schema.yml": schema_yml, "patch_path_selection_schema.yml": patch_path_selection_schema_yml, "base_users.sql": base_users_sql, "users.sql": users_sql, "versioned_v3.sql": base_users_sql, "users_rollup.sql": users_rollup_sql, "users_rollup_dependency.sql": users_rollup_dependency_sql, "emails.sql": emails_sql, "emails_alt.sql": emails_alt_sql, "alternative.users.sql": alternative_users_sql, "never_selected.sql": never_selected_sql, "test": { "subdir.sql": subdir_sql, "versioned_v2.sql": subdir_sql, "subdir": { "nested_users.sql": nested_users_sql, "versioned_v1.sql": nested_users_sql, }, }, } @pytest.fixture(scope="class") def seeds(self, test_data_dir): # Read seed file and return seeds = {"properties.yml": properties_yml} for seed_file in ["seed.csv", "summary_expected.csv"]: seeds[seed_file] = read_file(test_data_dir, seed_file) return seeds ================================================ FILE: tests/functional/graph_selection/test_graph_selection.py ================================================ import json import os import pytest from dbt.tests.util import check_result_nodes_by_name, run_dbt from tests.functional.graph_selection.fixtures import SelectionFixtures selectors_yml = """ selectors: - name: bi_selector description: This is a BI selector definition: method: tag value: bi """ def assert_correct_schemas(project): adapter = project.adapter with adapter.connection_named("__test"): exists = adapter.check_schema_exists(project.database, project.test_schema) assert exists schema = project.test_schema + "_and_then" exists = adapter.check_schema_exists(project.database, schema) assert not exists def clear_schema(project): project.run_sql("drop schema if exists {schema} cascade") project.run_sql("create schema {schema}") class TestGraphSelection(SelectionFixtures): # The tests here aiming to test whether the correct node is selected, # we don't need the run to pass @pytest.fixture(scope="class") def selectors(self): return selectors_yml def test_specific_model(self, project): results = run_dbt(["run", "--select", "users"], expect_pass=False) check_result_nodes_by_name(results, ["users"]) assert_correct_schemas(project) def test_tags(self, project, project_root): results = run_dbt(["run", "--selector", "bi_selector"], expect_pass=False) check_result_nodes_by_name(results, ["users", "users_rollup"]) assert_correct_schemas(project) manifest_path = project_root.join("target/manifest.json") assert os.path.exists(manifest_path) with open(manifest_path) as fp: manifest = json.load(fp) assert "selectors" in manifest def test_tags_and_children(self, project): results = run_dbt(["run", "--select", "tag:base+"], expect_pass=False) check_result_nodes_by_name( results, [ "emails_alt", "users_rollup", "users", "alternative.users", "users_rollup_dependency", ], ) assert_correct_schemas(project) def test_tags_and_children_limited(self, project): results = run_dbt(["run", "--select", "tag:base+2"], expect_pass=False) check_result_nodes_by_name( results, ["emails_alt", "users_rollup", "users", "alternative.users"] ) assert_correct_schemas(project) def test_group(self, project): expected = ["test.unique_users_id", "test.users"] results = run_dbt(["ls", "--select", "group:users_group"]) assert sorted(results) == expected def test_specific_model_and_children(self, project): results = run_dbt(["run", "--select", "users+"], expect_pass=False) check_result_nodes_by_name( results, ["users", "users_rollup", "emails_alt", "users_rollup_dependency"] ) assert_correct_schemas(project) def test_specific_model_and_children_limited(self, project): results = run_dbt(["run", "--select", "users+1"], expect_pass=False) check_result_nodes_by_name(results, ["users", "users_rollup", "emails_alt"]) assert_correct_schemas(project) def test_specific_model_and_parents(self, project): results = run_dbt(["run", "--select", "+users_rollup"], expect_pass=False) check_result_nodes_by_name(results, ["users_rollup", "users"]) assert_correct_schemas(project) def test_specific_model_and_parents_limited(self, project): results = run_dbt(["run", "--select", "1+users_rollup"], expect_pass=False) check_result_nodes_by_name(results, ["users", "users_rollup"]) assert_correct_schemas(project) def test_specific_model_with_exclusion(self, project): results = run_dbt( [ "run", "--select", "+users_rollup", "--exclude", "models/users_rollup.sql", ], expect_pass=False, ) check_result_nodes_by_name(results, ["users"]) assert_correct_schemas(project) def test_locally_qualified_name(self, project): results = run_dbt(["run", "--select", "test.subdir"]) check_result_nodes_by_name(results, ["nested_users", "subdir", "versioned"]) assert_correct_schemas(project) os.chdir( project.profiles_dir ) # Change to random directory to test that Path selector works with project-dir results = run_dbt( ["run", "--project-dir", str(project.project_root), "--select", "models/test/subdir*"] ) check_result_nodes_by_name(results, ["nested_users", "subdir", "versioned"]) assert_correct_schemas(project) results = run_dbt( [ "build", "--project-dir", str(project.project_root), "--select", "models/patch_path_selection_schema.yml", ] ) check_result_nodes_by_name(results, ["subdir"]) assert_correct_schemas(project) # Check that list command works os.chdir( project.profiles_dir ) # Change to random directory to test that Path selector works with project-dir results = run_dbt( [ "-q", "ls", "-s", "path:models/test/subdir.sql", "--project-dir", str(project.project_root), ] # ["list", "--project-dir", str(project.project_root), "--select", "models/test/subdir*"] ) assert len(results) == 1 def test_locally_qualified_name_model_with_dots(self, project): results = run_dbt(["run", "--select", "alternative.users"], expect_pass=False) check_result_nodes_by_name(results, ["alternative.users"]) assert_correct_schemas(project) results = run_dbt(["run", "--select", "models/alternative.*"], expect_pass=False) check_result_nodes_by_name(results, ["alternative.users"]) assert_correct_schemas(project) def test_childrens_parents(self, project): results = run_dbt(["run", "--select", "@base_users"], expect_pass=False) check_result_nodes_by_name( results, [ "alternative.users", "users_rollup", "users", "emails_alt", "users_rollup_dependency", ], ) results = run_dbt(["test", "--select", "test_name:not_null"], expect_pass=False) check_result_nodes_by_name(results, ["not_null_emails_email"]) def test_more_childrens_parents(self, project): results = run_dbt(["run", "--select", "@users"], expect_pass=False) check_result_nodes_by_name( results, ["users_rollup", "users", "emails_alt", "users_rollup_dependency"] ) results = run_dbt(["test", "--select", "test_name:unique"], expect_pass=False) check_result_nodes_by_name(results, ["unique_users_id", "unique_users_rollup_gender"]) def test_concat(self, project): results = run_dbt(["run", "--select", "@emails_alt", "users_rollup"], expect_pass=False) check_result_nodes_by_name(results, ["users_rollup", "users", "emails_alt"]) def test_concat_multiple(self, project): results = run_dbt( ["run", "--select", "@emails_alt", "--select", "users_rollup"], expect_pass=False ) check_result_nodes_by_name(results, ["users_rollup", "users", "emails_alt"]) def test_concat_exclude(self, project): results = run_dbt( [ "run", "--select", "@emails_alt", "users_rollup", "--exclude", "emails_alt", ], expect_pass=False, ) check_result_nodes_by_name(results, ["users_rollup", "users"]) def test_concat_exclude_multiple(self, project): results = run_dbt( [ "run", "--select", "@emails_alt", "users_rollup", "--exclude", "users", "--exclude", "emails_alt", ], expect_pass=False, ) check_result_nodes_by_name(results, ["users_rollup"]) def test_concat_exclude_concat(self, project): results = run_dbt( [ "run", "--select", "@emails_alt", "users_rollup", "--exclude", "emails_alt", "users_rollup", ], expect_pass=False, ) check_result_nodes_by_name(results, ["users"]) results = run_dbt( [ "test", "--select", "@emails_alt", "users_rollup", "--exclude", "emails_alt", "users_rollup", ], expect_pass=False, ) check_result_nodes_by_name(results, ["unique_users_id"]) def test_exposure_parents(self, project): results = run_dbt(["ls", "--select", "+exposure:seed_ml_exposure"]) assert sorted(results) == [ "exposure:test.seed_ml_exposure", "source:test.raw.seed", ] results = run_dbt(["ls", "--select", "1+exposure:user_exposure"]) assert sorted(results) == [ "exposure:test.user_exposure", "test.unique_users_id", "test.unique_users_rollup_gender", "test.users", "test.users_rollup", "test.versioned.v3", ] results = run_dbt(["run", "-m", "+exposure:user_exposure"], expect_pass=False) check_result_nodes_by_name( results, [ "users_rollup", "users", ], ) class TestListPathGraphSelection(SelectionFixtures): def test_list_select_with_project_dir(self, project): # Check that list command works os.chdir( project.profiles_dir ) # Change to random directory to test that Path selector works with project-dir results = run_dbt( [ "-q", "ls", "-s", "path:models/test/subdir.sql", "--project-dir", str(project.project_root), ] ) assert results == ["test.test.subdir"] ================================================ FILE: tests/functional/graph_selection/test_group_selection.py ================================================ import pytest from dbt.tests.util import read_file, run_dbt from tests.functional.graph_selection.fixtures import ( alternative_users_sql, base_users_sql, emails_alt_sql, emails_sql, nested_users_sql, never_selected_sql, properties_yml, schema_yml, subdir_sql, users_rollup_dependency_sql, users_rollup_sql, users_sql, ) selectors_yml = """ selectors: - name: group_specified_as_string_str definition: group:users_group - name: group_specified_as_string_dict definition: method: group value: users_group - name: users_grouped_childrens_parents definition: method: group value: users_group childrens_parents: true """ class TestGroupSelection: @pytest.fixture(scope="class") def models(self): return { "schema.yml": schema_yml, "base_users.sql": base_users_sql, "users.sql": users_sql, "users_rollup.sql": users_rollup_sql, "versioned_v3.sql": base_users_sql, "users_rollup_dependency.sql": users_rollup_dependency_sql, "emails.sql": emails_sql, "emails_alt.sql": emails_alt_sql, "alternative.users.sql": alternative_users_sql, "never_selected.sql": never_selected_sql, "test": { "subdir.sql": subdir_sql, "subdir": {"nested_users.sql": nested_users_sql}, }, } @pytest.fixture(scope="class") def seeds(self, test_data_dir): # Read seed file and return seeds = {"properties.yml": properties_yml} for seed_file in ["seed.csv", "summary_expected.csv"]: seeds[seed_file] = read_file(test_data_dir, seed_file) return seeds @pytest.fixture(scope="class") def selectors(self): return selectors_yml def test_select_models_by_group(self, project): results = run_dbt(["ls", "--model", "group:users_group"]) assert sorted(results) == ["test.users"] def test_select_group_selector_str(self, project): results = run_dbt(["ls", "--selector", "group_specified_as_string_str"]) assert sorted(results) == ["test.unique_users_id", "test.users"] def test_select_group_selector_dict(self, project): results = run_dbt(["ls", "--selector", "group_specified_as_string_dict"]) assert sorted(results) == ["test.unique_users_id", "test.users"] def test_select_models_by_group_and_children(self, project): # noqa results = run_dbt(["ls", "--models", "+group:users_group+"]) assert sorted(results) == [ "test.base_users", "test.emails_alt", "test.users", "test.users_rollup", "test.users_rollup_dependency", ] def test_select_group_and_children(self, project): # noqa expected = [ "exposure:test.user_exposure", "source:test.raw.seed", "test.base_users", "test.emails_alt", "test.unique_users_id", "test.unique_users_rollup_gender", "test.users", "test.users_rollup", "test.users_rollup_dependency", ] results = run_dbt(["ls", "--select", "+group:users_group+"]) assert sorted(results) == expected def test_select_group_and_children_selector_str(self, project): # noqa expected = [ "exposure:test.user_exposure", "source:test.raw.seed", "test.base_users", "test.emails_alt", "test.unique_users_id", "test.unique_users_rollup_gender", "test.users", "test.users_rollup", "test.users_rollup_dependency", "test.versioned.v3", ] results = run_dbt(["ls", "--selector", "users_grouped_childrens_parents"]) assert sorted(results) == expected # 2 groups def test_select_models_two_groups(self, project): expected = ["test.base_users", "test.emails", "test.users"] results = run_dbt(["ls", "--models", "@group:emails_group group:users_group"]) assert sorted(results) == expected ================================================ FILE: tests/functional/graph_selection/test_inline.py ================================================ import pytest from dbt.cli.exceptions import DbtUsageException from dbt.tests.util import run_dbt, run_dbt_and_capture, write_file selectors_yml = """ selectors: - name: test_selector description: Exclude everything default: true definition: method: package value: "foo" """ dbt_project_yml = """ name: test profile: test flags: send_anonymous_usage_stats: false """ dbt_project_yml_disabled_models = """ name: test profile: test flags: send_anonymous_usage_stats: false models: +enabled: false """ class TestCompileInlineWithSelector: @pytest.fixture(scope="class") def models(self): return { "first_model.sql": "select 1 as id", } @pytest.fixture(scope="class") def selectors(self): return selectors_yml def test_inline_selectors(self, project): (results, log_output) = run_dbt_and_capture( ["compile", "--inline", "select * from {{ ref('first_model') }}"] ) assert len(results) == 1 assert "Compiled inline node is:" in log_output # Set all models to disabled, check that we still get inline result write_file(dbt_project_yml_disabled_models, project.project_root, "dbt_project.yml") (results, log_output) = run_dbt_and_capture(["compile", "--inline", "select 1 as id"]) assert len(results) == 1 # put back non-disabled dbt_project and check for mutually exclusive error message # for --select and --inline write_file(dbt_project_yml, project.project_root, "dbt_project.yml") with pytest.raises(DbtUsageException): run_dbt(["compile", "--select", "first_model", "--inline", "select 1 as id"]) # check for mutually exclusive --selector and --inline with pytest.raises(DbtUsageException): run_dbt(["compile", "--selector", "test_selector", "--inline", "select 1 as id"]) ================================================ FILE: tests/functional/graph_selection/test_intersection_syntax.py ================================================ import pytest from dbt.tests.util import check_result_nodes_by_name, run_dbt from tests.functional.graph_selection.fixtures import SelectionFixtures selectors_yml = """ selectors: - name: same_intersection definition: intersection: - fqn: users - fqn: users - name: tags_intersection definition: intersection: - tag: bi - tag: users - name: triple_descending definition: intersection: - fqn: "*" - tag: bi - tag: users - name: triple_ascending definition: intersection: - tag: users - tag: bi - fqn: "*" - name: intersection_with_exclusion definition: intersection: - method: fqn value: users_rollup_dependency parents: true - method: fqn value: users children: true - exclude: - users_rollup_dependency - name: intersection_exclude_intersection definition: intersection: - tag:bi - "@users" - exclude: - intersection: - tag:bi - method: fqn value: users_rollup children: true - name: intersection_exclude_intersection_lack definition: intersection: - tag:bi - "@users" - exclude: - intersection: - method: fqn value: emails children_parents: true - method: fqn value: emails_alt children_parents: true """ # The project and run_seed fixtures will be executed for each test method class TestIntersectionSyncs(SelectionFixtures): # The tests here aiming to test whether the correct node is selected, # we don't need the run to pass @pytest.fixture(scope="class") def selectors(self): return selectors_yml def test_same_model_intersection(self, project): results = run_dbt(["run", "--models", "users,users"], expect_pass=False) check_result_nodes_by_name(results, ["users"]) def test_same_model_intersection_selectors(self, project): results = run_dbt(["run", "--selector", "same_intersection"], expect_pass=False) check_result_nodes_by_name(results, ["users"]) def test_tags_intersection(self, project): results = run_dbt(["run", "--models", "tag:bi,tag:users"], expect_pass=False) check_result_nodes_by_name(results, ["users"]) def test_tags_intersection_selectors(self, project): results = run_dbt(["run", "--selector", "tags_intersection"], expect_pass=False) check_result_nodes_by_name(results, ["users"]) def test_intersection_triple_descending(self, project): results = run_dbt(["run", "--models", "*,tag:bi,tag:users"], expect_pass=False) check_result_nodes_by_name(results, ["users"]) def test_intersection_triple_descending_schema(self, project): results = run_dbt(["run", "--models", "*,tag:bi,tag:users"], expect_pass=False) check_result_nodes_by_name(results, ["users"]) def test_intersection_triple_descending_schema_selectors(self, project): results = run_dbt(["run", "--selector", "triple_descending"], expect_pass=False) check_result_nodes_by_name(results, ["users"]) def test_intersection_triple_ascending(self, project): results = run_dbt(["run", "--models", "tag:users,tag:bi,*"], expect_pass=False) check_result_nodes_by_name(results, ["users"]) def test_intersection_triple_ascending_schema_selectors(self, project): results = run_dbt(["run", "--selector", "triple_ascending"], expect_pass=False) check_result_nodes_by_name(results, ["users"]) def test_intersection_with_exclusion(self, project): results = run_dbt( [ "run", "--models", "+users_rollup_dependency,users+", "--exclude", "users_rollup_dependency", ], expect_pass=False, ) check_result_nodes_by_name(results, ["users", "users_rollup"]) def test_intersection_with_exclusion_selectors(self, project): results = run_dbt(["run", "--selector", "intersection_with_exclusion"], expect_pass=False) check_result_nodes_by_name(results, ["users", "users_rollup"]) def test_intersection_exclude_intersection(self, project): results = run_dbt( ["run", "--models", "tag:bi,@users", "--exclude", "tag:bi,users_rollup+"], expect_pass=False, ) check_result_nodes_by_name(results, ["users"]) def test_intersection_exclude_intersection_selectors(self, project): results = run_dbt( ["run", "--selector", "intersection_exclude_intersection"], expect_pass=False, ) check_result_nodes_by_name(results, ["users"]) def test_intersection_exclude_intersection_lack(self, project): results = run_dbt( ["run", "--models", "tag:bi,@users", "--exclude", "@emails,@emails_alt"], expect_pass=False, ) check_result_nodes_by_name(results, ["users", "users_rollup"]) def test_intersection_exclude_intersection_lack_selector(self, project): results = run_dbt( ["run", "--selector", "intersection_exclude_intersection_lack"], expect_pass=False, ) check_result_nodes_by_name(results, ["users", "users_rollup"]) def test_intersection_exclude_triple_intersection(self, project): results = run_dbt( ["run", "--models", "tag:bi,@users", "--exclude", "*,tag:bi,users_rollup"], expect_pass=False, ) check_result_nodes_by_name(results, ["users"]) def test_intersection_concat(self, project): results = run_dbt(["run", "--models", "tag:bi,@users", "emails_alt"], expect_pass=False) check_result_nodes_by_name(results, ["users", "users_rollup", "emails_alt"]) def test_intersection_concat_intersection(self, project): results = run_dbt( ["run", "--models", "tag:bi,@users", "@emails_alt,emails_alt"], expect_pass=False, ) check_result_nodes_by_name(results, ["users", "users_rollup", "emails_alt"]) def test_intersection_concat_exclude(self, project): results = run_dbt( [ "run", "--models", "tag:bi,@users", "emails_alt", "--exclude", "users_rollup", ], expect_pass=False, ) check_result_nodes_by_name(results, ["users", "emails_alt"]) def test_intersection_concat_exclude_concat(self, project): results = run_dbt( [ "run", "--models", "tag:bi,@users", "emails_alt,@users", "--exclude", "users_rollup_dependency", "users_rollup", ], expect_pass=False, ) check_result_nodes_by_name(results, ["users", "emails_alt"]) def test_intersection_concat_exclude_intersection_concat(self, project): results = run_dbt( [ "run", "--models", "tag:bi,@users", "emails_alt,@users", "--exclude", "@users,users_rollup_dependency", "@users,users_rollup", ], expect_pass=False, ) check_result_nodes_by_name(results, ["users", "emails_alt"]) ================================================ FILE: tests/functional/graph_selection/test_schema_test_graph_selection.py ================================================ import pytest from dbt.tests.fixtures.project import write_project_files from dbt.tests.util import run_dbt from tests.fixtures.dbt_integration_project import dbt_integration_project # noqa: F401 from tests.functional.graph_selection.fixtures import SelectionFixtures def run_schema_and_assert(project, include, exclude, expected_tests): # deps must run before seed run_dbt(["deps"]) run_dbt(["seed"]) results = run_dbt(["run", "--exclude", "never_selected"]) assert len(results) == 12 test_args = ["test"] if include: test_args += ["--select", include] if exclude: test_args += ["--exclude", exclude] test_results = run_dbt(test_args) ran_tests = sorted([test.node.name for test in test_results]) expected_sorted = sorted(expected_tests) assert ran_tests == expected_sorted class TestSchemaTestGraphSelection(SelectionFixtures): @pytest.fixture(scope="class", autouse=True) def setUp(self, project_root, dbt_integration_project): # noqa: F811 write_project_files(project_root, "dbt_integration_project", dbt_integration_project) @pytest.fixture(scope="class") def packages(self): return {"packages": [{"local": "dbt_integration_project"}]} def test_schema_tests_no_specifiers(self, project): run_schema_and_assert( project, None, None, [ "not_null_emails_email", "unique_table_model_id", "unique_users_id", "unique_users_rollup_gender", ], ) def test_schema_tests_specify_model(self, project): run_schema_and_assert(project, "users", None, ["unique_users_id"]) def test_schema_tests_specify_tag(self, project): run_schema_and_assert( project, "tag:bi", None, ["unique_users_id", "unique_users_rollup_gender"] ) def test_schema_tests_specify_model_and_children(self, project): run_schema_and_assert( project, "users+", None, ["unique_users_id", "unique_users_rollup_gender"] ) def test_schema_tests_specify_tag_and_children(self, project): run_schema_and_assert( project, "tag:base+", None, ["not_null_emails_email", "unique_users_id", "unique_users_rollup_gender"], ) def test_schema_tests_specify_model_and_parents(self, project): run_schema_and_assert( project, "+users_rollup", None, ["unique_users_id", "unique_users_rollup_gender"], ) def test_schema_tests_specify_model_and_parents_with_exclude(self, project): run_schema_and_assert(project, "+users_rollup", "users_rollup", ["unique_users_id"]) def test_schema_tests_specify_exclude_only(self, project): run_schema_and_assert( project, None, "users_rollup", ["not_null_emails_email", "unique_table_model_id", "unique_users_id"], ) def test_schema_tests_specify_model_in_pkg(self, project): run_schema_and_assert( project, "test.users_rollup", None, # TODO: change this. there's no way to select only direct ancestors # atm. ["unique_users_rollup_gender"], ) def test_schema_tests_with_glob(self, project): run_schema_and_assert( project, "*", "users", [ "not_null_emails_email", "unique_table_model_id", "unique_users_rollup_gender", ], ) def test_schema_tests_dep_package_only(self, project): run_schema_and_assert(project, "dbt_integration_project", None, ["unique_table_model_id"]) def test_schema_tests_model_in_dep_pkg(self, project): run_schema_and_assert( project, "dbt_integration_project.table_model", None, ["unique_table_model_id"], ) def test_schema_tests_exclude_pkg(self, project): run_schema_and_assert( project, None, "dbt_integration_project", ["not_null_emails_email", "unique_users_id", "unique_users_rollup_gender"], ) ================================================ FILE: tests/functional/graph_selection/test_tag_selection.py ================================================ import pytest from dbt.events.types import NoNodesForSelectionCriteria from dbt.tests.util import check_result_nodes_by_name, run_dbt from dbt_common.events.event_catcher import EventCatcher from tests.functional.graph_selection.fixtures import SelectionFixtures selectors_yml = """ selectors: - name: tag_specified_as_string_str definition: tag:specified_as_string - name: tag_specified_as_string_dict definition: method: tag value: specified_as_string - name: tag_specified_in_project_children_str definition: +tag:specified_in_project+ - name: tag_specified_in_project_children_dict definition: method: tag value: specified_in_project parents: true children: true - name: tagged-bi definition: method: tag value: bi - name: user_tagged_childrens_parents definition: method: tag value: users childrens_parents: true - name: base_ephemerals definition: union: - tag: base - method: config.materialized value: ephemeral - name: warn-severity definition: config.severity: warn - name: roundabout-everything definition: union: - "@tag:users" - intersection: - tag: base - config.materialized: ephemeral """ class TestTagSelection(SelectionFixtures): # The tests here aiming to test whether the correct node is selected, # we don't need the run to pass @pytest.fixture(scope="class") def project_config_update(self): return { "config-version": 2, "models": { "test": { "users": {"tags": "specified_as_string"}, "users_rollup": { "tags": ["specified_in_project"], }, } }, } @pytest.fixture(scope="class") def selectors(self): return selectors_yml def test_select_tag(self, project): results = run_dbt(["run", "--models", "tag:specified_as_string"], expect_pass=False) check_result_nodes_by_name(results, ["users"]) def test_select_tag_selector_str(self, project): results = run_dbt(["run", "--selector", "tag_specified_as_string_str"], expect_pass=False) check_result_nodes_by_name(results, ["users"]) def test_select_tag_selector_dict(self, project): results = run_dbt(["run", "--selector", "tag_specified_as_string_dict"], expect_pass=False) check_result_nodes_by_name(results, ["users"]) def test_select_tag_and_children(self, project): # noqa results = run_dbt(["run", "--models", "+tag:specified_in_project+"], expect_pass=False) check_result_nodes_by_name(results, ["users", "users_rollup", "users_rollup_dependency"]) def test_select_tag_and_children_selector_str(self, project): # noqa results = run_dbt( ["run", "--selector", "tag_specified_in_project_children_str"], expect_pass=False, ) check_result_nodes_by_name(results, ["users", "users_rollup", "users_rollup_dependency"]) def test_select_tag_and_children_selector_dict(self, project): # noqa results = run_dbt( ["run", "--selector", "tag_specified_in_project_children_dict"], expect_pass=False, ) check_result_nodes_by_name(results, ["users", "users_rollup", "users_rollup_dependency"]) def test_select_tag_in_model_with_project_config(self, project): # noqa results = run_dbt(["run", "--models", "tag:bi"], expect_pass=False) check_result_nodes_by_name(results, ["users", "users_rollup"]) def test_select_tag_in_model_with_project_config_selector(self, project): # noqa results = run_dbt(["run", "--selector", "tagged-bi"], expect_pass=False) check_result_nodes_by_name(results, ["users", "users_rollup"]) # check that model configs aren't squashed by project configs def test_select_tag_in_model_with_project_config_parents_children(self, project): # noqa results = run_dbt(["run", "--models", "@tag:users"], expect_pass=False) check_result_nodes_by_name( results, ["users", "users_rollup", "emails_alt", "users_rollup_dependency"] ) # just the users/users_rollup tests results = run_dbt(["test", "--models", "@tag:users"], expect_pass=False) check_result_nodes_by_name(results, ["unique_users_rollup_gender", "unique_users_id"]) # just the email test results = run_dbt( ["test", "--models", "tag:base,config.materialized:ephemeral"], expect_pass=False, ) check_result_nodes_by_name(results, ["not_null_emails_email"]) # also just the email test results = run_dbt(["test", "--models", "config.severity:warn"], expect_pass=False) check_result_nodes_by_name(results, ["not_null_emails_email"]) # all 3 tests results = run_dbt( ["test", "--models", "@tag:users tag:base,config.materialized:ephemeral"], expect_pass=False, ) check_result_nodes_by_name( results, ["not_null_emails_email", "unique_users_id", "unique_users_rollup_gender"], ) def test_select_tag_in_model_with_project_config_parents_children_selectors(self, project): results = run_dbt( ["run", "--selector", "user_tagged_childrens_parents"], expect_pass=False ) check_result_nodes_by_name( results, ["users", "users_rollup", "emails_alt", "users_rollup_dependency"] ) # just the users/users_rollup tests results = run_dbt( ["test", "--selector", "user_tagged_childrens_parents"], expect_pass=False ) check_result_nodes_by_name(results, ["unique_users_id", "unique_users_rollup_gender"]) # just the email test results = run_dbt(["test", "--selector", "base_ephemerals"], expect_pass=False) check_result_nodes_by_name(results, ["not_null_emails_email"]) # also just the email test results = run_dbt(["test", "--selector", "warn-severity"], expect_pass=False) check_result_nodes_by_name(results, ["not_null_emails_email"]) # all 3 tests results = run_dbt(["test", "--selector", "roundabout-everything"], expect_pass=False) check_result_nodes_by_name( results, ["unique_users_rollup_gender", "unique_users_id", "not_null_emails_email"], ) class TestTagSelectionNoMatch: def test_no_match(self, project): no_match_catcher = EventCatcher(event_to_catch=NoNodesForSelectionCriteria) # check `run` command run_dbt(["run", "--select", "tag:no_such_tag"], callbacks=[no_match_catcher.catch]) assert len(no_match_catcher.caught_events) == 1 # clear the catcher no_match_catcher.flush() # check `test` command run_dbt(["test", "--select", "tag:no_such_tag"], callbacks=[no_match_catcher.catch]) assert len(no_match_catcher.caught_events) == 1 # clear the catcher no_match_catcher.flush() # check `build` command run_dbt(["build", "--select", "tag:no_such_tag"], callbacks=[no_match_catcher.catch]) assert len(no_match_catcher.caught_events) == 1 ================================================ FILE: tests/functional/graph_selection/test_version_selection.py ================================================ import pytest from dbt.tests.util import read_file, run_dbt from tests.functional.graph_selection.fixtures import ( base_users_sql, properties_yml, schema_yml, users_rollup_sql, users_sql, ) selectors_yml = """ selectors: - name: version_specified_as_string_str definition: version:latest - name: version_specified_as_string_dict definition: method: version value: latest - name: version_childrens_parents definition: method: version value: latest childrens_parents: true """ class TestVersionSelection: @pytest.fixture(scope="class") def models(self): return { "schema.yml": schema_yml, "versioned_v1.sql": users_sql, "versioned_v2.sql": users_sql, "versioned_v3.sql": users_sql, "versioned_v4.5.sql": users_sql, "versioned_v5.0.sql": users_sql, "versioned_v21.sql": users_sql, "versioned_vtest.sql": users_sql, "base_users.sql": base_users_sql, "users.sql": users_sql, "users_rollup.sql": users_rollup_sql, } @pytest.fixture(scope="class") def seeds(self, test_data_dir): # Read seed file and return seeds = {"properties.yml": properties_yml} for seed_file in ["seed.csv", "summary_expected.csv"]: seeds[seed_file] = read_file(test_data_dir, seed_file) return seeds @pytest.fixture(scope="class") def selectors(self): return selectors_yml def test_select_none_versions(self, project): results = run_dbt(["ls", "--select", "version:none"]) assert sorted(results) == [ "test.base_users", "test.unique_users_id", "test.unique_users_rollup_gender", "test.users", "test.users_rollup", ] def test_select_latest_versions(self, project): results = run_dbt(["ls", "--select", "version:latest"]) assert sorted(results) == ["test.versioned.v2"] def test_select_old_versions(self, project): results = run_dbt(["ls", "--select", "version:old"]) assert sorted(results) == ["test.versioned.v1"] def test_select_prerelease_versions(self, project): results = run_dbt(["ls", "--select", "version:prerelease"]) assert sorted(results) == [ "test.versioned.v21", "test.versioned.v3", "test.versioned.v4.5", "test.versioned.v5.0", "test.versioned.vtest", ] def test_select_version_selector_str(self, project): results = run_dbt(["ls", "--selector", "version_specified_as_string_str"]) assert sorted(results) == ["test.versioned.v2"] def test_select_version_selector_dict(self, project): results = run_dbt(["ls", "--selector", "version_specified_as_string_dict"]) assert sorted(results) == ["test.versioned.v2"] def test_select_models_by_version_and_children(self, project): # noqa results = run_dbt(["ls", "--models", "+version:latest+"]) assert sorted(results) == ["test.base_users", "test.versioned.v2"] def test_select_version_and_children(self, project): # noqa expected = ["source:test.raw.seed", "test.base_users", "test.versioned.v2"] results = run_dbt(["ls", "--select", "+version:latest+"]) assert sorted(results) == expected def test_select_group_and_children_selector_str(self, project): # noqa expected = ["source:test.raw.seed", "test.base_users", "test.versioned.v2"] results = run_dbt(["ls", "--selector", "version_childrens_parents"]) assert sorted(results) == expected # 2 versions def test_select_models_two_versions(self, project): results = run_dbt(["ls", "--models", "version:latest version:old"]) assert sorted(results) == ["test.versioned.v1", "test.versioned.v2"] my_model_yml = """ models: - name: my_model versions: - v: 0 """ class TestVersionZero: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": "select 1 as id", "another.sql": "select * from {{ ref('my_model') }}", "schema.yml": my_model_yml, } def test_version_zero(self, project): results = run_dbt(["run"]) assert len(results) == 2 ================================================ FILE: tests/functional/incremental_schema_tests/fixtures.py ================================================ # # Properties # _PROPERTIES__SCHEMA = """ version: 2 models: - name: model_a columns: - name: id tags: [column_level_tag] data_tests: - unique - name: incremental_ignore columns: - name: id tags: [column_level_tag] data_tests: - unique - name: incremental_ignore_target columns: - name: id tags: [column_level_tag] data_tests: - unique - name: incremental_append_new_columns columns: - name: id tags: [column_level_tag] data_tests: - unique - name: incremental_append_new_columns_target columns: - name: id tags: [column_level_tag] data_tests: - unique - name: incremental_sync_all_columns columns: - name: id tags: [column_level_tag] data_tests: - unique - name: incremental_sync_all_columns_target columns: - name: id tags: [column_leveL_tag] data_tests: - unique """ # # Models # _MODELS__INCREMENTAL_SYNC_REMOVE_ONLY = """ {{ config( materialized='incremental', unique_key='id', on_schema_change='sync_all_columns' ) }} WITH source_data AS (SELECT * FROM {{ ref('model_a') }} ) {% set string_type = 'varchar(10)' %} {% if is_incremental() %} SELECT id, cast(field1 as {{string_type}}) as field1 FROM source_data WHERE id NOT IN (SELECT id from {{ this }} ) {% else %} select id, cast(field1 as {{string_type}}) as field1, cast(field2 as {{string_type}}) as field2 from source_data where id <= 3 {% endif %} """ _MODELS__INCREMENTAL_IGNORE = """ {{ config( materialized='incremental', unique_key='id', on_schema_change='ignore' ) }} WITH source_data AS (SELECT * FROM {{ ref('model_a') }} ) {% if is_incremental() %} SELECT id, field1, field2, field3, field4 FROM source_data WHERE id NOT IN (SELECT id from {{ this }} ) {% else %} SELECT id, field1, field2 FROM source_data LIMIT 3 {% endif %} """ _MODELS__INCREMENTAL_SYNC_REMOVE_ONLY_TARGET = """ {{ config(materialized='table') }} with source_data as ( select * from {{ ref('model_a') }} ) {% set string_type = 'varchar(10)' %} select id ,cast(field1 as {{string_type}}) as field1 from source_data order by id """ _MODELS__INCREMENTAL_IGNORE_TARGET = """ {{ config(materialized='table') }} with source_data as ( select * from {{ ref('model_a') }} ) select id ,field1 ,field2 from source_data """ _MODELS__INCREMENTAL_FAIL = """ {{ config( materialized='incremental', unique_key='id', on_schema_change='fail' ) }} WITH source_data AS (SELECT * FROM {{ ref('model_a') }} ) {% if is_incremental() %} SELECT id, field1, field2 FROM source_data {% else %} SELECT id, field1, field3 FROm source_data {% endif %} """ _MODELS__INCREMENTAL_SYNC_ALL_COLUMNS = """ {{ config( materialized='incremental', unique_key='id', on_schema_change='sync_all_columns' ) }} WITH source_data AS (SELECT * FROM {{ ref('model_a') }} ) {% set string_type = 'varchar(10)' %} {% if is_incremental() %} SELECT id, cast(field1 as {{string_type}}) as field1, cast(field3 as {{string_type}}) as field3, -- to validate new fields cast(field4 as {{string_type}}) AS field4 -- to validate new fields FROM source_data WHERE id NOT IN (SELECT id from {{ this }} ) {% else %} select id, cast(field1 as {{string_type}}) as field1, cast(field2 as {{string_type}}) as field2 from source_data where id <= 3 {% endif %} """ _MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_REMOVE_ONE = """ {{ config( materialized='incremental', unique_key='id', on_schema_change='append_new_columns' ) }} {% set string_type = 'varchar(10)' %} WITH source_data AS (SELECT * FROM {{ ref('model_a') }} ) {% if is_incremental() %} SELECT id, cast(field1 as {{string_type}}) as field1, cast(field3 as {{string_type}}) as field3, cast(field4 as {{string_type}}) as field4 FROM source_data WHERE id NOT IN (SELECT id from {{ this }} ) {% else %} SELECT id, cast(field1 as {{string_type}}) as field1, cast(field2 as {{string_type}}) as field2 FROM source_data where id <= 3 {% endif %} """ _MODELS__A = """ {{ config(materialized='table') }} with source_data as ( select 1 as id, 'aaa' as field1, 'bbb' as field2, 111 as field3, 'TTT' as field4 union all select 2 as id, 'ccc' as field1, 'ddd' as field2, 222 as field3, 'UUU' as field4 union all select 3 as id, 'eee' as field1, 'fff' as field2, 333 as field3, 'VVV' as field4 union all select 4 as id, 'ggg' as field1, 'hhh' as field2, 444 as field3, 'WWW' as field4 union all select 5 as id, 'iii' as field1, 'jjj' as field2, 555 as field3, 'XXX' as field4 union all select 6 as id, 'kkk' as field1, 'lll' as field2, 666 as field3, 'YYY' as field4 ) select id ,field1 ,field2 ,field3 ,field4 from source_data """ _MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_TARGET = """ {{ config(materialized='table') }} {% set string_type = 'varchar(10)' %} with source_data as ( select * from {{ ref('model_a') }} ) select id ,cast(field1 as {{string_type}}) as field1 ,cast(field2 as {{string_type}}) as field2 ,cast(CASE WHEN id <= 3 THEN NULL ELSE field3 END as {{string_type}}) AS field3 ,cast(CASE WHEN id <= 3 THEN NULL ELSE field4 END as {{string_type}}) AS field4 from source_data """ _MODELS__INCREMENTAL_APPEND_NEW_COLUMNS = """ {{ config( materialized='incremental', unique_key='id', on_schema_change='append_new_columns' ) }} {% set string_type = 'varchar(10)' %} WITH source_data AS (SELECT * FROM {{ ref('model_a') }} ) {% if is_incremental() %} SELECT id, cast(field1 as {{string_type}}) as field1, cast(field2 as {{string_type}}) as field2, cast(field3 as {{string_type}}) as field3, cast(field4 as {{string_type}}) as field4 FROM source_data WHERE id NOT IN (SELECT id from {{ this }} ) {% else %} SELECT id, cast(field1 as {{string_type}}) as field1, cast(field2 as {{string_type}}) as field2 FROM source_data where id <= 3 {% endif %} """ _MODELS__INCREMENTAL_SYNC_ALL_COLUMNS_TARGET = """ {{ config(materialized='table') }} with source_data as ( select * from {{ ref('model_a') }} ) {% set string_type = 'varchar(10)' %} select id ,cast(field1 as {{string_type}}) as field1 --,field2 ,cast(case when id <= 3 then null else field3 end as {{string_type}}) as field3 ,cast(case when id <= 3 then null else field4 end as {{string_type}}) as field4 from source_data order by id """ _MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_REMOVE_ONE_TARGET = """ {{ config(materialized='table') }} {% set string_type = 'varchar(10)' %} with source_data as ( select * from {{ ref('model_a') }} ) select id, cast(field1 as {{string_type}}) as field1, cast(CASE WHEN id > 3 THEN NULL ELSE field2 END as {{string_type}}) AS field2, cast(CASE WHEN id <= 3 THEN NULL ELSE field3 END as {{string_type}}) AS field3, cast(CASE WHEN id <= 3 THEN NULL ELSE field4 END as {{string_type}}) AS field4 from source_data """ # # Tests # _TESTS__SELECT_FROM_INCREMENTAL_IGNORE = """ select * from {{ ref('incremental_ignore') }} where false """ _TESTS__SELECT_FROM_A = """ select * from {{ ref('model_a') }} where false """ _TESTS__SELECT_FROM_INCREMENTAL_APPEND_NEW_COLUMNS_TARGET = """ select * from {{ ref('incremental_append_new_columns_target') }} where false """ _TESTS__SELECT_FROM_INCREMENTAL_SYNC_ALL_COLUMNS = """ select * from {{ ref('incremental_sync_all_columns') }} where false """ _TESTS__SELECT_FROM_INCREMENTAL_SYNC_ALL_COLUMNS_TARGET = """ select * from {{ ref('incremental_sync_all_columns_target') }} where false """ _TESTS__SELECT_FROM_INCREMENTAL_IGNORE_TARGET = """ select * from {{ ref('incremental_ignore_target') }} where false """ _TESTS__SELECT_FROM_INCREMENTAL_APPEND_NEW_COLUMNS = """ select * from {{ ref('incremental_append_new_columns') }} where false """ ================================================ FILE: tests/functional/incremental_schema_tests/test_incremental_schema.py ================================================ import pytest from dbt.tests.util import check_relations_equal, run_dbt from tests.functional.incremental_schema_tests.fixtures import ( _MODELS__A, _MODELS__INCREMENTAL_APPEND_NEW_COLUMNS, _MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_REMOVE_ONE, _MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_REMOVE_ONE_TARGET, _MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_TARGET, _MODELS__INCREMENTAL_FAIL, _MODELS__INCREMENTAL_IGNORE, _MODELS__INCREMENTAL_IGNORE_TARGET, _MODELS__INCREMENTAL_SYNC_ALL_COLUMNS, _MODELS__INCREMENTAL_SYNC_ALL_COLUMNS_TARGET, _MODELS__INCREMENTAL_SYNC_REMOVE_ONLY, _MODELS__INCREMENTAL_SYNC_REMOVE_ONLY_TARGET, _PROPERTIES__SCHEMA, _TESTS__SELECT_FROM_A, _TESTS__SELECT_FROM_INCREMENTAL_APPEND_NEW_COLUMNS, _TESTS__SELECT_FROM_INCREMENTAL_APPEND_NEW_COLUMNS_TARGET, _TESTS__SELECT_FROM_INCREMENTAL_IGNORE, _TESTS__SELECT_FROM_INCREMENTAL_IGNORE_TARGET, _TESTS__SELECT_FROM_INCREMENTAL_SYNC_ALL_COLUMNS, _TESTS__SELECT_FROM_INCREMENTAL_SYNC_ALL_COLUMNS_TARGET, ) class TestIncrementalSchemaChange: @pytest.fixture(scope="class") def properties(self): return { "schema.yml": _PROPERTIES__SCHEMA, } @pytest.fixture(scope="class") def models(self): return { "incremental_sync_remove_only.sql": _MODELS__INCREMENTAL_SYNC_REMOVE_ONLY, "incremental_ignore.sql": _MODELS__INCREMENTAL_IGNORE, "incremental_sync_remove_only_target.sql": _MODELS__INCREMENTAL_SYNC_REMOVE_ONLY_TARGET, "incremental_ignore_target.sql": _MODELS__INCREMENTAL_IGNORE_TARGET, "incremental_fail.sql": _MODELS__INCREMENTAL_FAIL, "incremental_sync_all_columns.sql": _MODELS__INCREMENTAL_SYNC_ALL_COLUMNS, "incremental_append_new_columns_remove_one.sql": _MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_REMOVE_ONE, "model_a.sql": _MODELS__A, "incremental_append_new_columns_target.sql": _MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_TARGET, "incremental_append_new_columns.sql": _MODELS__INCREMENTAL_APPEND_NEW_COLUMNS, "incremental_sync_all_columns_target.sql": _MODELS__INCREMENTAL_SYNC_ALL_COLUMNS_TARGET, "incremental_append_new_columns_remove_one_target.sql": _MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_REMOVE_ONE_TARGET, } @pytest.fixture(scope="class") def tests(self): return { "select_from_incremental.sql": _TESTS__SELECT_FROM_INCREMENTAL_IGNORE, "select_from_a.sql": _TESTS__SELECT_FROM_A, "select_from_incremental_append_new_columns_target.sql": _TESTS__SELECT_FROM_INCREMENTAL_APPEND_NEW_COLUMNS_TARGET, "select_from_incremental_sync_all_columns.sql": _TESTS__SELECT_FROM_INCREMENTAL_SYNC_ALL_COLUMNS, "select_from_incremental_sync_all_columns_target.sql": _TESTS__SELECT_FROM_INCREMENTAL_SYNC_ALL_COLUMNS_TARGET, "select_from_incremental_ignore_target.sql": _TESTS__SELECT_FROM_INCREMENTAL_IGNORE_TARGET, "select_from_incremental_append_new_columns.sql": _TESTS__SELECT_FROM_INCREMENTAL_APPEND_NEW_COLUMNS, } def run_twice_and_assert(self, include, compare_source, compare_target, project): # dbt run (twice) run_args = ["run"] if include: run_args.extend(("--select", include)) results_one = run_dbt(run_args) assert len(results_one) == 3 results_two = run_dbt(run_args) assert len(results_two) == 3 check_relations_equal(project.adapter, [compare_source, compare_target]) def run_incremental_append_new_columns(self, project): select = "model_a incremental_append_new_columns incremental_append_new_columns_target" compare_source = "incremental_append_new_columns" compare_target = "incremental_append_new_columns_target" self.run_twice_and_assert(select, compare_source, compare_target, project) def run_incremental_append_new_columns_remove_one(self, project): select = "model_a incremental_append_new_columns_remove_one incremental_append_new_columns_remove_one_target" compare_source = "incremental_append_new_columns_remove_one" compare_target = "incremental_append_new_columns_remove_one_target" self.run_twice_and_assert(select, compare_source, compare_target, project) def run_incremental_sync_all_columns(self, project): select = "model_a incremental_sync_all_columns incremental_sync_all_columns_target" compare_source = "incremental_sync_all_columns" compare_target = "incremental_sync_all_columns_target" self.run_twice_and_assert(select, compare_source, compare_target, project) def run_incremental_sync_remove_only(self, project): select = "model_a incremental_sync_remove_only incremental_sync_remove_only_target" compare_source = "incremental_sync_remove_only" compare_target = "incremental_sync_remove_only_target" self.run_twice_and_assert(select, compare_source, compare_target, project) def test_run_incremental_ignore(self, project): select = "model_a incremental_ignore incremental_ignore_target" compare_source = "incremental_ignore" compare_target = "incremental_ignore_target" self.run_twice_and_assert(select, compare_source, compare_target, project) def test_run_incremental_append_new_columns(self, project): self.run_incremental_append_new_columns(project) self.run_incremental_append_new_columns_remove_one(project) def test_run_incremental_sync_all_columns(self, project): self.run_incremental_sync_all_columns(project) self.run_incremental_sync_remove_only(project) def test_run_incremental_fail_on_schema_change(self, project): select = "model_a incremental_fail" run_dbt(["run", "--models", select, "--full-refresh"]) results_two = run_dbt(["run", "--models", select], expect_pass=False) assert "Compilation Error" in results_two[1].message ================================================ FILE: tests/functional/init/test_init.py ================================================ import os from pathlib import Path from unittest import mock from unittest.mock import Mock, call import click import pytest import yaml from dbt.exceptions import DbtRuntimeError from dbt.task.init import InitTask from dbt.tests.util import run_dbt class TestInitProjectWithExistingProfilesYml: @mock.patch("dbt.task.init.InitTask._run_debug") @mock.patch("dbt.task.init._get_adapter_plugin_names") @mock.patch("click.confirm") @mock.patch("click.prompt") def test_init_task_in_project_with_existing_profiles_yml( self, mock_prompt, mock_confirm, mock_get_adapter, mock_run_debug, project ): manager = Mock() manager.attach_mock(mock_prompt, "prompt") manager.attach_mock(mock_confirm, "confirm") manager.confirm.side_effect = ["y"] manager.prompt.side_effect = [ 1, "localhost", 5432, "test_user", "test_password", "test_db", "test_schema", 4, ] mock_get_adapter.return_value = [project.adapter.type()] run_dbt(["init"]) manager.assert_has_calls( [ call.confirm( f"The profile test already exists in {os.path.join(project.profiles_dir, 'profiles.yml')}. Continue and overwrite it?" ), call.prompt( "Which database would you like to use?\n[1] postgres\n\n(Don't see the one you want? https://docs.getdbt.com/docs/available-adapters)\n\nEnter a number", type=click.INT, ), call.prompt( "host (hostname for the instance)", default=None, hide_input=False, type=None ), call.prompt("port", default=5432, hide_input=False, type=click.INT), call.prompt("user (dev username)", default=None, hide_input=False, type=None), call.prompt("pass (dev password)", default=None, hide_input=True, type=None), call.prompt( "dbname (default database that dbt will build objects in)", default=None, hide_input=False, type=None, ), call.prompt( "schema (default schema that dbt will build objects in)", default=None, hide_input=False, type=None, ), call.prompt("threads (1 or more)", default=1, hide_input=False, type=click.INT), ] ) with open(os.path.join(project.profiles_dir, "profiles.yml"), "r") as f: assert ( f.read() == """test: outputs: dev: dbname: test_db host: localhost pass: test_password port: 5432 schema: test_schema threads: 4 type: postgres user: test_user target: dev """ ) mock_run_debug.assert_called_once() def test_init_task_in_project_specifying_profile_errors(self, project): with pytest.raises(DbtRuntimeError) as error: run_dbt(["init", "--profile", "test"], expect_pass=False) assert "Can not init existing project with specified profile" in str(error) class TestInitProjectWithoutExistingProfilesYml: @mock.patch("dbt.task.init.InitTask._run_debug") @mock.patch("dbt.task.init._get_adapter_plugin_names") @mock.patch("click.prompt") @mock.patch.object(Path, "exists", autospec=True) def test_init_task_in_project_without_existing_profiles_yml( self, exists, mock_prompt, mock_get_adapter, mock_run_debug, project ): def exists_side_effect(path): # Override responses on specific files, default to 'real world' if not overriden return {"profiles.yml": False}.get(path.name, os.path.exists(path)) exists.side_effect = exists_side_effect manager = Mock() manager.attach_mock(mock_prompt, "prompt") manager.prompt.side_effect = [ 1, "localhost", 5432, "test_user", "test_password", "test_db", "test_schema", 4, ] mock_get_adapter.return_value = [project.adapter.type()] run_dbt(["init"]) manager.assert_has_calls( [ call.prompt( "Which database would you like to use?\n[1] postgres\n\n(Don't see the one you want? https://docs.getdbt.com/docs/available-adapters)\n\nEnter a number", type=click.INT, ), call.prompt( "host (hostname for the instance)", default=None, hide_input=False, type=None ), call.prompt("port", default=5432, hide_input=False, type=click.INT), call.prompt("user (dev username)", default=None, hide_input=False, type=None), call.prompt("pass (dev password)", default=None, hide_input=True, type=None), call.prompt( "dbname (default database that dbt will build objects in)", default=None, hide_input=False, type=None, ), call.prompt( "schema (default schema that dbt will build objects in)", default=None, hide_input=False, type=None, ), call.prompt("threads (1 or more)", default=1, hide_input=False, type=click.INT), ] ) with open(os.path.join(project.profiles_dir, "profiles.yml"), "r") as f: assert ( f.read() == """test: outputs: dev: dbname: test_db host: localhost pass: test_password port: 5432 schema: test_schema threads: 4 type: postgres user: test_user target: dev """ ) mock_run_debug.assert_called_once() def test_init_task_in_project_without_profile_yml_specifying_profile_errors(self, project): # Even without profiles.yml, init inside a project with --profile should error with pytest.raises(DbtRuntimeError) as error: run_dbt(["init", "--profile", "test"], expect_pass=False) assert "Can not init existing project with specified profile" in str(error) class TestInitProjectWithoutExistingProfilesYmlOrTemplate: @mock.patch("dbt.task.init.InitTask._run_debug") @mock.patch("dbt.task.init._get_adapter_plugin_names") @mock.patch("click.confirm") @mock.patch("click.prompt") @mock.patch.object(Path, "exists", autospec=True) def test_init_task_in_project_without_existing_profiles_yml_or_profile_template( self, exists, mock_prompt, mock_confirm, mock_get_adapter, mock_run_debug, project ): def exists_side_effect(path): # Override responses on specific files, default to 'real world' if not overriden return { "profiles.yml": False, "profile_template.yml": False, }.get(path.name, os.path.exists(path)) exists.side_effect = exists_side_effect manager = Mock() manager.attach_mock(mock_prompt, "prompt") manager.attach_mock(mock_confirm, "confirm") manager.prompt.side_effect = [ 1, ] mock_get_adapter.return_value = [project.adapter.type()] run_dbt(["init"]) manager.assert_has_calls( [ call.prompt( "Which database would you like to use?\n[1] postgres\n\n(Don't see the one you want? https://docs.getdbt.com/docs/available-adapters)\n\nEnter a number", type=click.INT, ), ] ) with open(os.path.join(project.profiles_dir, "profiles.yml"), "r") as f: assert ( f.read() == """test: outputs: dev: type: postgres threads: [1 or more] host: [host] port: [port] user: [dev_username] pass: [dev_password] dbname: [dbname] schema: [dev_schema] prod: type: postgres threads: [1 or more] host: [host] port: [port] user: [prod_username] pass: [prod_password] dbname: [dbname] schema: [prod_schema] target: dev """ ) mock_run_debug.assert_called_once() class TestInitProjectWithProfileTemplateWithoutExistingProfilesYml: @mock.patch("dbt.task.init.InitTask._run_debug") @mock.patch("dbt.task.init._get_adapter_plugin_names") @mock.patch("click.confirm") @mock.patch("click.prompt") @mock.patch.object(Path, "exists", autospec=True) def test_init_task_in_project_with_profile_template_without_existing_profiles_yml( self, exists, mock_prompt, mock_confirm, mock_get_adapter, mock_run_debug, project ): def exists_side_effect(path): # Override responses on specific files, default to 'real world' if not overriden return { "profiles.yml": False, }.get(path.name, os.path.exists(path)) exists.side_effect = exists_side_effect with open("profile_template.yml", "w") as f: f.write( """fixed: type: postgres threads: 4 host: localhost dbname: my_db schema: my_schema target: my_target prompts: target: hint: 'The target name' type: string port: hint: 'The port (for integer test purposes)' type: int default: 5432 user: hint: 'Your username' pass: hint: 'Your password' hide_input: true""" ) manager = Mock() manager.attach_mock(mock_prompt, "prompt") manager.attach_mock(mock_confirm, "confirm") manager.prompt.side_effect = ["my_target", 5432, "test_username", "test_password"] mock_get_adapter.return_value = [project.adapter.type()] run_dbt(["init"]) manager.assert_has_calls( [ call.prompt( "target (The target name)", default=None, hide_input=False, type=click.STRING ), call.prompt( "port (The port (for integer test purposes))", default=5432, hide_input=False, type=click.INT, ), call.prompt("user (Your username)", default=None, hide_input=False, type=None), call.prompt("pass (Your password)", default=None, hide_input=True, type=None), ] ) with open(os.path.join(project.profiles_dir, "profiles.yml"), "r") as f: assert ( f.read() == """test: outputs: my_target: dbname: my_db host: localhost pass: test_password port: 5432 schema: my_schema threads: 4 type: postgres user: test_username target: my_target """ ) mock_run_debug.assert_called_once() class TestInitInvalidProfileTemplate: @mock.patch("dbt.task.init.InitTask._run_debug") @mock.patch("dbt.task.init._get_adapter_plugin_names") @mock.patch("click.confirm") @mock.patch("click.prompt") def test_init_task_in_project_with_invalid_profile_template( self, mock_prompt, mock_confirm, mock_get_adapter, mock_run_debug, project ): """Test that when an invalid profile_template.yml is provided in the project, init command falls back to the target's profile_template.yml""" with open(os.path.join(project.project_root, "profile_template.yml"), "w") as f: f.write("""invalid template""") manager = Mock() manager.attach_mock(mock_prompt, "prompt") manager.attach_mock(mock_confirm, "confirm") manager.confirm.side_effect = ["y"] manager.prompt.side_effect = [ 1, "localhost", 5432, "test_username", "test_password", "test_db", "test_schema", 4, ] mock_get_adapter.return_value = [project.adapter.type()] run_dbt(["init"]) manager.assert_has_calls( [ call.confirm( f"The profile test already exists in {os.path.join(project.profiles_dir, 'profiles.yml')}. Continue and overwrite it?" ), call.prompt( "Which database would you like to use?\n[1] postgres\n\n(Don't see the one you want? https://docs.getdbt.com/docs/available-adapters)\n\nEnter a number", type=click.INT, ), call.prompt( "host (hostname for the instance)", default=None, hide_input=False, type=None ), call.prompt("port", default=5432, hide_input=False, type=click.INT), call.prompt("user (dev username)", default=None, hide_input=False, type=None), call.prompt("pass (dev password)", default=None, hide_input=True, type=None), call.prompt( "dbname (default database that dbt will build objects in)", default=None, hide_input=False, type=None, ), call.prompt( "schema (default schema that dbt will build objects in)", default=None, hide_input=False, type=None, ), call.prompt("threads (1 or more)", default=1, hide_input=False, type=click.INT), ] ) with open(os.path.join(project.profiles_dir, "profiles.yml"), "r") as f: assert ( f.read() == """test: outputs: dev: dbname: test_db host: localhost pass: test_password port: 5432 schema: test_schema threads: 4 type: postgres user: test_username target: dev """ ) mock_run_debug.assert_called_once() class TestInitInsideOfProjectBase: @pytest.fixture(scope="class") def project_name(self, unique_schema): return f"my_project_{unique_schema}" class TestInitOutsideOfProjectBase: @pytest.fixture(scope="class") def project_name(self, unique_schema): return f"my_project_{unique_schema}" @pytest.fixture(scope="class", autouse=True) def setup(self, project): # Start by removing the dbt_project.yml so that we're not in an existing project os.remove(os.path.join(project.project_root, "dbt_project.yml")) class TestInitOutsideOfProject(TestInitOutsideOfProjectBase): @pytest.fixture(scope="class") def dbt_profile_data(self, unique_schema): return { "test": { "outputs": { "default2": { "type": "postgres", "threads": 4, "host": "localhost", "port": int(os.getenv("POSTGRES_TEST_PORT", 5432)), "user": os.getenv("POSTGRES_TEST_USER", "root"), "pass": os.getenv("POSTGRES_TEST_PASS", "password"), "dbname": os.getenv("POSTGRES_TEST_DATABASE", "dbt"), "schema": unique_schema, }, "noaccess": { "type": "postgres", "threads": 4, "host": "localhost", "port": int(os.getenv("POSTGRES_TEST_PORT", 5432)), "user": "noaccess", "pass": "password", "dbname": os.getenv("POSTGRES_TEST_DATABASE", "dbt"), "schema": unique_schema, }, }, "target": "default2", }, } @mock.patch("dbt.task.init.InitTask._run_debug") @mock.patch("dbt.task.init._get_adapter_plugin_names") @mock.patch("click.confirm") @mock.patch("click.prompt") def test_init_task_outside_of_project( self, mock_prompt, mock_confirm, mock_get_adapter, mock_run_debug, project, project_name, unique_schema, ): manager = Mock() manager.attach_mock(mock_prompt, "prompt") manager.attach_mock(mock_confirm, "confirm") manager.prompt.side_effect = [ project_name, 1, "localhost", 5432, "test_username", "test_password", "test_db", "test_schema", 4, ] mock_get_adapter.return_value = [project.adapter.type()] run_dbt(["init"]) manager.assert_has_calls( [ call.prompt("Enter a name for your project (letters, digits, underscore)"), call.prompt( "Which database would you like to use?\n[1] postgres\n\n(Don't see the one you want? https://docs.getdbt.com/docs/available-adapters)\n\nEnter a number", type=click.INT, ), call.prompt( "host (hostname for the instance)", default=None, hide_input=False, type=None ), call.prompt("port", default=5432, hide_input=False, type=click.INT), call.prompt("user (dev username)", default=None, hide_input=False, type=None), call.prompt("pass (dev password)", default=None, hide_input=True, type=None), call.prompt( "dbname (default database that dbt will build objects in)", default=None, hide_input=False, type=None, ), call.prompt( "schema (default schema that dbt will build objects in)", default=None, hide_input=False, type=None, ), call.prompt("threads (1 or more)", default=1, hide_input=False, type=click.INT), ] ) mock_run_debug.assert_called_once() with open(os.path.join(project.profiles_dir, "profiles.yml"), "r") as f: assert ( f.read() == f"""{project_name}: outputs: dev: dbname: test_db host: localhost pass: test_password port: 5432 schema: test_schema threads: 4 type: postgres user: test_username target: dev test: outputs: default2: dbname: dbt host: localhost pass: password port: 5432 schema: {unique_schema} threads: 4 type: postgres user: root noaccess: dbname: dbt host: localhost pass: password port: 5432 schema: {unique_schema} threads: 4 type: postgres user: noaccess target: default2 """ ) with open(os.path.join(project.project_root, project_name, "dbt_project.yml"), "r") as f: assert ( f.read() == f""" # Name your project! Project names should contain only lowercase characters # and underscores. A good package name should reflect your organization's # name or the intended use of these models name: '{project_name}' version: '1.0.0' # This setting configures which "profile" dbt uses for this project. profile: '{project_name}' # These configurations specify where dbt should look for different types of files. # The `model-paths` config, for example, states that models in this project can be # found in the "models/" directory. You probably won't need to change these! model-paths: ["models"] analysis-paths: ["analyses"] test-paths: ["tests"] seed-paths: ["seeds"] macro-paths: ["macros"] snapshot-paths: ["snapshots"] clean-targets: # directories to be removed by `dbt clean` - "target" - "dbt_packages" # Configuring models # Full documentation: https://docs.getdbt.com/docs/configuring-models # In this example config, we tell dbt to build all models in the example/ # directory as views. These settings can be overridden in the individual model # files using the `{{{{ config(...) }}}}` macro. models: {project_name}: # Config indicated by + and applies to all files under models/example/ example: +materialized: view """ ) class TestInitInvalidProjectNameCLI(TestInitOutsideOfProjectBase): @mock.patch("dbt.task.init._get_adapter_plugin_names") @mock.patch("click.confirm") @mock.patch("click.prompt") def test_init_invalid_project_name_cli( self, mock_prompt, mock_confirm, mock_get_adapter, project_name, project ): manager = Mock() manager.attach_mock(mock_prompt, "prompt") manager.attach_mock(mock_confirm, "confirm") invalid_name = "name-with-hyphen" valid_name = project_name manager.prompt.side_effect = [valid_name] mock_get_adapter.return_value = [project.adapter.type()] run_dbt(["init", invalid_name, "--skip-profile-setup"]) manager.assert_has_calls( [ call.prompt("Enter a name for your project (letters, digits, underscore)"), ] ) class TestInitInvalidProjectNamePrompt(TestInitOutsideOfProjectBase): @mock.patch("dbt.task.init._get_adapter_plugin_names") @mock.patch("click.confirm") @mock.patch("click.prompt") def test_init_invalid_project_name_prompt( self, mock_prompt, mock_confirm, mock_get_adapter, project_name, project ): manager = Mock() manager.attach_mock(mock_prompt, "prompt") manager.attach_mock(mock_confirm, "confirm") invalid_name = "name-with-hyphen" valid_name = project_name manager.prompt.side_effect = [invalid_name, valid_name] mock_get_adapter.return_value = [project.adapter.type()] run_dbt(["init", "--skip-profile-setup"]) manager.assert_has_calls( [ call.prompt("Enter a name for your project (letters, digits, underscore)"), call.prompt("Enter a name for your project (letters, digits, underscore)"), ] ) class TestInitProvidedProjectNameAndSkipProfileSetup(TestInitOutsideOfProjectBase): @mock.patch("dbt.task.init._get_adapter_plugin_names") @mock.patch("click.confirm") @mock.patch("click.prompt") def test_init_provided_project_name_and_skip_profile_setup( self, mock_prompt, mock_confirm, mock_get, project, project_name ): manager = mock.Mock() manager.attach_mock(mock_prompt, "prompt") manager.attach_mock(mock_confirm, "confirm") manager.prompt.side_effect = [ 1, "localhost", 5432, "test_username", "test_password", "test_db", "test_schema", 4, ] mock_get.return_value = [project.adapter.type()] # provide project name through the init command run_dbt(["init", project_name, "--skip-profile-setup"]) assert len(manager.mock_calls) == 0 with open(os.path.join(project.project_root, project_name, "dbt_project.yml"), "r") as f: assert ( f.read() == f""" # Name your project! Project names should contain only lowercase characters # and underscores. A good package name should reflect your organization's # name or the intended use of these models name: '{project_name}' version: '1.0.0' # This setting configures which "profile" dbt uses for this project. profile: '{project_name}' # These configurations specify where dbt should look for different types of files. # The `model-paths` config, for example, states that models in this project can be # found in the "models/" directory. You probably won't need to change these! model-paths: ["models"] analysis-paths: ["analyses"] test-paths: ["tests"] seed-paths: ["seeds"] macro-paths: ["macros"] snapshot-paths: ["snapshots"] clean-targets: # directories to be removed by `dbt clean` - "target" - "dbt_packages" # Configuring models # Full documentation: https://docs.getdbt.com/docs/configuring-models # In this example config, we tell dbt to build all models in the example/ # directory as views. These settings can be overridden in the individual model # files using the `{{{{ config(...) }}}}` macro. models: {project_name}: # Config indicated by + and applies to all files under models/example/ example: +materialized: view """ ) class TestInitInsideProjectAndSkipProfileSetup(TestInitInsideOfProjectBase): @mock.patch("dbt.task.init.InitTask._run_debug") @mock.patch("dbt.task.init._get_adapter_plugin_names") @mock.patch("click.confirm") @mock.patch("click.prompt") def test_init_inside_project_and_skip_profile_setup( self, mock_prompt, mock_confirm, mock_get, mock_debug, project, project_name ): manager = mock.Mock() manager.attach_mock(mock_prompt, "prompt") manager.attach_mock(mock_confirm, "confirm") assert Path("dbt_project.yml").exists() # skip interactive profile setup run_dbt(["init", "--skip-profile-setup"]) assert len(manager.mock_calls) == 0 mock_debug.assert_not_called() class TestInitOutsideOfProjectWithSpecifiedProfile(TestInitOutsideOfProjectBase): @mock.patch("dbt.task.init.InitTask._run_debug") @mock.patch("dbt.task.init._get_adapter_plugin_names") @mock.patch("click.prompt") def test_init_task_outside_of_project_with_specified_profile( self, mock_prompt, mock_get_adapter, mock_run_debug, project, project_name, unique_schema, dbt_profile_data, ): manager = Mock() manager.attach_mock(mock_prompt, "prompt") manager.prompt.side_effect = [ project_name, ] mock_get_adapter.return_value = [project.adapter.type()] run_dbt(["init", "--profile", "test"]) manager.assert_has_calls( [ call.prompt("Enter a name for your project (letters, digits, underscore)"), ] ) # profiles.yml is NOT overwritten, so assert that the text matches that of the # original fixture with open(os.path.join(project.profiles_dir, "profiles.yml"), "r") as f: assert f.read() == yaml.safe_dump(dbt_profile_data) with open(os.path.join(project.project_root, project_name, "dbt_project.yml"), "r") as f: assert ( f.read() == f""" # Name your project! Project names should contain only lowercase characters # and underscores. A good package name should reflect your organization's # name or the intended use of these models name: '{project_name}' version: '1.0.0' # This setting configures which "profile" dbt uses for this project. profile: 'test' # These configurations specify where dbt should look for different types of files. # The `model-paths` config, for example, states that models in this project can be # found in the "models/" directory. You probably won't need to change these! model-paths: ["models"] analysis-paths: ["analyses"] test-paths: ["tests"] seed-paths: ["seeds"] macro-paths: ["macros"] snapshot-paths: ["snapshots"] clean-targets: # directories to be removed by `dbt clean` - "target" - "dbt_packages" # Configuring models # Full documentation: https://docs.getdbt.com/docs/configuring-models # In this example config, we tell dbt to build all models in the example/ # directory as views. These settings can be overridden in the individual model # files using the `{{{{ config(...) }}}}` macro. models: {project_name}: # Config indicated by + and applies to all files under models/example/ example: +materialized: view """ ) mock_run_debug.assert_called_once() class TestInitOutsideOfProjectSpecifyingInvalidProfile(TestInitOutsideOfProjectBase): @mock.patch("dbt.task.init._get_adapter_plugin_names") @mock.patch("click.prompt") def test_init_task_outside_project_specifying_invalid_profile_errors( self, mock_prompt, mock_get_adapter, project, project_name ): manager = Mock() manager.attach_mock(mock_prompt, "prompt") manager.prompt.side_effect = [ project_name, ] mock_get_adapter.return_value = [project.adapter.type()] with pytest.raises(DbtRuntimeError) as error: run_dbt(["init", "--profile", "invalid"], expect_pass=False) assert "Could not find profile named invalid" in str(error) manager.assert_has_calls( [ call.prompt("Enter a name for your project (letters, digits, underscore)"), ] ) class TestInitOutsideOfProjectSpecifyingProfileNoProfilesYml(TestInitOutsideOfProjectBase): @mock.patch("dbt.task.init._get_adapter_plugin_names") @mock.patch("click.prompt") def test_init_task_outside_project_specifying_profile_no_profiles_yml_errors( self, mock_prompt, mock_get_adapter, project, project_name ): manager = Mock() manager.attach_mock(mock_prompt, "prompt") manager.prompt.side_effect = [ project_name, ] mock_get_adapter.return_value = [project.adapter.type()] # Override responses on specific files, default to 'real world' if not overriden original_isfile = os.path.isfile with mock.patch( "os.path.isfile", new=lambda path: {"profiles.yml": False}.get( os.path.basename(path), original_isfile(path) ), ): with pytest.raises(DbtRuntimeError) as error: run_dbt(["init", "--profile", "test"], expect_pass=False) assert "Could not find profile named invalid" in str(error) manager.assert_has_calls( [ call.prompt("Enter a name for your project (letters, digits, underscore)"), ] ) class TestInitRunsDebugAfterInit(TestInitInsideOfProjectBase): @mock.patch("dbt.task.init._get_adapter_plugin_names") @mock.patch("click.confirm") @mock.patch("click.prompt") def test_debug_runs_after_init(self, mock_prompt, mock_confirm, mock_get_adapter, project): """Verify DebugTask is instantiated and run() is called after init with profile setup.""" manager = Mock() manager.attach_mock(mock_prompt, "prompt") manager.attach_mock(mock_confirm, "confirm") manager.confirm.side_effect = ["y"] manager.prompt.side_effect = [ 1, "localhost", 5432, "test_user", "test_password", "test_db", "test_schema", 4, ] mock_get_adapter.return_value = [project.adapter.type()] mock_debug_task_cls = Mock() with mock.patch("dbt.task.debug.DebugTask", mock_debug_task_cls): run_dbt(["init"]) mock_debug_task_cls.return_value.run.assert_called_once() class TestInitDebugSkippedWithSkipProfileSetup(TestInitInsideOfProjectBase): @mock.patch("dbt.task.init.InitTask._run_debug") @mock.patch("dbt.task.init._get_adapter_plugin_names") @mock.patch("click.confirm") @mock.patch("click.prompt") def test_debug_skipped_with_skip_profile_setup( self, mock_prompt, mock_confirm, mock_get_adapter, mock_run_debug, project ): """Verify _run_debug is NOT called when --skip-profile-setup is used.""" run_dbt(["init", "--skip-profile-setup"]) mock_run_debug.assert_not_called() class TestInitSkipDebugFlag(TestInitInsideOfProjectBase): @mock.patch("dbt.task.init._get_adapter_plugin_names") @mock.patch("click.confirm") @mock.patch("click.prompt") def test_debug_skipped_with_skip_debug_flag( self, mock_prompt, mock_confirm, mock_get_adapter, project ): """Verify DebugTask is NOT instantiated when --skip-debug is used.""" manager = Mock() manager.attach_mock(mock_prompt, "prompt") manager.attach_mock(mock_confirm, "confirm") manager.confirm.side_effect = ["y"] manager.prompt.side_effect = [ 1, "localhost", 5432, "test_user", "test_password", "test_db", "test_schema", 4, ] mock_get_adapter.return_value = [project.adapter.type()] mock_debug_task_cls = Mock() with mock.patch("dbt.task.debug.DebugTask", mock_debug_task_cls): run_dbt(["init", "--skip-debug"]) mock_debug_task_cls.assert_not_called() def test_run_debug_returns_none_when_skip_debug(self): """Verify _run_debug returns None when skip_debug is True.""" task = Mock(spec=InitTask) task.args = Mock(skip_debug=True) assert InitTask._run_debug(task) is None class TestInitDebugFailureDoesNotFailInit(TestInitInsideOfProjectBase): @mock.patch("dbt.task.init._get_adapter_plugin_names") @mock.patch("click.confirm") @mock.patch("click.prompt") def test_debug_failure_does_not_fail_init( self, mock_prompt, mock_confirm, mock_get_adapter, project ): """Verify that if DebugTask.run() raises, init still succeeds.""" manager = Mock() manager.attach_mock(mock_prompt, "prompt") manager.attach_mock(mock_confirm, "confirm") manager.confirm.side_effect = ["y"] manager.prompt.side_effect = [ 1, "localhost", 5432, "test_user", "test_password", "test_db", "test_schema", 4, ] mock_get_adapter.return_value = [project.adapter.type()] mock_debug_task_cls = Mock() mock_debug_task_cls.return_value.run.side_effect = Exception("connection failed") with mock.patch("dbt.task.debug.DebugTask", mock_debug_task_cls): # Should not raise — debug failure is informational only run_dbt(["init"]) ================================================ FILE: tests/functional/invalid_model_tests/test_invalid_models.py ================================================ import pytest from dbt.exceptions import CompilationError, ParsingError from dbt.tests.util import run_dbt # from `test/integration/011_invalid_model_tests`, invalid_model_tests # # Seeds # seeds__base_seed = """ first_name,last_name,email,gender,ip_address Jack,Hunter,jhunter0@pbs.org,Male,59.80.20.168 Kathryn,Walker,kwalker1@ezinearticles.com,Female,194.121.179.35 Gerald,Ryan,gryan2@com.com,Male,11.3.212.243 Bonnie,Spencer,bspencer3@ameblo.jp,Female,216.32.196.175 Harold,Taylor,htaylor4@people.com.cn,Male,253.10.246.136 Jacqueline,Griffin,jgriffin5@t.co,Female,16.13.192.220 Wanda,Arnold,warnold6@google.nl,Female,232.116.150.64 Craig,Ortiz,cortiz7@sciencedaily.com,Male,199.126.106.13 Gary,Day,gday8@nih.gov,Male,35.81.68.186 Rose,Wright,rwright9@yahoo.co.jp,Female,236.82.178.100 """ # # Properties # properties__seed_types_yml = """ version: 2 seeds: - name: seeds__base_seed config: +column_types: first_name: varchar(50), last_name: varchar(50), email: varchar(50), gender: varchar(50), ip_address: varchar(20) """ # see config in test class properties__disabled_source_yml = """ version: 2 sources: - name: test_source schema: "{{ target.schema }}" tables: - name: test_table identifier: seed """ # # Macros # macros__bad_macros = """ {% macro some_macro(arg) %} {{ arg }} {% endmacro %} """ # # Models # models__view_bad_enabled_value = """ {{ config( enabled = 'false' ) }} select * from {{ this.schema }}.seed """ models__view_disabled = """ {{ config( enabled = False ) }} select * from {{ this.schema }}.seed """ models__dependent_on_view = """ select * from {{ ref('models__view_disabled') }} """ models__with_bad_macro = """ {{ some_macro(invalid='test') }} select 1 as id """ models__referencing_disabled_source = """ select * from {{ source('test_source', 'test_table') }} """ # # Tests # class InvalidModelBase(object): @pytest.fixture(scope="class") def seeds(self): return { "seeds__base_seed.csv": seeds__base_seed, } @pytest.fixture(scope="class") def properties(self): return { "properties__seed_types.yml": properties__seed_types_yml, } class TestMalformedEnabledParam(InvalidModelBase): @pytest.fixture(scope="class") def models(self): return { "models__view_bad_enabled_value.sql": models__view_bad_enabled_value, } def test_view_disabled(self, project): with pytest.raises(ParsingError) as exc: run_dbt(["seed"]) assert "enabled" in str(exc.value) class TestReferencingDisabledModel(InvalidModelBase): """Expects that the upstream model is disabled""" @pytest.fixture(scope="class") def models(self): return { "models__view_disabled.sql": models__view_disabled, "models__dependent_on_view.sql": models__dependent_on_view, } def test_referencing_disabled_model(self, project): with pytest.raises(CompilationError) as exc: run_dbt() assert "which is disabled" in str(exc.value) class TestMissingModelReference(InvalidModelBase): """Expects that the upstream model is not found""" @pytest.fixture(scope="class") def models(self): return {"models__dependent_on_view.sql": models__dependent_on_view} def test_models_not_found(self, project): with pytest.raises(CompilationError) as exc: run_dbt() assert "which was not found" in str(exc.value) class TestInvalidMacroCall(InvalidModelBase): @pytest.fixture(scope="class") def macros(self): return {"macros__bad_macros.sql": macros__bad_macros} @pytest.fixture(scope="class") def models(self): return {"models__with_bad_macro.sql": models__with_bad_macro} def test_with_invalid_macro_call(self, project): with pytest.raises(CompilationError) as exc: run_dbt(["compile"]) assert "macro 'dbt_macro__some_macro' takes no keyword argument 'invalid'" in str( exc.value ) class TestInvalidDisabledSource(InvalidModelBase): @pytest.fixture(scope="class") def properties(self): return { "properties__seed_types.yml": properties__seed_types_yml, "properties__disabled_source.yml": properties__disabled_source_yml, } @pytest.fixture(scope="class") def models(self): return {"models__referencing_disabled_source.sql": models__referencing_disabled_source} @pytest.fixture(scope="class") def project_config_update(self): return { "sources": { "test": { "enabled": False, } } } def test_postgres_source_disabled(self, project): with pytest.raises(CompilationError) as exc: run_dbt() assert "which is disabled" in str(exc.value) class TestInvalidMissingSource(InvalidModelBase): """like TestInvalidDisabledSource but source omitted entirely""" @pytest.fixture(scope="class") def models(self): return {"models__referencing_disabled_source.sql": models__referencing_disabled_source} def test_source_missing(self, project): with pytest.raises(CompilationError) as exc: run_dbt() assert "which was not found" in str(exc.value) ================================================ FILE: tests/functional/invalid_model_tests/test_model_logging.py ================================================ import pytest from dbt.tests.util import run_dbt_and_capture warnings_sql = """ {{ config(group='my_group') }} {% do exceptions.warn('warning: everything is terrible but not that terrible') %} {{ exceptions.warn("warning: everything is terrible but not that terrible") }} select 1 as id """ schema_yml = """ version: 2 groups: - name: my_group owner: name: group_owner """ class TestModelLogging: @pytest.fixture(scope="class") def models(self): return { "warnings.sql": warnings_sql, "schema.yml": schema_yml, } def test_warn(self, project): results, log_output = run_dbt_and_capture(["run", "--log-format", "json"]) log_lines = log_output.split("\n") log_lines_with_warning = [line for line in log_lines if "JinjaLogWarning" in line] assert len(log_lines_with_warning) == 4 assert all("everything is terrible" in line for line in log_lines_with_warning) log_lines_with_group = [line for line in log_lines if "LogModelResult" in line] assert len(log_lines_with_group) == 1 assert "group_owner" in log_lines_with_group[0] assert "my_group" in log_lines_with_group[0] ================================================ FILE: tests/functional/list/fixtures.py ================================================ import pytest from dbt.tests.fixtures.project import write_project_files snapshots__snapshot_sql = """ {% snapshot my_snapshot %} {{ config( target_database=var('target_database', database), target_schema=schema, unique_key='id', strategy='timestamp', updated_at='updated_at', ) }} select * from {{database}}.{{schema}}.seed {% endsnapshot %} """ tests__t_sql = """ select 1 as id limit 0 """ models__schema_yml = """ version: 2 models: - name: outer description: The outer table columns: - name: id description: The id value data_tests: - unique - not_null sources: - name: my_source tables: - name: my_table """ models__ephemeral_sql = """ {{ config(materialized='ephemeral') }} select 1 as id, {{ dbt.date_trunc('day', dbt.current_timestamp()) }} as created_at """ models__metric_flow = """ select {{ dbt.date_trunc('day', dbt.current_timestamp()) }} as date_day """ models__incremental_sql = """ {{ config( materialized = "incremental", incremental_strategy = "delete+insert", ) }} select * from {{ ref('seed') }} {% if is_incremental() %} where a > (select max(a) from {{this}}) {% endif %} """ models__docs_md = """ {% docs my_docs %} some docs {% enddocs %} """ models__outer_sql = """ select * from {{ ref('ephemeral') }} """ models__sub__inner_sql = """ select * from {{ ref('outer') }} """ macros__macro_stuff_sql = """ {% macro cool_macro() %} wow! {% endmacro %} {% macro other_cool_macro(a, b) %} cool! {% endmacro %} """ seeds__seed_csv = """a,b 1,2 """ analyses__a_sql = """ select 4 as id """ semantic_models__sm_yml = """ semantic_models: - name: my_sm model: ref('outer') defaults: agg_time_dimension: created_at entities: - name: my_entity type: primary expr: id dimensions: - name: created_at type: time type_params: time_granularity: day measures: - name: total_outer_count agg: count expr: 1 """ metrics__m_yml = """ metrics: - name: total_outer type: simple description: The total count of outer label: Total Outer type_params: measure: total_outer_count """ saved_queries__sq_yml = """ saved_queries: - name: my_saved_query label: My Saved Query query_params: metrics: - total_outer group_by: - "Dimension('my_entity__created_at')" exports: - name: my_export config: alias: my_export_alias export_as: table schema: my_export_schema_name """ @pytest.fixture(scope="class") def snapshots(): return {"snapshot.sql": snapshots__snapshot_sql} @pytest.fixture(scope="class") def tests(): return {"t.sql": tests__t_sql} @pytest.fixture(scope="class") def models(): return { "schema.yml": models__schema_yml, "ephemeral.sql": models__ephemeral_sql, "incremental.sql": models__incremental_sql, "docs.md": models__docs_md, "outer.sql": models__outer_sql, "metricflow_time_spine.sql": models__metric_flow, "sq.yml": saved_queries__sq_yml, "sm.yml": semantic_models__sm_yml, "m.yml": metrics__m_yml, "sub": {"inner.sql": models__sub__inner_sql}, } @pytest.fixture(scope="class") def macros(): return {"macro_stuff.sql": macros__macro_stuff_sql} @pytest.fixture(scope="class") def seeds(): return {"seed.csv": seeds__seed_csv} @pytest.fixture(scope="class") def analyses(): return {"a.sql": analyses__a_sql} @pytest.fixture(scope="class") def semantic_models(): return {"sm.yml": semantic_models__sm_yml} @pytest.fixture(scope="class") def metrics(): return {"m.yml": metrics__m_yml} @pytest.fixture(scope="class") def saved_queries(): return {"sq.yml": saved_queries__sq_yml} @pytest.fixture(scope="class") def project_files( project_root, snapshots, tests, models, macros, seeds, analyses, ): write_project_files(project_root, "snapshots", snapshots) write_project_files(project_root, "tests", tests) write_project_files(project_root, "models", models) write_project_files(project_root, "macros", macros) write_project_files(project_root, "seeds", seeds) write_project_files(project_root, "analyses", analyses) ================================================ FILE: tests/functional/list/test_commands.py ================================================ import shutil import pytest from dbt.artifacts.resources.types import NodeType from dbt.cli.main import dbtRunner from dbt.cli.types import Command from dbt.events.types import NoNodesSelected from dbt.tests.util import run_dbt from dbt_common.events.event_catcher import EventCatcher """ Testing different commands against the happy path fixture The general flow 1. Declare the commands to be tested 2. Write a paramaterized test ensure a given command reaches causes and associated desired state. """ # These are commands we're skipping as they don't make sense or don't work with the # happy path fixture currently commands_to_skip = { "clone", "generate", "server", "init", "list", "run-operation", "show", "snapshot", "freshness", "serve", # this was actually serving up docs and caused issues locally } # Commands to happy path test commands = [command.to_list() for command in Command if command.value not in commands_to_skip] class TestRunCommands: @pytest.fixture(scope="class", autouse=True) def drop_snapshots(self, happy_path_project, project_root: str) -> None: """The snapshots are erroring out, so lets drop them. Seems to be database related. Ideally snapshots should work in these tests. It's a bad sign that they don't. That may have more to do with our fixture setup than the source code though. Note: that the `happy_path_fixture_files` are a _class_ based fixture. Thus although this fixture _modifies_ the files available to the happy path project, it doesn't affect that fixture for tests in other test classes. """ shutil.rmtree(f"{project_root}/snapshots") def test_run_commmand( self, happy_path_project, ): for command in commands: run_dbt(command) """ Testing command interactions with specific node types The general flow 1. Declare resource (node) types to be tested 2. Write a parameterized test that ensures commands interact successfully with each resource type """ # TODO: Figure out which of these are just missing from the happy path fixture vs which ones aren't selectable skipped_resource_types = { "analysis", "operation", "rpc", "sql_operation", "doc", "macro", "exposure", "group", "unit_test", "fixture", } resource_types = [ node_type.value for node_type in NodeType if node_type.value not in skipped_resource_types ] class TestSelectResourceType: @pytest.fixture(scope="function") def catcher(self) -> EventCatcher: return EventCatcher(event_to_catch=NoNodesSelected) @pytest.fixture(scope="function") def runner(self, catcher: EventCatcher) -> dbtRunner: return dbtRunner(callbacks=[catcher.catch]) @pytest.mark.parametrize("resource_type", resource_types) def test_select_by_resource_type( self, resource_type: str, happy_path_project, runner: dbtRunner, catcher: EventCatcher, ) -> None: runner.invoke(["list", "--select", f"resource_type:{resource_type}"]) assert len(catcher.caught_events) == 0 ================================================ FILE: tests/functional/list/test_list.py ================================================ import json import os from dbt.tests.util import run_dbt from tests.functional.fixtures.happy_path_fixture import ( # noqa: F401 happy_path_project, happy_path_project_files, ) # Marker to allow some objects to skip full comparison when a full comparison is # unlikely to improve practical test effectiveness. ANY = object() class TestList: def dir(self, value): return os.path.normpath(value) def test_packages_install_path_does_not_exist(self, happy_path_project): # noqa: F811 run_dbt(["list"]) packages_install_path = "dbt_packages" # the packages-install-path should not be created by `dbt list` assert not os.path.exists(packages_install_path) def run_dbt_ls(self, args=None, expect_pass=True): full_args = ["ls"] if args is not None: full_args += args result = run_dbt(args=full_args, expect_pass=expect_pass) return result def assert_json_equal(self, json_str, expected): assert json.loads(json_str) == expected def expect_given_output(self, args, expectations): for key, values in expectations.items(): ls_result = self.run_dbt_ls(args + ["--output", key]) if not isinstance(values, (list, tuple)): values = [values] assert len(ls_result) == len(values) for got, expected in zip(ls_result, values): if key == "json": if expected != ANY: self.assert_json_equal(got, expected) else: assert got == expected def expect_snapshot_output(self, happy_path_project): # noqa: F811 expectations = { "name": ["my_snapshot", "snapshot_2", "snapshot_3"], "selector": ["test.snapshot.my_snapshot", "test.snapshot_2", "test.snapshot_3"], "json": [ { "name": "my_snapshot", "package_name": "test", "depends_on": {"nodes": [], "macros": []}, "tags": [], "config": { "enabled": True, "group": None, "materialized": "snapshot", "post-hook": [], "tags": [], "pre-hook": [], "quoting": {}, "column_types": {}, "persist_docs": {}, "database": happy_path_project.database, "schema": happy_path_project.test_schema, "dbt_valid_to_current": None, "snapshot_meta_column_names": { "dbt_scd_id": None, "dbt_updated_at": None, "dbt_valid_from": None, "dbt_valid_to": None, "dbt_is_deleted": None, }, "unique_key": "id", "strategy": "timestamp", "updated_at": "updated_at", "full_refresh": None, "target_database": None, "target_schema": None, "alias": None, "check_cols": None, "on_schema_change": "ignore", "on_configuration_change": "apply", "meta": {}, "grants": {}, "packages": [], "incremental_strategy": None, "docs": {"node_color": None, "show": True}, "contract": {"enforced": False, "alias_types": True}, "event_time": None, "lookback": 1, "batch_size": None, "begin": None, "concurrent_batches": None, }, "unique_id": "snapshot.test.my_snapshot", "original_file_path": normalize("snapshots/snapshot.sql"), "alias": "my_snapshot", "resource_type": "snapshot", }, ANY, ANY, ], "path": [ self.dir("snapshots/snapshot.sql"), self.dir("snapshots/snapshot_2.yml"), self.dir("snapshots/snapshot_3.yml"), ], } self.expect_given_output(["--resource-type", "snapshot"], expectations) def expect_analyses_output(self): expectations = { "name": "a", "selector": "test.analysis.a", "json": { "name": "a", "package_name": "test", "depends_on": {"nodes": [], "macros": []}, "tags": ["tag"], "config": { "enabled": True, "group": "finance", "materialized": "view", "post-hook": [], "tags": ["tag"], "pre-hook": [], "quoting": {}, "column_types": {}, "persist_docs": {}, "full_refresh": None, "on_schema_change": "ignore", "on_configuration_change": "apply", "database": None, "schema": None, "alias": None, "meta": {"test": 1}, "unique_key": None, "grants": {}, "packages": [], "incremental_strategy": None, "docs": {"node_color": "purple", "show": True}, "contract": {"enforced": False, "alias_types": True}, "event_time": None, "lookback": 1, "batch_size": None, "begin": None, "concurrent_batches": None, }, "unique_id": "analysis.test.a", "original_file_path": normalize("analyses/a.sql"), "alias": "a", "resource_type": "analysis", }, "path": self.dir("analyses/a.sql"), } self.expect_given_output(["--resource-type", "analysis"], expectations) def expect_model_output(self): expectations = { "name": ( "ephemeral", "incremental", "inner", "metricflow_time_spine", "metricflow_time_spine_second", "model_to_unit_test", "model_with_lots_of_schema_configs", "outer", "snapshot_source", ), "selector": ( "test.ephemeral", "test.incremental", "test.sub.inner", "test.metricflow_time_spine", "test.metricflow_time_spine_second", "test.model_to_unit_test", "test.model_with_lots_of_schema_configs", "test.outer", "test.snapshot_source", ), "json": ( { "name": "ephemeral", "package_name": "test", "depends_on": { "nodes": [], "macros": ["macro.dbt.current_timestamp", "macro.dbt.date_trunc"], }, "tags": [], "config": { "enabled": True, "group": None, "materialized": "ephemeral", "post-hook": [], "tags": [], "pre-hook": [], "quoting": {}, "column_types": {}, "persist_docs": {}, "full_refresh": None, "unique_key": None, "on_schema_change": "ignore", "on_configuration_change": "apply", "database": None, "schema": None, "alias": None, "meta": {}, "grants": {}, "packages": [], "incremental_strategy": None, "docs": {"node_color": None, "show": True}, "contract": {"enforced": False, "alias_types": True}, "access": "protected", "event_time": None, "lookback": 1, "batch_size": None, "begin": None, "concurrent_batches": None, "freshness": None, }, "original_file_path": normalize("models/ephemeral.sql"), "unique_id": "model.test.ephemeral", "alias": "ephemeral", "resource_type": "model", }, { "name": "incremental", "package_name": "test", "depends_on": { "nodes": ["seed.test.seed"], "macros": ["macro.dbt.is_incremental"], }, "tags": [], "config": { "enabled": True, "group": None, "materialized": "incremental", "post-hook": [], "tags": [], "pre-hook": [], "quoting": {}, "column_types": {}, "persist_docs": {}, "full_refresh": None, "unique_key": None, "on_schema_change": "ignore", "on_configuration_change": "apply", "database": None, "schema": None, "alias": None, "meta": {}, "grants": {}, "packages": [], "incremental_strategy": "delete+insert", "docs": {"node_color": None, "show": True}, "contract": {"enforced": False, "alias_types": True}, "access": "protected", "event_time": None, "lookback": 1, "batch_size": None, "begin": None, "concurrent_batches": None, "freshness": None, }, "original_file_path": normalize("models/incremental.sql"), "unique_id": "model.test.incremental", "alias": "incremental", "resource_type": "model", }, { "name": "inner", "package_name": "test", "depends_on": { "nodes": ["model.test.outer"], "macros": [], }, "tags": [], "config": { "enabled": True, "group": None, "materialized": "view", "post-hook": [], "tags": [], "pre-hook": [], "quoting": {}, "column_types": {}, "persist_docs": {}, "full_refresh": None, "unique_key": None, "on_schema_change": "ignore", "on_configuration_change": "apply", "database": None, "schema": None, "alias": None, "meta": {}, "grants": {}, "packages": [], "incremental_strategy": None, "docs": {"node_color": None, "show": True}, "contract": {"enforced": False, "alias_types": True}, "access": "protected", "event_time": None, "lookback": 1, "batch_size": None, "begin": None, "concurrent_batches": None, "freshness": None, }, "original_file_path": normalize("models/sub/inner.sql"), "unique_id": "model.test.inner", "alias": "inner", "resource_type": "model", }, { "name": "metricflow_time_spine", "package_name": "test", "depends_on": { "nodes": [], "macros": ["macro.dbt.current_timestamp", "macro.dbt.date_trunc"], }, "config": { "enabled": True, "group": "finance", "materialized": "view", "post-hook": [ { "sql": "SELECT 'string_post_hook' as my_post_hook;", "transaction": True, "index": None, } ], "tags": ["list", "of", "tags"], "pre-hook": [ { "sql": "SELECT 'string_pre_hook' as my_pre_hook;", "transaction": True, "index": None, } ], "quoting": {}, "column_types": {}, "persist_docs": {}, "full_refresh": None, "unique_key": None, "on_schema_change": "ignore", "on_configuration_change": "apply", "database": None, "schema": None, "alias": None, "meta": {}, "grants": {}, "packages": [], "incremental_strategy": None, "docs": {"node_color": None, "show": True}, "contract": {"enforced": False, "alias_types": True}, "access": "protected", "event_time": None, "lookback": 1, "batch_size": None, "begin": None, "concurrent_batches": None, "freshness": None, }, "original_file_path": normalize("models/metricflow_time_spine.sql"), "unique_id": "model.test.metricflow_time_spine", "alias": "metricflow_time_spine", "resource_type": "model", "tags": ["list", "of", "tags"], }, { "name": "metricflow_time_spine_second", "package_name": "test", "depends_on": { "nodes": [], "macros": ["macro.dbt.current_timestamp", "macro.dbt.date_trunc"], }, "tags": [], "config": { "enabled": True, "group": None, "materialized": "view", "post-hook": [], "tags": [], "pre-hook": [], "quoting": {}, "column_types": {}, "persist_docs": {}, "full_refresh": None, "unique_key": None, "on_schema_change": "ignore", "on_configuration_change": "apply", "database": None, "schema": None, "alias": None, "meta": {}, "grants": {}, "packages": [], "incremental_strategy": None, "docs": {"node_color": None, "show": True}, "contract": {"enforced": False, "alias_types": True}, "access": "protected", "event_time": "ts_second", "lookback": 1, "batch_size": None, "begin": None, "concurrent_batches": None, "freshness": None, }, "original_file_path": normalize("models/metricflow_time_spine_second.sql"), "unique_id": "model.test.metricflow_time_spine_second", "alias": "metricflow_time_spine_second", "resource_type": "model", }, { "alias": "model_to_unit_test", "config": { "access": "protected", "alias": None, "batch_size": None, "begin": None, "column_types": {}, "concurrent_batches": None, "contract": {"alias_types": True, "enforced": False}, "database": None, "docs": {"node_color": None, "show": True}, "enabled": True, "event_time": None, "freshness": None, "full_refresh": None, "grants": {}, "group": None, "incremental_strategy": None, "lookback": 1, "materialized": "table", "meta": {}, "on_configuration_change": "apply", "on_schema_change": "ignore", "packages": [], "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "schema": None, "tags": [], "unique_key": None, }, "depends_on": {"macros": [], "nodes": ["seed.test.seed"]}, "name": "model_to_unit_test", "original_file_path": normalize("models/model_to_unit_test.sql"), "package_name": "test", "resource_type": "model", "tags": [], "unique_id": "model.test.model_to_unit_test", }, { "name": "model_with_lots_of_schema_configs", "resource_type": "model", "package_name": "test", "original_file_path": normalize( "models/model_with_lots_of_schema_configs.sql" ), "unique_id": "model.test.model_with_lots_of_schema_configs", "alias": "outer_alias", "config": { "enabled": True, "access": "public", "alias": "outer_alias", "schema": "test", "sql_header": "SELECT 1 as header;", "database": "dbt", "docs": {"node_color": "purple", "show": True}, "event_time": None, "tags": ["string_tag"], "meta": {"my_custom_property": "string_meta"}, "group": None, "materialized": "table", "incremental_strategy": None, "batch_size": "day", "lookback": 5, "begin": "2020-01-01", "persist_docs": {"columns": True, "relation": True}, "post-hook": [ { "sql": "SELECT 'string_post_hook' as my_post_hook;", "transaction": True, "index": None, } ], "pre-hook": [ { "sql": "SELECT 'string_pre_hook' as my_pre_hook;", "transaction": True, "index": None, } ], "quoting": {}, "column_types": {}, "concurrent_batches": False, "contract": {"alias_types": True, "enforced": False}, "full_refresh": False, "unique_key": "id", "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {"select": ["root"]}, "packages": [], "freshness": None, }, "depends_on": { "macros": [], "nodes": ["source.test.my_source.my_table", "model.test.ephemeral"], }, "tags": ["string_tag"], }, { "name": "outer", "package_name": "test", "depends_on": { "nodes": ["model.test.ephemeral"], "macros": [], }, "tags": [], "config": { "enabled": True, "group": None, "materialized": "view", "post-hook": [], "tags": [], "pre-hook": [], "quoting": {}, "column_types": {}, "persist_docs": {}, "full_refresh": None, "unique_key": None, "on_schema_change": "ignore", "on_configuration_change": "apply", "database": None, "schema": None, "alias": None, "meta": {}, "grants": {}, "packages": [], "incremental_strategy": None, "docs": {"node_color": None, "show": True}, "contract": {"enforced": False, "alias_types": True}, "access": "protected", "event_time": None, "lookback": 1, "batch_size": None, "begin": None, "concurrent_batches": None, "freshness": None, }, "original_file_path": normalize("models/outer.sql"), "unique_id": "model.test.outer", "alias": "outer", "resource_type": "model", }, ANY, ), "path": ( self.dir("models/ephemeral.sql"), self.dir("models/incremental.sql"), self.dir("models/sub/inner.sql"), self.dir("models/metricflow_time_spine.sql"), self.dir("models/metricflow_time_spine_second.sql"), self.dir("models/model_to_unit_test.sql"), self.dir("models/model_with_lots_of_schema_configs.sql"), self.dir("models/outer.sql"), self.dir("models/snapshot_source.sql"), ), } self.expect_given_output(["--resource-type", "model"], expectations) # Do not include ephemeral model - it was not selected def expect_model_ephemeral_output(self): expectations = { "name": ("outer"), "selector": ("test.outer"), "json": ( { "name": "outer", "package_name": "test", "depends_on": {"nodes": [], "macros": []}, "tags": [], "config": { "enabled": True, "materialized": "view", "post-hook": [], "tags": [], "pre-hook": [], "quoting": {}, "column_types": {}, "persist_docs": {}, "full_refresh": None, "on_schema_change": "ignore", "on_configuration_change": "apply", "database": None, "schema": None, "alias": None, "meta": {}, "grants": {}, "packages": [], "incremental_strategy": None, "docs": {"node_color": None, "show": True}, "access": "protected", }, "unique_id": "model.test.ephemeral", "original_file_path": normalize("models/ephemeral.sql"), "alias": "outer", "resource_type": "model", }, ), "path": (self.dir("models/outer.sql"),), } self.expect_given_output(["--model", "outer"], expectations) def expect_source_output(self): expectations = { "name": "my_source.my_table", "selector": "source:test.my_source.my_table", "json": { "config": { "enabled": True, "event_time": "column_name", "freshness": { "error_after": { "count": 2, "period": "hour", }, "warn_after": { "count": 1, "period": "minute", }, "filter": "column_name = 1", }, "meta": {"source_meta": 1, "table_meta": 1}, "tags": ["source_tag", "table_tag"], "loaded_at_query": None, "loaded_at_field": "column_name", }, "unique_id": "source.test.my_source.my_table", "original_file_path": normalize("models/schema.yml"), "package_name": "test", "name": "my_table", "source_name": "my_source", "resource_type": "source", "tags": ["source_tag", "table_tag"], }, "path": self.dir("models/schema.yml"), } # should we do this --select automatically for a user if if 'source' is # in the resource types and there is no '--select' or '--exclude'? self.expect_given_output( ["--resource-type", "source", "--select", "source:*"], expectations ) def expect_seed_output(self): expectations = { "name": "seed", "selector": "test.seed", "json": { "name": "seed", "package_name": "test", "tags": ["tag"], "config": { "enabled": True, "group": "finance", "materialized": "seed", "post-hook": [{"sql": "select 1", "transaction": True, "index": None}], "tags": ["tag"], "pre-hook": [{"sql": "select 1", "transaction": True, "index": None}], "quoting": {}, "column_types": {"a": "BIGINT"}, "delimiter": ",", "persist_docs": {}, "quote_columns": False, "full_refresh": True, "unique_key": None, "on_schema_change": "ignore", "on_configuration_change": "apply", "database": "dbt", "schema": "test", "alias": "test_alias", "meta": {"meta_key": "meta_value"}, "grants": {}, "packages": [], "incremental_strategy": None, "docs": {"node_color": "purple", "show": True}, "contract": {"enforced": False, "alias_types": True}, "event_time": "my_time_field", "lookback": 1, "batch_size": None, "begin": None, "concurrent_batches": None, }, "depends_on": {"macros": []}, "unique_id": "seed.test.seed", "original_file_path": normalize("seeds/seed.csv"), "alias": "test_alias", "resource_type": "seed", }, "path": self.dir("seeds/seed.csv"), } self.expect_given_output(["--resource-type", "seed"], expectations) def expect_test_output(self): # This is order sensitive :grimace: expectations = { "name": ( "expression_is_true_seed_b_2", "my_favorite_test", "my_second_favorite_test", "not_null_outer_id", "not_null_seed__a_", "not_null_seed__b_", "t", "unique_model_with_lots_of_schema_configs_id", "unique_outer_id", ), "selector": ( "test.expression_is_true_seed_b_2", "test.my_favorite_test", "test.my_second_favorite_test", "test.not_null_outer_id", "test.not_null_seed__a_", "test.not_null_seed__b_", "test.t", "test.unique_model_with_lots_of_schema_configs_id", "test.unique_outer_id", ), "json": ( { "alias": "expression_is_true_seed_b_2", "config": { "alias": None, "database": None, "enabled": True, "error_if": "!= 0", "fail_calc": "count(*)", "group": None, "limit": None, "materialized": "test", "meta": {}, "schema": "dbt_test__audit", "severity": "ERROR", "store_failures": None, "store_failures_as": None, "sql_header": None, "tags": [], "warn_if": "!= 0", "where": None, }, "depends_on": { "macros": [ "macro.test.test_expression_is_true", "macro.dbt.get_where_subquery", ], "nodes": ["seed.test.seed"], }, "name": "expression_is_true_seed_b_2", "original_file_path": normalize("seeds/s.yml"), "package_name": "test", "resource_type": "test", "tags": [], "unique_id": "test.test.expression_is_true_seed_b_2.4e0babbea4", }, { "alias": "not_null__id__alias", "config": { "alias": "not_null__id__alias", "database": "dbt", "enabled": True, "error_if": "!= 0", "fail_calc": "count(*)", "group": "important_tests", "limit": 10, "materialized": "test", "meta": {"my_custom_meta_key": "my_custom_meta_value"}, "schema": "dbt_test__audit", "severity": "warn", "store_failures": True, "store_failures_as": "table", "sql_header": None, "tags": ["test_tag"], "warn_if": "!= 0", "where": "1 = 1", }, "depends_on": { "macros": ["macro.dbt.test_not_null"], "nodes": ["model.test.model_with_lots_of_schema_configs"], }, "name": "my_favorite_test", "original_file_path": normalize("models/schema.yml"), "package_name": "test", "resource_type": "test", "tags": ["column_level_tag", "test_tag"], "unique_id": "test.test.my_favorite_test.b488d63233", }, { "alias": "my_generic_test__created_at__alias", "config": { "alias": "my_generic_test__created_at__alias", "database": "dbt", "enabled": True, "error_if": "!= 0", "fail_calc": "count(*)", "group": "important_tests", "limit": 10, "materialized": "test", "meta": {"my_custom_meta_key": "my_custom_meta_value"}, "schema": "dbt_test__audit", "severity": "warn", "store_failures": True, "store_failures_as": "table", "sql_header": None, "tags": ["test_tag", "test_tag"], "warn_if": "!= 0", "where": "1 = 1", }, "depends_on": { "macros": [ "macro.test.test_my_generic_test", "macro.dbt.get_where_subquery", ], "nodes": ["model.test.model_with_lots_of_schema_configs"], }, "name": "my_second_favorite_test", "original_file_path": normalize("models/schema.yml"), "package_name": "test", "resource_type": "test", "tags": ["test_tag"], "unique_id": "test.test.my_second_favorite_test.c8955109ad", }, { "name": "not_null_outer_id", "package_name": "test", "depends_on": { "nodes": ["model.test.outer"], "macros": ["macro.dbt.test_not_null"], }, "tags": [], "config": { "enabled": True, "group": None, "materialized": "test", "severity": "ERROR", "store_failures": None, "store_failures_as": None, "sql_header": None, "warn_if": "!= 0", "error_if": "!= 0", "fail_calc": "count(*)", "where": None, "limit": None, "tags": [], "database": None, "schema": "dbt_test__audit", "alias": None, "meta": {}, }, "unique_id": "test.test.not_null_outer_id.a226f4fb36", "original_file_path": normalize("models/schema.yml"), "alias": "not_null_outer_id", "resource_type": "test", }, { "alias": "not_null_seed__a_", "config": { "alias": None, "database": None, "enabled": True, "error_if": "!= 0", "fail_calc": "count(*)", "group": None, "limit": None, "materialized": "test", "meta": {}, "schema": "dbt_test__audit", "severity": "ERROR", "store_failures": None, "store_failures_as": None, "sql_header": None, # generic test builders don't propagate tags correctly "tags": [], "warn_if": "!= 0", "where": None, }, "depends_on": { "macros": ["macro.dbt.test_not_null"], "nodes": ["seed.test.seed"], }, "name": "not_null_seed__a_", "original_file_path": normalize("seeds/s.yml"), "package_name": "test", "resource_type": "test", "tags": ["tag"], "unique_id": "test.test.not_null_seed__a_.6b59640cde", }, { "alias": "not_null_seed__b_", "config": { "alias": None, "database": None, "enabled": True, "error_if": "!= 0", "fail_calc": "count(*)", "group": None, "limit": None, "materialized": "test", "meta": {}, "schema": "dbt_test__audit", "severity": "ERROR", "store_failures": None, "store_failures_as": None, "sql_header": None, # generic test builders don't propagate tags correctly "tags": [], "warn_if": "!= 0", "where": None, }, "depends_on": { "macros": ["macro.dbt.test_not_null"], "nodes": ["seed.test.seed"], }, "name": "not_null_seed__b_", "original_file_path": normalize("seeds/s.yml"), "package_name": "test", "resource_type": "test", "tags": ["tag"], "unique_id": "test.test.not_null_seed__b_.a088b263cb", }, { "name": "t", "package_name": "test", "alias": "test_alias", "config": { "alias": "test_alias", "database": "dbt", "enabled": True, "error_if": "!= 0", "fail_calc": "count(*)", "group": "important_tests", "limit": 10, "materialized": "test", "meta": {"my_custom_meta_key": "my_custom_meta_value"}, "schema": "dbt_test__audit", "severity": "warn", "store_failures": True, "store_failures_as": "table", "sql_header": None, "tags": ["test_tag"], "warn_if": "!= 0", "where": "1 = 1", }, "depends_on": {"macros": [], "nodes": []}, "name": "t", "original_file_path": normalize("tests/t.sql"), "package_name": "test", "resource_type": "test", "tags": ["test_tag"], "unique_id": "test.test.t", }, { "alias": "unique_model_with_lots_of_schema_configs_id", "config": { "alias": None, "database": None, "enabled": True, "error_if": "!= 0", "fail_calc": "count(*)", "group": None, "limit": None, "materialized": "test", "meta": {}, "schema": "dbt_test__audit", "severity": "ERROR", "store_failures": None, "store_failures_as": None, "sql_header": None, "tags": [], "warn_if": "!= 0", "where": None, }, "depends_on": { "macros": ["macro.dbt.test_unique"], "nodes": ["model.test.model_with_lots_of_schema_configs"], }, "name": "unique_model_with_lots_of_schema_configs_id", "original_file_path": normalize("models/schema.yml"), "package_name": "test", "resource_type": "test", "tags": ["column_level_tag"], "unique_id": "test.test.unique_model_with_lots_of_schema_configs_id.8328d84982", }, { "name": "unique_outer_id", "package_name": "test", "depends_on": { "nodes": ["model.test.outer"], "macros": ["macro.dbt.test_unique"], }, "tags": [], "config": { "enabled": True, "group": None, "materialized": "test", "severity": "ERROR", "store_failures": None, "store_failures_as": None, "sql_header": None, "warn_if": "!= 0", "error_if": "!= 0", "fail_calc": "count(*)", "where": None, "limit": None, "tags": [], "database": None, "schema": "dbt_test__audit", "alias": None, "meta": {}, }, "unique_id": "test.test.unique_outer_id.2195e332d3", "original_file_path": normalize("models/schema.yml"), "alias": "unique_outer_id", "resource_type": "test", }, ), "path": ( self.dir("seeds/s.yml"), self.dir("models/schema.yml"), self.dir("models/schema.yml"), self.dir("models/schema.yml"), self.dir("seeds/s.yml"), self.dir("seeds/s.yml"), self.dir("tests/t.sql"), self.dir("models/schema.yml"), self.dir("models/schema.yml"), ), } self.expect_given_output(["--resource-type", "test"], expectations) def expect_function_output(self): # using `--resource-type` results = self.run_dbt_ls(["--resource-type", "function"]) assert set(results) == {"test.area_of_circle"} # using `--select` with `resource_type` results = self.run_dbt_ls(["--select", "resource_type:function"]) assert set(results) == {"test.area_of_circle"} # using `--select` with path spec results = self.run_dbt_ls(["--select", "functions/area_of_circle.sql"]) assert set(results) == {"test.area_of_circle"} # using `--select` with path function name results = self.run_dbt_ls(["--select", "area_of_circle"]) assert set(results) == {"test.area_of_circle"} def expect_all_output(self): # generic test FQNS include the resource + column they're defined on # models are just package, subdirectory path, name # sources are like models, ending in source_name.table_name expected_default = { "exposure:test.weekly_jaffle_metrics", "test.ephemeral", "test.incremental", "test.snapshot.my_snapshot", "test.snapshot_2", "test.snapshot_3", "test.sub.inner", "test.outer", "test.snapshot_source", "test.seed", "source:test.my_source.my_table", "test.not_null_outer_id", "test.unique_outer_id", "test.metricflow_time_spine", "test.metricflow_time_spine_second", "test.model_to_unit_test", "test.model_with_lots_of_schema_configs", "test.unique_model_with_lots_of_schema_configs_id", "test.t", "test.my_favorite_test", "test.my_second_favorite_test", "test.area_of_circle", "semantic_model:test.my_sm", "metric:test.total_outer", "metric:test.conversion_metric", "metric:test.cumulative_metric", "metric:test.cumulative_metric_2", "metric:test.derived_metric", "metric:test.filtered_ratio_metric", "metric:test.simple_ratio_metric", "metric:test.discrete_order_value_p99", "saved_query:test.my_saved_query", "test.expression_is_true_seed_b_2", "test.not_null_seed__a_", "test.not_null_seed__b_", "unit_test:test.test_model_to_unit_test", "unit_test:test.test_model_to_unit_test_2", } # analyses have their type inserted into their fqn like tests expected_all = expected_default | {"test.analysis.a"} results = self.run_dbt_ls(["--resource-type", "all", "--select", "*", "source:*"]) assert set(results) == expected_all results = self.run_dbt_ls(["--select", "*", "source:*"]) assert set(results) == expected_default results = self.run_dbt_ls(["--resource-type", "default", "--select", "*", "source:*"]) assert set(results) == expected_default results = self.run_dbt_ls def expect_select(self): results = self.run_dbt_ls(["--resource-type", "test", "--select", "outer"]) assert set(results) == {"test.not_null_outer_id", "test.unique_outer_id"} self.run_dbt_ls(["--resource-type", "test", "--select", "inner"], expect_pass=True) results = self.run_dbt_ls(["--resource-type", "test", "--select", "+inner"]) assert set(results) == {"test.not_null_outer_id", "test.unique_outer_id"} results = self.run_dbt_ls(["--resource-type", "semantic_model"]) assert set(results) == {"semantic_model:test.my_sm"} results = self.run_dbt_ls(["--resource-type", "metric"]) assert set(results) == { "metric:test.total_outer", "metric:test.simple_ratio_metric", "metric:test.filtered_ratio_metric", "metric:test.conversion_metric", "metric:test.cumulative_metric", "metric:test.cumulative_metric_2", "metric:test.derived_metric", "metric:test.discrete_order_value_p99", } results = self.run_dbt_ls(["--resource-type", "saved_query"]) assert set(results) == {"saved_query:test.my_saved_query"} results = self.run_dbt_ls(["--resource-type", "model", "--select", "outer+"]) assert set(results) == {"test.outer", "test.sub.inner"} results = self.run_dbt_ls(["--resource-type", "model", "--exclude", "inner"]) assert set(results) == { "test.ephemeral", "test.outer", "test.snapshot_source", "test.metricflow_time_spine", "test.metricflow_time_spine_second", "test.model_to_unit_test", "test.model_with_lots_of_schema_configs", "test.incremental", } results = self.run_dbt_ls(["--select", "config.incremental_strategy:delete+insert"]) assert set(results) == {"test.incremental"} self.run_dbt_ls( ["--select", "config.incremental_strategy:insert_overwrite"], expect_pass=True ) def expect_resource_type_multiple(self): """Expect selected resources when --resource-type given multiple times""" results = self.run_dbt_ls(["--resource-type", "test", "--resource-type", "model"]) assert set(results) == { "test.ephemeral", "test.incremental", "test.not_null_outer_id", "test.outer", "test.snapshot_source", "test.sub.inner", "test.metricflow_time_spine", "test.metricflow_time_spine_second", "test.model_to_unit_test", "test.model_with_lots_of_schema_configs", "test.t", "test.unique_outer_id", "test.unique_model_with_lots_of_schema_configs_id", "test.expression_is_true_seed_b_2", "test.not_null_seed__a_", "test.not_null_seed__b_", "test.my_favorite_test", "test.my_second_favorite_test", } results = self.run_dbt_ls( [ "--resource-type", "test", "--resource-type", "model", "--exclude", "unique_outer_id", ] ) assert set(results) == { "test.ephemeral", "test.incremental", "test.not_null_outer_id", "test.outer", "test.snapshot_source", "test.metricflow_time_spine", "test.metricflow_time_spine_second", "test.model_to_unit_test", "test.model_with_lots_of_schema_configs", "test.sub.inner", "test.t", "test.unique_model_with_lots_of_schema_configs_id", "test.expression_is_true_seed_b_2", "test.not_null_seed__a_", "test.not_null_seed__b_", "test.my_favorite_test", "test.my_second_favorite_test", } results = self.run_dbt_ls( [ "--resource-type", "test", "model", "--select", "+inner", "outer+", "--exclude", "inner", ] ) assert set(results) == { "test.ephemeral", "test.not_null_outer_id", "test.unique_outer_id", "test.outer", } def expect_resource_type_env_var(self): """Expect selected resources when --resource-type given multiple times""" os.environ["DBT_RESOURCE_TYPES"] = "test model" results = self.run_dbt_ls() assert set(results) == { "test.ephemeral", "test.incremental", "test.not_null_outer_id", "test.outer", "test.snapshot_source", "test.sub.inner", "test.metricflow_time_spine", "test.metricflow_time_spine_second", "test.model_to_unit_test", "test.model_with_lots_of_schema_configs", "test.t", "test.unique_outer_id", "test.unique_model_with_lots_of_schema_configs_id", "test.expression_is_true_seed_b_2", "test.not_null_seed__a_", "test.not_null_seed__b_", "test.my_favorite_test", "test.my_second_favorite_test", } del os.environ["DBT_RESOURCE_TYPES"] os.environ["DBT_EXCLUDE_RESOURCE_TYPES"] = ( "test saved_query metric source semantic_model snapshot seed" ) results = self.run_dbt_ls() assert set(results) == { "exposure:test.weekly_jaffle_metrics", "test.area_of_circle", "test.ephemeral", "test.incremental", "test.outer", "test.snapshot_source", "test.sub.inner", "test.metricflow_time_spine", "test.metricflow_time_spine_second", "test.model_to_unit_test", "test.model_with_lots_of_schema_configs", "unit_test:test.test_model_to_unit_test", "unit_test:test.test_model_to_unit_test_2", } del os.environ["DBT_EXCLUDE_RESOURCE_TYPES"] def expect_selected_keys(self, happy_path_project): # noqa: F811 """Expect selected fields of the the selected model""" expectations = [ { "database": happy_path_project.database, "schema": happy_path_project.test_schema, "alias": "inner", } ] results = self.run_dbt_ls( [ "--model", "inner", "--output", "json", "--output-keys", "database", "schema", "alias", ] ) assert len(results) == len(expectations) for got, expected in zip(results, expectations): self.assert_json_equal(got, expected) """Expect selected fields when --output-keys given multiple times """ expectations = [ {"database": happy_path_project.database, "schema": happy_path_project.test_schema} ] results = self.run_dbt_ls( [ "--model", "inner", "--output", "json", "--output-keys", "database", "--output-keys", "schema", ] ) assert len(results) == len(expectations) for got, expected in zip(results, expectations): self.assert_json_equal(got, expected) """Expect selected fields of the test resource types """ expectations = [ {"name": "expression_is_true_seed_b_2", "column_name": None}, {"name": "my_favorite_test", "column_name": "id"}, {"name": "my_second_favorite_test", "column_name": "created_at"}, {"name": "not_null_outer_id", "column_name": "id"}, {"name": "not_null_seed__a_", "column_name": '"a"'}, {"name": "not_null_seed__b_", "column_name": '"b"'}, {"name": "t"}, {"name": "unique_model_with_lots_of_schema_configs_id", "column_name": "id"}, {"name": "unique_outer_id", "column_name": "id"}, ] results = self.run_dbt_ls( [ "--resource-type", "test", "--output", "json", "--output-keys", "name", "column_name", ] ) assert len(results) == len(expectations) for got, expected in zip( sorted(results, key=lambda x: json.loads(x).get("name")), sorted(expectations, key=lambda x: x.get("name")), ): self.assert_json_equal(got, expected) """Expect nothing (non-existent keys) for the selected models """ expectations = [{}, {}] results = self.run_dbt_ls( [ "--model", "inner outer", "--output", "json", "--output-keys", "non_existent_key", ] ) assert len(results) == len(expectations) for got, expected in zip(results, expectations): self.assert_json_equal(got, expected) def test_ls(self, happy_path_project): # noqa: F811 self.expect_snapshot_output(happy_path_project) self.expect_analyses_output() self.expect_model_output() self.expect_source_output() self.expect_seed_output() self.expect_test_output() self.expect_function_output() self.expect_select() self.expect_resource_type_multiple() self.expect_resource_type_env_var() self.expect_all_output() self.expect_selected_keys(happy_path_project) def normalize(path): """On windows, neither is enough on its own: >>> normcase('C:\\documents/ALL CAPS/subdir\\..') 'c:\\documents\\all caps\\subdir\\..' >>> normpath('C:\\documents/ALL CAPS/subdir\\..') 'C:\\documents\\ALL CAPS' >>> normpath(normcase('C:\\documents/ALL CAPS/subdir\\..')) 'c:\\documents\\all caps' """ return os.path.normcase(os.path.normpath(path)) ================================================ FILE: tests/functional/logging/test_logging.py ================================================ import json import os import pytest from dbt.events.types import InvalidOptionYAML from dbt.tests.util import get_manifest, read_file, run_dbt from dbt_common.events import EventLevel from dbt_common.events.functions import fire_event my_model_sql = """ select 1 as fun """ @pytest.fixture(scope="class") def models(): return {"my_model.sql": my_model_sql} # This test checks that various events contain node_info, # which is supplied by the log_contextvars context manager def test_basic(project, logs_dir): results = run_dbt(["--log-format=json", "run"]) assert len(results) == 1 manifest = get_manifest(project.project_root) assert "model.test.my_model" in manifest.nodes # get log file log_file = read_file(logs_dir, "dbt.log") assert log_file node_start = False node_finished = False connection_reused_data = [] for log_line in log_file.split("\n"): # skip empty lines if len(log_line) == 0: continue # The adapter logging also shows up, so skip non-json lines if "[debug]" in log_line: continue log_dct = json.loads(log_line) log_data = log_dct["data"] log_event = log_dct["info"]["name"] if log_event == "ConnectionReused": connection_reused_data.append(log_data) if log_event == "NodeStart": node_start = True if log_event == "NodeFinished": node_finished = True assert log_data["run_result"]["adapter_response"] if node_start and not node_finished: if log_event == "NodeExecuting": assert "node_info" in log_data if log_event == "JinjaLogDebug": assert "node_info" in log_data if log_event == "SQLQuery": assert "node_info" in log_data if log_event == "TimingInfoCollected": assert "node_info" in log_data assert "timing_info" in log_data # windows doesn't have the same thread/connection flow so the ConnectionReused # events don't show up if os.name != "nt": # Verify the ConnectionReused event occurs and has the right data assert connection_reused_data for data in connection_reused_data: assert "conn_name" in data and data["conn_name"] assert "orig_conn_name" in data and data["orig_conn_name"] def test_formatted_logs(project, logs_dir): # a basic run of dbt with a single model should have 5 `Formatting` events in the json logs results = run_dbt(["--log-format=json", "run"]) assert len(results) == 1 # get log file json_log_file = read_file(logs_dir, "dbt.log") formatted_json_lines = 0 for log_line in json_log_file.split("\n"): # skip the empty line at the end if len(log_line) == 0: continue log_dct = json.loads(log_line) log_event = log_dct["info"]["name"] if log_event == "Formatting": formatted_json_lines += 1 assert formatted_json_lines == 5 def test_invalid_event_value(project, logs_dir): results = run_dbt(["--log-format=json", "run"]) assert len(results) == 1 with pytest.raises(Exception): # This should raise because positional arguments are provided to the event fire_event(InvalidOptionYAML("testing")) # Provide invalid type to "option_name" with pytest.raises(Exception) as excinfo: fire_event(InvalidOptionYAML(option_name=1)) assert "[InvalidOptionYAML]: Unable to parse logging event dictionary." in str(excinfo.value) groups_yml = """ groups: - name: my_group_with_owner_metadata owner: name: my_name email: my.email@gmail.com slack: my_slack other_property: something_else models: - name: my_model group: my_group_with_owner_metadata access: public """ groups_yml_with_multiple_emails = """ groups: - name: my_group_with_multiple_emails owner: name: my_name email: - my.email@gmail.com - my.second.email@gmail.com slack: my_slack other_property: something_else models: - name: my_model group: my_group_with_multiple_emails access: public columns: - name: my_column tests: - not_null """ class TestRunResultErrorNodeInfo: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": "select not_found as id", } def test_node_info_on_results(self, project, logs_dir): results = run_dbt(["--log-format=json", "run"], expect_pass=False) assert len(results) == 1 log_file = read_file(logs_dir, "dbt.log") for log_line in log_file.split("\n"): if not log_line: continue log_json = json.loads(log_line) if log_json["info"]["level"] == EventLevel.DEBUG: continue if log_json["info"]["name"] == "RunResultError": assert "node_info" in log_json["data"] assert log_json["data"]["node_info"]["unique_id"] == "model.test.my_model" assert "Database Error" in log_json["data"]["msg"] def assert_group_data(group_data): assert group_data["name"] == "my_group_with_owner_metadata" assert group_data["owner"] == { "name": "my_name", "email": "my.email@gmail.com", "slack": "my_slack", "other_property": "something_else", } class TestRunResultErrorGroup: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": "select not_found as id", "groups.yml": groups_yml, } def test_node_info_on_results(self, project, logs_dir): results = run_dbt(["--log-format=json", "run"], expect_pass=False) assert len(results) == 1 log_file = read_file(logs_dir, "dbt.log") run_result_error_count = 0 for log_line in log_file.split("\n"): if not log_line: continue log_json = json.loads(log_line) if log_json["info"]["level"] == EventLevel.DEBUG: continue if log_json["info"]["name"] == "RunResultError": assert "group" in log_json["data"] assert_group_data(log_json["data"]["group"]) run_result_error_count += 1 assert run_result_error_count == 1 class TestRunResultFailureGroup: @pytest.fixture(scope="class") def models(self): schema_yml = ( groups_yml + """ columns: - name: my_column tests: - not_null """ ) print(schema_yml) return { "my_model.sql": "select 1 as id, null as my_column", "groups.yml": schema_yml, } def test_node_info_on_results(self, project, logs_dir): results = run_dbt(["--log-format=json", "build"], expect_pass=False) assert len(results) == 2 log_file = read_file(logs_dir, "dbt.log") run_result_error_count = 0 run_result_failure_count = 0 for log_line in log_file.split("\n"): if not log_line: continue log_json = json.loads(log_line) if log_json["info"]["level"] == EventLevel.DEBUG: continue if log_json["info"]["name"] == "RunResultError": assert "group" in log_json["data"] assert_group_data(log_json["data"]["group"]) run_result_error_count += 1 if log_json["info"]["name"] == "RunResultFailure": assert "group" in log_json["data"] assert_group_data(log_json["data"]["group"]) run_result_failure_count += 1 assert run_result_error_count == 1 assert run_result_failure_count == 1 class TestRunResultWarningGroup: @pytest.fixture(scope="class") def models(self): schema_yml = ( groups_yml + """ columns: - name: my_column tests: - not_null: config: severity: warn """ ) print(schema_yml) return { "my_model.sql": "select 1 as id, null as my_column", "groups.yml": schema_yml, } def test_node_info_on_results(self, project, logs_dir): results = run_dbt(["--log-format=json", "build"]) assert len(results) == 2 log_file = read_file(logs_dir, "dbt.log") run_result_warning_count = 0 for log_line in log_file.split("\n"): if not log_line: continue log_json = json.loads(log_line) if log_json["info"]["level"] == EventLevel.DEBUG: continue if log_json["info"]["name"] == "RunResultWarning": assert "group" in log_json["data"] assert_group_data(log_json["data"]["group"]) run_result_warning_count += 1 assert run_result_warning_count == 1 class TestRunResultNoGroup: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": "select 1 as id", } def test_node_info_on_results(self, project, logs_dir): results = run_dbt(["--no-write-json", "run"]) assert len(results) == 1 class TestRunResultGroupWithMultipleEmails: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": "select 1 as id, null as my_column", "groups.yml": groups_yml_with_multiple_emails, } def test_node_info_on_results(self, project, logs_dir): results = run_dbt(["--log-format=json", "build"], expect_pass=False) assert len(results) == 2 log_file = read_file(logs_dir, "dbt.log") run_result_error_count = 0 for log_line in log_file.split("\n"): if not log_line: continue log_json = json.loads(log_line) if log_json["info"]["level"] == EventLevel.DEBUG: continue if log_json["info"]["name"] == "RunResultError": assert "group" in log_json["data"] group_data = log_json["data"]["group"] assert group_data["name"] == "my_group_with_multiple_emails" assert group_data["owner"] == { "name": "my_name", "email": "['my.email@gmail.com', 'my.second.email@gmail.com']", "slack": "my_slack", "other_property": "something_else", } run_result_error_count += 1 assert run_result_error_count == 1 ================================================ FILE: tests/functional/logging/test_meta_logging.py ================================================ import json import pytest from dbt.tests.util import read_file, run_dbt model1 = "select 1 as fun" model2 = '{{ config(meta={"owners": ["team1", "team2"]})}} select 1 as fun' model3 = '{{ config(meta={"key": 1})}} select 1 as fun' @pytest.fixture(scope="class") # noqa def models(): return {"model1.sql": model1, "model2.sql": model2, "model3.sql": model3} def run_and_capture_node_info_logs(logs_dir): run_dbt(["--log-format=json", "run"]) # get log file log_file = read_file(logs_dir, "dbt.log") assert log_file for log_line in log_file.split("\n"): # skip empty lines if len(log_line) == 0: continue # The adapter logging also shows up, so skip non-json lines if "[debug]" in log_line: continue log_dct = json.loads(log_line) if "node_info" not in log_dct["data"]: continue yield log_dct["data"]["node_info"] # This test checks that various events contain node_info, # which is supplied by the log_contextvars context manager def test_meta(project, logs_dir): for node_info_log in run_and_capture_node_info_logs(logs_dir): if node_info_log["unique_id"] == "model.test.model1": assert node_info_log["meta"] == {} elif node_info_log["unique_id"] == "model.test.model2": assert node_info_log["meta"] == {"owners": ["team1", "team2"]} elif node_info_log["unique_id"] == "model.test.model3": assert node_info_log["meta"] == {"key": 1} def test_checksum(project, logs_dir): for node_info_log in run_and_capture_node_info_logs(logs_dir): if node_info_log["unique_id"] == "model.test.model1": assert ( node_info_log["node_checksum"] == "7a72de8ca68190cc1f3a600b99ad24ce701817a5674222778845eb939c64aa76" ) elif node_info_log["unique_id"] == "model.test.model2": assert ( node_info_log["node_checksum"] == "4e5b7658359b9a7fec6aa3cbad98ab07725927ccce59ec6e511e599e000b0fd3" ) elif node_info_log["unique_id"] == "model.test.model3": assert ( node_info_log["node_checksum"] == "99c67d153920066d43168cc495240f185cec9d8cd552e7778e08437e66f44da7" ) ================================================ FILE: tests/functional/macros/data/seed.sql ================================================ create table {schema}.expected_dep_macro ( foo TEXT, bar TEXT ); create table {schema}.expected_local_macro ( foo2 TEXT, bar2 TEXT ); create table {schema}.seed ( id integer, updated_at timestamp ); insert into {schema}.expected_dep_macro (foo, bar) values ('arg1', 'arg2'); insert into {schema}.expected_local_macro (foo2, bar2) values ('arg1', 'arg2'), ('arg3', 'arg4'); insert into {schema}.seed (id, updated_at) values (1, '2017-01-01'), (2, '2017-01-02'); ================================================ FILE: tests/functional/macros/fixtures.py ================================================ models__dep_macro = """ {{ dbt_integration_project.do_something("arg1", "arg2") }} """ models__materialization_macro = """ {{ materialization_macro() }} """ models__with_undefined_macro = """ {{ dispatch_to_nowhere() }} select 1 as id """ models__local_macro = """ {{ do_something2("arg1", "arg2") }} union all {{ test.do_something2("arg3", "arg4") }} """ models__ref_macro = """ select * from {{ with_ref() }} """ models__override_get_columns_macros = """ {% set result = adapter.get_columns_in_relation(this) %} {% if execute and result != 'a string' %} {% do exceptions.raise_compiler_error('overriding get_columns_in_relation failed') %} {% endif %} select 1 as id """ models__deprecated_adapter_macro_model = """ {% if some_macro('foo', 'bar') != 'foobar' %} {% do exceptions.raise_compiler_error('invalid foobar') %} {% endif %} select 1 as id """ # # Macros # macros__my_macros = """ {% macro do_something2(foo2, bar2) %} select '{{ foo2 }}' as foo2, '{{ bar2 }}' as bar2 {% endmacro %} {% macro with_ref() %} {{ ref('table_model') }} {% endmacro %} {% macro dispatch_to_parent() %} {% set macro = adapter.dispatch('dispatch_to_parent') %} {{ macro() }} {% endmacro %} {% macro default__dispatch_to_parent() %} {% set msg = 'No default implementation of dispatch_to_parent' %} {{ exceptions.raise_compiler_error(msg) }} {% endmacro %} {% macro postgres__dispatch_to_parent() %} {{ return('') }} {% endmacro %} """ macros__named_materialization = """ {% macro materialization_macro() %} select 1 as foo {% endmacro %} """ macros__no_default_macros = """ {% macro do_something2(foo2, bar2) %} select '{{ foo2 }}' as foo2, '{{ bar2 }}' as bar2 {% endmacro %} {% macro with_ref() %} {{ ref('table_model') }} {% endmacro %} {# there is no default__dispatch_to_nowhere! #} {% macro dispatch_to_nowhere() %} {% set macro = adapter.dispatch('dispatch_to_nowhere') %} {{ macro() }} {% endmacro %} {% macro dispatch_to_parent() %} {% set macro = adapter.dispatch('dispatch_to_parent') %} {{ macro() }} {% endmacro %} {% macro default__dispatch_to_parent() %} {% set msg = 'No default implementation of dispatch_to_parent' %} {{ exceptions.raise_compiler_error(msg) }} {% endmacro %} {% macro postgres__dispatch_to_parent() %} {{ return('') }} {% endmacro %} """ macros__override_get_columns_macros = """ {% macro get_columns_in_relation(relation) %} {{ return('a string') }} {% endmacro %} """ macros__package_override_get_columns_macros = """ {% macro postgres__get_columns_in_relation(relation) %} {{ return('a string') }} {% endmacro %} """ macros__deprecated_adapter_macro = """ {% macro some_macro(arg1, arg2) -%} {{ adapter_macro('some_macro', arg1, arg2) }} {%- endmacro %} """ macros__incorrect_dispatch = """ {% macro cowsay() %} {{ return(adapter.dispatch('cowsay', 'farm_utils')()) }} {%- endmacro %} {% macro default__cowsay() %} 'moo' {% endmacro %} """ # Note the difference between `test_utils` below and `farm_utils` above models__incorrect_dispatch = """ select {{ test_utils.cowsay() }} as cowsay """ dbt_project__incorrect_dispatch = """ name: 'test_utils' version: '1.0' config-version: 2 profile: 'default' macro-paths: ["macros"] """ macros__config_sql = """ {% macro macro_config() %} {% endmacro %} {% macro macro_top_only() %} {% endmacro %} {% macro macro_config_only() %} {% endmacro %} """ macros__config_yml = """ macros: - name: macro_top_only description: Macro with only top-level meta and docs meta: top_k: top_v docs: show: true node_color: "#AAAAAA" - name: macro_config_only description: Macro with only config meta and docs config: meta: cm_k: cm_v docs: show: false node_color: "#BBBBBB" - name: macro_config description: Macro with merged meta and docs meta: top_k: top_v docs: show: true node_color: "#AAAAAA" config: meta: cm_k: cm_v docs: node_color: "#BBBBBB" """ ================================================ FILE: tests/functional/macros/package_macro_overrides/dbt_project.yml ================================================ name: 'package_macro_overrides' version: '1.0' config-version: 2 profile: 'default' macro-paths: ["macros"] ================================================ FILE: tests/functional/macros/package_macro_overrides/macros/macros.sql ================================================ {% macro get_columns_in_relation(relation) %} {{ return('a string') }} {% endmacro %} ================================================ FILE: tests/functional/macros/test_macro_annotations.py ================================================ import pytest from dbt.events.types import InvalidMacroAnnotation from dbt.tests.util import run_dbt from dbt_common.events.event_catcher import EventCatcher from dbt_common.ui import warning_tag macros_sql = """ {% macro my_macro(my_arg_1, my_arg_2, my_arg_3) %} {% endmacro %} """ bad_arg_names_macros_yml = """ macros: - name: my_macro description: This is the macro description. arguments: - name: my_arg_1 - name: my_misnamed_arg_2 - name: my_misnamed_arg_3 """ bad_arg_count_macros_yml = """ macros: - name: my_macro arguments: - name: my_arg_1 type: string description: This is an argument description. """ bad_arg_types_macros_yml = """ macros: - name: my_macro arguments: - name: my_arg_1 type: string - name: my_arg_2 type: invalid_type - name: my_arg_3 type: int[int] """ bad_everything_types_macros_yml = """ macros: - name: my_macro arguments: - name: my_arg_1 type: string - name: my_wrong_arg_2 type: invalid_type """ class TestMacroDefaultArgMetadata: """Test that when the validate_macro_args behavior flag is enabled, macro argument names are included in the manifest even if there is no yml patch.""" @pytest.fixture(scope="class") def macros(self): return {"macros.sql": macros_sql} @pytest.fixture(scope="class") def project_config_update(self): return {"flags": {"validate_macro_args": True}} def test_macro_default_arg_metadata(self, project) -> None: manifest = run_dbt(["parse"]) my_macro_args = manifest.macros["macro.test.my_macro"].arguments assert my_macro_args[0].name == "my_arg_1" assert my_macro_args[1].name == "my_arg_2" assert my_macro_args[2].name == "my_arg_3" class TestMacroNameWarnings: @pytest.fixture(scope="class") def macros(self): return {"macros.sql": macros_sql, "macros.yml": bad_arg_names_macros_yml} @pytest.fixture(scope="class") def project_config_update(self): return {"flags": {"validate_macro_args": True}} def test_macro_name_enforcement(self, project) -> None: event_catcher = EventCatcher(event_to_catch=InvalidMacroAnnotation) run_dbt(["parse"], callbacks=[event_catcher.catch]) assert len(event_catcher.caught_events) == 2 msg = "Argument my_misnamed_arg_2 in yaml for macro my_macro does not match the jinja" assert any( [e for e in event_catcher.caught_events if e.info.msg.startswith(warning_tag(msg))] ) msg = "Argument my_misnamed_arg_3 in yaml for macro my_macro does not match the jinja" assert any( [e for e in event_catcher.caught_events if e.info.msg.startswith(warning_tag(msg))] ) class TestMacroTypeWarnings: @pytest.fixture(scope="class") def macros(self): return {"macros.sql": macros_sql, "macros.yml": bad_arg_types_macros_yml} @pytest.fixture(scope="class") def project_config_update(self): return {"flags": {"validate_macro_args": True}} def test_macro_type_warnings(self, project) -> None: event_catcher = EventCatcher(event_to_catch=InvalidMacroAnnotation) run_dbt(["parse"], callbacks=[event_catcher.catch]) assert len(event_catcher.caught_events) == 2 msg = "Argument my_arg_2 in the yaml for macro my_macro has an invalid type" assert any( [e for e in event_catcher.caught_events if e.info.msg.startswith(warning_tag(msg))] ) msg = "Argument my_arg_3 in the yaml for macro my_macro has an invalid type" assert any( [e for e in event_catcher.caught_events if e.info.msg.startswith(warning_tag(msg))] ) class TestMacroNonEnforcement: @pytest.fixture(scope="class") def macros(self): return {"macros.yml": bad_everything_types_macros_yml, "macros.sql": macros_sql} def test_macro_non_enforcement(self, project) -> None: event_catcher = EventCatcher(event_to_catch=InvalidMacroAnnotation) run_dbt(["parse"], callbacks=[event_catcher.catch]) assert len(event_catcher.caught_events) == 0 ================================================ FILE: tests/functional/macros/test_macros.py ================================================ import shutil from pathlib import Path import pytest import dbt_common.exceptions from dbt.tests.fixtures.project import write_project_files from dbt.tests.util import check_relations_equal, get_manifest, run_dbt from tests.functional.macros.fixtures import ( dbt_project__incorrect_dispatch, macros__config_sql, macros__config_yml, macros__deprecated_adapter_macro, macros__incorrect_dispatch, macros__my_macros, macros__named_materialization, macros__no_default_macros, macros__override_get_columns_macros, macros__package_override_get_columns_macros, models__dep_macro, models__deprecated_adapter_macro_model, models__incorrect_dispatch, models__local_macro, models__materialization_macro, models__override_get_columns_macros, models__ref_macro, models__with_undefined_macro, ) class TestMacros: @pytest.fixture(scope="class", autouse=True) def setUp(self, project): project.run_sql_file(project.test_data_dir / Path("seed.sql")) @pytest.fixture(scope="class") def models(self): return { "dep_macro.sql": models__dep_macro, "local_macro.sql": models__local_macro, "ref_macro.sql": models__ref_macro, } @pytest.fixture(scope="class") def macros(self): return {"my_macros.sql": macros__my_macros} @pytest.fixture(scope="class") def packages(self): return { "packages": [ { "git": "https://github.com/dbt-labs/dbt-integration-project", "revision": "dbt/1.0.0", }, ] } @pytest.fixture(scope="class") def project_config_update(self): return { "config-version": 2, "vars": { "test": { "test": "DUMMY", }, }, "macro-paths": ["macros"], } def test_working_macros(self, project): run_dbt(["deps"]) results = run_dbt() assert len(results) == 6 check_relations_equal(project.adapter, ["expected_dep_macro", "dep_macro"]) check_relations_equal(project.adapter, ["expected_local_macro", "local_macro"]) class TestMacrosNamedMaterialization: @pytest.fixture(scope="class") def models(self): return { "models_materialization_macro.sql": models__materialization_macro, } @pytest.fixture(scope="class") def macros(self): return {"macros_named_materialization.sql": macros__named_materialization} def test_macro_with_materialization_in_name_works(self, project): run_dbt(expect_pass=True) class TestInvalidMacros: @pytest.fixture(scope="class") def models(self): return { "dep_macro.sql": models__dep_macro, "local_macro.sql": models__local_macro, "ref_macro.sql": models__ref_macro, } def test_invalid_macro(self, project): run_dbt(expect_pass=False) class TestAdapterMacroNoDestination: @pytest.fixture(scope="class") def models(self): return {"model.sql": models__with_undefined_macro} @pytest.fixture(scope="class") def macros(self): return {"my_macros.sql": macros__no_default_macros} def test_invalid_macro(self, project): with pytest.raises(dbt_common.exceptions.CompilationError) as exc: run_dbt() assert "In dispatch: No macro named 'dispatch_to_nowhere' found" in str(exc.value) class TestMacroOverrideBuiltin: @pytest.fixture(scope="class") def models(self): return {"model.sql": models__override_get_columns_macros} @pytest.fixture(scope="class") def macros(self): return {"macros.sql": macros__override_get_columns_macros} def test_overrides(self, project): # the first time, the model doesn't exist run_dbt() run_dbt() class TestMacroOverridePackage: """ The macro in `override-postgres-get-columns-macros` should override the `get_columns_in_relation` macro by default. """ @pytest.fixture(scope="class") def models(self): return {"model.sql": models__override_get_columns_macros} @pytest.fixture(scope="class") def macros(self): return {"macros.sql": macros__package_override_get_columns_macros} def test_overrides(self, project): # the first time, the model doesn't exist run_dbt() run_dbt() class TestMacroNotOverridePackage: """ The macro in `override-postgres-get-columns-macros` does NOT override the `get_columns_in_relation` macro because we tell dispatch to not look at the postgres macros. """ @pytest.fixture(scope="class") def models(self): return {"model.sql": models__override_get_columns_macros} @pytest.fixture(scope="class") def macros(self): return {"macros.sql": macros__package_override_get_columns_macros} @pytest.fixture(scope="class") def project_config_update(self): return { "dispatch": [{"macro_namespace": "dbt", "search_order": ["dbt"]}], } def test_overrides(self, project): # the first time, the model doesn't exist run_dbt(expect_pass=False) run_dbt(expect_pass=False) class TestDispatchMacroOverrideBuiltin(TestMacroOverrideBuiltin): # test the same functionality as above, but this time, # dbt.get_columns_in_relation will dispatch to a default__ macro # from an installed package, per dispatch config search_order @pytest.fixture(scope="class", autouse=True) def setUp(self, project): shutil.copytree( project.test_dir / Path("package_macro_overrides"), project.project_root / Path("package_macro_overrides"), ) @pytest.fixture(scope="class") def project_config_update(self): return { "dispatch": [ { "macro_namespace": "dbt", "search_order": ["test", "package_macro_overrides", "dbt"], } ], } @pytest.fixture(scope="class") def packages(self): return { "packages": [ { "local": "./package_macro_overrides", }, ] } def test_overrides(self, project): run_dbt(["deps"]) run_dbt() run_dbt() class TestMisnamedMacroNamespace: @pytest.fixture(scope="class", autouse=True) def setUp(self, project_root): test_utils_files = { "dbt_project.yml": dbt_project__incorrect_dispatch, "macros": { "cowsay.sql": macros__incorrect_dispatch, }, } write_project_files(project_root, "test_utils", test_utils_files) @pytest.fixture(scope="class") def models(self): return { "my_model.sql": models__incorrect_dispatch, } @pytest.fixture(scope="class") def packages(self): return { "packages": [ {"local": "test_utils"}, ] } def test_misnamed_macro_namespace( self, project, ): run_dbt(["deps"]) with pytest.raises(dbt_common.exceptions.CompilationError) as exc: run_dbt() assert "In dispatch: No macro named 'cowsay' found" in str(exc.value) class TestAdapterMacroDeprecated: @pytest.fixture(scope="class") def models(self): return {"model.sql": models__deprecated_adapter_macro_model} @pytest.fixture(scope="class") def macros(self): return {"macro.sql": macros__deprecated_adapter_macro} def test_invalid_macro(self, project): with pytest.raises(dbt_common.exceptions.CompilationError) as exc: run_dbt() assert 'The "adapter_macro" macro has been deprecated' in str(exc.value) class TestMacroMetaDocsMerge: @pytest.fixture(scope="class") def macros(self): return {"macro_config.sql": macros__config_sql, "macro_config.yml": macros__config_yml} def test_only_top_level_defined(self, project) -> None: """If only top-level defined then it exists both in top-level and config.""" run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert manifest is not None macro = manifest.macros.get("macro.test.macro_top_only") assert macro is not None expected_meta = {"top_k": "top_v"} expected_docs_show = True expected_docs_node_color = "#AAAAAA" assert macro.meta == expected_meta assert macro.config.meta == expected_meta assert macro.docs.show == expected_docs_show assert macro.docs.node_color == expected_docs_node_color assert macro.config.docs.show == expected_docs_show assert macro.config.docs.node_color == expected_docs_node_color def test_only_config_defined(self, project) -> None: """If only config defined then it exists both in top-level and config.""" run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert manifest is not None macro = manifest.macros.get("macro.test.macro_config_only") assert macro is not None expected_meta = {"cm_k": "cm_v"} expected_docs_show = False expected_docs_node_color = "#BBBBBB" assert macro.meta == expected_meta assert macro.config.meta == expected_meta assert macro.docs.show == expected_docs_show assert macro.docs.node_color == expected_docs_node_color assert macro.config.docs.show == expected_docs_show assert macro.config.docs.node_color == expected_docs_node_color def test_both_config_and_top_level_defined(self, project) -> None: """If both defined then config overrides top-level.""" run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert manifest is not None macro = manifest.macros.get("macro.test.macro_config") assert macro is not None expected_meta = { "top_k": "top_v", "cm_k": "cm_v", } expected_docs_show = True expected_docs_node_color = "#BBBBBB" assert macro.meta == expected_meta assert macro.config.meta == expected_meta assert macro.docs.show == expected_docs_show assert macro.docs.node_color == expected_docs_node_color assert macro.config.docs.show == expected_docs_show assert macro.config.docs.node_color == expected_docs_node_color ================================================ FILE: tests/functional/manifest_validations/test_check_for_spaces_in_model_names.py ================================================ from typing import Dict import pytest from dbt import deprecations from dbt.cli.main import dbtRunner from dbt.events.types import ( ResourceNamesWithSpacesDeprecation, SpacesInResourceNameDeprecation, ) from dbt.tests.util import update_config_file from dbt_common.events.base_types import EventLevel from dbt_common.events.event_catcher import EventCatcher class TestSpacesInModelNamesHappyPath: def test_no_warnings_when_no_spaces_in_name(self, project) -> None: event_catcher = EventCatcher(SpacesInResourceNameDeprecation) runner = dbtRunner(callbacks=[event_catcher.catch]) runner.invoke(["parse"]) assert len(event_catcher.caught_events) == 0 class TestSpacesInModelNamesSadPath: @pytest.fixture(scope="class") def models(self) -> Dict[str, str]: return { "my model.sql": "select 1 as id", } def tests_warning_when_spaces_in_name(self, project) -> None: config_patch = {"flags": {"require_resource_names_without_spaces": False}} update_config_file(config_patch, project.project_root, "dbt_project.yml") event_catcher = EventCatcher(SpacesInResourceNameDeprecation) total_catcher = EventCatcher(ResourceNamesWithSpacesDeprecation) runner = dbtRunner(callbacks=[event_catcher.catch, total_catcher.catch]) runner.invoke(["parse"]) assert len(total_catcher.caught_events) == 1 assert len(event_catcher.caught_events) == 1 event = event_catcher.caught_events[0] assert "Found spaces in the name of `model.test.my model`" in event.info.msg assert event.info.level == EventLevel.WARN class TestSpaceInModelNamesWithDebug: @pytest.fixture(scope="class") def models(self) -> Dict[str, str]: return { "my model.sql": "select 1 as id", "my model2.sql": "select 1 as id", } def tests_debug_when_spaces_in_name(self, project) -> None: config_patch = {"flags": {"require_resource_names_without_spaces": False}} update_config_file(config_patch, project.project_root, "dbt_project.yml") deprecations.reset_deprecations() spaces_check_catcher = EventCatcher(SpacesInResourceNameDeprecation) total_catcher = EventCatcher(ResourceNamesWithSpacesDeprecation) runner = dbtRunner(callbacks=[spaces_check_catcher.catch, total_catcher.catch]) runner.invoke(["parse"]) assert len(spaces_check_catcher.caught_events) == 1 assert len(total_catcher.caught_events) == 1 assert "Spaces found in 2 resource name(s)" in total_catcher.caught_events[0].info.msg assert ( "Run again with `--debug` to see them all." in total_catcher.caught_events[0].info.msg ) deprecations.reset_deprecations() spaces_check_catcher = EventCatcher(SpacesInResourceNameDeprecation) total_catcher = EventCatcher(ResourceNamesWithSpacesDeprecation) runner = dbtRunner(callbacks=[spaces_check_catcher.catch, total_catcher.catch]) runner.invoke(["parse", "--debug"]) assert len(spaces_check_catcher.caught_events) == 2 assert len(total_catcher.caught_events) == 1 assert ( "Run again with `--debug` to see them all." not in total_catcher.caught_events[0].info.msg ) class TestAllowSpacesInModelNamesFalse: @pytest.fixture(scope="class") def models(self) -> Dict[str, str]: return { "my model.sql": "select 1 as id", } def test_require_resource_names_without_spaces(self, project): config_patch = {"flags": {"require_resource_names_without_spaces": False}} update_config_file(config_patch, project.project_root, "dbt_project.yml") spaces_check_catcher = EventCatcher(SpacesInResourceNameDeprecation) runner = dbtRunner(callbacks=[spaces_check_catcher.catch]) runner.invoke(["parse"]) assert len(spaces_check_catcher.caught_events) == 1 assert spaces_check_catcher.caught_events[0].info.level == EventLevel.WARN config_patch = {"flags": {"require_resource_names_without_spaces": True}} update_config_file(config_patch, project.project_root, "dbt_project.yml") spaces_check_catcher = EventCatcher(SpacesInResourceNameDeprecation) runner = dbtRunner(callbacks=[spaces_check_catcher.catch]) result = runner.invoke(["parse"]) assert not result.success assert "Resource names cannot contain spaces" in result.exception.__str__() assert "my model.sql" in result.exception.__str__() assert len(spaces_check_catcher.caught_events) == 0 ================================================ FILE: tests/functional/materializations/conftest.py ================================================ import pytest from dbt.tests.fixtures.project import write_project_files override_view_adapter_pass_dep__dbt_project_yml = """ name: view_adapter_override version: '1.0' macro-paths: ['macros'] config-version: 2 """ override_view_adapter_pass_dep__macros__override_view_sql = """ {# copy+pasting the default view impl #} {% materialization view, default %} {%- set identifier = model['alias'] -%} {%- set tmp_identifier = model['name'] + '__dbt_tmp' -%} {%- set backup_identifier = model['name'] + '__dbt_backup' -%} {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%} {%- set target_relation = api.Relation.create(identifier=identifier, schema=schema, database=database, type='view') -%} {%- set intermediate_relation = api.Relation.create(identifier=tmp_identifier, schema=schema, database=database, type='view') -%} /* This relation (probably) doesn't exist yet. If it does exist, it's a leftover from a previous run, and we're going to try to drop it immediately. At the end of this materialization, we're going to rename the "old_relation" to this identifier, and then we're going to drop it. In order to make sure we run the correct one of: - drop view ... - drop table ... We need to set the type of this relation to be the type of the old_relation, if it exists, or else "view" as a sane default if it does not. Note that if the old_relation does not exist, then there is nothing to move out of the way and subsequentally drop. In that case, this relation will be effectively unused. */ {%- set backup_relation_type = 'view' if old_relation is none else old_relation.type -%} {%- set backup_relation = api.Relation.create(identifier=backup_identifier, schema=schema, database=database, type=backup_relation_type) -%} {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%} {{ run_hooks(pre_hooks, inside_transaction=False) }} -- drop the temp relations if they exists for some reason {{ adapter.drop_relation(intermediate_relation) }} {{ adapter.drop_relation(backup_relation) }} -- `BEGIN` happens here: {{ run_hooks(pre_hooks, inside_transaction=True) }} -- build model {% call statement('main') -%} {{ create_view_as(intermediate_relation, sql) }} {%- endcall %} -- cleanup -- move the existing view out of the way {% if old_relation is not none %} {{ adapter.rename_relation(target_relation, backup_relation) }} {% endif %} {{ adapter.rename_relation(intermediate_relation, target_relation) }} {{ run_hooks(post_hooks, inside_transaction=True) }} {{ adapter.commit() }} {{ drop_relation_if_exists(backup_relation) }} {{ run_hooks(post_hooks, inside_transaction=False) }} {{ return({'relations': [target_relation]}) }} {%- endmaterialization -%} """ override_view_adapter_macros__override_view_sql = """ {%- materialization view, adapter='postgres' -%} {{ exceptions.raise_compiler_error('intentionally raising an error in the postgres view materialization') }} {%- endmaterialization -%} {# copy+pasting the default view impl #} {% materialization view, default %} {%- set identifier = model['alias'] -%} {%- set tmp_identifier = model['name'] + '__dbt_tmp' -%} {%- set backup_identifier = model['name'] + '__dbt_backup' -%} {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%} {%- set target_relation = api.Relation.create(identifier=identifier, schema=schema, database=database, type='view') -%} {%- set intermediate_relation = api.Relation.create(identifier=tmp_identifier, schema=schema, database=database, type='view') -%} /* This relation (probably) doesn't exist yet. If it does exist, it's a leftover from a previous run, and we're going to try to drop it immediately. At the end of this materialization, we're going to rename the "old_relation" to this identifier, and then we're going to drop it. In order to make sure we run the correct one of: - drop view ... - drop table ... We need to set the type of this relation to be the type of the old_relation, if it exists, or else "view" as a sane default if it does not. Note that if the old_relation does not exist, then there is nothing to move out of the way and subsequentally drop. In that case, this relation will be effectively unused. */ {%- set backup_relation_type = 'view' if old_relation is none else old_relation.type -%} {%- set backup_relation = api.Relation.create(identifier=backup_identifier, schema=schema, database=database, type=backup_relation_type) -%} {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%} {{ run_hooks(pre_hooks, inside_transaction=False) }} -- drop the temp relations if they exists for some reason {{ adapter.drop_relation(intermediate_relation) }} {{ adapter.drop_relation(backup_relation) }} -- `BEGIN` happens here: {{ run_hooks(pre_hooks, inside_transaction=True) }} -- build model {% call statement('main') -%} {{ create_view_as(intermediate_relation, sql) }} {%- endcall %} -- cleanup -- move the existing view out of the way {% if old_relation is not none %} {{ adapter.rename_relation(target_relation, backup_relation) }} {% endif %} {{ adapter.rename_relation(intermediate_relation, target_relation) }} {{ run_hooks(post_hooks, inside_transaction=True) }} {{ adapter.commit() }} {{ drop_relation_if_exists(backup_relation) }} {{ run_hooks(post_hooks, inside_transaction=False) }} {{ return({'relations': [target_relation]}) }} {%- endmaterialization -%} """ override_view_adapter_dep__dbt_project_yml = """ name: view_adapter_override version: '1.0' macro-paths: ['macros'] config-version: 2 """ override_view_adapter_dep__macros__override_view_sql = """ {%- materialization view, adapter='postgres' -%} {{ exceptions.raise_compiler_error('intentionally raising an error in the postgres view materialization') }} {%- endmaterialization -%} {# copy+pasting the default view impl #} {% materialization view, default %} {%- set identifier = model['alias'] -%} {%- set tmp_identifier = model['name'] + '__dbt_tmp' -%} {%- set backup_identifier = model['name'] + '__dbt_backup' -%} {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%} {%- set target_relation = api.Relation.create(identifier=identifier, schema=schema, database=database, type='view') -%} {%- set intermediate_relation = api.Relation.create(identifier=tmp_identifier, schema=schema, database=database, type='view') -%} /* This relation (probably) doesn't exist yet. If it does exist, it's a leftover from a previous run, and we're going to try to drop it immediately. At the end of this materialization, we're going to rename the "old_relation" to this identifier, and then we're going to drop it. In order to make sure we run the correct one of: - drop view ... - drop table ... We need to set the type of this relation to be the type of the old_relation, if it exists, or else "view" as a sane default if it does not. Note that if the old_relation does not exist, then there is nothing to move out of the way and subsequentally drop. In that case, this relation will be effectively unused. */ {%- set backup_relation_type = 'view' if old_relation is none else old_relation.type -%} {%- set backup_relation = api.Relation.create(identifier=backup_identifier, schema=schema, database=database, type=backup_relation_type) -%} {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%} {{ run_hooks(pre_hooks, inside_transaction=False) }} -- drop the temp relations if they exists for some reason {{ adapter.drop_relation(intermediate_relation) }} {{ adapter.drop_relation(backup_relation) }} -- `BEGIN` happens here: {{ run_hooks(pre_hooks, inside_transaction=True) }} -- build model {% call statement('main') -%} {{ create_view_as(intermediate_relation, sql) }} {%- endcall %} -- cleanup -- move the existing view out of the way {% if old_relation is not none %} {{ adapter.rename_relation(target_relation, backup_relation) }} {% endif %} {{ adapter.rename_relation(intermediate_relation, target_relation) }} {{ run_hooks(post_hooks, inside_transaction=True) }} {{ adapter.commit() }} {{ drop_relation_if_exists(backup_relation) }} {{ run_hooks(post_hooks, inside_transaction=False) }} {{ return({'relations': [target_relation]}) }} {%- endmaterialization -%} """ override_view_default_dep__dbt_project_yml = """ name: view_default_override config-version: 2 version: '1.0' macro-paths: ['macros'] """ override_view_default_dep__macros__default_view_sql = """ {%- materialization view, default -%} {{ exceptions.raise_compiler_error('intentionally raising an error in the default view materialization') }} {%- endmaterialization -%} """ override_view_return_no_relation__dbt_project_yml = """ name: view_adapter_override version: 2 macro-paths: ['macros'] config-version: 2 """ override_view_return_no_relation__macros__override_view_sql = """ {# copy+pasting the default view impl #} {% materialization view, default %} {%- set identifier = model['alias'] -%} {%- set tmp_identifier = model['name'] + '__dbt_tmp' -%} {%- set backup_identifier = model['name'] + '__dbt_backup' -%} {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%} {%- set target_relation = api.Relation.create(identifier=identifier, schema=schema, database=database, type='view') -%} {%- set intermediate_relation = api.Relation.create(identifier=tmp_identifier, schema=schema, database=database, type='view') -%} /* This relation (probably) doesn't exist yet. If it does exist, it's a leftover from a previous run, and we're going to try to drop it immediately. At the end of this materialization, we're going to rename the "old_relation" to this identifier, and then we're going to drop it. In order to make sure we run the correct one of: - drop view ... - drop table ... We need to set the type of this relation to be the type of the old_relation, if it exists, or else "view" as a sane default if it does not. Note that if the old_relation does not exist, then there is nothing to move out of the way and subsequentally drop. In that case, this relation will be effectively unused. */ {%- set backup_relation_type = 'view' if old_relation is none else old_relation.type -%} {%- set backup_relation = api.Relation.create(identifier=backup_identifier, schema=schema, database=database, type=backup_relation_type) -%} {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%} {{ run_hooks(pre_hooks, inside_transaction=False) }} -- drop the temp relations if they exists for some reason {{ adapter.drop_relation(intermediate_relation) }} {{ adapter.drop_relation(backup_relation) }} -- `BEGIN` happens here: {{ run_hooks(pre_hooks, inside_transaction=True) }} -- build model {% call statement('main') -%} {{ create_view_as(intermediate_relation, sql) }} {%- endcall %} -- cleanup -- move the existing view out of the way {% if old_relation is not none %} {{ adapter.rename_relation(target_relation, backup_relation) }} {% endif %} {{ adapter.rename_relation(intermediate_relation, target_relation) }} {{ run_hooks(post_hooks, inside_transaction=True) }} {{ adapter.commit() }} {{ drop_relation_if_exists(backup_relation) }} {{ run_hooks(post_hooks, inside_transaction=False) }} {# do not return anything! #} {# {{ return({'relations': [target_relation]}) }} #} {%- endmaterialization -%} """ custom_materialization_dep__dbt_project_yml = """ name: custom_materialization_default macro-paths: ['macros'] """ custom_materialization_sql = """ {% materialization custom_materialization, default %} {%- set target_relation = this.incorporate(type='table') %} {% call statement('main') -%} select 1 as column1 {%- endcall %} {{ return({'relations': [target_relation]}) }} {% endmaterialization %} """ @pytest.fixture(scope="class") def override_view_adapter_pass_dep(project_root): files = { "dbt_project.yml": override_view_adapter_pass_dep__dbt_project_yml, "macros": {"override_view.sql": override_view_adapter_pass_dep__macros__override_view_sql}, } write_project_files(project_root, "override-view-adapter-pass-dep", files) @pytest.fixture(scope="class") def override_view_adapter_macros(project_root): files = {"override_view.sql": override_view_adapter_macros__override_view_sql} write_project_files(project_root, "override-view-adapter-macros", files) @pytest.fixture(scope="class") def override_view_adapter_dep(project_root): files = { "dbt_project.yml": override_view_adapter_dep__dbt_project_yml, "macros": {"override_view.sql": override_view_adapter_dep__macros__override_view_sql}, } write_project_files(project_root, "override-view-adapter-dep", files) @pytest.fixture(scope="class") def override_view_default_dep(project_root): files = { "dbt_project.yml": override_view_default_dep__dbt_project_yml, "macros": {"default_view.sql": override_view_default_dep__macros__default_view_sql}, } write_project_files(project_root, "override-view-default-dep", files) @pytest.fixture(scope="class") def override_view_return_no_relation(project_root): files = { "dbt_project.yml": override_view_return_no_relation__dbt_project_yml, "macros": { "override_view.sql": override_view_return_no_relation__macros__override_view_sql }, } write_project_files(project_root, "override-view-return-no-relation", files) @pytest.fixture(scope="class") def custom_materialization_dep(project_root): files = { "dbt_project.yml": custom_materialization_dep__dbt_project_yml, "macros": {"custom_materialization.sql": custom_materialization_sql}, } write_project_files(project_root, "custom-materialization-dep", files) ================================================ FILE: tests/functional/materializations/fixtures.py ================================================ fct_eph_first_sql = """ -- fct_eph_first.sql {{ config(materialized='ephemeral') }} with int_eph_first as( select * from {{ ref('int_eph_first') }} ) select * from int_eph_first """ int_eph_first_sql = """ -- int_eph_first.sql {{ config(materialized='ephemeral') }} select 1 as first_column, 2 as second_column """ schema_yml = """ version: 2 models: - name: int_eph_first columns: - name: first_column data_tests: - not_null - name: second_column data_tests: - not_null - name: fct_eph_first columns: - name: first_column data_tests: - not_null - name: second_column data_tests: - not_null """ bar_sql = """ {{ config(materialized = 'table') }} WITH foo AS ( SELECT * FROM {{ ref('foo') }} ), foo_1 AS ( SELECT * FROM {{ ref('foo_1') }} ), foo_2 AS ( SELECT * FROM {{ ref('foo_2') }} ) SELECT * FROM foo UNION ALL SELECT * FROM foo_1 UNION ALL SELECT * FROM foo_2 """ bar1_sql = """ {{ config(materialized = 'table') }} WITH foo AS ( SELECT * FROM {{ ref('foo') }} ), foo_1 AS ( SELECT * FROM {{ ref('foo_1') }} ), foo_2 AS ( SELECT * FROM {{ ref('foo_2') }} ) SELECT * FROM foo UNION ALL SELECT * FROM foo_1 UNION ALL SELECT * FROM foo_2 """ bar2_sql = """ {{ config(materialized = 'table') }} WITH foo AS ( SELECT * FROM {{ ref('foo') }} ), foo_1 AS ( SELECT * FROM {{ ref('foo_1') }} ), foo_2 AS ( SELECT * FROM {{ ref('foo_2') }} ) SELECT * FROM foo UNION ALL SELECT * FROM foo_1 UNION ALL SELECT * FROM foo_2 """ bar3_sql = """ {{ config(materialized = 'table') }} WITH foo AS ( SELECT * FROM {{ ref('foo') }} ), foo_1 AS ( SELECT * FROM {{ ref('foo_1') }} ), foo_2 AS ( SELECT * FROM {{ ref('foo_2') }} ) SELECT * FROM foo UNION ALL SELECT * FROM foo_1 UNION ALL SELECT * FROM foo_2 """ bar4_sql = """ {{ config(materialized = 'table') }} WITH foo AS ( SELECT * FROM {{ ref('foo') }} ), foo_1 AS ( SELECT * FROM {{ ref('foo_1') }} ), foo_2 AS ( SELECT * FROM {{ ref('foo_2') }} ) SELECT * FROM foo UNION ALL SELECT * FROM foo_1 UNION ALL SELECT * FROM foo_2 """ bar5_sql = """ {{ config(materialized = 'table') }} WITH foo AS ( SELECT * FROM {{ ref('foo') }} ), foo_1 AS ( SELECT * FROM {{ ref('foo_1') }} ), foo_2 AS ( SELECT * FROM {{ ref('foo_2') }} ) SELECT * FROM foo UNION ALL SELECT * FROM foo_1 UNION ALL SELECT * FROM foo_2 """ baz_sql = """ {{ config(materialized = 'table') }} SELECT * FROM {{ ref('bar') }} """ baz1_sql = """ {{ config(materialized = 'table') }} SELECT * FROM {{ ref('bar_1') }} """ foo_sql = """ {{ config(materialized = 'ephemeral') }} with source as ( select 1 as id ), renamed as ( select id as uid from source ) select * from renamed """ foo1_sql = """ {{ config(materialized = 'ephemeral') }} WITH source AS ( SELECT 1 AS id ), RENAMED as ( SELECT id as UID FROM source ) SELECT * FROM renamed """ foo2_sql = """ {{ config(materialized = 'ephemeral') }} WITH source AS ( SELECT 1 AS id ), RENAMED as ( SELECT id as UID FROM source ) SELECT * FROM renamed """ ================================================ FILE: tests/functional/materializations/test_custom_materialization.py ================================================ from collections import defaultdict import pytest from dbt import deprecations from dbt.tests.util import run_dbt models__model_sql = """ {{ config(materialized='view') }} select 1 as id """ models_custom_materialization__model_sql = """ {{ config(materialized='custom_materialization') }} select 1 as id """ @pytest.fixture(scope="class") def models(): return {"model.sql": models__model_sql} @pytest.fixture(scope="class") def set_up_deprecations(): deprecations.reset_deprecations() assert deprecations.active_deprecations == defaultdict(int) class TestOverrideAdapterDependency: # make sure that if there's a dependency with an adapter-specific # materialization, we honor that materialization @pytest.fixture(scope="class") def packages(self): return {"packages": [{"local": "override-view-adapter-dep"}]} def test_adapter_dependency(self, project, override_view_adapter_dep, set_up_deprecations): run_dbt(["deps"]) # this should pass because implicit overrides are now deprecated (= disabled by default) run_dbt(["run"]) class TestOverrideAdapterDependencyDeprecated: # make sure that if there's a dependency with an adapter-specific # materialization, we honor that materialization @pytest.fixture(scope="class") def packages(self): return {"packages": [{"local": "override-view-adapter-dep"}]} @pytest.fixture(scope="class") def project_config_update(self): return { "flags": { "require_explicit_package_overrides_for_builtin_materializations": True, }, } def test_adapter_dependency_deprecate_overrides( self, project, override_view_adapter_dep, set_up_deprecations ): run_dbt(["deps"]) # this should pass because the override is buggy and unused run_dbt(["run"]) # no deprecation warning -- flag used correctly assert deprecations.active_deprecations == defaultdict(int) class TestOverrideAdapterDependencyLegacy: # make sure that if there's a dependency with an adapter-specific # materialization, we honor that materialization @pytest.fixture(scope="class") def packages(self): return {"packages": [{"local": "override-view-adapter-dep"}]} @pytest.fixture(scope="class") def project_config_update(self): return { "flags": { "require_explicit_package_overrides_for_builtin_materializations": False, }, } def test_adapter_dependency(self, project, override_view_adapter_dep, set_up_deprecations): run_dbt(["deps"]) # this should error because the override is buggy run_dbt(["run"], expect_pass=False) # overriding a built-in materialization scoped to adapter from package is deprecated assert "package-materialization-override" in deprecations.active_deprecations class TestOverrideDefaultDependency: @pytest.fixture(scope="class") def packages(self): return {"packages": [{"local": "override-view-default-dep"}]} def test_default_dependency(self, project, override_view_default_dep, set_up_deprecations): run_dbt(["deps"]) # this should pass because implicit overrides are now deprecated (= disabled by default) run_dbt(["run"]) class TestOverrideDefaultDependencyDeprecated: @pytest.fixture(scope="class") def packages(self): return {"packages": [{"local": "override-view-default-dep"}]} @pytest.fixture(scope="class") def project_config_update(self): return { "flags": { "require_explicit_package_overrides_for_builtin_materializations": True, }, } def test_default_dependency_deprecated( self, project, override_view_default_dep, set_up_deprecations ): run_dbt(["deps"]) # this should pass because the override is buggy and unused run_dbt(["run"]) # overriding a built-in materialization from package is deprecated assert deprecations.active_deprecations == defaultdict(int) class TestOverrideDefaultDependencyLegacy: @pytest.fixture(scope="class") def packages(self): return {"packages": [{"local": "override-view-default-dep"}]} @pytest.fixture(scope="class") def project_config_update(self): return { "flags": { "require_explicit_package_overrides_for_builtin_materializations": False, }, } def test_default_dependency(self, project, override_view_default_dep, set_up_deprecations): run_dbt(["deps"]) # this should error because the override is buggy run_dbt(["run"], expect_pass=False) # overriding a built-in materialization from package is deprecated assert "package-materialization-override" in deprecations.active_deprecations root_view_override_macro = """ {% materialization view, default %} {{ return(view_default_override.materialization_view_default()) }} {% endmaterialization %} """ class TestOverrideDefaultDependencyRootOverride: @pytest.fixture(scope="class") def packages(self): return {"packages": [{"local": "override-view-default-dep"}]} @pytest.fixture(scope="class") def macros(self): return {"my_view.sql": root_view_override_macro} def test_default_dependency_with_root_override( self, project, override_view_default_dep, set_up_deprecations ): run_dbt(["deps"]) # this should error because the override is buggy run_dbt(["run"], expect_pass=False) # using an package-overriden built-in materialization in a root matereialization is _not_ deprecated assert deprecations.active_deprecations == defaultdict(int) class TestCustomMaterializationDependency: @pytest.fixture(scope="class") def models(self): return {"model.sql": models_custom_materialization__model_sql} @pytest.fixture(scope="class") def packages(self): return {"packages": [{"local": "custom-materialization-dep"}]} def test_custom_materialization_deopendency( self, project, custom_materialization_dep, set_up_deprecations ): run_dbt(["deps"]) # custom materilization is valid run_dbt(["run"]) # using a custom materialization is from an installed package is _not_ deprecated assert deprecations.active_deprecations == defaultdict(int) class TestOverrideAdapterDependencyPassing: @pytest.fixture(scope="class") def packages(self): return {"packages": [{"local": "override-view-adapter-pass-dep"}]} def test_default_dependency(self, project, override_view_adapter_pass_dep): run_dbt(["deps"]) # this should pass because the override is ok run_dbt(["run"]) class TestOverrideAdapterLocal: # make sure that the local default wins over the dependency # adapter-specific @pytest.fixture(scope="class") def packages(self): return {"packages": [{"local": "override-view-adapter-pass-dep"}]} @pytest.fixture(scope="class") def project_config_update(self): return {"macro-paths": ["override-view-adapter-macros"]} def test_default_dependency( self, project, override_view_adapter_pass_dep, override_view_adapter_macros ): run_dbt(["deps"]) # this should error because the override is buggy run_dbt(["run"], expect_pass=False) class TestOverrideDefaultReturn: @pytest.fixture(scope="class") def project_config_update(self): return {"macro-paths": ["override-view-return-no-relation"]} def test_default_dependency(self, project, override_view_return_no_relation): run_dbt(["deps"]) results = run_dbt(["run"], expect_pass=False) assert "did not explicitly return a list of relations" in results[0].message ================================================ FILE: tests/functional/materializations/test_ephemeral_compilation.py ================================================ import pytest from dbt.contracts.graph.nodes import ModelNode from dbt.contracts.results import RunExecutionResult, RunResult from dbt.tests.util import run_dbt from tests.functional.materializations.fixtures import ( bar1_sql, bar2_sql, bar3_sql, bar4_sql, bar5_sql, bar_sql, baz1_sql, baz_sql, fct_eph_first_sql, foo1_sql, foo2_sql, foo_sql, int_eph_first_sql, schema_yml, ) # Note: This tests compilation only, so is a dbt Core test and not an adapter test. # There is some complicated logic in core/dbt/compilation.py having to do with # ephemeral nodes and handling multiple threads at the same time. This test # fails fairly regularly if that is broken, but does occasionally work (depending # on the order in which things are compiled). It requires multi-threading to fail. SUPPRESSED_CTE_EXPECTED_OUTPUT = """-- fct_eph_first.sql with int_eph_first as( select * from __dbt__cte__int_eph_first ) select * from int_eph_first""" class TestEphemeralCompilation: @pytest.fixture(scope="class") def models(self): return { "int_eph_first.sql": int_eph_first_sql, "fct_eph_first.sql": fct_eph_first_sql, "schema.yml": schema_yml, } def test_ephemeral_compilation(self, project): # Note: There are no models that run successfully. This testcase tests running tests. results = run_dbt(["run"]) assert len(results) == 0 def test__suppress_injected_ctes(self, project): compile_output = run_dbt( ["compile", "--no-inject-ephemeral-ctes", "--select", "fct_eph_first"] ) assert isinstance(compile_output, RunExecutionResult) node_result = compile_output.results[0] assert isinstance(node_result, RunResult) node = node_result.node assert isinstance(node, ModelNode) assert node.compiled_code == SUPPRESSED_CTE_EXPECTED_OUTPUT # From: https://github.com/jeremyyeo/ephemeral-invalid-sql-repro/tree/main/models class TestLargeEphemeralCompilation: @pytest.fixture(scope="class") def models(self): return { "bar.sql": bar_sql, "bar_1.sql": bar1_sql, "bar_2.sql": bar2_sql, "bar_3.sql": bar3_sql, "bar_4.sql": bar4_sql, "bar_5.sql": bar5_sql, "baz.sql": baz_sql, "baz_1.sql": baz1_sql, "foo.sql": foo_sql, "foo_1.sql": foo1_sql, "foo_2.sql": foo2_sql, } def test_ephemeral_compilation(self, project): # 8/11 table models are built as expected. no compilation errors results = run_dbt(["build"]) assert len(results) == 8 ================================================ FILE: tests/functional/materializations/test_incremental.py ================================================ import pytest from dbt.context.providers import generate_runtime_model_context from dbt.exceptions import DbtRuntimeError from dbt.tests.util import get_manifest, run_dbt my_model_sql = """ select 1 as fun """ @pytest.fixture(scope="class") def models(): return {"my_model.sql": my_model_sql} def test_basic(project): results = run_dbt(["run"]) assert len(results) == 1 manifest = get_manifest(project.project_root) model = manifest.nodes["model.test.my_model"] # Normally the context will be provided by the macro that calls the # get_incrmental_strategy_macro method, but for testing purposes # we create a runtime_model_context. context = generate_runtime_model_context( model, project.adapter.config, manifest, ) macro_func = project.adapter.get_incremental_strategy_macro(context, "default") assert macro_func assert type(macro_func).__name__ == "MacroGenerator" macro_func = project.adapter.get_incremental_strategy_macro(context, "append") assert macro_func assert type(macro_func).__name__ == "MacroGenerator" macro_func = project.adapter.get_incremental_strategy_macro(context, "delete+insert") assert macro_func assert type(macro_func).__name__ == "MacroGenerator" # This incremental strategy only works for Postgres >= 15 macro_func = project.adapter.get_incremental_strategy_macro(context, "merge") assert macro_func assert type(macro_func).__name__ == "MacroGenerator" # This incremental strategy is not valid for Postgres with pytest.raises(DbtRuntimeError) as excinfo: macro_func = project.adapter.get_incremental_strategy_macro(context, "insert_overwrite") assert "insert_overwrite" in str(excinfo.value) ================================================ FILE: tests/functional/materializations/test_incremental_with_contract.py ================================================ import pytest from dbt.tests.util import ( check_relations_equal, get_relation_columns, relation_from_name, run_dbt, ) seeds_base_csv = """ id,name_xxx,some_date 1,Easton,1981-05-20T06:46:51 2,Lillian,1978-09-03T18:10:33 3,Jeremiah,1982-03-11T03:59:51 4,Nolan,1976-05-06T20:21:35 5,Hannah,1982-06-23T05:41:26 6,Eleanor,1991-08-10T23:12:21 7,Lily,1971-03-29T14:58:02 8,Jonathan,1988-02-26T02:55:24 9,Adrian,1994-02-09T13:14:23 10,Nora,1976-03-01T16:51:39 """.lstrip() seeds_added_csv = ( seeds_base_csv + """ 11,Mateo,2014-09-07T17:04:27 12,Julian,2000-02-04T11:48:30 13,Gabriel,2001-07-10T07:32:52 14,Isaac,2002-11-24T03:22:28 15,Levi,2009-11-15T11:57:15 16,Elizabeth,2005-04-09T03:50:11 17,Grayson,2019-08-06T19:28:17 18,Dylan,2014-03-01T11:50:41 19,Jayden,2009-06-06T07:12:49 20,Luke,2003-12-05T21:42:18 """.lstrip() ) incremental_not_schema_change_sql = """ {{ config(materialized="incremental", unique_key="user_id_current_time",on_schema_change="sync_all_columns") }} select 1 || '-' || current_timestamp as user_id_current_time, {% if is_incremental() %} 'thisis18characters' as platform {% else %} 'okthisis20characters' as platform {% endif %} """ incremental_sql = """ {{ config(materialized="incremental") }} select * from {{ source('raw', 'seed') }} {% if is_incremental() %} where id > (select max(id) from {{ this }}) {% endif %} """ schema_base_yml = """ sources: - name: raw schema: "{{ target.schema }}" tables: - name: seed identifier: "{{ var('seed_name', 'base') }}" models: - name: incremental config: contract: enforced: true on_schema_change: append_new_columns columns: - name: id data_type: int - name: name_xxx data_type: character varying(10) - name: some_date data_type: timestamp """ class TestIncremental: @pytest.fixture(scope="class") def project_config_update(self): return {"name": "incremental"} @pytest.fixture(scope="class") def models(self): return {"incremental.sql": incremental_sql, "schema.yml": schema_base_yml} @pytest.fixture(scope="class") def seeds(self): return {"base.csv": seeds_base_csv, "added.csv": seeds_added_csv} def test_incremental(self, project): # seed command results = run_dbt(["seed"]) assert len(results) == 2 # base table rowcount relation = relation_from_name(project.adapter, "base") result = project.run_sql(f"select count(*) as num_rows from {relation}", fetch="one") assert result[0] == 10 # added table rowcount relation = relation_from_name(project.adapter, "added") result = project.run_sql(f"select count(*) as num_rows from {relation}", fetch="one") assert result[0] == 20 # run command # the "seed_name" var changes the seed identifier in the schema file results = run_dbt(["run", "--vars", "seed_name: base"]) assert len(results) == 1 # check relations equal check_relations_equal(project.adapter, ["base", "incremental"]) # change seed_name var # the "seed_name" var changes the seed identifier in the schema file results = run_dbt(["run", "--debug", "--vars", "seed_name: added"]) assert len(results) == 1 # Error before fix: Changing col type from character varying(10) to character varying(256) in table: # "dbt"."test<...>_test_incremental_with_contract"."incremental" columns = get_relation_columns(project.adapter, "incremental") # [('id', 'integer', None), ('name_xxx', 'character varying', 10), ('some_date', 'timestamp without time zone', None)] for column in columns: if column[0] == "name_xxx": assert column[2] == 10 ================================================ FILE: tests/functional/materializations/test_runtime_materialization.py ================================================ import pytest from dbt.tests.util import check_relations_equal, check_table_does_not_exist, run_dbt models__view_sql = """ {{ config( materialized = "view" ) }} select * from {{ this.schema }}.seed {% if is_incremental() %} {% do exceptions.raise_compiler_error("is_incremental() evaluated to True in a view") %} {% endif %} """ models__incremental_sql = """ {{ config( materialized = "incremental" ) }} select * from {{ this.schema }}.seed {% if is_incremental() %} where id > (select max(id) from {{this}}) {% endif %} """ models__materialized_sql = """ {{ config( materialized = "table" ) }} select * from {{ this.schema }}.seed {% if is_incremental() %} {% do exceptions.raise_compiler_error("is_incremental() evaluated to True in a table") %} {% endif %} """ seeds__seed_csv = """id,first_name,last_name,email,gender,ip_address 1,Jack,Hunter,jhunter0@pbs.org,Male,59.80.20.168 2,Kathryn,Walker,kwalker1@ezinearticles.com,Female,194.121.179.35 3,Gerald,Ryan,gryan2@com.com,Male,11.3.212.243 4,Bonnie,Spencer,bspencer3@ameblo.jp,Female,216.32.196.175 5,Harold,Taylor,htaylor4@people.com.cn,Male,253.10.246.136 6,Jacqueline,Griffin,jgriffin5@t.co,Female,16.13.192.220 7,Wanda,Arnold,warnold6@google.nl,Female,232.116.150.64 8,Craig,Ortiz,cortiz7@sciencedaily.com,Male,199.126.106.13 9,Gary,Day,gday8@nih.gov,Male,35.81.68.186 10,Rose,Wright,rwright9@yahoo.co.jp,Female,236.82.178.100 """ invalidate_incremental_sql = """ insert into {schema}.incremental (first_name, last_name, email, gender, ip_address) values ('Hank', 'Hund', 'hank@yahoo.com', 'Male', '101.239.70.175'); """ update_sql = """ -- create a view on top of the models create view {schema}.dependent_view as ( select count(*) from {schema}.materialized union all select count(*) from {schema}.view union all select count(*) from {schema}.incremental ); insert into {schema}.seed (id, first_name, last_name, email, gender, ip_address) values (101, 'Michael', 'Perez', 'mperez0@chronoengine.com', 'Male', '106.239.70.175'); insert into {schema}.seed (id, first_name, last_name, email, gender, ip_address) values (102, 'Shawn', 'Mccoy', 'smccoy1@reddit.com', 'Male', '24.165.76.182'); insert into {schema}.seed (id, first_name, last_name, email, gender, ip_address) values (103, 'Kathleen', 'Payne', 'kpayne2@cargocollective.com', 'Female', '113.207.168.106'); insert into {schema}.seed (id, first_name, last_name, email, gender, ip_address) values (104, 'Jimmy', 'Cooper', 'jcooper3@cargocollective.com', 'Male', '198.24.63.114'); insert into {schema}.seed (id, first_name, last_name, email, gender, ip_address) values (105, 'Katherine', 'Rice', 'krice4@typepad.com', 'Female', '36.97.186.238'); """ create_view__dbt_tmp_sql = """ create view {schema}.view__dbt_tmp as ( select 1 as id ); """ create_view__dbt_backup_sql = """ create view {schema}.view__dbt_backup as ( select 1 as id ); """ create_incremental__dbt_tmp_sql = """ create table {schema}.incremental__dbt_tmp as ( select 1 as id ); """ @pytest.fixture(scope="class") def models(): return { "view.sql": models__view_sql, "incremental.sql": models__incremental_sql, "materialized.sql": models__materialized_sql, } @pytest.fixture(scope="class") def seeds(): return {"seed.csv": seeds__seed_csv} @pytest.fixture(scope="class", autouse=True) def setup(project): run_dbt(["seed"]) class TestRuntimeMaterialization: @pytest.fixture(scope="class") def project_config_update(self): return { "seeds": { "quote_columns": False, } } def test_full_refresh( self, project, ): # initial full-refresh should have no effect results = run_dbt(["run", "-f"]) assert len(results) == 3 check_relations_equal(project.adapter, ["seed", "view", "incremental", "materialized"]) # adds one record to the incremental model. full-refresh should truncate then re-run project.run_sql(invalidate_incremental_sql) results = run_dbt(["run", "-f"]) assert len(results) == 3 check_relations_equal(project.adapter, ["seed", "incremental"]) project.run_sql(update_sql) results = run_dbt(["run", "-f"]) assert len(results) == 3 check_relations_equal(project.adapter, ["seed", "view", "incremental", "materialized"]) def test_delete_dbt_tmp_relation( self, project, ): # This creates a __dbt_tmp view - make sure it doesn't interfere with the dbt run project.run_sql(create_view__dbt_tmp_sql) results = run_dbt(["run", "--model", "view"]) assert len(results) == 1 check_table_does_not_exist(project.adapter, "view__dbt_tmp") check_relations_equal(project.adapter, ["seed", "view"]) # Again, but with a __dbt_backup view project.run_sql(create_view__dbt_backup_sql) results = run_dbt(["run", "--model", "view"]) assert len(results) == 1 check_table_does_not_exist(project.adapter, "view__dbt_backup") check_relations_equal(project.adapter, ["seed", "view"]) # Again, but against the incremental materialization results = run_dbt(["run", "--model", "incremental"]) project.run_sql(create_incremental__dbt_tmp_sql) assert len(results) == 1 results = run_dbt(["run", "--model", "incremental", "-f"]) assert len(results) == 1 check_table_does_not_exist(project.adapter, "incremental__dbt_tmp") check_relations_equal(project.adapter, ["seed", "incremental"]) # Run same tests with models configured with full_refresh class TestRuntimeMaterializationWithConfig(TestRuntimeMaterialization): @pytest.fixture(scope="class") def project_config_update(self): return { "seeds": { "quote_columns": False, }, "models": {"full_refresh": True}, } ================================================ FILE: tests/functional/materializations/test_supported_languages.py ================================================ import pytest from dbt.tests.util import run_dbt custom_mat_tmpl = """ {% materialization custom_mat{} %} {%- set target_relation = this.incorporate(type='table') %} {% call statement('main') -%} select 1 as column1 {%- endcall %} {{ return({'relations': [target_relation]}) }} {% endmaterialization %} """ models__sql_model = """ {{ config(materialized='custom_mat') }} select 1 as fun """ models__py_model = """ def model(dbt, session): dbt.config(materialized='custom_mat') return """ class SupportedLanguageBase: model_map = { "sql": ("sql_model.sql", models__sql_model), "python": ("py_model.py", models__py_model), } @pytest.fixture(scope="class") def macros(self): custom_mat = custom_mat_tmpl.replace("{}", "") if hasattr(self, "supported_langs"): custom_mat = custom_mat_tmpl.replace( "{}", f", supported_languages=[{self.lang_list()}]" ) return {"custom_mat.sql": custom_mat} @pytest.fixture(scope="class") def models(self): file_name, model = self.model_map[self.use_lang] return {file_name: model} def lang_list(self): return ", ".join([f"'{l}'" for l in self.supported_langs]) def test_language(self, project): result = run_dbt(["run"], expect_pass=self.expect_pass) if not self.expect_pass: assert "only supports languages" in result.results[0].message class TestSupportedLanguages_SupportsDefault_UsingSql(SupportedLanguageBase): use_lang = "sql" expect_pass = True class TestSupportedLanguages_SupportsDefault_UsingPython(SupportedLanguageBase): use_lang = "python" expect_pass = False class TestSupportedLanguages_SupportsSql_UsingSql(SupportedLanguageBase): supported_langs = ["sql"] use_lang = "sql" expect_pass = True class TestSupportedLanguages_SuppotsSql_UsingPython(SupportedLanguageBase): supported_langs = ["sql"] use_lang = "python" expect_pass = False class TestSupportedLanguages_SuppotsPython_UsingSql(SupportedLanguageBase): supported_langs = ["python"] use_lang = "sql" expect_pass = False class TestSupportedLanguages_SuppotsPython_UsingPython(SupportedLanguageBase): supported_langs = ["python"] use_lang = "python" expect_pass = True class TestSupportedLanguages_SuppotsSqlAndPython_UsingSql(SupportedLanguageBase): supported_langs = ["sql", "python"] use_lang = "sql" expect_pass = True class TestSupportedLanguages_SuppotsSqlAndPython_UsingPython(SupportedLanguageBase): supported_langs = ["sql", "python"] use_lang = "python" expect_pass = True ================================================ FILE: tests/functional/metrics/fixtures.py ================================================ # not strictly necessary, but this reflects the integration tests currently in the 'dbt-metrics' package right now # i'm including just the first 10 rows for more concise 'git diff' mock_purchase_data_csv = """purchased_at,payment_type,payment_total 2021-02-14 17:52:36,maestro,2418.94 2021-02-15 04:16:50,jcb,3043.28 2021-02-15 11:30:45,solo,1505.81 2021-02-16 13:08:18,,1532.85 2021-02-17 05:41:34,americanexpress,319.91 2021-02-18 06:47:32,jcb,2143.44 2021-02-19 01:37:09,jcb,840.1 2021-02-19 03:38:49,jcb,1388.18 2021-02-19 04:22:41,jcb,2834.96 2021-02-19 13:28:50,china-unionpay,2440.98 """.strip() models_people_sql = """ select 1 as id, 'Drew' as first_name, 'Banin' as last_name, 'yellow' as favorite_color, true as loves_dbt, 5 as tenure, current_timestamp as created_at union all select 2 as id, 'Jeremy' as first_name, 'Cohen' as last_name, 'indigo' as favorite_color, true as loves_dbt, 4 as tenure, current_timestamp as created_at union all select 3 as id, 'Callum' as first_name, 'McCann' as last_name, 'emerald' as favorite_color, true as loves_dbt, 0 as tenure, current_timestamp as created_at """ semantic_model_people_yml = """ version: 2 semantic_models: - name: semantic_people model: ref('people') dimensions: - name: favorite_color type: categorical - name: created_at type: TIME type_params: time_granularity: day measures: - name: years_tenure agg: SUM expr: tenure - name: people agg: count expr: id entities: - name: id type: primary defaults: agg_time_dimension: created_at """ basic_metrics_yml = """ version: 2 metrics: - name: number_of_people label: "Number of people" description: Total count of people type: simple type_params: measure: people meta: my_meta: 'testing' - name: collective_tenure label: "Collective tenure" description: Total number of years of team experience type: simple type_params: measure: name: "years_tenure" filter: "{{ Dimension('id__loves_dbt') }} is true" - name: average_tenure label: "Average tenure" description: "The average tenure per person" type: ratio type_params: numerator: collective_tenure denominator: number_of_people - name: average_tenure_plus_one label: "Average tenure, plus 1" description: "The average tenure per person" type: derived type_params: metrics: - average_tenure expr: "average_tenure + 1" - name: tenured_people label: Tenured People description: People who have been here more than 1 year type: simple type_params: measure: people filter: "{{ Metric('collective_tenure', ['id']) }} > 2" """ metricflow_time_spine_sql = """ SELECT to_date('02/20/2023, 'mm/dd/yyyy') as date_day """ models_people_metrics_yml = """ version: 2 metrics: - name: number_of_people label: "Number of people" description: Total count of people type: simple type_params: measure: people time_granularity: month config: meta: my_meta_config: 'config' - name: collective_tenure label: "Collective tenure" description: Total number of years of team experience type: simple type_params: measure: name: years_tenure filter: "{{ Dimension('id__loves_dbt') }} is true" join_to_timespine: true fill_nulls_with: 0 - name: collective_window label: "Collective window" description: Testing window type: simple type_params: measure: name: years_tenure filter: "{{ Dimension('id__loves_dbt') }} is true" window: 14 days - name: average_tenure label: Average Tenure description: The average tenure of our people type: ratio type_params: numerator: collective_tenure denominator: number_of_people - name: average_tenure_minus_people label: Average Tenure minus People description: Well this isn't really useful is it? type: derived type_params: expr: average_tenure - number_of_people metrics: - average_tenure - number_of_people """ models_people_metrics_meta_top_yml = """ version: 2 metrics: - name: number_of_people label: "Number of people" description: Total count of people type: simple type_params: measure: people meta: my_meta_top: 'top' - name: collective_tenure label: "Collective tenure" description: Total number of years of team experience type: simple type_params: measure: name: years_tenure filter: "{{ Dimension('id__loves_dbt') }} is true" join_to_timespine: true fill_nulls_with: 0 - name: collective_window label: "Collective window" description: Testing window type: simple type_params: measure: name: years_tenure filter: "{{ Dimension('id__loves_dbt') }} is true" window: 14 days - name: average_tenure label: Average Tenure description: The average tenure of our people type: ratio type_params: numerator: collective_tenure denominator: number_of_people - name: average_tenure_minus_people label: Average Tenure minus People description: Well this isn't really useful is it? type: derived type_params: expr: average_tenure - number_of_people metrics: - average_tenure - number_of_people """ invalid_models_people_metrics_yml = """ version: 2 metrics: - name: number_of_people label: "Number of people" description: Total count of people model: "ref(people)" calculation_method: count expression: "*" timestamp: created_at time_grains: [day, week, month] dimensions: - favorite_color - loves_dbt meta: my_meta: 'testing' - name: collective_tenure label: "Collective tenure" description: Total number of years of team experience model: "ref(people)" calculation_method: sum expression: tenure timestamp: created_at time_grains: [day] filters: - field: loves_dbt operator: 'is' value: 'true' """ invalid_metrics_missing_model_yml = """ version: 2 metrics: - name: number_of_people label: "Number of people" description: Total count of people calculation_method: count expression: "*" timestamp: created_at time_grains: [day, week, month] dimensions: - favorite_color - loves_dbt meta: my_meta: 'testing' - name: collective_tenure label: "Collective tenure" description: Total number of years of team experience calculation_method: sum expression: tenure timestamp: created_at time_grains: [day] filters: - field: loves_dbt operator: 'is' value: 'true' """ invalid_metrics_missing_expression_yml = """ version: 2 metrics: - name: number_of_people label: "Number of people" model: "ref(people)" description: Total count of people calculation_method: count timestamp: created_at time_grains: [day, week, month] dimensions: - favorite_color - loves_dbt meta: my_meta: 'testing' """ names_with_spaces_metrics_yml = """ version: 2 metrics: - name: number of people label: "Number of people" description: Total count of people type: simple type_params: measure: people meta: my_meta: 'testing' """ names_with_special_chars_metrics_yml = """ version: 2 metrics: - name: number_of_people! label: "Number of people" description: Total count of people type: simple type_params: measure: people meta: my_meta: 'testing' """ names_with_leading_numeric_metrics_yml = """ version: 2 metrics: - name: 1_number_of_people label: "Number of people" description: Total count of people type: simple type_params: measure: people meta: my_meta: 'testing' """ long_name_metrics_yml = """ version: 2 metrics: - name: this_name_is_going_to_contain_more_than_250_characters_but_be_otherwise_acceptable_and_then_will_throw_an_error_which_I_expect_to_happen_and_repeat_this_name_is_going_to_contain_more_than_250_characters_but_be_otherwise_acceptable_and_then_will_throw_an_error_which_I_expect_to_happen label: "Number of people" description: Total count of people type: simple type_params: measure: people meta: my_meta: 'testing' """ downstream_model_sql = """ -- this model will depend on these three metrics {% set some_metrics = [ metric('count_orders'), metric('sum_order_revenue'), metric('average_order_value') ] %} /* {% if not execute %} -- the only properties available to us at 'parse' time are: -- 'metric_name' -- 'package_name' (None if same package) {% set metric_names = [] %} {% for m in some_metrics %} {% do metric_names.append(m.metric_name) %} {% endfor %} -- this config does nothing, but it lets us check these values below {{ config(metric_names = metric_names) }} {% else %} -- these are the properties available to us at 'execution' time {% for m in some_metrics %} name: {{ m.name }} label: {{ m.label }} type: {{ m.type }} type_params: {{ m.type_params }} filter: {{ m.filter }} {% endfor %} {% endif %} select 1 as id """ invalid_derived_metric_contains_model_yml = """ version: 2 metrics: - name: count_orders label: Count orders model: ref('mock_purchase_data') calculation_method: count expression: "*" timestamp: purchased_at time_grains: [day, week, month, quarter, year] dimensions: - payment_type - name: sum_order_revenue label: Total order revenue model: ref('mock_purchase_data') calculation_method: sum expression: "payment_total" timestamp: purchased_at time_grains: [day, week, month, quarter, year] dimensions: - payment_type - name: average_order_value label: Average Order Value calculation_method: derived expression: "{{metric('sum_order_revenue')}} / {{metric('count_orders')}} " model: ref('mock_purchase_data') timestamp: purchased_at time_grains: [day, week, month, quarter, year] dimensions: - payment_type """ purchasing_model_sql = """ select purchased_at, payment_type, payment_total from {{ ref('mock_purchase_data') }} """ semantic_model_purchasing_yml = """ version: 2 semantic_models: - name: semantic_purchasing model: ref('purchasing') measures: - name: num_orders agg: COUNT expr: purchased_at - name: order_revenue agg: SUM expr: payment_total dimensions: - name: purchased_at type: TIME type_params: time_granularity: day entities: - name: purchase type: primary expr: '1' defaults: agg_time_dimension: purchased_at """ derived_metric_yml = """ version: 2 metrics: - name: count_orders label: Count orders type: simple type_params: measure: num_orders - name: sum_order_revenue label: Total order revenue type: simple type_params: measure: order_revenue - name: average_order_value label: Average Order Value type: ratio type_params: numerator: name: sum_order_revenue denominator: name: count_orders - name: sum_order_revenue_plus_one_custom_offset_window label: "Total order revenue, plus 1 with custom offset window" description: "The total order revenue plus 1 offset by 1 martian day" type: derived type_params: metrics: - name: sum_order_revenue offset_window: 1 martian_day expr: "sum_order_revenue + 1" """ disabled_metric_level_schema_yml = """ version: 2 metrics: - name: number_of_people label: "Number of people" description: Total count of people type: simple type_params: measure: people config: enabled: False meta: my_meta: 'testing' - name: collective_tenure label: "Collective tenure" description: Total number of years of team experience type: simple type_params: measure: name: years_tenure filter: "{{ Dimension('id__loves_dbt') }} is true" """ meta_metric_level_schema_yml = """ version: 2 metrics: - name: number_of_people label: "Number of people" description: Total count of people type: simple type_params: measure: people config: meta: my_meta_config: 'config meta: my_meta_direct: 'direct' - name: collective_tenure label: "Collective tenure" description: Total number of years of team experience type: simple type_params: measure: name: years_tenure filter: "{{ Dimension('id__loves_dbt') }} is true" """ enabled_metric_level_schema_yml = """ version: 2 metrics: - name: number_of_people label: "Number of people" description: Total count of people type: simple type_params: measure: people config: enabled: True meta: my_meta: 'testing' - name: collective_tenure label: "Collective tenure" description: Total number of years of team experience type: simple type_params: measure: name: years_tenure filter: "{{ Dimension('id__loves_dbt') }} is true" """ models_people_metrics_sql = """ -- this model will depend on these two metrics {% set some_metrics = [ metric('number_of_people'), metric('collective_tenure') ] %} /* {% if not execute %} -- the only properties available to us at 'parse' time are: -- 'metric_name' -- 'package_name' (None if same package) {% set metric_names = [] %} {% for m in some_metrics %} {% do metric_names.append(m.metric_name) %} {% endfor %} -- this config does nothing, but it lets us check these values below {{ config(metric_names = metric_names) }} {% else %} -- these are the properties available to us at 'execution' time {% for m in some_metrics %} name: {{ m.name }} label: {{ m.label }} type: {{ m.type }} type_params: {{ m.type_params }} filter: {{ m.filter }} window: {{ m.window }} {% endfor %} {% endif %} select 1 as id """ metrics_1_yml = """ version: 2 metrics: - name: some_metric label: Some Metric type: simple type_params: measure: some_measure """ metrics_2_yml = """ version: 2 metrics: - name: some_metric label: Some Metric type: simple type_params: measure: some_measure """ model_a_sql = """ select 1 as fun """ model_b_sql = """ -- {{ metric('some_metric') }} {% if execute %} {% set model_ref_node = graph.nodes.values() | selectattr('name', 'equalto', 'model_a') | first %} {% set relation = api.Relation.create( database = model_ref_node.database, schema = model_ref_node.schema, identifier = model_ref_node.alias ) %} {% else %} {% set relation = "" %} {% endif %} -- this one is a real ref select * from {{ ref('model_a') }} union all -- this one is synthesized via 'graph' var select * from {{ relation }} """ invalid_config_metric_yml = """ version: 2 metrics: - name: number_of_people label: "Number of people" config: enabled: True and False description: Total count of people type: simple type_params: measure: people meta: my_meta: 'testing' """ invalid_metric_without_timestamp_with_time_grains_yml = """ version: 2 metrics: - name: number_of_people label: "Number of people" description: Total count of people model: "ref('people')" time_grains: [day, week, month] calculation_method: count expression: "*" dimensions: - favorite_color - loves_dbt meta: my_meta: 'testing' """ invalid_metric_without_timestamp_with_window_yml = """ version: 2 metrics: - name: number_of_people label: "Number of people" description: Total count of people model: "ref('people')" window: count: 14 period: day calculation_method: count expression: "*" dimensions: - favorite_color - loves_dbt meta: my_meta: 'testing' """ conversion_semantic_model_purchasing_yml = """ version: 2 semantic_models: - name: semantic_purchasing model: ref('purchasing') measures: - name: num_orders agg: COUNT expr: purchased_at - name: num_visits agg: SUM expr: 1 dimensions: - name: purchased_at type: TIME type_params: time_granularity: day entities: - name: purchase type: primary expr: '1' defaults: agg_time_dimension: purchased_at """ cumulative_metric_yml = """ version: 2 metrics: - name: weekly_visits label: Rolling sum of visits over the last 7 days type: cumulative type_params: measure: num_visits cumulative_type_params: window: 7 days period_agg: average - name: cumulative_orders label: Rolling total of orders (all time) type: cumulative type_params: measure: num_orders cumulative_type_params: period_agg: last - name: orders_ytd label: Total orders since the start of the year type: cumulative type_params: measure: num_orders cumulative_type_params: grain_to_date: year period_agg: first - name: monthly_orders label: Orders in the past month type: cumulative type_params: measure: num_orders window: 1 month cumulative_type_params: period_agg: average - name: yearly_orders label: Orders in the past year type: cumulative type_params: measure: num_orders window: 1 year - name: visits_mtd label: Visits since start of month type: cumulative type_params: measure: num_visits grain_to_date: month - name: cumulative_visits label: Rolling total of visits (all time) type: cumulative type_params: measure: num_visits # TODO: Re-enable this when custom grain is supported for this type # - name: visits_martian_day # label: Visits since start of martian_day # type: cumulative # type_params: # measure: num_visits # cumulative_type_params: # grain_to_date: martian_day # - name: visits_martian_day_window # label: Visits since start of martian_day window # type: cumulative # type_params: # measure: num_visits # cumulative_type_params: # window: 1 martian_day """ conversion_metric_yml = """ version: 2 metrics: - name: converted_orders_over_visits label: Number of orders converted from visits type: conversion type_params: conversion_type_params: base_measure: num_visits conversion_measure: num_orders entity: purchase - name: converted_orders_over_visits_with_window label: Number of orders converted from visits with window type: conversion type_params: conversion_type_params: base_measure: num_visits conversion_measure: num_orders entity: purchase window: 4 day # TODO: Re-enable this when custom grain is supported for this type # - name: converted_orders_over_visits_with_custom_window # label: Number of orders converted from visits with custom window # type: conversion # type_params: # conversion_type_params: # base_measure: num_visits # conversion_measure: num_orders # entity: purchase # window: 4 martian_day """ filtered_metrics_yml = """ version: 2 metrics: - name: collective_tenure_measure_filter_str label: "Collective tenure1" description: Total number of years of team experience type: simple type_params: measure: name: "years_tenure" filter: "{{ Dimension('id__loves_dbt') }} is true" - name: collective_tenure_measure_filter_list label: "Collective tenure2" description: Total number of years of team experience type: simple type_params: measure: name: "years_tenure" filter: - "{{ Dimension('id__loves_dbt') }} is true" - name: collective_tenure_metric_filter_str label: Collective tenure3 description: Total number of years of team experience type: simple type_params: measure: name: "years_tenure" filter: "{{ Dimension('id__loves_dbt') }} is true" - name: collective_tenure_metric_filter_list label: Collective tenure4 description: Total number of years of team experience type: simple type_params: measure: name: "years_tenure" filter: - "{{ Dimension('id__loves_dbt') }} is true" - name: average_tenure_filter_str label: Average tenure of people who love dbt1 description: Average tenure of people who love dbt type: derived type_params: expr: "average_tenure" metrics: - name: average_tenure filter: "{{ Dimension('id__loves_dbt') }} is true" - name: average_tenure_filter_list label: Average tenure of people who love dbt2 description: Average tenure of people who love dbt type: derived type_params: expr: "average_tenure" metrics: - name: average_tenure filter: - "{{ Dimension('id__loves_dbt') }} is true" """ duplicate_measure_metric_yml = """ metrics: # Simple metrics - name: people_with_tenure description: "Count of people with tenure" type: simple label: People with tenure type_params: measure: people - name: ratio_tenure_to_people description: People to years of tenure label: New customers to all customers type: ratio type_params: numerator: people_with_tenure denominator: number_of_people """ ================================================ FILE: tests/functional/metrics/test_metric_configs.py ================================================ import pytest from dbt.artifacts.resources import MetricConfig from dbt.exceptions import CompilationError, ParsingError from dbt.tests.util import get_manifest, run_dbt, update_config_file from dbt_common.dataclass_schema import ValidationError from tests.functional.metrics.fixtures import ( disabled_metric_level_schema_yml, enabled_metric_level_schema_yml, invalid_config_metric_yml, metricflow_time_spine_sql, models_people_metrics_meta_top_yml, models_people_metrics_sql, models_people_metrics_yml, models_people_sql, semantic_model_people_yml, ) class MetricConfigTests: @pytest.fixture(scope="class", autouse=True) def setUp(self): pytest.expected_config = MetricConfig( enabled=True, ) # Test enabled config in dbt_project.yml class TestMetricEnabledConfigProjectLevel(MetricConfigTests): @pytest.fixture(scope="class") def models(self): return { "people.sql": models_people_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "semantic_model_people.yml": semantic_model_people_yml, "schema.yml": models_people_metrics_yml, } @pytest.fixture(scope="class") def project_config_update(self): return { "metrics": { "test": { "average_tenure_minus_people": { "enabled": False, }, } } } def test_enabled_metric_config_dbt_project(self, project): run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "metric.test.average_tenure_minus_people" not in manifest.metrics new_enabled_config = { "metrics": { "test": { "average_tenure_minus_people": { "enabled": True, }, } } } update_config_file(new_enabled_config, project.project_root, "dbt_project.yml") run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "metric.test.average_tenure_minus_people" in manifest.metrics assert "metric.test.collective_tenure" in manifest.metrics # Test enabled config at metrics level in yml file class TestConfigYamlMetricLevel(MetricConfigTests): @pytest.fixture(scope="class") def models(self): return { "people.sql": models_people_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "semantic_model_people.yml": semantic_model_people_yml, "schema.yml": disabled_metric_level_schema_yml, } def test_metric_config_yaml_metric_level(self, project): run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "metric.test.number_of_people" not in manifest.metrics assert "metric.test.collective_tenure" in manifest.metrics # Test inheritence - set configs at project and metric level - expect metric level to win class TestMetricConfigsInheritence(MetricConfigTests): @pytest.fixture(scope="class") def models(self): return { "people.sql": models_people_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "semantic_model_people.yml": semantic_model_people_yml, "schema.yml": enabled_metric_level_schema_yml, } @pytest.fixture(scope="class") def project_config_update(self): return {"metrics": {"enabled": False}} def test_metrics_all_configs(self, project): run_dbt(["parse"]) manifest = get_manifest(project.project_root) # This should be overridden assert "metric.test.number_of_people" in manifest.metrics # This should stay disabled assert "metric.test.collective_tenure" not in manifest.metrics config_test_table = manifest.metrics.get("metric.test.number_of_people").config assert isinstance(config_test_table, MetricConfig) assert config_test_table == pytest.expected_config # Test CompilationError if a model references a disabled metric class TestDisabledMetricRef(MetricConfigTests): @pytest.fixture(scope="class") def models(self): return { "people.sql": models_people_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "semantic_model_people.yml": semantic_model_people_yml, "people_metrics.sql": models_people_metrics_sql, "schema.yml": models_people_metrics_yml, } def test_disabled_metric_ref_model(self, project): run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "metric.test.number_of_people" in manifest.metrics assert "metric.test.collective_tenure" in manifest.metrics assert "model.test.people_metrics" in manifest.nodes assert "metric.test.average_tenure" in manifest.metrics assert "metric.test.average_tenure_minus_people" in manifest.metrics new_enabled_config = { "metrics": { "test": { "number_of_people": { "enabled": False, }, "average_tenure_minus_people": { "enabled": False, }, "average_tenure": { "enabled": False, }, } } } update_config_file(new_enabled_config, project.project_root, "dbt_project.yml") with pytest.raises(CompilationError): run_dbt(["parse"]) # Test invalid metric configs class TestInvalidMetric(MetricConfigTests): @pytest.fixture(scope="class") def models(self): return { "people.sql": models_people_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "semantic_model_people.yml": semantic_model_people_yml, "schema.yml": invalid_config_metric_yml, } def test_invalid_config_metric(self, project): with pytest.raises(ValidationError) as excinfo: run_dbt(["parse"]) expected_msg = "'True and False' is not of type 'boolean'" assert expected_msg in str(excinfo.value) class TestDisabledMetric(MetricConfigTests): @pytest.fixture(scope="class") def models(self): return { "people.sql": models_people_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "semantic_model_people.yml": semantic_model_people_yml, "schema.yml": models_people_metrics_yml, } def test_disabling_upstream_metric_errors(self, project): run_dbt(["parse"]) # shouldn't error out yet new_enabled_config = { "metrics": { "test": { "number_of_people": { "enabled": False, }, } } } update_config_file(new_enabled_config, project.project_root, "dbt_project.yml") with pytest.raises(ParsingError) as excinfo: run_dbt(["parse"]) expected_msg = ( "The metric `number_of_people` is disabled and thus cannot be referenced." ) assert expected_msg in str(excinfo.value) # Test meta config in dbt_project.yml class TestMetricMetaConfigProjectLevel(MetricConfigTests): @pytest.fixture(scope="class") def models(self): return { "people.sql": models_people_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "semantic_model_people.yml": semantic_model_people_yml, "schema.yml": models_people_metrics_yml, } @pytest.fixture(scope="class") def project_config_update(self): return { "metrics": { "test": { "average_tenure_minus_people": { "+meta": {"project_field": "project_value"}, }, } } } def test_meta_metric_config_dbt_project(self, project): run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "metric.test.average_tenure_minus_people" in manifest.metrics # for backwards compatibility the config level meta gets copied to the top level meta assert manifest.metrics.get("metric.test.average_tenure_minus_people").config.meta == { "project_field": "project_value" } assert manifest.metrics.get("metric.test.average_tenure_minus_people").meta == { "project_field": "project_value" } # Test setting config at config level class TestMetricMetaConfigLevel(MetricConfigTests): @pytest.fixture(scope="class") def models(self): return { "people.sql": models_people_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "semantic_model_people.yml": semantic_model_people_yml, "schema.yml": models_people_metrics_yml, } def test_meta_metric_config_yaml(self, project): run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "metric.test.number_of_people" in manifest.metrics assert manifest.metrics.get("metric.test.number_of_people").config.meta == { "my_meta_config": "config" } assert manifest.metrics.get("metric.test.number_of_people").meta == { "my_meta_config": "config" } # Test setting config at metric level- expect to exist in config after parsing class TestMetricMetaTopLevel(MetricConfigTests): @pytest.fixture(scope="class") def models(self): return { "people.sql": models_people_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "semantic_model_people.yml": semantic_model_people_yml, "schema.yml": models_people_metrics_meta_top_yml, } def test_meta_metric_config_yaml(self, project): run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "metric.test.number_of_people" in manifest.metrics # for backwards compatibility the config level meta gets copied to the top level meta assert manifest.metrics.get("metric.test.number_of_people").config.meta != { "my_meta_top": "top" } assert manifest.metrics.get("metric.test.number_of_people").meta == {"my_meta_top": "top"} ================================================ FILE: tests/functional/metrics/test_metric_deferral.py ================================================ import os from pathlib import Path import pytest from dbt.tests.util import copy_file, run_dbt, write_file from tests.functional.metrics.fixtures import ( metrics_1_yml, metrics_2_yml, model_a_sql, model_b_sql, ) class TestMetricDeferral: @pytest.fixture(scope="class", autouse=True) def setup(self, project): # Create "prod" schema prod_schema_name = project.test_schema + "_prod" project.create_test_schema(schema_name=prod_schema_name) # Create "state" directory path = Path(project.project_root) / "state" Path.mkdir(path) @pytest.fixture(scope="class") def dbt_profile_data(self, unique_schema): return { "test": { "outputs": { "default": { "type": "postgres", "threads": 4, "host": "localhost", "port": int(os.getenv("POSTGRES_TEST_PORT", 5432)), "user": os.getenv("POSTGRES_TEST_USER", "root"), "pass": os.getenv("POSTGRES_TEST_PASS", "password"), "dbname": os.getenv("POSTGRES_TEST_DATABASE", "dbt"), "schema": unique_schema, }, "prod": { "type": "postgres", "threads": 4, "host": "localhost", "port": int(os.getenv("POSTGRES_TEST_PORT", 5432)), "user": os.getenv("POSTGRES_TEST_USER", "root"), "pass": os.getenv("POSTGRES_TEST_PASS", "password"), "dbname": os.getenv("POSTGRES_TEST_DATABASE", "dbt"), "schema": unique_schema + "_prod", }, }, "target": "default", }, } @pytest.fixture(scope="class") def models(self): return { "model_a.sql": model_a_sql, "model_b.sql": model_b_sql, "metrics.yml": metrics_1_yml, } @pytest.mark.skip("TODO") def test_metric_deferral(self, project): results = run_dbt(["run", "--target", "prod"]) assert len(results) == 2 # copy manifest.json to "state" directory target_path = os.path.join(project.project_root, "target") copy_file(target_path, "manifest.json", project.project_root, ["state", "manifest.json"]) # Change metrics file write_file(metrics_2_yml, project.project_root, "models", "metrics.yml") # Confirm that some_metric + model_b are both selected, and model_a is not selected results = run_dbt(["ls", "-s", "state:modified+", "--state", "state/", "--target", "prod"]) assert results == ["metric:test.some_metric", "test.model_b"] # Run in default schema results = run_dbt( ["run", "-s", "state:modified+", "--state", "state/", "--defer", "--target", "default"] ) assert len(results) == 1 ================================================ FILE: tests/functional/metrics/test_metric_helper_functions.py ================================================ import pytest from dbt.contracts.graph.manifest import Manifest from dbt.contracts.graph.metrics import ResolvedMetricReference from dbt.tests.util import run_dbt from tests.functional.metrics.fixtures import ( basic_metrics_yml, metricflow_time_spine_sql, models_people_sql, semantic_model_people_yml, ) class TestMetricHelperFunctions: @pytest.fixture(scope="class") def models(self): return { "metrics.yml": basic_metrics_yml, "semantic_people.yml": semantic_model_people_yml, "metricflow_time_spine.sql": metricflow_time_spine_sql, "people.sql": models_people_sql, } def test_derived_metric( self, project, ): # initial parse manifest = run_dbt(["parse"]) assert isinstance(manifest, Manifest) parsed_metric = manifest.metrics["metric.test.average_tenure_plus_one"] testing_metric = ResolvedMetricReference(parsed_metric, manifest) full_metric_dependency = set(testing_metric.full_metric_dependency()) expected_full_metric_dependency = set( ["average_tenure_plus_one", "average_tenure", "collective_tenure", "number_of_people"] ) assert full_metric_dependency == expected_full_metric_dependency base_metric_dependency = set(testing_metric.base_metric_dependency()) expected_base_metric_dependency = set(["collective_tenure", "number_of_people"]) assert base_metric_dependency == expected_base_metric_dependency derived_metric_dependency = set(testing_metric.derived_metric_dependency()) expected_derived_metric_dependency = set(["average_tenure_plus_one", "average_tenure"]) assert derived_metric_dependency == expected_derived_metric_dependency derived_metric_dependency_depth = list(testing_metric.derived_metric_dependency_depth()) expected_derived_metric_dependency_depth = list( [{"average_tenure_plus_one": 1}, {"average_tenure": 2}] ) assert derived_metric_dependency_depth == expected_derived_metric_dependency_depth ================================================ FILE: tests/functional/metrics/test_metrics.py ================================================ import pytest from dbt.artifacts.resources.v1.metric import CumulativeTypeParams, MetricTimeWindow from dbt.cli.main import dbtRunner from dbt.contracts.graph.manifest import Manifest from dbt.exceptions import ParsingError from dbt.tests.util import get_manifest, run_dbt from dbt_semantic_interfaces.type_enums.period_agg import PeriodAggregation from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity from tests.functional.metrics.fixtures import ( basic_metrics_yml, conversion_metric_yml, conversion_semantic_model_purchasing_yml, cumulative_metric_yml, derived_metric_yml, downstream_model_sql, duplicate_measure_metric_yml, filtered_metrics_yml, invalid_derived_metric_contains_model_yml, invalid_metric_without_timestamp_with_time_grains_yml, invalid_metric_without_timestamp_with_window_yml, invalid_metrics_missing_expression_yml, invalid_metrics_missing_model_yml, invalid_models_people_metrics_yml, long_name_metrics_yml, metricflow_time_spine_sql, mock_purchase_data_csv, models_people_metrics_yml, models_people_sql, names_with_leading_numeric_metrics_yml, names_with_spaces_metrics_yml, names_with_special_chars_metrics_yml, purchasing_model_sql, semantic_model_people_yml, semantic_model_purchasing_yml, ) from tests.functional.time_spines.fixtures import time_spine_yml class TestSimpleMetrics: @pytest.fixture(scope="class") def models(self): return { "people_metrics.yml": models_people_metrics_yml, "metricflow_time_spine.sql": metricflow_time_spine_sql, "semantic_model_people.yml": semantic_model_people_yml, "people.sql": models_people_sql, } def test_simple_metric( self, project, ): runner = dbtRunner() result = runner.invoke(["parse"]) assert result.success assert isinstance(result.result, Manifest) manifest = get_manifest(project.project_root) metric_ids = list(manifest.metrics.keys()) expected_metric_ids = [ "metric.test.number_of_people", "metric.test.collective_tenure", "metric.test.collective_window", "metric.test.average_tenure", "metric.test.average_tenure_minus_people", ] assert metric_ids == expected_metric_ids assert ( len(manifest.metrics["metric.test.number_of_people"].type_params.input_measures) == 1 ) assert ( len(manifest.metrics["metric.test.collective_tenure"].type_params.input_measures) == 1 ) assert ( len(manifest.metrics["metric.test.collective_window"].type_params.input_measures) == 1 ) assert len(manifest.metrics["metric.test.average_tenure"].type_params.input_measures) == 2 assert ( len( manifest.metrics[ "metric.test.average_tenure_minus_people" ].type_params.input_measures ) == 2 ) assert ( manifest.metrics["metric.test.number_of_people"].time_granularity == TimeGranularity.MONTH.value ) assert manifest.metrics["metric.test.collective_tenure"].time_granularity is None class TestInvalidRefMetrics: @pytest.fixture(scope="class") def models(self): return { "people_metrics.yml": invalid_models_people_metrics_yml, "people.sql": models_people_sql, } # tests that we get a ParsingError with an invalid model ref, where # the model name does not have quotes def test_simple_metric( self, project, ): # initial run with pytest.raises(ParsingError): run_dbt(["run"]) class TestInvalidMetricMissingModel: @pytest.fixture(scope="class") def models(self): return { "people_metrics.yml": invalid_metrics_missing_model_yml, "people.sql": models_people_sql, } # tests that we get a ParsingError with an invalid model ref, where # the model name does not have quotes def test_simple_metric( self, project, ): # initial run with pytest.raises(ParsingError): run_dbt(["run"]) class TestInvalidMetricMissingExpression: @pytest.fixture(scope="class") def models(self): return { "people_metrics.yml": invalid_metrics_missing_expression_yml, "people.sql": models_people_sql, } # tests that we get a ParsingError with a missing expression def test_simple_metric( self, project, ): # initial run with pytest.raises(ParsingError): run_dbt(["run"]) class TestNamesWithSpaces: @pytest.fixture(scope="class") def models(self): return { "people_metrics.yml": names_with_spaces_metrics_yml, "people.sql": models_people_sql, } def test_names_with_spaces(self, project): with pytest.raises(ParsingError) as exc: run_dbt(["run"]) assert "cannot contain spaces" in str(exc.value) class TestNamesWithSpecialChar: @pytest.fixture(scope="class") def models(self): return { "people_metrics.yml": names_with_special_chars_metrics_yml, "people.sql": models_people_sql, } def test_names_with_special_char(self, project): with pytest.raises(ParsingError) as exc: run_dbt(["run"]) assert "must contain only letters, numbers and underscores" in str(exc.value) class TestNamesWithLeandingNumber: @pytest.fixture(scope="class") def models(self): return { "people_metrics.yml": names_with_leading_numeric_metrics_yml, "people.sql": models_people_sql, } def test_names_with_leading_number(self, project): with pytest.raises(ParsingError) as exc: run_dbt(["run"]) assert "must begin with a letter" in str(exc.value) class TestLongName: @pytest.fixture(scope="class") def models(self): return { "people_metrics.yml": long_name_metrics_yml, "people.sql": models_people_sql, } def test_long_name(self, project): with pytest.raises(ParsingError) as exc: run_dbt(["run"]) assert "cannot contain more than 250 characters" in str(exc.value) class TestInvalidDerivedMetrics: @pytest.fixture(scope="class") def models(self): return { "derived_metric.yml": invalid_derived_metric_contains_model_yml, "downstream_model.sql": downstream_model_sql, } def test_invalid_derived_metrics(self, project): with pytest.raises(ParsingError): run_dbt(["run"]) class TestMetricDependsOn: @pytest.fixture(scope="class") def models(self): return { "people.sql": models_people_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "semantic_models.yml": semantic_model_people_yml, "people_metrics.yml": models_people_metrics_yml, } def test_metric_depends_on(self, project): manifest = run_dbt(["parse"]) assert isinstance(manifest, Manifest) expected_depends_on_for_number_of_people = ["semantic_model.test.semantic_people"] expected_depends_on_for_average_tenure = [ "metric.test.collective_tenure", "metric.test.number_of_people", ] number_of_people_metric = manifest.metrics["metric.test.number_of_people"] assert number_of_people_metric.depends_on.nodes == expected_depends_on_for_number_of_people average_tenure_metric = manifest.metrics["metric.test.average_tenure"] assert average_tenure_metric.depends_on.nodes == expected_depends_on_for_average_tenure class TestDerivedMetric: @pytest.fixture(scope="class") def models(self): return { "downstream_model.sql": downstream_model_sql, "purchasing.sql": purchasing_model_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "semantic_models.yml": semantic_model_purchasing_yml, "derived_metric.yml": derived_metric_yml, "time_spine.yml": time_spine_yml, } # not strictly necessary to use "real" mock data for this test # we just want to make sure that the 'metric' calls match our expectations # but this sort of thing is possible, to have actual data flow through and validate results @pytest.fixture(scope="class") def seeds(self): return { "mock_purchase_data.csv": mock_purchase_data_csv, } def test_derived_metric( self, project, ): # initial parse results = run_dbt(["parse"]) # make sure all the metrics are in the manifest manifest = get_manifest(project.project_root) metric_ids = list(manifest.metrics.keys()) expected_metric_ids = [ "metric.test.count_orders", "metric.test.sum_order_revenue", "metric.test.average_order_value", "metric.test.sum_order_revenue_plus_one_custom_offset_window", ] assert metric_ids == expected_metric_ids # make sure the downstream_model depends on these metrics metric_names = ["average_order_value", "count_orders", "sum_order_revenue"] downstream_model = manifest.nodes["model.test.downstream_model"] assert sorted(downstream_model.metrics) == [[metric_name] for metric_name in metric_names] assert sorted(downstream_model.depends_on.nodes) == [ "metric.test.average_order_value", "metric.test.count_orders", "metric.test.sum_order_revenue", ] assert sorted(downstream_model.config["metric_names"]) == metric_names # make sure the 'expression' metric depends on the two upstream metrics derived_metric = manifest.metrics["metric.test.average_order_value"] assert sorted(derived_metric.depends_on.nodes) == [ "metric.test.count_orders", "metric.test.sum_order_revenue", ] derived_metric_with_custom_offset_window = manifest.metrics[ "metric.test.sum_order_revenue_plus_one_custom_offset_window" ] assert len(derived_metric_with_custom_offset_window.input_metrics) == 1 assert derived_metric_with_custom_offset_window.input_metrics[ 0 ].offset_window == MetricTimeWindow(count=1, granularity="martian_day") # actually compile results = run_dbt(["compile", "--select", "downstream_model"]) compiled_code = results[0].node.compiled_code # make sure all these metrics properties show up in compiled SQL for metric_name in manifest.metrics: if metric_name == "metric.test.sum_order_revenue_plus_one_custom_offset_window": # Skip this metric continue parsed_metric_node = manifest.metrics[metric_name] for property in [ "name", "label", "type", "type_params", "filter", ]: expected_value = getattr(parsed_metric_node, property) assert f"{property}: {expected_value}" in compiled_code class TestInvalidTimestampTimeGrainsMetrics: @pytest.fixture(scope="class") def models(self): return { "people_metrics.yml": invalid_metric_without_timestamp_with_time_grains_yml, "people.sql": models_people_sql, } # Tests that we get a ParsingError with an invalid metric definition. # This metric definition is missing timestamp but HAS a time_grains property def test_simple_metric( self, project, ): # initial run with pytest.raises(ParsingError): run_dbt(["run"]) class TestInvalidTimestampWindowMetrics: @pytest.fixture(scope="class") def models(self): return { "people_metrics.yml": invalid_metric_without_timestamp_with_window_yml, "people.sql": models_people_sql, } # Tests that we get a ParsingError with an invalid metric definition. # This metric definition is missing timestamp but HAS a window property def test_simple_metric( self, project, ): # initial run with pytest.raises(ParsingError): run_dbt(["run"]) class TestConversionMetric: @pytest.fixture(scope="class") def models(self): return { "purchasing.sql": purchasing_model_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "semantic_models.yml": conversion_semantic_model_purchasing_yml, "conversion_metric.yml": conversion_metric_yml, "time_spine.yml": time_spine_yml, } @pytest.fixture(scope="class") def seeds(self): return { "mock_purchase_data.csv": mock_purchase_data_csv, } def test_conversion_metric( self, project, ): # initial parse runner = dbtRunner() result = runner.invoke(["parse"]) assert result.success assert isinstance(result.result, Manifest) # make sure the metric is in the manifest manifest = get_manifest(project.project_root) metric_ids = list(manifest.metrics.keys()) expected_metric_ids = { "metric.test.converted_orders_over_visits": None, "metric.test.converted_orders_over_visits_with_window": MetricTimeWindow( count=4, granularity=TimeGranularity.DAY.value ), } assert set(metric_ids) == set(expected_metric_ids.keys()) assert manifest.metrics[ "metric.test.converted_orders_over_visits" ].type_params.conversion_type_params assert ( len( manifest.metrics[ "metric.test.converted_orders_over_visits" ].type_params.input_measures ) == 2 ) assert ( manifest.metrics[ "metric.test.converted_orders_over_visits" ].type_params.conversion_type_params.window is None ) assert ( manifest.metrics[ "metric.test.converted_orders_over_visits" ].type_params.conversion_type_params.entity == "purchase" ) for ( metric_id, expected_window, ) in expected_metric_ids.items(): assert ( manifest.metrics[metric_id].type_params.conversion_type_params.window == expected_window ), f"Found unexpected conversion window for {metric_id}" class TestCumulativeMetric: @pytest.fixture(scope="class") def models(self): return { "purchasing.sql": purchasing_model_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "semantic_models.yml": conversion_semantic_model_purchasing_yml, "cumulative_metric.yml": cumulative_metric_yml, "time_spine.yml": time_spine_yml, } @pytest.fixture(scope="class") def seeds(self): return {"mock_purchase_data.csv": mock_purchase_data_csv} def test_cumulative_metric(self, project): # initial parse runner = dbtRunner() result = runner.invoke(["parse"]) assert result.success assert isinstance(result.result, Manifest) manifest = get_manifest(project.project_root) metric_ids = set(manifest.metrics.keys()) expected_metric_ids_to_cumulative_type_params = { "metric.test.weekly_visits": CumulativeTypeParams( window=MetricTimeWindow(count=7, granularity=TimeGranularity.DAY.value), period_agg=PeriodAggregation.AVERAGE, ), "metric.test.cumulative_orders": CumulativeTypeParams( period_agg=PeriodAggregation.LAST ), "metric.test.orders_ytd": CumulativeTypeParams( grain_to_date=TimeGranularity.YEAR.value, period_agg=PeriodAggregation.FIRST ), "metric.test.monthly_orders": CumulativeTypeParams( window=MetricTimeWindow(count=1, granularity=TimeGranularity.MONTH.value), period_agg=PeriodAggregation.AVERAGE, ), "metric.test.yearly_orders": CumulativeTypeParams( window=MetricTimeWindow(count=1, granularity=TimeGranularity.YEAR.value), period_agg=PeriodAggregation.FIRST, ), "metric.test.visits_mtd": CumulativeTypeParams( grain_to_date=TimeGranularity.MONTH.value, period_agg=PeriodAggregation.FIRST ), "metric.test.cumulative_visits": CumulativeTypeParams( period_agg=PeriodAggregation.FIRST ), } assert metric_ids == set(expected_metric_ids_to_cumulative_type_params.keys()) for ( metric_id, expected_cumulative_type_params, ) in expected_metric_ids_to_cumulative_type_params.items(): assert ( manifest.metrics[metric_id].type_params.cumulative_type_params == expected_cumulative_type_params ), f"Found unexpected cumulative type params for {metric_id}" class TestFilterParsing: @pytest.fixture(scope="class") def models(self): return { "basic_metrics.yml": basic_metrics_yml, "filtered_metrics.yml": filtered_metrics_yml, "metricflow_time_spine.sql": metricflow_time_spine_sql, "semantic_model_people.yml": semantic_model_people_yml, "people.sql": models_people_sql, } # Tests that filters are parsed to their appropriate types def test_filter_parsing( self, project, ): runner = dbtRunner() result = runner.invoke(["parse"]) assert result.success assert isinstance(result.result, Manifest) manifest = get_manifest(project.project_root) assert manifest # Test metrics with input measure filters. filters1 = ( manifest.metrics["metric.test.collective_tenure_measure_filter_str"] .input_measures[0] .filter.where_filters ) assert len(filters1) == 1 assert filters1[0].where_sql_template == "{{ Dimension('id__loves_dbt') }} is true" filters2 = ( manifest.metrics["metric.test.collective_tenure_measure_filter_list"] .input_measures[0] .filter.where_filters ) assert len(filters2) == 1 assert filters2[0].where_sql_template == "{{ Dimension('id__loves_dbt') }} is true" # Test metrics with metric-level filters. filters3 = manifest.metrics[ "metric.test.collective_tenure_metric_filter_str" ].filter.where_filters assert len(filters3) == 1 assert filters3[0].where_sql_template == "{{ Dimension('id__loves_dbt') }} is true" filters4 = manifest.metrics[ "metric.test.collective_tenure_metric_filter_list" ].filter.where_filters assert len(filters4) == 1 assert filters4[0].where_sql_template == "{{ Dimension('id__loves_dbt') }} is true" # Test derived metrics with input metric filters. filters5 = ( manifest.metrics["metric.test.average_tenure_filter_str"] .input_metrics[0] .filter.where_filters ) assert len(filters5) == 1 assert filters5[0].where_sql_template == "{{ Dimension('id__loves_dbt') }} is true" filters6 = ( manifest.metrics["metric.test.average_tenure_filter_list"] .input_metrics[0] .filter.where_filters ) assert len(filters6) == 1 assert filters6[0].where_sql_template == "{{ Dimension('id__loves_dbt') }} is true" class TestDuplicateInputMeasures: @pytest.fixture(scope="class") def models(self): return { "basic_metrics.yml": basic_metrics_yml, "filtered_metrics.yml": duplicate_measure_metric_yml, "metricflow_time_spine.sql": metricflow_time_spine_sql, "semantic_model_people.yml": semantic_model_people_yml, "people.sql": models_people_sql, } def test_duplicate_input_measures(self, project): runner = dbtRunner() result = runner.invoke(["parse"]) assert result.success assert isinstance(result.result, Manifest) ================================================ FILE: tests/functional/microbatch/test_microbatch.py ================================================ import json import os from typing import Dict from unittest import mock import pytest from pytest_mock import MockerFixture from dbt.events.types import ( GenericExceptionOnRun, InvalidConcurrentBatchesConfig, JinjaLogDebug, LogBatchResult, LogModelResult, MicrobatchExecutionDebug, MicrobatchMacroOutsideOfBatchesDeprecation, MicrobatchModelNoEventTimeInputs, ) from dbt.tests.fixtures.project import TestProjInfo from dbt.tests.util import ( get_artifact, patch_microbatch_end_time, read_file, relation_from_name, run_dbt, run_dbt_and_capture, write_file, ) from dbt_common.events.event_catcher import EventCatcher input_model_sql = """ {{ config(materialized='table', event_time='event_time') }} select 1 as id, TIMESTAMP '2020-01-01 00:00:00-0' as event_time union all select 2 as id, TIMESTAMP '2020-01-02 00:00:00-0' as event_time union all select 3 as id, TIMESTAMP '2020-01-03 00:00:00-0' as event_time """ input_model_invalid_sql = """ {{ config(materialized='table', event_time='event_time') }} select invalid as event_time """ input_model_without_event_time_sql = """ {{ config(materialized='table') }} select 1 as id, TIMESTAMP '2020-01-01 00:00:00-0' as event_time union all select 2 as id, TIMESTAMP '2020-01-02 00:00:00-0' as event_time union all select 3 as id, TIMESTAMP '2020-01-03 00:00:00-0' as event_time """ microbatch_model_sql = """ {{ config(materialized='incremental', incremental_strategy='microbatch', unique_key='id', event_time='event_time', batch_size='day', begin=modules.datetime.datetime(2020, 1, 1, 0, 0, 0)) }} select * from {{ ref('input_model') }} """ microbatch_model_hour_sql = """ {{ config(materialized='incremental', incremental_strategy='microbatch', unique_key='id', event_time='event_time', batch_size='hour', begin=modules.datetime.datetime(2020, 1, 1, 0, 0, 0)) }} select * from {{ ref('input_model') }} """ microbatch_model_month_sql = """ {{ config(materialized='incremental', incremental_strategy='microbatch', unique_key='id', event_time='event_time', batch_size='month', begin=modules.datetime.datetime(2020, 1, 1, 0, 0, 0)) }} select * from {{ ref('input_model') }} """ microbatch_model_year_sql = """ {{ config(materialized='incremental', incremental_strategy='microbatch', unique_key='id', event_time='event_time', batch_size='year', begin=modules.datetime.datetime(2020, 1, 1, 0, 0, 0)) }} select * from {{ ref('input_model') }} """ microbatch_model_with_pre_and_post_sql = """ {{ config( materialized='incremental', incremental_strategy='microbatch', unique_key='id', event_time='event_time', batch_size='day', begin=modules.datetime.datetime(2020, 1, 1, 0, 0, 0), pre_hook='{{log("execute: " ~ execute ~ ", pre-hook run by batch " ~ model.batch.id)}}', post_hook='{{log("execute: " ~ execute ~ ", post-hook run by batch " ~ model.batch.id)}}', ) }} select * from {{ ref('input_model') }} """ microbatch_model_force_concurrent_batches_sql = """ {{ config(materialized='incremental', incremental_strategy='microbatch', unique_key='id', event_time='event_time', batch_size='day', begin=modules.datetime.datetime(2020, 1, 1, 0, 0, 0), concurrent_batches=true) }} select * from {{ ref('input_model') }} """ microbatch_yearly_model_sql = """ {{ config(materialized='incremental', incremental_strategy='microbatch', unique_key='id', event_time='event_time', batch_size='year', begin=modules.datetime.datetime(2020, 1, 1, 0, 0, 0)) }} select * from {{ ref('input_model') }} """ microbatch_yearly_model_downstream_sql = """ {{ config(materialized='incremental', incremental_strategy='microbatch', unique_key='id', event_time='event_time', batch_size='year', begin=modules.datetime.datetime(2020, 1, 1, 0, 0, 0)) }} select * from {{ ref('microbatch_model') }} """ invalid_batch_jinja_context_macro_sql = """ {% macro check_invalid_batch_jinja_context() %} {% if model is not mapping %} {{ exceptions.raise_compiler_error("`model` is invalid: expected mapping type") }} {% elif compiled_code and compiled_code is not string %} {{ exceptions.raise_compiler_error("`compiled_code` is invalid: expected string type") }} {% elif sql and sql is not string %} {{ exceptions.raise_compiler_error("`sql` is invalid: expected string type") }} {% elif is_incremental is not callable %} {{ exceptions.raise_compiler_error("`is_incremental()` is invalid: expected callable type") }} {% elif should_full_refresh is not callable %} {{ exceptions.raise_compiler_error("`should_full_refresh()` is invalid: expected callable type") }} {% endif %} {% endmacro %} """ microbatch_model_with_context_checks_sql = """ {{ config(pre_hook="{{ check_invalid_batch_jinja_context() }}", materialized='incremental', incremental_strategy='microbatch', unique_key='id', event_time='event_time', batch_size='day', begin=modules.datetime.datetime(2020, 1, 1, 0, 0, 0)) }} {{ check_invalid_batch_jinja_context() }} select * from {{ ref('input_model') }} """ microbatch_model_downstream_sql = """ {{ config(materialized='incremental', incremental_strategy='microbatch', unique_key='id', event_time='event_time', batch_size='day', begin=modules.datetime.datetime(2020, 1, 1, 0, 0, 0)) }} select * from {{ ref('microbatch_model') }} """ microbatch_model_ref_render_sql = """ {{ config(materialized='incremental', incremental_strategy='microbatch', unique_key='id', event_time='event_time', batch_size='day', begin=modules.datetime.datetime(2020, 1, 1, 0, 0, 0)) }} select * from {{ ref('input_model').render() }} """ seed_csv = """id,event_time 1,'2020-01-01 00:00:00-0' 2,'2020-01-02 00:00:00-0' 3,'2020-01-03 00:00:00-0' """ seeds_yaml = """ seeds: - name: raw_source config: column_types: event_time: TIMESTAMP """ sources_yaml = """ sources: - name: seed_sources schema: "{{ target.schema }}" tables: - name: raw_source config: event_time: event_time """ microbatch_model_calling_source_sql = """ {{ config(materialized='incremental', incremental_strategy='microbatch', unique_key='id', event_time='event_time', batch_size='day', begin=modules.datetime.datetime(2020, 1, 1, 0, 0, 0)) }} select * from {{ source('seed_sources', 'raw_source') }} """ custom_microbatch_strategy = """ {% macro get_incremental_microbatch_sql(arg_dict) %} {% do log('custom microbatch strategy', info=True) %} {%- set dest_cols_csv = get_quoted_csv(arg_dict["dest_columns"] | map(attribute="name")) -%} insert into {{ arg_dict["target_relation"] }} ({{ dest_cols_csv }}) ( select {{ dest_cols_csv }} from {{ arg_dict["temp_relation"] }} ) {% endmacro %} """ downstream_model_of_microbatch_sql = """ SELECT * FROM {{ ref('microbatch_model') }} """ microbatch_model_full_refresh_false_sql = """ {{ config(materialized='incremental', incremental_strategy='microbatch', unique_key='id', event_time='event_time', batch_size='day', begin=modules.datetime.datetime(2020, 1, 1, 0, 0, 0), full_refresh=False) }} select * from {{ ref('input_model') }} """ class BaseMicrobatchCustomUserStrategy: @pytest.fixture(scope="class") def models(self): return { "input_model.sql": input_model_sql, "microbatch_model.sql": microbatch_model_sql, } @pytest.fixture(scope="class") def macros(self): return {"microbatch.sql": custom_microbatch_strategy} @pytest.fixture(scope="class") def project_config_update(self): return { "flags": { "require_batched_execution_for_custom_microbatch_strategy": True, } } @pytest.fixture(scope="class") def deprecation_catcher(self) -> EventCatcher: return EventCatcher(MicrobatchMacroOutsideOfBatchesDeprecation) class TestMicrobatchCustomUserStrategyDefault(BaseMicrobatchCustomUserStrategy): @pytest.fixture(scope="class") def project_config_update(self): return { "flags": { "require_batched_execution_for_custom_microbatch_strategy": False, } } def test_use_custom_microbatch_strategy_by_default( self, project, deprecation_catcher: EventCatcher, ): # Initial run fires deprecation run_dbt(["run"], callbacks=[deprecation_catcher.catch]) # Deprecation warning about custom microbatch macro fired assert len(deprecation_catcher.caught_events) == 1 # Incremental run uses custom strategy _, logs = run_dbt_and_capture(["run"]) assert "custom microbatch strategy" in logs # The custom strategy wasn't used with batch functionality assert "START batch" not in logs class TestMicrobatchCustomUserStrategyProjectFlagTrueValid(BaseMicrobatchCustomUserStrategy): def test_use_custom_microbatch_strategy_project_flag_true_invalid_incremental_strategy( self, project, deprecation_catcher: EventCatcher, ): with mock.patch.object( type(project.adapter), "valid_incremental_strategies", lambda _: ["microbatch"] ): # Initial run with patch_microbatch_end_time("2020-01-03 13:57:00"): run_dbt(["run"], callbacks=[deprecation_catcher.catch]) # Deprecation warning about custom microbatch macro not fired assert len(deprecation_catcher.caught_events) == 0 # Incremental run uses custom strategy with patch_microbatch_end_time("2020-01-03 13:57:00"): _, logs = run_dbt_and_capture(["run"]) assert "custom microbatch strategy" in logs # The custom strategy was used with batch functionality assert "START batch" in logs class TestMicrobatchCustomUserStrategyProjectFlagTrueNoValidBuiltin( BaseMicrobatchCustomUserStrategy ): def test_use_custom_microbatch_strategy_project_flag_true_invalid_incremental_strategy( self, project ): with mock.patch.object( type(project.adapter), "valid_incremental_strategies", lambda _: [] ): # Run of microbatch model while adapter doesn't have a "valid" # microbatch strategy causes no error when behaviour flag set to true # and there is a custom microbatch macro with patch_microbatch_end_time("2020-01-03 13:57:00"): _, logs = run_dbt_and_capture(["run"]) assert "'microbatch' is not valid" not in logs assert ( "The use of a custom microbatch macro outside of batched execution is deprecated" not in logs ) class BaseMicrobatchTest: @pytest.fixture(scope="class") def models(self): return { "input_model.sql": input_model_sql, "microbatch_model.sql": microbatch_model_sql, } def assert_row_count(self, project, relation_name: str, expected_row_count: int): relation = relation_from_name(project.adapter, relation_name) result = project.run_sql(f"select count(*) as num_rows from {relation}", fetch="one") if result[0] != expected_row_count: # running show for debugging run_dbt(["show", "--inline", f"select * from {relation}"]) assert result[0] == expected_row_count class TestMicrobatchCLI(BaseMicrobatchTest): CLI_COMMAND_NAME = "run" def test_run_with_event_time(self, project): # run without --event-time-start or --event-time-end - 3 expected rows in output model_catcher = EventCatcher(event_to_catch=LogModelResult) batch_catcher = EventCatcher(event_to_catch=LogBatchResult) with patch_microbatch_end_time("2020-01-03 13:57:00"): run_dbt([self.CLI_COMMAND_NAME], callbacks=[model_catcher.catch, batch_catcher.catch]) self.assert_row_count(project, "microbatch_model", 3) assert len(model_catcher.caught_events) == 2 assert len(batch_catcher.caught_events) == 3 batch_creation_events = 0 for caught_event in batch_catcher.caught_events: if "batch 2020" in caught_event.data.description: batch_creation_events += 1 assert caught_event.data.execution_time > 0 # 3 batches should have been run, so there should be 3 batch # creation events assert batch_creation_events == 3 # build model between 2020-01-02 >= event_time < 2020-01-03 run_dbt( [ self.CLI_COMMAND_NAME, "--event-time-start", "2020-01-02", "--event-time-end", "2020-01-03", "--full-refresh", ] ) self.assert_row_count(project, "microbatch_model", 1) class TestMicrobatchCLIBuild(TestMicrobatchCLI): CLI_COMMAND_NAME = "build" class TestMicrobatchCLIRunOutputJSON(BaseMicrobatchTest): def test_list_output_json(self, project: TestProjInfo): """Test whether the command `dbt list --output json` works""" model_catcher = EventCatcher(event_to_catch=LogModelResult) batch_catcher = EventCatcher(event_to_catch=LogBatchResult) _, microbatch_json = run_dbt( ["list", "--output", "json"], callbacks=[model_catcher.catch, batch_catcher.catch] ) microbatch_dict = json.loads(microbatch_json) assert microbatch_dict["config"]["begin"] == "2020-01-01T00:00:00" class TestMicroBatchBoundsDefault(BaseMicrobatchTest): def test_run_with_event_time(self, project): # initial run -- backfills all data with patch_microbatch_end_time("2020-01-03 13:57:00"): run_dbt(["run"]) self.assert_row_count(project, "microbatch_model", 3) # our partition grain is "day" so running the same day without new data should produce the same results with patch_microbatch_end_time("2020-01-03 14:57:00"): run_dbt(["run"]) self.assert_row_count(project, "microbatch_model", 3) # add next two days of data test_schema_relation = project.adapter.Relation.create( database=project.database, schema=project.test_schema ) project.run_sql( f"insert into {test_schema_relation}.input_model(id, event_time) values (4, TIMESTAMP '2020-01-04 00:00:00-0'), (5, TIMESTAMP '2020-01-05 00:00:00-0')" ) self.assert_row_count(project, "input_model", 5) # re-run without changing current time => no insert with patch_microbatch_end_time("2020-01-03 14:57:00"): run_dbt(["run", "--select", "microbatch_model"]) self.assert_row_count(project, "microbatch_model", 3) # re-run by advancing time by one day changing current time => insert 1 row with patch_microbatch_end_time("2020-01-04 14:57:00"): run_dbt(["run", "--select", "microbatch_model"]) self.assert_row_count(project, "microbatch_model", 4) # re-run by advancing time by one more day changing current time => insert 1 more row with patch_microbatch_end_time("2020-01-05 14:57:00"): run_dbt(["run", "--select", "microbatch_model"]) self.assert_row_count(project, "microbatch_model", 5) class TestMicrobatchWithSource(BaseMicrobatchTest): @pytest.fixture(scope="class") def seeds(self): return { "raw_source.csv": seed_csv, } @pytest.fixture(scope="class") def models(self): return { "microbatch_model.sql": microbatch_model_calling_source_sql, "sources.yml": sources_yaml, "seeds.yml": seeds_yaml, } def test_run_with_event_time(self, project): # ensure seed is created for source run_dbt(["seed"]) # initial run -- backfills all data catcher = EventCatcher(event_to_catch=MicrobatchModelNoEventTimeInputs) with patch_microbatch_end_time("2020-01-03 13:57:00"): run_dbt(["run"], callbacks=[catcher.catch]) self.assert_row_count(project, "microbatch_model", 3) assert len(catcher.caught_events) == 0 # our partition grain is "day" so running the same day without new data should produce the same results with patch_microbatch_end_time("2020-01-03 14:57:00"): run_dbt(["run"]) self.assert_row_count(project, "microbatch_model", 3) # add next two days of data test_schema_relation = project.adapter.Relation.create( database=project.database, schema=project.test_schema ) project.run_sql( f"insert into {test_schema_relation}.raw_source(id, event_time) values (4, TIMESTAMP '2020-01-04 00:00:00-0'), (5, TIMESTAMP '2020-01-05 00:00:00-0')" ) self.assert_row_count(project, "raw_source", 5) # re-run without changing current time => no insert with patch_microbatch_end_time("2020-01-03 14:57:00"): run_dbt(["run", "--select", "microbatch_model"]) self.assert_row_count(project, "microbatch_model", 3) # re-run by advancing time by one day changing current time => insert 1 row with patch_microbatch_end_time("2020-01-04 14:57:00"): run_dbt(["run", "--select", "microbatch_model"]) self.assert_row_count(project, "microbatch_model", 4) # re-run by advancing time by one more day changing current time => insert 1 more row with patch_microbatch_end_time("2020-01-05 14:57:00"): run_dbt(["run", "--select", "microbatch_model"]) self.assert_row_count(project, "microbatch_model", 5) class TestMicrobatchJinjaContext(BaseMicrobatchTest): @pytest.fixture(scope="class") def macros(self): return {"check_batch_jinja_context.sql": invalid_batch_jinja_context_macro_sql} @pytest.fixture(scope="class") def models(self): return { "input_model.sql": input_model_sql, "microbatch_model.sql": microbatch_model_with_context_checks_sql, } def test_run_with_event_time(self, project): # initial run -- backfills all data with patch_microbatch_end_time("2020-01-03 13:57:00"): run_dbt(["run"]) self.assert_row_count(project, "microbatch_model", 3) class TestMicrobatchWithInputWithoutEventTime(BaseMicrobatchTest): @pytest.fixture(scope="class") def models(self): return { "input_model.sql": input_model_without_event_time_sql, "microbatch_model.sql": microbatch_model_sql, } def test_run_with_event_time(self, project): catcher = EventCatcher(event_to_catch=MicrobatchModelNoEventTimeInputs) # initial run -- backfills all data with patch_microbatch_end_time("2020-01-03 13:57:00"): run_dbt(["run"], callbacks=[catcher.catch]) self.assert_row_count(project, "microbatch_model", 3) assert len(catcher.caught_events) == 1 # our partition grain is "day" so running the same day without new data should produce the same results catcher.caught_events = [] with patch_microbatch_end_time("2020-01-03 14:57:00"): run_dbt(["run"], callbacks=[catcher.catch]) self.assert_row_count(project, "microbatch_model", 3) assert len(catcher.caught_events) == 1 # add next two days of data test_schema_relation = project.adapter.Relation.create( database=project.database, schema=project.test_schema ) project.run_sql( f"insert into {test_schema_relation}.input_model(id, event_time) values (4, TIMESTAMP '2020-01-04 00:00:00-0'), (5, TIMESTAMP '2020-01-05 00:00:00-0')" ) self.assert_row_count(project, "input_model", 5) # re-run without changing current time => INSERT BECAUSE INPUT MODEL ISN'T BEING FILTERED with patch_microbatch_end_time("2020-01-03 14:57:00"): run_dbt(["run", "--select", "microbatch_model"]) self.assert_row_count(project, "microbatch_model", 5) class TestMicrobatchUsingRefRenderSkipsFilter(BaseMicrobatchTest): def test_run_with_event_time(self, project): # initial run -- backfills all data with patch_microbatch_end_time("2020-01-03 13:57:00"): run_dbt(["run"]) self.assert_row_count(project, "microbatch_model", 3) # our partition grain is "day" so running the same day without new data should produce the same results with patch_microbatch_end_time("2020-01-03 14:57:00"): run_dbt(["run"]) self.assert_row_count(project, "microbatch_model", 3) # add next two days of data test_schema_relation = project.adapter.Relation.create( database=project.database, schema=project.test_schema ) project.run_sql( f"insert into {test_schema_relation}.input_model(id, event_time) values (4, TIMESTAMP '2020-01-04 00:00:00-0'), (5, TIMESTAMP '2020-01-05 00:00:00-0')" ) self.assert_row_count(project, "input_model", 5) # re-run without changing current time => no insert with patch_microbatch_end_time("2020-01-03 14:57:00"): run_dbt(["run", "--select", "microbatch_model"]) self.assert_row_count(project, "microbatch_model", 3) # Update microbatch model to call .render() on ref('input_model') write_file( microbatch_model_ref_render_sql, project.project_root, "models", "microbatch_model.sql" ) # re-run without changing current time => INSERT because .render() skips filtering with patch_microbatch_end_time("2020-01-03 14:57:00"): run_dbt(["run", "--select", "microbatch_model"]) self.assert_row_count(project, "microbatch_model", 5) microbatch_model_context_vars = """ {{ config(materialized='incremental', incremental_strategy='microbatch', unique_key='id', event_time='event_time', batch_size='day', begin=modules.datetime.datetime(2020, 1, 1, 0, 0, 0)) }} {{ log("start: "~ model.config.__dbt_internal_microbatch_event_time_start, info=True)}} {{ log("end: "~ model.config.__dbt_internal_microbatch_event_time_end, info=True)}} {% if model.batch %} {{ log("batch.event_time_start: "~ model.batch.event_time_start, info=True)}} {{ log("batch.event_time_end: "~ model.batch.event_time_end, info=True)}} {{ log("batch.id: "~ model.batch.id, info=True)}} {{ log("start timezone: "~ model.batch.event_time_start.tzinfo, info=True)}} {{ log("end timezone: "~ model.batch.event_time_end.tzinfo, info=True)}} {% endif %} select * from {{ ref('input_model') }} """ class TestMicrobatchJinjaContextVarsAvailable(BaseMicrobatchTest): @pytest.fixture(scope="class") def models(self): return { "input_model.sql": input_model_sql, "microbatch_model.sql": microbatch_model_context_vars, } def test_run_with_event_time_logs(self, project): with patch_microbatch_end_time("2020-01-03 13:57:00"): _, logs = run_dbt_and_capture(["run"]) assert "start: 2020-01-01 00:00:00+00:00" in logs assert "end: 2020-01-02 00:00:00+00:00" in logs assert "batch.event_time_start: 2020-01-01 00:00:00+00:00" in logs assert "batch.event_time_end: 2020-01-02 00:00:00+00:00" in logs assert "batch.id: 20200101" in logs assert "start timezone: UTC" in logs assert "end timezone: UTC" in logs assert "start: 2020-01-02 00:00:00+00:00" in logs assert "end: 2020-01-03 00:00:00+00:00" in logs assert "batch.event_time_start: 2020-01-02 00:00:00+00:00" in logs assert "batch.event_time_end: 2020-01-03 00:00:00+00:00" in logs assert "batch.id: 20200102" in logs assert "start: 2020-01-03 00:00:00+00:00" in logs assert "end: 2020-01-03 13:57:00+00:00" in logs assert "batch.event_time_start: 2020-01-03 00:00:00+00:00" in logs assert "batch.event_time_end: 2020-01-03 13:57:00+00:00" in logs assert "batch.id: 20200103" in logs # compile does not have access to populated batch context vars, but should not break on access with patch_microbatch_end_time("2020-01-03 13:57:00"): _, compile_logs = run_dbt_and_capture(["compile"]) assert "start:" in compile_logs assert "end:" in compile_logs assert "batch.event_time_start:" not in compile_logs assert "batch.event_time_end:" not in compile_logs microbatch_model_failing_incremental_partition_sql = """ {{ config(materialized='incremental', incremental_strategy='microbatch', unique_key='id', event_time='event_time', batch_size='day', begin=modules.datetime.datetime(2020, 1, 1, 0, 0, 0)) }} {% if '2020-01-02' in (model.config.__dbt_internal_microbatch_event_time_start | string) %} invalid_sql {% endif %} select * from {{ ref('input_model') }} """ class TestMicrobatchIncrementalBatchFailure(BaseMicrobatchTest): @pytest.fixture(scope="class") def models(self): return { "input_model.sql": input_model_sql, "microbatch_model.sql": microbatch_model_failing_incremental_partition_sql, "downstream_model.sql": downstream_model_of_microbatch_sql, } def test_run_with_event_time(self, project): event_catcher = EventCatcher( GenericExceptionOnRun, predicate=lambda event: event.data.node_info is not None ) # run all partitions from start - 2 expected rows in output, one failed with patch_microbatch_end_time("2020-01-03 13:57:00"): run_dbt(["run"], callbacks=[event_catcher.catch], expect_pass=False) assert len(event_catcher.caught_events) == 1 self.assert_row_count(project, "microbatch_model", 2) run_results = get_artifact(project.project_root, "target", "run_results.json") microbatch_run_result = run_results["results"][1] assert microbatch_run_result["status"] == "partial success" batch_results = microbatch_run_result["batch_results"] assert batch_results is not None assert len(batch_results["successful"]) == 2 assert len(batch_results["failed"]) == 1 assert run_results["results"][2]["status"] == "skipped" class TestMicrobatchRetriesPartialSuccesses(BaseMicrobatchTest): @pytest.fixture(scope="class") def models(self): return { "input_model.sql": input_model_sql, "microbatch_model.sql": microbatch_model_failing_incremental_partition_sql, } def test_run_with_event_time(self, project): # run all partitions from start - 2 expected rows in output, one failed with patch_microbatch_end_time("2020-01-03 13:57:00"): _, console_output = run_dbt_and_capture(["run"], expect_pass=False) assert "PARTIAL SUCCESS (2/3)" in console_output assert "Completed with 1 partial success" in console_output self.assert_row_count(project, "microbatch_model", 2) run_results = get_artifact(project.project_root, "target", "run_results.json") microbatch_run_result = run_results["results"][1] assert microbatch_run_result["status"] == "partial success" batch_results = microbatch_run_result["batch_results"] assert batch_results is not None assert len(batch_results["successful"]) == 2 assert len(batch_results["failed"]) == 1 # update the microbatch model so that it no longer fails write_file(microbatch_model_sql, project.project_root, "models", "microbatch_model.sql") with patch_microbatch_end_time("2020-01-03 13:57:00"): _, console_output = run_dbt_and_capture(["retry"]) assert "PARTIAL SUCCESS" not in console_output assert "Completed with 1 partial success" not in console_output assert "Completed successfully" in console_output self.assert_row_count(project, "microbatch_model", 3) class TestMicrobatchMultipleRetries(BaseMicrobatchTest): @pytest.fixture(scope="class") def models(self): return { "input_model.sql": input_model_sql, "microbatch_model.sql": microbatch_model_failing_incremental_partition_sql, } def test_run_with_event_time(self, project): # run all partitions from start - 2 expected rows in output, one failed with patch_microbatch_end_time("2020-01-03 13:57:00"): _, console_output = run_dbt_and_capture(["run"], expect_pass=False) assert "PARTIAL SUCCESS (2/3)" in console_output assert "Completed with 1 partial success" in console_output self.assert_row_count(project, "microbatch_model", 2) with patch_microbatch_end_time("2020-01-03 13:57:00"): _, console_output = run_dbt_and_capture(["retry"], expect_pass=False) assert "PARTIAL SUCCESS" not in console_output assert "ERROR" in console_output assert "Completed with 1 error, 0 partial successes, and 0 warnings" in console_output self.assert_row_count(project, "microbatch_model", 2) with patch_microbatch_end_time("2020-01-03 13:57:00"): _, console_output = run_dbt_and_capture(["retry"], expect_pass=False) assert "PARTIAL SUCCESS" not in console_output assert "ERROR" in console_output assert "Completed with 1 error, 0 partial successes, and 0 warnings" in console_output self.assert_row_count(project, "microbatch_model", 2) microbatch_model_first_partition_failing_sql = """ {{ config(materialized='incremental', incremental_strategy='microbatch', unique_key='id', event_time='event_time', batch_size='day', begin=modules.datetime.datetime(2020, 1, 1, 0, 0, 0)) }} {% if '2020-01-01' in (model.config.__dbt_internal_microbatch_event_time_start | string) %} invalid_sql {% endif %} select * from {{ ref('input_model') }} """ microbatch_model_second_batch_failing_sql = """ {{ config(materialized='incremental', incremental_strategy='microbatch', unique_key='id', event_time='event_time', batch_size='day', begin=modules.datetime.datetime(2020, 1, 1, 0, 0, 0)) }} {% if '20200102' == model.batch.id %} invalid_sql {% endif %} select * from {{ ref('input_model') }} """ class TestMicrobatchInitialBatchFailure(BaseMicrobatchTest): @pytest.fixture(scope="class") def models(self): return { "input_model.sql": input_model_sql, "microbatch_model.sql": microbatch_model_first_partition_failing_sql, } def test_run_with_event_time(self, project): # When the first batch of a microbatch model fails, the rest of the batches should # be skipped and the model marked as failed (not _partial success_) general_exc_catcher = EventCatcher( GenericExceptionOnRun, predicate=lambda event: event.data.node_info is not None, ) batch_catcher = EventCatcher( event_to_catch=LogBatchResult, predicate=lambda event: event.data.status == "skipped", ) # run all partitions from start - 2 expected rows in output, one failed with patch_microbatch_end_time("2020-01-03 13:57:00"): run_dbt( ["run"], expect_pass=False, callbacks=[general_exc_catcher.catch, batch_catcher.catch], ) assert len(general_exc_catcher.caught_events) == 1 assert len(batch_catcher.caught_events) == 2 # Because the first batch failed, and the rest of the batches were skipped, the table shouldn't # exist in the data warehosue relation_info = relation_from_name(project.adapter, "microbatch_model") relation = project.adapter.get_relation( relation_info.database, relation_info.schema, relation_info.name ) assert relation is None class TestMicrobatchSecondBatchFailure(BaseMicrobatchTest): @pytest.fixture(scope="class") def models(self): return { "input_model.sql": input_model_sql, "microbatch_model.sql": microbatch_model_second_batch_failing_sql, } def test_run_with_event_time(self, project): event_catcher = EventCatcher( GenericExceptionOnRun, predicate=lambda event: event.data.node_info is not None ) # run all partitions from start - 2 expected rows in output, one failed with patch_microbatch_end_time("2020-01-03 13:57:00"): run_dbt(["run"], expect_pass=False, callbacks=[event_catcher.catch]) assert len(event_catcher.caught_events) == 1 self.assert_row_count(project, "microbatch_model", 2) class TestMicrobatchCompiledRunPaths(BaseMicrobatchTest): def test_run_with_event_time(self, project): # run all partitions from start - 2 expected rows in output, one failed with patch_microbatch_end_time("2020-01-03 13:57:00"): run_dbt(["run"]) # Compiled paths - batch compilations assert read_file( project.project_root, "target", "compiled", "test", "models", "microbatch_model", "microbatch_model_2020-01-01.sql", ) assert read_file( project.project_root, "target", "compiled", "test", "models", "microbatch_model", "microbatch_model_2020-01-02.sql", ) assert read_file( project.project_root, "target", "compiled", "test", "models", "microbatch_model", "microbatch_model_2020-01-03.sql", ) assert read_file( project.project_root, "target", "run", "test", "models", "microbatch_model", "microbatch_model_2020-01-01.sql", ) assert read_file( project.project_root, "target", "run", "test", "models", "microbatch_model", "microbatch_model_2020-01-02.sql", ) assert read_file( project.project_root, "target", "run", "test", "models", "microbatch_model", "microbatch_model_2020-01-03.sql", ) class TestMicrobatchCompiledRunPathsHourly(BaseMicrobatchTest): @pytest.fixture(scope="class") def models(self): return { "input_model.sql": input_model_sql, "microbatch_model.sql": microbatch_model_hour_sql, } def test_run_with_event_time(self, project): # run all partitions from start - 2 expected rows in output, one failed with patch_microbatch_end_time("2020-01-03 13:57:00"): run_dbt(["run"]) # Compiled paths - batch compilations assert read_file( project.project_root, "target", "compiled", "test", "models", "microbatch_model", "microbatch_model_2020-01-03T13.sql", ) assert read_file( project.project_root, "target", "run", "test", "models", "microbatch_model", "microbatch_model_2020-01-03T13.sql", ) class TestMicrobatchCompiledRunPathsMonthly(BaseMicrobatchTest): @pytest.fixture(scope="class") def models(self): return { "input_model.sql": input_model_sql, "microbatch_model.sql": microbatch_model_month_sql, } def test_run_with_event_time(self, project): # run all partitions from start - 2 expected rows in output, one failed with patch_microbatch_end_time("2020-01-03 13:57:00"): run_dbt(["run"]) # Compiled paths - batch compilations assert read_file( project.project_root, "target", "compiled", "test", "models", "microbatch_model", "microbatch_model_2020-01.sql", ) assert read_file( project.project_root, "target", "run", "test", "models", "microbatch_model", "microbatch_model_2020-01.sql", ) class TestMicrobatchCompiledRunPathsYearly(BaseMicrobatchTest): @pytest.fixture(scope="class") def models(self): return { "input_model.sql": input_model_sql, "microbatch_model.sql": microbatch_model_year_sql, } def test_run_with_event_time(self, project): # run all partitions from start - 2 expected rows in output, one failed with patch_microbatch_end_time("2020-01-03 13:57:00"): run_dbt(["run"]) # Compiled paths - batch compilations assert read_file( project.project_root, "target", "compiled", "test", "models", "microbatch_model", "microbatch_model_2020.sql", ) assert read_file( project.project_root, "target", "run", "test", "models", "microbatch_model", "microbatch_model_2020.sql", ) class TestMicrobatchFullRefreshConfigFalse(BaseMicrobatchTest): @pytest.fixture(scope="class") def models(self): return { "input_model.sql": input_model_sql, "microbatch_model.sql": microbatch_model_full_refresh_false_sql, "downstream_model.sql": downstream_model_of_microbatch_sql, } def test_run_with_event_time(self, project): # run all partitions from 2020-01-02 to spoofed "now" - 2 expected rows in output with patch_microbatch_end_time("2020-01-03 13:57:00"): run_dbt( [ "run", "--event-time-start", "2020-01-02", "--event-time-end", "2020-01-03 13:57:00", ] ) self.assert_row_count(project, "microbatch_model", 2) # re-running shouldn't change what it's in the data set because there is nothing new with patch_microbatch_end_time("2020-01-03 13:57:00"): run_dbt(["run"]) self.assert_row_count(project, "microbatch_model", 2) # running with --full-refresh shouldn't pick up 2020-01-01 BECAUSE the model has # full_refresh = false with patch_microbatch_end_time("2020-01-03 13:57:00"): run_dbt(["run", "--full-refresh"]) self.assert_row_count(project, "microbatch_model", 2) # update the microbatch model to no longer have full_refresh=False config write_file(microbatch_model_sql, project.project_root, "models", "microbatch_model.sql") # running with full refresh should now pick up the 2020-01-01 data with patch_microbatch_end_time("2020-01-03 13:57:00"): run_dbt(["run", "--full-refresh"]) self.assert_row_count(project, "microbatch_model", 3) class TestMicrbobatchModelsRunWithSameCurrentTime(BaseMicrobatchTest): @pytest.fixture(scope="class") def models(self): return { "input_model.sql": input_model_sql, "microbatch_model.sql": microbatch_yearly_model_sql, "second_microbatch_model.sql": microbatch_yearly_model_downstream_sql, } def test_microbatch(self, project) -> None: run_dbt(["run"]) run_results = get_artifact(project.project_root, "target", "run_results.json") microbatch_model_last_batch = run_results["results"][1]["batch_results"]["successful"][-1] second_microbatch_model_last_batch = run_results["results"][2]["batch_results"][ "successful" ][-1] # they should have the same last batch because they are using the _same_ "current_time" assert microbatch_model_last_batch == second_microbatch_model_last_batch class TestMicrobatchModelSkipped(BaseMicrobatchTest): @pytest.fixture(scope="class") def models(self): return { "input_model.sql": input_model_invalid_sql, "microbatch_model.sql": microbatch_model_sql, } def test_microbatch_model_skipped(self, project) -> None: run_dbt(["run"], expect_pass=False) run_results = get_artifact(project.project_root, "target", "run_results.json") microbatch_result = run_results["results"][1] assert microbatch_result["status"] == "skipped" assert microbatch_result["batch_results"] is None class TestMicrobatchCanRunParallelOrSequential(BaseMicrobatchTest): @pytest.fixture def batch_exc_catcher(self) -> EventCatcher: return EventCatcher(MicrobatchExecutionDebug) def test_microbatch( self, mocker: MockerFixture, project, batch_exc_catcher: EventCatcher ) -> None: mocked_srip = mocker.patch("dbt.task.run.MicrobatchBatchRunner.should_run_in_parallel") # Should be run in parallel mocked_srip.return_value = True with patch_microbatch_end_time("2020-01-03 13:57:00"): _ = run_dbt(["run"], callbacks=[batch_exc_catcher.catch]) assert len(batch_exc_catcher.caught_events) > 1 some_batches_run_concurrently = False for caugh_event in batch_exc_catcher.caught_events: if "is being run concurrently" in caugh_event.data.msg: # type: ignore some_batches_run_concurrently = True break assert some_batches_run_concurrently, "Found no batches being run concurrently!" # reset caught events batch_exc_catcher.caught_events = [] # Should _not_ run in parallel mocked_srip.return_value = False with patch_microbatch_end_time("2020-01-03 13:57:00"): _ = run_dbt(["run"], callbacks=[batch_exc_catcher.catch]) assert len(batch_exc_catcher.caught_events) > 1 some_batches_run_concurrently = False for caugh_event in batch_exc_catcher.caught_events: if "is being run concurrently" in caugh_event.data.msg: # type: ignore some_batches_run_concurrently = True break assert not some_batches_run_concurrently, "Found a batch being run concurrently!" class TestFirstAndLastBatchAlwaysSequential(BaseMicrobatchTest): @pytest.fixture def batch_exc_catcher(self) -> EventCatcher: return EventCatcher( event_to_catch=MicrobatchExecutionDebug, predicate=lambda event: "is being run" in event.data.msg, ) def test_microbatch( self, mocker: MockerFixture, project, batch_exc_catcher: EventCatcher ) -> None: mocked_srip = mocker.patch("dbt.task.run.MicrobatchBatchRunner.should_run_in_parallel") # Should be run in parallel mocked_srip.return_value = True with patch_microbatch_end_time("2020-01-03 13:57:00"): _ = run_dbt(["run"], callbacks=[batch_exc_catcher.catch]) assert len(batch_exc_catcher.caught_events) > 1 first_batch_event = batch_exc_catcher.caught_events[0] last_batch_event = batch_exc_catcher.caught_events[-1] for event in [first_batch_event, last_batch_event]: assert "is being run sequentially" in event.data.msg # type: ignore for event in batch_exc_catcher.caught_events[1:-1]: assert "is being run concurrently" in event.data.msg # type: ignore class TestFirstBatchRunsPreHookLastBatchRunsPostHook(BaseMicrobatchTest): @pytest.fixture(scope="class") def models(self): return { "input_model.sql": input_model_sql, "microbatch_model.sql": microbatch_model_with_pre_and_post_sql, } @pytest.fixture def batch_log_catcher(self) -> EventCatcher: def pre_or_post_hook(event) -> bool: return "execute: True" in event.data.msg and ( "pre-hook" in event.data.msg or "post-hook" in event.data.msg ) return EventCatcher(event_to_catch=JinjaLogDebug, predicate=pre_or_post_hook) def test_microbatch( self, mocker: MockerFixture, project, batch_log_catcher: EventCatcher ) -> None: with patch_microbatch_end_time("2020-01-04 13:57:00"): _ = run_dbt(["run"], callbacks=[batch_log_catcher.catch]) # There should be two logs as the pre-hook and post-hook should # both only be run once assert len(batch_log_catcher.caught_events) == 2 for event in batch_log_catcher.caught_events: # batch id that should be firing pre-hook if "20200101" in event.data.msg: # type: ignore assert "pre-hook" in event.data.msg # type: ignore # batch id that should be firing the post-hook if "20200104" in event.data.msg: # type: ignore assert "post-hook" in event.data.msg # type: ignore class TestWhenOnlyOneBatchRunBothPostAndPreHooks(BaseMicrobatchTest): @pytest.fixture(scope="class") def models(self): return { "input_model.sql": input_model_sql, "microbatch_model.sql": microbatch_model_with_pre_and_post_sql, } @pytest.fixture def batch_log_catcher(self) -> EventCatcher: def pre_or_post_hook(event) -> bool: return "execute: True" in event.data.msg and ( "pre-hook" in event.data.msg or "post-hook" in event.data.msg ) return EventCatcher(event_to_catch=JinjaLogDebug, predicate=pre_or_post_hook) @pytest.fixture def generic_exception_catcher(self) -> EventCatcher: return EventCatcher(event_to_catch=GenericExceptionOnRun) def test_microbatch( self, project, batch_log_catcher: EventCatcher, generic_exception_catcher: EventCatcher, ) -> None: with patch_microbatch_end_time("2020-01-01 13:57:00"): _ = run_dbt( ["run"], callbacks=[batch_log_catcher.catch, generic_exception_catcher.catch] ) # There should be two logs as the pre-hook and post-hook should # both only be run once assert len(batch_log_catcher.caught_events) == 2 assert "20200101" in batch_log_catcher.caught_events[0].data.msg # type: ignore assert "pre-hook" in batch_log_catcher.caught_events[0].data.msg # type: ignore assert "20200101" in batch_log_catcher.caught_events[1].data.msg # type: ignore assert "post-hook" in batch_log_catcher.caught_events[1].data.msg # type: ignore # we had a bug where having only one batch caused a generic exception assert len(generic_exception_catcher.caught_events) == 0 class TestCanSilenceInvalidConcurrentBatchesConfigWarning(BaseMicrobatchTest): @pytest.fixture(scope="class") def models(self): return { "input_model.sql": input_model_sql, "microbatch_model.sql": microbatch_model_force_concurrent_batches_sql, } @pytest.fixture def event_catcher(self) -> EventCatcher: return EventCatcher(event_to_catch=InvalidConcurrentBatchesConfig) def test_microbatch( self, project, event_catcher: EventCatcher, ) -> None: # This test works because postgres doesn't support concurrent batch execution # If the postgres adapter starts supporting concurrent batch execution we'll # need to start mocking the return value of `adapter.supports()` with patch_microbatch_end_time("2020-01-01 13:57:00"): _ = run_dbt(["run"], callbacks=[event_catcher.catch]) # We didn't silence the warning, so we get it assert len(event_catcher.caught_events) == 1 # Clear caught events event_catcher.caught_events = [] # Run again with silencing with patch_microbatch_end_time("2020-01-01 13:57:00"): _ = run_dbt( ["run", "--warn-error-options", "{'silence': ['InvalidConcurrentBatchesConfig']}"], callbacks=[event_catcher.catch], ) # Because we silenced the warning, it shouldn't get fired assert len(event_catcher.caught_events) == 0 single_batch_microbatch_model_sql = """ {{ config( materialized='incremental', incremental_strategy='microbatch', unique_key='tmp', event_time='tmp', begin=modules.datetime.datetime.now(), lookback=0, batch_size='day', meta={'param': 'invalid_param'}, pre_hook=[ validate_param('param1') ] ) }} SELECT current_date as tmp """ validate_param_macro_sql = """ {% macro validate_param(valid_param) %} {% set current_param = config.get('meta')['param'] %} {% if execute %} {% if current_param != valid_param %} {% set exceptions_message = "Invalid param: " ~ current_param ~ ", valid param: " ~ valid_param %} {% do exceptions.raise_compiler_error(exceptions_message) %} {% endif %} {% endif %} {% endmacro %} """ class TestCompilationErrorOnSingleBatchRun(BaseMicrobatchTest): @pytest.fixture(scope="class") def models(self) -> Dict[str, str]: return { "microbatch_model.sql": single_batch_microbatch_model_sql, } @pytest.fixture(scope="class") def macros(self) -> Dict[str, str]: return { "validate_param.sql": validate_param_macro_sql, } def test_microbatch(self, project) -> None: _, console_output = run_dbt_and_capture(["run"], expect_pass=False) assert "Completed with 1 error, 0 partial successes, and 0 warnings" in console_output microbatch_model_all_failing_sql = """ {{ config(materialized='incremental', incremental_strategy='microbatch', unique_key='id', event_time='event_time', batch_size='day', begin=modules.datetime.datetime(2020, 1, 1, 0, 0, 0)) }} invalid_sql select * from {{ ref('input_model') }} """ class TestMicrobatchRetryUsesOriginalInvocationTime(BaseMicrobatchTest): """When all batches fail and dbt retry is run later, the retry should recompute batches using the original invocation time, not the current time.""" @pytest.fixture(scope="class") def models(self): return { "input_model.sql": input_model_sql, "microbatch_model.sql": microbatch_model_all_failing_sql, } def test_retry_uses_original_invocation_time(self, project): # Run with end time of 2020-01-03 — produces 3 batches, all fail with patch_microbatch_end_time("2020-01-03 13:57:00"): run_dbt(["run"], expect_pass=False) run_results = get_artifact(project.project_root, "target", "run_results.json") microbatch_result = next( r for r in run_results["results"] if "microbatch_model" in r["unique_id"] ) assert microbatch_result["status"] == "error" # Simulate the original run happening at 2020-01-03 by updating # invocation_started_at in the saved run_results.json run_results_path = os.path.join(project.project_root, "target", "run_results.json") run_results["metadata"]["invocation_started_at"] = "2020-01-03T13:57:00Z" with open(run_results_path, "w") as f: json.dump(run_results, f) # Fix the model so retry can succeed write_file( microbatch_model_sql, project.project_root, "models", "microbatch_model.sql", ) # Run retry WITHOUT patching end time. The fix ensures retry # uses the original invocation time from run_results.json (2020-01-03), # so only 3 batches are produced. Without the fix, it would use # the current time and produce many more batches. batch_catcher = EventCatcher(event_to_catch=LogBatchResult) run_dbt(["retry"], callbacks=[batch_catcher.catch]) # Should only have 3 batches (2020-01-01, 2020-01-02, 2020-01-03) assert len(batch_catcher.caught_events) == 3 self.assert_row_count(project, "microbatch_model", 3) class TestMicrobatchBuildRetryUsesOriginalInvocationTime(BaseMicrobatchTest): """Same as above but using dbt build + retry to verify the issubclass check ensures BuildTask also gets original invocation time behavior.""" @pytest.fixture(scope="class") def models(self): return { "input_model.sql": input_model_sql, "microbatch_model.sql": microbatch_model_all_failing_sql, } def test_build_retry_uses_original_invocation_time(self, project): # Run build with end time of 2020-01-03 — produces 3 batches, all fail with patch_microbatch_end_time("2020-01-03 13:57:00"): run_dbt(["build"], expect_pass=False) run_results = get_artifact(project.project_root, "target", "run_results.json") microbatch_result = next( r for r in run_results["results"] if "microbatch_model" in r["unique_id"] ) assert microbatch_result["status"] == "error" # Simulate the original build happening at 2020-01-03 by updating # invocation_started_at in the saved run_results.json run_results_path = os.path.join(project.project_root, "target", "run_results.json") run_results["metadata"]["invocation_started_at"] = "2020-01-03T13:57:00Z" with open(run_results_path, "w") as f: json.dump(run_results, f) # Fix the model so retry can succeed write_file( microbatch_model_sql, project.project_root, "models", "microbatch_model.sql", ) # Retry should use the original invocation time, not "now" batch_catcher = EventCatcher(event_to_catch=LogBatchResult) run_dbt(["retry"], callbacks=[batch_catcher.catch]) # Should only have 3 batches (2020-01-01, 2020-01-02, 2020-01-03) assert len(batch_catcher.caught_events) == 3 self.assert_row_count(project, "microbatch_model", 3) ================================================ FILE: tests/functional/microbatch/test_microbatch_config_validation.py ================================================ import pytest from dbt.exceptions import ParsingError from dbt.tests.util import run_dbt valid_microbatch_model_sql = """ {{ config(materialized='incremental', incremental_strategy='microbatch', batch_size='day', event_time='event_time') }} select * from {{ ref('input_model') }} """ valid_microbatch_model_no_config_sql = """ select * from {{ ref('input_model') }} """ valid_microbatch_model_config_yml = """ models: - name: microbatch config: materialized: incremental incremental_strategy: microbatch batch_size: day event_time: event_time begin: 2020-01-01 """ invalid_microbatch_model_config_yml = """ models: - name: microbatch config: materialized: incremental incremental_strategy: microbatch batch_size: day event_time: event_time begin: 2020-01-01 11 PM """ missing_event_time_microbatch_model_sql = """ {{ config(materialized='incremental', incremental_strategy='microbatch', batch_size='day') }} select * from {{ ref('input_model') }} """ invalid_event_time_microbatch_model_sql = """ {{ config(materialized='incremental', incremental_strategy='microbatch', batch_size='day', event_time=2) }} select * from {{ ref('input_model') }} """ missing_begin_microbatch_model_sql = """ {{ config(materialized='incremental', incremental_strategy='microbatch', batch_size='day', event_time='event_time') }} select * from {{ ref('input_model') }} """ invalid_begin_microbatch_model_sql = """ {{ config(materialized='incremental', incremental_strategy='microbatch', batch_size='day', event_time='event_time', begin=2) }} select * from {{ ref('input_model') }} """ missing_batch_size_microbatch_model_sql = """ {{ config(materialized='incremental', incremental_strategy='microbatch', event_time='event_time') }} select * from {{ ref('input_model') }} """ invalid_batch_size_microbatch_model_sql = """ {{ config(materialized='incremental', incremental_strategy='microbatch', batch_size='invalid', event_time='event_time') }} select * from {{ ref('input_model') }} """ invalid_event_time_input_model_sql = """ {{ config(materialized='table', event_time=1) }} select 1 as id, TIMESTAMP '2020-01-01 00:00:00-0' as event_time """ valid_input_model_sql = """ {{ config(materialized='table') }} select 1 as id, TIMESTAMP '2020-01-01 00:00:00-0' as event_time """ class BaseMicrobatchTestParseError: @pytest.fixture(scope="class") def models(self): return {} @pytest.fixture(scope="class") def project_config_update(self): return { "flags": { "require_batched_execution_for_custom_microbatch_strategy": True, } } def test_parsing_error_raised(self, project): with pytest.raises(ParsingError): run_dbt(["parse"]) class BaseMicrobatchTestNoError: @pytest.fixture(scope="class") def models(self): return {} @pytest.fixture(scope="class") def project_config_update(self): return { "flags": { "require_batched_execution_for_custom_microbatch_strategy": True, } } def test_parsing_error_not_raised(self, project): run_dbt(["parse"]) class TestMissingEventTimeMicrobatch(BaseMicrobatchTestParseError): @pytest.fixture(scope="class") def models(self): return { "input_model.sql": valid_input_model_sql, "microbatch.sql": missing_event_time_microbatch_model_sql, } class TestInvalidEventTimeMicrobatch(BaseMicrobatchTestParseError): @pytest.fixture(scope="class") def models(self): return { "input_model.sql": valid_input_model_sql, "microbatch.sql": invalid_event_time_microbatch_model_sql, } class TestMissingBeginMicrobatch(BaseMicrobatchTestParseError): @pytest.fixture(scope="class") def models(self): return { "input_model.sql": valid_input_model_sql, "microbatch.sql": missing_begin_microbatch_model_sql, } class TestInvaliBeginTypeMicrobatch(BaseMicrobatchTestParseError): @pytest.fixture(scope="class") def models(self): return { "input_model.sql": valid_input_model_sql, "microbatch.sql": invalid_begin_microbatch_model_sql, } class TestInvaliBegiFormatMicrobatch(BaseMicrobatchTestParseError): @pytest.fixture(scope="class") def models(self): return { "input_model.sql": valid_input_model_sql, "microbatch.sql": valid_microbatch_model_no_config_sql, "microbatch.yml": invalid_microbatch_model_config_yml, } class TestMissingBatchSizeMicrobatch(BaseMicrobatchTestParseError): @pytest.fixture(scope="class") def models(self): return { "input_model.sql": valid_input_model_sql, "microbatch.sql": missing_batch_size_microbatch_model_sql, } class TestInvalidBatchSizeMicrobatch(BaseMicrobatchTestParseError): @pytest.fixture(scope="class") def models(self): return { "input_model.sql": valid_input_model_sql, "microbatch.sql": invalid_batch_size_microbatch_model_sql, } class TestInvalidInputEventTimeMicrobatch(BaseMicrobatchTestParseError): @pytest.fixture(scope="class") def models(self): return { "input_model.sql": invalid_event_time_input_model_sql, "microbatch.sql": valid_microbatch_model_sql, } class TestValidBeginMicrobatch(BaseMicrobatchTestNoError): @pytest.fixture(scope="class") def models(self): return { "input_model.sql": valid_input_model_sql, "microbatch.sql": valid_microbatch_model_no_config_sql, "schema.yml": valid_microbatch_model_config_yml, } ================================================ FILE: tests/functional/minimal_cli/fixtures.py ================================================ import pytest from click.testing import CliRunner models__schema_yml = """ version: 2 models: - name: sample_model columns: - name: sample_num data_tests: - accepted_values: values: [1, 2] - not_null - name: sample_bool data_tests: - not_null - unique """ models__sample_model = """ select * from {{ ref('sample_seed') }} """ snapshots__sample_snapshot = """ {% snapshot orders_snapshot %} {{ config( target_database='dbt', target_schema='snapshots', unique_key='sample_num', strategy='timestamp', updated_at='updated_at', ) }} select * from {{ ref('sample_model') }} {% endsnapshot %} """ seeds__sample_seed = """sample_num,sample_bool 1,true 2,false ,true """ tests__failing_sql = """ {{ config(severity = 'warn') }} select 1 """ class BaseConfigProject: @pytest.fixture() def runner(self): return CliRunner() @pytest.fixture(scope="class") def project_config_update(self): return { "name": "jaffle_shop", "profile": "jaffle_shop", "version": "0.1.0", "config-version": 2, "clean-targets": ["target", "dbt_packages", "logs"], } @pytest.fixture(scope="class") def profiles_config_update(self): return { "jaffle_shop": { "outputs": { "dev": { "type": "postgres", "dbname": "dbt", "schema": "jaffle_shop", "host": "localhost", "user": "root", "port": 5432, "pass": "password", } }, "target": "dev", } } @pytest.fixture(scope="class") def packages(self): return {"packages": [{"package": "dbt-labs/dbt_utils", "version": "1.0.0"}]} @pytest.fixture(scope="class") def models(self): return { "schema.yml": models__schema_yml, "sample_model.sql": models__sample_model, } @pytest.fixture(scope="class") def snapshots(self): return {"sample_snapshot.sql": snapshots__sample_snapshot} @pytest.fixture(scope="class") def seeds(self): return {"sample_seed.csv": seeds__sample_seed} @pytest.fixture(scope="class") def tests(self): return { "failing.sql": tests__failing_sql, } ================================================ FILE: tests/functional/minimal_cli/test_minimal_cli.py ================================================ from dbt.cli.main import cli from tests.functional.minimal_cli.fixtures import BaseConfigProject from tests.functional.utils import up_one class TestClean(BaseConfigProject): """Test the minimal/happy-path for the CLI using the Click CliRunner""" def test_clean(self, runner, project): result = runner.invoke(cli, ["clean"]) assert "target" in result.output assert "dbt_packages" in result.output assert "logs" in result.output class TestCleanUpLevel(BaseConfigProject): def test_clean_one_level_up(self, runner, project): with up_one(): result = runner.invoke(cli, ["clean"]) assert result.exit_code == 2 assert "Runtime Error" in result.output assert "No dbt_project.yml" in result.output class TestDeps(BaseConfigProject): def test_deps(self, runner, project): result = runner.invoke(cli, ["deps"]) assert "dbt-labs/dbt_utils" in result.output assert "1.0.0" in result.output class TestLS(BaseConfigProject): def test_ls(self, runner, project): runner.invoke(cli, ["deps"]) ls_result = runner.invoke(cli, ["ls"]) assert "1 seed" in ls_result.output assert "1 model" in ls_result.output assert "5 data tests" in ls_result.output assert "1 snapshot" in ls_result.output class TestBuild(BaseConfigProject): def test_build(self, runner, project): runner.invoke(cli, ["deps"]) result = runner.invoke(cli, ["build"]) # 1 seed, 1 model, 2 data tests assert "PASS=4" in result.output # 2 data tests assert "ERROR=2" in result.output # Singular test assert "WARN=1" in result.output # 1 snapshot assert "SKIP=1" in result.output class TestBuildFailFast(BaseConfigProject): def test_build(self, runner, project): runner.invoke(cli, ["deps"]) result = runner.invoke(cli, ["build", "--fail-fast"]) # 1 seed, 1 model, 2 data tests assert "PASS=4" in result.output # 2 data tests assert "ERROR=2" in result.output # Singular test assert "WARN=1" in result.output # 1 snapshot assert "SKIP=1" in result.output # Skipping due to fail_fast is not shown when --debug is not specified. assert "Skipping due to fail_fast" not in result.output class TestBuildFailFastDebug(BaseConfigProject): def test_build(self, runner, project): runner.invoke(cli, ["deps"]) result = runner.invoke(cli, ["build", "--fail-fast", "--debug"]) # 1 seed, 1 model, 2 data tests assert "PASS=4" in result.output # 2 data tests assert "ERROR=2" in result.output # Singular test assert "WARN=1" in result.output # 1 snapshot assert "SKIP=1" in result.output # Skipping due to fail_fast is shown when --debug is specified. assert "Skipping due to fail_fast" in result.output class TestDocsGenerate(BaseConfigProject): def test_docs_generate(self, runner, project): runner.invoke(cli, ["deps"]) result = runner.invoke(cli, ["docs", "generate"]) assert "Building catalog" in result.output assert "Catalog written" in result.output ================================================ FILE: tests/functional/model_config/test_freshness_config.py ================================================ import pytest from dbt.tests.util import run_dbt from dbt_common.dataclass_schema import ValidationError # Seed data for source tables seeds__source_table_csv = """id,_loaded_at 1,2024-03-20 00:00:00 2,2024-03-20 00:00:00 3,2024-03-20 00:00:00 """ models__no_freshness_sql = """ select 1 as id """ # Scenario 2: Model freshness defined with just model freshness spec models__model_freshness_schema_yml = """ version: 2 sources: - name: my_source database: "{{ target.database }}" schema: "{{ target.schema }}" config: freshness: warn_after: {count: 24, period: hour} error_after: {count: 48, period: hour} loaded_at_field: _loaded_at tables: - name: source_table identifier: source_table models: - name: model_a description: Model with no freshness defined - name: model_b description: Model with only model freshness defined config: freshness: build_after: count: 1 period: day updates_on: all - name: model_c description: Model with only source freshness defined config: freshness: warn_after: {count: 24, period: hour} error_after: {count: 48, period: hour} loaded_at_field: _loaded_at tables: - name: source_table identifier: source_table """ models__model_freshness_sql = """ select 1 as id """ models__model_freshness_sql_inline = """ {{ config( materialized='table', freshness={ 'warn_after': {'count': 24, 'period': 'hour'} } ) }} select 1 as id """ models__source_freshness_sql = """ select * from {{ source('my_source', 'source_table') }} """ models__both_freshness_sql = """ select * from {{ source('my_source', 'source_table') }} """ models__model_freshness_schema_yml_build_after_only = """ models: - name: model_a description: Model with only model freshness defined config: freshness: build_after: updates_on: all """ models__model_freshness_schema_yml_build_period_requires_count = """ models: - name: model_a description: Model with only model freshness defined config: freshness: build_after: period: day """ models__model_freshness_schema_yml_build_period_has_0_count = """ models: - name: model_a description: Model with only model freshness defined config: freshness: build_after: period: day count: 0 """ models__model_freshness_schema_yml_build_count_requires_period = """ models: - name: model_a description: Model with only model freshness defined config: freshness: build_after: count: 1 """ class TestModelFreshnessConfig: @pytest.fixture(scope="class") def models(self): return { "schema.yml": models__model_freshness_schema_yml, "model_a.sql": models__no_freshness_sql, "model_b.sql": models__model_freshness_sql, "model_c.sql": models__source_freshness_sql, "model_d.sql": models__both_freshness_sql, "model_e.sql": models__model_freshness_sql_inline, } def test_model_freshness_configs(self, project): run_dbt(["parse"]) compile_results = run_dbt(["compile"]) assert len(compile_results) == 5 # All 4 models compiled successfully class TestModelFreshnessConfigParseBuildAfterOnly: @pytest.fixture(scope="class") def models(self): return { "schema.yml": models__model_freshness_schema_yml_build_after_only, "model_a.sql": models__no_freshness_sql, } def test_model_freshness_configs(self, project): run_dbt(["parse"]) class TestModelFreshnessConfigParseBuildPeriodRequiresCount: @pytest.fixture(scope="class") def models(self): return { "schema.yml": models__model_freshness_schema_yml_build_period_requires_count, "model_a.sql": models__no_freshness_sql, } def test_model_freshness_configs(self, project): with pytest.raises(ValidationError) as excinfo: run_dbt(["parse"]) expected_msg = ( "`freshness.build_after` must have a value for `count` if a `period` is provided" ) assert expected_msg in str(excinfo.value) class TestModelFreshnessConfigParseBuildPeriodHas0Count: @pytest.fixture(scope="class") def models(self): return { "schema.yml": models__model_freshness_schema_yml_build_period_has_0_count, "model_a.sql": models__no_freshness_sql, } def test_model_freshness_configs(self, project): run_dbt(["parse"]) class TestModelFreshnessConfigParseBuildCountRequiresPeriod: @pytest.fixture(scope="class") def models(self): return { "schema.yml": models__model_freshness_schema_yml_build_count_requires_period, "model_a.sql": models__no_freshness_sql, } def test_model_freshness_configs(self, project): with pytest.raises(ValidationError) as excinfo: run_dbt(["parse"]) expected_msg = ( "`freshness.build_after` must have a value for `period` if a `count` is provided" ) assert expected_msg in str(excinfo.value) ================================================ FILE: tests/functional/partial_parsing/fixtures.py ================================================ local_dependency__dbt_project_yml = """ name: 'local_dep' version: '1.0' config-version: 2 profile: 'default' model-paths: ["models"] analysis-paths: ["analyses"] test-paths: ["tests"] seed-paths: ["seeds"] macro-paths: ["macros"] require-dbt-version: '>=0.1.0' target-path: "target" # directory which will store compiled SQL files clean-targets: # directories to be removed by `dbt clean` - "target" - "dbt_packages" seeds: quote_columns: False """ local_dependency__models__schema_yml = """ sources: - name: seed_source schema: "{{ var('schema_override', target.schema) }}" tables: - name: "seed" columns: - name: id data_tests: - unique """ local_dependency__models__model_to_import_sql = """ select * from {{ ref('seed') }} """ local_dependency__macros__dep_macro_sql = """ {% macro some_overridden_macro() -%} 100 {%- endmacro %} """ local_dependency__seeds__seed_csv = """id 1 """ empty_schema_with_version_yml = """ """ schema_sources5_yml = """ sources: - name: seed_sources schema: "{{ target.schema }}" tables: - name: raw_customers columns: - name: id data_tests: - not_null: severity: "{{ 'error' if target.name == 'prod' else 'warn' }}" - unique - name: first_name - name: last_name - name: email seeds: - name: rad_customers description: "Raw customer data" columns: - name: id data_tests: - unique - not_null - name: first_name - name: last_name - name: email """ my_macro2_sql = """ {% macro do_something(foo2, bar2) %} select 'foo' as foo2, 'var' as bar2 {% endmacro %} """ raw_customers_csv = """id,first_name,last_name,email 1,Michael,Perez,mperez0@chronoengine.com 2,Shawn,Mccoy,smccoy1@reddit.com 3,Kathleen,Payne,kpayne2@cargocollective.com 4,Jimmy,Cooper,jcooper3@cargocollective.com 5,Katherine,Rice,krice4@typepad.com 6,Sarah,Ryan,sryan5@gnu.org 7,Martin,Mcdonald,mmcdonald6@opera.com 8,Frank,Robinson,frobinson7@wunderground.com 9,Jennifer,Franklin,jfranklin8@mail.ru 10,Henry,Welch,hwelch9@list-manage.com """ model_three_disabled2_sql = """ - Disabled model {{ config(materialized='table', enabled=False) }} with source_data as ( select 1 as id union all select null as id ) select * from source_data """ schema_sources4_yml = """ sources: - name: seed_sources schema: "{{ target.schema }}" tables: - name: raw_customers columns: - name: id data_tests: - not_null: severity: "{{ 'error' if target.name == 'prod' else 'warn' }}" - unique - every_value_is_blue - name: first_name - name: last_name - name: email seeds: - name: raw_customers description: "Raw customer data" columns: - name: id data_tests: - unique - not_null - name: first_name - name: last_name - name: email """ env_var_schema_yml = """ models: - name: model_one config: materialized: "{{ env_var('TEST_SCHEMA_VAR') }}" """ my_test_sql = """ select * from {{ ref('customers') }} where first_name = '{{ macro_something() }}' """ empty_schema_yml = """ """ schema_models_c_yml = """ sources: - name: seed_source description: "This is a source override" overrides: local_dep schema: "{{ var('schema_override', target.schema) }}" tables: - name: "seed" columns: - name: id data_tests: - unique - not_null """ env_var_sources_yml = """ sources: - name: seed_sources schema: "{{ target.schema }}" database: "{{ env_var('ENV_VAR_DATABASE') }}" tables: - name: raw_customers columns: - name: id data_tests: - not_null: severity: "{{ env_var('ENV_VAR_SEVERITY') }}" - unique - name: first_name - name: last_name - name: email """ generic_test_edited_sql = """ {% test is_odd(model, column_name) %} with validation as ( select {{ column_name }} as odd_field2 from {{ model }} ), validation_errors as ( select odd_field2 from validation -- if this is true, then odd_field is actually even! where (odd_field2 % 2) = 0 ) select * from validation_errors {% endtest %} """ schema_sources1_yml = """ sources: - name: seed_sources schema: "{{ target.schema }}" tables: - name: raw_customers columns: - name: id data_tests: - not_null: severity: "{{ 'error' if target.name == 'prod' else 'warn' }}" - unique - name: first_name - name: last_name - name: email """ schema_sources3_yml = """ sources: - name: seed_sources schema: "{{ target.schema }}" tables: - name: raw_customers columns: - name: id data_tests: - not_null: severity: "{{ 'error' if target.name == 'prod' else 'warn' }}" - unique - name: first_name - name: last_name - name: email exposures: - name: proxy_for_dashboard description: "This is for the XXX dashboard" type: "dashboard" owner: name: "Dashboard Tester" email: "tester@dashboard.com" depends_on: - ref("model_one") - source("seed_sources", "raw_customers") """ my_analysis_sql = """ select * from customers """ schema_sources2_yml = """ sources: - name: seed_sources schema: "{{ target.schema }}" tables: - name: raw_customers columns: - name: id data_tests: - not_null: severity: "{{ 'error' if target.name == 'prod' else 'warn' }}" - unique - name: first_name - name: last_name - name: email exposures: - name: proxy_for_dashboard description: "This is for the XXX dashboard" type: "dashboard" owner: name: "Dashboard Tester" email: "tester@dashboard.com" depends_on: - ref("model_one") - ref("raw_customers") - source("seed_sources", "raw_customers") """ model_color_sql = """ select 'blue' as fun """ my_metric_yml = """ metrics: - name: new_customers label: New Customers model: customers description: "The number of paid customers who are using the product" type: simple type_params: measure: name: customers filter: "{{ Dimension('id__loves_dbt') }} is true" +meta: is_okr: True tags: - okrs """ env_var_schema2_yml = """ models: - name: model_one config: materialized: "{{ env_var('TEST_SCHEMA_VAR') }}" data_tests: - check_color: column_name: fun color: "env_var('ENV_VAR_COLOR')" """ gsm_override_sql = """ - custom macro {% macro generate_schema_name(schema_name, node) %} {{ schema_name }}_{{ target.schema }} {% endmacro %} """ model_four1_sql = """ select * from {{ ref('model_three') }} """ model_one_sql = """ select 1 as fun """ metricflow_time_spine_sql = """ SELECT to_date('02/20/2023', 'mm/dd/yyyy') as date_day """ env_var_schema3_yml = """ models: - name: model_one config: materialized: "{{ env_var('TEST_SCHEMA_VAR') }}" data_tests: - check_color: column_name: fun color: "env_var('ENV_VAR_COLOR')" exposures: - name: proxy_for_dashboard description: "This is for the XXX dashboard" type: "dashboard" owner: name: "{{ env_var('ENV_VAR_OWNER') }}" email: "tester@dashboard.com" depends_on: - ref("model_color") - source("seed_sources", "raw_customers") """ people_semantic_models_yml = """ version: 2 semantic_models: - name: semantic_people model: ref('people') dimensions: - name: favorite_color type: categorical - name: created_at type: TIME type_params: time_granularity: day measures: - name: years_tenure agg: SUM expr: tenure - name: people agg: count expr: id entities: - name: id type: primary defaults: agg_time_dimension: created_at """ people_sl_yml = """ version: 2 semantic_models: - name: semantic_people model: ref('people') dimensions: - name: favorite_color type: categorical - name: created_at type: TIME type_params: time_granularity: day measures: - name: years_tenure agg: SUM expr: tenure - name: people agg: count expr: id entities: - name: id type: primary defaults: agg_time_dimension: created_at metrics: - name: number_of_people description: Total count of people label: "Number of people" type: simple type_params: measure: people meta: my_meta: 'testing' - name: collective_tenure description: Total number of years of team experience label: "Collective tenure" type: simple type_params: measure: name: years_tenure filter: "{{ Dimension('id__loves_dbt') }} is true" - name: average_tenure label: Average Tenure type: ratio type_params: numerator: collective_tenure denominator: number_of_people """ env_var_metrics_yml = """ metrics: - name: number_of_people description: Total count of people label: "Number of people" type: simple type_params: measure: people meta: my_meta: '{{ env_var("ENV_VAR_METRICS") }}' - name: collective_tenure description: Total number of years of team experience label: "Collective tenure" type: simple type_params: measure: name: years_tenure filter: "{{ Dimension('id__loves_dbt') }} is true" """ customers_sql = """ with source as ( select * from {{ source('seed_sources', 'raw_customers') }} ), renamed as ( select id as customer_id, first_name, last_name, email from source ) select * from renamed """ customers_yml = """ models: - name: customers description: "This table contains customer data" """ model_four2_sql = """ select fun from {{ ref('model_one') }} """ env_var_model_sql = """ select '{{ env_var('ENV_VAR_TEST') }}' as vartest """ env_var_model_one_sql = """ select 'blue' as fun """ custom_schema_tests2_sql = """ {% test type_one(model) %} select * from ( select * from {{ model }} union all select * from {{ ref('model_b') }} ) as Foo {% endtest %} {% test type_two(model) %} {{ config(severity = "ERROR") }} select * from {{ model }} {% endtest %} """ metric_model_a_sql = """ {% set metric_list = [ metric('number_of_people'), metric('collective_tenure') ] %} {% if not execute %} {% set metric_names = [] %} {% for m in metric_list %} {% do metric_names.append(m.metric_name) %} {% endfor %} -- this config does nothing, but it lets us check these values {{ config(metric_names = metric_names) }} {% endif %} select 1 as fun """ model_b_sql = """ select 1 as notfun """ customers2_md = """ {% docs customer_table %} LOTS of customer data {% enddocs %} """ custom_schema_tests1_sql = """ {% test type_one(model) %} select * from ( select * from {{ model }} union all select * from {{ ref('model_b') }} ) as Foo {% endtest %} {% test type_two(model) %} {{ config(severity = "WARN") }} select * from {{ model }} {% endtest %} """ people_metrics_yml = """ metrics: - name: number_of_people description: Total count of people label: "Number of people" type: simple type_params: measure: people meta: my_meta: 'testing' - name: collective_tenure description: Total number of years of team experience label: "Collective tenure" type: simple type_params: measure: name: years_tenure filter: "{{ Dimension('id__loves_dbt') }} is true" """ people_sql = """ select 1 as id, 'Drew' as first_name, 'Banin' as last_name, 'yellow' as favorite_color, true as loves_dbt, 5 as tenure, current_timestamp as created_at union all select 1 as id, 'Jeremy' as first_name, 'Cohen' as last_name, 'indigo' as favorite_color, true as loves_dbt, 4 as tenure, current_timestamp as created_at """ orders_sql = """ select 1 as id, 101 as user_id, 'pending' as status """ orders_sql_modified = """ select 1 as id, 101 as user_id, 'completed' as status """ orders_singular_test_sql = """ select * from {{ ref('orders') }} where status = 'invalid' """ orders_downstream_sql = """ select * from {{ ref('orders') }} """ model_a_sql = """ select 1 as fun """ model_three_disabled_sql = """ {{ config(materialized='table', enabled=False) }} with source_data as ( select 1 as id union all select null as id ) select * from source_data """ models_schema2b_yml = """ models: - name: model_one description: "The first model" - name: model_three description: "The third model" columns: - name: id data_tests: - not_null """ env_var_macros_yml = """ macros: - name: do_something description: "This is a test macro" meta: some_key: "{{ env_var('ENV_VAR_SOME_KEY') }}" """ models_schema4_yml = """ models: - name: model_one description: "The first model" - name: model_three description: "The third model" config: enabled: false columns: - name: id data_tests: - unique """ model_two_sql = """ select 1 as notfun """ model_two_sql_missing_space = """ select1asnotfun """ model_two_sql_extra_whitespace = """ select 1 as notfun """ model_two_disabled_sql = """ {{ config( enabled=false ) }} select 1 as notfun """ generic_test_schema_yml = """ models: - name: orders description: "Some order data" columns: - name: id data_tests: - unique - is_odd """ customers1_md = """ {% docs customer_table %} This table contains customer data {% enddocs %} """ model_three_modified_sql = """ {{ config(materialized='table') }} with source_data as ( {#- This is model three #} select 1 as id union all select null as id ) select * from source_data """ macros_yml = """ macros: - name: do_something description: "This is a test macro" """ test_color_sql = """ {% test check_color(model, column_name, color) %} select * from {{ model }} where {{ column_name }} = '{{ color }}' {% endtest %} """ models_schema2_yml = """ models: - name: model_one description: "The first model" - name: model_three description: "The third model" columns: - name: id data_tests: - unique """ gsm_override2_sql = """ - custom macro xxxx {% macro generate_schema_name(schema_name, node) %} {{ schema_name }}_{{ target.schema }} {% endmacro %} """ models_schema3_yml = """ models: - name: model_one description: "The first model" - name: model_three description: "The third model" data_tests: - unique macros: - name: do_something description: "This is a test macro" """ generic_test_sql = """ {% test is_odd(model, column_name) %} with validation as ( select {{ column_name }} as odd_field from {{ model }} ), validation_errors as ( select odd_field from validation -- if this is true, then odd_field is actually even! where (odd_field % 2) = 0 ) select * from validation_errors {% endtest %} """ env_var_model_test_yml = """ models: - name: model_color columns: - name: fun data_tests: - unique: enabled: "{{ env_var('ENV_VAR_ENABLED', True) }}" """ model_three_sql = """ {{ config(materialized='table') }} with source_data as ( select 1 as id union all select null as id ) select * from source_data """ ref_override2_sql = """ - Macro to override ref xxxx {% macro ref(modelname) %} {% do return(builtins.ref(modelname)) %} {% endmacro %} """ models_schema1_yml = """ models: - name: model_one description: "The first model" """ macros_schema_yml = """ models: - name: model_a data_tests: - type_one - type_two """ models_versions_schema_yml = """ models: - name: model_one description: "The first model" versions: - v: 1 - v: 2 """ models_versions_defined_in_schema_yml = """ models: - name: model_one description: "The first model" versions: - v: 1 - v: 2 defined_in: model_one_different """ models_versions_updated_schema_yml = """ models: - name: model_one latest_version: 1 description: "The first model" versions: - v: 1 - v: 2 defined_in: model_one_different """ my_macro_sql = """ {% macro do_something(foo2, bar2) %} select '{{ foo2 }}' as foo2, '{{ bar2 }}' as bar2 {% endmacro %} """ snapshot_sql = """ {% snapshot orders_snapshot %} {{ config( target_schema=schema, strategy='check', unique_key='id', check_cols=['status'], ) }} select * from {{ ref('orders') }} {% endsnapshot %} {% snapshot orders2_snapshot %} {{ config( target_schema=schema, strategy='check', unique_key='id', check_cols=['order_date'], ) }} select * from {{ ref('orders') }} {% endsnapshot %} """ models_schema4b_yml = """ models: - name: model_one description: "The first model" - name: model_three description: "The third model" config: enabled: true columns: - name: id data_tests: - unique """ test_macro_sql = """ {% macro macro_something() %} {% do return('macro_something') %} {% endmacro %} """ people_metrics2_yml = """ metrics: - name: number_of_people description: Total count of people label: "Number of people" type: simple type_params: measure: people meta: my_meta: 'replaced' - name: collective_tenure description: Total number of years of team experience label: "Collective tenure" type: simple type_params: measure: name: years_tenure filter: "{{ Dimension('id__loves_dbt') }} is true" """ generic_schema_yml = """ models: - name: orders description: "Some order data" columns: - name: id data_tests: - unique """ groups_schema_yml_one_group = """ groups: - name: test_group owner: name: test_group_owner models: - name: orders description: "Some order data" """ groups_schema_yml_two_groups = """ groups: - name: test_group owner: name: test_group_owner - name: test_group2 owner: name: test_group_owner2 models: - name: orders description: "Some order data" """ groups_schema_yml_two_groups_private_orders_valid_access = """ groups: - name: test_group owner: name: test_group_owner - name: test_group2 owner: name: test_group_owner2 models: - name: orders group: test_group access: private description: "Some order data" - name: orders_downstream group: test_group description: "Some order data" """ groups_schema_yml_two_groups_private_orders_invalid_access = """ groups: - name: test_group owner: name: test_group_owner - name: test_group2 owner: name: test_group_owner2 models: - name: orders group: test_group2 access: private description: "Some order data" - name: orders_downstream group: test_group description: "Some order data" """ groups_schema_yml_one_group_model_in_group2 = """ groups: - name: test_group owner: name: test_group_owner models: - name: orders description: "Some order data" config: group: test_group2 """ groups_schema_yml_two_groups_edited = """ groups: - name: test_group owner: name: test_group_owner - name: test_group2_edited owner: name: test_group_owner2 models: - name: orders description: "Some order data" """ snapshot2_sql = """ - add a comment {% snapshot orders_snapshot %} {{ config( target_schema=schema, strategy='check', unique_key='id', check_cols=['status'], ) }} select * from {{ ref('orders') }} {% endsnapshot %} {% snapshot orders2_snapshot %} {{ config( target_schema=schema, strategy='check', unique_key='id', check_cols=['order_date'], ) }} select * from {{ ref('orders') }} {% endsnapshot %} """ sources_tests2_sql = """ {% test every_value_is_blue(model, column_name) %} select * from {{ model }} where {{ column_name }} != 99 {% endtest %} """ people_metrics3_yml = """ metrics: - name: number_of_people description: Total count of people label: "Number of people" type: simple type_params: measure: people meta: my_meta: 'replaced' """ ref_override_sql = """ - Macro to override ref {% macro ref(modelname) %} {% do return(builtins.ref(modelname)) %} {% endmacro %} """ test_macro2_sql = """ {% macro macro_something() %} {% do return('some_name') %} {% endmacro %} """ env_var_macro_sql = """ {% macro do_something(foo2, bar2) %} select '{{ foo2 }}' as foo2, '{{ bar2 }}' as bar2 {% endmacro %} """ sources_tests1_sql = """ {% test every_value_is_blue(model, column_name) %} select * from {{ model }} where {{ column_name }} = 9999 {% endtest %} """ macros_sql = """ {% macro foo() %} foo {% endmacro %} {% macro bar() %} bar {% endmacro %} """ macros_schema1_yml = """ macros: - name: foo description: Lorem. - name: bar description: Lorem. """ macros_schema2_yml = """ macros: - name: foo description: Lorem. - name: bar description: Lorem ipsum. """ my_func_sql = """ value * 2 """ my_func_yml = """ functions: - name: my_func description: "Doubles an integer" arguments: - name: value data_type: int description: "An integer to be doubled" returns: data_type: int """ updated_my_func_sql = """ number * 2.0 """ updated_my_func_yml = """ functions: - name: my_func description: "Doubles a float" arguments: - name: number data_type: float description: "A float to be doubled" returns: data_type: float """ model_using_function_sql = """ SELECT {{ function('my_func') }}(1) as result """ ================================================ FILE: tests/functional/partial_parsing/test_file_diff.py ================================================ import os import pytest from dbt.tests.util import run_dbt, write_artifact, write_file from tests.functional.partial_parsing.fixtures import model_one_sql, model_two_sql first_file_diff = { "deleted": [], "changed": [], "added": [{"path": "models/model_one.sql", "content": "select 1 as fun"}], } second_file_diff = { "deleted": [], "changed": [], "added": [{"path": "models/model_two.sql", "content": "select 123 as notfun"}], } class TestFileDiffPaths: def test_file_diffs(self, project): os.environ["DBT_PP_FILE_DIFF_TEST"] = "true" run_dbt(["deps"]) run_dbt(["seed"]) # We start with an empty project results = run_dbt() write_artifact(first_file_diff, "file_diff.json") results = run_dbt() assert len(results) == 1 write_artifact(second_file_diff, "file_diff.json") results = run_dbt() assert len(results) == 2 class TestFileDiffs: @pytest.fixture(scope="class") def models(self): return { "model_one.sql": model_one_sql, } def test_no_file_diffs(self, project): # We start with a project with one model manifest = run_dbt(["parse"]) assert len(manifest.nodes) == 1 # add a model file write_file(model_two_sql, project.project_root, "models", "model_two.sql") # parse without computing a file diff manifest = run_dbt(["--partial-parse", "--no-partial-parse-file-diff", "parse"]) assert len(manifest.nodes) == 1 # default behaviour - parse with computing a file diff manifest = run_dbt(["--partial-parse", "parse"]) assert len(manifest.nodes) == 2 ================================================ FILE: tests/functional/partial_parsing/test_partial_parsing.py ================================================ import os import re from argparse import Namespace from unittest import mock import pytest import yaml import dbt.flags as flags from dbt.contracts.files import ParseFileType from dbt.contracts.results import TestStatus from dbt.events.types import UnableToPartialParse from dbt.exceptions import CompilationError from dbt.plugins.manifest import ModelNodeArgs, PluginNodes from dbt.tests.fixtures.project import write_project_files from dbt.tests.util import ( get_manifest, rename_dir, rm_file, run_dbt, run_dbt_and_capture, write_file, ) from dbt_common.events.event_catcher import EventCatcher from tests.functional.partial_parsing.fixtures import ( custom_schema_tests1_sql, custom_schema_tests2_sql, customers1_md, customers2_md, customers_sql, empty_schema_with_version_yml, empty_schema_yml, generic_schema_yml, generic_test_edited_sql, generic_test_schema_yml, generic_test_sql, gsm_override2_sql, gsm_override_sql, local_dependency__dbt_project_yml, local_dependency__macros__dep_macro_sql, local_dependency__models__model_to_import_sql, local_dependency__models__schema_yml, local_dependency__seeds__seed_csv, macros_schema1_yml, macros_schema2_yml, macros_schema_yml, macros_sql, macros_yml, model_a_sql, model_b_sql, model_four1_sql, model_four2_sql, model_one_sql, model_three_disabled2_sql, model_three_disabled_sql, model_three_modified_sql, model_three_sql, model_two_disabled_sql, model_two_sql, model_two_sql_extra_whitespace, model_two_sql_missing_space, models_schema1_yml, models_schema2_yml, models_schema2b_yml, models_schema3_yml, models_schema4_yml, models_schema4b_yml, my_analysis_sql, my_macro2_sql, my_macro_sql, my_test_sql, orders_singular_test_sql, orders_sql, orders_sql_modified, raw_customers_csv, ref_override2_sql, ref_override_sql, schema_models_c_yml, schema_sources1_yml, schema_sources2_yml, schema_sources3_yml, schema_sources4_yml, schema_sources5_yml, snapshot2_sql, snapshot_sql, sources_tests1_sql, sources_tests2_sql, test_macro2_sql, test_macro_sql, ) from tests.functional.utils import up_one os.environ["DBT_PP_TEST"] = "true" def normalize(path): return os.path.normcase(os.path.normpath(path)) class TestModels: @pytest.fixture(scope="class") def models(self): return { "model_one.sql": model_one_sql, } def test_pp_models(self, project): # initial run # run_dbt(['clean']) results = run_dbt(["run"]) assert len(results) == 1 # add a model file with missing space write_file(model_two_sql_missing_space, project.project_root, "models", "model_two.sql") run_dbt(["--partial-parse", "run"], expect_pass=False) # update model file - fix missing space issue write_file(model_two_sql, project.project_root, "models", "model_two.sql") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 2 # update model file - add additional spaces, should not change prior checksum manifest = get_manifest(project.project_root) model_two_checksum = manifest.nodes["model.test.model_two"].checksum write_file(model_two_sql_extra_whitespace, project.project_root, "models", "model_two.sql") manifest = run_dbt(["--partial-parse", "parse"]) assert manifest.nodes["model.test.model_two"].checksum == model_two_checksum # add a schema file write_file(models_schema1_yml, project.project_root, "models", "schema.yml") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 2 manifest = get_manifest(project.project_root) assert "model.test.model_one" in manifest.nodes model_one_node = manifest.nodes["model.test.model_one"] assert model_one_node.description == "The first model" assert model_one_node.patch_path == "test://" + normalize("models/schema.yml") # add a model and a schema file (with a test) at the same time write_file(models_schema2_yml, project.project_root, "models", "schema.yml") write_file(model_three_sql, project.project_root, "models", "model_three.sql") results = run_dbt(["--partial-parse", "test"], expect_pass=False) assert len(results) == 1 manifest = get_manifest(project.project_root) project_files = [f for f in manifest.files if f.startswith("test://")] assert len(project_files) == 4 model_3_file_id = "test://" + normalize("models/model_three.sql") assert model_3_file_id in manifest.files model_three_file = manifest.files[model_3_file_id] assert model_three_file.parse_file_type == ParseFileType.Model assert type(model_three_file).__name__ == "SourceFile" model_three_node = manifest.nodes[model_three_file.nodes[0]] schema_file_id = "test://" + normalize("models/schema.yml") assert model_three_node.patch_path == schema_file_id assert model_three_node.description == "The third model" schema_file = manifest.files[schema_file_id] assert type(schema_file).__name__ == "SchemaSourceFile" assert len(schema_file.data_tests) == 1 tests = schema_file.get_all_test_ids() assert tests == ["test.test.unique_model_three_id.6776ac8160"] unique_test_id = tests[0] assert unique_test_id in manifest.nodes # modify model sql file, ensure description still there write_file(model_three_modified_sql, project.project_root, "models", "model_three.sql") results = run_dbt(["--partial-parse", "run"]) manifest = get_manifest(project.project_root) model_id = "model.test.model_three" assert model_id in manifest.nodes model_three_node = manifest.nodes[model_id] assert model_three_node.description == "The third model" # Change the model 3 test from unique to not_null write_file(models_schema2b_yml, project.project_root, "models", "schema.yml") results = run_dbt(["--partial-parse", "test"], expect_pass=False) manifest = get_manifest(project.project_root) schema_file_id = "test://" + normalize("models/schema.yml") schema_file = manifest.files[schema_file_id] tests = schema_file.get_all_test_ids() assert tests == ["test.test.not_null_model_three_id.3162ce0a6f"] not_null_test_id = tests[0] assert not_null_test_id in manifest.nodes.keys() assert unique_test_id not in manifest.nodes.keys() assert len(results) == 1 # go back to previous version of schema file, removing patch, test, and model for model three write_file(models_schema1_yml, project.project_root, "models", "schema.yml") rm_file(project.project_root, "models", "model_three.sql") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 2 # remove schema file, still have 3 models write_file(model_three_sql, project.project_root, "models", "model_three.sql") rm_file(project.project_root, "models", "schema.yml") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 3 manifest = get_manifest(project.project_root) schema_file_id = "test://" + normalize("models/schema.yml") assert schema_file_id not in manifest.files project_files = [f for f in manifest.files if f.startswith("test://")] assert len(project_files) == 3 # Put schema file back and remove a model # referred to in schema file write_file(models_schema2_yml, project.project_root, "models", "schema.yml") rm_file(project.project_root, "models", "model_three.sql") with pytest.raises(CompilationError): results = run_dbt(["--partial-parse", "--warn-error", "run"]) # Put model back again write_file(model_three_sql, project.project_root, "models", "model_three.sql") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 3 # Add model four refing model three write_file(model_four1_sql, project.project_root, "models", "model_four.sql") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 4 # Remove model_three and change model_four to ref model_one # and change schema file to remove model_three rm_file(project.project_root, "models", "model_three.sql") write_file(model_four2_sql, project.project_root, "models", "model_four.sql") write_file(models_schema1_yml, project.project_root, "models", "schema.yml") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 3 # Remove model four, put back model three, put back schema file write_file(model_three_sql, project.project_root, "models", "model_three.sql") write_file(models_schema2_yml, project.project_root, "models", "schema.yml") rm_file(project.project_root, "models", "model_four.sql") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 3 # disable model three in the schema file write_file(models_schema4_yml, project.project_root, "models", "schema.yml") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 2 # update enabled config to be true for model three in the schema file write_file(models_schema4b_yml, project.project_root, "models", "schema.yml") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 3 # disable model three in the schema file again write_file(models_schema4_yml, project.project_root, "models", "schema.yml") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 2 # remove disabled config for model three in the schema file to check it gets enabled write_file(models_schema4b_yml, project.project_root, "models", "schema.yml") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 3 # Add a macro write_file(my_macro_sql, project.project_root, "macros", "my_macro.sql") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 3 manifest = get_manifest(project.project_root) macro_id = "macro.test.do_something" assert macro_id in manifest.macros # Modify the macro write_file(my_macro2_sql, project.project_root, "macros", "my_macro.sql") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 3 # Add a macro patch write_file(models_schema3_yml, project.project_root, "models", "schema.yml") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 3 # Remove the macro rm_file(project.project_root, "macros", "my_macro.sql") with pytest.raises(CompilationError): results = run_dbt(["--partial-parse", "--warn-error", "run"]) # put back macro file, go back to schema file with no macro # add separate macro patch schema file write_file(models_schema2_yml, project.project_root, "models", "schema.yml") write_file(my_macro_sql, project.project_root, "macros", "my_macro.sql") write_file(macros_yml, project.project_root, "macros", "macros.yml") results = run_dbt(["--partial-parse", "run"]) # add macro file with two macros and schema file with patches for both write_file(macros_schema1_yml, project.project_root, "macros", "macros_x.yml") write_file(macros_sql, project.project_root, "macros", "macros_x.sql") results = run_dbt(["--partial-parse", "parse"]) # modify one of the patches write_file(macros_schema2_yml, project.project_root, "macros", "macros_x.yml") results = run_dbt(["--partial-parse", "parse"]) rm_file(project.project_root, "macros", "macros_x.sql") rm_file(project.project_root, "macros", "macros_x.yml") # delete macro and schema file rm_file(project.project_root, "macros", "my_macro.sql") rm_file(project.project_root, "macros", "macros.yml") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 3 # Add an empty schema file write_file(empty_schema_yml, project.project_root, "models", "eschema.yml") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 3 # Add version to empty schema file write_file(empty_schema_with_version_yml, project.project_root, "models", "eschema.yml") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 3 # Disable model_three write_file(model_three_disabled_sql, project.project_root, "models", "model_three.sql") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 2 manifest = get_manifest(project.project_root) model_id = "model.test.model_three" assert model_id in manifest.disabled assert model_id not in manifest.nodes # Edit disabled model three write_file(model_three_disabled2_sql, project.project_root, "models", "model_three.sql") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 2 manifest = get_manifest(project.project_root) model_id = "model.test.model_three" assert model_id in manifest.disabled assert model_id not in manifest.nodes # Remove disabled from model three write_file(model_three_sql, project.project_root, "models", "model_three.sql") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 3 manifest = get_manifest(project.project_root) model_id = "model.test.model_three" assert model_id in manifest.nodes assert model_id not in manifest.disabled class TestSources: @pytest.fixture(scope="class") def models(self): return { "model_one.sql": model_one_sql, } def test_pp_sources(self, project): # initial run write_file(raw_customers_csv, project.project_root, "seeds", "raw_customers.csv") write_file(sources_tests1_sql, project.project_root, "macros", "tests.sql") results = run_dbt(["run"]) assert len(results) == 1 # Partial parse running 'seed' run_dbt(["--partial-parse", "seed"]) manifest = get_manifest(project.project_root) seed_file_id = "test://" + normalize("seeds/raw_customers.csv") assert seed_file_id in manifest.files # Add another seed file write_file(raw_customers_csv, project.project_root, "seeds", "more_customers.csv") run_dbt(["--partial-parse", "run"]) seed_file_id = "test://" + normalize("seeds/more_customers.csv") manifest = get_manifest(project.project_root) assert seed_file_id in manifest.files seed_id = "seed.test.more_customers" assert seed_id in manifest.nodes # Remove seed file and add a schema files with a source referring to raw_customers rm_file(project.project_root, "seeds", "more_customers.csv") write_file(schema_sources1_yml, project.project_root, "models", "sources.yml") results = run_dbt(["--partial-parse", "run"]) manifest = get_manifest(project.project_root) assert len(manifest.sources) == 1 file_id = "test://" + normalize("models/sources.yml") assert file_id in manifest.files # add a model referring to raw_customers source write_file(customers_sql, project.project_root, "models", "customers.sql") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 2 # remove sources schema file rm_file(project.project_root, "models", "sources.yml") with pytest.raises(CompilationError): results = run_dbt(["--partial-parse", "run"]) # put back sources and add an exposures file write_file(schema_sources2_yml, project.project_root, "models", "sources.yml") results = run_dbt(["--partial-parse", "run"]) # remove seed referenced in exposures file rm_file(project.project_root, "seeds", "raw_customers.csv") with pytest.raises(CompilationError): results = run_dbt(["--partial-parse", "run"]) # put back seed and remove depends_on from exposure write_file(raw_customers_csv, project.project_root, "seeds", "raw_customers.csv") write_file(schema_sources3_yml, project.project_root, "models", "sources.yml") results = run_dbt(["--partial-parse", "run"]) # Add seed config with test to schema.yml, remove exposure write_file(schema_sources4_yml, project.project_root, "models", "sources.yml") results = run_dbt(["--partial-parse", "run"]) # Change seed name to wrong name write_file(schema_sources5_yml, project.project_root, "models", "sources.yml") with pytest.raises(CompilationError): results = run_dbt(["--partial-parse", "--warn-error", "run"]) # Put back seed name to right name write_file(schema_sources4_yml, project.project_root, "models", "sources.yml") results = run_dbt(["--partial-parse", "run"]) # Add docs file customers.md write_file(customers1_md, project.project_root, "models", "customers.md") results = run_dbt(["--partial-parse", "run"]) # Change docs file customers.md write_file(customers2_md, project.project_root, "models", "customers.md") results = run_dbt(["--partial-parse", "run"]) # Delete docs file rm_file(project.project_root, "models", "customers.md") results = run_dbt(["--partial-parse", "run"]) # Add a data test write_file(test_macro_sql, project.project_root, "macros", "test-macro.sql") write_file(my_test_sql, project.project_root, "tests", "my_test.sql") results = run_dbt(["--partial-parse", "test"]) manifest = get_manifest(project.project_root) assert len(manifest.nodes) == 9 test_id = "test.test.my_test" assert test_id in manifest.nodes # Change macro that data test depends on write_file(test_macro2_sql, project.project_root, "macros", "test-macro.sql") results = run_dbt(["--partial-parse", "test"]) manifest = get_manifest(project.project_root) # Add an analysis write_file(my_analysis_sql, project.project_root, "analyses", "my_analysis.sql") results = run_dbt(["--partial-parse", "run"]) manifest = get_manifest(project.project_root) # Remove data test rm_file(project.project_root, "tests", "my_test.sql") results = run_dbt(["--partial-parse", "test"]) manifest = get_manifest(project.project_root) assert len(manifest.nodes) == 9 # Remove analysis rm_file(project.project_root, "analyses", "my_analysis.sql") results = run_dbt(["--partial-parse", "run"]) manifest = get_manifest(project.project_root) assert len(manifest.nodes) == 8 # Change source test write_file(sources_tests2_sql, project.project_root, "macros", "tests.sql") results = run_dbt(["--partial-parse", "run"]) class TestPartialParsingDependency: @pytest.fixture(scope="class") def models(self): return { "model_one.sql": model_one_sql, } @pytest.fixture(scope="class", autouse=True) def setUp(self, project_root): local_dependency_files = { "dbt_project.yml": local_dependency__dbt_project_yml, "models": { "schema.yml": local_dependency__models__schema_yml, "model_to_import.sql": local_dependency__models__model_to_import_sql, }, "macros": {"dep_macro.sql": local_dependency__macros__dep_macro_sql}, "seeds": {"seed.csv": local_dependency__seeds__seed_csv}, } write_project_files(project_root, "local_dependency", local_dependency_files) @pytest.fixture(scope="class") def packages(self): return {"packages": [{"local": "local_dependency"}]} def test_parsing_with_dependency(self, project): run_dbt(["clean"]) run_dbt(["deps"]) run_dbt(["seed"]) run_dbt(["run"]) # Add a source override write_file(schema_models_c_yml, project.project_root, "models", "schema.yml") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 2 manifest = get_manifest(project.project_root) assert len(manifest.sources) == 1 source_id = "source.local_dep.seed_source.seed" assert source_id in manifest.sources # We have 1 root model, 1 local_dep model, 1 local_dep seed, 1 local_dep source test, 2 root source tests assert len(manifest.nodes) == 5 test_id = "test.local_dep.source_unique_seed_source_seed_id.afa94935ed" assert test_id in manifest.nodes # Remove a source override rm_file(project.project_root, "models", "schema.yml") results = run_dbt(["--partial-parse", "run"]) manifest = get_manifest(project.project_root) assert len(manifest.sources) == 1 class TestNestedMacros: @pytest.fixture(scope="class") def models(self): return { "model_a.sql": model_a_sql, "model_b.sql": model_b_sql, "schema.yml": macros_schema_yml, } @pytest.fixture(scope="class") def macros(self): return { "custom_schema_tests.sql": custom_schema_tests1_sql, } def test_nested_macros(self, project): results = run_dbt() assert len(results) == 2 manifest = get_manifest(project.project_root) macro_child_map = manifest.build_macro_child_map() macro_unique_id = "macro.test.test_type_two" assert macro_unique_id in macro_child_map results = run_dbt(["test"], expect_pass=False) results = sorted(results, key=lambda r: r.node.name) assert len(results) == 2 # type_one_model_a_ assert results[0].status == TestStatus.Fail assert re.search(r"union all", results[0].node.compiled_code) # type_two_model_a_ assert results[1].status == TestStatus.Warn assert results[1].node.config.severity == "WARN" write_file( custom_schema_tests2_sql, project.project_root, "macros", "custom_schema_tests.sql" ) results = run_dbt(["--partial-parse", "test"], expect_pass=False) manifest = get_manifest(project.project_root) test_node_id = "test.test.type_two_model_a_.842bc6c2a7" assert test_node_id in manifest.nodes results = sorted(results, key=lambda r: r.node.name) assert len(results) == 2 # type_two_model_a_ assert results[1].status == TestStatus.Fail assert results[1].node.config.severity == "ERROR" class TestSkipMacros: @pytest.fixture(scope="class") def models(self): return { "model_one.sql": model_one_sql, "eschema.yml": empty_schema_yml, } def test_skip_macros(self, project): # initial run so we have a msgpack file # includes empty_schema file for bug #4850 results = run_dbt() # add a new ref override macro write_file(ref_override_sql, project.project_root, "macros", "ref_override.sql") results, log_output = run_dbt_and_capture(["--partial-parse", "run"]) assert "Starting full parse." in log_output # modify a ref override macro write_file(ref_override2_sql, project.project_root, "macros", "ref_override.sql") results, log_output = run_dbt_and_capture(["--partial-parse", "run"]) assert "Starting full parse." in log_output # remove a ref override macro rm_file(project.project_root, "macros", "ref_override.sql") results, log_output = run_dbt_and_capture(["--partial-parse", "run"]) assert "Starting full parse." in log_output # custom generate_schema_name macro write_file(gsm_override_sql, project.project_root, "macros", "gsm_override.sql") results, log_output = run_dbt_and_capture(["--partial-parse", "run"]) assert "Starting full parse." in log_output # change generate_schema_name macro write_file(gsm_override2_sql, project.project_root, "macros", "gsm_override.sql") results, log_output = run_dbt_and_capture(["--partial-parse", "run"]) assert "Starting full parse." in log_output class TestMacroDescriptionUpdate: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": model_one_sql, } @pytest.fixture(scope="class") def macros(self): return { "macros.sql": macros_sql, "schema.yml": macros_schema1_yml, } def test_pp_macro_description_update(self, project): # initial parse run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "macro.test.foo" in manifest.macros assert "macro.test.bar" in manifest.macros assert manifest.macros["macro.test.foo"].description == "Lorem." assert manifest.macros["macro.test.bar"].description == "Lorem." assert manifest.macros["macro.test.foo"].patch_path == "test://" + normalize( "macros/schema.yml" ) assert manifest.macros["macro.test.bar"].patch_path == "test://" + normalize( "macros/schema.yml" ) # edit YAML in macros-path write_file(macros_schema2_yml, project.project_root, "macros", "schema.yml") # parse again run_dbt(["--partial-parse", "parse"]) manifest = get_manifest(project.project_root) assert "macro.test.foo" in manifest.macros assert "macro.test.bar" in manifest.macros assert manifest.macros["macro.test.foo"].description == "Lorem." assert manifest.macros["macro.test.bar"].description == "Lorem ipsum." assert manifest.macros["macro.test.foo"].patch_path == "test://" + normalize( "macros/schema.yml" ) assert manifest.macros["macro.test.bar"].patch_path == "test://" + normalize( "macros/schema.yml" ) # compile run_dbt(["--partial-parse", "compile"]) class TestSnapshots: @pytest.fixture(scope="class") def models(self): return { "orders.sql": orders_sql, } def test_pp_snapshots(self, project): # initial run results = run_dbt() assert len(results) == 1 # add snapshot write_file(snapshot_sql, project.project_root, "snapshots", "snapshot.sql") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 1 manifest = get_manifest(project.project_root) snapshot_id = "snapshot.test.orders_snapshot" assert snapshot_id in manifest.nodes snapshot2_id = "snapshot.test.orders2_snapshot" assert snapshot2_id in manifest.nodes # run snapshot results = run_dbt(["--partial-parse", "snapshot"]) assert len(results) == 2 # modify snapshot write_file(snapshot2_sql, project.project_root, "snapshots", "snapshot.sql") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 1 # delete snapshot rm_file(project.project_root, "snapshots", "snapshot.sql") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 1 class TestGenericTests: @pytest.fixture(scope="class") def models(self): return { "orders.sql": orders_sql, "schema.yml": generic_schema_yml, } @pytest.fixture(scope="class") def tests(self): # Make sure "generic" directory is created return {"generic": {"readme.md": ""}} def test_pp_generic_tests(self, project): # initial run results = run_dbt() assert len(results) == 1 manifest = get_manifest(project.project_root) expected_nodes = ["model.test.orders", "test.test.unique_orders_id.1360ecc70e"] assert expected_nodes == list(manifest.nodes.keys()) # add generic test in test-path write_file(generic_test_sql, project.project_root, "tests", "generic", "generic_test.sql") write_file(generic_test_schema_yml, project.project_root, "models", "schema.yml") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 1 manifest = get_manifest(project.project_root) test_id = "test.test.is_odd_orders_id.82834fdc5b" assert test_id in manifest.nodes expected_nodes = [ "model.test.orders", "test.test.unique_orders_id.1360ecc70e", "test.test.is_odd_orders_id.82834fdc5b", ] assert expected_nodes == list(manifest.nodes.keys()) # edit generic test in test-path write_file( generic_test_edited_sql, project.project_root, "tests", "generic", "generic_test.sql" ) results = run_dbt(["--partial-parse", "run"]) assert len(results) == 1 manifest = get_manifest(project.project_root) test_id = "test.test.is_odd_orders_id.82834fdc5b" assert test_id in manifest.nodes expected_nodes = [ "model.test.orders", "test.test.unique_orders_id.1360ecc70e", "test.test.is_odd_orders_id.82834fdc5b", ] assert expected_nodes == list(manifest.nodes.keys()) class TestSingularTests: @pytest.fixture(scope="class") def models(self): return { "orders.sql": orders_sql, "schema.yml": generic_schema_yml, } @pytest.fixture(scope="class") def tests(self): # Make sure "generic" directory is created return {"generic": {"readme.md": ""}} def test_pp_singular_tests(self, project): # initial run results = run_dbt() assert len(results) == 1 manifest = get_manifest(project.project_root) expected_nodes = ["model.test.orders", "test.test.unique_orders_id.1360ecc70e"] assert expected_nodes == list(manifest.nodes.keys()) # add singular test in test-path write_file(orders_singular_test_sql, project.project_root, "tests", "singular_test.sql") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 1 # modify model being tested by singular test write_file(orders_sql_modified, project.project_root, "models", "orders.sql") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 1 class TestExternalModels: @pytest.fixture(scope="class") def external_model_node(self): return ModelNodeArgs( name="external_model", package_name="external", identifier="test_identifier", schema="test_schema", ) @pytest.fixture(scope="class") def external_model_node_versioned(self): return ModelNodeArgs( name="external_model_versioned", package_name="external", identifier="test_identifier_v1", schema="test_schema", version=1, ) @pytest.fixture(scope="class") def external_model_node_depends_on(self): return ModelNodeArgs( name="external_model_depends_on", package_name="external", identifier="test_identifier_depends_on", schema="test_schema", depends_on_nodes=["model.external.external_model_depends_on_parent"], ) @pytest.fixture(scope="class") def external_model_node_depends_on_parent(self): return ModelNodeArgs( name="external_model_depends_on_parent", package_name="external", identifier="test_identifier_depends_on_parent", schema="test_schema", ) @pytest.fixture(scope="class") def external_model_node_merge(self): return ModelNodeArgs( name="model_two", package_name="test", identifier="test_identifier", schema="test_schema", database="dbt", ) @pytest.fixture(scope="class") def models(self): return {"model_one.sql": model_one_sql} @mock.patch("dbt.plugins.get_plugin_manager") def test_pp_external_models( self, get_plugin_manager, project, external_model_node, external_model_node_versioned, external_model_node_depends_on, external_model_node_depends_on_parent, external_model_node_merge, ): # initial plugin - one external model external_nodes = PluginNodes() external_nodes.add_model(external_model_node) get_plugin_manager.return_value.get_nodes.return_value = external_nodes # initial parse manifest = run_dbt(["parse"]) assert len(manifest.nodes) == 2 assert set(manifest.nodes.keys()) == { "model.external.external_model", "model.test.model_one", } assert len(manifest.external_node_unique_ids) == 1 assert manifest.external_node_unique_ids == ["model.external.external_model"] # add a model file - test.model_two write_file(model_two_sql, project.project_root, "models", "model_two.sql") manifest = run_dbt(["--partial-parse", "parse"]) assert len(manifest.nodes) == 3 # add an external model that is already in project - test.model_two # project model should be preferred to external model external_nodes.add_model(external_model_node_merge) manifest = run_dbt(["--partial-parse", "parse"]) assert len(manifest.nodes) == 3 assert len(manifest.external_node_unique_ids) == 1 # disable test.model_two in project # project models should still be preferred to external model write_file(model_two_disabled_sql, project.project_root, "models", "model_two.sql") manifest = run_dbt(["--partial-parse", "parse"]) assert len(manifest.nodes) == 2 assert len(manifest.disabled) == 1 assert len(manifest.external_node_unique_ids) == 1 # re-enable model_2.sql write_file(model_two_sql, project.project_root, "models", "model_two.sql") # add a new external model external_nodes.add_model(external_model_node_versioned) manifest = run_dbt(["--partial-parse", "parse"]) assert len(manifest.nodes) == 4 assert len(manifest.external_node_unique_ids) == 2 # add a model file that depends on external model write_file( "SELECT * FROM {{ref('external', 'external_model')}}", project.project_root, "models", "model_depends_on_external.sql", ) manifest = run_dbt(["--partial-parse", "parse"]) assert len(manifest.nodes) == 5 assert len(manifest.external_node_unique_ids) == 2 # remove a model file that depends on external model rm_file(project.project_root, "models", "model_depends_on_external.sql") manifest = run_dbt(["--partial-parse", "parse"]) assert len(manifest.nodes) == 4 # add an external node with depends on external_nodes.add_model(external_model_node_depends_on) external_nodes.add_model(external_model_node_depends_on_parent) manifest = run_dbt(["--partial-parse", "parse"]) assert len(manifest.nodes) == 6 assert len(manifest.external_node_unique_ids) == 4 # skip files parsing - ensure no issues run_dbt(["--partial-parse", "parse"]) assert len(manifest.nodes) == 6 assert len(manifest.external_node_unique_ids) == 4 class TestPortablePartialParsing: @pytest.fixture(scope="class") def models(self): return { "model_one.sql": model_one_sql, } @pytest.fixture(scope="class") def packages(self): return {"packages": [{"local": "local_dependency"}]} @pytest.fixture(scope="class") def local_dependency_files(self): return { "dbt_project.yml": local_dependency__dbt_project_yml, "models": { "schema.yml": local_dependency__models__schema_yml, "model_to_import.sql": local_dependency__models__model_to_import_sql, }, "macros": {"dep_macro.sql": local_dependency__macros__dep_macro_sql}, "seeds": {"seed.csv": local_dependency__seeds__seed_csv}, } def rename_project_root(self, project, new_project_root): with up_one(new_project_root): rename_dir(project.project_root, new_project_root) project.project_root = new_project_root # flags.project_dir is set during the project test fixture, and is persisted across run_dbt calls, # so it needs to be reset between invocations flags.set_from_args(Namespace(PROJECT_DIR=new_project_root), None) @pytest.fixture(scope="class", autouse=True) def initial_run_and_rename_project_dir(self, project, local_dependency_files): initial_project_root = project.project_root renamed_project_root = os.path.join(project.project_root.dirname, "renamed_project_dir") write_project_files(project.project_root, "local_dependency", local_dependency_files) # initial run run_dbt(["deps"]) assert len(run_dbt(["seed"])) == 1 assert len(run_dbt(["run"])) == 2 self.rename_project_root(project, renamed_project_root) yield self.rename_project_root(project, initial_project_root) def test_pp_renamed_project_dir_unchanged_project_contents(self, project): # partial parse same project in new absolute dir location, using partial_parse.msgpack created in previous dir run_dbt(["deps"]) assert len(run_dbt(["--partial-parse", "seed"])) == 1 assert len(run_dbt(["--partial-parse", "run"])) == 2 def test_pp_renamed_project_dir_changed_project_contents(self, project): write_file(model_two_sql, project.project_root, "models", "model_two.sql") # partial parse changed project in new absolute dir location, using partial_parse.msgpack created in previous dir run_dbt(["deps"]) len(run_dbt(["--partial-parse", "seed"])) == 1 len(run_dbt(["--partial-parse", "run"])) == 3 class TestProfileChanges: @pytest.fixture(scope="class") def models(self): return { "model.sql": "select 1 as id", } def test_profile_change(self, project, dbt_profile_data): # Fist run not partial parsing _, stdout = run_dbt_and_capture(["parse"]) assert "Unable to do partial parsing because saved manifest not found" in stdout _, stdout = run_dbt_and_capture(["parse"]) assert "Unable to do partial parsing" not in stdout # change dbname which is included in the connection_info dbt_profile_data["test"]["outputs"]["default"]["dbname"] = "dbt2" write_file(yaml.safe_dump(dbt_profile_data), project.profiles_dir, "profiles.yml") _, stdout = run_dbt_and_capture(["parse"]) assert "Unable to do partial parsing because profile has changed" in stdout # Change the password which is not included in the connection_info dbt_profile_data["test"]["outputs"]["default"]["pass"] = "another_password" write_file(yaml.safe_dump(dbt_profile_data), project.profiles_dir, "profiles.yml") _, stdout = run_dbt_and_capture(["parse"]) assert "Unable to do partial parsing" not in stdout class TestExplicitDefaultProfileAndTarget: def test_explicit_default_profile_allows_partial_parse(self, project): event_catcher = EventCatcher(UnableToPartialParse) run_dbt(["parse"]) run_dbt(["parse", "--profile", "test"], callbacks=[event_catcher.catch]) assert len(event_catcher.caught_events) == 0 def test_explicit_default_target_allows_partial_parse(self, project): event_catcher = EventCatcher(UnableToPartialParse) run_dbt(["parse"]) run_dbt(["parse", "--target", "default"], callbacks=[event_catcher.catch]) assert len(event_catcher.caught_events) == 0 ================================================ FILE: tests/functional/partial_parsing/test_pp_disabled_config.py ================================================ import pytest from dbt.tests.util import get_manifest, run_dbt, write_file model_one_sql = """ select 1 as fun """ metricflow_time_spine_sql = """ SELECT to_date('02/20/2023', 'mm/dd/yyyy') as date_day """ schema1_yml = """ version: 2 models: - name: model_one semantic_models: - name: semantic_people model: ref('model_one') dimensions: - name: created_at type: TIME type_params: time_granularity: day measures: - name: people agg: count expr: fun entities: - name: fun type: primary defaults: agg_time_dimension: created_at metrics: - name: number_of_people label: "Number of people" description: Total count of people type: simple type_params: measure: people meta: my_meta: 'testing' exposures: - name: proxy_for_dashboard description: "My Exposure" type: "dashboard" owner: name: "Dashboard Tester" email: "tester@dashboard.com" depends_on: - ref("model_one") """ schema2_yml = """ version: 2 models: - name: model_one semantic_models: - name: semantic_people model: ref('model_one') dimensions: - name: created_at type: TIME type_params: time_granularity: day measures: - name: people agg: count expr: fun entities: - name: fun type: primary defaults: agg_time_dimension: created_at metrics: - name: number_of_people label: "Number of people" description: Total count of people config: enabled: false type: simple type_params: measure: people meta: my_meta: 'testing' exposures: - name: proxy_for_dashboard description: "My Exposure" config: enabled: false type: "dashboard" owner: name: "Dashboard Tester" email: "tester@dashboard.com" depends_on: - ref("model_one") """ schema3_yml = """ version: 2 models: - name: model_one semantic_models: - name: semantic_people model: ref('model_one') dimensions: - name: created_at type: TIME type_params: time_granularity: day measures: - name: people agg: count expr: fun entities: - name: fun type: primary defaults: agg_time_dimension: created_at metrics: - name: number_of_people label: "Number of people" description: Total count of people type: simple type_params: measure: people meta: my_meta: 'testing' """ schema4_yml = """ version: 2 models: - name: model_one exposures: - name: proxy_for_dashboard description: "My Exposure" config: enabled: false type: "dashboard" owner: name: "Dashboard Tester" email: "tester@dashboard.com" depends_on: - ref("model_one") """ class TestDisabled: @pytest.fixture(scope="class") def models(self): return { "model_one.sql": model_one_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "schema.yml": schema1_yml, } def test_pp_disabled(self, project): expected_exposure = "exposure.test.proxy_for_dashboard" expected_metric = "metric.test.number_of_people" run_dbt(["seed"]) manifest = run_dbt(["parse"]) assert expected_exposure in manifest.exposures assert expected_metric in manifest.metrics assert expected_exposure not in manifest.disabled assert expected_metric not in manifest.disabled # Update schema file with disabled metric and exposure write_file(schema2_yml, project.project_root, "models", "schema.yml") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 2 manifest = get_manifest(project.project_root) assert expected_exposure not in manifest.exposures assert expected_metric not in manifest.metrics assert expected_exposure in manifest.disabled assert expected_metric in manifest.disabled # Update schema file with enabled metric and exposure write_file(schema1_yml, project.project_root, "models", "schema.yml") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 2 manifest = get_manifest(project.project_root) assert expected_exposure in manifest.exposures assert expected_metric in manifest.metrics assert expected_exposure not in manifest.disabled assert expected_metric not in manifest.disabled # Update schema file - remove exposure, enable metric write_file(schema3_yml, project.project_root, "models", "schema.yml") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 2 manifest = get_manifest(project.project_root) assert expected_exposure not in manifest.exposures assert expected_metric in manifest.metrics assert expected_exposure not in manifest.disabled assert expected_metric not in manifest.disabled # Update schema file - add back exposure, remove metric write_file(schema4_yml, project.project_root, "models", "schema.yml") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 2 manifest = get_manifest(project.project_root) assert expected_exposure not in manifest.exposures assert expected_metric not in manifest.metrics assert expected_exposure in manifest.disabled assert expected_metric not in manifest.disabled ================================================ FILE: tests/functional/partial_parsing/test_pp_docs.py ================================================ import pytest from dbt.tests.util import get_manifest, rm_file, run_dbt, write_file model_one_sql = """ select 1 as fun """ raw_customers_csv = """id,first_name,last_name,email 1,Michael,Perez,mperez0@chronoengine.com 2,Shawn,Mccoy,smccoy1@reddit.com 3,Kathleen,Payne,kpayne2@cargocollective.com 4,Jimmy,Cooper,jcooper3@cargocollective.com 5,Katherine,Rice,krice4@typepad.com 6,Sarah,Ryan,sryan5@gnu.org 7,Martin,Mcdonald,mmcdonald6@opera.com 8,Frank,Robinson,frobinson7@wunderground.com 9,Jennifer,Franklin,jfranklin8@mail.ru 10,Henry,Welch,hwelch9@list-manage.com """ my_macro_sql = """ {% macro my_macro(something) %} select '{{ something }}' as something2 {% endmacro %} """ customers1_md = """ {% docs customer_table %} This table contains customer data {% enddocs %} """ customers2_md = """ {% docs customer_table %} LOTS of customer data {% enddocs %} """ schema1_yml = """ version: 2 models: - name: model_one description: "{{ doc('customer_table') }}" """ schema2_yml = """ version: 2 models: - name: model_one description: "{{ doc('customer_table') }}" macros: - name: my_macro description: "{{ doc('customer_table') }}" sources: - name: seed_sources description: "{{ doc('customer_table') }}" schema: "{{ target.schema }}" tables: - name: raw_customers columns: - name: id data_tests: - not_null: severity: "{{ 'error' if target.name == 'prod' else 'warn' }}" - unique - name: first_name - name: last_name - name: email exposures: - name: proxy_for_dashboard description: "{{ doc('customer_table') }}" type: "dashboard" owner: name: "Dashboard Tester" email: "tester@dashboard.com" depends_on: - ref("model_one") - ref("raw_customers") - source("seed_sources", "raw_customers") """ class TestDocs: @pytest.fixture(scope="class") def models(self): return { "model_one.sql": model_one_sql, } @pytest.fixture(scope="class") def seeds(self): return { "raw_customers.csv": raw_customers_csv, } @pytest.fixture(scope="class") def macros(self): return { "my_macro.sql": my_macro_sql, } def test_pp_docs(self, project): run_dbt(["seed"]) results = run_dbt(["run"]) assert len(results) == 1 # Add docs file customers.md write_file(customers1_md, project.project_root, "models", "customers.md") results = run_dbt(["--partial-parse", "run"]) manifest = get_manifest(project.project_root) assert len(manifest.docs) == 2 # Add schema file with 'docs' description write_file(schema1_yml, project.project_root, "models", "schema.yml") results = run_dbt(["--partial-parse", "run"]) manifest = get_manifest(project.project_root) assert len(manifest.docs) == 2 doc_id = "doc.test.customer_table" assert doc_id in manifest.docs doc = manifest.docs[doc_id] doc_file_id = doc.file_id assert doc_file_id in manifest.files source_file = manifest.files[doc_file_id] assert len(source_file.nodes) == 1 model_one_id = "model.test.model_one" assert model_one_id in source_file.nodes model_node = manifest.nodes[model_one_id] assert model_node.description == "This table contains customer data" # Update the doc file write_file(customers2_md, project.project_root, "models", "customers.md") results = run_dbt(["--partial-parse", "run"]) manifest = get_manifest(project.project_root) assert len(manifest.docs) == 2 assert model_one_id in manifest.nodes model_node = manifest.nodes[model_one_id] assert "LOTS" in model_node.description # Add a macro patch, source and exposure with doc write_file(schema2_yml, project.project_root, "models", "schema.yml") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 1 manifest = get_manifest(project.project_root) doc_file = manifest.files[doc_file_id] expected_nodes = [ "model.test.model_one", "source.test.seed_sources.raw_customers", "macro.test.my_macro", "exposure.test.proxy_for_dashboard", ] assert expected_nodes == doc_file.nodes source_id = "source.test.seed_sources.raw_customers" assert manifest.sources[source_id].source_description == "LOTS of customer data" macro_id = "macro.test.my_macro" assert manifest.macros[macro_id].description == "LOTS of customer data" exposure_id = "exposure.test.proxy_for_dashboard" assert manifest.exposures[exposure_id].description == "LOTS of customer data" # update the doc file again write_file(customers1_md, project.project_root, "models", "customers.md") results = run_dbt(["--partial-parse", "run"]) manifest = get_manifest(project.project_root) source_file = manifest.files[doc_file_id] assert model_one_id in source_file.nodes model_node = manifest.nodes[model_one_id] assert model_node.description == "This table contains customer data" assert ( manifest.sources[source_id].source_description == "This table contains customer data" ) assert manifest.macros[macro_id].description == "This table contains customer data" assert manifest.exposures[exposure_id].description == "This table contains customer data" # check that _lock is working with manifest._lock: assert manifest._lock my_model_yml = """ version: 2 models: - name: my_model columns: - name: id description: "{{ doc('whatever') }}" """ my_model_no_description_yml = """ version: 2 models: - name: my_model columns: - name: id """ my_model_md = """ {% docs whatever %} cool stuff {% enddocs %} """ class TestDocsRemoveReplace: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": "select 1 as id", "my_model.yml": my_model_yml, "my_model.md": my_model_md, } def test_remove_replace(self, project): run_dbt(["parse"]) manifest = get_manifest(project.project_root) doc_id = "doc.test.whatever" assert doc_id in manifest.docs doc = manifest.docs[doc_id] doc_file = manifest.files[doc.file_id] model_id = "model.test.my_model" assert model_id in manifest.nodes assert doc_file.nodes == [model_id] model = manifest.nodes[model_id] model_file_id = model.file_id assert model_file_id in manifest.files # remove the doc file rm_file(project.project_root, "models", "my_model.md") # remove description from schema file write_file(my_model_no_description_yml, project.project_root, "models", "my_model.yml") run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert doc_id not in manifest.docs # The bug was that the file still existed in manifest.files assert doc.file_id not in manifest.files # put back the doc file write_file(my_model_md, project.project_root, "models", "my_model.md") # put back the description in the schema file write_file(my_model_yml, project.project_root, "models", "my_model.yml") run_dbt(["parse"]) ================================================ FILE: tests/functional/partial_parsing/test_pp_functions.py ================================================ import pytest from dbt.artifacts.resources import FunctionArgument, FunctionReturns from dbt.contracts.graph.manifest import Manifest from dbt.tests.util import run_dbt, update_config_file, write_file from dbt_common.events.event_catcher import EventCatcher from dbt_common.events.types import Note from tests.functional.partial_parsing.fixtures import ( model_using_function_sql, my_func_sql, my_func_yml, updated_my_func_sql, updated_my_func_yml, ) class TestPartialParsingFunctions: @pytest.fixture(scope="class") def functions(self): return { "my_func.sql": my_func_sql, "my_func.yml": my_func_yml, } def test_pp_functions(self, project): # initial run manifest = run_dbt(["parse"]) assert isinstance(manifest, Manifest) assert len(manifest.functions) == 1 function = manifest.functions["function.test.my_func"] assert function.raw_code == "value * 2" assert function.description == "Doubles an integer" assert function.arguments == [ FunctionArgument(name="value", data_type="int", description="An integer to be doubled") ] assert function.returns == FunctionReturns(data_type="int") # update sql write_file(updated_my_func_sql, project.project_root, "functions", "my_func.sql") manifest = run_dbt(["parse"]) assert isinstance(manifest, Manifest) assert len(manifest.functions) == 1 function = manifest.functions["function.test.my_func"] assert function.raw_code == "number * 2.0" assert function.description == "Doubles an integer" assert function.arguments == [ FunctionArgument(name="value", data_type="int", description="An integer to be doubled") ] assert function.returns == FunctionReturns(data_type="int") # update yml write_file(updated_my_func_yml, project.project_root, "functions", "my_func.yml") manifest = run_dbt(["parse"]) assert isinstance(manifest, Manifest) assert len(manifest.functions) == 1 function = manifest.functions["function.test.my_func"] assert function.raw_code == "number * 2.0" assert function.description == "Doubles a float" assert function.arguments == [ FunctionArgument(name="number", data_type="float", description="A float to be doubled") ] assert function.returns == FunctionReturns(data_type="float") # if we parse again, partial parsing should be skipped note_catcher = EventCatcher(Note) manifest = run_dbt(["parse"], callbacks=[note_catcher.catch]) assert isinstance(manifest, Manifest) assert len(manifest.functions) == 1 assert len(note_catcher.caught_events) == 1 assert ( note_catcher.caught_events[0].info.msg == "Nothing changed, skipping partial parsing." ) class TestPartialParsingFunctionsAndCompilationOfDownstreamNodes: @pytest.fixture(scope="class") def functions(self): return { "my_func.sql": my_func_sql, "my_func.yml": my_func_yml, } @pytest.fixture(scope="class") def models(self): return { "model_using_function.sql": model_using_function_sql, } def test_pp_functions(self, project): result = run_dbt(["compile"]) # one function node and one model node assert len(result.results) == 2 assert result.results[0].node.name == "my_func" assert result.results[0].node.config.alias is None assert result.results[1].node.name == "model_using_function" # `my_func` should be the third part of the name for the function in the compiled code assert "my_func" in result.results[1].node.compiled_code # Add an alias to `my_func` add_function_alias = { "functions": { "+alias": "aliased_my_func", } } update_config_file(add_function_alias, "dbt_project.yml") # Recompile result = run_dbt(["compile"]) # one function node and one model node assert len(result.results) == 2 assert result.results[0].node.name == "my_func" assert result.results[0].node.config.alias == "aliased_my_func" assert result.results[1].node.name == "model_using_function" # `aliased_my_func` should be the third part of the name for the function in the compiled code assert "aliased_my_func" in result.results[1].node.compiled_code ================================================ FILE: tests/functional/partial_parsing/test_pp_groups.py ================================================ import pytest from dbt.exceptions import ParsingError from dbt.tests.util import get_manifest, run_dbt, write_file from tests.functional.partial_parsing.fixtures import ( groups_schema_yml_one_group, groups_schema_yml_one_group_model_in_group2, groups_schema_yml_two_groups, groups_schema_yml_two_groups_edited, groups_schema_yml_two_groups_private_orders_invalid_access, groups_schema_yml_two_groups_private_orders_valid_access, orders_downstream_sql, orders_sql, ) class TestGroups: @pytest.fixture(scope="class") def models(self): return { "orders.sql": orders_sql, "orders_downstream.sql": orders_downstream_sql, "schema.yml": groups_schema_yml_one_group, } def test_pp_groups(self, project): # initial run results = run_dbt() assert len(results) == 2 manifest = get_manifest(project.project_root) expected_nodes = ["model.test.orders", "model.test.orders_downstream"] expected_groups = ["group.test.test_group"] assert expected_nodes == sorted(list(manifest.nodes.keys())) assert expected_groups == sorted(list(manifest.groups.keys())) # add group to schema write_file(groups_schema_yml_two_groups, project.project_root, "models", "schema.yml") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 2 manifest = get_manifest(project.project_root) expected_nodes = ["model.test.orders", "model.test.orders_downstream"] expected_groups = ["group.test.test_group", "group.test.test_group2"] assert expected_nodes == sorted(list(manifest.nodes.keys())) assert expected_groups == sorted(list(manifest.groups.keys())) # edit group in schema write_file( groups_schema_yml_two_groups_edited, project.project_root, "models", "schema.yml" ) results = run_dbt(["--partial-parse", "run"]) assert len(results) == 2 manifest = get_manifest(project.project_root) expected_nodes = ["model.test.orders", "model.test.orders_downstream"] expected_groups = ["group.test.test_group", "group.test.test_group2_edited"] assert expected_nodes == sorted(list(manifest.nodes.keys())) assert expected_groups == sorted(list(manifest.groups.keys())) # delete group in schema write_file(groups_schema_yml_one_group, project.project_root, "models", "schema.yml") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 2 manifest = get_manifest(project.project_root) expected_nodes = ["model.test.orders", "model.test.orders_downstream"] expected_groups = ["group.test.test_group"] assert expected_nodes == sorted(list(manifest.nodes.keys())) assert expected_groups == sorted(list(manifest.groups.keys())) # add back second group write_file(groups_schema_yml_two_groups, project.project_root, "models", "schema.yml") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 2 # remove second group with model still configured to second group write_file( groups_schema_yml_one_group_model_in_group2, project.project_root, "models", "schema.yml", ) with pytest.raises(ParsingError): results = run_dbt(["--partial-parse", "run"]) # add back second group, make orders private with valid ref write_file( groups_schema_yml_two_groups_private_orders_valid_access, project.project_root, "models", "schema.yml", ) results = run_dbt(["--partial-parse", "run"]) assert len(results) == 2 write_file( groups_schema_yml_two_groups_private_orders_invalid_access, project.project_root, "models", "schema.yml", ) with pytest.raises(ParsingError): results = run_dbt(["--partial-parse", "run"]) my_model_c = """ select * from {{ ref("my_model_a") }} union all select * from {{ ref("my_model_b") }} """ models_yml = """ models: - name: my_model_a - name: my_model_b - name: my_model_c """ models_and_groups_yml = """ groups: - name: sales_analytics owner: name: Sales Analytics email: sales@jaffleshop.com models: - name: my_model_a access: private group: sales_analytics - name: my_model_b access: private group: sales_analytics - name: my_model_c access: private group: sales_analytics """ class TestAddingModelsToNewGroups: @pytest.fixture(scope="class") def models(self): return { "my_model_a.sql": "select 1 as id", "my_model_b.sql": "select 2 as id", "my_model_c.sql": my_model_c, "models.yml": models_yml, } def test_adding_models_to_new_groups(self, project): run_dbt(["compile"]) # This tests that the correct patch is added to my_model_c. The bug # was that it was using the old patch, so model_c didn't have the # correct group and access. write_file(models_and_groups_yml, project.project_root, "models", "models.yml") run_dbt(["compile"]) manifest = get_manifest(project.project_root) model_c_node = manifest.nodes["model.test.my_model_c"] assert model_c_node.group == "sales_analytics" assert model_c_node.access == "private" ================================================ FILE: tests/functional/partial_parsing/test_pp_metrics.py ================================================ import pytest from dbt.cli.main import dbtRunner from dbt.contracts.graph.manifest import Manifest from dbt.exceptions import CompilationError from dbt.tests.util import get_manifest, rm_file, run_dbt, write_file from tests.functional.partial_parsing.fixtures import ( metric_model_a_sql, metricflow_time_spine_sql, people_metrics2_yml, people_metrics3_yml, people_metrics_yml, people_semantic_models_yml, people_sl_yml, people_sql, ) class TestMetrics: @pytest.fixture(scope="class") def models(self): return { "people.sql": people_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, } def test_metrics(self, project): # initial run results = run_dbt(["run"]) assert len(results) == 2 manifest = get_manifest(project.project_root) assert len(manifest.nodes) == 2 # Add metrics yaml file (and necessary semantic models yaml) write_file( people_semantic_models_yml, project.project_root, "models", "people_semantic_models.yml", ) write_file(people_metrics_yml, project.project_root, "models", "people_metrics.yml") results = run_dbt(["run"]) assert len(results) == 2 manifest = get_manifest(project.project_root) assert len(manifest.metrics) == 2 metric_people_id = "metric.test.number_of_people" metric_people = manifest.metrics[metric_people_id] expected_meta = {"my_meta": "testing"} assert metric_people.meta == expected_meta # TODO: Bring back when we resolving `depends_on_nodes` # metric_tenure_id = "metric.test.collective_tenure" # metric_tenure = manifest.metrics[metric_tenure_id] # assert metric_people.refs == [RefArgs(name="people")] # assert metric_tenure.refs == [RefArgs(name="people")] # expected_depends_on_nodes = ["model.test.people"] # assert metric_people.depends_on.nodes == expected_depends_on_nodes # Change metrics yaml files write_file(people_metrics2_yml, project.project_root, "models", "people_metrics.yml") results = run_dbt(["run"]) assert len(results) == 2 manifest = get_manifest(project.project_root) metric_people = manifest.metrics[metric_people_id] expected_meta = {"my_meta": "replaced"} assert metric_people.meta == expected_meta # TODO: Bring back when we resolving `depends_on_nodes` # expected_depends_on_nodes = ["model.test.people"] # assert metric_people.depends_on.nodes == expected_depends_on_nodes # Add model referring to metric write_file(metric_model_a_sql, project.project_root, "models", "metric_model_a.sql") results = run_dbt(["run"]) manifest = get_manifest(project.project_root) # TODO: Bring back when we resolving `depends_on_nodes` # model_a = manifest.nodes["model.test.metric_model_a"] # expected_depends_on_nodes = [ # "metric.test.number_of_people", # "metric.test.collective_tenure", # ] # assert model_a.depends_on.nodes == expected_depends_on_nodes # Then delete a metric write_file(people_metrics3_yml, project.project_root, "models", "people_metrics.yml") with pytest.raises(CompilationError): # We use "parse" here and not "run" because we're checking that the CompilationError # occurs at parse time, not compilation results = run_dbt(["parse"]) class TestDeleteFileWithMetricsAndSemanticModels: @pytest.fixture(scope="class") def models(self): return { "people.sql": people_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "people_sl.yml": people_sl_yml, } def test_metrics(self, project): # Initial parsing runner = dbtRunner() result = runner.invoke(["parse"]) assert result.success manifest = result.result assert isinstance(manifest, Manifest) assert len(manifest.metrics) == 3 # Remove metric file rm_file(project.project_root, "models", "people_sl.yml") # Rerun parse, shouldn't fail result = runner.invoke(["parse"]) assert result.exception is None, result.exception ================================================ FILE: tests/functional/partial_parsing/test_pp_schema_file_order.py ================================================ import os import pytest from dbt.tests.util import get_manifest, rm_file, run_dbt, write_file os.environ["DBT_PP_TEST"] = "true" colors_sql = """ select 'green' as first, 'red' as second, 'blue' as third """ another_v1_sql = """ select * from {{ ref("colors") }} """ another_ref_sql = """ select * from {{ ref("another") }} """ colors_yml = """ models: - name: colors description: "a list of colors" - name: another description: "another model" versions: - v: 1 """ colors_alt_yml = """ models: - name: colors description: "a list of colors" - name: another description: "YET another model" versions: - v: 1 """ foo_model_sql = """ select 1 as id """ another_ref_yml = """ models: - name: another_ref description: "model with reference to another ref" - name: foo_model description: "some random model" """ another_ref_alt_yml = """ models: - name: another_ref description: "model with reference to another ref" - name: foo_model description: "some random other model" """ class TestSchemaFileOrder: @pytest.fixture(scope="class") def models(self): return { "colors.sql": colors_sql, "colors.yml": colors_yml, "another_v1.sql": another_v1_sql, "another_ref.sql": another_ref_sql, "foo_model.sql": foo_model_sql, "another_ref.yml": another_ref_yml, } def test_schema_file_order(self, project): # initial run results = run_dbt(["run"]) assert len(results) == 4 manifest = get_manifest(project.project_root) model_id = "model.test.another_ref" model = manifest.nodes.get(model_id) assert model.description == "model with reference to another ref" write_file(colors_alt_yml, project.project_root, "models", "colors.yml") write_file(another_ref_alt_yml, project.project_root, "models", "another_ref.yml") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 4 manifest = get_manifest(project.project_root) model = manifest.nodes.get(model_id) assert model.name == "another_ref" # The description here would be '' without the bug fix assert model.description == "model with reference to another ref" foo_sql = """ select 1 c """ bar_sql = """ select 1 c """ bar_with_ref_sql = """ select * from {{ ref('foo') }} """ foo_v2_sql = """ select 1 c """ schema_yml = """ # models/schema.yml models: - name: foo latest_version: 1 versions: - v: 1 - v: 2 """ foo_yml = """ # models/foo.yml models: - name: foo """ bar_yml = """ # models/bar.yml models: - name: bar columns: - name: c tests: - relationships: to: ref('foo') field: c """ foo_alt_yml = """ # models/foo.yml models: - name: foo latest_version: 1 versions: - v: 1 - v: 2 """ class TestNewVersionedSchemaFile: @pytest.fixture(scope="class") def models(self): return { "foo.sql": foo_sql, "bar.sql": bar_with_ref_sql, } def test_schema_file_order_new_versions(self, project): # This tests that when a model referring to an existing model # which has had a version added in a yaml file has been re-parsed # in order to fix the depends_on to the correct versioned model # initial run results = run_dbt(["compile"]) assert len(results) == 2 write_file(foo_v2_sql, project.project_root, "models", "foo_v2.sql") write_file(schema_yml, project.project_root, "models", "schema.yml") results = run_dbt(["compile"]) class TestMoreNewVersionedSchemaFile: @pytest.fixture(scope="class") def models(self): return { "foo.sql": foo_sql, "bar.sql": bar_sql, "foo.yml": foo_yml, "bar.yml": bar_yml, } def test_more_schema_file_new_versions(self, project): # initial run results = run_dbt(["compile"]) assert len(results) == 3 rm_file(project.project_root, "models", "foo.sql") write_file(foo_sql, project.project_root, "models", "foo_v1.sql") write_file(foo_sql, project.project_root, "models", "foo_v2.sql") write_file(foo_alt_yml, project.project_root, "models", "foo.yml") results = run_dbt(["compile"]) sources_yml = """ sources: - name: top_source tables: - name: abcd - name: efgh - name: ijkl """ abcd_sql = """ select * from {{ source("top_source", "abcd") }} """ efgh_sql = """ select * from {{ source("top_source", "efgh") }} """ ijkl_sql = """ select * from {{ source("top_source", "ijkl") }} """ models_yml = """ models: - name: abcd description: "abcd model" versions: - v: 1 - name: efgh description: "efgh model" versions: - v: 1 - name: ijkl description: "ijkl model" versions: - v: 1 """ append_sources_yml = """ - name: mnop """ append_models_yml = """ - name: mnop description: "mnop model" versions: - v: 1 """ mnop_sql = """ select * from {{ source("top_source", "mnop") }} """ class TestSourcesAndSchemaFiles: @pytest.fixture(scope="class") def models(self): return { "sources.yml": sources_yml, "abcd_v1.sql": abcd_sql, "efgh_v1.sql": efgh_sql, "ijkl_v1.sql": ijkl_sql, "_models.yml": models_yml, } def test_schema_file_order_new_versions(self, project): # initial run manifest = run_dbt(["parse"]) assert len(manifest.nodes) == 3 write_file(models_yml + append_models_yml, project.project_root, "models", "_models.yml") write_file(mnop_sql, project.project_root, "models", "mnop_v1.sql") write_file(sources_yml + append_sources_yml, project.project_root, "models", "sources.yml") manifest = run_dbt(["parse"]) assert len(manifest.nodes) == 4 # Without the fix the three original nodes will all be missing the # the patch updates, including description, so description will be "" for node in manifest.nodes.values(): assert node.description == f"{node.name} model" assert node.unique_id.endswith(".v1") ================================================ FILE: tests/functional/partial_parsing/test_pp_semantic_models.py ================================================ ================================================ FILE: tests/functional/partial_parsing/test_pp_undefined_serialization.py ================================================ """ Functional tests for handling jinja2.Undefined objects during manifest msgpack serialization (write_manifest_for_partial_parse). Reproduces: TypeError: can not serialize 'Undefined' object The fix adds isinstance(obj, jinja2.Undefined) -> None handling to extended_msgpack_encoder in core/dbt/parser/manifest.py. """ import pytest from dbt.tests.util import get_manifest, run_dbt, write_file # A model whose meta references a Jinja variable that is not in the schema # rendering context. The SchemaYamlRenderer renders these values with # native=True, so the result is a raw jinja2.Undefined object stored in the # node's meta dict rather than a string. model_with_undefined_meta_sql = """ select 1 as id """ # The value "{{ undefined_jinja_var }}" is NOT in the schema YAML rendering # context, so it evaluates to jinja2.Undefined and ends up stored in the # manifest node's meta dict. schema_with_undefined_meta_yml = """ version: 2 models: - name: model_with_undefined_meta meta: key: "{{ undefined_jinja_var }}" """ class TestUndefinedMetaSerializationInPartialParse: """ When a schema.yml meta value resolves to jinja2.Undefined during parse time, write_manifest_for_partial_parse must not raise TypeError. Without the fix (the isinstance(obj, jinja2.Undefined) branch in extended_msgpack_encoder), this test fails with: TypeError: can not serialize 'Undefined' object """ @pytest.fixture(scope="class") def models(self): return { "model_with_undefined_meta.sql": model_with_undefined_meta_sql, "schema.yml": schema_with_undefined_meta_yml, } def test_parse_with_undefined_meta_does_not_raise(self, project): # First parse - also triggers write_manifest_for_partial_parse which # is where the TypeError would previously be raised. run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert manifest is not None # The manifest was written successfully; the model node exists. assert "model.test.model_with_undefined_meta" in manifest.nodes # The meta value that was jinja2.Undefined should have been serialized # as None (the fix converts Undefined -> None before msgpack packing). node = manifest.nodes["model.test.model_with_undefined_meta"] assert node.meta.get("key") is None def test_partial_parse_with_undefined_meta_does_not_raise(self, project): # Write a trivial change to a different file to trigger the partial # parse path, which re-invokes write_manifest_for_partial_parse. write_file( model_with_undefined_meta_sql + "\n-- trigger partial reparse\n", project.project_root, "models", "model_with_undefined_meta.sql", ) run_dbt(["--partial-parse", "parse"]) manifest = get_manifest(project.project_root) assert manifest is not None assert "model.test.model_with_undefined_meta" in manifest.nodes ================================================ FILE: tests/functional/partial_parsing/test_pp_vars.py ================================================ import os from pathlib import Path import pytest from dbt.adapters.exceptions import FailedToConnectError from dbt.exceptions import ParsingError from dbt.tests.util import get_manifest, run_dbt, run_dbt_and_capture, write_file from dbt_common.constants import SECRET_ENV_PREFIX from tests.functional.partial_parsing.fixtures import ( env_var_macro_sql, env_var_macros_yml, env_var_metrics_yml, env_var_model_one_sql, env_var_model_sql, env_var_model_test_yml, env_var_schema2_yml, env_var_schema3_yml, env_var_schema_yml, env_var_sources_yml, metricflow_time_spine_sql, model_color_sql, model_one_sql, people_semantic_models_yml, people_sql, raw_customers_csv, test_color_sql, ) os.environ["DBT_PP_TEST"] = "true" class TestEnvVars: @pytest.fixture(scope="class") def models(self): return { "model_color.sql": model_color_sql, } def test_env_vars_models(self, project): # initial run results = run_dbt(["run"]) assert len(results) == 1 # copy a file with an env_var call without an env_var write_file(env_var_model_sql, project.project_root, "models", "env_var_model.sql") with pytest.raises(ParsingError): results = run_dbt(["--partial-parse", "run"]) # set the env var os.environ["ENV_VAR_TEST"] = "TestingEnvVars" results = run_dbt(["--partial-parse", "run"]) assert len(results) == 2 manifest = get_manifest(project.project_root) expected_env_vars = {"ENV_VAR_TEST": "TestingEnvVars"} assert expected_env_vars == manifest.env_vars model_id = "model.test.env_var_model" model = manifest.nodes[model_id] model_created_at = model.created_at # change the env var os.environ["ENV_VAR_TEST"] = "second" results = run_dbt(["--partial-parse", "run"]) assert len(results) == 2 manifest = get_manifest(project.project_root) expected_env_vars = {"ENV_VAR_TEST": "second"} assert expected_env_vars == manifest.env_vars assert model_created_at != manifest.nodes[model_id].created_at # set an env_var in a schema file write_file(env_var_schema_yml, project.project_root, "models", "schema.yml") write_file(env_var_model_one_sql, project.project_root, "models", "model_one.sql") with pytest.raises(ParsingError): results = run_dbt(["--partial-parse", "run"]) # actually set the env_var os.environ["TEST_SCHEMA_VAR"] = "view" results = run_dbt(["--partial-parse", "run"]) manifest = get_manifest(project.project_root) expected_env_vars = {"ENV_VAR_TEST": "second", "TEST_SCHEMA_VAR": "view"} assert expected_env_vars == manifest.env_vars # env vars in a source os.environ["ENV_VAR_DATABASE"] = "dbt" os.environ["ENV_VAR_SEVERITY"] = "warn" write_file(raw_customers_csv, project.project_root, "seeds", "raw_customers.csv") write_file(env_var_sources_yml, project.project_root, "models", "sources.yml") run_dbt(["--partial-parse", "seed"]) results = run_dbt(["--partial-parse", "run"]) assert len(results) == 3 manifest = get_manifest(project.project_root) expected_env_vars = { "ENV_VAR_TEST": "second", "TEST_SCHEMA_VAR": "view", "ENV_VAR_DATABASE": "dbt", "ENV_VAR_SEVERITY": "warn", } assert expected_env_vars == manifest.env_vars assert len(manifest.sources) == 1 source_id = "source.test.seed_sources.raw_customers" source = manifest.sources[source_id] assert source.database == "dbt" schema_file = manifest.files[source.file_id] test_id = "test.test.source_not_null_seed_sources_raw_customers_id.e39ee7bf0d" test_node = manifest.nodes[test_id] assert test_node.config.severity == "warn" # Change severity env var os.environ["ENV_VAR_SEVERITY"] = "error" results = run_dbt(["--partial-parse", "run"]) manifest = get_manifest(project.project_root) expected_env_vars = { "ENV_VAR_TEST": "second", "TEST_SCHEMA_VAR": "view", "ENV_VAR_DATABASE": "dbt", "ENV_VAR_SEVERITY": "error", } assert expected_env_vars == manifest.env_vars source_id = "source.test.seed_sources.raw_customers" source = manifest.sources[source_id] schema_file = manifest.files[source.file_id] expected_schema_file_env_vars = { "sources": {"seed_sources": ["ENV_VAR_DATABASE", "ENV_VAR_SEVERITY"]} } assert expected_schema_file_env_vars == schema_file.env_vars test_node = manifest.nodes[test_id] assert test_node.config.severity == "error" # Change database env var os.environ["ENV_VAR_DATABASE"] = "test_dbt" results = run_dbt(["--partial-parse", "run"]) manifest = get_manifest(project.project_root) expected_env_vars = { "ENV_VAR_TEST": "second", "TEST_SCHEMA_VAR": "view", "ENV_VAR_DATABASE": "test_dbt", "ENV_VAR_SEVERITY": "error", } assert expected_env_vars == manifest.env_vars source = manifest.sources[source_id] assert source.database == "test_dbt" # Delete database env var del os.environ["ENV_VAR_DATABASE"] with pytest.raises(ParsingError): results = run_dbt(["--partial-parse", "run"]) os.environ["ENV_VAR_DATABASE"] = "test_dbt" # Add generic test with test kwarg that's rendered late (no curly brackets) os.environ["ENV_VAR_DATABASE"] = "dbt" write_file(test_color_sql, project.project_root, "macros", "test_color.sql") results = run_dbt(["--partial-parse", "run"]) # Add source test using test_color and an env_var for color write_file(env_var_schema2_yml, project.project_root, "models/schema.yml") with pytest.raises(ParsingError): results = run_dbt(["--partial-parse", "run"]) os.environ["ENV_VAR_COLOR"] = "green" results = run_dbt(["--partial-parse", "run"]) manifest = get_manifest(project.project_root) test_color_id = "test.test.check_color_model_one_env_var_ENV_VAR_COLOR___fun.89638de387" test_node = manifest.nodes[test_color_id] # kwarg was rendered but not changed (it will be rendered again when compiled) assert test_node.test_metadata.kwargs["color"] == "env_var('ENV_VAR_COLOR')" results = run_dbt(["--partial-parse", "test"]) # Add an exposure with an env_var os.environ["ENV_VAR_OWNER"] = "John Doe" write_file(env_var_schema3_yml, project.project_root, "models", "schema.yml") results = run_dbt(["--partial-parse", "run"]) manifest = get_manifest(project.project_root) expected_env_vars = { "ENV_VAR_TEST": "second", "TEST_SCHEMA_VAR": "view", "ENV_VAR_DATABASE": "dbt", "ENV_VAR_SEVERITY": "error", "ENV_VAR_COLOR": "green", "ENV_VAR_OWNER": "John Doe", } assert expected_env_vars == manifest.env_vars exposure = list(manifest.exposures.values())[0] schema_file = manifest.files[exposure.file_id] expected_sf_env_vars = { "models": {"model_one": ["TEST_SCHEMA_VAR", "ENV_VAR_COLOR"]}, "exposures": {"proxy_for_dashboard": ["ENV_VAR_OWNER"]}, } assert expected_sf_env_vars == schema_file.env_vars # add a macro and a macro schema file os.environ["ENV_VAR_SOME_KEY"] = "toodles" write_file(env_var_macro_sql, project.project_root, "macros", "env_var_macro.sql") write_file(env_var_macros_yml, project.project_root, "macros", "env_var_macros.yml") results = run_dbt(["--partial-parse", "run"]) manifest = get_manifest(project.project_root) expected_env_vars = { "ENV_VAR_TEST": "second", "TEST_SCHEMA_VAR": "view", "ENV_VAR_DATABASE": "dbt", "ENV_VAR_SEVERITY": "error", "ENV_VAR_COLOR": "green", "ENV_VAR_OWNER": "John Doe", "ENV_VAR_SOME_KEY": "toodles", } assert expected_env_vars == manifest.env_vars macro_id = "macro.test.do_something" macro = manifest.macros[macro_id] assert macro.meta == {"some_key": "toodles"} # change the env var os.environ["ENV_VAR_SOME_KEY"] = "dumdedum" results = run_dbt(["--partial-parse", "run"]) manifest = get_manifest(project.project_root) macro = manifest.macros[macro_id] assert macro.meta == {"some_key": "dumdedum"} # Add a schema file with a test on model_color and env_var in test enabled config write_file(env_var_model_test_yml, project.project_root, "models", "schema.yml") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 3 manifest = get_manifest(project.project_root) model_color = manifest.nodes["model.test.model_color"] schema_file = manifest.files[model_color.patch_path] expected_env_vars = { "models": { "model_one": ["TEST_SCHEMA_VAR", "ENV_VAR_COLOR"], "model_color": ["ENV_VAR_ENABLED"], }, "exposures": {"proxy_for_dashboard": ["ENV_VAR_OWNER"]}, } assert expected_env_vars == schema_file.env_vars # Add a metrics file with env_vars os.environ["ENV_VAR_METRICS"] = "TeStInG" write_file(people_sql, project.project_root, "models", "people.sql") write_file( metricflow_time_spine_sql, project.project_root, "models", "metricflow_time_spine.sql" ) write_file( people_semantic_models_yml, project.project_root, "models", "semantic_models.yml" ) write_file(env_var_metrics_yml, project.project_root, "models", "metrics.yml") results = run_dbt(["run"]) manifest = get_manifest(project.project_root) assert "ENV_VAR_METRICS" in manifest.env_vars assert manifest.env_vars["ENV_VAR_METRICS"] == "TeStInG" metric_node = manifest.metrics["metric.test.number_of_people"] assert metric_node.meta == {"my_meta": "TeStInG"} # Change metrics env var os.environ["ENV_VAR_METRICS"] = "Changed!" results = run_dbt(["run"]) manifest = get_manifest(project.project_root) metric_node = manifest.metrics["metric.test.number_of_people"] assert metric_node.meta == {"my_meta": "Changed!"} # delete the env vars to cleanup del os.environ["ENV_VAR_TEST"] del os.environ["ENV_VAR_SEVERITY"] del os.environ["ENV_VAR_DATABASE"] del os.environ["TEST_SCHEMA_VAR"] del os.environ["ENV_VAR_COLOR"] del os.environ["ENV_VAR_SOME_KEY"] del os.environ["ENV_VAR_OWNER"] del os.environ["ENV_VAR_METRICS"] class TestProjectEnvVars: @pytest.fixture(scope="class") def environment(self): custom_env = os.environ.copy() custom_env["ENV_VAR_NAME"] = "Jane Smith" return custom_env @pytest.fixture(scope="class") def project_config_update(self): # Need to set the environment variable here initially because # the project fixture loads the config. return {"models": {"+meta": {"meta_name": "{{ env_var('ENV_VAR_NAME') }}"}}} @pytest.fixture(scope="class") def models(self): return { "model_one.sql": model_one_sql, } def test_project_env_vars(self, project): # Initial run os.environ["ENV_VAR_NAME"] = "Jane Smith" results = run_dbt(["run"]) assert len(results) == 1 manifest = get_manifest(project.project_root) state_check = manifest.state_check model_id = "model.test.model_one" model = manifest.nodes[model_id] assert model.config.meta["meta_name"] == "Jane Smith" env_vars_hash_checksum = state_check.project_env_vars_hash.checksum # Change the environment variable os.environ["ENV_VAR_NAME"] = "Jane Doe" results = run_dbt(["run"]) assert len(results) == 1 manifest = get_manifest(project.project_root) model = manifest.nodes[model_id] assert model.config.meta["meta_name"] == "Jane Doe" assert env_vars_hash_checksum != manifest.state_check.project_env_vars_hash.checksum # cleanup del os.environ["ENV_VAR_NAME"] class TestProfileEnvVars: @pytest.fixture(scope="class") def models(self): return { "model_one.sql": model_one_sql, } @pytest.fixture(scope="class") def environment(self): custom_env = os.environ.copy() custom_env["ENV_VAR_HOST"] = "localhost" return custom_env @pytest.fixture(scope="class") def dbt_profile_target(self): return { "type": "postgres", "threads": 4, "host": "{{ env_var('ENV_VAR_HOST') }}", "port": 5432, "user": "root", "pass": "password", "dbname": "dbt", } def test_profile_env_vars(self, project, logs_dir): # Initial run os.environ["ENV_VAR_HOST"] = "localhost" run_dbt(["run"]) # Change env_vars, the user doesn't exist, this should fail os.environ["ENV_VAR_HOST"] = "wrong_host" # N.B. run_dbt_and_capture won't work here because FailedToConnectError ends the test entirely with pytest.raises(FailedToConnectError): run_dbt(["run"], expect_pass=False) log_output = Path(logs_dir, "dbt.log").read_text() assert "Unable to do partial parsing because profile has changed" in log_output class TestProfileSecretEnvVars: @pytest.fixture(scope="class") def models(self): return { "model_one.sql": model_one_sql, } @property def dbt_profile_target(self): # Need to set these here because the base integration test class # calls 'load_config' before the tests are run. # Note: only the specified profile is rendered, so there's no # point in setting env_vars in non-used profiles. # user is secret and password is not. postgres on macos doesn't care if the password # changes so we have to change the user. related: https://github.com/dbt-labs/dbt-core/pull/4250 os.environ[SECRET_ENV_PREFIX + "_USER"] = "root" os.environ["ENV_VAR_PASS"] = "password" return { "type": "postgres", "threads": 4, "host": "localhost", "port": 5432, "user": "{{ env_var('DBT_ENV_SECRET_USER') }}", "pass": "{{ env_var('ENV_VAR_PASS') }}", "dbname": "dbt", } def test_profile_secret_env_vars(self, project): # Initial run os.environ[SECRET_ENV_PREFIX + "_USER"] = "root" os.environ["ENV_VAR_PASS"] = "password" results = run_dbt(["run"]) manifest = get_manifest(project.project_root) env_vars_checksum = manifest.state_check.profile_env_vars_hash.checksum # Change a secret var, it shouldn't register because we shouldn't save secrets. os.environ[SECRET_ENV_PREFIX + "_USER"] = "fake_user" # we just want to see if the manifest has included # the secret in the hash of environment variables. (results, log_output) = run_dbt_and_capture(["run"], expect_pass=True) # I020 is the event code for "env vars used in profiles.yml have changed" assert not ("I020" in log_output) manifest = get_manifest(project.project_root) assert env_vars_checksum == manifest.state_check.profile_env_vars_hash.checksum # Model that uses a var with a default value model_with_var_sql = """ select '{{ var("my_var", "default_value") }}' as var_value """ class TestVarsFilePartialParsing: """Tests for partial parsing behavior when vars.yml file changes.""" @pytest.fixture(scope="class") def models(self): return { "model_with_var.sql": model_with_var_sql, } def test_vars_file_partial_parsing(self, project, logs_dir): # Initial run without vars.yml - uses default value results = run_dbt(["run"]) assert len(results) == 1 manifest = get_manifest(project.project_root) initial_vars_hash = manifest.state_check.vars_hash.checksum # Add vars.yml file - should trigger full reparse vars_yml_content = """ vars: my_var: "from_vars_file" """ write_file(vars_yml_content, project.project_root, "vars.yml") (results, log_output) = run_dbt_and_capture(["--partial-parse", "run"]) assert len(results) == 1 assert "Unable to do partial parsing" in log_output manifest = get_manifest(project.project_root) current_vars_hash = manifest.state_check.vars_hash.checksum assert current_vars_hash != initial_vars_hash # Run again with no changes - should use partial parsing (no reparse message) (results, log_output) = run_dbt_and_capture(["--partial-parse", "run"]) assert len(results) == 1 assert "Unable to do partial parsing" not in log_output manifest = get_manifest(project.project_root) previous_vars_hash = current_vars_hash current_vars_hash = manifest.state_check.vars_hash.checksum assert current_vars_hash == previous_vars_hash # Modify vars.yml - should trigger full reparse vars_yml_content_modified = """ vars: my_var: "modified_value" """ write_file(vars_yml_content_modified, project.project_root, "vars.yml") (results, log_output) = run_dbt_and_capture(["--partial-parse", "run"]) assert len(results) == 1 assert "Unable to do partial parsing" in log_output manifest = get_manifest(project.project_root) previous_vars_hash = current_vars_hash current_vars_hash = manifest.state_check.vars_hash.checksum assert previous_vars_hash != current_vars_hash # Delete vars.yml - should trigger full reparse os.remove(os.path.join(project.project_root, "vars.yml")) (results, log_output) = run_dbt_and_capture(["--partial-parse", "run"]) assert len(results) == 1 assert "Unable to do partial parsing" in log_output manifest = get_manifest(project.project_root) previous_vars_hash = current_vars_hash current_vars_hash = manifest.state_check.vars_hash.checksum assert previous_vars_hash != current_vars_hash class TestVarsFileBackwardCompatibility: """Tests that users not using vars.yml are unaffected by the feature.""" @pytest.fixture(scope="class") def models(self): return { "model_one.sql": model_one_sql, } def test_no_vars_file_hash_unchanged(self, project): # First run - no vars.yml results, output = run_dbt_and_capture(["run"]) assert len(results) == 1 # Second run - still no vars.yml, hash should remain empty results, output = run_dbt_and_capture(["--partial-parse", "run"]) assert len(results) == 1 assert "Unable to do partial parsing" not in output def test_cli_vars_still_trigger_reparse(self, project, logs_dir): # First run with no CLI vars results = run_dbt(["run"]) assert len(results) == 1 manifest = get_manifest(project.project_root) initial_vars_hash = manifest.state_check.vars_hash.checksum # Second run with CLI vars - should trigger reparse due to vars_hash change (results, log_output) = run_dbt_and_capture( [ "--partial-parse", "run", "--vars", '{"cli_var": "value"}', ] ) assert len(results) == 1 assert "Unable to do partial parsing" in log_output manifest = get_manifest(project.project_root) assert manifest.state_check.vars_hash.checksum != initial_vars_hash ================================================ FILE: tests/functional/partial_parsing/test_versioned_models.py ================================================ import pathlib from typing import Dict import pytest from dbt.exceptions import DuplicateVersionedUnversionedError from dbt.tests.util import get_manifest, read_file, rm_file, run_dbt, write_file model_one_sql = """ select 1 as fun """ model_one_downstream_sql = """ select fun from {{ ref('model_one') }} """ models_versions_schema_yml = """ models: - name: model_one description: "The first model" versions: - v: 1 - v: 2 """ models_versions_defined_in_schema_yml = """ models: - name: model_one description: "The first model" versions: - v: 1 - v: 2 defined_in: model_one_different """ models_versions_updated_schema_yml = """ models: - name: model_one latest_version: 1 description: "The first model" versions: - v: 1 - v: 2 defined_in: model_one_different """ model_two_sql = """ select 1 as notfun """ class TestVersionedModels: @pytest.fixture(scope="class") def models(self): return { "model_one_v1.sql": model_one_sql, "model_one.sql": model_one_sql, "model_one_downstream.sql": model_one_downstream_sql, "schema.yml": models_versions_schema_yml, } def test_pp_versioned_models(self, project): results = run_dbt(["run"]) assert len(results) == 3 manifest = get_manifest(project.project_root) model_one_node = manifest.nodes["model.test.model_one.v1"] assert not model_one_node.is_latest_version model_two_node = manifest.nodes["model.test.model_one.v2"] assert model_two_node.is_latest_version # assert unpinned ref points to latest version model_one_downstream_node = manifest.nodes["model.test.model_one_downstream"] assert model_one_downstream_node.depends_on.nodes == ["model.test.model_one.v2"] # update schema.yml block - model_one is now 'defined_in: model_one_different' rm_file(project.project_root, "models", "model_one.sql") write_file(model_one_sql, project.project_root, "models", "model_one_different.sql") write_file( models_versions_defined_in_schema_yml, project.project_root, "models", "schema.yml" ) results = run_dbt(["--partial-parse", "run"]) assert len(results) == 3 # update versions schema.yml block - latest_version from 2 to 1 write_file( models_versions_updated_schema_yml, project.project_root, "models", "schema.yml" ) # This is where the test was failings in a CI run with: # relation \"test..._test_partial_parsing.model_one_downstream\" does not exist # because in core/dbt/include/global_project/macros/materializations/models/view/view.sql # "existing_relation" didn't actually exist by the time it gets to the rename of the # existing relation. (pathlib.Path(project.project_root) / "log_output").mkdir(parents=True, exist_ok=True) results = run_dbt( ["--partial-parse", "--log-format-file", "json", "--log-path", "log_output", "run"] ) assert len(results) == 3 manifest = get_manifest(project.project_root) model_one_node = manifest.nodes["model.test.model_one.v1"] assert model_one_node.is_latest_version model_two_node = manifest.nodes["model.test.model_one.v2"] assert not model_two_node.is_latest_version # assert unpinned ref points to latest version model_one_downstream_node = manifest.nodes["model.test.model_one_downstream"] assert model_one_downstream_node.depends_on.nodes == ["model.test.model_one.v1"] # assert unpinned ref to latest-not-max version yields an "FYI" info-level log log_output = read_file("log_output", "dbt.log").replace("\n", " ").replace("\\n", " ") assert "UnpinnedRefNewVersionAvailable" in log_output # update versioned model write_file(model_two_sql, project.project_root, "models", "model_one_different.sql") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 3 manifest = get_manifest(project.project_root) assert len(manifest.nodes) == 3 # create a new model_one in model_one.sql and re-parse write_file(model_one_sql, project.project_root, "models", "model_one.sql") with pytest.raises(DuplicateVersionedUnversionedError): run_dbt(["parse"]) model_unversioned_schema_yml = """ models: - name: model_one description: "The first model" """ model_versioned_schema_yml = """ models: - name: model_one description: "The first model" latest_version: 1 versions: - v: 1 """ class TestAddingVersioningToModel: @pytest.fixture(scope="class") def models(self) -> Dict[str, str]: return { "model_one.sql": model_one_sql, "model_one_downstream.sql": model_one_downstream_sql, "schema.yml": model_unversioned_schema_yml, } def test_pp_newly_versioned_models(self, project) -> None: results = run_dbt(["run"]) assert len(results) == 2 # update schema.yml block - model_one is now versioned write_file(model_versioned_schema_yml, project.project_root, "models", "schema.yml") results = run_dbt(["--partial-parse", "run"]) assert len(results) == 2 ================================================ FILE: tests/functional/permission/data/seed.sql ================================================ create schema if not exists {schema}; revoke create on database dbt from noaccess; revoke usage on schema {schema} from noaccess; create table {schema}.seed ( id BIGSERIAL PRIMARY KEY, first_name VARCHAR(50), last_name VARCHAR(50), email VARCHAR(50), gender VARCHAR(50), ip_address VARCHAR(20) ); insert into {schema}.seed (first_name, last_name, email, gender, ip_address) values ('Kathryn', 'Walker', 'kwalker1@ezinearticles.com', 'Female', '194.121.179.35'); insert into {schema}.seed (first_name, last_name, email, gender, ip_address) values ('Gerald', 'Ryan', 'gryan2@com.com', 'Male', '11.3.212.243'); insert into {schema}.seed (first_name, last_name, email, gender, ip_address) values ('Bonnie', 'Spencer', 'bspencer3@ameblo.jp', 'Female', '216.32.196.175'); insert into {schema}.seed (first_name, last_name, email, gender, ip_address) values ('Harold', 'Taylor', 'htaylor4@people.com.cn', 'Male', '253.10.246.136'); insert into {schema}.seed (first_name, last_name, email, gender, ip_address) values ('Jacqueline', 'Griffin', 'jgriffin5@t.co', 'Female', '16.13.192.220'); insert into {schema}.seed (first_name, last_name, email, gender, ip_address) values ('Wanda', 'Arnold', 'warnold6@google.nl', 'Female', '232.116.150.64'); insert into {schema}.seed (first_name, last_name, email, gender, ip_address) values ('Craig', 'Ortiz', 'cortiz7@sciencedaily.com', 'Male', '199.126.106.13'); insert into {schema}.seed (first_name, last_name, email, gender, ip_address) values ('Gary', 'Day', 'gday8@nih.gov', 'Male', '35.81.68.186'); insert into {schema}.seed (first_name, last_name, email, gender, ip_address) values ('Rose', 'Wright', 'rwright9@yahoo.co.jp', 'Female', '236.82.178.100'); insert into {schema}.seed (first_name, last_name, email, gender, ip_address) values ('Raymond', 'Kelley', 'rkelleya@fc2.com', 'Male', '213.65.166.67'); ================================================ FILE: tests/functional/permission/fixtures.py ================================================ import pytest from dbt.tests.fixtures.project import write_project_files models__view_model_sql = """ select * from {{ this.schema }}.seed """ @pytest.fixture(scope="class") def models(): return {"view_model.sql": models__view_model_sql} @pytest.fixture(scope="class") def project_files( project_root, models, ): write_project_files(project_root, "models", models) ================================================ FILE: tests/functional/postgres/fixtures.py ================================================ models__incremental_sql = """ {{ config( materialized = "incremental", indexes=[ {'columns': ['column_a'], 'type': 'hash'}, {'columns': ['column_a', 'column_b'], 'unique': True}, ] ) }} select * from ( select 1 as column_a, 2 as column_b ) t {% if is_incremental() %} where column_a > (select max(column_a) from {{this}}) {% endif %} """ models__table_sql = """ {{ config( materialized = "table", indexes=[ {'columns': ['column_a']}, {'columns': ['column_b']}, {'columns': ['column_a', 'column_b']}, {'columns': ['column_b', 'column_a'], 'type': 'btree', 'unique': True}, {'columns': ['column_a'], 'type': 'hash'} ] ) }} select 1 as column_a, 2 as column_b """ models_invalid__invalid_columns_type_sql = """ {{ config( materialized = "table", indexes=[ {'columns': 'column_a, column_b'}, ] ) }} select 1 as column_a, 2 as column_b """ models_invalid__invalid_type_sql = """ {{ config( materialized = "table", indexes=[ {'columns': ['column_a'], 'type': 'non_existent_type'}, ] ) }} select 1 as column_a, 2 as column_b """ models_invalid__invalid_unique_config_sql = """ {{ config( materialized = "table", indexes=[ {'columns': ['column_a'], 'unique': 'yes'}, ] ) }} select 1 as column_a, 2 as column_b """ models_invalid__missing_columns_sql = """ {{ config( materialized = "table", indexes=[ {'unique': True}, ] ) }} select 1 as column_a, 2 as column_b """ snapshots__colors_sql = """ {% snapshot colors %} {{ config( target_database=database, target_schema=schema, unique_key='id', strategy='check', check_cols=['color'], indexes=[ {'columns': ['id'], 'type': 'hash'}, {'columns': ['id', 'color'], 'unique': True}, ] ) }} {% if var('version') == 1 %} select 1 as id, 'red' as color union all select 2 as id, 'green' as color {% else %} select 1 as id, 'blue' as color union all select 2 as id, 'green' as color {% endif %} {% endsnapshot %} """ seeds__seed_csv = """country_code,country_name US,United States CA,Canada GB,United Kingdom """ ================================================ FILE: tests/functional/postgres/test_postgres_indexes.py ================================================ import re import pytest from dbt.tests.util import run_dbt, run_dbt_and_capture from tests.functional.postgres.fixtures import ( models__incremental_sql, models__table_sql, models_invalid__invalid_columns_type_sql, models_invalid__invalid_type_sql, models_invalid__invalid_unique_config_sql, models_invalid__missing_columns_sql, seeds__seed_csv, snapshots__colors_sql, ) INDEX_DEFINITION_PATTERN = re.compile(r"using\s+(\w+)\s+\((.+)\)\Z") class TestPostgresIndex: @pytest.fixture(scope="class") def models(self): return { "table.sql": models__table_sql, "incremental.sql": models__incremental_sql, } @pytest.fixture(scope="class") def seeds(self): return {"seed.csv": seeds__seed_csv} @pytest.fixture(scope="class") def snapshots(self): return {"colors.sql": snapshots__colors_sql} @pytest.fixture(scope="class") def project_config_update(self): return { "config-version": 2, "seeds": { "quote_columns": False, "indexes": [ {"columns": ["country_code"], "unique": False, "type": "hash"}, {"columns": ["country_code", "country_name"], "unique": True}, ], }, "vars": { "version": 1, }, } def test_table(self, project, unique_schema): results = run_dbt(["run", "--models", "table"]) assert len(results) == 1 indexes = self.get_indexes("table", project, unique_schema) expected = [ {"columns": "column_a", "unique": False, "type": "btree"}, {"columns": "column_b", "unique": False, "type": "btree"}, {"columns": "column_a, column_b", "unique": False, "type": "btree"}, {"columns": "column_b, column_a", "unique": True, "type": "btree"}, {"columns": "column_a", "unique": False, "type": "hash"}, ] assert len(indexes) == len(expected) def test_incremental(self, project, unique_schema): for additional_argument in [[], [], ["--full-refresh"]]: results = run_dbt(["run", "--models", "incremental"] + additional_argument) assert len(results) == 1 indexes = self.get_indexes("incremental", project, unique_schema) expected = [ {"columns": "column_a", "unique": False, "type": "hash"}, {"columns": "column_a, column_b", "unique": True, "type": "btree"}, ] assert len(indexes) == len(expected) def test_seed(self, project, unique_schema): for additional_argument in [[], [], ["--full-refresh"]]: results = run_dbt(["seed"] + additional_argument) assert len(results) == 1 indexes = self.get_indexes("seed", project, unique_schema) expected = [ {"columns": "country_code", "unique": False, "type": "hash"}, {"columns": "country_code, country_name", "unique": True, "type": "btree"}, ] assert len(indexes) == len(expected) def test_snapshot(self, project, unique_schema): for version in [1, 2]: results = run_dbt(["snapshot", "--vars", f"version: {version}"]) assert len(results) == 1 indexes = self.get_indexes("colors", project, unique_schema) expected = [ {"columns": "id", "unique": False, "type": "hash"}, {"columns": "id, color", "unique": True, "type": "btree"}, ] assert len(indexes) == len(expected) def get_indexes(self, table_name, project, unique_schema): sql = f""" SELECT pg_get_indexdef(idx.indexrelid) as index_definition FROM pg_index idx JOIN pg_class tab ON tab.oid = idx.indrelid WHERE tab.relname = '{table_name}' AND tab.relnamespace = ( SELECT oid FROM pg_namespace WHERE nspname = '{unique_schema}' ); """ results = project.run_sql(sql, fetch="all") return [self.parse_index_definition(row[0]) for row in results] def parse_index_definition(self, index_definition): index_definition = index_definition.lower() is_unique = "unique" in index_definition m = INDEX_DEFINITION_PATTERN.search(index_definition) return { "columns": m.group(2), "unique": is_unique, "type": m.group(1), } def assertCountEqual(self, a, b): assert len(a) == len(b) class TestPostgresInvalidIndex: @pytest.fixture(scope="class") def models(self): return { "invalid_unique_config.sql": models_invalid__invalid_unique_config_sql, "invalid_type.sql": models_invalid__invalid_type_sql, "invalid_columns_type.sql": models_invalid__invalid_columns_type_sql, "missing_columns.sql": models_invalid__missing_columns_sql, } def test_invalid_index_configs(self, project): results, output = run_dbt_and_capture(expect_pass=False) assert len(results) == 4 assert re.search(r"columns.*is not of type 'array'", output) assert re.search(r"unique.*is not of type 'boolean'", output) assert re.search(r"'columns' is a required property", output) assert re.search(r"Database Error in model invalid_type", output) ================================================ FILE: tests/functional/postgres/test_postgres_unlogged_table.py ================================================ import pytest from dbt.tests.util import run_dbt schema_yml = """ version: 2 models: - name: table_unlogged description: "Unlogged table model" columns: - name: column_a description: "Sample description" quote: true """ table_unlogged_sql = """ {{ config(materialized = 'table', unlogged = True) }} select 1 as column_a """ class TestPostgresUnloggedTable: @pytest.fixture(scope="class") def models(self): return { "schema.yml": schema_yml, "table_unlogged.sql": table_unlogged_sql, } @pytest.fixture(scope="class") def project_config_update(self): return { "models": { "test": { "materialized": "table", "+persist_docs": { "relation": True, "columns": True, }, } } } def test_postgres_unlogged_table_catalog(self, project): table_name = "table_unlogged" results = run_dbt(["run", "--models", table_name]) assert len(results) == 1 result = self.get_table_persistence(project, table_name) assert result == "u" catalog = run_dbt(["docs", "generate"]) assert len(catalog.nodes) == 1 table_node = catalog.nodes["model.test.table_unlogged"] assert table_node assert "column_a" in table_node.columns def get_table_persistence(self, project, table_name): sql = """ SELECT relpersistence FROM pg_class WHERE relname = '{table_name}' """ sql = sql.format(table_name=table_name, schema=project.test_schema) result = project.run_sql(sql, fetch="one") assert len(result) == 1 return result[0] ================================================ FILE: tests/functional/primary_keys/fixtures.py ================================================ simple_model_sql = """ select 1 as id, 'blue' as color """ simple_model_unique_test = """ models: - name: simple_model columns: - name: id tests: - unique """ invalid_model_unique_test = """ models: - name: simple_model data_tests: - unique: column_name: null columns: - name: id """ simple_model_disabled_unique_test = """ models: - name: simple_model columns: - name: id tests: - unique: enabled: false """ simple_model_unique_not_null_tests = """ models: - name: simple_model columns: - name: id tests: - unique - not_null """ simple_model_unique_combo_of_columns = """ models: - name: simple_model tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: [id, color] """ invalid_model_unique_combo_of_columns = """ models: - name: simple_model tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: [null] - dbt_utils.unique_combination_of_columns: combination_of_columns: "test" """ simple_model_constraints = """ models: - name: simple_model config: contract: enforced: true columns: - name: id data_type: int constraints: - type: not_null - type: primary_key - name: color data_type: text """ simple_model_two_versions_both_configured = """ models: - name: simple_model latest_version: 1 columns: - name: id tests: - unique - not_null versions: - v: 1 - v: 2 """ simple_model_two_versions_exclude_col = """ models: - name: simple_model latest_version: 1 columns: - name: id tests: - unique - not_null versions: - v: 1 - v: 2 columns: - include: all exclude: [id] """ ================================================ FILE: tests/functional/primary_keys/test_primary_keys.py ================================================ import pytest from dbt.tests.util import get_manifest, run_dbt from tests.functional.primary_keys.fixtures import ( invalid_model_unique_combo_of_columns, invalid_model_unique_test, simple_model_constraints, simple_model_disabled_unique_test, simple_model_sql, simple_model_two_versions_both_configured, simple_model_two_versions_exclude_col, simple_model_unique_combo_of_columns, simple_model_unique_not_null_tests, simple_model_unique_test, ) class TestSimpleModelNoYml: @pytest.fixture(scope="class") def models(self): return { "simple_model.sql": simple_model_sql, } def test_simple_model_no_yml(self, project): run_dbt(["deps"]) run_dbt(["run"]) manifest = get_manifest(project.project_root) node = manifest.nodes["model.test.simple_model"] assert node.primary_key == [] class TestSimpleModelConstraints: @pytest.fixture(scope="class") def models(self): return { "simple_model.sql": simple_model_sql, "schema.yml": simple_model_constraints, } def test_simple_model_constraints(self, project): run_dbt(["deps"]) run_dbt(["run"]) manifest = get_manifest(project.project_root) node = manifest.nodes["model.test.simple_model"] assert node.primary_key == ["id"] class TestSimpleModelUniqueNotNullTests: @pytest.fixture(scope="class") def models(self): return { "simple_model.sql": simple_model_sql, "schema.yml": simple_model_unique_not_null_tests, } def test_simple_model_unique_not_null_tests(self, project): run_dbt(["deps"]) run_dbt(["run"]) manifest = get_manifest(project.project_root) node = manifest.nodes["model.test.simple_model"] assert node.primary_key == ["id"] class TestSimpleModelUniqueTests: @pytest.fixture(scope="class") def models(self): return { "simple_model.sql": simple_model_sql, "schema.yml": simple_model_unique_test, } def test_simple_model_unique_test(self, project): run_dbt(["deps"]) run_dbt(["run"]) manifest = get_manifest(project.project_root) node = manifest.nodes["model.test.simple_model"] assert node.primary_key == ["id"] class TestSimpleModelDisabledUniqueTests: @pytest.fixture(scope="class") def models(self): return { "simple_model.sql": simple_model_sql, "schema.yml": simple_model_disabled_unique_test, } def test_simple_model_disabled_unique_test(self, project): run_dbt(["deps"]) run_dbt(["run"]) manifest = get_manifest(project.project_root) node = manifest.nodes["model.test.simple_model"] assert node.primary_key == ["id"] class TestVersionedSimpleModel: @pytest.fixture(scope="class") def models(self): return { "simple_model_v1.sql": simple_model_sql, "simple_model_v2.sql": simple_model_sql, "schema.yml": simple_model_two_versions_both_configured, } def test_versioned_simple_model(self, project): run_dbt(["deps"]) run_dbt(["run"]) manifest = get_manifest(project.project_root) node_v1 = manifest.nodes["model.test.simple_model.v1"] node_v2 = manifest.nodes["model.test.simple_model.v2"] assert node_v1.primary_key == ["id"] assert node_v2.primary_key == ["id"] class TestVersionedSimpleModelExcludeTests: @pytest.fixture(scope="class") def models(self): return { "simple_model_v1.sql": simple_model_sql, "simple_model_v2.sql": simple_model_sql, "schema.yml": simple_model_two_versions_exclude_col, } def test_versioned_simple_model_exclude_col(self, project): run_dbt(["deps"]) run_dbt(["run"]) manifest = get_manifest(project.project_root) node_v1 = manifest.nodes["model.test.simple_model.v1"] node_v2 = manifest.nodes["model.test.simple_model.v2"] assert node_v1.primary_key == ["id"] assert node_v2.primary_key == [] class TestSimpleModelCombinationOfColumns: @pytest.fixture(scope="class") def packages(self): return { "packages": [ { "git": "https://github.com/dbt-labs/dbt-utils.git", "revision": "1.1.0", }, ] } @pytest.fixture(scope="class") def models(self): return { "simple_model.sql": simple_model_sql, "schema.yml": simple_model_unique_combo_of_columns, } def test_versioned_simple_combo_of_columns(self, project): run_dbt(["deps"]) run_dbt(["run"]) manifest = get_manifest(project.project_root) node = manifest.nodes["model.test.simple_model"] assert node.primary_key == ["color", "id"] class TestInvalidModelCombinationOfColumns: @pytest.fixture(scope="class") def packages(self): return { "packages": [ { "git": "https://github.com/dbt-labs/dbt-utils.git", "revision": "1.1.0", }, ] } @pytest.fixture(scope="class") def models(self): return { "simple_model.sql": simple_model_sql, "schema.yml": invalid_model_unique_combo_of_columns, } def test_invalid_combo_of_columns(self, project): run_dbt(["deps"]) run_dbt(["run"]) manifest = get_manifest(project.project_root) node = manifest.nodes["model.test.simple_model"] assert node.primary_key == [] class TestInvalidModelUniqueTest: @pytest.fixture(scope="class") def packages(self): return { "packages": [ { "git": "https://github.com/dbt-labs/dbt-utils.git", "revision": "1.1.0", }, ] } @pytest.fixture(scope="class") def models(self): return { "simple_model.sql": simple_model_sql, "schema.yml": invalid_model_unique_test, } def test_invalid_combo_of_columns(self, project): run_dbt(["deps"]) run_dbt(["run"]) manifest = get_manifest(project.project_root) node = manifest.nodes["model.test.simple_model"] assert node.primary_key == [] ================================================ FILE: tests/functional/profiles/test_profile_dir.py ================================================ import os from argparse import Namespace from contextlib import contextmanager from pathlib import Path import pytest import yaml import dbt.flags as flags from dbt.tests.util import rm_file, run_dbt, run_dbt_and_capture, write_file @pytest.fixture(scope="class") def profiles_yml(profiles_root, dbt_profile_data): write_file(yaml.safe_dump(dbt_profile_data), profiles_root, "profiles.yml") return dbt_profile_data @pytest.fixture(scope="class") def profiles_home_root(): return os.path.join(os.path.expanduser("~"), ".dbt") @pytest.fixture(scope="class") def profiles_env_root(tmpdir_factory): path = tmpdir_factory.mktemp("profile_env") # environment variables are lowercased for some reason in _get_flag_value_from_env within dbt.flags return str(path).lower() @pytest.fixture(scope="class") def profiles_flag_root(tmpdir_factory): return tmpdir_factory.mktemp("profile_flag") @pytest.fixture(scope="class") def profiles_project_root(project): return project.project_root @pytest.fixture(scope="class") def cwd(): return os.getcwd() @pytest.fixture(scope="class") def cwd_parent(cwd): return os.path.dirname(cwd) @pytest.fixture(scope="class") def cwd_child(): # pick any child directory of the dbt project return Path(os.getcwd()) / "macros" @pytest.fixture def write_profiles_yml(request): def _write_profiles_yml(profiles_dir, dbt_profile_contents): def cleanup(): rm_file(Path(profiles_dir) / "profiles.yml") request.addfinalizer(cleanup) write_file(yaml.safe_dump(dbt_profile_contents), profiles_dir, "profiles.yml") return _write_profiles_yml # https://gist.github.com/igniteflow/7267431?permalink_comment_id=2551951#gistcomment-2551951 @contextmanager def environ(env): """Temporarily set environment variables inside the context manager and fully restore previous environment afterwards """ original_env = {key: os.getenv(key) for key in env} os.environ.update(env) try: yield finally: for key, value in original_env.items(): if value is None: del os.environ[key] else: os.environ[key] = value class TestProfilesMayNotExist: def test_debug(self, project): # The database will not be able to connect; expect neither a pass or a failure (but not an exception) run_dbt(["debug", "--profiles-dir", "does_not_exist"], expect_pass=None) def test_deps(self, project): run_dbt(["deps", "--profiles-dir", "does_not_exist"]) class TestProfiles: def dbt_debug(self, project_dir_cli_arg=None, profiles_dir_cli_arg=None): # begin with no command-line args or user config (from profiles.yml) flags.set_from_args(Namespace(), {}) command = ["debug"] if project_dir_cli_arg: command.extend(["--project-dir", str(project_dir_cli_arg)]) if profiles_dir_cli_arg: command.extend(["--profiles-dir", str(profiles_dir_cli_arg)]) # get the output of `dbt debug` regardless of the exit code return run_dbt_and_capture(command, expect_pass=None) @pytest.mark.parametrize( "project_dir_cli_arg, working_directory", [ # 3 different scenarios for `--project-dir` flag and current working directory (None, "cwd"), # no --project-dir flag and cwd is project directory (None, "cwd_child"), # no --project-dir flag and cwd is a project subdirectory ("cwd", "cwd_parent"), # use --project-dir flag and cwd is outside of it ], ) def test_profiles( self, project_dir_cli_arg, working_directory, write_profiles_yml, dbt_profile_data, profiles_home_root, profiles_project_root, profiles_flag_root, profiles_env_root, request, ): """Verify priority order to search for profiles.yml configuration. Reverse priority order: 1. HOME directory 2. DBT_PROFILES_DIR environment variable 3. --profiles-dir command-line argument Specification later in this list will take priority over earlier ones, even when both are provided. """ # https://pypi.org/project/pytest-lazy-fixture/ is an alternative to using request.getfixturevalue if project_dir_cli_arg is not None: project_dir_cli_arg = request.getfixturevalue(project_dir_cli_arg) if working_directory is not None: working_directory = request.getfixturevalue(working_directory) # start in the specified directory if working_directory is not None: os.chdir(working_directory) # default case with profiles.yml in the HOME directory _, stdout = self.dbt_debug(project_dir_cli_arg) assert f"Using profiles.yml file at {profiles_home_root}" in stdout # set DBT_PROFILES_DIR environment variable for the remainder of the cases env_vars = {"DBT_PROFILES_DIR": profiles_env_root} with environ(env_vars): _, stdout = self.dbt_debug(project_dir_cli_arg) assert f"Using profiles.yml file at {profiles_env_root}" in stdout # This additional case is also within the context manager because we want to verify # that it takes priority even when the relevant environment variable is also set # set --profiles-dir on the command-line _, stdout = self.dbt_debug( project_dir_cli_arg, profiles_dir_cli_arg=profiles_flag_root ) assert f"Using profiles.yml file at {profiles_flag_root}" in stdout ================================================ FILE: tests/functional/profiles/test_profiles_yml.py ================================================ import pathlib from test_profile_dir import environ from dbt.cli.main import dbtRunner jinjaesque_password = "no{{jinja{%re{#ndering" profile_with_jinjaesque_password = f"""test: outputs: default: dbname: my_db host: localhost password: {jinjaesque_password} port: 12345 schema: dummy threads: 4 type: postgres user: peter.webb target: default """ profile_with_env_password = """test: outputs: default: dbname: my_db host: localhost password: "{{ env_var('DBT_PASSWORD') }}" port: 12345 schema: dummy threads: 4 type: postgres user: peter.webb target: default """ class TestProfileParsing: def write_profiles_yml(self, profiles_root, content) -> None: with open(pathlib.Path(profiles_root, "profiles.yml"), "w") as profiles_yml: profiles_yml.write(content) def test_password_not_jinja_rendered_when_invalid(self, project, profiles_root) -> None: """Verifies that passwords that contain Jinja control characters, but which are not valid Jinja, do not cause errors.""" self.write_profiles_yml(profiles_root, profile_with_jinjaesque_password) events = [] result = dbtRunner(callbacks=[events.append]).invoke(["parse"]) assert result.success for e in events: assert "no{{jinja{%re{#ndering" not in e.info.msg def test_password_jinja_rendered_when_valid(self, project, profiles_root) -> None: """Verifies that a password value that is valid Jinja is rendered as such, and that it doesn't cause problems if the resulting value looks like Jinja""" self.write_profiles_yml(profiles_root, profile_with_env_password) events = [] with environ({"DBT_PASSWORD": jinjaesque_password}): result = dbtRunner(callbacks=[events.append]).invoke(["parse"]) assert result.success assert project.adapter.config.credentials.password == jinjaesque_password ================================================ FILE: tests/functional/record/test_record.py ================================================ import os import pytest from dbt.tests.util import run_dbt TEMP_ENV_VARS = {} ENV_VARS_TO_SUSPEND = ["DBT_RECORDER_MODE"] @pytest.fixture(scope="session", autouse=True) def tests_setup_and_teardown(): # Will be executed before the first test old_environ = dict(os.environ) os.environ.update(TEMP_ENV_VARS) for env_var in ENV_VARS_TO_SUSPEND: os.environ.pop(env_var, default=None) yield # Will be executed after the last test os.environ.clear() os.environ.update(old_environ) @pytest.fixture(scope="session") def set_dbt_recorder_mode(): old_environ = os.environ os.environ["DBT_RECORDER_MODE"] = "record" yield os.environ = old_environ class TestRecord: def test_record_when_env_var_set(self, project, set_dbt_recorder_mode): run_dbt(["run"]) assert os.path.isfile(os.path.join(project.project_root, "recording.json")) ================================================ FILE: tests/functional/ref_override/test_custom_ref_kwargs.py ================================================ import pytest from dbt.tests.util import run_dbt # Custom ref macro that accepts and ignores a 'label' kwarg macros__custom_ref_sql = """ {% macro ref() %} {% set label = kwargs.get('label') %} {% set version = kwargs.get('version') or kwargs.get('v') %} {% set packagename = none %} {%- if (varargs | length) == 1 -%} {% set modelname = varargs[0] %} {%- else -%} {% set packagename = varargs[0] %} {% set modelname = varargs[1] %} {% endif %} {% set rel = None %} {% if packagename is not none %} {% set rel = builtins.ref(packagename, modelname, version=version) %} {% else %} {% set rel = builtins.ref(modelname, version=version) %} {% endif %} {% do return(rel) %} {% endmacro %} """ models__model_a_sql = """ select 1 as id, 'alice' as name """ models__model_b_sql = """ select * from {{ ref('model_a', label='staging') }} """ models__schema_yml = """ models: - name: model_a - name: model_b columns: - name: id data_tests: - relationships: to: ref('model_a', label='staging') field: id unit_tests: - name: my_unit_test model: model_b given: - input: ref('model_a', label='staging') rows: - {id: 1, name: 'alice'} expect: rows: - {id: 1, name: 'alice'} """ # Separate schema with only unit test for the "without flag" test models__schema_unit_only_yml = """ models: - name: model_a - name: model_b unit_tests: - name: my_unit_test model: model_b given: - input: ref('model_a', label='staging') rows: - {id: 1, name: 'alice'} expect: rows: - {id: 1, name: 'alice'} """ class TestCustomRefKwargsUnitTest: """Test that unit tests work with custom ref kwargs when flag is enabled.""" @pytest.fixture(scope="class") def project_config_update(self): return { "flags": { "support_custom_ref_kwargs": True, }, } @pytest.fixture(scope="class") def models(self): return { "model_a.sql": models__model_a_sql, "model_b.sql": models__model_b_sql, "schema.yml": models__schema_yml, } @pytest.fixture(scope="class") def macros(self): return {"ref.sql": macros__custom_ref_sql} def test_unit_test_with_custom_ref_kwargs(self, project): results = run_dbt(["run"]) assert len(results) == 2 results = run_dbt(["test", "--select", "test_type:unit"]) assert len(results) == 1 assert results[0].status == "pass" class TestCustomRefKwargsGenericTest: """Test that generic data tests work with custom ref kwargs when flag is enabled.""" @pytest.fixture(scope="class") def project_config_update(self): return { "flags": { "support_custom_ref_kwargs": True, }, } @pytest.fixture(scope="class") def models(self): return { "model_a.sql": models__model_a_sql, "model_b.sql": models__model_b_sql, "schema.yml": models__schema_yml, } @pytest.fixture(scope="class") def macros(self): return {"ref.sql": macros__custom_ref_sql} def test_generic_test_with_custom_ref_kwargs(self, project): results = run_dbt(["run"]) assert len(results) == 2 results = run_dbt(["test", "--select", "test_type:data"]) assert len(results) == 1 assert results[0].status == "pass" class TestCustomRefKwargsWithoutFlag: """Test that custom ref kwargs in unit tests fail without the behavior flag.""" @pytest.fixture(scope="class") def models(self): return { "model_a.sql": models__model_a_sql, "model_b.sql": models__model_b_sql, "schema.yml": models__schema_unit_only_yml, } @pytest.fixture(scope="class") def macros(self): return {"ref.sql": macros__custom_ref_sql} def test_unit_test_fails_without_flag(self, project): results = run_dbt(["run"]) assert len(results) == 2 results = run_dbt(["test", "--select", "test_type:unit"], expect_pass=False) assert len(results) == 1 assert ( "Unit test given inputs must be either a 'ref', 'source' or 'this' call" in results.results[0].message ) ================================================ FILE: tests/functional/ref_override/test_ref_override.py ================================================ import pytest from dbt.tests.util import check_relations_equal, run_dbt models__ref_override_sql = """ select * from {{ ref('seed_1') }} """ macros__ref_override_macro_sql = """ -- Macro to override ref and always return the same result {% macro ref(modelname) %} {% do return(builtins.ref(modelname).replace_path(identifier='seed_2')) %} {% endmacro %} """ seeds__seed_2_csv = """a,b 6,2 12,4 18,6""" seeds__seed_1_csv = """a,b 1,2 2,4 3,6""" class TestRefOverride: @pytest.fixture(scope="class") def models(self): return {"ref_override.sql": models__ref_override_sql} @pytest.fixture(scope="class") def macros(self): return {"ref_override_macro.sql": macros__ref_override_macro_sql} @pytest.fixture(scope="class") def seeds(self): return {"seed_2.csv": seeds__seed_2_csv, "seed_1.csv": seeds__seed_1_csv} def test_ref_override( self, project, ): run_dbt(["seed"]) run_dbt(["run"]) # We want it to equal seed_2 and not seed_1. If it's # still pointing at seed_1 then the override hasn't worked. check_relations_equal(project.adapter, ["ref_override", "seed_2"]) models__version_ref_override_sql = """ select * from {{ ref('versioned_model', version=1) }} """ models__package_ref_override_sql = """ select * from {{ ref('package', 'versioned_model') }} """ models__package_version_ref_override_sql = """ select * from {{ ref('package', 'versioned_model', version=1) }} """ models__v1_sql = """ select 1 """ models__v2_sql = """ select 2 """ schema__versions_yml = """ models: - name: versioned_model versions: - v: 1 - v: 2 """ macros__package_version_ref_override_macro_sql = """ -- Macro to override ref and always return the same result {% macro ref() %} -- extract user-provided positional and keyword arguments {% set version = kwargs.get('version') %} {% set packagename = none %} {%- if (varargs | length) == 1 -%} {% set modelname = varargs[0] %} {%- else -%} {% set packagename = varargs[0] %} {% set modelname = varargs[1] %} {% endif %} {%- set version_override = 2 -%} {%- set packagename_override = 'test' -%} -- call builtins.ref based on provided positional arguments {% if packagename is not none %} {% do return(builtins.ref(packagename_override, modelname, version=version_override)) %} {% else %} {% do return(builtins.ref(modelname, version=version_override)) %} {% endif %} {% endmacro %} """ class TestAdvancedRefOverride: @pytest.fixture(scope="class") def models(self): return { "version_ref_override.sql": models__version_ref_override_sql, "package_ref_override.sql": models__package_ref_override_sql, "package_version_ref_override.sql": models__package_version_ref_override_sql, "versioned_model_v1.sql": models__v1_sql, "versioned_model_v2.sql": models__v2_sql, "model.sql": models__v1_sql, "schema.yml": schema__versions_yml, } @pytest.fixture(scope="class") def macros(self): return {"ref_override_macro.sql": macros__package_version_ref_override_macro_sql} def test_ref_override( self, project, ): run_dbt(["run"]) # We want versioned_ref_override to equal to versioned_model_v2, otherwise the # ref override macro has not worked check_relations_equal(project.adapter, ["version_ref_override", "versioned_model_v2"]) check_relations_equal(project.adapter, ["package_ref_override", "versioned_model_v2"]) check_relations_equal( project.adapter, ["package_version_ref_override", "versioned_model_v2"] ) ================================================ FILE: tests/functional/relation_names/test_relation_name.py ================================================ import pytest from dbt.contracts.results import RunStatus from dbt.tests.util import run_dbt # Test coverage: A relation is a name for a database entity, i.e. a table or view. Every relation has # a name. These tests verify the default Postgres rules for relation names are followed. Adapters # may override connection rules and thus may have their own tests. seeds__seed = """col_A,col_B 1,2 3,4 5,6 """ models__basic_incremental = """ select * from {{ this.schema }}.seed {{ config({ "unique_key": "col_A", "materialized": "incremental" }) }} """ models__basic_table = """ select * from {{ this.schema }}.seed {{ config({ "materialized": "table" }) }} """ class TestGeneratedDDLNameRules: @classmethod def setup_class(self): self.incremental_filename = "my_name_is_51_characters_incremental_abcdefghijklmn" # length is 63 self.max_length_filename = ( "my_name_is_max_length_chars_abcdefghijklmnopqrstuvwxyz123456789" ) # length is 64 self.over_max_length_filename = ( "my_name_is_one_over_max_length_chats_abcdefghijklmnopqrstuvwxyz1" ) self.filename_for_backup_file = "my_name_is_52_characters_abcdefghijklmnopqrstuvwxyz0" @pytest.fixture(scope="class", autouse=True) def setUp(self, project): run_dbt(["seed"]) @pytest.fixture(scope="class") def seeds(self): return {"seed.csv": seeds__seed} @pytest.fixture(scope="class") def models(self): return { f"{self.incremental_filename}.sql": models__basic_incremental, f"{self.filename_for_backup_file}.sql": models__basic_table, f"{self.max_length_filename}.sql": models__basic_table, f"{self.over_max_length_filename}.sql": models__basic_table, } @pytest.fixture(scope="class") def project_config_update(self): return { "seeds": { "quote_columns": False, }, } # Backup table name generation: # 1. for len(relation name) <= 51, backfills # 2. for len(relation name) > 51 characters, overwrites # the last 12 characters with __dbt_backup def test_name_shorter_or_equal_to_63_passes(self, project): run_dbt( [ "run", "-s", f"{self.max_length_filename}", f"{self.filename_for_backup_file}", ], ) def test_long_name_passes_when_temp_tables_are_generated(self): run_dbt( [ "run", "-s", f"{self.incremental_filename}", ], ) # Run again to trigger incremental materialization run_dbt( [ "run", "-s", f"{self.incremental_filename}", ], ) # 63 characters is the character limit for a table name in a postgres database # (assuming compiled without changes from source) def test_name_longer_than_63_does_not_build(self): err_msg = ( "Relation name 'my_name_is_one_over_max" "_length_chats_abcdefghijklmnopqrstuvwxyz1' is longer than 63 characters" ) res = run_dbt( [ "run", "-s", self.over_max_length_filename, ], expect_pass=False, ) assert res[0].status == RunStatus.Error assert err_msg in res[0].message ================================================ FILE: tests/functional/relation_quoting/test_relation_quoting.py ================================================ import pytest from dbt.tests.util import read_file, run_dbt _SOURCES_YML = """ sources: - name: source_name database: source_database schema: source_schema tables: - name: customers """ class TestSourceQuotingGlobalConfigs: @pytest.fixture(scope="class") def project_config_update(self): # Postgres quoting configs are True by default -- turn them all to False to show they are not respected during source rendering return { "quoting": { "database": False, "schema": False, "identifier": False, }, } @pytest.fixture(scope="class") def models(self): return { "sources.yml": _SOURCES_YML, "model.sql": "select * from {{ source('source_name', 'customers') }}", } def test_sources_ignore_global_quoting_configs(self, project): run_dbt(["compile"]) generated_sql = read_file("target", "compiled", "test", "models", "model.sql") assert generated_sql == 'select * from "source_database"."source_schema"."customers"' class TestModelQuoting: @pytest.fixture(scope="class") def project_config_update(self): # Postgres quoting configs are True by default -- turn them all to False to show they are respected during model rendering return { "quoting": { "database": False, "schema": False, "identifier": False, }, } @pytest.fixture(scope="class") def models(self): return { "model.sql": "select 1 as id", "model_downstream.sql": "select * from {{ ref('model') }}", } def test_models_respect_global_quoting_configs(self, project): run_dbt(["compile"]) generated_sql = read_file("target", "compiled", "test", "models", "model_downstream.sql") assert generated_sql == f"select * from dbt.{project.test_schema}.model" ================================================ FILE: tests/functional/retry/fixtures.py ================================================ models__sample_model = """select 1 as id, baz as foo""" models__second_model = """select 1 as id, 2 as bar""" models__thread_model = """select idx as id""" models__union_model = """ select foo + bar as sum3 from {{ ref('sample_model') }} left join {{ ref('second_model') }} on sample_model.id = second_model.id """ schema_yml = """ models: - name: sample_model columns: - name: foo data_tests: - accepted_values: arguments: values: [3] quote: false config: severity: warn - name: second_model columns: - name: bar data_tests: - accepted_values: arguments: values: [3] quote: false config: severity: warn - name: union_model columns: - name: sum3 data_tests: - accepted_values: arguments: values: [3] quote: false """ macros__alter_timezone_sql = """ {% macro alter_timezone(timezone='America/Los_Angeles') %} {{ print('running a macro!') }} {% set sql %} SET TimeZone='{{ timezone }}'; {% endset %} {% do run_query(sql) %} {% do log("Timezone set to: " + timezone, info=True) %} {% endmacro %} """ macros__success_macro_sql = """ {% macro success_macro() %} {{ print('running a macro!') }} select 1 {% endmacro %} """ simple_model = """ select null as id """ simple_schema = """ models: - name: some_model columns: - name: id data_tests: - not_null """ schema_test_thread_yml = """ models: - name: thread_model columns: - name: id data_tests: - not_null """ ================================================ FILE: tests/functional/retry/test_retry.py ================================================ from pathlib import Path from shutil import copytree, move import pytest from dbt.contracts.results import RunStatus, TestStatus from dbt.exceptions import DbtRuntimeError, TargetNotFoundError from dbt.tests.util import ( rm_file, run_dbt, run_dbt_and_capture, update_config_file, write_file, ) from tests.functional.retry.fixtures import ( macros__alter_timezone_sql, macros__success_macro_sql, models__sample_model, models__second_model, models__union_model, schema_yml, simple_model, simple_schema, ) class TestCustomTargetRetry: @pytest.fixture(scope="class") def models(self): return { "sample_model.sql": models__sample_model, "second_model.sql": models__second_model, "schema.yml": schema_yml, } def test_custom_target(self, project): run_dbt(["build", "--select", "second_model"]) run_dbt( ["build", "--select", "sample_model", "--target-path", "target2"], expect_pass=False ) # Regular retry - this is a no op because it's actually running `dbt build --select second_model` # agian because it's looking at the default target since the custom_target wasn't passed in results = run_dbt(["retry"]) assert len(results) == 0 # Retry with custom target after fixing the error fixed_sql = "select 1 as id, 1 as foo" write_file(fixed_sql, "models", "sample_model.sql") results = run_dbt(["retry", "--state", "target2"]) expected_statuses = { "sample_model": RunStatus.Success, "accepted_values_sample_model_foo__False__3": TestStatus.Warn, } assert {n.node.name: n.status for n in results.results} == expected_statuses write_file(models__sample_model, "models", "sample_model.sql") class BaseTestRetry: @pytest.fixture(scope="class") def models(self): return { "sample_model.sql": models__sample_model, "second_model.sql": models__second_model, "union_model.sql": models__union_model, "schema.yml": schema_yml, } @pytest.fixture(scope="class") def macros(self): return { "alter_timezone.sql": macros__alter_timezone_sql, "success_macro.sql": macros__success_macro_sql, } class TestRetryNoPreviousRun(BaseTestRetry): def test_no_previous_run(self, project): with pytest.raises( DbtRuntimeError, match="Could not find previous run in 'target' target directory" ): run_dbt(["retry"]) with pytest.raises( DbtRuntimeError, match="Could not find previous run in 'walmart' target directory" ): run_dbt(["retry", "--state", "walmart"]) class TestRetryPreviousRun(BaseTestRetry): def test_previous_run(self, project): # Regular build results = run_dbt(["build"], expect_pass=False) expected_statuses = { "sample_model": RunStatus.Error, "second_model": RunStatus.Success, "union_model": RunStatus.Skipped, "accepted_values_sample_model_foo__False__3": RunStatus.Skipped, "accepted_values_second_model_bar__False__3": TestStatus.Warn, "accepted_values_union_model_sum3__False__3": RunStatus.Skipped, } assert {n.node.name: n.status for n in results.results} == expected_statuses # Ignore second_model which succeeded results = run_dbt(["retry"], expect_pass=False) expected_statuses = { "sample_model": RunStatus.Error, "union_model": RunStatus.Skipped, "accepted_values_union_model_sum3__False__3": RunStatus.Skipped, "accepted_values_sample_model_foo__False__3": RunStatus.Skipped, } assert {n.node.name: n.status for n in results.results} == expected_statuses # Fix sample model and retry, everything should pass fixed_sql = "select 1 as id, 1 as foo" write_file(fixed_sql, "models", "sample_model.sql") results = run_dbt(["retry"]) expected_statuses = { "sample_model": RunStatus.Success, "union_model": RunStatus.Success, "accepted_values_union_model_sum3__False__3": TestStatus.Pass, "accepted_values_sample_model_foo__False__3": TestStatus.Warn, } assert {n.node.name: n.status for n in results.results} == expected_statuses # No failures in previous run, nothing to retry results = run_dbt(["retry"]) expected_statuses = {} assert {n.node.name: n.status for n in results.results} == expected_statuses write_file(models__sample_model, "models", "sample_model.sql") class TestRetryWarnError(BaseTestRetry): def test_warn_error(self, project): # Our test command should succeed when run normally... results = run_dbt(["build", "--select", "second_model"]) # ...but it should fail when run with warn-error, due to a warning... results = run_dbt(["--warn-error", "build", "--select", "second_model"], expect_pass=False) expected_statuses = { "second_model": RunStatus.Success, "accepted_values_second_model_bar__False__3": TestStatus.Fail, } assert {n.node.name: n.status for n in results.results} == expected_statuses # Retry regular, should pass run_dbt(["retry"]) # Retry with --warn-error, should fail run_dbt(["--warn-error", "retry"], expect_pass=False) class TestRetryRunOperation(BaseTestRetry): def test_run_operation(self, project): results, log_output = run_dbt_and_capture( ["run-operation", "alter_timezone", "--args", "{timezone: abc}"], expect_pass=False ) assert "running a macro!" in log_output expected_statuses = { "macro.test.alter_timezone": RunStatus.Error, } assert {n.unique_id: n.status for n in results.results} == expected_statuses results, log_output = run_dbt_and_capture(["retry"], expect_pass=False) assert "running a macro!" in log_output assert {n.unique_id: n.status for n in results.results} == expected_statuses class TestRetrySuccessfulRunOperation(BaseTestRetry): def test_run_operation(self, project): results, log_output = run_dbt_and_capture(["run-operation", "success_macro"]) assert "running a macro!" in log_output expected_statuses = { "macro.test.success_macro": RunStatus.Success, } assert {n.unique_id: n.status for n in results.results} == expected_statuses results, log_output = run_dbt_and_capture(["retry"]) assert "running a macro!" not in log_output assert {n.unique_id: n.status for n in results.results} == expected_statuses class TestRetryRemovedFile(BaseTestRetry): def test_removed_file(self, project): run_dbt(["build"], expect_pass=False) rm_file("models", "sample_model.sql") with pytest.raises( TargetNotFoundError, match="depends on a node named 'sample_model' which was not found" ): run_dbt(["retry"], expect_pass=False) write_file(models__sample_model, "models", "sample_model.sql") class TestRetryRemovedFileLeafNode(BaseTestRetry): def test_removed_file_leaf_node(self, project): write_file(models__sample_model, "models", "third_model.sql") run_dbt(["build"], expect_pass=False) rm_file("models", "third_model.sql") with pytest.raises(ValueError, match="Couldn't find model 'model.test.third_model'"): run_dbt(["retry"], expect_pass=False) class TestFailFast: @pytest.fixture(scope="class") def models(self): return { "sample_model.sql": models__sample_model, "second_model.sql": models__second_model, "union_model.sql": models__union_model, "final_model.sql": "select * from {{ ref('union_model') }};", } def test_fail_fast(self, project): results = run_dbt(["--fail-fast", "build"], expect_pass=False) assert {r.node.unique_id: r.status for r in results.results} == { "model.test.sample_model": RunStatus.Error, "model.test.second_model": RunStatus.Success, "model.test.union_model": RunStatus.Skipped, "model.test.final_model": RunStatus.Skipped, } # Check that retry inherits fail-fast from upstream command (build) results = run_dbt(["retry"], expect_pass=False) assert {r.node.unique_id: r.status for r in results.results} == { "model.test.sample_model": RunStatus.Error, "model.test.union_model": RunStatus.Skipped, "model.test.final_model": RunStatus.Skipped, } fixed_sql = "select 1 as id, 1 as foo" write_file(fixed_sql, "models", "sample_model.sql") results = run_dbt(["retry"], expect_pass=False) assert {r.node.unique_id: r.status for r in results.results} == { "model.test.sample_model": RunStatus.Success, "model.test.union_model": RunStatus.Success, "model.test.final_model": RunStatus.Error, } results = run_dbt(["retry"], expect_pass=False) assert {r.node.unique_id: r.status for r in results.results} == { "model.test.final_model": RunStatus.Error, } fixed_sql = "select * from {{ ref('union_model') }}" write_file(fixed_sql, "models", "final_model.sql") results = run_dbt(["retry"]) assert {r.node.unique_id: r.status for r in results.results} == { "model.test.final_model": RunStatus.Success, } results = run_dbt(["retry"]) assert {r.node.unique_id: r.status for r in results.results} == {} class TestRetryResourceType: @pytest.fixture(scope="class") def models(self): return { "null_model.sql": simple_model, "schema.yml": simple_schema, } def test_resource_type(self, project): # test multiple options in single string results = run_dbt(["build", "--select", "null_model", "--resource-type", "test model"]) assert len(results) == 1 # nothing to do results = run_dbt(["retry"]) assert len(results) == 0 # test multiple options in multiple args results = run_dbt( [ "build", "--select", "null_model", "--resource-type", "test", "--resource-type", "model", ] ) assert len(results) == 1 # nothing to do results = run_dbt(["retry"]) assert len(results) == 0 # test single all option results = run_dbt(["build", "--select", "null_model", "--resource-type", "all"]) assert len(results) == 1 # nothing to do results = run_dbt(["retry"]) assert len(results) == 0 class TestRetryOverridePath: @pytest.fixture(scope="class") def models(self): return { "sample_model.sql": models__sample_model, } def test_retry(self, project): project_root = project.project_root proj_location_1 = project_root / "proj_location_1" proj_location_2 = project_root / "proj_location_2" copytree(project_root, proj_location_1) run_dbt(["run", "--project-dir", "proj_location_1"], expect_pass=False) move(proj_location_1, proj_location_2) run_dbt(["retry", "--project-dir", "proj_location_2"], expect_pass=False) class TestRetryVars: @pytest.fixture(scope="class") def models(self): return { "sample_model.sql": "select {{ var('myvar_a', '1') + var('myvar_b', '2') }} as mycol", } def test_retry(self, project): # pass because default vars works run_dbt(["run"]) run_dbt(["run", "--vars", '{"myvar_a": "12", "myvar_b": "3 4"}'], expect_pass=False) # fail because vars are invalid, this shows that the last passed vars are being used # instead of using the default vars run_dbt(["retry"], expect_pass=False) results = run_dbt(["retry", "--vars", '{"myvar_a": "12", "myvar_b": "34"}']) assert len(results) == 1 class TestRetryFullRefresh: @pytest.fixture(scope="class") def models(self): return { "sample_model.sql": "{% if flags.FULL_REFRESH %} this is invalid sql {% else %} select 1 as mycol {% endif %}", } def test_retry(self, project): # This run should fail with invalid sql... run_dbt(["run", "--full-refresh"], expect_pass=False) # ...and so should this one, since the effect of the full-refresh parameter should persist. results = run_dbt(["retry"], expect_pass=False) assert len(results) == 1 class TestRetryTargetPathEnvVar: @pytest.fixture(scope="class") def models(self): return { "sample_model.sql": models__sample_model, } def test_retry_target_path_env_var(self, project, monkeypatch): monkeypatch.setenv("DBT_TARGET_PATH", "artifacts") run_dbt(["run"], expect_pass=False) write_file(models__second_model, "models", "sample_model.sql") results = run_dbt(["retry"]) assert len(results) == 1 class TestRetryTargetPathFlag: @pytest.fixture(scope="class") def models(self): return { "sample_model.sql": models__sample_model, } def test_retry_target_path_flag(self, project): run_dbt(["run", "--target-path", "target"], expect_pass=False) project_root = project.project_root move(project_root / "target", project_root / "artifacts") write_file(models__second_model, "models", "sample_model.sql") results = run_dbt(["retry", "--state", "artifacts", "--target-path", "my_target_path"]) assert len(results) == 1 assert Path("my_target_path").is_dir() class TestRetryHooksAlwaysRun: @pytest.fixture(scope="class") def project_config_update(self): return { "on-run-start": ["select 1;"], "on-run-end": ["select 2;"], } @pytest.fixture(scope="class") def models(self): return { "sample_model.sql": models__sample_model, } def test_retry_hooks_always_run(self, project): res = run_dbt(["run", "--target-path", "target"], expect_pass=False) assert len(res) == 3 write_file(models__second_model, "models", "sample_model.sql") res = run_dbt(["retry", "--state", "target"]) assert len(res) == 3 class TestFixRetryHook: @pytest.fixture(scope="class") def project_config_update(self): return { "flags": { "skip_nodes_if_on_run_start_fails": True, }, "on-run-start": [ "select 1 as id", "select column_does_not_exist", "select 2 as id", ], } @pytest.fixture(scope="class") def models(self): return { "sample_model.sql": "select 1 as id, 1 as foo", "second_model.sql": models__second_model, "union_model.sql": models__union_model, } def test_fix_retry_hook(self, project): res = run_dbt(["run"], expect_pass=False) assert {r.node.unique_id: r.status for r in res.results} == { "operation.test.test-on-run-start-0": RunStatus.Success, "operation.test.test-on-run-start-1": RunStatus.Error, "operation.test.test-on-run-start-2": RunStatus.Skipped, "model.test.sample_model": RunStatus.Skipped, "model.test.second_model": RunStatus.Skipped, "model.test.union_model": RunStatus.Skipped, } res = run_dbt(["retry"], expect_pass=False) assert {r.node.unique_id: r.status for r in res.results} == { "operation.test.test-on-run-start-0": RunStatus.Success, "operation.test.test-on-run-start-1": RunStatus.Error, "operation.test.test-on-run-start-2": RunStatus.Skipped, "model.test.sample_model": RunStatus.Skipped, "model.test.second_model": RunStatus.Skipped, "model.test.union_model": RunStatus.Skipped, } new_dbt_project_yml = { "flags": { "skip_nodes_if_on_run_start_fails": True, }, "on-run-start": [ "select 1 as id", "select 3 as id", "select 2 as id", ], } update_config_file(new_dbt_project_yml, project.project_root, "dbt_project.yml") res = run_dbt(["retry"]) assert {r.node.unique_id: r.status for r in res.results} == { "operation.test.test-on-run-start-0": RunStatus.Success, "operation.test.test-on-run-start-1": RunStatus.Success, "operation.test.test-on-run-start-2": RunStatus.Success, "model.test.sample_model": RunStatus.Success, "model.test.second_model": RunStatus.Success, "model.test.union_model": RunStatus.Success, } ================================================ FILE: tests/functional/retry/test_retry_threads.py ================================================ import pytest from dbt.contracts.results import RunStatus, TestStatus from dbt.tests.util import run_dbt, write_file from tests.functional.retry.fixtures import models__thread_model, schema_test_thread_yml class TestCustomThreadRetry: @pytest.fixture(scope="class") def models(self): return { "thread_model.sql": models__thread_model, "schema.yml": schema_test_thread_yml, } def test_thread_target(self, project): # Passing Threads to check results = run_dbt( ["build", "--select", "thread_model", "--threads", "3"], expect_pass=False ) expected_statuses = { "thread_model": RunStatus.Error, "not_null_thread_model_id": TestStatus.Skipped, } assert {n.node.name: n.status for n in results.results} == expected_statuses # Retry Running the Dbt with simple Retry results = run_dbt(["retry", "--threads", "2"], expect_pass=False) expected_statuses = { "thread_model": RunStatus.Error, "not_null_thread_model_id": TestStatus.Skipped, } assert {n.node.name: n.status for n in results.results} == expected_statuses assert results.args["threads"] == 2 # running with retry withour threads results = run_dbt(["retry"], expect_pass=False) expected_statuses = { "thread_model": RunStatus.Error, "not_null_thread_model_id": TestStatus.Skipped, } assert {n.node.name: n.status for n in results.results} == expected_statuses assert results.args["threads"] == 2 # Retry with fixing the model and running with --threads 1 fixed_sql = "select 1 as id" write_file(fixed_sql, "models", "thread_model.sql") results = run_dbt(["retry", "--threads", "1"]) expected_statuses = { "thread_model": RunStatus.Success, "not_null_thread_model_id": TestStatus.Pass, } assert {n.node.name: n.status for n in results.results} == expected_statuses assert results.args["threads"] == 1 ================================================ FILE: tests/functional/run_operations/fixtures.py ================================================ happy_macros_sql = """ {% macro no_args() %} {% if execute %} {% call statement(auto_begin=True) %} create table "{{ schema }}"."no_args" (id int); commit; {% endcall %} {% endif %} {% endmacro %} {% macro table_name_args(table_name) %} {% if execute %} {% call statement(auto_begin=True) %} create table "{{ schema }}"."{{ table_name }}" (id int); commit; {% endcall %} {% endif %} {% endmacro %} {% macro select_something(name) %} {% set query %} select 'hello, {{ name }}' as name {% endset %} {% set table = run_query(query) %} {% if table.columns['name'][0] != 'hello, world' %} {% do exceptions.raise_compiler_error("unexpected result: " ~ table) %} {% endif %} {% endmacro %} {% macro vacuum(table_name) %} {% set query %} vacuum "{{ schema }}"."{{ table_name }}" {% endset %} {% do run_query(query) %} {% endmacro %} {% macro vacuum_ref(ref_target) %} {% set query %} vacuum {{ ref(ref_target) }} {% endset %} {% do run_query(query) %} {% endmacro %} {% macro log_graph() %} {% for node in graph.nodes.values() %} {{ log((node | string), info=True)}} {% endfor %} {% endmacro %} {% macro print_something() %} {{ print("You're doing awesome!") }} {% endmacro %} """ sad_macros_sql = """ {% macro syntax_error() %} {% if execute %} {% call statement() %} select NOPE NOT A VALID QUERY {% endcall %} {% endif %} {% endmacro %} """ model_sql = """ select 1 as id """ private_model_sql = """ select 1 as id """ groups_yml = """ groups: - name: analytics owner: name: analytics_owner """ private_model_schema_yml = """ models: - name: private_model access: private group: analytics """ ref_private_model_macro_sql = """ {% macro ref_private_model() %} {% set result = run_query("select * from " ~ ref('private_model')) %} {{ log("Successfully referenced private model", info=true) }} {% endmacro %} """ ================================================ FILE: tests/functional/run_operations/test_run_operations.py ================================================ import os import pytest import yaml from dbt.artifacts.schemas.results import RunStatus from dbt.tests.util import ( check_table_does_exist, mkdir, rm_dir, rm_file, run_dbt, run_dbt_and_capture, write_file, ) from dbt_common.exceptions import UndefinedMacroError from tests.functional.run_operations.fixtures import ( groups_yml, happy_macros_sql, model_sql, private_model_schema_yml, private_model_sql, ref_private_model_macro_sql, sad_macros_sql, ) class TestOperations: @pytest.fixture(scope="class") def models(self): return {"model.sql": model_sql} @pytest.fixture(scope="class") def macros(self): return {"happy_macros.sql": happy_macros_sql, "sad_macros.sql": sad_macros_sql} @pytest.fixture(scope="class") def dbt_profile_data(self, unique_schema): return { "test": { "outputs": { "default": { "type": "postgres", "threads": 4, "host": "localhost", "port": int(os.getenv("POSTGRES_TEST_PORT", 5432)), "user": os.getenv("POSTGRES_TEST_USER", "root"), "pass": os.getenv("POSTGRES_TEST_PASS", "password"), "dbname": os.getenv("POSTGRES_TEST_DATABASE", "dbt"), "schema": unique_schema, }, "noaccess": { "type": "postgres", "threads": 4, "host": "localhost", "port": int(os.getenv("POSTGRES_TEST_PORT", 5432)), "user": "noaccess", "pass": "password", "dbname": os.getenv("POSTGRES_TEST_DATABASE", "dbt"), "schema": unique_schema, }, }, "target": "default", }, } def run_operation(self, macro, expect_pass=True, extra_args=None, **kwargs): args = ["run-operation", macro] if kwargs: args.extend(("--args", yaml.safe_dump(kwargs))) if extra_args: args.extend(extra_args) return run_dbt(args, expect_pass=expect_pass) def test_macro_noargs(self, project): self.run_operation("no_args") check_table_does_exist(project.adapter, "no_args") def test_macro_args(self, project): self.run_operation("table_name_args", table_name="my_fancy_table") check_table_does_exist(project.adapter, "my_fancy_table") def test_macro_exception(self, project): self.run_operation("syntax_error", False) def test_macro_missing(self, project): with pytest.raises( UndefinedMacroError, match="dbt could not find a macro with the name 'this_macro_does_not_exist' in any package", ): self.run_operation("this_macro_does_not_exist", False) def test_cannot_connect(self, project): self.run_operation("no_args", extra_args=["--target", "noaccess"], expect_pass=False) def test_vacuum(self, project): run_dbt(["run"]) # this should succeed self.run_operation("vacuum", table_name="model") def test_vacuum_ref(self, project): run_dbt(["run"]) # this should succeed self.run_operation("vacuum_ref", ref_target="model") def test_select(self, project): self.run_operation("select_something", name="world") def test_access_graph(self, project): self.run_operation("log_graph") def test_print(self, project): # Tests that calling the `print()` macro does not cause an exception self.run_operation("print_something") def test_run_operation_local_macro(self, project): pkg_macro = """ {% macro something_cool() %} {{ log("something cool", info=true) }} {% endmacro %} """ mkdir("pkg/macros") write_file(pkg_macro, "pkg/macros/something_cool.sql") pkg_yaml = """ packages: - local: pkg """ write_file(pkg_yaml, "packages.yml") pkg_dbt_project = """ name: 'pkg' """ write_file(pkg_dbt_project, "pkg/dbt_project.yml") run_dbt(["deps"]) results, log_output = run_dbt_and_capture(["run-operation", "something_cool"]) for result in results: if result.status == RunStatus.Skipped: continue timing_keys = [timing.name for timing in result.timing] assert timing_keys == ["compile", "execute"] assert "something cool" in log_output results, log_output = run_dbt_and_capture(["run-operation", "pkg.something_cool"]) for result in results: if result.status == RunStatus.Skipped: continue timing_keys = [timing.name for timing in result.timing] assert timing_keys == ["compile", "execute"] assert "something cool" in log_output rm_dir("pkg") rm_file("packages.yml") class TestRunOperationRefPrivateModel: """Regression test for https://github.com/dbt-labs/dbt-core/issues/8248 Macros invoked via run-operation should be able to ref() private models, since macros are outside the group/access control system. """ @pytest.fixture(scope="class") def models(self): return { "private_model.sql": private_model_sql, "schema.yml": private_model_schema_yml, "groups.yml": groups_yml, } @pytest.fixture(scope="class") def macros(self): return {"ref_private_model.sql": ref_private_model_macro_sql} def test_run_operation_ref_private_model(self, project): run_dbt(["run"]) results = run_dbt(["run-operation", "ref_private_model"]) assert results.results[0].status == RunStatus.Success ================================================ FILE: tests/functional/run_query/test_types.py ================================================ import pytest from dbt.contracts.results import NodeStatus from dbt.tests.util import run_dbt macros_sql = """ {% macro test_array_results() %} {% set sql %} select ARRAY[1, 2, 3, 4] as mydata {% endset %} {% set result = run_query(sql) %} {% set value = result.columns['mydata'][0] %} {# This will be json-stringified #} {% if value != "[1, 2, 3, 4]" %} {% do exceptions.raise_compiler_error("Value was " ~ value) %} {% endif %} {% endmacro %} """ class TestTypes: @pytest.fixture(scope="class") def macros(self): return { "macros.sql": macros_sql, } def test_nested_types(self, project): result = run_dbt(["run-operation", "test_array_results"]) assert result.results[0].status == NodeStatus.Success ================================================ FILE: tests/functional/sample_mode/test_sample_mode.py ================================================ from datetime import datetime from typing import Optional import freezegun import pytest import pytz from dbt.artifacts.resources.types import BatchSize from dbt.event_time.sample_window import SampleWindow from dbt.events.types import JinjaLogInfo from dbt.materializations.incremental.microbatch import MicrobatchBuilder from dbt.tests.util import read_file, relation_from_name, run_dbt, write_file from dbt_common.events.event_catcher import EventCatcher input_model_sql = """ {{ config(materialized='table', event_time='event_time') }} select 1 as id, TIMESTAMP '2020-01-01 01:25:00-0' as event_time UNION ALL select 2 as id, TIMESTAMP '2025-01-02 13:47:00-0' as event_time UNION ALL select 3 as id, TIMESTAMP '2025-01-03 01:32:00-0' as event_time """ later_input_model_sql = """ {{ config(materialized='table', event_time='event_time') }} select 1 as id, TIMESTAMP '2020-01-01 01:25:00-0' as event_time UNION ALL select 2 as id, TIMESTAMP '2025-01-02 13:47:00-0' as event_time UNION ALL select 3 as id, TIMESTAMP '2025-01-03 01:32:00-0' as event_time UNION ALL select 4 as id, TIMESTAMP '2025-01-04 14:32:00-0' as event_time UNION ALL select 5 as id, TIMESTAMP '2025-01-05 20:32:00-0' as event_time UNION ALL select 6 as id, TIMESTAMP '2025-01-06 12:32:00-0' as event_time """ input_seed_csv = """id,event_time 1,'2020-01-01 01:25:00-0' 2,'2025-01-02 13:47:00-0' 3,'2025-01-03 01:32:00-0' """ seed_properties_yml = """ seeds: - name: input_seed config: event_time: event_time column_types: event_time: timestamp """ sample_mode_model_sql = """ {{ config(materialized='table', event_time='event_time') }} {% if execute %} {{ log("Sample: " ~ invocation_args_dict.get("sample"), info=true) }} {% endif %} SELECT * FROM {{ ref("input_model") }} """ sample_input_seed_sql = """ {{ config(materialized='table') }} SELECT * FROM {{ ref("input_seed") }} """ sample_microbatch_model_sql = """ {{ config(materialized='incremental', incremental_strategy='microbatch', event_time='event_time', batch_size='day', lookback=3, begin='2024-12-25', unique_key='id')}} {% if execute %} {{ log("batch.event_time_start: "~ model.batch.event_time_start, info=True)}} {{ log("batch.event_time_end: "~ model.batch.event_time_end, info=True)}} {% endif %} SELECT * FROM {{ ref("input_model") }} """ sample_incremental_merge_sql = """ {{ config(materialized='incremental', incremental_strategy='merge', unique_key='id')}} {% if execute %} {{ log("is_incremental: " ~ is_incremental(), info=true) }} {{ log("sample: " ~ invocation_args_dict.get("sample"), info=true) }} {% endif %} SELECT * FROM {{ ref("input_model") }} {% if is_incremental() %} WHERE event_time >= (SELECT max(event_time) FROM {{ this }}) {% endif %} """ snapshot_input_model_sql = """ {% snapshot snapshot_input_model %} {{ config(strategy='timestamp', unique_key='id', updated_at='event_time', event_time='event_time') }} select * from {{ ref('input_model') }} {% endsnapshot %} """ model_from_snapshot_sql = """ {{ config(materialized='table') }} SELECT * FROM {{ ref('snapshot_input_model') }} """ class BaseSampleMode: # TODO This is now used in 3 test files, it might be worth turning into a full test utility method def assert_row_count(self, project, relation_name: str, expected_row_count: int): relation = relation_from_name(project.adapter, relation_name) result = project.run_sql(f"select count(*) as num_rows from {relation}", fetch="one") if result[0] != expected_row_count: # running show for debugging run_dbt(["show", "--inline", f"select * from {relation}"]) assert result[0] == expected_row_count def drop_table(self, project, relation_name: str): relation = relation_from_name(project.adapter, "snapshot_input_model") project.run_sql(f"drop table if exists {relation}") class TestBasicSampleMode(BaseSampleMode): @pytest.fixture(scope="class") def models(self): return { "input_model.sql": input_model_sql, "sample_mode_model.sql": sample_mode_model_sql, } @pytest.fixture def event_catcher(self) -> EventCatcher: return EventCatcher(event_to_catch=JinjaLogInfo) # type: ignore @pytest.mark.parametrize( "dbt_command,run_sample_mode,expected_row_count", [ ("run", True, 2), ("run", False, 3), ("build", True, 2), ("build", False, 3), ], ) @freezegun.freeze_time("2025-01-03T02:03:0Z") def test_sample_mode( self, project, event_catcher: EventCatcher, dbt_command: str, run_sample_mode: bool, expected_row_count: int, ): run_args = [dbt_command] expected_sample = None if run_sample_mode: run_args.append("--sample=1 day") expected_sample = SampleWindow( start=datetime(2025, 1, 2, 2, 3, 0, 0, tzinfo=pytz.UTC), end=datetime(2025, 1, 3, 2, 3, 0, 0, tzinfo=pytz.UTC), ) _ = run_dbt(run_args, callbacks=[event_catcher.catch]) assert len(event_catcher.caught_events) == 1 assert event_catcher.caught_events[0].info.msg == f"Sample: {expected_sample}" # type: ignore self.assert_row_count( project=project, relation_name="sample_mode_model", expected_row_count=expected_row_count, ) class TestMicrobatchSampleMode(BaseSampleMode): @pytest.fixture(scope="class") def models(self): return { "input_model.sql": input_model_sql, "sample_microbatch_model.sql": sample_microbatch_model_sql, } @pytest.fixture def event_time_start_catcher(self) -> EventCatcher: return EventCatcher(event_to_catch=JinjaLogInfo, predicate=lambda event: "batch.event_time_start" in event.info.msg) # type: ignore @pytest.fixture def event_time_end_catcher(self) -> EventCatcher: return EventCatcher(event_to_catch=JinjaLogInfo, predicate=lambda event: "batch.event_time_end" in event.info.msg) # type: ignore @freezegun.freeze_time("2025-01-03T02:03:0Z") def test_sample_mode( self, project, event_time_end_catcher: EventCatcher, event_time_start_catcher: EventCatcher, ): expected_batches = [ ("2025-01-01 00:00:00", "2025-01-02 00:00:00"), ("2025-01-02 00:00:00", "2025-01-03 00:00:00"), ("2025-01-03 00:00:00", "2025-01-04 00:00:00"), ] expected_filters = [ "event_time >= '2025-01-01 02:03:00+00:00' and event_time < '2025-01-02 00:00:00+00:00'", "event_time >= '2025-01-02 00:00:00+00:00' and event_time < '2025-01-03 00:00:00+00:00'", "event_time >= '2025-01-03 00:00:00+00:00' and event_time < '2025-01-03 02:03:00+00:00'", ] _ = run_dbt( ["run", "--sample=2 day"], callbacks=[event_time_end_catcher.catch, event_time_start_catcher.catch], ) assert len(event_time_start_catcher.caught_events) == len(expected_batches) assert len(event_time_end_catcher.caught_events) == len(expected_batches) for index in range(len(expected_batches)): assert expected_batches[index][0] in event_time_start_catcher.caught_events[index].info.msg # type: ignore assert expected_batches[index][1] in event_time_end_catcher.caught_events[index].info.msg # type: ignore batch_id = MicrobatchBuilder.format_batch_start( datetime.fromisoformat(expected_batches[index][0]), BatchSize.day ) batch_file_name = f"sample_microbatch_model_{batch_id}.sql" compiled_sql = read_file( project.project_root, "target", "compiled", "test", "models", "sample_microbatch_model", batch_file_name, ) assert expected_filters[index] in compiled_sql # The first row of the "input_model" should be excluded from the sample because # it falls outside of the filter for the first batch (which is only doing a _partial_ batch selection) self.assert_row_count( project=project, relation_name="sample_microbatch_model", expected_row_count=2, ) class TestIncrementalModelSampleModeRelative(BaseSampleMode): @pytest.fixture(scope="class") def models(self): return { "input_model.sql": input_model_sql, "sample_incremental_merge.sql": sample_incremental_merge_sql, } @pytest.fixture def event_catcher(self) -> EventCatcher: return EventCatcher(event_to_catch=JinjaLogInfo, predicate=lambda event: "is_incremental: True" in event.info.msg) # type: ignore @pytest.mark.parametrize( "sample,expected_rows", [ (None, 6), ("3 days", 6), ("2 days", 5), ], ) @freezegun.freeze_time("2025-01-06T18:03:0Z") def test_incremental_model_sample( self, project, event_catcher: EventCatcher, sample: Optional[str], expected_rows: int, ): # writing the input_model is necessary because we've parametrized the test # thus the "later_input_model" will still be present on the "non-first" runs write_file(input_model_sql, "models", "input_model.sql") # --full-refresh is necessary because we've parametrized the test _ = run_dbt(["run", "--full-refresh"], callbacks=[event_catcher.catch]) assert len(event_catcher.caught_events) == 0 self.assert_row_count( project=project, relation_name="sample_incremental_merge", expected_row_count=3, ) # update the input file to have more rows write_file(later_input_model_sql, "models", "input_model.sql") run_args = ["run"] if sample is not None: run_args.extend([f"--sample={sample}"]) _ = run_dbt(run_args, callbacks=[event_catcher.catch]) assert len(event_catcher.caught_events) == 1 self.assert_row_count( project=project, relation_name="sample_incremental_merge", expected_row_count=expected_rows, ) class TestIncrementalModelSampleModeSpecific(BaseSampleMode): # This had to be split out from the "relative" tests because `freezegun.freezetime` # breaks how timestamps get created. @pytest.fixture(scope="class") def models(self): return { "input_model.sql": input_model_sql, "sample_incremental_merge.sql": sample_incremental_merge_sql, } @pytest.fixture def event_catcher(self) -> EventCatcher: return EventCatcher(event_to_catch=JinjaLogInfo, predicate=lambda event: "is_incremental: True" in event.info.msg) # type: ignore @pytest.mark.parametrize( "sample,expected_rows", [ (None, 6), ("{'start': '2025-01-03', 'end': '2025-01-07'}", 6), ("{'start': '2025-01-04', 'end': '2025-01-06'}", 5), ("{'start': '2025-01-05', 'end': '2025-01-07'}", 5), ("{'start': '2024-12-31', 'end': '2025-01-03'}", 3), ], ) def test_incremental_model_sample( self, project, event_catcher: EventCatcher, sample: Optional[str], expected_rows: int, ): # writing the input_model is necessary because we've parametrized the test # thus the "later_input_model" will still be present on the "non-first" runs write_file(input_model_sql, "models", "input_model.sql") # --full-refresh is necessary because we've parametrized the test _ = run_dbt(["run", "--full-refresh"], callbacks=[event_catcher.catch]) assert len(event_catcher.caught_events) == 0 self.assert_row_count( project=project, relation_name="sample_incremental_merge", expected_row_count=3, ) # update the input file to have more rows write_file(later_input_model_sql, "models", "input_model.sql") run_args = ["run"] if sample is not None: run_args.extend([f"--sample={sample}"]) _ = run_dbt(run_args, callbacks=[event_catcher.catch]) assert len(event_catcher.caught_events) == 1 self.assert_row_count( project=project, relation_name="sample_incremental_merge", expected_row_count=expected_rows, ) class TestSampleSeedRefs(BaseSampleMode): @pytest.fixture(scope="class") def seeds(self): return { "input_seed.csv": input_seed_csv, "properties.yml": seed_properties_yml, } @pytest.fixture(scope="class") def models(self): return { "sample_input_seed.sql": sample_input_seed_sql, } @pytest.mark.parametrize( "run_sample_mode,expected_row_count", [ (True, 2), (False, 3), ], ) @freezegun.freeze_time("2025-01-03T02:03:0Z") def test_sample_mode( self, project, run_sample_mode: bool, expected_row_count: int, ): run_args = ["run"] if run_sample_mode: run_args.append("--sample=1 day") _ = run_dbt(["seed"]) _ = run_dbt(run_args) self.assert_row_count( project=project, relation_name="sample_input_seed", expected_row_count=expected_row_count, ) class TestSamplingModelFromSnapshot(BaseSampleMode): @pytest.fixture(scope="class") def models(self): return { "input_model.sql": input_model_sql, } @pytest.fixture(scope="class") def snapshots(self): return { "snapshot_input_model.sql": snapshot_input_model_sql, } @pytest.mark.parametrize( "run_sample_mode,expected_row_count", [ (True, 2), (False, 3), ], ) @freezegun.freeze_time("2025-01-03T02:03:0Z") def test_sample_mode( self, project, run_sample_mode: bool, expected_row_count: int, ): run_args = ["build"] if run_sample_mode: run_args.append("--sample=1 day") _ = run_dbt(run_args) self.assert_row_count( project=project, relation_name="snapshot_input_model", expected_row_count=expected_row_count, ) self.drop_table(project, "snapshot_input_model") class TestSamplingSnapshot(BaseSampleMode): @pytest.fixture(scope="class") def models(self): return { "input_model.sql": input_model_sql, } @pytest.fixture(scope="class") def snapshots(self): return { "snapshot_input_model.sql": snapshot_input_model_sql, } @pytest.mark.parametrize( "run_sample_mode,expected_row_count", [ (True, 2), (False, 3), ], ) @freezegun.freeze_time("2025-01-03T02:03:0Z") def test_sample_mode( self, project, run_sample_mode: bool, expected_row_count: int, ): run_args = ["build"] # create the snapshot before building a model that depends on it _ = run_dbt(run_args) # Snapshot should always have 3 in this test because we don't sample it self.assert_row_count( project=project, relation_name="snapshot_input_model", expected_row_count=3, ) if run_sample_mode: run_args.append("--sample=1 day") # create model that depends on the snapshot write_file( model_from_snapshot_sql, project.project_root, "models", "model_from_snapshot.sql" ) _ = run_dbt(run_args) self.assert_row_count( project=project, relation_name="model_from_snapshot", expected_row_count=expected_row_count, ) self.drop_table(project, "snapshot_input_model") ================================================ FILE: tests/functional/saved_queries/__init__.py ================================================ ================================================ FILE: tests/functional/saved_queries/fixtures.py ================================================ saved_query_description = """ {% docs saved_query_description %} My SavedQuery Description {% enddocs %} """ saved_queries_yml = """ saved_queries: - name: test_saved_query description: "{{ doc('saved_query_description') }}" label: Test Saved Query query_params: metrics: - simple_metric group_by: - "Dimension('id__ds')" where: - "{{ TimeDimension('id__ds', 'DAY') }} <= now()" - "{{ TimeDimension('id__ds', 'DAY') }} >= '2023-01-01'" - "{{ Metric('txn_revenue', ['id']) }} > 1" order_by: - Metric('simple_metric').descending(True) - "Dimension('id__ds').descending(True)" limit: 10 exports: - name: my_export config: alias: my_export_alias export_as: table schema: my_export_schema_name """ saved_queries_with_defaults_yml = """ saved_queries: - name: test_saved_query description: "{{ doc('saved_query_description') }}" label: Test Saved Query query_params: metrics: - simple_metric group_by: - "Dimension('id__ds')" where: - "{{ TimeDimension('id__ds', 'DAY') }} <= now()" - "{{ TimeDimension('id__ds', 'DAY') }} >= '2023-01-01'" - "{{ Metric('txn_revenue', ['id']) }} > 1" exports: - name: my_export config: alias: my_export_alias export_as: table """ saved_queries_with_diff_filters_yml = """ saved_queries: - name: test_saved_query_where_list description: "{{ doc('saved_query_description') }}" label: Test Saved Query query_params: metrics: - simple_metric group_by: - "Dimension('id__ds')" where: - "{{ TimeDimension('id__ds', 'DAY') }} <= now()" - "{{ TimeDimension('id__ds', 'DAY') }} >= '2023-01-01'" exports: - name: my_export config: alias: my_export_alias export_as: table schema: my_export_schema_name - name: test_saved_query_where_str description: "{{ doc('saved_query_description') }}" label: Test Saved Query2 query_params: metrics: - simple_metric group_by: - "Dimension('id__ds')" where: "{{ TimeDimension('id__ds', 'DAY') }} <= now()" """ saved_query_with_extra_config_attributes_yml = """ saved_queries: - name: test_saved_query description: "{{ doc('saved_query_description') }}" label: Test Saved Query query_params: metrics: - simple_metric group_by: - "Dimension('id__ds')" where: - "{{ TimeDimension('id__ds', 'DAY') }} <= now()" - "{{ TimeDimension('id__ds', 'DAY') }} >= '2023-01-01'" exports: - name: my_export config: my_random_config: 'I have this for some reason' export_as: table """ saved_query_with_export_configs_defined_at_saved_query_level_yml = """ saved_queries: - name: test_saved_query description: "{{ doc('saved_query_description') }}" label: Test Saved Query config: export_as: table schema: my_default_export_schema query_params: metrics: - simple_metric group_by: - "Dimension('id__ds')" where: - "{{ TimeDimension('id__ds', 'DAY') }} <= now()" - "{{ TimeDimension('id__ds', 'DAY') }} >= '2023-01-01'" exports: - name: my_export config: export_as: view schema: my_custom_export_schema - name: my_export2 """ saved_query_without_export_configs_defined_yml = """ saved_queries: - name: test_saved_query description: "{{ doc('saved_query_description') }}" label: Test Saved Query query_params: metrics: - simple_metric group_by: - "Dimension('id__ds')" where: - "{{ TimeDimension('id__ds', 'DAY') }} <= now()" - "{{ TimeDimension('id__ds', 'DAY') }} >= '2023-01-01'" exports: - name: my_export """ saved_query_with_cache_configs_defined_yml = """ saved_queries: - name: test_saved_query description: "{{ doc('saved_query_description') }}" label: Test Saved Query config: cache: enabled: True query_params: metrics: - simple_metric group_by: - "Dimension('id__ds')" where: - "{{ TimeDimension('id__ds', 'DAY') }} <= now()" - "{{ TimeDimension('id__ds', 'DAY') }} >= '2023-01-01'" exports: - name: my_export config: alias: my_export_alias export_as: table schema: my_export_schema_name """ saved_query_with_tags_defined_yml = """ saved_queries: - name: test_saved_query description: "{{ doc('saved_query_description') }}" label: Test Saved Query tags: - tag_a - tag_c query_params: metrics: - simple_metric group_by: - "Dimension('id__ds')" where: - "{{ TimeDimension('id__ds', 'DAY') }} <= now()" - "{{ TimeDimension('id__ds', 'DAY') }} >= '2023-01-01'" exports: - name: my_export config: alias: my_export_alias export_as: table schema: my_export_schema_name """ ================================================ FILE: tests/functional/saved_queries/test_configs.py ================================================ import pytest from dbt.contracts.graph.manifest import Manifest from dbt.tests.util import update_config_file from dbt_semantic_interfaces.type_enums.export_destination_type import ( ExportDestinationType, ) from tests.functional.assertions.test_runner import dbtTestRunner from tests.functional.configs.fixtures import BaseConfigProject from tests.functional.saved_queries.fixtures import ( saved_queries_with_defaults_yml, saved_queries_yml, saved_query_description, saved_query_with_cache_configs_defined_yml, saved_query_with_export_configs_defined_at_saved_query_level_yml, saved_query_with_extra_config_attributes_yml, saved_query_with_tags_defined_yml, saved_query_without_export_configs_defined_yml, ) from tests.functional.semantic_models.fixtures import ( fct_revenue_sql, metricflow_time_spine_sql, schema_yml, ) class TestSavedQueryConfigs(BaseConfigProject): @pytest.fixture(scope="class") def project_config_update(self): return { "saved-queries": { "test": { "test_saved_query": { "+enabled": True, "+export_as": ExportDestinationType.VIEW.value, "+schema": "my_default_export_schema", "+cache": {"enabled": True}, } }, }, } @pytest.fixture(scope="class") def models(self): return { "saved_queries.yml": saved_query_with_extra_config_attributes_yml, "schema.yml": schema_yml, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "docs.md": saved_query_description, } def test_basic_saved_query_config( self, project, ): runner = dbtTestRunner() # parse with default fixture project config result = runner.invoke(["parse"]) assert result.success assert isinstance(result.result, Manifest) assert len(result.result.saved_queries) == 1 saved_query = result.result.saved_queries["saved_query.test.test_saved_query"] assert saved_query.config.export_as == ExportDestinationType.VIEW assert saved_query.config.schema == "my_default_export_schema" assert saved_query.config.cache.enabled is True # disable the saved_query via project config and rerun config_patch = {"saved-queries": {"test": {"test_saved_query": {"+enabled": False}}}} update_config_file(config_patch, project.project_root, "dbt_project.yml") result = runner.invoke(["parse"]) assert result.success assert len(result.result.saved_queries) == 0 # Test that the cache will default to enabled = false if not set in the saved_query config class TestSavedQueryDefaultCacheConfigs(BaseConfigProject): @pytest.fixture(scope="class") def models(self): return { "saved_queries.yml": saved_query_with_extra_config_attributes_yml, "schema.yml": schema_yml, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "docs.md": saved_query_description, } def test_basic_saved_query_config( self, project, ): runner = dbtTestRunner() # parse with default fixture project config result = runner.invoke(["parse"]) assert result.success assert isinstance(result.result, Manifest) assert len(result.result.saved_queries) == 1 saved_query = result.result.saved_queries["saved_query.test.test_saved_query"] assert saved_query.config.cache.enabled is False class TestExportConfigsWithAdditionalProperties(BaseConfigProject): @pytest.fixture(scope="class") def models(self): return { "saved_queries.yml": saved_queries_yml, "schema.yml": schema_yml, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "docs.md": saved_query_description, } def test_extra_config_properties_dont_break_parsing(self, project): runner = dbtTestRunner() # parse with default fixture project config result = runner.invoke(["parse"]) assert result.success assert isinstance(result.result, Manifest) assert len(result.result.saved_queries) == 1 saved_query = result.result.saved_queries["saved_query.test.test_saved_query"] assert len(saved_query.exports) == 1 assert saved_query.exports[0].config.__dict__.get("my_random_config") is None class TestExportConfigsWithDefaultProperties(BaseConfigProject): @pytest.fixture(scope="class") def models(self): return { "saved_queries.yml": saved_queries_with_defaults_yml, "schema.yml": schema_yml, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "docs.md": saved_query_description, } def test_default_properties(self, project): runner = dbtTestRunner() # parse with default fixture project config result = runner.invoke(["parse"]) assert result.success assert isinstance(result.result, Manifest) assert len(result.result.saved_queries) == 1 saved_query = result.result.saved_queries["saved_query.test.test_saved_query"] assert len(saved_query.exports) == 1 export = saved_query.exports[0] assert export.config.alias == "my_export_alias" assert export.config.schema_name == project.test_schema assert export.config.database == project.database class TestInheritingExportConfigFromSavedQueryConfig(BaseConfigProject): @pytest.fixture(scope="class") def models(self): return { "saved_queries.yml": saved_query_with_export_configs_defined_at_saved_query_level_yml, "schema.yml": schema_yml, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "docs.md": saved_query_description, } def test_export_config_inherits_from_saved_query(self, project): runner = dbtTestRunner() # parse with default fixture project config result = runner.invoke(["parse"]) assert result.success assert isinstance(result.result, Manifest) assert len(result.result.saved_queries) == 1 saved_query = result.result.saved_queries["saved_query.test.test_saved_query"] assert len(saved_query.exports) == 2 # assert Export `my_export` has its configs defined from itself because they should take priority export1 = next( (export for export in saved_query.exports if export.name == "my_export"), None ) assert export1 is not None assert export1.config.export_as == ExportDestinationType.VIEW assert export1.config.export_as != saved_query.config.export_as assert export1.config.schema_name == "my_custom_export_schema" assert export1.config.schema_name != saved_query.config.schema assert export1.config.database == project.database # assert Export `my_export` has its configs defined from the saved_query because they should take priority export2 = next( (export for export in saved_query.exports if export.name == "my_export2"), None ) assert export2 is not None assert export2.config.export_as == ExportDestinationType.TABLE assert export2.config.export_as == saved_query.config.export_as assert export2.config.schema_name == "my_default_export_schema" assert export2.config.schema_name == saved_query.config.schema assert export2.config.database == project.database class TestInheritingExportConfigsFromProject(BaseConfigProject): @pytest.fixture(scope="class") def project_config_update(self): return { "saved-queries": { "test": { "test_saved_query": { "+export_as": ExportDestinationType.VIEW.value, } }, }, } @pytest.fixture(scope="class") def models(self): return { "saved_queries.yml": saved_query_without_export_configs_defined_yml, "schema.yml": schema_yml, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "docs.md": saved_query_description, } def test_export_config_inherits_from_project( self, project, ): runner = dbtTestRunner() # parse with default fixture project config result = runner.invoke(["parse"]) assert result.success assert isinstance(result.result, Manifest) assert len(result.result.saved_queries) == 1 saved_query = result.result.saved_queries["saved_query.test.test_saved_query"] assert saved_query.config.export_as == ExportDestinationType.VIEW # change export's `export_as` to `TABLE` via project config config_patch = { "saved-queries": { "test": {"test_saved_query": {"+export_as": ExportDestinationType.TABLE.value}} } } update_config_file(config_patch, project.project_root, "dbt_project.yml") result = runner.invoke(["parse"]) assert result.success assert isinstance(result.result, Manifest) assert len(result.result.saved_queries) == 1 saved_query = result.result.saved_queries["saved_query.test.test_saved_query"] assert saved_query.config.export_as == ExportDestinationType.TABLE # cache can be specified just in a SavedQuery config class TestSavedQueryLevelCacheConfigs(BaseConfigProject): @pytest.fixture(scope="class") def models(self): return { "saved_queries.yml": saved_query_with_cache_configs_defined_yml, "schema.yml": schema_yml, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "docs.md": saved_query_description, } def test_basic_saved_query_config( self, project, ): runner = dbtTestRunner() # parse with default fixture project config result = runner.invoke(["parse"]) assert result.success assert isinstance(result.result, Manifest) assert len(result.result.saved_queries) == 1 saved_query = result.result.saved_queries["saved_query.test.test_saved_query"] assert saved_query.config.cache.enabled is True # the cache defined in yaml for the SavedQuery overrides settings from the dbt_project.toml class TestSavedQueryCacheConfigsOverride(BaseConfigProject): @pytest.fixture(scope="class") def project_config_update(self): return { "saved-queries": { "test": { "test_saved_query": { "+cache": {"enabled": True}, } }, }, } @pytest.fixture(scope="class") def models(self): return { "saved_queries.yml": saved_query_with_cache_configs_defined_yml, "schema.yml": schema_yml, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "docs.md": saved_query_description, } def test_override_saved_query_config( self, project, ): runner = dbtTestRunner() # parse with default fixture project config result = runner.invoke(["parse"]) assert result.success assert isinstance(result.result, Manifest) assert len(result.result.saved_queries) == 1 saved_query = result.result.saved_queries["saved_query.test.test_saved_query"] assert saved_query.config.cache.enabled is True # set cache to enabled=False via project config but since it's set to true at the saved_query # level, it should stay enabled config_patch = { "saved-queries": {"test": {"test_saved_query": {"+cache": {"enabled": False}}}} } update_config_file(config_patch, project.project_root, "dbt_project.yml") result = runner.invoke(["parse"]) assert result.success assert saved_query.config.cache.enabled is True # the tags defined in project yaml for the SavedQuery is additive to the query's class TestSavedQueryTagsAdditiveWithConfig(BaseConfigProject): @pytest.fixture(scope="class") def project_config_update(self): return { "saved-queries": {"+tags": ["tag_b", "tag_c"]}, } @pytest.fixture(scope="class") def models(self): return { "saved_queries.yml": saved_query_with_tags_defined_yml, "schema.yml": schema_yml, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "docs.md": saved_query_description, } def test_saved_query_tags_are_additive_unique_and_sorted( self, project, ): runner = dbtTestRunner() # parse with default fixture project config result = runner.invoke(["parse"]) assert result.success assert isinstance(result.result, Manifest) assert len(result.result.saved_queries) == 1 saved_query = result.result.saved_queries["saved_query.test.test_saved_query"] assert saved_query.tags == ["tag_a", "tag_b", "tag_c"] ================================================ FILE: tests/functional/saved_queries/test_saved_query_build.py ================================================ import pytest from dbt.artifacts.schemas.results import RunStatus from dbt.contracts.graph.nodes import SavedQuery from dbt.tests.util import run_dbt from tests.functional.saved_queries.fixtures import ( saved_queries_yml, saved_query_description, ) from tests.functional.semantic_models.fixtures import ( fct_revenue_sql, metricflow_time_spine_sql, schema_yml, ) class TestSavedQueryBuild: @pytest.fixture(scope="class") def models(self): return { "saved_queries.yml": saved_queries_yml, "schema.yml": schema_yml, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "docs.md": saved_query_description, } @pytest.fixture(scope="class") def packages(self): return """ packages: - package: dbt-labs/dbt_utils version: 1.1.1 """ def test_build_saved_queries_no_op(self, project) -> None: """Test building saved query exports with no flag, so should be no-op.""" run_dbt(["deps"]) result = run_dbt(["build"]) assert len(result.results) == 3 saved_query_results = ( result for result in result.results if isinstance(result.node, SavedQuery) ) assert {result.node.name for result in saved_query_results} == {"test_saved_query"} assert all("NO-OP" in result.message for result in saved_query_results) assert all(result.status == RunStatus.NoOp for result in saved_query_results) ================================================ FILE: tests/functional/saved_queries/test_saved_query_parsing.py ================================================ import os import shutil from copy import deepcopy from typing import List import pytest from dbt.contracts.graph.manifest import Manifest from dbt.tests.util import run_dbt, write_file from dbt_common.events.base_types import BaseEvent from dbt_semantic_interfaces.type_enums.export_destination_type import ( ExportDestinationType, ) from tests.functional.assertions.test_runner import dbtTestRunner from tests.functional.saved_queries.fixtures import ( saved_queries_with_defaults_yml, saved_queries_with_diff_filters_yml, saved_queries_yml, saved_query_description, saved_query_with_cache_configs_defined_yml, ) from tests.functional.semantic_models.fixtures import ( fct_revenue_sql, metricflow_time_spine_sql, schema_yml, ) class TestSavedQueryParsing: @pytest.fixture(scope="class") def models(self): return { "saved_queries.yml": saved_queries_yml, "schema.yml": schema_yml, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "docs.md": saved_query_description, } @pytest.fixture(scope="class") def other_schema(self, unique_schema): return unique_schema + "_other" @pytest.fixture(scope="class") def profiles_config_update(self, dbt_profile_target, unique_schema, other_schema): outputs = {"default": dbt_profile_target, "prod": deepcopy(dbt_profile_target)} outputs["default"]["schema"] = unique_schema outputs["prod"]["schema"] = other_schema return {"test": {"outputs": outputs, "target": "default"}} def copy_state(self): if not os.path.exists("state"): os.makedirs("state") shutil.copyfile("target/manifest.json", "state/manifest.json") def test_semantic_model_parsing(self, project): runner = dbtTestRunner() result = runner.invoke(["parse", "--no-partial-parse"]) assert result.success assert isinstance(result.result, Manifest) manifest = result.result assert len(manifest.saved_queries) == 1 saved_query = manifest.saved_queries["saved_query.test.test_saved_query"] assert saved_query.name == "test_saved_query" assert len(saved_query.query_params.metrics) == 1 assert len(saved_query.query_params.group_by) == 1 assert len(saved_query.query_params.where.where_filters) == 3 assert len(saved_query.depends_on.nodes) == 1 assert len(saved_query.query_params.order_by) == 2 assert saved_query.query_params.limit is not None assert saved_query.description == "My SavedQuery Description" assert len(saved_query.exports) == 1 assert saved_query.exports[0].name == "my_export" assert saved_query.exports[0].config.alias == "my_export_alias" assert saved_query.exports[0].config.export_as == ExportDestinationType.TABLE assert saved_query.exports[0].config.schema_name == "my_export_schema_name" assert saved_query.exports[0].unrendered_config == { "alias": "my_export_alias", "export_as": "table", "schema": "my_export_schema_name", } # Save state self.copy_state() # Nothing has changed, so no state:modified results results = run_dbt(["ls", "--select", "state:modified", "--state", "./state"]) assert len(results) == 0 # Change saved_query write_file( saved_query_with_cache_configs_defined_yml, project.project_root, "models", "saved_queries.yml", ) # State modified finds changed saved_query results = run_dbt(["ls", "--select", "state:modified", "--state", "./state"]) assert len(results) == 1 # change exports write_file( saved_queries_with_defaults_yml, project.project_root, "models", "saved_queries.yml" ) # State modified finds changed saved_query results = run_dbt(["ls", "--select", "state:modified", "--state", "./state"]) assert len(results) == 1 def test_semantic_model_parsing_change_export(self, project, other_schema): runner = dbtTestRunner() result = runner.invoke(["parse", "--no-partial-parse"]) assert result.success assert isinstance(result.result, Manifest) manifest = result.result assert len(manifest.saved_queries) == 1 saved_query = manifest.saved_queries["saved_query.test.test_saved_query"] assert saved_query.name == "test_saved_query" assert saved_query.exports[0].name == "my_export" # Save state self.copy_state() # Nothing has changed, so no state:modified results results = run_dbt(["ls", "--select", "state:modified", "--state", "./state"]) assert len(results) == 0 # Change export name write_file( saved_queries_yml.replace("name: my_export", "name: my_expor2"), project.project_root, "models", "saved_queries.yml", ) # State modified finds changed saved_query results = run_dbt(["ls", "--select", "state:modified", "--state", "./state"]) assert len(results) == 1 # Change export schema write_file( saved_queries_yml.replace( "schema: my_export_schema_name", "schema: my_export_schema_name2" ), project.project_root, "models", "saved_queries.yml", ) # State modified finds changed saved_query results = run_dbt(["ls", "--select", "state:modified", "--state", "./state"]) assert len(results) == 1 def test_semantic_model_parsing_with_default_schema(self, project, other_schema): write_file( saved_queries_with_defaults_yml, project.project_root, "models", "saved_queries.yml" ) runner = dbtTestRunner() result = runner.invoke(["parse", "--no-partial-parse", "--target", "prod"]) assert result.success assert isinstance(result.result, Manifest) manifest = result.result assert len(manifest.saved_queries) == 1 saved_query = manifest.saved_queries["saved_query.test.test_saved_query"] assert saved_query.name == "test_saved_query" assert len(saved_query.query_params.metrics) == 1 assert len(saved_query.query_params.group_by) == 1 assert len(saved_query.query_params.where.where_filters) == 3 assert len(saved_query.depends_on.nodes) == 1 assert saved_query.description == "My SavedQuery Description" assert len(saved_query.exports) == 1 assert saved_query.exports[0].name == "my_export" assert saved_query.exports[0].config.alias == "my_export_alias" assert saved_query.exports[0].config.export_as == ExportDestinationType.TABLE assert saved_query.exports[0].config.schema_name == other_schema assert saved_query.exports[0].unrendered_config == { "alias": "my_export_alias", "export_as": "table", } # Save state self.copy_state() # Nothing has changed, so no state:modified results results = run_dbt( ["ls", "--select", "state:modified", "--state", "./state", "--target", "prod"] ) assert len(results) == 0 # There should also be no state:modified results when using the default schema results = run_dbt(["ls", "--select", "state:modified", "--state", "./state"]) assert len(results) == 0 def test_saved_query_error(self, project): error_schema_yml = saved_queries_yml.replace("simple_metric", "metric_not_found") write_file(error_schema_yml, project.project_root, "models", "saved_queries.yml") events: List[BaseEvent] = [] runner = dbtTestRunner(callbacks=[events.append]) result = runner.invoke(["parse", "--no-partial-parse"]) assert not result.success validation_errors = [e for e in events if e.info.name == "MainEncounteredError"] assert validation_errors class TestSavedQueryPartialParsing: @pytest.fixture(scope="class") def models(self): return { "saved_queries.yml": saved_queries_yml, "saved_queries_with_diff_filters.yml": saved_queries_with_diff_filters_yml, "schema.yml": schema_yml, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "docs.md": saved_query_description, } def test_saved_query_filter_types(self, project): runner = dbtTestRunner() result = runner.invoke(["parse"]) assert result.success manifest = result.result saved_query1 = manifest.saved_queries["saved_query.test.test_saved_query_where_list"] saved_query2 = manifest.saved_queries["saved_query.test.test_saved_query_where_str"] # List filter assert len(saved_query1.query_params.where.where_filters) == 2 assert { where_filter.where_sql_template for where_filter in saved_query1.query_params.where.where_filters } == { "{{ TimeDimension('id__ds', 'DAY') }} <= now()", "{{ TimeDimension('id__ds', 'DAY') }} >= '2023-01-01'", } # String filter assert len(saved_query2.query_params.where.where_filters) == 1 assert ( saved_query2.query_params.where.where_filters[0].where_sql_template == "{{ TimeDimension('id__ds', 'DAY') }} <= now()" ) def test_saved_query_metrics_changed(self, project): # First, use the default saved_queries.yml to define our saved_queries, and # run the dbt parse command runner = dbtTestRunner() result = runner.invoke(["parse"]) assert result.success # Next, modify the default saved_queries.yml to change a detail of the saved # query. modified_saved_queries_yml = saved_queries_yml.replace("simple_metric", "txn_revenue") write_file(modified_saved_queries_yml, project.project_root, "models", "saved_queries.yml") # Now, run the dbt parse command again. result = runner.invoke(["parse"]) assert result.success # Finally, verify that the manifest reflects the partially parsed change manifest = result.result saved_query = manifest.saved_queries["saved_query.test.test_saved_query"] assert len(saved_query.metrics) == 1 assert saved_query.metrics[0] == "txn_revenue" def test_saved_query_deleted_partial_parsing(self, project): # First, use the default saved_queries.yml to define our saved_query, and # run the dbt parse command runner = dbtTestRunner() result = runner.invoke(["parse"]) assert result.success assert "saved_query.test.test_saved_query" in result.result.saved_queries # Next, modify the default saved_queries.yml to remove the saved query. write_file("", project.project_root, "models", "saved_queries.yml") # Now, run the dbt parse command again. result = runner.invoke(["parse"]) assert result.success # Finally, verify that the manifest reflects the deletion assert "saved_query.test.test_saved_query" not in result.result.saved_queries ================================================ FILE: tests/functional/schema/fixtures/macros.py ================================================ _CUSTOM_MACRO = """ {% macro generate_schema_name(schema_name, node) %} {{ schema_name }}_{{ target.schema }}_macro {% endmacro %} """ _CUSTOM_MACRO_W_CONFIG = """ {% macro generate_schema_name(schema_name, node) %} {{ node.config['schema'] }}_{{ target.schema }}_macro {% endmacro %} """ _CUSTOM_MACRO_MULTI_SCHEMA = """ {% macro generate_alias_name(custom_alias_name=none, node=none) -%} {%- set node_name = node.name | trim -%} {%- set split_name = node_name.split('.') -%} {%- set n_parts = split_name | length -%} {{ split_name[1] if n_parts>1 else node_name }} {%- endmacro -%} {% macro generate_schema_name(custom_schema_name=none, node=none) -%} {%- set default_schema = target.schema -%} {%- set node_name = node.name | trim -%} {%- set split_name = node_name.split('.') -%} {%- set n_parts = split_name | length -%} {{ split_name[0] if n_parts>1 else default_schema }} {%- endmacro -%} """ ================================================ FILE: tests/functional/schema/fixtures/sql.py ================================================ _TABLE_ONE = """ select * from {{ ref('seed') }} """ _TABLE_ONE_DOT_MODEL_SCHEMA = "first_schema" _TABLE_ONE_DOT_MODEL_NAME = f"{_TABLE_ONE_DOT_MODEL_SCHEMA}.view_1" _TABLE_ONE_DOT_MODEL = """ select * from {{ target.schema }}.seed """ _TABLE_TWO_SCHEMA = "custom" _TABLE_TWO = ( """ {{ config(schema='""" + _TABLE_TWO_SCHEMA + """') }} select * from {{ ref('view_1') }} """ ) _TABLE_TWO_DOT_MODEL_SCHEMA = "second_schema" _TABLE_TWO_DOT_MODEL_NAME = f"{_TABLE_TWO_DOT_MODEL_SCHEMA}.view_2" _TABLE_TWO_DOT_MODEL = "select * from {{ ref('" + _TABLE_ONE_DOT_MODEL_NAME + "') }}" _TABLE_THREE_SCHEMA = "test" _TABLE_THREE = ( """ {{ config(materialized='table', schema='""" + _TABLE_THREE_SCHEMA + """') }} with v1 as ( select * from{{ ref('view_1') }} ), v2 as ( select * from {{ ref('view_2') }} ), combined as ( select last_name from v1 union all select last_name from v2 ) select last_name, count(*) as count from combined group by 1 """ ) _TABLE_THREE_DOT_MODEL = """ {{ config(materialized='table') }} with v1 as ( select * from {{ ref('first_schema.view_1') }} ), v2 as ( select * from {{ ref('second_schema.view_2') }} ), combined as ( select last_name from v1 union all select last_name from v2 ) select last_name, count(*) as count from combined group by 1 """ _SEED_CSV = """id,first_name,last_name,email,gender,ip_address 1,Jack,Hunter,jhunter0@pbs.org,Male,59.80.20.168 2,Kathryn,Walker,kwalker1@ezinearticles.com,Female,194.121.179.35 3,Gerald,Ryan,gryan2@com.com,Male,11.3.212.243""" _CUSTOM_CONFIG = """ {{ config(schema='custom') }} select * from {{ ref('view_1') }} """ _VALIDATION_SQL = """ drop table if exists {database}.{schema}.seed cascade; create table {database}.{schema}.seed ( id BIGSERIAL PRIMARY KEY, first_name VARCHAR(50), last_name VARCHAR(50), email VARCHAR(50), gender VARCHAR(50), ip_address VARCHAR(20) ); drop table if exists {database}.{schema}.agg cascade; create table {database}.{schema}.agg ( last_name VARCHAR(50), count BIGINT ); insert into {database}.{schema}.seed (first_name, last_name, email, gender, ip_address) values ('Jack', 'Hunter', 'jhunter0@pbs.org', 'Male', '59.80.20.168'), ('Kathryn', 'Walker', 'kwalker1@ezinearticles.com', 'Female', '194.121.179.35'), ('Gerald', 'Ryan', 'gryan2@com.com', 'Male', '11.3.212.243'); insert into {database}.{schema}.agg (last_name, count) values ('Hunter', 2), ('Walker', 2), ('Ryan', 2); """ ================================================ FILE: tests/functional/schema/test_custom_schema.py ================================================ import pytest from dbt.tests.util import check_relations_equal, run_dbt from tests.functional.schema.fixtures.macros import ( _CUSTOM_MACRO, _CUSTOM_MACRO_MULTI_SCHEMA, _CUSTOM_MACRO_W_CONFIG, ) from tests.functional.schema.fixtures.sql import ( _SEED_CSV, _TABLE_ONE, _TABLE_ONE_DOT_MODEL_NAME, _TABLE_ONE_DOT_MODEL_SCHEMA, _TABLE_THREE, _TABLE_THREE_DOT_MODEL, _TABLE_THREE_SCHEMA, _TABLE_TWO, _TABLE_TWO_DOT_MODEL, _TABLE_TWO_DOT_MODEL_NAME, _TABLE_TWO_DOT_MODEL_SCHEMA, _TABLE_TWO_SCHEMA, _VALIDATION_SQL, ) _CUSTOM_SCHEMA = "dbt_test" class BaseTestCustomSchema: @pytest.fixture(scope="class") def seeds(self): return {"seed.csv": _SEED_CSV} @pytest.fixture(scope="class") def models(self): return { "view_1.sql": _TABLE_ONE, "view_2.sql": _TABLE_TWO, "table_3.sql": _TABLE_THREE, } @pytest.fixture(scope="class") def project_config_update(self): return {"models": {"schema": _CUSTOM_SCHEMA}} class TestCustomSchema(BaseTestCustomSchema): def test__postgres_handles__custom_schema_with_no_prefix(self, project, macros): project.run_sql(_VALIDATION_SQL) run_dbt(["seed"]) results = run_dbt(["run"]) assert len(results) == 3 table_results = {r.node.name: r.node.schema for r in results.results} assert table_results["view_1"] == f"{project.test_schema}_{_CUSTOM_SCHEMA}" assert table_results["view_2"] == f"{project.test_schema}_{_TABLE_TWO_SCHEMA}" assert table_results["table_3"] == f"{project.test_schema}_{_TABLE_THREE_SCHEMA}" check_relations_equal( adapter=project.adapter, relation_names=("seed", f"{project.test_schema}_{_CUSTOM_SCHEMA}.view_1"), ) check_relations_equal( adapter=project.adapter, relation_names=("seed", f"{project.test_schema}_{_TABLE_TWO_SCHEMA}.view_2"), ) check_relations_equal( adapter=project.adapter, relation_names=("agg", f"{project.test_schema}_{_TABLE_THREE_SCHEMA}.table_3"), ) class TestCustomSchemaWithCustomMacro(BaseTestCustomSchema): @pytest.fixture(scope="class") def macros(self): return { "custom_macro.sql": _CUSTOM_MACRO, } def test__postgres_handles__custom_schema_with_custom_macro(self, project, macros): project.run_sql(_VALIDATION_SQL) run_dbt(["seed"]) results = run_dbt(["run"]) assert len(results) == 3 table_results = {r.node.name: r.node.schema for r in results.results} assert table_results["view_1"] == f"{_CUSTOM_SCHEMA}_{project.test_schema}_macro" assert table_results["view_2"] == f"{_TABLE_TWO_SCHEMA}_{project.test_schema}_macro" assert table_results["table_3"] == f"{_TABLE_THREE_SCHEMA}_{project.test_schema}_macro" check_relations_equal( adapter=project.adapter, relation_names=("seed", f"{_CUSTOM_SCHEMA}_{project.test_schema}_macro.view_1"), ) check_relations_equal( adapter=project.adapter, relation_names=("seed", f"{_TABLE_TWO_SCHEMA}_{project.test_schema}_macro.view_2"), ) check_relations_equal( adapter=project.adapter, relation_names=("agg", f"{_TABLE_THREE_SCHEMA}_{project.test_schema}_macro.table_3"), ) class TestCustomSchemaWithPrefix(BaseTestCustomSchema): @pytest.fixture(scope="class") def macros(self): return { "custom_macro.sql": _CUSTOM_MACRO_W_CONFIG, } def test__postgres__custom_schema_with_prefix(self, project, macros): project.run_sql(_VALIDATION_SQL) run_dbt(["seed"]) results = run_dbt(["run"]) assert len(results) == 3 table_results = {r.node.name: r.node.schema for r in results.results} assert table_results["view_1"] == f"{_CUSTOM_SCHEMA}_{project.test_schema}_macro" assert table_results["view_2"] == f"{_TABLE_TWO_SCHEMA}_{project.test_schema}_macro" assert table_results["table_3"] == f"{_TABLE_THREE_SCHEMA}_{project.test_schema}_macro" check_relations_equal( adapter=project.adapter, relation_names=("seed", f"{_CUSTOM_SCHEMA}_{project.test_schema}_macro.view_1"), ) check_relations_equal( adapter=project.adapter, relation_names=("seed", f"{_TABLE_TWO_SCHEMA}_{project.test_schema}_macro.view_2"), ) check_relations_equal( adapter=project.adapter, relation_names=("agg", f"{_TABLE_THREE_SCHEMA}_{project.test_schema}_macro.table_3"), ) class TestCustomSchemaWithPrefixAndDispatch(BaseTestCustomSchema): @pytest.fixture(scope="class") def macros(self): return { "custom_macro.sql": _CUSTOM_MACRO_W_CONFIG, } @pytest.fixture(scope="class") def project_config_update(self): return { "models": {"schema": _CUSTOM_SCHEMA}, "dispatch": [ { "macro_namespace": "dbt", "search_order": ["test", "package_macro_overrides", "dbt"], } ], } def test__postgres__custom_schema_with_prefix_and_dispatch( self, project, macros, project_config_update ): project.run_sql(_VALIDATION_SQL) run_dbt(["deps"]) run_dbt(["seed"]) results = run_dbt(["run"]) assert len(results) == 3 table_results = {r.node.name: r.node.schema for r in results.results} assert table_results["view_1"] == f"{_CUSTOM_SCHEMA}_{project.test_schema}_macro" assert table_results["view_2"] == f"{_TABLE_TWO_SCHEMA}_{project.test_schema}_macro" assert table_results["table_3"] == f"{_TABLE_THREE_SCHEMA}_{project.test_schema}_macro" check_relations_equal( adapter=project.adapter, relation_names=("seed", f"{_CUSTOM_SCHEMA}_{project.test_schema}_macro.view_1"), ) check_relations_equal( adapter=project.adapter, relation_names=("seed", f"{_TABLE_TWO_SCHEMA}_{project.test_schema}_macro.view_2"), ) check_relations_equal( adapter=project.adapter, relation_names=("agg", f"{_TABLE_THREE_SCHEMA}_{project.test_schema}_macro.table_3"), ) class TestCustomSchemaWithCustomMacroFromModelName(BaseTestCustomSchema): @pytest.fixture(scope="class") def macros(self): return { "custom_macro.sql": _CUSTOM_MACRO_MULTI_SCHEMA, } @pytest.fixture(scope="class") def project_config_update(self): return { "models": {"schema": _CUSTOM_SCHEMA}, "seeds": { "quote_columns": False, }, } @pytest.fixture(scope="class") def models(self): return { f"{_TABLE_ONE_DOT_MODEL_NAME}.sql": _TABLE_ONE, f"{_TABLE_TWO_DOT_MODEL_NAME}.sql": _TABLE_TWO_DOT_MODEL, "table_3.sql": _TABLE_THREE_DOT_MODEL, } def test__postgres__custom_schema_from_model_name( self, project, macros, project_config_update ): project.run_sql(_VALIDATION_SQL) run_dbt(["seed"]) results = run_dbt(["run"]) assert len(results) == 3 table_results = {r.node.name: r.node.schema for r in results.results} assert table_results[_TABLE_ONE_DOT_MODEL_NAME] == _TABLE_ONE_DOT_MODEL_SCHEMA assert table_results[_TABLE_TWO_DOT_MODEL_NAME] == _TABLE_TWO_DOT_MODEL_SCHEMA assert table_results["table_3"] == f"{project.test_schema}" check_relations_equal( adapter=project.adapter, relation_names=("seed", _TABLE_ONE_DOT_MODEL_NAME) ) check_relations_equal( adapter=project.adapter, relation_names=("seed", _TABLE_TWO_DOT_MODEL_NAME) ) check_relations_equal( adapter=project.adapter, relation_names=("agg", f"{project.test_schema}.table_3") ) ================================================ FILE: tests/functional/schema_tests/data/seed.sql ================================================ create table {schema}.seed ( favorite_color VARCHAR(10), id INTEGER, first_name VARCHAR(11), email VARCHAR(31), net_worth NUMERIC(12, 2) DEFAULT '100.00', fav_number NUMERIC DEFAULT '3.14159265', ip_address VARCHAR(15), updated_at TIMESTAMP WITHOUT TIME ZONE ); INSERT INTO {schema}.seed ("favorite_color", "id","first_name","email","ip_address","updated_at") VALUES ('blue', 1,'Larry',null,'69.135.206.194','2008-09-12 19:08:31'), ('blue', 2,'Larry',null,'64.210.133.162','1978-05-09 04:15:14'), ('blue', 3,'Anna','amontgomery2@miitbeian.gov.cn','168.104.64.114','2011-10-16 04:07:57'), ('blue', 4,'Sandra','sgeorge3@livejournal.com','229.235.252.98','1973-07-19 10:52:43'), ('blue', 5,'Fred','fwoods4@google.cn','78.229.170.124','2012-09-30 16:38:29'), ('blue', 6,'Stephen','shanson5@livejournal.com','182.227.157.105','1995-11-07 21:40:50'), ('blue', 7,'William','wmartinez6@upenn.edu','135.139.249.50','1982-09-05 03:11:59'), ('blue', 8,'Jessica','jlong7@hao123.com','203.62.178.210','1991-10-16 11:03:15'), ('blue', 9,'Douglas','dwhite8@tamu.edu','178.187.247.1','1979-10-01 09:49:48'), ('blue', 10,'Lisa','lcoleman9@nydailynews.com','168.234.128.249','2011-05-26 07:45:49'), ('blue', 11,'Ralph','rfieldsa@home.pl','55.152.163.149','1972-11-18 19:06:11'), ('blue', 12,'Louise','lnicholsb@samsung.com','141.116.153.154','2014-11-25 20:56:14'), ('blue', 13,'Clarence','cduncanc@sfgate.com','81.171.31.133','2011-11-17 07:02:36'), ('blue', 14,'Daniel','dfranklind@omniture.com','8.204.211.37','1980-09-13 00:09:04'), ('blue', 15,'Katherine','klanee@auda.org.au','176.96.134.59','1997-08-22 19:36:56'), ('blue', 16,'Billy','bwardf@wikia.com','214.108.78.85','2003-10-19 02:14:47'), ('blue', 17,'Annie','agarzag@ocn.ne.jp','190.108.42.70','1988-10-28 15:12:35'), ('blue', 18,'Shirley','scolemanh@fastcompany.com','109.251.164.84','1988-08-24 10:50:57'), ('blue', 19,'Roger','rfrazieri@scribd.com','38.145.218.108','1985-12-31 15:17:15'), ('blue', 20,'Lillian','lstanleyj@goodreads.com','47.57.236.17','1970-06-08 02:09:05'), ('blue', 21,'Aaron','arodriguezk@nps.gov','205.245.118.221','1985-10-11 23:07:49'), ('blue', 22,'Patrick','pparkerl@techcrunch.com','19.8.100.182','2006-03-29 12:53:56'), ('blue', 23,'Phillip','pmorenom@intel.com','41.38.254.103','2011-11-07 15:35:43'), ('blue', 24,'Henry','hgarcian@newsvine.com','1.191.216.252','2008-08-28 08:30:44'), ('blue', 25,'Irene','iturnero@opera.com','50.17.60.190','1994-04-01 07:15:02'), ('blue', 26,'Andrew','adunnp@pen.io','123.52.253.176','2000-11-01 06:03:25'), ('blue', 27,'David','dgutierrezq@wp.com','238.23.203.42','1988-01-25 07:29:18'), ('blue', 28,'Henry','hsanchezr@cyberchimps.com','248.102.2.185','1983-01-01 13:36:37'), ('blue', 29,'Evelyn','epetersons@gizmodo.com','32.80.46.119','1979-07-16 17:24:12'), ('blue', 30,'Tammy','tmitchellt@purevolume.com','249.246.167.88','2001-04-03 10:00:23'), ('blue', 31,'Jacqueline','jlittleu@domainmarket.com','127.181.97.47','1986-02-11 21:35:50'), ('blue', 32,'Earl','eortizv@opera.com','166.47.248.240','1996-07-06 08:16:27'), ('blue', 33,'Juan','jgordonw@sciencedirect.com','71.77.2.200','1987-01-31 03:46:44'), ('blue', 34,'Diane','dhowellx@nyu.edu','140.94.133.12','1994-06-11 02:30:05'), ('blue', 35,'Randy','rkennedyy@microsoft.com','73.255.34.196','2005-05-26 20:28:39'), ('blue', 36,'Janice','jriveraz@time.com','22.214.227.32','1990-02-09 04:16:52'), ('blue', 37,'Laura','lperry10@diigo.com','159.148.145.73','2015-03-17 05:59:25'), ('blue', 38,'Gary','gray11@statcounter.com','40.193.124.56','1970-01-27 10:04:51'), ('blue', 39,'Jesse','jmcdonald12@typepad.com','31.7.86.103','2009-03-14 08:14:29'), ('blue', 40,'Sandra','sgonzalez13@goodreads.com','223.80.168.239','1993-05-21 14:08:54'), ('blue', 41,'Scott','smoore14@archive.org','38.238.46.83','1980-08-30 11:16:56'), ('blue', 42,'Phillip','pevans15@cisco.com','158.234.59.34','2011-12-15 23:26:31'), ('blue', 43,'Steven','sriley16@google.ca','90.247.57.68','2011-10-29 19:03:28'), ('blue', 44,'Deborah','dbrown17@hexun.com','179.125.143.240','1995-04-10 14:36:07'), ('blue', 45,'Lori','lross18@ow.ly','64.80.162.180','1980-12-27 16:49:15'), ('blue', 46,'Sean','sjackson19@tumblr.com','240.116.183.69','1988-06-12 21:24:45'), ('blue', 47,'Terry','tbarnes1a@163.com','118.38.213.137','1997-09-22 16:43:19'), ('blue', 48,'Dorothy','dross1b@ebay.com','116.81.76.49','2005-02-28 13:33:24'), ('blue', 49,'Samuel','swashington1c@house.gov','38.191.253.40','1989-01-19 21:15:48'), ('blue', 50,'Ralph','rcarter1d@tinyurl.com','104.84.60.174','2007-08-11 10:21:49'), ('green', 51,'Wayne','whudson1e@princeton.edu','90.61.24.102','1983-07-03 16:58:12'), ('green', 52,'Rose','rjames1f@plala.or.jp','240.83.81.10','1995-06-08 11:46:23'), ('green', 53,'Louise','lcox1g@theglobeandmail.com','105.11.82.145','2016-09-19 14:45:51'), ('green', 54,'Kenneth','kjohnson1h@independent.co.uk','139.5.45.94','1976-08-17 11:26:19'), ('green', 55,'Donna','dbrown1i@amazon.co.uk','19.45.169.45','2006-05-27 16:51:40'), ('green', 56,'Johnny','jvasquez1j@trellian.com','118.202.238.23','1975-11-17 08:42:32'), ('green', 57,'Patrick','pramirez1k@tamu.edu','231.25.153.198','1997-08-06 11:51:09'), ('green', 58,'Helen','hlarson1l@prweb.com','8.40.21.39','1993-08-04 19:53:40'), ('green', 59,'Patricia','pspencer1m@gmpg.org','212.198.40.15','1977-08-03 16:37:27'), ('green', 60,'Joseph','jspencer1n@marriott.com','13.15.63.238','2005-07-23 20:22:06'), ('green', 61,'Phillip','pschmidt1o@blogtalkradio.com','177.98.201.190','1976-05-19 21:47:44'), ('green', 62,'Joan','jwebb1p@google.ru','105.229.170.71','1972-09-07 17:53:47'), ('green', 63,'Phyllis','pkennedy1q@imgur.com','35.145.8.244','2000-01-01 22:33:37'), ('green', 64,'Katherine','khunter1r@smh.com.au','248.168.205.32','1991-01-09 06:40:24'), ('green', 65,'Laura','lvasquez1s@wiley.com','128.129.115.152','1997-10-23 12:04:56'), ('green', 66,'Juan','jdunn1t@state.gov','44.228.124.51','2004-11-10 05:07:35'), ('green', 67,'Judith','jholmes1u@wiley.com','40.227.179.115','1977-08-02 17:01:45'), ('green', 68,'Beverly','bbaker1v@wufoo.com','208.34.84.59','2016-03-06 20:07:23'), ('green', 69,'Lawrence','lcarr1w@flickr.com','59.158.212.223','1988-09-13 06:07:21'), ('green', 70,'Gloria','gwilliams1x@mtv.com','245.231.88.33','1995-03-18 22:32:46'), ('green', 71,'Steven','ssims1y@cbslocal.com','104.50.58.255','2001-08-05 21:26:20'), ('green', 72,'Betty','bmills1z@arstechnica.com','103.177.214.220','1981-12-14 21:26:54'), ('green', 73,'Mildred','mfuller20@prnewswire.com','151.158.8.130','2000-04-19 10:13:55'), ('green', 74,'Donald','dday21@icq.com','9.178.102.255','1972-12-03 00:58:24'), ('green', 75,'Eric','ethomas22@addtoany.com','85.2.241.227','1992-11-01 05:59:30'), ('green', 76,'Joyce','jarmstrong23@sitemeter.com','169.224.20.36','1985-10-24 06:50:01'), ('green', 77,'Maria','mmartinez24@amazonaws.com','143.189.167.135','2005-10-05 05:17:42'), ('green', 78,'Harry','hburton25@youtube.com','156.47.176.237','1978-03-26 05:53:33'), ('green', 79,'Kevin','klawrence26@hao123.com','79.136.183.83','1994-10-12 04:38:52'), ('green', 80,'David','dhall27@prweb.com','133.149.172.153','1976-12-15 16:24:24'), ('green', 81,'Kathy','kperry28@twitter.com','229.242.72.228','1979-03-04 02:58:56'), ('green', 82,'Adam','aprice29@elegantthemes.com','13.145.21.10','1982-11-07 11:46:59'), ('green', 83,'Brandon','bgriffin2a@va.gov','73.249.128.212','2013-10-30 05:30:36'), ('green', 84,'Henry','hnguyen2b@discovery.com','211.36.214.242','1985-01-09 06:37:27'), ('green', 85,'Eric','esanchez2c@edublogs.org','191.166.188.251','2004-05-01 23:21:42'), ('green', 86,'Jason','jlee2d@jimdo.com','193.92.16.182','1973-01-08 09:05:39'), ('green', 87,'Diana','drichards2e@istockphoto.com','19.130.175.245','1994-10-05 22:50:49'), ('green', 88,'Andrea','awelch2f@abc.net.au','94.155.233.96','2002-04-26 08:41:44'), ('green', 89,'Louis','lwagner2g@miitbeian.gov.cn','26.217.34.111','2003-08-25 07:56:39'), ('green', 90,'Jane','jsims2h@seesaa.net','43.4.220.135','1987-03-20 20:39:04'), ('green', 91,'Larry','lgrant2i@si.edu','97.126.79.34','2000-09-07 20:26:19'), ('green', 92,'Louis','ldean2j@prnewswire.com','37.148.40.127','2011-09-16 20:12:14'), ('green', 93,'Jennifer','jcampbell2k@xing.com','38.106.254.142','1988-07-15 05:06:49'), ('green', 94,'Wayne','wcunningham2l@google.com.hk','223.28.26.187','2009-12-15 06:16:54'), ('green', 95,'Lori','lstevens2m@icq.com','181.250.181.58','1984-10-28 03:29:19'), ('green', 96,'Judy','jsimpson2n@marriott.com','180.121.239.219','1986-02-07 15:18:10'), ('green', 97,'Phillip','phoward2o@usa.gov','255.247.0.175','2002-12-26 08:44:45'), ('green', 98,'Gloria','gwalker2p@usa.gov','156.140.7.128','1997-10-04 07:58:58'), ('green', 99,'Paul','pjohnson2q@umn.edu','183.59.198.197','1991-11-14 12:33:55'), ('green', 100,'Frank','fgreene2r@blogspot.com','150.143.68.121','2010-06-12 23:55:39'); ================================================ FILE: tests/functional/schema_tests/data/seed_failure.sql ================================================ create table {schema}.seed_failure ( favorite_color VARCHAR(10), id INTEGER, first_name VARCHAR(11), email VARCHAR(31), ip_address VARCHAR(15), updated_at TIMESTAMP WITHOUT TIME ZONE ); INSERT INTO {schema}.seed_failure ("favorite_color", "id","first_name","email","ip_address","updated_at") VALUES -- unaccepted 'red' favorite_color ('red', 1,'Larry','lking0@miitbeian.gov.cn','69.135.206.194','2008-09-12 19:08:31'), -- dupicate unique field (id=1) ('blue', 1,'Larry','lperkins1@toplist.cz','64.210.133.162','1978-05-09 04:15:14'), -- null not_null field (id) ('blue', null,'Anna','amontgomery2@miitbeian.gov.cn','168.104.64.114','2011-10-16 04:07:57'), ('blue', 4,'Sandra','sgeorge3@livejournal.com','229.235.252.98','1973-07-19 10:52:43'), ('blue', 5,'Fred','fwoods4@google.cn','78.229.170.124','2012-09-30 16:38:29'), ('blue', 6,'Stephen','shanson5@livejournal.com','182.227.157.105','1995-11-07 21:40:50'), ('blue', 7,'William','wmartinez6@upenn.edu','135.139.249.50','1982-09-05 03:11:59'), ('blue', 8,'Jessica','jlong7@hao123.com','203.62.178.210','1991-10-16 11:03:15'), ('blue', 9,'Douglas','dwhite8@tamu.edu','178.187.247.1','1979-10-01 09:49:48'), ('blue', 10,'Lisa','lcoleman9@nydailynews.com','168.234.128.249','2011-05-26 07:45:49'), ('blue', 11,'Ralph','rfieldsa@home.pl','55.152.163.149','1972-11-18 19:06:11'), ('blue', 12,'Louise','lnicholsb@samsung.com','141.116.153.154','2014-11-25 20:56:14'), ('blue', 13,'Clarence','cduncanc@sfgate.com','81.171.31.133','2011-11-17 07:02:36'), ('blue', 14,'Daniel','dfranklind@omniture.com','8.204.211.37','1980-09-13 00:09:04'), ('blue', 15,'Katherine','klanee@auda.org.au','176.96.134.59','1997-08-22 19:36:56'), ('blue', 16,'Billy','bwardf@wikia.com','214.108.78.85','2003-10-19 02:14:47'), ('blue', 17,'Annie','agarzag@ocn.ne.jp','190.108.42.70','1988-10-28 15:12:35'), ('blue', 18,'Shirley','scolemanh@fastcompany.com','109.251.164.84','1988-08-24 10:50:57'), ('blue', 19,'Roger','rfrazieri@scribd.com','38.145.218.108','1985-12-31 15:17:15'), ('blue', 20,'Lillian','lstanleyj@goodreads.com','47.57.236.17','1970-06-08 02:09:05'), ('blue', 21,'Aaron','arodriguezk@nps.gov','205.245.118.221','1985-10-11 23:07:49'), ('blue', 22,'Patrick','pparkerl@techcrunch.com','19.8.100.182','2006-03-29 12:53:56'), ('blue', 23,'Phillip','pmorenom@intel.com','41.38.254.103','2011-11-07 15:35:43'), ('blue', 24,'Henry','hgarcian@newsvine.com','1.191.216.252','2008-08-28 08:30:44'), ('blue', 25,'Irene','iturnero@opera.com','50.17.60.190','1994-04-01 07:15:02'), ('blue', 26,'Andrew','adunnp@pen.io','123.52.253.176','2000-11-01 06:03:25'), ('blue', 27,'David','dgutierrezq@wp.com','238.23.203.42','1988-01-25 07:29:18'), ('blue', 28,'Henry','hsanchezr@cyberchimps.com','248.102.2.185','1983-01-01 13:36:37'), ('blue', 29,'Evelyn','epetersons@gizmodo.com','32.80.46.119','1979-07-16 17:24:12'), ('blue', 30,'Tammy','tmitchellt@purevolume.com','249.246.167.88','2001-04-03 10:00:23'), ('blue', 31,'Jacqueline','jlittleu@domainmarket.com','127.181.97.47','1986-02-11 21:35:50'), ('blue', 32,'Earl','eortizv@opera.com','166.47.248.240','1996-07-06 08:16:27'), ('blue', 33,'Juan','jgordonw@sciencedirect.com','71.77.2.200','1987-01-31 03:46:44'), ('blue', 34,'Diane','dhowellx@nyu.edu','140.94.133.12','1994-06-11 02:30:05'), ('blue', 35,'Randy','rkennedyy@microsoft.com','73.255.34.196','2005-05-26 20:28:39'), ('blue', 36,'Janice','jriveraz@time.com','22.214.227.32','1990-02-09 04:16:52'), ('blue', 37,'Laura','lperry10@diigo.com','159.148.145.73','2015-03-17 05:59:25'), ('blue', 38,'Gary','gray11@statcounter.com','40.193.124.56','1970-01-27 10:04:51'), ('blue', 39,'Jesse','jmcdonald12@typepad.com','31.7.86.103','2009-03-14 08:14:29'), ('blue', 40,'Sandra','sgonzalez13@goodreads.com','223.80.168.239','1993-05-21 14:08:54'), ('blue', 41,'Scott','smoore14@archive.org','38.238.46.83','1980-08-30 11:16:56'), ('blue', 42,'Phillip','pevans15@cisco.com','158.234.59.34','2011-12-15 23:26:31'), ('blue', 43,'Steven','sriley16@google.ca','90.247.57.68','2011-10-29 19:03:28'), ('blue', 44,'Deborah','dbrown17@hexun.com','179.125.143.240','1995-04-10 14:36:07'), ('blue', 45,'Lori','lross18@ow.ly','64.80.162.180','1980-12-27 16:49:15'), ('blue', 46,'Sean','sjackson19@tumblr.com','240.116.183.69','1988-06-12 21:24:45'), ('blue', 47,'Terry','tbarnes1a@163.com','118.38.213.137','1997-09-22 16:43:19'), ('blue', 48,'Dorothy','dross1b@ebay.com','116.81.76.49','2005-02-28 13:33:24'), ('blue', 49,'Samuel','swashington1c@house.gov','38.191.253.40','1989-01-19 21:15:48'), ('blue', 50,'Ralph','rcarter1d@tinyurl.com','104.84.60.174','2007-08-11 10:21:49'), ('green', 51,'Wayne','whudson1e@princeton.edu','90.61.24.102','1983-07-03 16:58:12'), ('green', 52,'Rose','rjames1f@plala.or.jp','240.83.81.10','1995-06-08 11:46:23'), ('green', 53,'Louise','lcox1g@theglobeandmail.com','105.11.82.145','2016-09-19 14:45:51'), ('green', 54,'Kenneth','kjohnson1h@independent.co.uk','139.5.45.94','1976-08-17 11:26:19'), ('green', 55,'Donna','dbrown1i@amazon.co.uk','19.45.169.45','2006-05-27 16:51:40'), ('green', 56,'Johnny','jvasquez1j@trellian.com','118.202.238.23','1975-11-17 08:42:32'), ('green', 57,'Patrick','pramirez1k@tamu.edu','231.25.153.198','1997-08-06 11:51:09'), ('green', 58,'Helen','hlarson1l@prweb.com','8.40.21.39','1993-08-04 19:53:40'), ('green', 59,'Patricia','pspencer1m@gmpg.org','212.198.40.15','1977-08-03 16:37:27'), ('green', 60,'Joseph','jspencer1n@marriott.com','13.15.63.238','2005-07-23 20:22:06'), ('green', 61,'Phillip','pschmidt1o@blogtalkradio.com','177.98.201.190','1976-05-19 21:47:44'), ('green', 62,'Joan','jwebb1p@google.ru','105.229.170.71','1972-09-07 17:53:47'), ('green', 63,'Phyllis','pkennedy1q@imgur.com','35.145.8.244','2000-01-01 22:33:37'), ('green', 64,'Katherine','khunter1r@smh.com.au','248.168.205.32','1991-01-09 06:40:24'), ('green', 65,'Laura','lvasquez1s@wiley.com','128.129.115.152','1997-10-23 12:04:56'), ('green', 66,'Juan','jdunn1t@state.gov','44.228.124.51','2004-11-10 05:07:35'), ('green', 67,'Judith','jholmes1u@wiley.com','40.227.179.115','1977-08-02 17:01:45'), ('green', 68,'Beverly','bbaker1v@wufoo.com','208.34.84.59','2016-03-06 20:07:23'), ('green', 69,'Lawrence','lcarr1w@flickr.com','59.158.212.223','1988-09-13 06:07:21'), ('green', 70,'Gloria','gwilliams1x@mtv.com','245.231.88.33','1995-03-18 22:32:46'), ('green', 71,'Steven','ssims1y@cbslocal.com','104.50.58.255','2001-08-05 21:26:20'), ('green', 72,'Betty','bmills1z@arstechnica.com','103.177.214.220','1981-12-14 21:26:54'), ('green', 73,'Mildred','mfuller20@prnewswire.com','151.158.8.130','2000-04-19 10:13:55'), ('green', 74,'Donald','dday21@icq.com','9.178.102.255','1972-12-03 00:58:24'), ('green', 75,'Eric','ethomas22@addtoany.com','85.2.241.227','1992-11-01 05:59:30'), ('green', 76,'Joyce','jarmstrong23@sitemeter.com','169.224.20.36','1985-10-24 06:50:01'), ('green', 77,'Maria','mmartinez24@amazonaws.com','143.189.167.135','2005-10-05 05:17:42'), ('green', 78,'Harry','hburton25@youtube.com','156.47.176.237','1978-03-26 05:53:33'), ('green', 79,'Kevin','klawrence26@hao123.com','79.136.183.83','1994-10-12 04:38:52'), ('green', 80,'David','dhall27@prweb.com','133.149.172.153','1976-12-15 16:24:24'), ('green', 81,'Kathy','kperry28@twitter.com','229.242.72.228','1979-03-04 02:58:56'), ('green', 82,'Adam','aprice29@elegantthemes.com','13.145.21.10','1982-11-07 11:46:59'), ('green', 83,'Brandon','bgriffin2a@va.gov','73.249.128.212','2013-10-30 05:30:36'), ('green', 84,'Henry','hnguyen2b@discovery.com','211.36.214.242','1985-01-09 06:37:27'), ('green', 85,'Eric','esanchez2c@edublogs.org','191.166.188.251','2004-05-01 23:21:42'), ('green', 86,'Jason','jlee2d@jimdo.com','193.92.16.182','1973-01-08 09:05:39'), ('green', 87,'Diana','drichards2e@istockphoto.com','19.130.175.245','1994-10-05 22:50:49'), ('green', 88,'Andrea','awelch2f@abc.net.au','94.155.233.96','2002-04-26 08:41:44'), ('green', 89,'Louis','lwagner2g@miitbeian.gov.cn','26.217.34.111','2003-08-25 07:56:39'), ('green', 90,'Jane','jsims2h@seesaa.net','43.4.220.135','1987-03-20 20:39:04'), ('green', 91,'Larry','lgrant2i@si.edu','97.126.79.34','2000-09-07 20:26:19'), ('green', 92,'Louis','ldean2j@prnewswire.com','37.148.40.127','2011-09-16 20:12:14'), ('green', 93,'Jennifer','jcampbell2k@xing.com','38.106.254.142','1988-07-15 05:06:49'), ('green', 94,'Wayne','wcunningham2l@google.com.hk','223.28.26.187','2009-12-15 06:16:54'), ('green', 95,'Lori','lstevens2m@icq.com','181.250.181.58','1984-10-28 03:29:19'), ('green', 96,'Judy','jsimpson2n@marriott.com','180.121.239.219','1986-02-07 15:18:10'), ('green', 97,'Phillip','phoward2o@usa.gov','255.247.0.175','2002-12-26 08:44:45'), ('green', 98,'Gloria','gwalker2p@usa.gov','156.140.7.128','1997-10-04 07:58:58'), ('green', 99,'Paul','pjohnson2q@umn.edu','183.59.198.197','1991-11-14 12:33:55'), ('green', 100,'Frank','fgreene2r@blogspot.com','150.143.68.121','2010-06-12 23:55:39'); ================================================ FILE: tests/functional/schema_tests/fixtures.py ================================================ wrong_specification_block__schema_yml = """ version: 2 models: - name: some_seed description: "This is my seed under a model" """ test_context_where_subq_models__schema_yml = """ version: 2 models: - name: model_a data_tests: - self_referential """ test_context_where_subq_models__model_a_sql = """ select 1 as fun """ test_utils__dbt_project_yml = """ name: 'test_utils' version: '1.0' config-version: 2 profile: 'default' macro-paths: ["macros"] """ test_utils__macros__current_timestamp_sql = """ {% macro current_timestamp() -%} {{ return(adapter.dispatch('current_timestamp', 'test_utils')()) }} {%- endmacro %} {% macro default__current_timestamp() -%} now() {%- endmacro %} """ test_utils__macros__custom_test_sql = """ {% macro test_dispatch(model) -%} {{ return(adapter.dispatch('test_dispatch', macro_namespace = 'test_utils')()) }} {%- endmacro %} {% macro default__test_dispatch(model) %} select {{ adapter.dispatch('current_timestamp', macro_namespace = 'test_utils')() }} {% endmacro %} """ local_dependency__dbt_project_yml = """ name: 'local_dep' version: '1.0' config-version: 2 profile: 'default' macro-paths: ["macros"] """ local_dependency__macros__equality_sql = """ {#-- taken from dbt-utils --#} {% test equality(model, compare_model, compare_columns=None) %} {{ return(adapter.dispatch('test_equality')(model, compare_model, compare_columns)) }} {% endtest %} {% macro default__test_equality(model, compare_model, compare_columns=None) %} {% set set_diff %} count(*) + abs( sum(case when which_diff = 'a_minus_b' then 1 else 0 end) - sum(case when which_diff = 'b_minus_a' then 1 else 0 end) ) {% endset %} {#-- Needs to be set at parse time, before we return '' below --#} {{ config(fail_calc = set_diff) }} {#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} {%- if not execute -%} {{ return('') }} {% endif %} -- setup {%- do dbt_utils._is_relation(model, 'test_equality') -%} {#- If the compare_cols arg is provided, we can run this test without querying the information schema — this allows the model to be an ephemeral model -#} {%- if not compare_columns -%} {%- do dbt_utils._is_ephemeral(model, 'test_equality') -%} {%- set compare_columns = adapter.get_columns_in_relation(model) | map(attribute='quoted') -%} {%- endif -%} {% set compare_cols_csv = compare_columns | join(', ') %} with a as ( select * from {{ model }} ), b as ( select * from {{ compare_model }} ), a_minus_b as ( select {{compare_cols_csv}} from a {{ dbt_utils.except() }} select {{compare_cols_csv}} from b ), b_minus_a as ( select {{compare_cols_csv}} from b {{ dbt_utils.except() }} select {{compare_cols_csv}} from a ), unioned as ( select 'a_minus_b' as which_diff, * from a_minus_b union all select 'b_minus_a' as which_diff, * from b_minus_a ) select * from unioned {% endmacro %} """ case_sensitive_models__schema_yml = """ version: 2 models: - name: lowercase columns: - name: id quote: true data_tests: - unique - name: uppercase columns: - name: id quote: true data_tests: - unique """ case_sensitive_models__uppercase_SQL = """ select 1 as id """ case_sensitive_models__lowercase_sql = """ select 1 as id """ test_context_macros__my_test_sql = """ {% macro test_call_pkg_macro(model) %} select {{ adapter.dispatch('current_timestamp', macro_namespace = 'local_utils')() }} {% endmacro %} """ test_context_macros__test_my_datediff_sql = """ {% macro test_my_datediff(model) %} select {{ local_utils.datediff() }} {% endmacro %} """ test_context_macros__custom_schema_tests_sql = """ {% test type_one(model) %} select * from ( select * from {{ model }} union all select * from {{ ref('model_b') }} ) as Foo {% endtest %} {% test type_two(model) %} {{ config(severity = "WARN") }} select * from {{ model }} {% endtest %} """ test_context_models_namespaced__schema_yml = """ version: 2 models: - name: model_a data_tests: - type_one - type_two - name: model_c data_tests: - call_pkg_macro - test_utils.dispatch """ test_context_models_namespaced__model_c_sql = """ select 1 as fun """ test_context_models_namespaced__model_b_sql = """ select 1 as notfun """ test_context_models_namespaced__model_a_sql = """ select 1 as fun """ macros_v2__override_get_test_macros_fail__get_test_sql_sql = """ {% macro get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%} select {{ fail_calc }} as failures, case when {{ fail_calc }} {{ warn_if }} then 'x' else 'y' end as should_warn, case when {{ fail_calc }} {{ error_if }} then 'x' else 'y' end as should_error from ( {{ main_sql }} {{ "limit " ~ limit if limit != none }} ) dbt_internal_test {% endmacro %} """ macros_v2__macros__tests_sql = """ {% test every_value_is_blue(model, column_name) %} select * from {{ model }} where {{ column_name }} != 'blue' {% endtest %} {% test rejected_values(model, column_name, values) %} select * from {{ model }} where {{ column_name }} in ( {% for value in values %} '{{ value }}' {% if not loop.last %} , {% endif %} {% endfor %} ) {% endtest %} {% test equivalent(model, value) %} {% set expected = 'foo-bar' %} {% set eq = 1 if value == expected else 0 %} {% set validation_message -%} 'got "{{ value }}", expected "{{ expected }}"' {%- endset %} {% if eq == 0 and execute %} {{ log(validation_message, info=True) }} {% endif %} select {{ validation_message }} as validation_error where {{ eq }} = 0 {% endtest %} """ macros_v2__override_get_test_macros__get_test_sql_sql = """ {% macro get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%} select {{ fail_calc }} as failures, case when {{ fail_calc }} {{ warn_if }} then 1 else 0 end as should_warn, case when {{ fail_calc }} {{ error_if }} then 1 else 0 end as should_error from ( {{ main_sql }} {{ "limit " ~ limit if limit != none }} ) dbt_internal_test {%- endmacro %} """ macros_v2__custom_configs__test_sql = """ {% test where(model, column_name) %} {{ config(where = "1 = 0") }} select * from {{ model }} {% endtest %} {% test error_if(model, column_name) %} {{ config(error_if = "<= 0", warn_if = "<= 0") }} select * from {{ model }} {% endtest %} {% test warn_if(model, column_name) %} {{ config(warn_if = "<= 0", severity = "WARN") }} select * from {{ model }} {% endtest %} {% test limit(model, column_name) %} {{ config(limit = 0) }} select * from {{ model }} {% endtest %} {% test fail_calc(model, column_name) %} {{ config(fail_calc = "count(*) - count(*)") }} select * from {{ model }} {% endtest %} """ test_context_macros_namespaced__my_test_sql = """ {% macro test_call_pkg_macro(model) %} select {{ test_utils.current_timestamp() }} {% endmacro %} """ test_context_macros_namespaced__custom_schema_tests_sql = """ {% test type_one(model) %} select * from ( select * from {{ model }} union all select * from {{ ref('model_b') }} ) as Foo {% endtest %} {% test type_two(model) %} {{ config(severity = "WARN") }} select * from {{ model }} {% endtest %} """ seeds__some_seed_csv = """ col_int,col_str 1,hello 2,goodbye """ test_context_models__schema_yml = """ version: 2 models: - name: model_a data_tests: - type_one - type_two - name: model_c data_tests: - call_pkg_macro - local_utils.dispatch - my_datediff """ test_context_models__model_c_sql = """ select 1 as fun """ test_context_models__model_b_sql = """ select 1 as notfun """ test_context_models__model_a_sql = """ select 1 as fun """ name_collision__schema_yml = """ version: 2 models: - name: base columns: - name: extension_id data_tests: - not_null - name: base_extension columns: - name: id data_tests: - not_null """ name_collision__base_sql = """ SELECT 'hello_world' AS extension_id """ name_collision__base_extension_sql = """ SELECT 'NOT_NULL' AS id """ dupe_generic_tests_collide__schema_yml = """ version: 2 models: - name: model_a columns: - name: id data_tests: - not_null: config: where: "1=1" - not_null: config: where: "1=2" """ dupe_generic_tests_collide__model_a = """ SELECT 'NOT_NULL' AS id """ custom_generic_test_config_custom_macro__schema_yml = """ version: 2 models: - name: model_a columns: - name: id data_tests: - not_null: config: where: "id = (select id from {{ ref('model_a') }} limit 1)" """ custom_generic_test_config_custom_macro__model_a = """ SELECT 1 AS id """ custom_generic_test_names__schema_yml = """ version: 2 models: - name: model_a columns: - name: id data_tests: - not_null: name: not_null_where_1_equals_1 config: where: "1=1" - not_null: name: not_null_where_1_equals_2 config: where: "1=2" """ custom_generic_test_names__model_a = """ SELECT 'NOT_NULL' AS id """ custom_generic_test_names_alt_format__schema_yml = """ version: 2 models: - name: model_a columns: - name: id data_tests: - name: not_null_where_1_equals_1 test_name: not_null config: where: "1=1" - name: not_null_where_1_equals_2 test_name: not_null config: where: "1=2" """ custom_generic_test_names_alt_format__model_a = """ SELECT 'NOT_NULL' AS id """ test_context_where_subq_macros__custom_generic_test_sql = """ /*{# This test will fail if get_where_subquery() is missing from TestContext + TestMacroNamespace #}*/ {% test self_referential(model) %} {%- set relation = api.Relation.create(schema=model.schema, identifier=model.table) -%} {%- set columns = adapter.get_columns_in_relation(relation) -%} {%- set columns_csv = columns | map(attribute='name') | list | join(', ') -%} select {{ columns_csv }} from {{ model }} limit 0 {% endtest %} """ invalid_schema_models__schema_yml = """ version: 2 models: name: model columns: - name: Id quote: true data_tests: - unique - not_null """ invalid_schema_models__model_sql = """ select 1 as "Id" """ all_quotes_schema__schema_yml = """# models/schema.yml # only comments here, which should be okay! # https://github.com/dbt-labs/dbt-core/issues/3568""" models_v2__render_test_cli_arg_models__schema_yml = """ version: 2 models: - name: model data_tests: - equivalent: value: "{{ var('myvar', 'baz') }}-bar" """ models_v2__render_test_cli_arg_models__model_sql = """ select 1 as id """ models_v2__override_get_test_models__schema_yml = """ version: 2 models: - name: my_model_pass description: "The table has 1 null values, and we're okay with that, until it's more than 1." columns: - name: id description: "The number of responses for this favorite color - purple will be null" data_tests: - not_null: error_if: '>1' warn_if: '>1' - name: my_model_warning description: "The table has 1 null values, and we're okay with that, but let us know" columns: - name: id description: "The number of responses for this favorite color - purple will be null" data_tests: - not_null: error_if: '>1' - name: my_model_failure description: "The table has 2 null values, and we're not okay with that" columns: - name: id description: "The number of responses for this favorite color - purple will be null" data_tests: - not_null: error_if: '>1' """ models_v2__override_get_test_models__my_model_warning_sql = """ select * from {{ ref('my_model_pass') }} """ models_v2__override_get_test_models__my_model_pass_sql = """ select 1 as id UNION ALL select null as id """ models_v2__override_get_test_models__my_model_failure_sql = """ select * from {{ ref('my_model_pass') }} UNION ALL select null as id """ models_v2__models__schema_yml = """ version: 2 models: - name: table_copy description: "A copy of the table" columns: - name: id description: "The ID" data_tests: - not_null - unique tags: - table_id - name: first_name description: "The user's first name" data_tests: - not_null tags: - table_first_name - name: ip_address description: "The user's IP address" data_tests: - not_null - name: updated_at description: "The update time of the user" data_tests: - not_null - name: email description: "The user's email address" data_tests: - unique - name: favorite_color description: "The user's favorite color" data_tests: - accepted_values: { values: ['blue', 'green'], quote: true, tags: table_copy_favorite_color # tags can be a single string } tags: - table_favorite_color - name: fav_number description: "The user's favorite number" data_tests: - accepted_values: values: [3.14159265] quote: false tags: # tags can be a list of strings - favorite_number_is_pi - name: table_summary description: "The summary table" columns: - name: favorite_color_copy description: "The favorite color" data_tests: - not_null - unique - accepted_values: { values: ['blue', 'green'] } - relationships: { field: favorite_color, to: ref('table_copy') } tags: - table_favorite_color - name: count description: "The number of responses for this favorite color" data_tests: - not_null # all of these constraints will fail - name: table_failure_copy description: "The table copy that does not comply with the schema" columns: - name: id description: "The user ID" data_tests: - not_null - unique tags: - xfail - name: favorite_color description: "The user's favorite color" data_tests: - accepted_values: { values: ['blue', 'green'] } tags: - xfail # all of these constraints will fail - name: table_failure_summary description: "The table summary that does not comply with the schema" columns: - name: favorite_color description: "The favorite color" data_tests: - accepted_values: { values: ['red'] } - relationships: { field: favorite_color, to: ref('table_copy') } tags: - xfail # this table is disabled so these tests should be ignored - name: table_disabled description: "A disabled table" columns: - name: favorite_color description: "The favorite color" data_tests: - accepted_values: { values: ['red'] } - relationships: { field: favorite_color, to: ref('table_copy') } # all of these constraints will fail - name: table_failure_null_relation description: "A table with a null value where it should be a foreign key" columns: - name: id description: "The user ID" data_tests: - relationships: { field: id, to: ref('table_failure_copy') } tags: - xfail """ models_v2__models__table_summary_sql = """ {{ config( materialized='table' ) }} select favorite_color as favorite_color_copy, count(*) as count from {{ ref('table_copy') }} group by 1 """ models_v2__models__table_failure_summary_sql = """ {{ config( materialized='table' ) }} -- force a foreign key constraint failure here select 'purple' as favorite_color, count(*) as count from {{ ref('table_failure_copy') }} group by 1 """ models_v2__models__table_disabled_sql = """ {{ config( enabled=False ) }} -- force a foreign key constraint failure here select 'purple' as favorite_color, count(*) as count from {{ ref('table_failure_copy') }} group by 1 """ models_v2__models__table_failure_null_relation_sql = """ {{ config( materialized='table' ) }} -- force a foreign key constraint failure here select 105 as id, count(*) as count from {{ ref('table_failure_copy') }} group by 1 """ models_v2__models__table_failure_copy_sql = """ {{ config( materialized='table' ) }} select * from {{ this.schema }}.seed_failure """ models_v2__models__table_copy_sql = """ {{ config( materialized='table' ) }} select * from {{ this.schema }}.seed """ models_v2__malformed__schema_yml = """ version: 2 models: # this whole model should fail and not run - name: table_copy description: "A copy of the table" columns: - name: id description: "The ID" data_tests: - not_null - unique - name: favorite_color data_tests: # this is missing a "-" and is malformed accepted_values: { values: ['blue', 'green'] } # this whole model should pass and run - name: table_summary description: "The summary table" columns: - name: favorite_color description: "The favorite color" data_tests: - not_null - unique - accepted_values: { values: ['blue', 'green'] } - relationships: { field: favorite_color, to: ref('table_copy') } - name: count description: "The number of responses for this favorite color" data_tests: - not_null """ models_v2__malformed__table_summary_sql = """ {{ config( materialized='table' ) }} select favorite_color, count(*) as count from {{ ref('table_copy') }} group by 1 """ models_v2__malformed__table_copy_sql = """ {{ config( materialized='table' ) }} select * from {{ this.schema }}.seed """ models_v2__override_get_test_models_fail__schema_yml = """ version: 2 models: - name: my_model description: "The table has 1 null values, and we're not okay with that." columns: - name: id description: "The number of responses for this favorite color - purple will be null" data_tests: - not_null """ models_v2__override_get_test_models_fail__my_model_sql = """ select 1 as id UNION ALL select null as id """ models_v2__custom_configs__schema_yml = """ version: 2 models: - name: table_copy description: "A copy of the table" # passes data_tests: - where - error_if - warn_if - limit - fail_calc columns: - name: id data_tests: # relationships with where - relationships: to: ref('table_copy') # itself field: id where: 1=1 - name: table_copy_another_one data_tests: - where: # test override + weird quoting config: where: "\\"favorite_color\\" = 'red'" - name: "table.copy.with.dots" description: "A copy of the table with a gross name" # passes, see https://github.com/dbt-labs/dbt-core/issues/3857 data_tests: - where """ models_v2__custom_configs__table_copy_another_one_sql = """ select * from {{ ref('table_copy') }} """ models_v2__custom_configs__table_copy_sql = """ {{ config( materialized='table' ) }} select * from {{ this.schema }}.seed """ models_v2__custom_configs__table_copy_with_dots_sql = """ select * from {{ ref('table_copy') }} """ models_v2__render_test_configured_arg_models__schema_yml = """ version: 2 models: - name: model data_tests: - equivalent: value: "{{ var('myvar', 'baz') }}-bar" """ models_v2__render_test_configured_arg_models__model_sql = """ select 1 as id """ models_v2__custom__schema_yml = """ version: 2 models: - name: table_copy description: "A copy of the table" columns: - name: email data_tests: - not_null - name: id description: "The ID" data_tests: - unique - name: favorite_color data_tests: - every_value_is_blue - rejected_values: { values: ['orange', 'purple'] } # passes data_tests: - local_dep.equality: { compare_model: ref('table_copy') } """ models_v2__custom__table_copy_sql = """ {{ config( materialized='table' ) }} select * from {{ this.schema }}.seed """ models_v2__limit_null__schema_yml = """ version: 2 models: - name: table_limit_null description: "The table has 1 null values, and we're okay with that, until it's more than 1." columns: - name: favorite_color_full_list description: "The favorite color" - name: count description: "The number of responses for this favorite color - purple will be null" data_tests: - not_null: error_if: '>1' warn_if: '>1' - name: table_warning_limit_null description: "The table has 1 null value, and we're okay with 1, but want to know of any." columns: - name: favorite_color_full_list description: "The favorite color" - name: count description: "The number of responses for this favorite color - purple will be null" data_tests: - not_null: error_if: '>1' - name: table_failure_limit_null description: "The table has some 2 null values, and that's not ok. Warn and error." columns: - name: favorite_color_full_list description: "The favorite color" - name: count description: "The number of responses for this favorite color - purple will be null" data_tests: - not_null: error_if: '>1' """ models_v2__limit_null__table_warning_limit_null_sql = """ {{ config( materialized='table' ) }} select * from {{ref('table_limit_null')}} """ models_v2__limit_null__table_limit_null_sql = """ {{ config( materialized='table' ) }} select favorite_color as favorite_color_full_list, count(*) as count from {{ this.schema }}.seed group by 1 UNION ALL select 'purple' as favorite_color_full_list, null as count """ models_v2__limit_null__table_failure_limit_null_sql = """ {{ config( materialized='table' ) }} select * from {{ref('table_limit_null')}} UNION ALL select 'magenta' as favorite_color_full_list, null as count """ local_utils__dbt_project_yml = """ name: 'local_utils' version: '1.0' config-version: 2 profile: 'default' macro-paths: ["macros"] """ local_utils__macros__datediff_sql = """ {% macro datediff(first_date, second_date, datepart) %} {{ return(adapter.dispatch('datediff', 'local_utils')(first_date, second_date, datepart)) }} {% endmacro %} {% macro default__datediff(first_date, second_date, datepart) %} datediff( {{ datepart }}, {{ first_date }}, {{ second_date }} ) {% endmacro %} {% macro postgres__datediff(first_date, second_date, datepart) %} {% if datepart == 'year' %} (date_part('year', ({{second_date}})::date) - date_part('year', ({{first_date}})::date)) {% elif datepart == 'quarter' %} ({{ adapter.dispatch('datediff', 'local_utils')(first_date, second_date, 'year') }} * 4 + date_part('quarter', ({{second_date}})::date) - date_part('quarter', ({{first_date}})::date)) {% else %} ( 1000 ) {% endif %} {% endmacro %} """ local_utils__macros__current_timestamp_sql = """ {% macro current_timestamp() -%} {{ return(adapter.dispatch('current_timestamp')) }} {%- endmacro %} {% macro default__current_timestamp() -%} now() {%- endmacro %} """ local_utils__macros__custom_test_sql = """ {% macro test_dispatch(model) -%} {{ return(adapter.dispatch('test_dispatch', macro_namespace = 'local_utils')()) }} {%- endmacro %} {% macro default__test_dispatch(model) %} select {{ adapter.dispatch('current_timestamp', macro_namespace = 'local_utils')() }} {% endmacro %} """ ephemeral__schema_yml = """ version: 2 models: - name: ephemeral columns: - name: id data_tests: - unique """ ephemeral__ephemeral_sql = """ {{ config(materialized='ephemeral') }} select 1 as id """ quote_required_models__schema_yml = """ version: 2 models: - name: model columns: - name: Id quote: true data_tests: - unique - not_null - name: model_again quote_columns: true columns: - name: Id data_tests: - unique - not_null - name: model_noquote quote_columns: true columns: - name: Id quote: false data_tests: - unique - not_null sources: # this should result in column quoting = true - name: my_source schema: "{{ target.schema }}" quoting: column: true tables: - name: model quoting: column: false columns: - name: Id quote: true data_tests: - unique - name: my_source_2 schema: "{{ target.schema }}" quoting: column: false tables: # this should result in column quoting = true - name: model quoting: column: true columns: - name: Id data_tests: - unique # this should result in column quoting = false - name: model_noquote columns: - name: Id data_tests: - unique """ quote_required_models__model_again_sql = """ select 1 as "Id" """ quote_required_models__model_noquote_sql = """ select 1 as id """ quote_required_models__model_sql = """ select 1 as "Id" """ alt_local_utils__macros__type_timestamp_sql = """ {%- macro type_timestamp() -%} {{ return(adapter.dispatch('type_timestamp', 'local_utils')()) }} {%- endmacro -%} {% macro default__type_timestamp() %} {{ return(adapter.dispatch('type_timestamp', 'dbt')()) }} {% endmacro %} """ macro_resolution_order_macros__my_custom_test_sql = """ {% test my_custom_test(model) %} select cast(current_timestamp as {{ dbt.type_timestamp() }}) limit 0 {% endtest %} """ macro_resolution_order_models__my_model_sql = """ select 1 as id """ macro_resolution_order_models__config_yml = """ version: 2 models: - name: my_model data_tests: - my_custom_test """ store_failures_models__my_model_sql = """ select 1 as id """ store_failures_models_true__config_yml = """ version: 2 models: - name: my_model columns: - name: id tests: - not_null: config: store_failures: true """ store_failures_models_false__config_yml = """ version: 2 models: - name: my_model columns: - name: id tests: - not_null: config: store_failures: false """ ================================================ FILE: tests/functional/schema_tests/test_custom_test_config.py ================================================ import re import pytest from dbt.contracts.graph.manifest import Manifest from dbt.contracts.graph.nodes import TestNode from dbt.exceptions import CompilationError from dbt.tests.util import get_manifest, run_dbt, update_config_file custom_config_yml = """ models: - name: table columns: - name: color data_tests: - accepted_values: values: ['blue', 'red'] config: custom_config_key: some_value - custom_color_from_config: severity: error config: test_color: orange store_failures: true unlogged: True - not_null: config: not_null_key: abc """ mixed_config_yml = """ models: - name: table columns: - name: color data_tests: - accepted_values: values: ['blue', 'red'] config: custom_config_key: some_value severity: warn - custom_color_from_config: severity: error config: test_color: blue """ same_key_error_yml = """ models: - name: table columns: - name: color data_tests: - accepted_values: values: ['blue', 'red'] severity: warn config: severity: error """ seed_csv = """ id,color,value 1,blue,10 2,red,20 3,green,30 4,yellow,40 5,blue,50 6,red,60 7,blue,70 8,green,80 9,yellow,90 10,blue,100 """.strip() table_sql = """ -- content of the table.sql select * from {{ ref('seed') }} """ test_custom_color_from_config = """ {% test custom_color_from_config(model, column_name) %} select * from {{ model }} where color = '{{ config.get('test_color') }}' {% endtest %} """ def _select_test_node(manifest: Manifest, pattern: re.Pattern[str]): # Find the test_id dynamically test_id = None for node_id in manifest.nodes: if pattern.match(node_id): test_id = node_id break # Ensure the test_id was found assert test_id is not None, "Test ID matching the pattern was not found in the manifest nodes" return manifest.nodes[test_id] def get_table_persistence(project, table_name): sql = f""" SELECT relpersistence FROM pg_class WHERE relname like '%{table_name}%' """ result = project.run_sql(sql, fetch="one") assert len(result) == 1 return result[0] class BaseDataTestsConfig: @pytest.fixture(scope="class") def seeds(self): return {"seed.csv": seed_csv} @pytest.fixture(scope="class") def macros(self): return {"custom_color_from_config.sql": test_custom_color_from_config} @pytest.fixture(scope="class", autouse=True) def setUp(self, project): run_dbt(["seed"]) class TestCustomDataTestConfig(BaseDataTestsConfig): @pytest.fixture(scope="class") def models(self): return {"table.sql": table_sql, "custom_config.yml": custom_config_yml} def test_custom_config(self, project): run_dbt(["run"]) run_dbt(["test"], expect_pass=False) manifest = get_manifest(project.project_root) # Pattern to match the test_id without the specific suffix pattern = re.compile(r"test\.test\.accepted_values_table_color__blue__red\.\d+") test_node: TestNode = _select_test_node(manifest, pattern) # Proceed with the assertions assert "custom_config_key" in test_node.config assert test_node.config["custom_config_key"] == "some_value" custom_color_pattern = re.compile(r"test\.test\.custom_color_from_config.*") custom_color_test_node = _select_test_node(manifest, custom_color_pattern) assert custom_color_test_node.config.get("test_color") == "orange" assert custom_color_test_node.config.get("unlogged") is True persistence = get_table_persistence(project, "custom_color_from_config_table_color") assert persistence == "u" not_null_pattern = re.compile(r"test\.test\.not_null.*") not_null_test_node = _select_test_node(manifest, not_null_pattern) assert not_null_test_node.config.get("not_null_key") == "abc" # set dbt_project.yml config and ensure that schema configs override project configs config_patch = { "data_tests": {"test_color": "blue", "some_key": "strange", "not_null_key": "def"} } update_config_file(config_patch, project.project_root, "dbt_project.yml") manifest = run_dbt(["parse"]) custom_color_test_node = _select_test_node(manifest, custom_color_pattern) assert custom_color_test_node.config.get("test_color") == "orange" assert custom_color_test_node.config.get("some_key") == "strange" not_null_test_node = _select_test_node(manifest, not_null_pattern) assert not_null_test_node.config.get("not_null_key") == "abc" class TestMixedDataTestConfig(BaseDataTestsConfig): @pytest.fixture(scope="class") def models(self): return {"table.sql": table_sql, "mixed_config.yml": mixed_config_yml} def test_mixed_config(self, project): run_dbt(["parse"]) manifest = get_manifest(project.project_root) # Pattern to match the test_id without the specific suffix pattern = re.compile(r"test\.test\.accepted_values_table_color__blue__red\.\d+") test_node = _select_test_node(manifest, pattern) assert "custom_config_key" in test_node.config assert test_node.config["custom_config_key"] == "some_value" assert "severity" in test_node.config assert test_node.config["severity"] == "warn" string_config_yml = """ models: - name: table columns: - name: color data_tests: - not_null: config: "severity" """ class TestConfigNotDictError: """ Regression test: when 'config' is a non-dict scalar (e.g. a string), dbt should raise a clear ParsingError rather than an opaque AttributeError: 'str' object has no attribute 'pop'. """ @pytest.fixture(scope="class") def models(self): return {"table.sql": table_sql, "string_config.yml": string_config_yml} def test_config_not_dict_raises_error(self, project): from dbt_common.exceptions import DbtBaseException with pytest.raises(DbtBaseException) as exc_info: run_dbt(["parse"]) exception_message = str(exc_info.value) assert "config" in exception_message.lower() assert "dict" in exception_message.lower() class TestSameKeyErrorDataTestConfig: @pytest.fixture(scope="class") def models(self): return {"table.sql": table_sql, "same_key_error.yml": same_key_error_yml} def test_same_key_error(self, project): """ Test that verifies dbt raises a CompilationError when the test configuration contains the same key at the top level and inside the config dictionary. """ # Run dbt and expect a CompilationError due to the invalid configuration with pytest.raises(CompilationError) as exc_info: run_dbt(["parse"]) # Extract the exception message exception_message = str(exc_info.value) # Assert that the error message contains the expected text assert "Test cannot have the same key at the top-level and in config" in exception_message ================================================ FILE: tests/functional/schema_tests/test_schema_v2_tests.py ================================================ import os import re import pytest from dbt.contracts.results import TestStatus from dbt.exceptions import CompilationError, DuplicateResourceNameError, ParsingError from dbt.tests.fixtures.project import write_project_files from dbt.tests.util import run_dbt, write_file from tests.fixtures.dbt_integration_project import dbt_integration_project # noqa: F401 from tests.functional.schema_tests.fixtures import ( all_quotes_schema__schema_yml, alt_local_utils__macros__type_timestamp_sql, case_sensitive_models__lowercase_sql, case_sensitive_models__schema_yml, case_sensitive_models__uppercase_SQL, custom_generic_test_config_custom_macro__model_a, custom_generic_test_config_custom_macro__schema_yml, custom_generic_test_names__model_a, custom_generic_test_names__schema_yml, custom_generic_test_names_alt_format__model_a, custom_generic_test_names_alt_format__schema_yml, dupe_generic_tests_collide__model_a, dupe_generic_tests_collide__schema_yml, ephemeral__ephemeral_sql, ephemeral__schema_yml, invalid_schema_models__model_sql, invalid_schema_models__schema_yml, local_dependency__dbt_project_yml, local_dependency__macros__equality_sql, local_utils__dbt_project_yml, local_utils__macros__current_timestamp_sql, local_utils__macros__custom_test_sql, local_utils__macros__datediff_sql, macro_resolution_order_macros__my_custom_test_sql, macro_resolution_order_models__config_yml, macro_resolution_order_models__my_model_sql, macros_v2__custom_configs__test_sql, macros_v2__macros__tests_sql, macros_v2__override_get_test_macros__get_test_sql_sql, macros_v2__override_get_test_macros_fail__get_test_sql_sql, models_v2__custom__schema_yml, models_v2__custom__table_copy_sql, models_v2__custom_configs__schema_yml, models_v2__custom_configs__table_copy_another_one_sql, models_v2__custom_configs__table_copy_sql, models_v2__custom_configs__table_copy_with_dots_sql, models_v2__limit_null__schema_yml, models_v2__limit_null__table_failure_limit_null_sql, models_v2__limit_null__table_limit_null_sql, models_v2__limit_null__table_warning_limit_null_sql, models_v2__malformed__schema_yml, models_v2__malformed__table_copy_sql, models_v2__malformed__table_summary_sql, models_v2__models__schema_yml, models_v2__models__table_copy_sql, models_v2__models__table_disabled_sql, models_v2__models__table_failure_copy_sql, models_v2__models__table_failure_null_relation_sql, models_v2__models__table_failure_summary_sql, models_v2__models__table_summary_sql, models_v2__override_get_test_models__my_model_failure_sql, models_v2__override_get_test_models__my_model_pass_sql, models_v2__override_get_test_models__my_model_warning_sql, models_v2__override_get_test_models__schema_yml, models_v2__override_get_test_models_fail__my_model_sql, models_v2__override_get_test_models_fail__schema_yml, models_v2__render_test_cli_arg_models__model_sql, models_v2__render_test_cli_arg_models__schema_yml, models_v2__render_test_configured_arg_models__model_sql, models_v2__render_test_configured_arg_models__schema_yml, name_collision__base_extension_sql, name_collision__base_sql, name_collision__schema_yml, quote_required_models__model_again_sql, quote_required_models__model_noquote_sql, quote_required_models__model_sql, quote_required_models__schema_yml, seeds__some_seed_csv, store_failures_models__my_model_sql, store_failures_models_false__config_yml, store_failures_models_true__config_yml, test_context_macros__custom_schema_tests_sql, test_context_macros__my_test_sql, test_context_macros__test_my_datediff_sql, test_context_macros_namespaced__custom_schema_tests_sql, test_context_macros_namespaced__my_test_sql, test_context_models__model_a_sql, test_context_models__model_b_sql, test_context_models__model_c_sql, test_context_models__schema_yml, test_context_models_namespaced__model_a_sql, test_context_models_namespaced__model_b_sql, test_context_models_namespaced__model_c_sql, test_context_models_namespaced__schema_yml, test_context_where_subq_macros__custom_generic_test_sql, test_context_where_subq_models__model_a_sql, test_context_where_subq_models__schema_yml, test_utils__dbt_project_yml, test_utils__macros__current_timestamp_sql, test_utils__macros__custom_test_sql, wrong_specification_block__schema_yml, ) class TestSchemaTests: @pytest.fixture(scope="class", autouse=True) def setUp(self, project): project.run_sql_file(os.path.join(project.test_data_dir, "seed.sql")) project.run_sql_file(os.path.join(project.test_data_dir, "seed_failure.sql")) @pytest.fixture(scope="class") def models(self): return { "schema.yml": models_v2__models__schema_yml, "table_summary.sql": models_v2__models__table_summary_sql, "table_failure_summary.sql": models_v2__models__table_failure_summary_sql, "table_disabled.sql": models_v2__models__table_disabled_sql, "table_failure_null_relation.sql": models_v2__models__table_failure_null_relation_sql, "table_failure_copy.sql": models_v2__models__table_failure_copy_sql, "table_copy.sql": models_v2__models__table_copy_sql, } def assertTestFailed(self, result): assert result.status == "fail" assert not result.skipped assert result.failures > 0, "test {} did not fail".format(result.node.name) def assertTestPassed(self, result): assert result.status == "pass" assert not result.skipped assert result.failures == 0, "test {} failed".format(result.node.name) def test_schema_tests( self, project, ): results = run_dbt() assert len(results) == 5 test_results = run_dbt(["test"], expect_pass=False) # If the disabled model's tests ran, there would be 20 of these. assert len(test_results) == 19 for result in test_results: # assert that all deliberately failing tests actually fail if "failure" in result.node.name: self.assertTestFailed(result) # assert that actual tests pass else: self.assertTestPassed(result) assert sum(x.failures for x in test_results) == 6 def test_schema_test_selection( self, project, ): results = run_dbt() assert len(results) == 5 test_results = run_dbt(["test", "--models", "tag:table_favorite_color"]) # 1 in table_copy, 4 in table_summary assert len(test_results) == 5 for result in test_results: self.assertTestPassed(result) test_results = run_dbt(["test", "--models", "tag:favorite_number_is_pi"]) assert len(test_results) == 1 self.assertTestPassed(test_results[0]) test_results = run_dbt(["test", "--models", "tag:table_copy_favorite_color"]) assert len(test_results) == 1 self.assertTestPassed(test_results[0]) def test_schema_test_exclude_failures( self, project, ): results = run_dbt() assert len(results) == 5 test_results = run_dbt(["test", "--exclude", "tag:xfail"]) # If the failed + disabled model's tests ran, there would be 20 of these. assert len(test_results) == 13 for result in test_results: self.assertTestPassed(result) test_results = run_dbt(["test", "--models", "tag:xfail"], expect_pass=False) assert len(test_results) == 6 for result in test_results: self.assertTestFailed(result) class TestLimitedSchemaTests: @pytest.fixture(scope="class", autouse=True) def setUp(self, project): project.run_sql_file(os.path.join(project.test_data_dir, "seed.sql")) @pytest.fixture(scope="class") def models(self): return { "schema.yml": models_v2__limit_null__schema_yml, "table_warning_limit_null.sql": models_v2__limit_null__table_warning_limit_null_sql, "table_limit_null.sql": models_v2__limit_null__table_limit_null_sql, "table_failure_limit_null.sql": models_v2__limit_null__table_failure_limit_null_sql, } def assertTestFailed(self, result): assert result.status == "fail" assert not result.skipped assert result.failures > 0, "test {} did not fail".format(result.node.name) def assertTestWarn(self, result): assert result.status == "warn" assert not result.skipped assert result.failures > 0, "test {} passed without expected warning".format( result.node.name ) def assertTestPassed(self, result): assert result.status == "pass" assert not result.skipped assert result.failures == 0, "test {} failed".format(result.node.name) def test_limit_schema_tests( self, project, ): results = run_dbt() assert len(results) == 3 test_results = run_dbt(["test"], expect_pass=False) assert len(test_results) == 3 for result in test_results: # assert that all deliberately failing tests actually fail if "failure" in result.node.name: self.assertTestFailed(result) # assert that tests with warnings have them elif "warning" in result.node.name: self.assertTestWarn(result) # assert that actual tests pass else: self.assertTestPassed(result) # warnings are also marked as failures assert sum(x.failures for x in test_results) == 3 class TestDefaultBoolType: # test with default True/False in get_test_sql macro @pytest.fixture(scope="class") def models(self): return { "schema.yml": models_v2__override_get_test_models__schema_yml, "my_model_warning.sql": models_v2__override_get_test_models__my_model_warning_sql, "my_model_pass.sql": models_v2__override_get_test_models__my_model_pass_sql, "my_model_failure.sql": models_v2__override_get_test_models__my_model_failure_sql, } def assertTestFailed(self, result): assert result.status == "fail" assert not result.skipped assert result.failures > 0, "test {} did not fail".format(result.node.name) def assertTestWarn(self, result): assert result.status == "warn" assert not result.skipped assert result.failures > 0, "test {} passed without expected warning".format( result.node.name ) def assertTestPassed(self, result): assert result.status == "pass" assert not result.skipped assert result.failures == 0, "test {} failed".format(result.node.name) def test_limit_schema_tests( self, project, ): results = run_dbt() assert len(results) == 3 test_results = run_dbt(["test"], expect_pass=False) assert len(test_results) == 3 for result in test_results: # assert that all deliberately failing tests actually fail if "failure" in result.node.name: self.assertTestFailed(result) # assert that tests with warnings have them elif "warning" in result.node.name: self.assertTestWarn(result) # assert that actual tests pass else: self.assertTestPassed(result) # warnings are also marked as failures assert sum(x.failures for x in test_results) == 3 class TestOtherBoolType: @pytest.fixture(scope="class", autouse=True) def setUp(self, project_root): macros_v2_file = { "override_get_test_macros": { "get_test_sql.sql": macros_v2__override_get_test_macros__get_test_sql_sql }, } write_project_files(project_root, "macros-v2", macros_v2_file) # test with expected 0/1 in custom get_test_sql macro @pytest.fixture(scope="class") def models(self): return { "schema.yml": models_v2__override_get_test_models__schema_yml, "my_model_warning.sql": models_v2__override_get_test_models__my_model_warning_sql, "my_model_pass.sql": models_v2__override_get_test_models__my_model_pass_sql, "my_model_failure.sql": models_v2__override_get_test_models__my_model_failure_sql, } @pytest.fixture(scope="class") def project_config_update(self): return { "config-version": 2, "macro-paths": ["macros-v2/override_get_test_macros"], } def assertTestFailed(self, result): assert result.status == "fail" assert not result.skipped assert result.failures > 0, "test {} did not fail".format(result.node.name) def assertTestWarn(self, result): assert result.status == "warn" assert not result.skipped assert result.failures > 0, "test {} passed without expected warning".format( result.node.name ) def assertTestPassed(self, result): assert result.status == "pass" assert not result.skipped assert result.failures == 0, "test {} failed".format(result.node.name) def test_limit_schema_tests( self, project, ): results = run_dbt() assert len(results) == 3 test_results = run_dbt(["test"], expect_pass=False) assert len(test_results) == 3 for result in test_results: # assert that all deliberately failing tests actually fail if "failure" in result.node.name: self.assertTestFailed(result) # assert that tests with warnings have them elif "warning" in result.node.name: self.assertTestWarn(result) # assert that actual tests pass else: self.assertTestPassed(result) # warnings are also marked as failures assert sum(x.failures for x in test_results) == 3 class TestNonBoolType: @pytest.fixture(scope="class", autouse=True) def setUp(self, project_root): macros_v2_file = { "override_get_test_macros_fail": { "get_test_sql.sql": macros_v2__override_get_test_macros_fail__get_test_sql_sql }, } write_project_files(project_root, "macros-v2", macros_v2_file) # test with invalid 'x'/'y' in custom get_test_sql macro @pytest.fixture(scope="class") def models(self): return { "schema.yml": models_v2__override_get_test_models_fail__schema_yml, "my_model.sql": models_v2__override_get_test_models_fail__my_model_sql, } @pytest.fixture(scope="class") def project_config_update(self): return { "config-version": 2, "macro-paths": ["macros-v2/override_get_test_macros_fail"], } def test_limit_schema_tests( self, project, ): results = run_dbt() assert len(results) == 1 run_result = run_dbt(["test"], expect_pass=False) results = run_result.results assert len(results) == 1 assert results[0].status == TestStatus.Error assert re.search(r"'get_test_sql' returns 'x'", results[0].message) class TestMalformedSchemaTests: @pytest.fixture(scope="class", autouse=True) def setUp(self, project): project.run_sql_file(os.path.join(project.test_data_dir, "seed.sql")) @pytest.fixture(scope="class") def models(self): return { "schema.yml": models_v2__malformed__schema_yml, "table_summary.sql": models_v2__malformed__table_summary_sql, "table_copy.sql": models_v2__malformed__table_copy_sql, } def test_malformed_schema_will_break_run( self, project, ): with pytest.raises(ParsingError): run_dbt() class TestCustomConfigSchemaTests: @pytest.fixture(scope="class", autouse=True) def setUp(self, project, project_root): project.run_sql_file(os.path.join(project.test_data_dir, "seed.sql")) macros_v2_file = {"custom-configs": {"test.sql": macros_v2__custom_configs__test_sql}} write_project_files(project_root, "macros-v2", macros_v2_file) @pytest.fixture(scope="class") def models(self): return { "schema.yml": models_v2__custom_configs__schema_yml, "table_copy_another_one.sql": models_v2__custom_configs__table_copy_another_one_sql, "table_copy.sql": models_v2__custom_configs__table_copy_sql, "table.copy.with.dots.sql": models_v2__custom_configs__table_copy_with_dots_sql, } @pytest.fixture(scope="class") def project_config_update(self): return { "config-version": 2, "macro-paths": ["macros-v2/custom-configs"], } def test_config( self, project, ): """Test that tests use configs properly. All tests for this project will fail, configs are set to make test pass.""" results = run_dbt(["test"], expect_pass=False) assert len(results) == 8 for result in results: assert not result.skipped class TestHooksInTests: @pytest.fixture(scope="class") def models(self): return { "schema.yml": ephemeral__schema_yml, "ephemeral.sql": ephemeral__ephemeral_sql, } @pytest.fixture(scope="class") def project_config_update(self): return { "config-version": 2, "on-run-start": ["{{ log('hooks called in tests -- good!') if execute }}"], "on-run-end": ["{{ log('hooks called in tests -- good!') if execute }}"], } def test_hooks_do_run_for_tests( self, project, ): # This passes now that hooks run, a behavior we changed in v1.0 results = run_dbt(["test", "--model", "ephemeral"]) assert len(results) == 3 for result in results: assert result.status in ("pass", "success") assert not result.skipped assert result.failures == 0, "test {} failed".format(result.node.name) class TestHooksForWhich: @pytest.fixture(scope="class") def models(self): return { "schema.yml": ephemeral__schema_yml, "ephemeral.sql": ephemeral__ephemeral_sql, } @pytest.fixture(scope="class") def project_config_update(self): return { "config-version": 2, "on-run-start": [ "{{exceptions.raise_compiler_error('hooks called in tests -- error') if (execute and flags.WHICH != 'test') }}" ], "on-run-end": [ "{{exceptions.raise_compiler_error('hooks called in tests -- error') if (execute and flags.WHICH != 'test') }}" ], } def test_these_hooks_dont_run_for_tests( self, project, ): # This would fail if the hooks ran results = run_dbt(["test", "--model", "ephemeral"]) assert len(results) == 3 for result in results: assert result.status in ("pass", "success") assert not result.skipped assert result.failures == 0, "test {} failed".format(result.node.name) class TestCustomSchemaTests: @pytest.fixture(scope="class", autouse=True) def setUp(self, project, project_root, dbt_integration_project): # noqa: F811 write_project_files(project_root, "dbt_integration_project", dbt_integration_project) project.run_sql_file(os.path.join(project.test_data_dir, "seed.sql")) local_dependency_files = { "dbt_project.yml": local_dependency__dbt_project_yml, "macros": {"equality.sql": local_dependency__macros__equality_sql}, } write_project_files(project_root, "local_dependency", local_dependency_files) macros_v2_file = { "macros": {"tests.sql": macros_v2__macros__tests_sql}, } write_project_files(project_root, "macros-v2", macros_v2_file) @pytest.fixture(scope="class") def packages(self): return { "packages": [ { "local": "./local_dependency", }, { "local": "./dbt_integration_project", }, ] } @pytest.fixture(scope="class") def project_config_update(self): # dbt-utils contains a schema test (equality) # dbt-integration-project contains a schema.yml file # both should work! return { "config-version": 2, "macro-paths": ["macros-v2/macros"], } @pytest.fixture(scope="class") def models(self): return { "schema.yml": models_v2__custom__schema_yml, "table_copy.sql": models_v2__custom__table_copy_sql, } def test_schema_tests( self, project, ): run_dbt(["deps"]) results = run_dbt() assert len(results) == 4 test_results = run_dbt(["test"], expect_pass=False) assert len(test_results) == 6 expected_failures = [ "not_null_table_copy_email", "every_value_is_blue_table_copy_favorite_color", ] for result in test_results: if result.status == "fail": assert result.node.name in expected_failures class TestQuotedSchemaTestColumns: @pytest.fixture(scope="class") def models(self): return { "schema.yml": quote_required_models__schema_yml, "model_again.sql": quote_required_models__model_again_sql, "model_noquote.sql": quote_required_models__model_noquote_sql, "model.sql": quote_required_models__model_sql, } def test_quote_required_column( self, project, ): results = run_dbt() assert len(results) == 3 results = run_dbt(["test", "-m", "model"]) assert len(results) == 2 results = run_dbt(["test", "-m", "model_again"]) assert len(results) == 2 results = run_dbt(["test", "-m", "model_noquote"]) assert len(results) == 2 results = run_dbt(["test", "-m", "source:my_source"]) assert len(results) == 1 results = run_dbt(["test", "-m", "source:my_source_2"]) assert len(results) == 2 class TestCliVarsSchemaTests: @pytest.fixture(scope="class", autouse=True) def setUp(self, project_root): macros_v2_file = { "macros": {"tests.sql": macros_v2__macros__tests_sql}, } write_project_files(project_root, "macros-v2", macros_v2_file) @pytest.fixture(scope="class") def models(self): return { "schema.yml": models_v2__render_test_cli_arg_models__schema_yml, "model.sql": models_v2__render_test_cli_arg_models__model_sql, } @pytest.fixture(scope="class") def project_config_update(self): return { "config-version": 2, "macro-paths": ["macros-v2/macros"], } def test_argument_rendering( self, project, ): results = run_dbt() assert len(results) == 1 results = run_dbt(["test", "--vars", "{myvar: foo}"]) assert len(results) == 1 run_dbt(["test"], expect_pass=False) class TestConfiguredVarsSchemaTests: @pytest.fixture(scope="class", autouse=True) def setUp(self, project_root): macros_v2_file = { "macros": {"tests.sql": macros_v2__macros__tests_sql}, } write_project_files(project_root, "macros-v2", macros_v2_file) @pytest.fixture(scope="class") def models(self): return { "schema.yml": models_v2__render_test_configured_arg_models__schema_yml, "model.sql": models_v2__render_test_configured_arg_models__model_sql, } @pytest.fixture(scope="class") def project_config_update(self): return { "config-version": 2, "macro-paths": ["macros-v2/macros"], "vars": {"myvar": "foo"}, } def test_argument_rendering( self, project, ): results = run_dbt() assert len(results) == 1 results = run_dbt(["test"]) assert len(results) == 1 class TestSchemaCaseInsensitive: @pytest.fixture(scope="class") def models(self): return { "schema.yml": case_sensitive_models__schema_yml, "lowercase.sql": case_sensitive_models__lowercase_sql, } @pytest.fixture(scope="class", autouse=True) def setUP(self, project): # Create the uppercase SQL file model_dir = os.path.join(project.project_root, "models") write_file(case_sensitive_models__uppercase_SQL, model_dir, "uppercase.SQL") def test_schema_lowercase_sql( self, project, ): results = run_dbt() assert len(results) == 2 results = run_dbt(["test", "-m", "lowercase"]) assert len(results) == 1 def test_schema_uppercase_sql( self, project, ): results = run_dbt() assert len(results) == 2 results = run_dbt(["test", "-m", "uppercase"]) assert len(results) == 1 class TestSchemaTestContext: @pytest.fixture(scope="class", autouse=True) def setUp(self, project_root): local_utils_files = { "dbt_project.yml": local_utils__dbt_project_yml, "macros": { "datediff.sql": local_utils__macros__datediff_sql, "current_timestamp.sql": local_utils__macros__current_timestamp_sql, "custom_test.sql": local_utils__macros__custom_test_sql, }, } write_project_files(project_root, "local_utils", local_utils_files) test_context_macros_files = { "my_test.sql": test_context_macros__my_test_sql, "test_my_datediff.sql": test_context_macros__test_my_datediff_sql, "custom_schema_tests.sql": test_context_macros__custom_schema_tests_sql, } write_project_files(project_root, "test-context-macros", test_context_macros_files) @pytest.fixture(scope="class") def models(self): return { "schema.yml": test_context_models__schema_yml, "model_c.sql": test_context_models__model_c_sql, "model_b.sql": test_context_models__model_b_sql, "model_a.sql": test_context_models__model_a_sql, } @pytest.fixture(scope="class") def project_config_update(self): return { "config-version": 2, "macro-paths": ["test-context-macros"], "vars": {"local_utils_dispatch_list": ["local_utils"]}, } @pytest.fixture(scope="class") def packages(self): return {"packages": [{"local": "local_utils"}]} def test_test_context_tests(self, project): # This test tests the the TestContext and TestMacroNamespace # are working correctly run_dbt(["deps"]) results = run_dbt() assert len(results) == 3 run_result = run_dbt(["test"], expect_pass=False) results = run_result.results results = sorted(results, key=lambda r: r.node.name) assert len(results) == 5 # call_pkg_macro_model_c_ assert results[0].status == TestStatus.Fail # dispatch_model_c_ assert results[1].status == TestStatus.Fail # my_datediff assert re.search(r"1000", results[2].node.compiled_code) # type_one_model_a_ assert results[3].status == TestStatus.Fail assert re.search(r"union all", results[3].node.compiled_code) # type_two_model_a_ assert results[4].status == TestStatus.Warn assert results[4].node.config.severity == "WARN" class TestSchemaTestContextWithMacroNamespace: @pytest.fixture(scope="class", autouse=True) def setUp(self, project_root): test_utils_files = { "dbt_project.yml": test_utils__dbt_project_yml, "macros": { "current_timestamp.sql": test_utils__macros__current_timestamp_sql, "custom_test.sql": test_utils__macros__custom_test_sql, }, } write_project_files(project_root, "test_utils", test_utils_files) local_utils_files = { "dbt_project.yml": local_utils__dbt_project_yml, "macros": { "datediff.sql": local_utils__macros__datediff_sql, "current_timestamp.sql": local_utils__macros__current_timestamp_sql, "custom_test.sql": local_utils__macros__custom_test_sql, }, } write_project_files(project_root, "local_utils", local_utils_files) test_context_macros_namespaced_file = { "my_test.sql": test_context_macros_namespaced__my_test_sql, "custom_schema_tests.sql": test_context_macros_namespaced__custom_schema_tests_sql, } write_project_files( project_root, "test-context-macros-namespaced", test_context_macros_namespaced_file ) @pytest.fixture(scope="class") def models(self): return { "schema.yml": test_context_models_namespaced__schema_yml, "model_c.sql": test_context_models_namespaced__model_c_sql, "model_b.sql": test_context_models_namespaced__model_b_sql, "model_a.sql": test_context_models_namespaced__model_a_sql, } @pytest.fixture(scope="class") def project_config_update(self): return { "config-version": 2, "macro-paths": ["test-context-macros-namespaced"], "dispatch": [ { "macro_namespace": "test_utils", "search_order": ["local_utils", "test_utils"], } ], } @pytest.fixture(scope="class") def packages(self): return { "packages": [ {"local": "test_utils"}, {"local": "local_utils"}, ] } def test_test_context_with_macro_namespace( self, project, ): # This test tests the the TestContext and TestMacroNamespace # are working correctly run_dbt(["deps"]) results = run_dbt() assert len(results) == 3 run_result = run_dbt(["test"], expect_pass=False) results = run_result.results results = sorted(results, key=lambda r: r.node.name) assert len(results) == 4 # call_pkg_macro_model_c_ assert results[0].status == TestStatus.Fail # dispatch_model_c_ assert results[1].status == TestStatus.Fail # type_one_model_a_ assert results[2].status == TestStatus.Fail assert re.search(r"union all", results[2].node.compiled_code) # type_two_model_a_ assert results[3].status == TestStatus.Warn assert results[3].node.config.severity == "WARN" class TestSchemaTestNameCollision: @pytest.fixture(scope="class") def models(self): return { "schema.yml": name_collision__schema_yml, "base.sql": name_collision__base_sql, "base_extension.sql": name_collision__base_extension_sql, } def test_collision_test_names_get_hash( self, project, ): """The models should produce unique IDs with a has appended""" results = run_dbt() test_results = run_dbt(["test"]) # both models and both tests run assert len(results) == 2 assert len(test_results) == 2 # both tests have the same unique id except for the hash expected_unique_ids = [ "test.test.not_null_base_extension_id.922d83a56c", "test.test.not_null_base_extension_id.c8d18fe069", ] assert test_results[0].node.unique_id in expected_unique_ids assert test_results[1].node.unique_id in expected_unique_ids class TestGenericTestsCollide: @pytest.fixture(scope="class") def models(self): return { "schema.yml": dupe_generic_tests_collide__schema_yml, "model_a.sql": dupe_generic_tests_collide__model_a, } def test_generic_test_collision( self, project, ): """These tests collide, since only the configs differ""" with pytest.raises(DuplicateResourceNameError) as exc: run_dbt() assert "dbt found two data_tests with the name" in str(exc.value) class TestGenericTestsConfigCustomMacros: @pytest.fixture(scope="class") def models(self): return { "schema.yml": custom_generic_test_config_custom_macro__schema_yml, "model_a.sql": custom_generic_test_config_custom_macro__model_a, } def test_generic_test_config_custom_macros( self, project, ): """This test has a reference to a custom macro its configs""" with pytest.raises(CompilationError) as exc: run_dbt() assert "Invalid generic test configuration" in str(exc) class TestGenericTestsCustomNames: @pytest.fixture(scope="class") def models(self): return { "schema.yml": custom_generic_test_names__schema_yml, "model_a.sql": custom_generic_test_names__model_a, } # users can define custom names for specific instances of generic tests def test_generic_tests_with_custom_names( self, project, ): """These tests don't collide, since they have user-provided custom names""" results = run_dbt() test_results = run_dbt(["test"]) # model + both tests run assert len(results) == 1 assert len(test_results) == 2 # custom names propagate to the unique_id expected_unique_ids = [ "test.test.not_null_where_1_equals_1.7b96089006", "test.test.not_null_where_1_equals_2.8ae586e17f", ] assert test_results[0].node.unique_id in expected_unique_ids assert test_results[1].node.unique_id in expected_unique_ids class TestGenericTestsCustomNamesAltFormat(TestGenericTestsCustomNames): @pytest.fixture(scope="class") def models(self): return { "schema.yml": custom_generic_test_names_alt_format__schema_yml, "model_a.sql": custom_generic_test_names_alt_format__model_a, } # exactly as above, just alternative format for yaml definition def test_collision_test_names_get_hash( self, project, ): """These tests don't collide, since they have user-provided custom names, defined using an alternative format""" super().test_generic_tests_with_custom_names(project) class TestInvalidSchema: @pytest.fixture(scope="class") def models(self): return { "schema.yml": invalid_schema_models__schema_yml, "model.sql": invalid_schema_models__model_sql, } def test_invalid_schema_file( self, project, ): with pytest.raises(ParsingError) as exc: run_dbt() assert re.search(r"'models' is not a list", str(exc)) class TestCommentedSchema: @pytest.fixture(scope="class") def models(self): return { "schema.yml": all_quotes_schema__schema_yml, "model.sql": invalid_schema_models__model_sql, } def test_quoted_schema_file(self, project): try: # A schema file consisting entirely of quotes should not be a problem run_dbt(["parse"]) except TypeError: assert ( False ), "`dbt parse` failed with a yaml file that is all comments with the same exception as 3568" except Exception: assert False, "`dbt parse` failed with a yaml file that is all comments" class TestWrongSpecificationBlock: @pytest.fixture(scope="class") def models(self): return {"schema.yml": wrong_specification_block__schema_yml} @pytest.fixture(scope="class") def seeds(self): return {"some_seed.csv": seeds__some_seed_csv} def test_wrong_specification_block( self, project, ): with pytest.warns(Warning): results = run_dbt( [ "ls", "-s", "some_seed", "--output", "json", "--output-keys", "name", "description", ] ) assert len(results) == 1 assert results[0] == '{"name": "some_seed", "description": ""}' class TestSchemaTestContextWhereSubq: @pytest.fixture(scope="class", autouse=True) def setUp(self, project_root): test_context_where_subq_macros_file = { "custom_generic_test.sql": test_context_where_subq_macros__custom_generic_test_sql } write_project_files( project_root, "test-context-where-subq-macros", test_context_where_subq_macros_file ) @pytest.fixture(scope="class") def models(self): return { "schema.yml": test_context_where_subq_models__schema_yml, "model_a.sql": test_context_where_subq_models__model_a_sql, } @pytest.fixture(scope="class") def project_config_update(self): return { "config-version": 2, "macro-paths": ["test-context-where-subq-macros"], } def test_test_context_tests( self, project, ): # This test tests that get_where_subquery() is included in TestContext + TestMacroNamespace, # otherwise api.Relation.create() will return an error results = run_dbt() assert len(results) == 1 results = run_dbt(["test"]) assert len(results) == 1 class TestCustomSchemaTestMacroResolutionOrder: @pytest.fixture(scope="class", autouse=True) def setUp(self, project_root): alt_local_utils_file = { "dbt_project.yml": local_utils__dbt_project_yml, "macros": { "datediff.sql": alt_local_utils__macros__type_timestamp_sql, }, } write_project_files(project_root, "alt_local_utils", alt_local_utils_file) macros_resolution_order_file = { "my_custom_test.sql": macro_resolution_order_macros__my_custom_test_sql, } write_project_files( project_root, "macro_resolution_order_macros", macros_resolution_order_file ) @pytest.fixture(scope="class") def models(self): return { "schema.yml": macro_resolution_order_models__config_yml, "my_model.sql": macro_resolution_order_models__my_model_sql, } @pytest.fixture(scope="class") def project_config_update(self): return { "config-version": 2, "macro-paths": ["macro_resolution_order_macros"], } @pytest.fixture(scope="class") def packages(self): return {"packages": [{"local": "alt_local_utils"}]} def test_macro_resolution_test_namespace( self, project, ): # https://github.com/dbt-labs/dbt-core/issues/5720 # Previously, macros called as 'dbt.some_macro' would not correctly # resolve to 'some_macro' from the 'dbt' namespace during static analysis, # if 'some_macro' also existed in an installed package, # leading to the macro being missing in the TestNamespace run_dbt(["deps"]) run_dbt(["parse"]) class TestSchemaTestStoreFailuresTrueParsing: @pytest.fixture(scope="class") def models(self): return { "schema.yml": store_failures_models_true__config_yml, "my_model.sql": store_failures_models__my_model_sql, } def test_parse_store_failures_True_as_table(self, project): manifest = run_dbt(["parse"]) test_node = [ node for node in manifest.nodes.values() if "not_null_my_model_id" in node.fqn ][0] assert test_node.config.store_failures assert test_node.config.store_failures_as == "table" class TestSchemaTestStoreFailuresFalseParsing: @pytest.fixture(scope="class") def models(self): return { "schema.yml": store_failures_models_false__config_yml, "my_model.sql": store_failures_models__my_model_sql, } def test_parse_store_failures_False_as_ephemeral(self, project): manifest = run_dbt(["parse"]) test_node = [ node for node in manifest.nodes.values() if "not_null_my_model_id" in node.fqn ][0] assert not test_node.config.store_failures assert test_node.config.store_failures_as == "ephemeral" class TestSchemaTestStoreFailuresTrueHierarchicalParsing: @pytest.fixture(scope="class") def models(self): return { "schema.yml": store_failures_models_true__config_yml, "my_model.sql": store_failures_models__my_model_sql, } @pytest.fixture(scope="class") def project_config_update(self): return { "tests": {"+store_failures": False}, } def test_parse_store_failures_True_as_table(self, project): manifest = run_dbt(["parse"]) test_node = [ node for node in manifest.nodes.values() if "not_null_my_model_id" in node.fqn ][0] assert test_node.config.store_failures assert test_node.config.store_failures_as == "table" class TestSchemaTestStoreFailuresFalseHierarchicalParsing: @pytest.fixture(scope="class") def models(self): return { "schema.yml": store_failures_models_false__config_yml, "my_model.sql": store_failures_models__my_model_sql, } @pytest.fixture(scope="class") def project_config_update(self): return { "tests": {"+store_failures": True}, } def test_parse_store_failures_False_as_ephemeral(self, project): manifest = run_dbt(["parse"]) test_node = [ node for node in manifest.nodes.values() if "not_null_my_model_id" in node.fqn ][0] assert not test_node.config.store_failures assert test_node.config.store_failures_as == "ephemeral" ================================================ FILE: tests/functional/schema_tests/test_sql_header_config.py ================================================ import re import pytest from dbt.contracts.graph.nodes import TestNode from dbt.events.types import CustomKeyInConfigDeprecation from dbt.tests.util import get_manifest, run_dbt from dbt_common.events.event_catcher import EventCatcher SQL_HEADER_MARKER = "-- SQL_HEADER_TEST_MARKER" seed_csv = """ id,value 1,10 2,20 3,30 """.strip() table_sql = """ select * from {{ ref('seed') }} """ # Singular data test using set_sql_header macro singular_test_with_sql_header = """ {{ config(store_failures_as="ephemeral") }} {% call set_sql_header(config) %} -- SQL_HEADER_TEST_MARKER {% endcall %} select id from {{ ref('table') }} where id > 100 """ # Schema YAML with generic test using sql_header config generic_test_with_sql_header_yml = """ models: - name: table columns: - name: id data_tests: - not_null: name: generic_test_with_sql_header config: sql_header: "-- SQL_HEADER_TEST_MARKER" """ class TestSingularDataTestSqlHeader: """Test that singular data tests properly parse and store sql_header from set_sql_header macro.""" @pytest.fixture(scope="class") def project_config_update(self): return {"flags": {"require_sql_header_in_test_configs": True}} @pytest.fixture(scope="class") def seeds(self): return {"seed.csv": seed_csv} @pytest.fixture(scope="class") def models(self): return {"table.sql": table_sql} @pytest.fixture(scope="class") def tests(self): return {"singular_test_with_sql_header.sql": singular_test_with_sql_header} @pytest.fixture(scope="class", autouse=True) def setUp(self, project): run_dbt(["seed"]) def test_singular_test_sql_header_in_config(self, project): run_dbt(["run"]) run_dbt(["test"]) manifest = get_manifest(project.project_root) pattern = re.compile(r"test\.test\.singular_test_with_sql_header") test_node = None for node_id, node in manifest.nodes.items(): if pattern.search(node_id) and isinstance(node, TestNode): test_node = node break assert test_node is not None, "Singular test node not found in manifest" sql_header = test_node.config.get("sql_header") assert sql_header is not None, "sql_header not found in singular test config" assert SQL_HEADER_MARKER in sql_header class TestGenericDataTestSqlHeader: """Test that generic data tests properly parse and store sql_header from YAML config.""" @pytest.fixture(scope="class") def project_config_update(self): return {"flags": {"require_sql_header_in_test_configs": True}} @pytest.fixture(scope="class") def seeds(self): return {"seed.csv": seed_csv} @pytest.fixture(scope="class") def models(self): return { "table.sql": table_sql, "schema.yml": generic_test_with_sql_header_yml, } @pytest.fixture(scope="class", autouse=True) def setUp(self, project): run_dbt(["seed"]) def test_generic_test_sql_header_in_config(self, project): run_dbt(["run"]) run_dbt(["test"]) manifest = get_manifest(project.project_root) pattern = re.compile(r"test\.test\.generic_test_with_sql_header") test_node = None for node_id, node in manifest.nodes.items(): if pattern.search(node_id) and isinstance(node, TestNode): test_node = node break assert test_node is not None, "Generic test node not found in manifest" assert test_node.config.sql_header == SQL_HEADER_MARKER class TestGenericDataTestSqlHeaderDeprecation: """Test that sql_header in test config emits deprecation warning when flag is off.""" @pytest.fixture(scope="class") def seeds(self): return {"seed.csv": seed_csv} @pytest.fixture(scope="class") def models(self): return { "table.sql": table_sql, "schema.yml": generic_test_with_sql_header_yml, } @pytest.fixture(scope="class", autouse=True) def setUp(self, project): run_dbt(["seed"]) def test_sql_header_deprecation_warning(self, project): event_catcher = EventCatcher(CustomKeyInConfigDeprecation) run_dbt( ["parse", "--no-partial-parse", "--show-all-deprecations"], callbacks=[event_catcher.catch], ) assert len(event_catcher.caught_events) == 1 assert ( "Custom key `sql_header` found in `config`" in event_catcher.caught_events[0].info.msg ) # Verify sql_header is preserved in the manifest manifest = get_manifest(project.project_root) pattern = re.compile(r"test\.test\.generic_test_with_sql_header") test_node = None for node_id, node in manifest.nodes.items(): if pattern.search(node_id) and isinstance(node, TestNode): test_node = node break assert test_node is not None, "Generic test node not found in manifest" assert test_node.config.sql_header == SQL_HEADER_MARKER ================================================ FILE: tests/functional/seeds/test_seed_column_type_validation.py ================================================ import pytest from dbt.tests.util import run_dbt, run_dbt_and_capture seeds__my_seed_csv = """id,name,value 1,john,100 2,jane,200 3,bob,300 """ my_seed_yml = """ version: 2 seeds: - name: my_seed config: column_types: id: integer name: text value: float # This column doesn't exist in the CSV non_existent_column: integer """ seeds__other_seed_csv = """id,name 1,john 2,jane """ other_seed_yml = """ version: 2 seeds: - name: other_seed config: column_types: id: integer name: text """ seeds__mismatched_columns_csv = """col_a,col_b,col_c 1,2,3 4,5,6 """ mismatched_columns_yml = """ version: 2 seeds: - name: mismatched_columns config: column_types: col_a: integer col_b: text col_d: integer # This doesn't exist """ class TestSeedColumnTypeValidation: """Test that column type validation works for seeds.""" @pytest.fixture(scope="class") def models(self): return { "my_seed.yml": my_seed_yml, "other_seed.yml": other_seed_yml, "mismatched_columns.yml": mismatched_columns_yml, } @pytest.fixture(scope="class") def seeds(self): return { "my_seed.csv": seeds__my_seed_csv, "other_seed.csv": seeds__other_seed_csv, "mismatched_columns.csv": seeds__mismatched_columns_csv, } def test_seed_with_invalid_column_type(self, project): results = run_dbt(["seed"]) my_seed_result = next((r for r in results if r.node.name == "my_seed"), None) assert my_seed_result is not None assert my_seed_result.agate_table is not None column_names = [col.name for col in my_seed_result.agate_table.columns] assert "id" in column_names assert "name" in column_names assert "value" in column_names # non_existent_column should not be in the table assert "non_existent_column" not in column_names rows = list(my_seed_result.agate_table.rows) assert len(rows) == 3 assert int(rows[0]["id"]) == 1 assert rows[0]["name"] == "john" assert int(rows[0]["value"]) == 100 class TestSeedColumnTypesBasic: """Test basic seed column types functionality.""" @pytest.fixture(scope="class") def models(self): return { "other_seed.yml": other_seed_yml, } @pytest.fixture(scope="class") def seeds(self): return { "other_seed.csv": seeds__other_seed_csv, } def test_seed_basic_column_types(self, project): results = run_dbt(["seed"]) assert len(results) == 1 other_seed_result = next((r for r in results if r.node.name == "other_seed"), None) assert other_seed_result is not None assert other_seed_result.agate_table is not None column_names = [col.name for col in other_seed_result.agate_table.columns] assert "id" in column_names assert "name" in column_names class TestSeedColumnTypesWithMismatchedColumns: """Test seed with column types for columns that don't exist in CSV.""" @pytest.fixture(scope="class") def models(self): return { "mismatched_columns.yml": mismatched_columns_yml, } @pytest.fixture(scope="class") def seeds(self): return { "mismatched_columns.csv": seeds__mismatched_columns_csv, } def test_seed_invalid_column_type_warning(self, project): results, log_output = run_dbt_and_capture(["seed"]) assert len(results) == 1 result = next((r for r in results if r.node.name == "mismatched_columns"), None) assert result is not None assert result.agate_table is not None column_names = [col.name for col in result.agate_table.columns] assert "col_a" in column_names assert "col_b" in column_names assert "col_c" in column_names assert "col_d" not in column_names rows = list(result.agate_table.rows) assert len(rows) == 2 assert int(rows[0]["col_a"]) == 1 assert rows[0]["col_b"] == "2" assert int(rows[0]["col_c"]) == 3 expected_msg_part = ( "Column types specified for non-existent columns in seed 'mismatched_columns'" ) assert expected_msg_part in log_output assert "col_d" in log_output ================================================ FILE: tests/functional/selected_resources/fixtures.py ================================================ on_run_start_macro_assert_selected_models_expected_list = """ {% macro assert_selected_models_expected_list(expected_list) %} {% if execute and (expected_list is not none) %} {% set sorted_selected_resources = selected_resources | sort %} {% set sorted_expected_list = expected_list | sort %} {% if sorted_selected_resources != sorted_expected_list %} {{ exceptions.raise_compiler_error("FAIL: sorted_selected_resources" ~ sorted_selected_resources ~ " is different from " ~ sorted_expected_list) }} {% endif %} {% endif %} {% endmacro %} """ my_model1 = """ select 1 as id """ my_model2 = """ select * from {{ ref('model1') }} """ my_snapshot = """ {% snapshot cc_all_snapshot %} {{ config( check_cols='all', unique_key='id', strategy='check', target_database=database, target_schema=schema ) }} select * from {{ ref('model2') }} {% endsnapshot %} """ ================================================ FILE: tests/functional/selected_resources/test_selected_resources.py ================================================ import pytest from dbt.tests.util import run_dbt from tests.functional.selected_resources.fixtures import ( my_model1, my_model2, my_snapshot, on_run_start_macro_assert_selected_models_expected_list, ) @pytest.fixture(scope="class") def macros(): return { "assert_selected_models_expected_list.sql": on_run_start_macro_assert_selected_models_expected_list, } @pytest.fixture(scope="class") def models(): return {"model1.sql": my_model1, "model2.sql": my_model2} @pytest.fixture(scope="class") def snapshots(): return { "my_snapshot.sql": my_snapshot, } @pytest.fixture(scope="class") def project_config_update(): return { "on-run-start": "{{ assert_selected_models_expected_list(var('expected_list',None)) }}", } @pytest.fixture def build_all(project): run_dbt(["build"]) @pytest.mark.usefixtures("build_all") class TestSelectedResources: def test_selected_resources_build_selector(self, project): results = run_dbt( [ "build", "--select", "model1+", "--vars", '{"expected_list": ["model.test.model1", "model.test.model2", "snapshot.test.cc_all_snapshot"]}', ] ) assert results[0].status == "success" def test_selected_resources_build_selector_subgraph(self, project): results = run_dbt( [ "build", "--select", "model2+", "--vars", '{"expected_list": ["model.test.model2", "snapshot.test.cc_all_snapshot"]}', ] ) assert results[0].status == "success" def test_selected_resources_run(self, project): results = run_dbt( [ "run", "--select", "model1+", "--vars", '{"expected_list": ["model.test.model2", "model.test.model1"]}', ] ) assert results[0].status == "success" def test_selected_resources_build_no_selector(self, project): results = run_dbt( [ "build", "--vars", '{"expected_list": ["model.test.model1", "model.test.model2", "snapshot.test.cc_all_snapshot"]}', ] ) assert results[0].status == "success" def test_selected_resources_build_no_model(self, project): results = run_dbt( [ "build", "--select", "model_that_does_not_exist", "--vars", '{"expected_list": []}', ] ) assert not results def test_selected_resources_test_no_model(self, project): results = run_dbt(["test", "--select", "model1+", "--vars", '{"expected_list": []}']) assert not results ================================================ FILE: tests/functional/selectors/test_default_selectors.py ================================================ import pytest from dbt.tests.util import run_dbt models__schema_yml = """ version: 2 sources: - name: src schema: "{{ target.schema }}" freshness: warn_after: {count: 24, period: hour} loaded_at_field: _loaded_at tables: - name: source_a identifier: model_c columns: - name: fun - name: _loaded_at - name: src schema: "{{ target.schema }}" freshness: warn_after: {count: 24, period: hour} loaded_at_field: _loaded_at tables: - name: source_b identifier: model_c columns: - name: fun - name: _loaded_at models: - name: model_a columns: - name: fun tags: [marketing] - name: model_b columns: - name: fun tags: [finance] """ models__model_a_sql = """ SELECT 1 AS fun """ models__model_b_sql = """ SELECT 1 AS fun """ seeds__model_c_csv = """fun,_loaded_at 1,2021-04-19 01:00:00""" @pytest.fixture(scope="class") def models(): return { "schema.yml": models__schema_yml, "model_b.sql": models__model_b_sql, "model_a.sql": models__model_a_sql, } @pytest.fixture(scope="class") def seeds(): return {"model_c.csv": seeds__model_c_csv} @pytest.fixture(scope="class") def selectors(): return """ selectors: - name: default_selector description: test default selector definition: union: - method: source value: "test.src.source_a" - method: fqn value: "model_a" default: true """ class TestDefaultSelectors: def test_model__list(self, project): result = run_dbt(["ls", "--resource-type", "model"]) assert "test.model_a" in result def test_model__compile(self, project): result = run_dbt(["compile"]) assert len(result) == 1 assert result.results[0].node.name == "model_a" def test_source__freshness(self, project): run_dbt(["seed", "-s", "test.model_c"]) result = run_dbt(["source", "freshness"]) assert len(result) == 1 assert result.results[0].node.name == "source_a" ================================================ FILE: tests/functional/selectors/test_selector_selector_method.py ================================================ from typing import Any import pytest from dbt.events.types import SelectExcludeIgnoredWithSelectorWarning from dbt.exceptions import DbtSelectorsError from dbt.tests.util import run_dbt from dbt_common.events.event_catcher import EventCatcher from dbt_common.exceptions import RecursionError as DbtRecursionError models__model_a_sql = """ SELECT 1 AS id """ models_model_b_sql = """ SELECT 1 AS id """ models_model_a_plus_1_sql = """ SELECT * FROM {{ ref('model_a') }} """ models_model_a_plus_2_sql = """ SELECT * FROM {{ ref('model_a_plus_1') }} """ models_model_c_sql = """ {{ config(tags=['tag_c']) }} SELECT * FROM {{ ref('model_a') }} UNION ALL SELECT * FROM {{ ref('model_b') }} """ selectors__yml = """ selectors: - name: model_a_selector description: Selects model_a definition: method: fqn value: model_a - name: model_b_selector description: Selects model_b definition: method: fqn value: model_b - name: model_c_selector description: Selects model_c definition: method: fqn value: model_c - name: model_a_b_selector description: Selects model_a and model_b definition: union: - method: fqn value: model_a - method: fqn value: model_b - name: model_a_plus_1_selector description: Selects model_a_plus_1 definition: method: fqn value: model_a_plus_1 - name: model_a_plus_2_selector description: Selects model_a_plus_2 definition: method: fqn value: model_a_plus_2 parents: true parents_depth: 2 - name: recursive_selector description: Selects recursive models definition: union: - model_b - selector:model_a_plus_2_selector - name: recursive_with_wildcards description: Selects recursive models with wildcards definition: union: - model_c - selector:model_[ab]_selector - name: circular_selection_hop description: Selects circular selection hop models definition: union: - selector:circular_dependency_selector - name: circular_dependency_selector description: Selects circular dependency models definition: union: - selector:circular_selection_hop - name: model_a_explicit_selector description: Selects model_a with an explicit selector method definition: method: selector value: model_a_selector """ @pytest.fixture(scope="class") def models(): return { "model_a.sql": models__model_a_sql, "model_a_plus_1.sql": models_model_a_plus_1_sql, "model_a_plus_2.sql": models_model_a_plus_2_sql, "model_b.sql": models_model_b_sql, "model_c.sql": models_model_c_sql, } @pytest.fixture(scope="class") def selectors(): return selectors__yml def assert_result_set(actual: Any, expected: set[str]): assert isinstance(actual, list) assert set(actual) == expected class TestSelectorSelectorMethod: def test_ls_with_selector_returns_model_a(self, project): result = run_dbt(["ls", "--select", "selector:model_a_selector"]) assert_result_set(result, {"test.model_a"}) def test_ls_with_selector_union(self, project): result = run_dbt(["ls", "--select", "selector:model_a_selector selector:model_b_selector"]) assert_result_set(result, {"test.model_a", "test.model_b"}) def test_ls_with_selector_intersection(self, project): result = run_dbt( ["ls", "--select", "selector:model_a_b_selector,selector:model_b_selector"] ) assert_result_set(result, {"test.model_b"}) def test_ls_with_graph_operator(self, project): # one child one parent result = run_dbt(["ls", "--select", "1+selector:model_a_plus_1_selector+1"]) assert_result_set( result, { "test.model_a", "test.model_a_plus_1", "test.model_a_plus_2", }, ) # two parents result = run_dbt(["ls", "--select", "1+selector:model_c_selector"]) assert_result_set(result, {"test.model_a", "test.model_b", "test.model_c"}) def test_selector_depth_overrides_operator_depth(self, project): result = run_dbt(["ls", "--select", "selector:model_a_plus_2_selector"]) assert_result_set( result, { "test.model_a", "test.model_a_plus_1", "test.model_a_plus_2", }, ) result = run_dbt(["ls", "--select", "1+selector:model_a_plus_2_selector"]) assert_result_set(result, {"test.model_a", "test.model_a_plus_1", "test.model_a_plus_2"}) def test_combine_with_other_methods(self, project): result = run_dbt(["ls", "--select", "selector:model_a_selector tag:tag_c"]) assert_result_set(result, {"test.model_a", "test.model_c"}) result = run_dbt(["ls", "--select", "selector:model_a_plus_2_selector model_a model_b"]) assert_result_set( result, { "test.model_a", "test.model_b", "test.model_a_plus_1", "test.model_a_plus_2", }, ) def test_recursive_selector(self, project): result = run_dbt(["ls", "--select", "selector:recursive_selector"]) assert_result_set( result, { "test.model_a", "test.model_b", "test.model_a_plus_1", "test.model_a_plus_2", }, ) def test_select_and_exclude(self, project): result = run_dbt( [ "ls", "--select", "1+selector:model_a_plus_1_selector", "--exclude", "selector:model_a_selector", ] ) assert_result_set(result, {"test.model_a_plus_1"}) def test_wildcards(self, project): result = run_dbt(["ls", "--select", "selector:model_?_selector"]) assert_result_set(result, {"test.model_a", "test.model_b", "test.model_c"}) result = run_dbt(["ls", "--select", "selector:*_c_*"]) assert_result_set(result, {"test.model_c"}) result = run_dbt(["ls", "--select", "selector:model_[ab]_selector"]) assert_result_set(result, {"test.model_a", "test.model_b"}) result = run_dbt(["ls", "--select", "selector:model_[a-c]_selector"]) assert_result_set(result, {"test.model_a", "test.model_b", "test.model_c"}) def test_recursive_with_wildcards(self, project): result = run_dbt(["ls", "--select", "selector:recursive_with_wildcards"]) assert_result_set(result, {"test.model_a", "test.model_b", "test.model_c"}) def test_circular_dependency(self, project): with pytest.raises(DbtRecursionError): run_dbt(["ls", "--select", "selector:circular_dependency_selector"], expect_pass=False) def test_selector_with_explicit_selector_method(self, project): result = run_dbt(["ls", "--select", "selector:model_a_explicit_selector"]) assert_result_set(result, {"test.model_a"}) def test_raises_if_no_selector_matches(self, project): with pytest.raises(DbtSelectorsError): run_dbt(["ls", "--select", "selector:nonexistent_selector"]) class TestSelectExcludeIgnoredWithSelectorWarning: """Test that SelectExcludeIgnoredWithSelectorWarning is raised when CLI is invoked with --selector together with --select or --exclude. """ def test_warning_raised_when_selector_and_select(self, project): event_catcher = EventCatcher(SelectExcludeIgnoredWithSelectorWarning) run_dbt( ["ls", "--selector", "model_a_selector", "--select", "model_a"], callbacks=[event_catcher.catch], ) assert len(event_catcher.caught_events) == 1 def test_warning_raised_when_selector_and_exclude(self, project): event_catcher = EventCatcher(SelectExcludeIgnoredWithSelectorWarning) run_dbt( ["ls", "--selector", "model_a_selector", "--exclude", "model_b"], callbacks=[event_catcher.catch], ) assert len(event_catcher.caught_events) == 1 def test_no_warning_when_selector_only(self, project): event_catcher = EventCatcher(SelectExcludeIgnoredWithSelectorWarning) run_dbt( ["ls", "--selector", "model_a_selector"], callbacks=[event_catcher.catch], ) assert len(event_catcher.caught_events) == 0 def test_warning_raised_when_selector_and_select_and_exclude(self, project): event_catcher = EventCatcher(SelectExcludeIgnoredWithSelectorWarning) run_dbt( [ "ls", "--selector", "model_a_selector", "--select", "model_a", "--exclude", "model_b", ], callbacks=[event_catcher.catch], ) assert len(event_catcher.caught_events) == 1 def test_no_warning_when_no_selector(self, project): event_catcher = EventCatcher(SelectExcludeIgnoredWithSelectorWarning) run_dbt( ["ls", "--select", "model_a", "--exclude", "model_b"], callbacks=[event_catcher.catch], ) assert len(event_catcher.caught_events) == 0 ================================================ FILE: tests/functional/semantic_models/fixtures.py ================================================ import textwrap simple_metricflow_time_spine_sql = """ SELECT to_date('02/20/2023', 'mm/dd/yyyy') as date_day """ models_people_sql = """ select 1 as id, 'Drew' as first_name, 'Banin' as last_name, 'yellow' as favorite_color, true as loves_dbt, 5 as tenure, current_timestamp as created_at union all select 2 as id, 'Jeremy' as first_name, 'Cohen' as last_name, 'indigo' as favorite_color, true as loves_dbt, 4 as tenure, current_timestamp as created_at union all select 3 as id, 'Callum' as first_name, 'McCann' as last_name, 'emerald' as favorite_color, true as loves_dbt, 0 as tenure, current_timestamp as created_at """ groups_yml = """ version: 2 groups: - name: some_group owner: email: me@gmail.com - name: some_other_group owner: email: me@gmail.com """ models_people_metrics_yml = """ version: 2 metrics: - name: number_of_people label: "Number of people" description: Total count of people type: simple type_params: measure: people meta: my_meta: 'testing' """ disabled_models_people_metrics_yml = """ version: 2 metrics: - name: number_of_people config: enabled: false group: some_group label: "Number of people" description: Total count of people type: simple type_params: measure: people meta: my_meta: 'testing' """ semantic_model_people_yml = """ version: 2 semantic_models: - name: semantic_people label: "Semantic People" model: ref('people') dimensions: - name: favorite_color label: "Favorite Color" type: categorical - name: created_at label: "Created At" type: TIME type_params: time_granularity: day measures: - name: years_tenure label: "Years Tenure" agg: SUM expr: tenure - name: people label: "People" agg: count expr: id entities: - name: id label: "Primary ID" type: primary defaults: agg_time_dimension: created_at """ semantic_model_people_diff_name_yml = """ version: 2 semantic_models: - name: semantic_people_diff_name label: "Semantic People" model: ref('people') dimensions: - name: favorite_color label: "Favorite Color" type: categorical - name: created_at label: "Created At" type: TIME type_params: time_granularity: day measures: - name: years_tenure label: "Years Tenure" agg: SUM expr: tenure - name: people label: "People" agg: count expr: id entities: - name: id label: "Primary ID" type: primary defaults: agg_time_dimension: created_at """ semantic_model_descriptions = """ {% docs semantic_model_description %} foo {% enddocs %} {% docs dimension_description %} bar {% enddocs %} {% docs measure_description %} baz {% enddocs %} {% docs entity_description %} qux {% enddocs %} {% docs simple_metric_description %} describe away! {% enddocs %} """ semantic_model_people_yml_with_docs = """ version: 2 semantic_models: - name: semantic_people model: ref('people') description: "{{ doc('semantic_model_description') }}" dimensions: - name: favorite_color type: categorical description: "{{ doc('dimension_description') }}" - name: created_at type: TIME type_params: time_granularity: day measures: - name: years_tenure agg: SUM expr: tenure description: "{{ doc('measure_description') }}" - name: people agg: count expr: id entities: - name: id description: "{{ doc('entity_description') }}" type: primary defaults: agg_time_dimension: created_at """ enabled_semantic_model_people_yml = """ version: 2 semantic_models: - name: semantic_people label: "Semantic People" model: ref('people') config: enabled: true group: some_group meta: my_meta: 'testing' my_other_meta: 'testing more' dimensions: - name: favorite_color type: categorical - name: created_at type: TIME type_params: time_granularity: day measures: - name: years_tenure agg: SUM expr: tenure - name: people agg: count expr: id entities: - name: id type: primary defaults: agg_time_dimension: created_at """ disabled_semantic_model_people_yml = """ version: 2 semantic_models: - name: semantic_people label: "Semantic People" model: ref('people') config: enabled: false dimensions: - name: favorite_color type: categorical - name: created_at type: TIME type_params: time_granularity: day measures: - name: years_tenure agg: SUM expr: tenure - name: people agg: count expr: id entities: - name: id type: primary defaults: agg_time_dimension: created_at """ semantic_model_with_disabled_ref_yml = """ version: 2 models: - name: people config: enabled: false semantic_models: - name: semantic_people label: "Semantic People" model: ref('people') dimensions: - name: favorite_color type: categorical - name: created_at type: TIME type_params: time_granularity: day measures: - name: people label: "People" agg: count expr: id entities: - name: id type: primary defaults: agg_time_dimension: created_at """ base_schema_yml = """models: - name: fct_revenue description: This is the model fct_revenue. It should be able to use doc blocks semantic_models: - name: revenue description: This is the revenue semantic model. It should be able to use doc blocks model: ref('fct_revenue') defaults: agg_time_dimension: ds measures: - name: txn_revenue expr: revenue agg: sum agg_time_dimension: ds create_metric: true - name: txn_revenue_with_label label: "Transaction Revenue with label" expr: revenue agg: sum agg_time_dimension: ds create_metric: true - name: sum_of_things expr: 2 agg: sum agg_time_dimension: ds - name: count_of_things agg: count expr: 1 agg_time_dimension: ds - name: count_of_things_2 agg: count expr: 1 agg_time_dimension: ds - name: has_revenue expr: true agg: sum_boolean agg_time_dimension: ds - name: discrete_order_value_p99 expr: order_total agg: percentile agg_time_dimension: ds agg_params: percentile: 0.99 use_discrete_percentile: True use_approximate_percentile: False - name: test_agg_params_optional_are_empty expr: order_total agg: percentile agg_time_dimension: ds agg_params: percentile: 0.99 - name: test_non_additive expr: txn_revenue agg: sum non_additive_dimension: name: ds window_choice: max dimensions: - name: ds type: time expr: created_at type_params: time_granularity: day entities: - name: user type: foreign expr: user_id - name: id type: primary metrics: - name: simple_metric label: Simple Metric type: simple type_params: measure: sum_of_things - name: test_cumulative_metric label: Cumulative Metric type: cumulative type_params: measure: sum_of_things cumulative_type_params: grain_to_date: day period_agg: first """ conversion_metric_yml = """ - name: test_conversion_metric label: Conversion Metric type: conversion type_params: conversion_type_params: base_measure: count_of_things conversion_measure: count_of_things_2 entity: user calculation: conversion_rate """ ratio_metric_yml = """ - name: test_ratio_metric label: Ratio Metric type: ratio type_params: numerator: simple_metric denominator: test_conversion_metric """ derived_metric_yml = """ - name: test_derived_metric label: Derived Metric type: derived type_params: metrics: - simple_metric - test_conversion_metric expr: simple_metric + 1 """ schema_yml = base_schema_yml + conversion_metric_yml + ratio_metric_yml + derived_metric_yml schema_without_semantic_model_yml = """models: - name: fct_revenue description: This is the model fct_revenue. It should be able to use doc blocks """ fct_revenue_sql = """select 1 as id, 10 as user_id, 1000 as revenue, current_timestamp as created_at""" metricflow_time_spine_sql = """ with days as ( {{dbt_utils.date_spine('day' , "to_date('01/01/2000','mm/dd/yyyy')" , "to_date('01/01/2027','mm/dd/yyyy')" ) }} ), final as ( select cast(date_day as date) as date_day from days ) select * from final """ multi_sm_schema_yml = """ models: - name: fct_revenue description: This is the model fct_revenue. semantic_models: - name: revenue description: This is the first semantic model. model: ref('fct_revenue') defaults: agg_time_dimension: ds measures: - name: txn_revenue expr: revenue agg: sum agg_time_dimension: ds create_metric: true - name: sum_of_things expr: 2 agg: sum agg_time_dimension: ds dimensions: - name: ds type: time expr: created_at type_params: time_granularity: day entities: - name: user type: foreign expr: user_id - name: id type: primary - name: alt_revenue description: This is the second revenue semantic model. model: ref('fct_revenue') defaults: agg_time_dimension: ads measures: - name: alt_txn_revenue expr: revenue agg: sum agg_time_dimension: ads create_metric: true - name: alt_sum_of_things expr: 2 agg: sum agg_time_dimension: ads dimensions: - name: ads type: time expr: created_at type_params: time_granularity: day entities: - name: user type: foreign expr: user_id - name: id type: primary metrics: - name: simple_metric label: Simple Metric type: simple type_params: measure: sum_of_things """ semantic_model_dimensions_entities_measures_meta_config = """ version: 2 semantic_models: - name: semantic_people label: "Semantic People" model: ref('people') dimensions: - name: favorite_color label: "Favorite Color" type: categorical config: meta: dimension: one - name: created_at label: "Created At" type: TIME type_params: time_granularity: day measures: - name: years_tenure label: "Years Tenure" agg: SUM expr: tenure config: meta: measure: two - name: people label: "People" agg: count expr: id entities: - name: id label: "Primary ID" type: primary config: meta: entity: three defaults: agg_time_dimension: created_at """ semantic_model_meta_clobbering_yml = """ version: 2 semantic_models: - name: semantic_people label: "Semantic People" model: ref('people') config: meta: model_level: "should_be_inherited" component_level: "should_be_overridden" dimensions: - name: favorite_color label: "Favorite Color" type: categorical config: meta: component_level: "dimension_override" - name: created_at label: "Created At" type: TIME type_params: time_granularity: day measures: - name: years_tenure label: "Years Tenure" agg: SUM expr: tenure config: meta: component_level: "measure_override" - name: people label: "People" agg: count expr: id entities: - name: id label: "Primary ID" type: primary config: meta: component_level: "entity_override" defaults: agg_time_dimension: created_at """ semantic_model_schema_yml_v2_template_for_model_configs = """models: - name: fct_revenue description: This is the model fct_revenue. It should be able to use doc blocks {semantic_model_value} columns: - name: id description: This is the id column dim. config: meta: component_level: "original_meta" dimension: name: id_dim label: "ID Dimension" type: categorical is_partition: true config: meta: component_level: "dimension_override" entity: name: id_entity description: This is the id entity, and it is the primary entity. label: ID Entity type: primary config: meta: component_level: "entity_override" - name: second_col description: This is the second column. granularity: day dimension: name: second_dim description: This is the second column (dim). label: Second Dimension type: time validity_params: is_start: true is_end: true - name: foreign_id_col description: This is a foreign id column. entity: foreign - name: col_with_default_dimensions description: This is the column with default dimension settings. dimension: categorical entity: name: col_with_default_entity_testing_default_desc type: natural """ # You can replace the semantic_model variable in the template like this: semantic_model_schema_yml_v2 = semantic_model_schema_yml_v2_template_for_model_configs.format( semantic_model_value="semantic_model: true", ) semantic_model_schema_yml_v2_disabled = ( semantic_model_schema_yml_v2_template_for_model_configs.format( semantic_model_value="semantic_model: false", ) ) semantic_model_test_groups_yml = """groups: - name: finance owner: # 'name' or 'email' is required; additional properties will no longer be allowed in a future release email: finance@jaffleshop.com config: meta: # optional data_owner: Finance team """ semantic_model_schema_yml_v2_renamed = semantic_model_schema_yml_v2_template_for_model_configs.format( semantic_model_value="""semantic_model: name: renamed_semantic_model enabled: true group: finance config: meta: meta_tag_1: this_meta """, ) semantic_model_schema_yml_v2_default_values = semantic_model_schema_yml_v2_template_for_model_configs.format( semantic_model_value="""semantic_model: enabled: true """ ) semantic_model_schema_yml_v2_disabled = semantic_model_schema_yml_v2_template_for_model_configs.format( semantic_model_value="""semantic_model: enabled: false """, ) semantic_model_schema_yml_v2_false_config = ( semantic_model_schema_yml_v2_template_for_model_configs.format( semantic_model_value="semantic_model: false", ) ) semantic_model_config_does_not_exist = ( semantic_model_schema_yml_v2_template_for_model_configs.format( semantic_model_value="", ) ) semantic_model_schema_yml_v2_template_for_primary_entity_tests = """models: - name: fct_revenue description: This is the model fct_revenue. It should be able to use doc blocks semantic_model: true {primary_entity_setting} columns: - name: id description: This is the id column dim. config: meta: component_level: "original_meta" dimension: type: categorical entity: name: id_entity description: This is the id entity, and it is the primary entity. label: ID Entity type: {id_entity_type} - name: second_col description: This is the second column. granularity: day dimension: name: second_dim description: This is the second column (dim). label: Second Dimension type: time validity_params: is_start: true is_end: true - name: other_id_col description: This is the other id column. entity: name: other_id_entity type: {other_id_entity_type} - name: col_with_default_dimensions description: This is the column with default dimension settings. dimension: categorical entity: name: col_with_default_entity_testing_default_desc type: natural """ semantic_model_schema_yml_v2_with_primary_entity_only_on_column = ( semantic_model_schema_yml_v2_template_for_primary_entity_tests.format( primary_entity_setting="", id_entity_type="primary", other_id_entity_type="foreign", ) ) semantic_model_schema_yml_v2_primary_entity_only_on_model = ( semantic_model_schema_yml_v2_template_for_primary_entity_tests.format( primary_entity_setting="primary_entity: id_entity", id_entity_type="foreign", other_id_entity_type="foreign", ) ) semantic_model_schema_yml_v2 = """models: - name: fct_revenue description: This is the model fct_revenue. It should be able to use doc blocks semantic_model: true columns: - name: id description: This is the id column dim. config: meta: component_level: "original_meta" dimension: name: id_dim label: "ID Dimension" type: categorical is_partition: true config: meta: component_level: "dimension_override" entity: name: id_entity description: This is the id entity, and it is the primary entity. label: ID Entity type: primary config: meta: component_level: "entity_override" - name: second_col description: This is the second column. granularity: day dimension: name: second_dim description: This is the second column (dim). label: Second Dimension type: time validity_params: is_start: true is_end: true - name: foreign_id_col description: This is a foreign id column. entity: foreign - name: col_with_default_dimensions description: This is the column with default dimension settings. dimension: categorical entity: name: col_with_default_entity_testing_default_desc type: natural """ # Separate from the full-spectrum entities and dimensions test because some settings # interact with metric validations. base_schema_yml_v2 = """models: - name: fct_revenue description: This is the model fct_revenue. It should be able to use doc blocks semantic_model: true agg_time_dimension: second_dim columns: - name: id description: This is the id column dim. config: meta: component_level: "original_meta" dimension: name: id_dim label: "ID Dimension" type: categorical is_partition: true config: meta: component_level: "dimension_override" entity: name: id_entity description: This is the id entity, and it is the primary entity. label: ID Entity type: primary config: meta: component_level: "entity_override" - name: second_col description: This is the second column. granularity: day dimension: name: second_dim description: This is the second column (dim). label: Second Dimension type: time - name: foreign_id_col description: This is a foreign id column. entity: foreign - name: created_at description: This is the time the entry was created. granularity: day dimension: name: ds description: the ds column label: DS Column type: time """ schema_yml_v2_simple_metric_on_model_1 = """ metrics: - name: simple_metric description: This is our first simple metric. label: Simple Metric type: simple agg: count expr: id - name: simple_metric_2 description: This is our second simple metric. agg_time_dimension: ds label: Simple Metric 2 type: simple agg: count expr: second_col - name: percentile_metric description: This is our percentile metric. label: Percentile Metric type: simple agg: percentile percentile: 0.99 percentile_type: discrete expr: second_col - name: cumulative_metric description: This is our cumulative metric. label: Cumulative Metric type: cumulative grain_to_date: day period_agg: first input_metric: simple_metric - name: conversion_metric description: This is our conversion metric. label: Conversion Metric type: conversion entity: id_entity calculation: conversion_rate base_metric: simple_metric conversion_metric: simple_metric_2 """ schema_yml_v2_metrics_with_hidden = """ metrics: - name: public_metric description: A metric that is not hidden. label: Public Metric type: simple agg: count expr: id hidden: false - name: private_metric description: A metric that is hidden. label: Private Metric type: simple agg: count expr: id hidden: true """ schema_yml_v2_metric_with_doc_jinja = """ - name: simple_metric_with_doc_jinja description: "{{ doc('simple_metric_description') }}" label: Simple Metric With Doc Jinja type: simple agg: count expr: id """ schema_yml_v2_metric_with_filter_dimension_jinja = """ - name: simple_metric_with_filter_dimension_jinja description: This is a description label: Simple Metric With Doc Jinja type: simple agg: count expr: id filter: | {{ Dimension('id_entity__id_dim') }} > 0 and {{ TimeDimension('id_entity__id_dim', 'day') }} > '2020-01-01' """ schema_yml_v2_metric_with_input_metrics_filter_dimension_jinja = """ metrics: - name: simple_metric description: This is our first simple metric. label: Simple Metric type: simple agg: count expr: id - name: derived_metric_with_jinja_filter description: This is a derived metric with a jinja filter on an input metric. label: Derived Metric With Jinja Filter type: derived expr: simple_metric - offset_metric input_metrics: - name: simple_metric - name: simple_metric alias: offset_metric filter: | {{ Dimension('id_entity__id_dim') }} > 0 offset_window: 1 week """ schema_yml_v2_metric_with_numerator_filter_dimension_jinja = """ metrics: - name: simple_metric description: This is our first simple metric. label: Simple Metric type: simple agg: count expr: id - name: simple_metric_2 description: This is our second simple metric. label: Simple Metric 2 type: simple agg: count expr: second_col - name: ratio_metric_with_jinja_filter description: This is a ratio metric with a jinja filter on the numerator. label: Ratio Metric With Jinja Filter type: ratio numerator: name: simple_metric filter: | {{ Dimension('id_entity__id_dim') }} > 0 denominator: simple_metric_2 """ schema_yml_v2_cumulative_metric_missing_input_metric = """ metrics: - name: cumulative_metric description: This is our cumulative metric. label: Cumulative Metric type: cumulative grain_to_date: day period_agg: first """ schema_yml_v2_conversion_metric_missing_base_metric = """ metrics: - name: simple_metric_2 description: This is our second simple metric. label: Simple Metric 2 type: simple agg: count expr: second_col - name: conversion_metric description: This is our conversion metric. label: Conversion Metric type: conversion entity: id_entity calculation: conversion_rate conversion_metric: simple_metric_2 """ schema_yml_v2_standalone_simple_metric = textwrap.dedent(schema_yml_v2_simple_metric_on_model_1) schema_yml_v2_standalone_metrics_template = """ metrics: - name: standalone_conversion_metric description: {description} label: Standalone Conversion Metric type: conversion entity: id_entity calculation: conversion_rate base_metric: simple_metric conversion_metric: simple_metric_2 {filter} """ schema_yml_v2_standalone_metrics = schema_yml_v2_standalone_metrics_template.format( description="This is our standalone conversion metric.", filter="filter: id > 0", ) schema_yml_v2_standalone_metrics_with_doc_jinja = schema_yml_v2_standalone_metrics_template.format( description="\"{{ doc('simple_metric_description') }}\"", filter="""filter: | {{ Dimension('id_entity__id_dim') }} > 0""", ) base_schema_yml_v2_with_custom_sm_name = """models: - name: fct_revenue description: This is the model fct_revenue. It should be able to use doc blocks semantic_model: name: custom_semantic_model agg_time_dimension: second_dim columns: - name: id description: This is the id column dim. config: meta: component_level: "original_meta" dimension: name: id_dim label: "ID Dimension" type: categorical is_partition: true config: meta: component_level: "dimension_override" entity: name: id_entity description: This is the id entity, and it is the primary entity. label: ID Entity type: primary config: meta: component_level: "entity_override" - name: second_col description: This is the second column. granularity: day dimension: name: second_dim description: This is the second column (dim). label: Second Dimension type: time - name: foreign_id_col description: This is a foreign id column. entity: foreign - name: created_at description: This is the time the entry was created. granularity: day dimension: name: ds description: the ds column label: DS Column type: time """ derived_semantics_yml = """ derived_semantics: entities: - name: derived_id_entity description: This is the id entity, and it is the primary entity. label: ID Entity type: foreign expr: "id + foreign_id_col" config: meta: test_label_thing: derived_entity_1 - name: derived_id_entity_with_no_optional_fields type: foreign expr: id + foreign_id_col dimensions: - name: derived_id_dimension type: categorical expr: id granularity: day validity_params: is_start: true is_end: true """ derived_semantics_with_doc_jinja_yml = """ derived_semantics: entities: - name: derived_id_entity description: "{{ doc('entity_description') }}" type: foreign expr: "id + foreign_id_col" dimensions: - name: derived_id_dimension description: "{{ doc('dimension_description') }}" type: categorical expr: id """ ================================================ FILE: tests/functional/semantic_models/test_semantic_model_configs.py ================================================ import pytest from dbt.artifacts.resources import SemanticModelConfig from dbt.exceptions import ParsingError from dbt.tests.util import get_manifest, run_dbt, update_config_file from tests.functional.semantic_models.fixtures import ( disabled_models_people_metrics_yml, disabled_semantic_model_people_yml, enabled_semantic_model_people_yml, groups_yml, metricflow_time_spine_sql, models_people_metrics_yml, models_people_sql, semantic_model_dimensions_entities_measures_meta_config, semantic_model_meta_clobbering_yml, semantic_model_people_yml, ) # Test disabled config at semantic_models level in yaml file class TestConfigYamlLevel: @pytest.fixture(scope="class") def models(self): return { "people.sql": models_people_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "semantic_models.yml": disabled_semantic_model_people_yml, "people_metrics.yml": disabled_models_people_metrics_yml, "groups.yml": groups_yml, } def test_yaml_level(self, project): run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "semantic_model.test.semantic_people" not in manifest.semantic_models assert "semantic_model.test.semantic_people" in manifest.disabled assert "group.test.some_group" in manifest.groups assert "semantic_model.test.semantic_people" not in manifest.groups # Test disabled config at semantic_models level with a still enabled metric class TestDisabledConfigYamlLevelEnabledMetric: @pytest.fixture(scope="class") def models(self): return { "people.sql": models_people_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "semantic_models.yml": disabled_semantic_model_people_yml, "people_metrics.yml": models_people_metrics_yml, "groups.yml": groups_yml, } def test_yaml_level(self, project): with pytest.raises( ParsingError, match="The measure `people` is referenced on disabled semantic model `semantic_people`.", ): run_dbt(["parse"]) # Test disabling semantic model config but not metric config in dbt_project.yml class TestMismatchesConfigProjectLevel: @pytest.fixture(scope="class") def models(self): return { "people.sql": models_people_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "semantic_models.yml": semantic_model_people_yml, "people_metrics.yml": models_people_metrics_yml, "groups.yml": groups_yml, } @pytest.fixture(scope="class") def project_config_update(self): return { "semantic-models": { "test": { "enabled": True, } } } def test_project_level(self, project): run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "semantic_model.test.semantic_people" in manifest.semantic_models assert "group.test.some_group" in manifest.groups assert manifest.semantic_models["semantic_model.test.semantic_people"].group is None new_enabled_config = { "semantic-models": { "test": { "enabled": False, } } } update_config_file(new_enabled_config, project.project_root, "dbt_project.yml") with pytest.raises( ParsingError, match="The measure `people` is referenced on disabled semantic model `semantic_people`.", ): run_dbt(["parse"]) # Test disabling semantic model and metric configs in dbt_project.yml class TestConfigProjectLevel: @pytest.fixture(scope="class") def models(self): return { "people.sql": models_people_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "semantic_models.yml": semantic_model_people_yml, "people_metrics.yml": models_people_metrics_yml, "groups.yml": groups_yml, } @pytest.fixture(scope="class") def project_config_update(self): return { "semantic-models": { "test": { "enabled": True, } }, "metrics": { "test": { "enabled": True, } }, } def test_project_level(self, project): run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "semantic_model.test.semantic_people" in manifest.semantic_models assert "group.test.some_group" in manifest.groups assert "group.test.some_other_group" in manifest.groups assert manifest.semantic_models["semantic_model.test.semantic_people"].group is None new_group_config = { "semantic-models": { "test": { "group": "some_other_group", } }, } update_config_file(new_group_config, project.project_root, "dbt_project.yml") run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "semantic_model.test.semantic_people" in manifest.semantic_models assert "group.test.some_other_group" in manifest.groups assert "group.test.some_group" in manifest.groups assert ( manifest.semantic_models["semantic_model.test.semantic_people"].group == "some_other_group" ) new_enabled_config = { "semantic-models": { "test": { "enabled": False, } }, "metrics": { "test": { "enabled": False, } }, } update_config_file(new_enabled_config, project.project_root, "dbt_project.yml") run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "semantic_model.test.semantic_people" not in manifest.semantic_models assert "semantic_model.test.semantic_people" in manifest.disabled assert "group.test.some_group" in manifest.groups assert "semantic_model.test.semantic_people" not in manifest.groups # Test inheritence - set configs at project and semantic_model level - expect semantic_model level to win class TestConfigsInheritence: @pytest.fixture(scope="class") def models(self): return { "people.sql": models_people_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "semantic_models.yml": enabled_semantic_model_people_yml, "people_metrics.yml": models_people_metrics_yml, "groups.yml": groups_yml, } @pytest.fixture(scope="class") def project_config_update(self): return {"semantic-models": {"enabled": False}} def test_project_plus_yaml_level(self, project): run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "semantic_model.test.semantic_people" in manifest.semantic_models config_test_table = manifest.semantic_models.get( "semantic_model.test.semantic_people" ).config assert isinstance(config_test_table, SemanticModelConfig) # test setting meta attributes in semantic model config class TestMetaConfig: @pytest.fixture(scope="class") def models(self): return { "people.sql": models_people_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "semantic_models.yml": enabled_semantic_model_people_yml, "people_metrics.yml": models_people_metrics_yml, "groups.yml": groups_yml, } def test_meta_config(self, project): run_dbt(["parse"]) manifest = get_manifest(project.project_root) sm_id = "semantic_model.test.semantic_people" assert sm_id in manifest.semantic_models sm_node = manifest.semantic_models[sm_id] meta_expected = {"my_meta": "testing", "my_other_meta": "testing more"} assert sm_node.config.meta == meta_expected # test meta configs on semantic model components (dimensions, measures, entities) class TestMetaConfigForComponents: @pytest.fixture(scope="class") def models(self): return { "people.sql": models_people_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "semantic_models.yml": semantic_model_dimensions_entities_measures_meta_config, "people_metrics.yml": models_people_metrics_yml, "groups.yml": groups_yml, } def test_component_meta_configs(self, project): run_dbt(["parse"]) manifest = get_manifest(project.project_root) sm_id = "semantic_model.test.semantic_people" assert sm_id in manifest.semantic_models sm_node = manifest.semantic_models[sm_id] # Check dimension meta config favorite_color_dim = next(d for d in sm_node.dimensions if d.name == "favorite_color") assert favorite_color_dim.config.meta == {"dimension": "one"} # Check measure meta config years_tenure_measure = next(m for m in sm_node.measures if m.name == "years_tenure") assert years_tenure_measure.config.meta == {"measure": "two"} # Check entity meta config id_entity = next(e for e in sm_node.entities if e.name == "id") assert id_entity.config.meta == {"entity": "three"} # test meta config clobbering behavior between semantic model and component levels class TestMetaConfigClobbering: @pytest.fixture(scope="class") def models(self): return { "people.sql": models_people_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "semantic_models.yml": semantic_model_meta_clobbering_yml, "people_metrics.yml": models_people_metrics_yml, "groups.yml": groups_yml, } def test_meta_config_clobbering(self, project): run_dbt(["parse"]) manifest = get_manifest(project.project_root) sm_id = "semantic_model.test.semantic_people" assert sm_id in manifest.semantic_models sm_node = manifest.semantic_models[sm_id] # Check semantic model level meta config assert sm_node.config.meta == { "model_level": "should_be_inherited", "component_level": "should_be_overridden", } # Check dimension inherits model-level meta and overrides component-level meta favorite_color_dim = next(d for d in sm_node.dimensions if d.name == "favorite_color") assert favorite_color_dim.config.meta == { "model_level": "should_be_inherited", "component_level": "dimension_override", } # Check measure inherits model-level meta and overrides component-level meta years_tenure_measure = next(m for m in sm_node.measures if m.name == "years_tenure") assert years_tenure_measure.config.meta == { "model_level": "should_be_inherited", "component_level": "measure_override", } # Check entity inherits model-level meta and overrides component-level meta id_entity = next(e for e in sm_node.entities if e.name == "id") assert id_entity.config.meta == { "model_level": "should_be_inherited", "component_level": "entity_override", } # Check component without meta config still inherits model-level meta created_at_dim = next(d for d in sm_node.dimensions if d.name == "created_at") assert created_at_dim.config.meta == { "model_level": "should_be_inherited", "component_level": "should_be_overridden", } ================================================ FILE: tests/functional/semantic_models/test_semantic_model_parsing.py ================================================ from typing import List import pytest from dbt.artifacts.resources.v1.semantic_model import MetricType from dbt.contracts.graph.manifest import Manifest from dbt.exceptions import CompilationError from dbt.tests.util import run_dbt, write_file from dbt_common.events.base_types import BaseEvent from dbt_semantic_interfaces.type_enums.conversion_calculation_type import ( ConversionCalculationType, ) from dbt_semantic_interfaces.type_enums.period_agg import PeriodAggregation from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity from tests.functional.assertions.test_runner import dbtTestRunner from tests.functional.semantic_models.fixtures import ( base_schema_yml, conversion_metric_yml, fct_revenue_sql, metricflow_time_spine_sql, models_people_sql, multi_sm_schema_yml, ratio_metric_yml, schema_without_semantic_model_yml, schema_yml, semantic_model_with_disabled_ref_yml, ) class TestSemanticModelParsingWorks: @pytest.fixture(scope="class") def models(self): return { "schema.yml": schema_yml, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, } def test_semantic_model_parsing(self, project): runner = dbtTestRunner() result = runner.invoke(["parse"]) assert result.success assert isinstance(result.result, Manifest) manifest = result.result assert len(manifest.semantic_models) == 1 semantic_model = manifest.semantic_models["semantic_model.test.revenue"] assert semantic_model.node_relation.alias == "fct_revenue" assert ( semantic_model.node_relation.relation_name == f'"dbt"."{project.test_schema}"."fct_revenue"' ) assert len(semantic_model.measures) == 9 # manifest should have two metrics created from measures assert len(manifest.metrics) == 7 metric = manifest.metrics["metric.test.txn_revenue"] assert metric.name == "txn_revenue" metric_with_label = manifest.metrics["metric.test.txn_revenue_with_label"] assert metric_with_label.name == "txn_revenue_with_label" assert metric_with_label.label == "Transaction Revenue with label" class TestSemanticModelParsingErrors: @pytest.fixture(scope="class") def models(self): return { "schema.yml": schema_yml, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, } def test_semantic_model_error(self, project): # Next, modify the default schema.yml to remove the semantic model. error_schema_yml = schema_yml.replace("sum_of_things", "has_revenue") write_file(error_schema_yml, project.project_root, "models", "schema.yml") events: List[BaseEvent] = [] runner = dbtTestRunner(callbacks=[events.append]) result = runner.invoke(["parse"]) assert not result.success validation_errors = [e for e in events if e.info.name == "SemanticValidationFailure"] assert validation_errors class TestSemanticModelWithDisabledRef: @pytest.fixture(scope="class") def models(self): return { "schema.yml": semantic_model_with_disabled_ref_yml, "people.sql": models_people_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, } def test_compilation_error_on_disabled_ref(self, project): with pytest.raises(CompilationError) as excinfo: run_dbt(["parse"]) assert "depends on a node named 'people' which is disabled" in str(excinfo.value) class TestSemanticModelParsingForCumulativeMetrics: @pytest.fixture(scope="class") def models(self): return { "schema.yml": schema_yml, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, } def test_cumulative_metric_parsing(self, project): runner = dbtTestRunner() result = runner.invoke(["parse"]) assert result.success manifest = result.result assert "metric.test.test_cumulative_metric" in manifest.metrics metric = manifest.metrics["metric.test.test_cumulative_metric"] assert metric.name == "test_cumulative_metric" # Check type and params for cumulative metric assert metric.type is MetricType.CUMULATIVE cumulative_params = metric.type_params.cumulative_type_params assert cumulative_params is not None assert cumulative_params.grain_to_date == "day" assert cumulative_params.period_agg is PeriodAggregation.FIRST assert len(metric.type_params.input_measures) == 1 assert "sum_of_things" in [im.name for im in metric.type_params.input_measures] # Not sure where dbt uses this, but for now, let's just make sure we know # we're keeping the contract that these metrics are marked as depending on # the semantic model. assert "semantic_model.test.revenue" in metric.depends_on.nodes class TestSemanticModelParsingForConversionMetrics: @pytest.fixture(scope="class") def models(self): return { "schema.yml": schema_yml, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, } def test_conversion_metric_parsing(self, project): runner = dbtTestRunner() result = runner.invoke(["parse"]) assert result.success manifest = result.result assert "metric.test.test_conversion_metric" in manifest.metrics metric = manifest.metrics["metric.test.test_conversion_metric"] assert metric.name == "test_conversion_metric" # Check type and params for conversion metric assert metric.type is MetricType.CONVERSION conversion_params = metric.type_params.conversion_type_params assert conversion_params is not None # Confirm measures, entity, and calculation are correct assert conversion_params.base_measure.name == "count_of_things" assert conversion_params.conversion_measure.name == "count_of_things_2" assert conversion_params.entity == "user" assert conversion_params.calculation == ConversionCalculationType.CONVERSION_RATE assert len(metric.type_params.input_measures) == 2 assert "count_of_things" in [im.name for im in metric.type_params.input_measures] assert "count_of_things_2" in [im.name for im in metric.type_params.input_measures] # Not sure where dbt uses this, but for now, let's just make sure # we're keeping the contract that these metrics are marked as depending on # the semantic model. assert "semantic_model.test.revenue" in metric.depends_on.nodes class TestSemanticModelParsingForRatioMetrics: @pytest.fixture(scope="class") def models(self): return { "schema.yml": schema_yml, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, } def test_ratio_metric_parsing(self, project): runner = dbtTestRunner() result = runner.invoke(["parse"]) assert result.success manifest = result.result assert "metric.test.test_ratio_metric" in manifest.metrics metric = manifest.metrics["metric.test.test_ratio_metric"] assert metric.name == "test_ratio_metric" assert metric.type is MetricType.RATIO type_params = metric.type_params assert type_params.numerator.name == "simple_metric" assert type_params.denominator.name == "test_conversion_metric" assert len(metric.type_params.input_measures) == 3 assert "sum_of_things" in [im.name for im in metric.type_params.input_measures] assert "count_of_things" in [im.name for im in metric.type_params.input_measures] assert "count_of_things_2" in [im.name for im in metric.type_params.input_measures] # I don't know for sure why, but it seems like we've never marked the # 'depends_on' semantic model for ratio metrics, so let's document that here # as a test. These are not used in metricflow. assert len(metric.depends_on.nodes) == 2 assert "metric.test.simple_metric" in metric.depends_on.nodes assert "metric.test.test_conversion_metric" in metric.depends_on.nodes class TestSemanticModelParsingForDerivedMetrics: @pytest.fixture(scope="class") def models(self): return { "schema.yml": schema_yml, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, } def test_derived_metric_parsing(self, project): runner = dbtTestRunner() result = runner.invoke(["parse"]) assert result.success manifest = result.result assert "metric.test.test_derived_metric" in manifest.metrics metric = manifest.metrics["metric.test.test_derived_metric"] assert metric.name == "test_derived_metric" assert metric.type is MetricType.DERIVED assert len(metric.type_params.input_measures) == 3 assert "sum_of_things" in [im.name for im in metric.type_params.input_measures] assert "count_of_things" in [im.name for im in metric.type_params.input_measures] assert "count_of_things_2" in [im.name for im in metric.type_params.input_measures] assert len(metric.type_params.metrics) == 2 assert "simple_metric" in [m.name for m in metric.type_params.metrics] assert "test_conversion_metric" in [m.name for m in metric.type_params.metrics] # I don't know for sure why, but it seems like we've never marked the # 'depends_on' semantic model for derived metrics, so let's document that here # as a test. These are not used in metricflow. assert len(metric.depends_on.nodes) == 2 assert "metric.test.simple_metric" in metric.depends_on.nodes assert "metric.test.test_conversion_metric" in metric.depends_on.nodes # ------------------------------------------------------------------------------ # Partial Parsing tests below # ------------------------------------------------------------------------------ class TestSemanticModelPartialParsingWithModelChanged: @pytest.fixture(scope="class") def models(self): return { "schema.yml": schema_yml, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, } def test_semantic_model_changed_partial_parsing(self, project): # First, use the default schema.yml to define our semantic model, and # run the dbt parse command runner = dbtTestRunner() result = runner.invoke(["parse"]) assert result.success # Next, modify the default schema.yml to change a detail of the semantic # model. modified_schema_yml = schema_yml.replace("time_granularity: day", "time_granularity: week") write_file(modified_schema_yml, project.project_root, "models", "schema.yml") # Now, run the dbt parse command again. result = runner.invoke(["parse"]) assert result.success # Finally, verify that the manifest reflects the partially parsed change manifest = result.result semantic_model = manifest.semantic_models["semantic_model.test.revenue"] assert semantic_model.dimensions[0].type_params.time_granularity == TimeGranularity.WEEK # TODO DI-4421 (Linear) / SEMANTIC-2997 (Jira): fix the partial parsing bug that breaks this. # class TestSemanticModelPartialParsingWithModelDeleted: # @pytest.fixture(scope="class") # def models(self): # return { # "schema.yml": schema_yml, # "fct_revenue.sql": fct_revenue_sql, # "metricflow_time_spine.sql": metricflow_time_spine_sql, # } # def test_semantic_model_deleted_partial_parsing__dependency_bug_causes_failure(self, project): # # First, use the default schema.yml to define our semantic model, and # # run the dbt parse command # runner = dbtTestRunner() # result = runner.invoke(["parse"]) # assert result.success # assert "semantic_model.test.revenue" in result.result.semantic_models # # Next, modify the default schema.yml to remove the semantic model. # write_file(schema_without_semantic_model_yml, project.project_root, "models", "schema.yml") # # Now, run the dbt parse command again. # result = runner.invoke(["parse"]) # # Known bug: we don't remove metrics in any particular order, so we'll remove # # simple_metric before the metrics that rely on it and have problems... but only SOMETIMES. # assert result.success class TestSemanticModelPartialParsingWithModelDeletedIteratively: @pytest.fixture(scope="class") def models(self): return { "schema.yml": base_schema_yml + conversion_metric_yml + ratio_metric_yml, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, } def test_semantic_model_deleted_partial_parsing(self, project): # First, use the default schema.yml to define our semantic model, and # run the dbt parse command runner = dbtTestRunner() result = runner.invoke(["parse"]) assert result.success assert "semantic_model.test.revenue" in result.result.semantic_models # Next, modify the default schema.yml to remove the semantic model ITERATIVELY # (removing it all at once doesn't work because of a bug right now. # see test_semantic_model_deleted_partial_parsing__dependency_bug_causes_failure.) write_file( base_schema_yml + conversion_metric_yml, project.project_root, "models", "schema.yml", ) # Now, run the dbt parse command again. result = runner.invoke(["parse"]) assert result.success write_file(base_schema_yml, project.project_root, "models", "schema.yml") result = runner.invoke(["parse"]) assert result.success write_file(schema_without_semantic_model_yml, project.project_root, "models", "schema.yml") result = runner.invoke(["parse"]) assert result.success # Finally, verify that the manifest reflects the deletion assert "semantic_model.test.revenue" not in result.result.semantic_models class TestSemanticModelPartialParsingWithModelFlippingCreateMetric: @pytest.fixture(scope="class") def models(self): return { "schema.yml": schema_yml, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, } def test_semantic_model_flipping_create_metric_partial_parsing(self, project): generated_metric = "metric.test.txn_revenue" generated_metric_with_label = "metric.test.txn_revenue_with_label" # First, use the default schema.yml to define our semantic model, and # run the dbt parse command runner = dbtTestRunner() result = runner.invoke(["parse"]) assert result.success # Verify the metric created by `create_metric: true` exists metric = result.result.metrics[generated_metric] assert metric.name == "txn_revenue" assert metric.label == "txn_revenue" metric_with_label = result.result.metrics[generated_metric_with_label] assert metric_with_label.name == "txn_revenue_with_label" assert metric_with_label.label == "Transaction Revenue with label" # --- Next, modify the default schema.yml to have no `create_metric: true` --- no_create_metric_schema_yml = schema_yml.replace( "create_metric: true", "create_metric: false" ) write_file(no_create_metric_schema_yml, project.project_root, "models", "schema.yml") # Now, run the dbt parse command again. result = runner.invoke(["parse"]) assert result.success # Verify the metric originally created by `create_metric: true` was removed assert result.result.metrics.get(generated_metric) is None # Verify that partial parsing didn't clobber the normal metric assert result.result.metrics.get("metric.test.simple_metric") is not None # --- Now bring it back --- create_metric_schema_yml = schema_yml.replace( "create_metric: false", "create_metric: true" ) write_file(create_metric_schema_yml, project.project_root, "models", "schema.yml") # Now, run the dbt parse command again. result = runner.invoke(["parse"]) assert result.success # Verify the metric originally created by `create_metric: true` was removed metric = result.result.metrics[generated_metric] assert metric.name == "txn_revenue" class TestSemanticModelPartialParsingGeneratedMetrics: @pytest.fixture(scope="class") def models(self): return { "schema.yml": multi_sm_schema_yml, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, } def test_generated_metrics(self, project): manifest = run_dbt(["parse"]) expected = { "metric.test.simple_metric", "metric.test.txn_revenue", "metric.test.alt_txn_revenue", } assert set(manifest.metrics.keys()) == expected # change description of 'revenue' semantic model modified_schema_yml = multi_sm_schema_yml.replace("first", "FIRST") write_file(modified_schema_yml, project.project_root, "models", "schema.yml") manifest = run_dbt(["parse"]) assert set(manifest.metrics.keys()) == expected ================================================ FILE: tests/functional/semantic_models/test_semantic_model_v2_parsing.py ================================================ import pytest from core.dbt.contracts.graph.semantic_manifest import SemanticManifest from dbt.contracts.graph.manifest import Manifest from dbt_semantic_interfaces.type_enums import ( AggregationType, ConversionCalculationType, DimensionType, EntityType, MetricType, PeriodAggregation, ) from tests.functional.assertions.test_runner import dbtTestRunner from tests.functional.semantic_models.fixtures import ( base_schema_yml_v2, base_schema_yml_v2_with_custom_sm_name, derived_semantics_with_doc_jinja_yml, derived_semantics_yml, fct_revenue_sql, metricflow_time_spine_sql, schema_yml_v2_conversion_metric_missing_base_metric, schema_yml_v2_cumulative_metric_missing_input_metric, schema_yml_v2_metric_with_doc_jinja, schema_yml_v2_metric_with_filter_dimension_jinja, schema_yml_v2_metric_with_input_metrics_filter_dimension_jinja, schema_yml_v2_metric_with_numerator_filter_dimension_jinja, schema_yml_v2_metrics_with_hidden, schema_yml_v2_simple_metric_on_model_1, schema_yml_v2_standalone_metrics, schema_yml_v2_standalone_metrics_with_doc_jinja, schema_yml_v2_standalone_simple_metric, semantic_model_config_does_not_exist, semantic_model_descriptions, semantic_model_schema_yml_v2, semantic_model_schema_yml_v2_default_values, semantic_model_schema_yml_v2_disabled, semantic_model_schema_yml_v2_false_config, semantic_model_schema_yml_v2_primary_entity_only_on_model, semantic_model_schema_yml_v2_renamed, semantic_model_schema_yml_v2_with_primary_entity_only_on_column, semantic_model_test_groups_yml, ) class TestSemanticModelParsingWorks: @pytest.fixture(scope="class") def models(self): return { "schema.yml": semantic_model_schema_yml_v2, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, } def test_semantic_model_parsing(self, project) -> None: runner = dbtTestRunner() result = runner.invoke(["parse"]) assert result.success assert isinstance(result.result, Manifest) manifest = result.result assert len(manifest.semantic_models) == 1 semantic_model = manifest.semantic_models["semantic_model.test.fct_revenue"] assert semantic_model.name == "fct_revenue" assert semantic_model.node_relation.alias == "fct_revenue" assert ( semantic_model.node_relation.relation_name == f'"dbt"."{project.test_schema}"."fct_revenue"' ) assert ( semantic_model.description == "This is the model fct_revenue. It should be able to use doc blocks" ) assert semantic_model.config.enabled is True assert semantic_model.config.group is None assert semantic_model.config.meta == {} # Dimensions assert len(semantic_model.dimensions) == 3 dimensions = {dimension.name: dimension for dimension in semantic_model.dimensions} id_dim = dimensions["id_dim"] assert id_dim.type == DimensionType.CATEGORICAL assert id_dim.description == "This is the id column dim." assert id_dim.label == "ID Dimension" assert id_dim.is_partition is True assert id_dim.config.meta == {"component_level": "dimension_override"} # dimension name "id_dim" differs from column name "id", so expr must # be set to the column name for MetricFlow to query the correct column. assert id_dim.expr == "id" second_dim = dimensions["second_dim"] assert second_dim.type == DimensionType.TIME assert second_dim.description == "This is the second column (dim)." assert second_dim.label == "Second Dimension" assert second_dim.is_partition is False assert second_dim.config.meta == {} assert second_dim.type_params.validity_params.is_start is True assert second_dim.type_params.validity_params.is_end is True # dimension name "second_dim" differs from column name "second_col", # so expr must be set to the column name. assert second_dim.expr == "second_col" col_with_default_dimensions = dimensions["col_with_default_dimensions"] assert col_with_default_dimensions.type == DimensionType.CATEGORICAL assert ( col_with_default_dimensions.description == "This is the column with default dimension settings." ) assert col_with_default_dimensions.label is None assert col_with_default_dimensions.is_partition is False assert col_with_default_dimensions.config.meta == {} assert col_with_default_dimensions.validity_params is None # dimension name matches column name, so expr should not be set. assert col_with_default_dimensions.expr is None # Entities assert len(semantic_model.entities) == 3 entities = {entity.name: entity for entity in semantic_model.entities} primary_entity = entities["id_entity"] assert primary_entity.type == EntityType.PRIMARY assert primary_entity.description == "This is the id entity, and it is the primary entity." assert primary_entity.label == "ID Entity" assert primary_entity.config.meta == {"component_level": "entity_override"} # entity name "id_entity" differs from column name "id", so expr must # be set to the column name for MetricFlow to query the correct column. assert primary_entity.expr == "id" foreign_id_col = entities["foreign_id_col"] assert foreign_id_col.type == EntityType.FOREIGN assert foreign_id_col.description == "This is a foreign id column." assert foreign_id_col.label is None assert foreign_id_col.config.meta == {} # entity name matches column name, so expr should not be set. assert foreign_id_col.expr is None col_with_default_entity_testing_default_desc = entities[ "col_with_default_entity_testing_default_desc" ] assert col_with_default_entity_testing_default_desc.type == EntityType.NATURAL assert ( col_with_default_entity_testing_default_desc.description == "This is the column with default dimension settings." ) assert col_with_default_entity_testing_default_desc.label is None assert col_with_default_entity_testing_default_desc.config.meta == {} # entity name differs from column name, so expr must be set. assert col_with_default_entity_testing_default_desc.expr == "col_with_default_dimensions" # No measures in v2 YAML assert len(semantic_model.measures) == 0 assert len(manifest.metrics) == 0 # TODO: Dimensions are not parsed yet (for those attached to model columns) # TODO: Dimensions are not parsed yet (for those defined in derived semantics) class TestSemanticModelConfigOverrides: @pytest.fixture(scope="class") def models(self): return { "schema.yml": semantic_model_test_groups_yml + semantic_model_schema_yml_v2_renamed, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, } def test_semantic_model_parsing(self, project) -> None: runner = dbtTestRunner() result = runner.invoke(["parse"]) assert result.success assert isinstance(result.result, Manifest) manifest = result.result assert len(manifest.semantic_models) == 1 semantic_model = manifest.semantic_models["semantic_model.test.renamed_semantic_model"] assert semantic_model.node_relation.alias == "fct_revenue" assert ( semantic_model.node_relation.relation_name == f'"dbt"."{project.test_schema}"."fct_revenue"' ) assert semantic_model.config.enabled is True assert semantic_model.config.group == "finance" assert semantic_model.config.meta == {"meta_tag_1": "this_meta"} class TestSemanticModelConfigDefaultValues: @pytest.fixture(scope="class") def models(self): return { "schema.yml": semantic_model_schema_yml_v2_default_values, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, } def test_semantic_model_parsing_defaults(self, project) -> None: runner = dbtTestRunner() result = runner.invoke(["parse"]) assert result.success assert isinstance(result.result, Manifest) manifest = result.result assert len(manifest.semantic_models) == 1 semantic_model = list(manifest.semantic_models.values())[0] # With no custom name, alias should be based on model name (default test. + model name) assert semantic_model.node_relation.alias == "fct_revenue" assert ( semantic_model.description == "This is the model fct_revenue. It should be able to use doc blocks" ) # Should use default config values assert semantic_model.config.enabled is True assert semantic_model.config.group is None assert semantic_model.config.meta == {} class TestSemanticModelConfigDoesNotExistPassesWithoutParsingSemanticModel: @pytest.fixture(scope="class") def models(self): return { "schema.yml": semantic_model_config_does_not_exist, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, } def test_semantic_model_parsing(self, project) -> None: runner = dbtTestRunner() result = runner.invoke(["parse"]) assert result.success assert isinstance(result.result, Manifest) manifest = result.result assert len(manifest.semantic_models) == 0 class TestSemanticModelDisabledConfigIsNotParsed: @pytest.fixture(scope="class") def models(self): return { "schema.yml": semantic_model_schema_yml_v2_disabled, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, } def test_semantic_model_parsing(self, project) -> None: runner = dbtTestRunner() result = runner.invoke(["parse"]) assert result.success assert isinstance(result.result, Manifest) manifest = result.result assert len(manifest.semantic_models) == 0 class TestSemanticModelFalseConfigIsNotParsed: @pytest.fixture(scope="class") def models(self): return { "schema.yml": semantic_model_schema_yml_v2_false_config, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, } def test_semantic_model_parsing(self, project) -> None: runner = dbtTestRunner() result = runner.invoke(["parse"]) assert result.success assert isinstance(result.result, Manifest) manifest = result.result assert len(manifest.semantic_models) == 0 class TestMetricOnModelParsingWorks: @pytest.fixture(scope="class") def models(self): return { "schema.yml": base_schema_yml_v2 + schema_yml_v2_simple_metric_on_model_1, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, } def test_metric_on_model_parsing(self, project): runner = dbtTestRunner() result = runner.invoke(["parse"]) assert result.success manifest = result.result semantic_model = manifest.semantic_models["semantic_model.test.fct_revenue"] assert semantic_model.defaults.agg_time_dimension == "second_dim" metrics = manifest.metrics semantic_manifest = SemanticManifest(manifest) semantic_manifest_metrics = { metric.name: metric for metric in semantic_manifest._get_pydantic_semantic_manifest().metrics } assert len(metrics) == 5 simple_metric = metrics["metric.test.simple_metric"] assert simple_metric.name == "simple_metric" assert simple_metric.description == "This is our first simple metric." assert simple_metric.type == MetricType.SIMPLE assert simple_metric.type_params.metric_aggregation_params.agg == AggregationType.COUNT assert simple_metric.type_params.metric_aggregation_params.semantic_model == "fct_revenue" assert "semantic_model.test.fct_revenue" in simple_metric.depends_on.nodes assert ( simple_metric.type_params.metric_aggregation_params.agg_time_dimension == "second_dim" ) simple_metric_pydantic = semantic_manifest_metrics["simple_metric"] assert simple_metric_pydantic.name == "simple_metric" assert simple_metric_pydantic.description == "This is our first simple metric." assert simple_metric_pydantic.type == MetricType.SIMPLE assert ( simple_metric_pydantic.type_params.metric_aggregation_params.agg == AggregationType.COUNT ) assert ( simple_metric_pydantic.type_params.metric_aggregation_params.semantic_model == "fct_revenue" ) assert ( simple_metric_pydantic.type_params.metric_aggregation_params.agg_time_dimension == "second_dim" ) # No 'depends_on' in the pydantic metric simple_metric_2 = metrics["metric.test.simple_metric_2"] assert simple_metric_2.name == "simple_metric_2" assert simple_metric_2.description == "This is our second simple metric." assert simple_metric_2.type == MetricType.SIMPLE assert simple_metric_2.type_params.metric_aggregation_params.agg == AggregationType.COUNT assert ( simple_metric_2.type_params.metric_aggregation_params.semantic_model == "fct_revenue" ) assert "semantic_model.test.fct_revenue" in simple_metric_2.depends_on.nodes assert simple_metric_2.type_params.metric_aggregation_params.agg_time_dimension == "ds" simple_metric_2_pydantic = semantic_manifest_metrics["simple_metric_2"] assert simple_metric_2_pydantic.name == "simple_metric_2" assert simple_metric_2_pydantic.description == "This is our second simple metric." assert simple_metric_2_pydantic.type == MetricType.SIMPLE assert ( simple_metric_2_pydantic.type_params.metric_aggregation_params.agg == AggregationType.COUNT ) assert ( simple_metric_2_pydantic.type_params.metric_aggregation_params.semantic_model == "fct_revenue" ) assert ( simple_metric_2_pydantic.type_params.metric_aggregation_params.agg_time_dimension == "ds" ) # No 'depends_on' in the pydantic metric percentile_metric = metrics["metric.test.percentile_metric"] assert percentile_metric.name == "percentile_metric" assert percentile_metric.description == "This is our percentile metric." assert percentile_metric.type == MetricType.SIMPLE assert ( percentile_metric.type_params.metric_aggregation_params.agg == AggregationType.PERCENTILE ) assert ( percentile_metric.type_params.metric_aggregation_params.semantic_model == "fct_revenue" ) assert ( percentile_metric.type_params.metric_aggregation_params.agg_params.percentile == 0.99 ) assert ( percentile_metric.type_params.metric_aggregation_params.agg_params.use_discrete_percentile is True ) assert ( percentile_metric.type_params.metric_aggregation_params.agg_params.use_approximate_percentile is False ) assert "semantic_model.test.fct_revenue" in percentile_metric.depends_on.nodes assert ( percentile_metric.type_params.metric_aggregation_params.agg_time_dimension == "second_dim" ) percentile_metric_pydantic = semantic_manifest_metrics["percentile_metric"] assert percentile_metric_pydantic.name == "percentile_metric" assert percentile_metric_pydantic.description == "This is our percentile metric." assert percentile_metric_pydantic.type == MetricType.SIMPLE assert ( percentile_metric_pydantic.type_params.metric_aggregation_params.agg == AggregationType.PERCENTILE ) assert ( percentile_metric_pydantic.type_params.metric_aggregation_params.semantic_model == "fct_revenue" ) assert ( percentile_metric_pydantic.type_params.metric_aggregation_params.agg_params.percentile == 0.99 ) assert ( percentile_metric_pydantic.type_params.metric_aggregation_params.agg_params.use_discrete_percentile is True ) assert ( percentile_metric_pydantic.type_params.metric_aggregation_params.agg_params.use_approximate_percentile is False ) assert ( percentile_metric_pydantic.type_params.metric_aggregation_params.agg_time_dimension == "second_dim" ) cumulative_metric = metrics["metric.test.cumulative_metric"] assert cumulative_metric.name == "cumulative_metric" assert cumulative_metric.description == "This is our cumulative metric." assert cumulative_metric.type == MetricType.CUMULATIVE assert cumulative_metric.type_params.cumulative_type_params.grain_to_date == "day" assert ( cumulative_metric.type_params.cumulative_type_params.period_agg == PeriodAggregation.FIRST ) assert cumulative_metric.type_params.cumulative_type_params.metric.name == "simple_metric" assert "metric.test.simple_metric" in cumulative_metric.depends_on.nodes assert cumulative_metric.type_params.metric_aggregation_params is None cumulative_metric_pydantic = semantic_manifest_metrics["cumulative_metric"] assert cumulative_metric_pydantic.name == "cumulative_metric" assert cumulative_metric_pydantic.description == "This is our cumulative metric." assert cumulative_metric_pydantic.type == MetricType.CUMULATIVE assert cumulative_metric_pydantic.type_params.cumulative_type_params.grain_to_date == "day" assert ( cumulative_metric_pydantic.type_params.cumulative_type_params.period_agg == PeriodAggregation.FIRST ) assert ( cumulative_metric_pydantic.type_params.cumulative_type_params.metric.name == "simple_metric" ) assert cumulative_metric_pydantic.type_params.metric_aggregation_params is None conversion_metric = metrics["metric.test.conversion_metric"] assert conversion_metric.name == "conversion_metric" assert conversion_metric.description == "This is our conversion metric." assert conversion_metric.type == MetricType.CONVERSION assert conversion_metric.type_params.conversion_type_params.entity == "id_entity" assert ( conversion_metric.type_params.conversion_type_params.calculation is ConversionCalculationType.CONVERSION_RATE ) assert ( conversion_metric.type_params.conversion_type_params.base_metric.name == "simple_metric" ) assert ( conversion_metric.type_params.conversion_type_params.conversion_metric.name == "simple_metric_2" ) assert "metric.test.simple_metric" in conversion_metric.depends_on.nodes assert "metric.test.simple_metric_2" in conversion_metric.depends_on.nodes assert conversion_metric.type_params.metric_aggregation_params is None conversion_metric_pydantic = semantic_manifest_metrics["conversion_metric"] assert conversion_metric_pydantic.name == "conversion_metric" assert conversion_metric_pydantic.description == "This is our conversion metric." assert conversion_metric_pydantic.type == MetricType.CONVERSION assert conversion_metric_pydantic.type_params.conversion_type_params.entity == "id_entity" assert ( conversion_metric_pydantic.type_params.conversion_type_params.calculation is ConversionCalculationType.CONVERSION_RATE ) assert ( conversion_metric_pydantic.type_params.conversion_type_params.base_metric.name == "simple_metric" ) assert ( conversion_metric_pydantic.type_params.conversion_type_params.conversion_metric.name == "simple_metric_2" ) assert conversion_metric_pydantic.type_params.metric_aggregation_params is None class TestMetricHiddenMapsToIsPrivate: """Test that a metric's 'hidden' field in YAML is reflected as 'is_private' on the parsed metric.""" @pytest.fixture(scope="class") def models(self): return { "schema.yml": base_schema_yml_v2 + schema_yml_v2_metrics_with_hidden, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, } def test_metric_hidden_yaml_maps_to_is_private(self, project): runner = dbtTestRunner() result = runner.invoke(["parse"]) assert result.success manifest = result.result metrics = manifest.metrics assert len(metrics) == 2 public_metric = metrics["metric.test.public_metric"] assert public_metric.type_params.is_private is False private_metric = metrics["metric.test.private_metric"] assert private_metric.type_params.is_private is True class TestStandaloneMetricParsingSimpleMetricFails: @pytest.fixture(scope="class") def models(self): return { "schema.yml": base_schema_yml_v2 + schema_yml_v2_standalone_simple_metric, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, } def test_standalone_metric_parsing(self, project): runner = dbtTestRunner() result = runner.invoke(["parse"]) assert not result.success assert ( "simple metrics in v2 YAML must be attached to semantic_model" in result.exception.msg ) class TestStandaloneMetricParsingWorks: @pytest.fixture(scope="class") def models(self): return { "schema.yml": base_schema_yml_v2 + schema_yml_v2_simple_metric_on_model_1 + schema_yml_v2_standalone_metrics, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, } def test_included_metric_parsing(self, project): runner = dbtTestRunner() result = runner.invoke(["parse"]) assert result.success manifest = result.result semantic_model = manifest.semantic_models["semantic_model.test.fct_revenue"] assert semantic_model.defaults.agg_time_dimension == "second_dim" metrics = manifest.metrics semantic_manifest = SemanticManifest(manifest) semantic_manifest_metrics = { metric.name: metric for metric in semantic_manifest._get_pydantic_semantic_manifest().metrics } assert len(metrics) == 6 metric = metrics["metric.test.standalone_conversion_metric"] assert metric.name == "standalone_conversion_metric" assert metric.description == "This is our standalone conversion metric." assert metric.type == MetricType.CONVERSION assert metric.type_params.conversion_type_params.entity == "id_entity" assert ( metric.type_params.conversion_type_params.calculation == ConversionCalculationType.CONVERSION_RATE ) assert metric.type_params.conversion_type_params.base_metric.name == "simple_metric" assert ( metric.type_params.conversion_type_params.conversion_metric.name == "simple_metric_2" ) assert set(metric.depends_on.nodes) == set( [ "metric.test.simple_metric", "metric.test.simple_metric_2", ] ) assert metric.type_params.metric_aggregation_params is None assert metric.filter.where_filters[0].where_sql_template == "id > 0" metric_pydantic = semantic_manifest_metrics["standalone_conversion_metric"] assert metric_pydantic.name == "standalone_conversion_metric" assert metric_pydantic.description == "This is our standalone conversion metric." assert metric_pydantic.type == MetricType.CONVERSION assert metric_pydantic.type_params.conversion_type_params.entity == "id_entity" assert ( metric_pydantic.type_params.conversion_type_params.calculation == ConversionCalculationType.CONVERSION_RATE ) assert ( metric_pydantic.type_params.conversion_type_params.base_metric.name == "simple_metric" ) assert ( metric_pydantic.type_params.conversion_type_params.conversion_metric.name == "simple_metric_2" ) assert metric_pydantic.type_params.metric_aggregation_params is None assert metric_pydantic.filter.where_filters[0].where_sql_template == "id > 0" class TestCumulativeMetricNoInputMetricFails: @pytest.fixture(scope="class") def models(self): return { "schema.yml": base_schema_yml_v2 + schema_yml_v2_cumulative_metric_missing_input_metric, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, } def test_cumulative_metric_no_input_metric_parsing_fails(self, project) -> None: runner = dbtTestRunner() result = runner.invoke(["parse"]) assert not result.success assert "input_metric is required for cumulative metrics." in str(result.exception) class TestConversionMetricNoBaseMetricFails: @pytest.fixture(scope="class") def models(self): return { "schema.yml": base_schema_yml_v2 + schema_yml_v2_conversion_metric_missing_base_metric, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, } def test_conversion_metric_no_base_metric_parsing_fails(self, project) -> None: runner = dbtTestRunner() result = runner.invoke(["parse"]) assert not result.success assert "base_metric is required for conversion metrics." in str(result.exception) class TestDerivedSemanticsWithDocJinjaParsingWorks: @pytest.fixture(scope="class") def models(self): return { "schema.yml": semantic_model_schema_yml_v2 + derived_semantics_with_doc_jinja_yml, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "docs.md": semantic_model_descriptions, } def test_derived_semantics_doc_jinja_parsing(self, project) -> None: runner = dbtTestRunner() result = runner.invoke(["parse"]) assert result.success manifest = result.result assert len(manifest.semantic_models) == 1 semantic_model = manifest.semantic_models["semantic_model.test.fct_revenue"] entities = {entity.name: entity for entity in semantic_model.entities} assert entities["derived_id_entity"].description == "qux" dimensions = {dimension.name: dimension for dimension in semantic_model.dimensions} assert dimensions["derived_id_dimension"].description == "bar" class TestDerivedSemanticsParsingWorks: @pytest.fixture(scope="class") def models(self): return { "schema.yml": semantic_model_schema_yml_v2 + derived_semantics_yml, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, } def test_derived_semantics_parsing(self, project) -> None: runner = dbtTestRunner() result = runner.invoke(["parse"]) assert result.success manifest = result.result assert len(manifest.semantic_models) == 1 semantic_model = manifest.semantic_models["semantic_model.test.fct_revenue"] entities = {entity.name: entity for entity in semantic_model.entities} assert ( len(entities) == 5 ) # length is so long because it is column entities + derived entities id_entity = entities["derived_id_entity"] assert id_entity.type == EntityType.FOREIGN assert id_entity.description == "This is the id entity, and it is the primary entity." assert id_entity.expr == "id + foreign_id_col" assert id_entity.config.meta == {"test_label_thing": "derived_entity_1"} id_entity_with_no_optional_fields = entities["derived_id_entity_with_no_optional_fields"] assert id_entity_with_no_optional_fields.type == EntityType.FOREIGN assert id_entity_with_no_optional_fields.expr == "id + foreign_id_col" assert id_entity_with_no_optional_fields.config.meta == {} dimensions = {dimension.name: dimension for dimension in semantic_model.dimensions} assert len(dimensions) == 4 # includes non-derived dimensions assert dimensions["derived_id_dimension"].type == DimensionType.CATEGORICAL assert dimensions["derived_id_dimension"].expr == "id" assert dimensions["derived_id_dimension"].config.meta == {} assert dimensions["derived_id_dimension"].type_params.validity_params.is_start is True assert dimensions["derived_id_dimension"].type_params.validity_params.is_end is True class TestSemanticModelWithPrimaryEntityOnlyOnColumn: @pytest.fixture(scope="class") def models(self): return { "schema.yml": semantic_model_schema_yml_v2_with_primary_entity_only_on_column, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, } def test_primary_entity_type_is_id_entity(self, project): runner = dbtTestRunner() result = runner.invoke(["parse"]) assert result.success manifest = result.result assert len(manifest.semantic_models) == 1 semantic_model = list(manifest.semantic_models.values())[0] entities = {entity.name: entity for entity in semantic_model.entities} primary_entity = [ entity for entity in entities.values() if entity.type == EntityType.PRIMARY ] assert len(primary_entity) == 1 primary_entity = primary_entity[0] assert primary_entity.name == "id_entity" assert semantic_model.primary_entity is None class TestSemanticModelWithPrimaryEntityOnlyOnModel: @pytest.fixture(scope="class") def models(self): return { "schema.yml": semantic_model_schema_yml_v2_primary_entity_only_on_model, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, } def test_primary_entities_empty(self, project): runner = dbtTestRunner() result = runner.invoke(["parse"]) assert result.success manifest = result.result assert len(manifest.semantic_models) == 1 semantic_model = list(manifest.semantic_models.values())[0] entities = {entity.name: entity for entity in semantic_model.entities} primary_entity = [ entity for entity in entities.values() if entity.type == EntityType.PRIMARY ] assert len(primary_entity) == 0 assert semantic_model.primary_entity == "id_entity" class TestSimpleSemanticModelWithMetricWithDocJinja: @pytest.fixture(scope="class") def models(self): return { "schema.yml": base_schema_yml_v2 + schema_yml_v2_simple_metric_on_model_1 + schema_yml_v2_metric_with_doc_jinja, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "docs.md": semantic_model_descriptions, } def test_simple_metric_with_doc_jinja_parsing(self, project): runner = dbtTestRunner() result = runner.invoke(["parse"]) assert result.success manifest = result.result assert len(manifest.semantic_models) == 1 metric = manifest.metrics["metric.test.simple_metric_with_doc_jinja"] assert metric.description == "describe away!" class TestSimpleSemanticModelWithFilterWithFilterDimensionJinja: @pytest.fixture(scope="class") def models(self): return { "schema.yml": base_schema_yml_v2 + schema_yml_v2_simple_metric_on_model_1 + schema_yml_v2_metric_with_filter_dimension_jinja, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "docs.md": semantic_model_descriptions, } def test_simple_metric_with_filter_with_filter_dimension_jinja_parsing(self, project): runner = dbtTestRunner() result = runner.invoke(["parse"]) assert result.success manifest = result.result assert len(manifest.semantic_models) == 1 metric = manifest.metrics["metric.test.simple_metric_with_filter_dimension_jinja"] assert ( metric.filter.where_filters[0].where_sql_template == "{{ Dimension('id_entity__id_dim') }} > 0 and {{ TimeDimension('id_entity__id_dim', 'day') }} > '2020-01-01'" ) class TestTopLevelSemanticsMetricWithDocJinja: @pytest.fixture(scope="class") def models(self): return { "schema.yml": base_schema_yml_v2 + schema_yml_v2_simple_metric_on_model_1 + schema_yml_v2_standalone_metrics_with_doc_jinja, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "docs.md": semantic_model_descriptions, } def test_top_level_metric_with_doc_jinja_parsing(self, project): runner = dbtTestRunner() result = runner.invoke(["parse"]) assert result.success manifest = result.result assert len(manifest.semantic_models) == 1 metric = manifest.metrics["metric.test.standalone_conversion_metric"] assert metric.description == "describe away!" assert ( metric.filter.where_filters[0].where_sql_template == "{{ Dimension('id_entity__id_dim') }} > 0" ) semantic_manifest = SemanticManifest(manifest) semantic_manifest_metrics = { metric.name: metric for metric in semantic_manifest._get_pydantic_semantic_manifest().metrics } assert ( semantic_manifest_metrics["standalone_conversion_metric"] .filter.where_filters[0] .where_sql_template == "{{ Dimension('id_entity__id_dim') }} > 0" ) class TestDerivedMetricWithInputMetricsFilterDimensionJinja: """Test that {{ Dimension(...) }} jinja in input_metrics[].filter is not rendered at parse time.""" @pytest.fixture(scope="class") def models(self): return { "schema.yml": base_schema_yml_v2 + schema_yml_v2_metric_with_input_metrics_filter_dimension_jinja, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, } def test_input_metrics_filter_jinja_not_rendered(self, project): runner = dbtTestRunner() result = runner.invoke(["parse"]) assert result.success manifest = result.result metric = manifest.metrics["metric.test.derived_metric_with_jinja_filter"] assert metric.type == MetricType.DERIVED # The input metric filter should preserve the Dimension jinja template offset_input = [m for m in metric.type_params.metrics if m.alias == "offset_metric"][0] assert "{{ Dimension('id_entity__id_dim') }} > 0" in ( offset_input.filter.where_filters[0].where_sql_template ) class TestRatioMetricWithNumeratorFilterDimensionJinja: """Test that {{ Dimension(...) }} jinja in numerator.filter is not rendered at parse time.""" @pytest.fixture(scope="class") def models(self): return { "schema.yml": base_schema_yml_v2 + schema_yml_v2_metric_with_numerator_filter_dimension_jinja, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, } def test_numerator_filter_jinja_not_rendered(self, project): runner = dbtTestRunner() result = runner.invoke(["parse"]) assert result.success manifest = result.result metric = manifest.metrics["metric.test.ratio_metric_with_jinja_filter"] assert metric.type == MetricType.RATIO # The numerator filter should preserve the Dimension jinja template assert "{{ Dimension('id_entity__id_dim') }} > 0" in ( metric.type_params.numerator.filter.where_filters[0].where_sql_template ) class TestMetricOnModelWithCustomSemanticModelName: """Test that metrics correctly reference the custom semantic model name when semantic_model.name is set, rather than using the dbt model name.""" @pytest.fixture(scope="class") def models(self): return { "schema.yml": base_schema_yml_v2_with_custom_sm_name + schema_yml_v2_simple_metric_on_model_1, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, } def test_metrics_use_custom_semantic_model_name(self, project): runner = dbtTestRunner() result = runner.invoke(["parse"]) assert result.success manifest = result.result # Semantic model should be registered under the custom name assert "semantic_model.test.custom_semantic_model" in manifest.semantic_models semantic_model = manifest.semantic_models["semantic_model.test.custom_semantic_model"] assert semantic_model.name == "custom_semantic_model" # But it still points to the fct_revenue model assert semantic_model.node_relation.alias == "fct_revenue" # Simple metrics should reference the custom semantic model name simple_metric = manifest.metrics["metric.test.simple_metric"] assert ( simple_metric.type_params.metric_aggregation_params.semantic_model == "custom_semantic_model" ) assert "semantic_model.test.custom_semantic_model" in simple_metric.depends_on.nodes simple_metric_2 = manifest.metrics["metric.test.simple_metric_2"] assert ( simple_metric_2.type_params.metric_aggregation_params.semantic_model == "custom_semantic_model" ) assert "semantic_model.test.custom_semantic_model" in simple_metric_2.depends_on.nodes # Conversion metric should pass validation (it references simple metrics # which are linked to the custom-named semantic model) conversion_metric = manifest.metrics["metric.test.conversion_metric"] assert conversion_metric.type == MetricType.CONVERSION assert ( conversion_metric.type_params.conversion_type_params.base_metric.name == "simple_metric" ) assert ( conversion_metric.type_params.conversion_type_params.conversion_metric.name == "simple_metric_2" ) # Verify semantic manifest validation also passes semantic_manifest = SemanticManifest(manifest) semantic_manifest_metrics = { metric.name: metric for metric in semantic_manifest._get_pydantic_semantic_manifest().metrics } assert ( semantic_manifest_metrics[ "simple_metric" ].type_params.metric_aggregation_params.semantic_model == "custom_semantic_model" ) class TestMetricOnModelWithoutCustomSemanticModelName: """Test that metrics correctly use the dbt model name as the semantic model name when no custom semantic_model.name is set (semantic_model: true).""" @pytest.fixture(scope="class") def models(self): return { "schema.yml": base_schema_yml_v2 + schema_yml_v2_simple_metric_on_model_1, "fct_revenue.sql": fct_revenue_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, } def test_metrics_use_model_name_as_semantic_model_name(self, project): runner = dbtTestRunner() result = runner.invoke(["parse"]) assert result.success manifest = result.result # Semantic model should be registered under the model name assert "semantic_model.test.fct_revenue" in manifest.semantic_models # Simple metrics should reference the model name simple_metric = manifest.metrics["metric.test.simple_metric"] assert simple_metric.type_params.metric_aggregation_params.semantic_model == "fct_revenue" assert "semantic_model.test.fct_revenue" in simple_metric.depends_on.nodes # Conversion metric should pass validation conversion_metric = manifest.metrics["metric.test.conversion_metric"] assert conversion_metric.type == MetricType.CONVERSION assert ( conversion_metric.type_params.conversion_type_params.base_metric.name == "simple_metric" ) # Verify semantic manifest validation also passes semantic_manifest = SemanticManifest(manifest) semantic_manifest_metrics = { metric.name: metric for metric in semantic_manifest._get_pydantic_semantic_manifest().metrics } assert ( semantic_manifest_metrics[ "simple_metric" ].type_params.metric_aggregation_params.semantic_model == "fct_revenue" ) # TODO DI-4605: add enforcement and a test for when there are validity params with no column granularity # TODO DI-4603: add enforcement and a test for a TIME type dimension and a column that has no granularity set ================================================ FILE: tests/functional/semantic_models/test_semantic_models.py ================================================ import pytest from dbt.contracts.graph.manifest import Manifest from dbt.exceptions import CompilationError from dbt.tests.util import run_dbt, write_file from tests.functional.semantic_models.fixtures import ( models_people_metrics_yml, models_people_sql, semantic_model_descriptions, semantic_model_people_diff_name_yml, semantic_model_people_yml, semantic_model_people_yml_with_docs, simple_metricflow_time_spine_sql, ) class TestSemanticModelDependsOn: @pytest.fixture(scope="class") def models(self): return { "people.sql": models_people_sql, "metricflow_time_spine.sql": simple_metricflow_time_spine_sql, "semantic_models.yml": semantic_model_people_yml, "people_metrics.yml": models_people_metrics_yml, } def test_depends_on(self, project): manifest = run_dbt(["parse"]) assert isinstance(manifest, Manifest) expected_depends_on_for_people_semantic_model = ["model.test.people"] number_of_people_metric = manifest.semantic_models["semantic_model.test.semantic_people"] assert ( number_of_people_metric.depends_on.nodes == expected_depends_on_for_people_semantic_model ) class TestSemanticModelNestedDocs: @pytest.fixture(scope="class") def models(self): return { "people.sql": models_people_sql, "metricflow_time_spine.sql": simple_metricflow_time_spine_sql, "semantic_models.yml": semantic_model_people_yml_with_docs, "people_metrics.yml": models_people_metrics_yml, "docs.md": semantic_model_descriptions, } def test_depends_on(self, project): manifest = run_dbt(["parse"]) node = manifest.semantic_models["semantic_model.test.semantic_people"] assert node.description == "foo" assert node.dimensions[0].description == "bar" assert node.measures[0].description == "baz" assert node.entities[0].description == "qux" class TestSemanticModelUnknownModel: @pytest.fixture(scope="class") def models(self): return { "not_people.sql": models_people_sql, "metricflow_time_spine.sql": simple_metricflow_time_spine_sql, "semantic_models.yml": semantic_model_people_yml, "people_metrics.yml": models_people_metrics_yml, } def test_unknown_model_raises_issue(self, project): with pytest.raises(CompilationError) as excinfo: run_dbt(["parse"]) assert "depends on a node named 'people' which was not found" in str(excinfo.value) class TestSemanticModelPartialParsing: @pytest.fixture(scope="class") def models(self): return { "people.sql": models_people_sql, "metricflow_time_spine.sql": simple_metricflow_time_spine_sql, "semantic_models.yml": semantic_model_people_yml, "people_metrics.yml": models_people_metrics_yml, } def test_semantic_model_deleted_partial_parsing(self, project): # First, use the default saved_queries.yml to define our saved_query, and # run the dbt parse command run_dbt(["parse"]) # Next, modify the default semantic_models.yml to remove the saved query. write_file( semantic_model_people_diff_name_yml, project.project_root, "models", "semantic_models.yml", ) run_dbt(["compile"]) ================================================ FILE: tests/functional/severity/test_severity.py ================================================ import pytest from dbt.tests.util import run_dbt models__sample_model_sql = """ select * from {{ source("raw", "sample_seed") }} """ models__schema_yml = """ version: 2 sources: - name: raw database: "{{ target.database }}" schema: "{{ target.schema }}" tables: - name: sample_seed columns: - name: email data_tests: - not_null: severity: "{{ 'error' if var('strict', false) else 'warn' }}" models: - name: sample_model columns: - name: email data_tests: - not_null: severity: "{{ 'error' if var('strict', false) else 'warn' }}" """ seeds__sample_seed_csv = """id,first_name,last_name,email,gender,ip_address,updated_at 1,Judith,Kennedy,jkennedy0@phpbb.com,Female,54.60.24.128,2015-12-24 12:19:28 2,Arthur,Kelly,akelly1@eepurl.com,Male,62.56.24.215,2015-10-28 16:22:15 3,Rachel,Moreno,rmoreno2@msu.edu,Female,31.222.249.23,2016-04-05 02:05:30 4,Ralph,Turner,rturner3@hp.com,Male,157.83.76.114,2016-08-08 00:06:51 5,Laura,Gonzales,lgonzales4@howstuffworks.com,Female,30.54.105.168,2016-09-01 08:25:38 6,Katherine,Lopez,null,Female,169.138.46.89,2016-08-30 18:52:11 7,Jeremy,Hamilton,jhamilton6@mozilla.org,Male,231.189.13.133,2016-07-17 02:09:46 8,Heather,Rose,hrose7@goodreads.com,Female,87.165.201.65,2015-12-29 22:03:56 9,Gregory,Kelly,gkelly8@trellian.com,Male,154.209.99.7,2016-03-24 21:18:16 10,Rachel,Lopez,rlopez9@themeforest.net,Female,237.165.82.71,2016-08-20 15:44:49 11,Donna,Welch,dwelcha@shutterfly.com,Female,103.33.110.138,2016-02-27 01:41:48 12,Russell,Lawrence,rlawrenceb@qq.com,Male,189.115.73.4,2016-06-11 03:07:09 13,Michelle,Montgomery,mmontgomeryc@scientificamerican.com,Female,243.220.95.82,2016-06-18 16:27:19 14,Walter,Castillo,null,Male,71.159.238.196,2016-10-06 01:55:44 15,Robin,Mills,rmillse@vkontakte.ru,Female,172.190.5.50,2016-10-31 11:41:21 16,Raymond,Holmes,rholmesf@usgs.gov,Male,148.153.166.95,2016-10-03 08:16:38 17,Gary,Bishop,gbishopg@plala.or.jp,Male,161.108.182.13,2016-08-29 19:35:20 18,Anna,Riley,arileyh@nasa.gov,Female,253.31.108.22,2015-12-11 04:34:27 19,Sarah,Knight,sknighti@foxnews.com,Female,222.220.3.177,2016-09-26 00:49:06 20,Phyllis,Fox,pfoxj@creativecommons.org,Female,163.191.232.95,2016-08-21 10:35:19 """ tests__sample_test_sql = """ {{ config(severity='error' if var('strict', false) else 'warn') }} select * from {{ ref("sample_model") }} where email is null """ @pytest.fixture(scope="class") def models(): return {"sample_model.sql": models__sample_model_sql, "schema.yml": models__schema_yml} @pytest.fixture(scope="class") def seeds(): return {"sample_seed.csv": seeds__sample_seed_csv} @pytest.fixture(scope="class") def tests(): return {"null_email.sql": tests__sample_test_sql} @pytest.fixture(scope="class") def project_config_update(): return { "config-version": 2, "seed-paths": ["seeds"], "test-paths": ["tests"], "seeds": { "quote_columns": False, }, } class TestSeverity: @pytest.fixture(scope="class", autouse=True) def seed_and_run(self, project): run_dbt(["seed"]) run_dbt(["run"]) def test_generic_default(self, project): results = run_dbt(["test", "--select", "test_type:generic"]) assert len(results) == 2 assert all([r.status == "warn" for r in results]) assert all([r.failures == 2 for r in results]) def test_generic_strict(self, project): results = run_dbt( ["test", "--select", "test_type:generic", "--vars", '{"strict": True}'], expect_pass=False, ) assert len(results) == 2 assert all([r.status == "fail" for r in results]) assert all([r.failures == 2 for r in results]) def test_singular_default(self, project): results = run_dbt(["test", "--select", "test_type:singular"]) assert len(results) == 1 assert all([r.status == "warn" for r in results]) assert all([r.failures == 2 for r in results]) def test_singular_strict(self, project): results = run_dbt( ["test", "--select", "test_type:singular", "--vars", '{"strict": True}'], expect_pass=False, ) assert len(results) == 1 assert all([r.status == "fail" for r in results]) assert all([r.failures == 2 for r in results]) ================================================ FILE: tests/functional/show/fixtures.py ================================================ models__sample_model = """ select * from {{ ref('sample_seed') }} """ models__sample_number_model = """ select cast(1.0 as int) as float_to_int_field, 3.0 as float_field, 4.3 as float_with_dec_field, 5 as int_field """ models__sample_number_model_with_nulls = """ select cast(1.0 as int) as float_to_int_field, 3.0 as float_field, 4.3 as float_with_dec_field, 5 as int_field union all select cast(null as int) as float_to_int_field, cast(null as float) as float_field, cast(null as float) as float_with_dec_field, cast(null as int) as int_field """ models__second_model = """ select sample_num as col_one, sample_bool as col_two, 42 as answer from {{ ref('sample_model') }} """ models__sql_header = """ {% call set_sql_header(config) %} set session time zone '{{ var("timezone", "Europe/Paris") }}'; {%- endcall %} select current_setting('timezone') as timezone """ private_model_yml = """ groups: - name: my_cool_group owner: {name: me} models: - name: private_model access: private config: group: my_cool_group """ schema_yml = """ models: - name: sample_model latest_version: 1 # declare the versions, and fully specify them versions: - v: 2 config: materialized: table columns: - name: sample_num data_type: int - name: sample_bool data_type: bool - name: answer data_type: int - v: 1 config: materialized: table contract: {enforced: true} columns: - name: sample_num data_type: int - name: sample_bool data_type: bool """ models__ephemeral_model = """ {{ config(materialized = 'ephemeral') }} select coalesce(sample_num, 0) + 10 as col_deci from {{ ref('sample_model') }} """ models__second_ephemeral_model = """ {{ config(materialized = 'ephemeral') }} select col_deci + 100 as col_hundo from {{ ref('ephemeral_model') }} """ seeds__sample_seed = """sample_num,sample_bool 1,true 2,false 3,true 4,false 5,true 6,false 7,true """ ================================================ FILE: tests/functional/show/test_show.py ================================================ import json import pytest from dbt.tests.util import run_dbt, run_dbt_and_capture from dbt_common.exceptions import DbtBaseException as DbtException from dbt_common.exceptions import DbtRuntimeError from tests.functional.show.fixtures import ( models__ephemeral_model, models__sample_model, models__sample_number_model, models__sample_number_model_with_nulls, models__second_ephemeral_model, models__second_model, private_model_yml, schema_yml, seeds__sample_seed, ) class ShowBase: @pytest.fixture(scope="class") def models(self): return { "sample_model.sql": models__sample_model, "sample_number_model.sql": models__sample_number_model, "sample_number_model_with_nulls.sql": models__sample_number_model_with_nulls, "second_model.sql": models__second_model, "ephemeral_model.sql": models__ephemeral_model, } @pytest.fixture(scope="class") def seeds(self): return {"sample_seed.csv": seeds__sample_seed} @pytest.fixture(scope="class", autouse=True) def setup(self, project): run_dbt(["seed"]) class TestShowNone(ShowBase): def test_none(self, project): with pytest.raises( DbtRuntimeError, match="Either --select or --inline must be passed to show" ): run_dbt(["show"]) class TestShowSelectText(ShowBase): def test_select_model_text(self, project): run_dbt(["build"]) (_, log_output) = run_dbt_and_capture(["show", "--select", "second_model"]) assert "Previewing node 'sample_model'" not in log_output assert "Previewing node 'second_model'" in log_output assert "col_one" in log_output assert "col_two" in log_output assert "answer" in log_output class TestShowMultiple(ShowBase): def test_select_multiple_model_text(self, project): run_dbt(["build"]) (_, log_output) = run_dbt_and_capture(["show", "--select", "sample_model second_model"]) assert "Previewing node 'sample_model'" in log_output assert "sample_num" in log_output assert "sample_bool" in log_output class TestShowSingle(ShowBase): def test_select_single_model_json(self, project): run_dbt(["build"]) (_, log_output) = run_dbt_and_capture( ["show", "--select", "sample_model", "--output", "json"] ) assert "Previewing node 'sample_model'" not in log_output assert "sample_num" in log_output assert "sample_bool" in log_output with pytest.raises(json.JSONDecodeError): json.loads(log_output) def test_select_single_model_json_quiet(self, project): run_dbt(["build"]) (_, log_output) = run_dbt_and_capture( ["show", "--quiet", "--select", "sample_model", "--output", "json"] ) assert "Previewing node 'sample_model'" not in log_output assert "sample_num" in log_output assert "sample_bool" in log_output json.loads(log_output) class TestShowNumeric(ShowBase): def test_numeric_values(self, project): run_dbt(["build"]) (_, log_output) = run_dbt_and_capture( ["show", "--select", "sample_number_model", "--output", "json"] ) # json log output needs the escapes removed for string matching log_output = log_output.replace("\\", "") assert "Previewing node 'sample_number_model'" not in log_output assert '"float_to_int_field": 1.0' not in log_output assert '"float_to_int_field": 1' in log_output assert '"float_field": 3.0' in log_output assert '"float_with_dec_field": 4.3' in log_output assert '"int_field": 5' in log_output assert '"int_field": 5.0' not in log_output class TestShowNumericNulls(ShowBase): def test_numeric_values_with_nulls(self, project): run_dbt(["build"]) (_, log_output) = run_dbt_and_capture( ["show", "--select", "sample_number_model_with_nulls", "--output", "json"] ) # json log output needs the escapes removed for string matching log_output = log_output.replace("\\", "") assert "Previewing node 'sample_number_model_with_nulls'" not in log_output assert '"float_to_int_field": 1.0' not in log_output assert '"float_to_int_field": 1' in log_output assert '"float_field": 3.0' in log_output assert '"float_with_dec_field": 4.3' in log_output assert '"int_field": 5' in log_output assert '"int_field": 5.0' not in log_output class TestShowInline(ShowBase): def test_inline_pass(self, project): run_dbt(["build"]) (_, log_output) = run_dbt_and_capture( ["show", "--inline", "select * from {{ ref('sample_model') }}"] ) assert "Previewing inline node" in log_output assert "sample_num" in log_output assert "sample_bool" in log_output def test_inline_pass_quiet(self, project): run_dbt(["build"]) (_, log_output) = run_dbt_and_capture( ["show", "--quiet", "--inline", "select * from {{ ref('sample_model') }}"] ) assert "Previewing inline node" not in log_output assert "sample_num" in log_output assert "sample_bool" in log_output class TestShowInlineFail(ShowBase): def test_inline_fail(self, project): with pytest.raises(DbtException, match="Error parsing inline query"): run_dbt(["show", "--inline", "select * from {{ ref('third_model') }}"]) class TestShowInlineFailDB(ShowBase): def test_inline_fail_database_error(self, project): with pytest.raises(DbtRuntimeError, match="Database Error"): run_dbt(["show", "--inline", "slect asdlkjfsld;j"]) class TestShowInlineDirect(ShowBase): def test_inline_direct_pass(self, project): query = f"select * from {project.test_schema}.sample_seed" (_, log_output) = run_dbt_and_capture(["show", "--inline-direct", query]) assert "Previewing inline node" in log_output assert "sample_num" in log_output assert "sample_bool" in log_output # This is a bit of a hack. Unfortunately, the test teardown code # expects that dbt loaded an adapter with a macro context the last # time it was called. The '--inline-direct' parameter used on the # previous run explicitly disables macros. So now we call 'dbt seed', # which will load the adapter fully and satisfy the teardown code. run_dbt(["seed"]) def test_inline_direct_pass_quiet(self, project): query = f"select * from {project.test_schema}.sample_seed" (_, log_output) = run_dbt_and_capture(["show", "--quiet", "--inline-direct", query]) assert "Previewing inline node" not in log_output assert "sample_num" in log_output assert "sample_bool" in log_output # See prior test for explanation of why this is here run_dbt(["seed"]) def test_inline_direct_pass_no_limit(self, project): query = f"select * from {project.test_schema}.sample_seed" (_, log_output) = run_dbt_and_capture(["show", "--inline-direct", query, "--limit", -1]) assert "Previewing inline node" in log_output assert "sample_num" in log_output assert "sample_bool" in log_output # See prior test for explanation of why this is here run_dbt(["seed"]) class TestShowInlineDirectFail(ShowBase): def test_inline_fail_database_error(self, project): with pytest.raises(DbtRuntimeError, match="Database Error"): run_dbt(["show", "--inline-direct", "slect asdlkjfsld;j"]) # See prior test for explanation of why this is here run_dbt(["seed"]) class TestShowEphemeral(ShowBase): def test_ephemeral_model(self, project): run_dbt(["build"]) (_, log_output) = run_dbt_and_capture(["show", "--select", "ephemeral_model"]) assert "col_deci" in log_output class TestShowSecondEphemeral(ShowBase): def test_second_ephemeral_model(self, project): run_dbt(["build"]) (_, log_output) = run_dbt_and_capture(["show", "--inline", models__second_ephemeral_model]) assert "col_hundo" in log_output class TestShowSeed(ShowBase): def test_seed(self, project): (_, log_output) = run_dbt_and_capture(["show", "--select", "sample_seed"]) assert "Previewing node 'sample_seed'" in log_output class TestShowModelVersions: @pytest.fixture(scope="class") def models(self): return { "schema.yml": schema_yml, "sample_model.sql": models__sample_model, "sample_model_v2.sql": models__second_model, } @pytest.fixture(scope="class") def seeds(self): return {"sample_seed.csv": seeds__sample_seed} def test_version_unspecified(self, project): run_dbt(["build"]) (results, log_output) = run_dbt_and_capture(["show", "--select", "sample_model"]) assert "Previewing node 'sample_model.v1'" in log_output assert "Previewing node 'sample_model.v2'" in log_output def test_none(self, project): run_dbt(["build"]) (results, log_output) = run_dbt_and_capture(["show", "--select", "sample_model.v2"]) assert "Previewing node 'sample_model.v1'" not in log_output assert "Previewing node 'sample_model.v2'" in log_output class TestShowPrivateModel: @pytest.fixture(scope="class") def models(self): return { "schema.yml": private_model_yml, "private_model.sql": models__sample_model, } @pytest.fixture(scope="class") def seeds(self): return {"sample_seed.csv": seeds__sample_seed} def test_version_unspecified(self, project): run_dbt(["build"]) run_dbt(["show", "--inline", "select * from {{ ref('private_model') }}"]) ================================================ FILE: tests/functional/snapshots/data/invalidate_postgres.sql ================================================ -- update records 11 - 21. Change email and updated_at field update {schema}.seed set updated_at = updated_at + interval '1 hour', email = case when id = 20 then 'pfoxj@creativecommons.org' else 'new_' || email end where id >= 10 and id <= 20; -- invalidate records 11 - 21 update {schema}.snapshot_expected set dbt_valid_to = updated_at + interval '1 hour' where id >= 10 and id <= 20; update {schema}.snapshot_castillo_expected set dbt_valid_to = "1-updated_at" + interval '1 hour' where id >= 10 and id <= 20; update {schema}.snapshot_alvarez_expected set dbt_valid_to = updated_at + interval '1 hour' where id >= 10 and id <= 20; update {schema}.snapshot_kelly_expected set dbt_valid_to = updated_at + interval '1 hour' where id >= 10 and id <= 20; ================================================ FILE: tests/functional/snapshots/data/seed_cn.sql ================================================ create table {database}.{schema}.seed ( id INTEGER, first_name VARCHAR(50), last_name VARCHAR(50), email VARCHAR(50), gender VARCHAR(50), ip_address VARCHAR(20), updated_at TIMESTAMP WITHOUT TIME ZONE ); create table {database}.{schema}.snapshot_expected ( id INTEGER, first_name VARCHAR(50), last_name VARCHAR(50), email VARCHAR(50), gender VARCHAR(50), ip_address VARCHAR(20), -- snapshotting fields updated_at TIMESTAMP WITHOUT TIME ZONE, test_valid_from TIMESTAMP WITHOUT TIME ZONE, test_valid_to TIMESTAMP WITHOUT TIME ZONE, test_scd_id TEXT, test_updated_at TIMESTAMP WITHOUT TIME ZONE ); -- seed inserts -- use the same email for two users to verify that duplicated check_cols values -- are handled appropriately insert into {database}.{schema}.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values (1, 'Judith', 'Kennedy', '(not provided)', 'Female', '54.60.24.128', '2015-12-24 12:19:28'), (2, 'Arthur', 'Kelly', '(not provided)', 'Male', '62.56.24.215', '2015-10-28 16:22:15'), (3, 'Rachel', 'Moreno', 'rmoreno2@msu.edu', 'Female', '31.222.249.23', '2016-04-05 02:05:30'), (4, 'Ralph', 'Turner', 'rturner3@hp.com', 'Male', '157.83.76.114', '2016-08-08 00:06:51'), (5, 'Laura', 'Gonzales', 'lgonzales4@howstuffworks.com', 'Female', '30.54.105.168', '2016-09-01 08:25:38'), (6, 'Katherine', 'Lopez', 'klopez5@yahoo.co.jp', 'Female', '169.138.46.89', '2016-08-30 18:52:11'), (7, 'Jeremy', 'Hamilton', 'jhamilton6@mozilla.org', 'Male', '231.189.13.133', '2016-07-17 02:09:46'), (8, 'Heather', 'Rose', 'hrose7@goodreads.com', 'Female', '87.165.201.65', '2015-12-29 22:03:56'), (9, 'Gregory', 'Kelly', 'gkelly8@trellian.com', 'Male', '154.209.99.7', '2016-03-24 21:18:16'), (10, 'Rachel', 'Lopez', 'rlopez9@themeforest.net', 'Female', '237.165.82.71', '2016-08-20 15:44:49'), (11, 'Donna', 'Welch', 'dwelcha@shutterfly.com', 'Female', '103.33.110.138', '2016-02-27 01:41:48'), (12, 'Russell', 'Lawrence', 'rlawrenceb@qq.com', 'Male', '189.115.73.4', '2016-06-11 03:07:09'), (13, 'Michelle', 'Montgomery', 'mmontgomeryc@scientificamerican.com', 'Female', '243.220.95.82', '2016-06-18 16:27:19'), (14, 'Walter', 'Castillo', 'wcastillod@pagesperso-orange.fr', 'Male', '71.159.238.196', '2016-10-06 01:55:44'), (15, 'Robin', 'Mills', 'rmillse@vkontakte.ru', 'Female', '172.190.5.50', '2016-10-31 11:41:21'), (16, 'Raymond', 'Holmes', 'rholmesf@usgs.gov', 'Male', '148.153.166.95', '2016-10-03 08:16:38'), (17, 'Gary', 'Bishop', 'gbishopg@plala.or.jp', 'Male', '161.108.182.13', '2016-08-29 19:35:20'), (18, 'Anna', 'Riley', 'arileyh@nasa.gov', 'Female', '253.31.108.22', '2015-12-11 04:34:27'), (19, 'Sarah', 'Knight', 'sknighti@foxnews.com', 'Female', '222.220.3.177', '2016-09-26 00:49:06'), (20, 'Phyllis', 'Fox', null, 'Female', '163.191.232.95', '2016-08-21 10:35:19'); -- populate snapshot table insert into {database}.{schema}.snapshot_expected ( id, first_name, last_name, email, gender, ip_address, updated_at, test_valid_from, test_valid_to, test_updated_at, test_scd_id ) select id, first_name, last_name, email, gender, ip_address, updated_at, -- fields added by snapshotting updated_at as test_valid_from, null::timestamp as test_valid_to, updated_at as test_updated_at, md5(id || '-' || first_name || '|' || updated_at::text) as test_scd_id from {database}.{schema}.seed; ================================================ FILE: tests/functional/snapshots/data/seed_dbt_valid_to.sql ================================================ create table {database}.{schema}.seed ( id INTEGER, first_name VARCHAR(50), last_name VARCHAR(50), email VARCHAR(50), gender VARCHAR(50), ip_address VARCHAR(20), updated_at TIMESTAMP WITHOUT TIME ZONE ); create table {database}.{schema}.snapshot_expected ( id INTEGER, first_name VARCHAR(50), last_name VARCHAR(50), email VARCHAR(50), gender VARCHAR(50), ip_address VARCHAR(20), -- snapshotting fields updated_at TIMESTAMP WITHOUT TIME ZONE, test_valid_from TIMESTAMP WITHOUT TIME ZONE, test_valid_to TIMESTAMP WITHOUT TIME ZONE, test_scd_id TEXT, test_updated_at TIMESTAMP WITHOUT TIME ZONE ); -- seed inserts -- use the same email for two users to verify that duplicated check_cols values -- are handled appropriately insert into {database}.{schema}.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values (1, 'Judith', 'Kennedy', '(not provided)', 'Female', '54.60.24.128', '2015-12-24 12:19:28'), (2, 'Arthur', 'Kelly', '(not provided)', 'Male', '62.56.24.215', '2015-10-28 16:22:15'), (3, 'Rachel', 'Moreno', 'rmoreno2@msu.edu', 'Female', '31.222.249.23', '2016-04-05 02:05:30'), (4, 'Ralph', 'Turner', 'rturner3@hp.com', 'Male', '157.83.76.114', '2016-08-08 00:06:51'), (5, 'Laura', 'Gonzales', 'lgonzales4@howstuffworks.com', 'Female', '30.54.105.168', '2016-09-01 08:25:38'), (6, 'Katherine', 'Lopez', 'klopez5@yahoo.co.jp', 'Female', '169.138.46.89', '2016-08-30 18:52:11'), (7, 'Jeremy', 'Hamilton', 'jhamilton6@mozilla.org', 'Male', '231.189.13.133', '2016-07-17 02:09:46'), (8, 'Heather', 'Rose', 'hrose7@goodreads.com', 'Female', '87.165.201.65', '2015-12-29 22:03:56'), (9, 'Gregory', 'Kelly', 'gkelly8@trellian.com', 'Male', '154.209.99.7', '2016-03-24 21:18:16'), (10, 'Rachel', 'Lopez', 'rlopez9@themeforest.net', 'Female', '237.165.82.71', '2016-08-20 15:44:49'), (11, 'Donna', 'Welch', 'dwelcha@shutterfly.com', 'Female', '103.33.110.138', '2016-02-27 01:41:48'), (12, 'Russell', 'Lawrence', 'rlawrenceb@qq.com', 'Male', '189.115.73.4', '2016-06-11 03:07:09'), (13, 'Michelle', 'Montgomery', 'mmontgomeryc@scientificamerican.com', 'Female', '243.220.95.82', '2016-06-18 16:27:19'), (14, 'Walter', 'Castillo', 'wcastillod@pagesperso-orange.fr', 'Male', '71.159.238.196', '2016-10-06 01:55:44'), (15, 'Robin', 'Mills', 'rmillse@vkontakte.ru', 'Female', '172.190.5.50', '2016-10-31 11:41:21'), (16, 'Raymond', 'Holmes', 'rholmesf@usgs.gov', 'Male', '148.153.166.95', '2016-10-03 08:16:38'), (17, 'Gary', 'Bishop', 'gbishopg@plala.or.jp', 'Male', '161.108.182.13', '2016-08-29 19:35:20'), (18, 'Anna', 'Riley', 'arileyh@nasa.gov', 'Female', '253.31.108.22', '2015-12-11 04:34:27'), (19, 'Sarah', 'Knight', 'sknighti@foxnews.com', 'Female', '222.220.3.177', '2016-09-26 00:49:06'), (20, 'Phyllis', 'Fox', null, 'Female', '163.191.232.95', '2016-08-21 10:35:19'); -- populate snapshot table insert into {database}.{schema}.snapshot_expected ( id, first_name, last_name, email, gender, ip_address, updated_at, test_valid_from, test_valid_to, test_updated_at, test_scd_id ) select id, first_name, last_name, email, gender, ip_address, updated_at, -- fields added by snapshotting updated_at as test_valid_from, date('2099-12-31') as test_valid_to, updated_at as test_updated_at, md5(id || '-' || first_name || '|' || updated_at::text) as test_scd_id from {database}.{schema}.seed; ================================================ FILE: tests/functional/snapshots/data/seed_pg.sql ================================================ create table {database}.{schema}.seed ( id INTEGER, first_name VARCHAR(50), last_name VARCHAR(50), email VARCHAR(50), gender VARCHAR(50), ip_address VARCHAR(20), updated_at TIMESTAMP WITHOUT TIME ZONE ); create table {database}.{schema}.snapshot_expected ( id INTEGER, first_name VARCHAR(50), last_name VARCHAR(50), email VARCHAR(50), gender VARCHAR(50), ip_address VARCHAR(20), -- snapshotting fields updated_at TIMESTAMP WITHOUT TIME ZONE, dbt_valid_from TIMESTAMP WITHOUT TIME ZONE, dbt_valid_to TIMESTAMP WITHOUT TIME ZONE, dbt_scd_id TEXT, dbt_updated_at TIMESTAMP WITHOUT TIME ZONE ); -- seed inserts -- use the same email for two users to verify that duplicated check_cols values -- are handled appropriately insert into {database}.{schema}.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values (1, 'Judith', 'Kennedy', '(not provided)', 'Female', '54.60.24.128', '2015-12-24 12:19:28'), (2, 'Arthur', 'Kelly', '(not provided)', 'Male', '62.56.24.215', '2015-10-28 16:22:15'), (3, 'Rachel', 'Moreno', 'rmoreno2@msu.edu', 'Female', '31.222.249.23', '2016-04-05 02:05:30'), (4, 'Ralph', 'Turner', 'rturner3@hp.com', 'Male', '157.83.76.114', '2016-08-08 00:06:51'), (5, 'Laura', 'Gonzales', 'lgonzales4@howstuffworks.com', 'Female', '30.54.105.168', '2016-09-01 08:25:38'), (6, 'Katherine', 'Lopez', 'klopez5@yahoo.co.jp', 'Female', '169.138.46.89', '2016-08-30 18:52:11'), (7, 'Jeremy', 'Hamilton', 'jhamilton6@mozilla.org', 'Male', '231.189.13.133', '2016-07-17 02:09:46'), (8, 'Heather', 'Rose', 'hrose7@goodreads.com', 'Female', '87.165.201.65', '2015-12-29 22:03:56'), (9, 'Gregory', 'Kelly', 'gkelly8@trellian.com', 'Male', '154.209.99.7', '2016-03-24 21:18:16'), (10, 'Rachel', 'Lopez', 'rlopez9@themeforest.net', 'Female', '237.165.82.71', '2016-08-20 15:44:49'), (11, 'Donna', 'Welch', 'dwelcha@shutterfly.com', 'Female', '103.33.110.138', '2016-02-27 01:41:48'), (12, 'Russell', 'Lawrence', 'rlawrenceb@qq.com', 'Male', '189.115.73.4', '2016-06-11 03:07:09'), (13, 'Michelle', 'Montgomery', 'mmontgomeryc@scientificamerican.com', 'Female', '243.220.95.82', '2016-06-18 16:27:19'), (14, 'Walter', 'Castillo', 'wcastillod@pagesperso-orange.fr', 'Male', '71.159.238.196', '2016-10-06 01:55:44'), (15, 'Robin', 'Mills', 'rmillse@vkontakte.ru', 'Female', '172.190.5.50', '2016-10-31 11:41:21'), (16, 'Raymond', 'Holmes', 'rholmesf@usgs.gov', 'Male', '148.153.166.95', '2016-10-03 08:16:38'), (17, 'Gary', 'Bishop', 'gbishopg@plala.or.jp', 'Male', '161.108.182.13', '2016-08-29 19:35:20'), (18, 'Anna', 'Riley', 'arileyh@nasa.gov', 'Female', '253.31.108.22', '2015-12-11 04:34:27'), (19, 'Sarah', 'Knight', 'sknighti@foxnews.com', 'Female', '222.220.3.177', '2016-09-26 00:49:06'), (20, 'Phyllis', 'Fox', null, 'Female', '163.191.232.95', '2016-08-21 10:35:19'); -- populate snapshot table insert into {database}.{schema}.snapshot_expected ( id, first_name, last_name, email, gender, ip_address, updated_at, dbt_valid_from, dbt_valid_to, dbt_updated_at, dbt_scd_id ) select id, first_name, last_name, email, gender, ip_address, updated_at, -- fields added by snapshotting updated_at as dbt_valid_from, null::timestamp as dbt_valid_to, updated_at as dbt_updated_at, md5(id || '-' || first_name || '|' || updated_at::text) as dbt_scd_id from {database}.{schema}.seed; create table {database}.{schema}.snapshot_castillo_expected ( id INTEGER, first_name VARCHAR(50), last_name VARCHAR(50), email VARCHAR(50), gender VARCHAR(50), ip_address VARCHAR(20), -- snapshotting fields "1-updated_at" TIMESTAMP WITHOUT TIME ZONE, dbt_valid_from TIMESTAMP WITHOUT TIME ZONE, dbt_valid_to TIMESTAMP WITHOUT TIME ZONE, dbt_scd_id TEXT, dbt_updated_at TIMESTAMP WITHOUT TIME ZONE ); -- one entry insert into {database}.{schema}.snapshot_castillo_expected ( id, first_name, last_name, email, gender, ip_address, "1-updated_at", dbt_valid_from, dbt_valid_to, dbt_updated_at, dbt_scd_id ) select id, first_name, last_name, email, gender, ip_address, updated_at, -- fields added by snapshotting updated_at as dbt_valid_from, null::timestamp as dbt_valid_to, updated_at as dbt_updated_at, md5(id || '-' || first_name || '|' || updated_at::text) as dbt_scd_id from {database}.{schema}.seed where last_name = 'Castillo'; create table {database}.{schema}.snapshot_alvarez_expected ( id INTEGER, first_name VARCHAR(50), last_name VARCHAR(50), email VARCHAR(50), gender VARCHAR(50), ip_address VARCHAR(20), -- snapshotting fields updated_at TIMESTAMP WITHOUT TIME ZONE, dbt_valid_from TIMESTAMP WITHOUT TIME ZONE, dbt_valid_to TIMESTAMP WITHOUT TIME ZONE, dbt_scd_id TEXT, dbt_updated_at TIMESTAMP WITHOUT TIME ZONE ); -- 0 entries insert into {database}.{schema}.snapshot_alvarez_expected ( id, first_name, last_name, email, gender, ip_address, updated_at, dbt_valid_from, dbt_valid_to, dbt_updated_at, dbt_scd_id ) select id, first_name, last_name, email, gender, ip_address, updated_at, -- fields added by snapshotting updated_at as dbt_valid_from, null::timestamp as dbt_valid_to, updated_at as dbt_updated_at, md5(id || '-' || first_name || '|' || updated_at::text) as dbt_scd_id from {database}.{schema}.seed where last_name = 'Alvarez'; create table {database}.{schema}.snapshot_kelly_expected ( id INTEGER, first_name VARCHAR(50), last_name VARCHAR(50), email VARCHAR(50), gender VARCHAR(50), ip_address VARCHAR(20), -- snapshotting fields updated_at TIMESTAMP WITHOUT TIME ZONE, dbt_valid_from TIMESTAMP WITHOUT TIME ZONE, dbt_valid_to TIMESTAMP WITHOUT TIME ZONE, dbt_scd_id TEXT, dbt_updated_at TIMESTAMP WITHOUT TIME ZONE ); -- 2 entries insert into {database}.{schema}.snapshot_kelly_expected ( id, first_name, last_name, email, gender, ip_address, updated_at, dbt_valid_from, dbt_valid_to, dbt_updated_at, dbt_scd_id ) select id, first_name, last_name, email, gender, ip_address, updated_at, -- fields added by snapshotting updated_at as dbt_valid_from, null::timestamp as dbt_valid_to, updated_at as dbt_updated_at, md5(id || '-' || first_name || '|' || updated_at::text) as dbt_scd_id from {database}.{schema}.seed where last_name = 'Kelly'; ================================================ FILE: tests/functional/snapshots/data/shared_macros.sql ================================================ {% macro get_snapshot_unique_id() -%} {{ return(adapter.dispatch('get_snapshot_unique_id')()) }} {%- endmacro %} {% macro default__get_snapshot_unique_id() -%} {% do return("id || '-' || first_name") %} {%- endmacro %} {# mostly copy+pasted from dbt_utils, but I removed some parameters and added a query that calls get_snapshot_unique_id #} {% test mutually_exclusive_ranges(model) %} with base as ( select {{ get_snapshot_unique_id() }} as dbt_unique_id, * from {{ model }} ), window_functions as ( select dbt_valid_from as lower_bound, coalesce(dbt_valid_to, '2099-1-1T00:00:01') as upper_bound, lead(dbt_valid_from) over ( partition by dbt_unique_id order by dbt_valid_from ) as next_lower_bound, row_number() over ( partition by dbt_unique_id order by dbt_valid_from desc ) = 1 as is_last_record from base ), calc as ( -- We want to return records where one of our assumptions fails, so we'll use -- the `not` function with `and` statements so we can write our assumptions nore cleanly select *, -- For each record: lower_bound should be < upper_bound. -- Coalesce it to return an error on the null case (implicit assumption -- these columns are not_null) coalesce( lower_bound < upper_bound, is_last_record ) as lower_bound_less_than_upper_bound, -- For each record: upper_bound {{ allow_gaps_operator }} the next lower_bound. -- Coalesce it to handle null cases for the last record. coalesce( upper_bound = next_lower_bound, is_last_record, false ) as upper_bound_equal_to_next_lower_bound from window_functions ), validation_errors as ( select * from calc where not( -- THE FOLLOWING SHOULD BE TRUE -- lower_bound_less_than_upper_bound and upper_bound_equal_to_next_lower_bound ) ) select * from validation_errors {% endtest %} ================================================ FILE: tests/functional/snapshots/data/update.sql ================================================ -- insert v2 of the 11 - 21 records insert into {database}.{schema}.snapshot_expected ( id, first_name, last_name, email, gender, ip_address, updated_at, dbt_valid_from, dbt_valid_to, dbt_updated_at, dbt_scd_id ) select id, first_name, last_name, email, gender, ip_address, updated_at, -- fields added by snapshotting updated_at as dbt_valid_from, null::timestamp as dbt_valid_to, updated_at as dbt_updated_at, md5(id || '-' || first_name || '|' || updated_at::text) as dbt_scd_id from {database}.{schema}.seed where id >= 10 and id <= 20; insert into {database}.{schema}.snapshot_castillo_expected ( id, first_name, last_name, email, gender, ip_address, "1-updated_at", dbt_valid_from, dbt_valid_to, dbt_updated_at, dbt_scd_id ) select id, first_name, last_name, email, gender, ip_address, updated_at, -- fields added by snapshotting updated_at as dbt_valid_from, null::timestamp as dbt_valid_to, updated_at as dbt_updated_at, md5(id || '-' || first_name || '|' || updated_at::text) as dbt_scd_id from {database}.{schema}.seed where id >= 10 and id <= 20 and last_name = 'Castillo'; insert into {database}.{schema}.snapshot_alvarez_expected ( id, first_name, last_name, email, gender, ip_address, updated_at, dbt_valid_from, dbt_valid_to, dbt_updated_at, dbt_scd_id ) select id, first_name, last_name, email, gender, ip_address, updated_at, -- fields added by snapshotting updated_at as dbt_valid_from, null::timestamp as dbt_valid_to, updated_at as dbt_updated_at, md5(id || '-' || first_name || '|' || updated_at::text) as dbt_scd_id from {database}.{schema}.seed where id >= 10 and id <= 20 and last_name = 'Alvarez'; insert into {database}.{schema}.snapshot_kelly_expected ( id, first_name, last_name, email, gender, ip_address, updated_at, dbt_valid_from, dbt_valid_to, dbt_updated_at, dbt_scd_id ) select id, first_name, last_name, email, gender, ip_address, updated_at, -- fields added by snapshotting updated_at as dbt_valid_from, null::timestamp as dbt_valid_to, updated_at as dbt_updated_at, md5(id || '-' || first_name || '|' || updated_at::text) as dbt_scd_id from {database}.{schema}.seed where id >= 10 and id <= 20 and last_name = 'Kelly'; -- insert 10 new records insert into {database}.{schema}.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values (21, 'Judy', 'Robinson', 'jrobinsonk@blogs.com', 'Female', '208.21.192.232', '2016-09-18 08:27:38'), (22, 'Kevin', 'Alvarez', 'kalvarezl@buzzfeed.com', 'Male', '228.106.146.9', '2016-07-29 03:07:37'), (23, 'Barbara', 'Carr', 'bcarrm@pen.io', 'Female', '106.165.140.17', '2015-09-24 13:27:23'), (24, 'William', 'Watkins', 'wwatkinsn@guardian.co.uk', 'Male', '78.155.84.6', '2016-03-08 19:13:08'), (25, 'Judy', 'Cooper', 'jcoopero@google.com.au', 'Female', '24.149.123.184', '2016-10-05 20:49:33'), (26, 'Shirley', 'Castillo', 'scastillop@samsung.com', 'Female', '129.252.181.12', '2016-06-20 21:12:21'), (27, 'Justin', 'Harper', 'jharperq@opera.com', 'Male', '131.172.103.218', '2016-05-21 22:56:46'), (28, 'Marie', 'Medina', 'mmedinar@nhs.uk', 'Female', '188.119.125.67', '2015-10-08 13:44:33'), (29, 'Kelly', 'Edwards', 'kedwardss@phoca.cz', 'Female', '47.121.157.66', '2015-09-15 06:33:37'), (30, 'Carl', 'Coleman', 'ccolemant@wikipedia.org', 'Male', '82.227.154.83', '2016-05-26 16:46:40'); -- add these new records to the snapshot table insert into {database}.{schema}.snapshot_expected ( id, first_name, last_name, email, gender, ip_address, updated_at, dbt_valid_from, dbt_valid_to, dbt_updated_at, dbt_scd_id ) select id, first_name, last_name, email, gender, ip_address, updated_at, -- fields added by snapshotting updated_at as dbt_valid_from, null::timestamp as dbt_valid_to, updated_at as dbt_updated_at, md5(id || '-' || first_name || '|' || updated_at::text) as dbt_scd_id from {database}.{schema}.seed where id > 20; -- add these new records to the snapshot table insert into {database}.{schema}.snapshot_castillo_expected ( id, first_name, last_name, email, gender, ip_address, "1-updated_at", dbt_valid_from, dbt_valid_to, dbt_updated_at, dbt_scd_id ) select id, first_name, last_name, email, gender, ip_address, updated_at, -- fields added by snapshotting updated_at as dbt_valid_from, null::timestamp as dbt_valid_to, updated_at as dbt_updated_at, md5(id || '-' || first_name || '|' || updated_at::text) as dbt_scd_id from {database}.{schema}.seed where id > 20 and last_name = 'Castillo'; insert into {database}.{schema}.snapshot_alvarez_expected ( id, first_name, last_name, email, gender, ip_address, updated_at, dbt_valid_from, dbt_valid_to, dbt_updated_at, dbt_scd_id ) select id, first_name, last_name, email, gender, ip_address, updated_at, -- fields added by snapshotting updated_at as dbt_valid_from, null::timestamp as dbt_valid_to, updated_at as dbt_updated_at, md5(id || '-' || first_name || '|' || updated_at::text) as dbt_scd_id from {database}.{schema}.seed where id > 20 and last_name = 'Alvarez'; insert into {database}.{schema}.snapshot_kelly_expected ( id, first_name, last_name, email, gender, ip_address, updated_at, dbt_valid_from, dbt_valid_to, dbt_updated_at, dbt_scd_id ) select id, first_name, last_name, email, gender, ip_address, updated_at, -- fields added by snapshotting updated_at as dbt_valid_from, null::timestamp as dbt_valid_to, updated_at as dbt_updated_at, md5(id || '-' || first_name || '|' || updated_at::text) as dbt_scd_id from {database}.{schema}.seed where id > 20 and last_name = 'Kelly'; ================================================ FILE: tests/functional/snapshots/fixtures.py ================================================ snapshots_select__snapshot_sql = """ {% snapshot snapshot_castillo %} {{ config( target_database=var('target_database', database), target_schema=schema, unique_key='id || ' ~ "'-'" ~ ' || first_name', strategy='timestamp', updated_at='"1-updated_at"', ) }} select id,first_name,last_name,email,gender,ip_address,updated_at as "1-updated_at" from {{target.database}}.{{schema}}.seed where last_name = 'Castillo' {% endsnapshot %} {% snapshot snapshot_alvarez %} {{ config( target_database=var('target_database', database), target_schema=schema, unique_key='id || ' ~ "'-'" ~ ' || first_name', strategy='timestamp', updated_at='updated_at', ) }} select * from {{target.database}}.{{schema}}.seed where last_name = 'Alvarez' {% endsnapshot %} {% snapshot snapshot_kelly %} {# This has no target_database set, which is allowed! #} {{ config( target_schema=schema, unique_key='id || ' ~ "'-'" ~ ' || first_name', strategy='timestamp', updated_at='updated_at', ) }} select * from {{target.database}}.{{schema}}.seed where last_name = 'Kelly' {% endsnapshot %} """ snapshots_pg_custom__snapshot_sql = """ {% snapshot snapshot_actual %} {{ config( target_database=var('target_database', database), target_schema=var('target_schema', schema), unique_key='id || ' ~ "'-'" ~ ' || first_name', strategy='custom', updated_at='updated_at', ) }} select * from {{target.database}}.{{target.schema}}.seed {% endsnapshot %} """ macros_custom_snapshot__custom_sql = """ {# A "custom" strategy that's really just the timestamp one #} {% macro snapshot_custom_strategy(node, snapshotted_rel, current_rel, config, target_exists) %} {% set primary_key = config['unique_key'] %} {% set updated_at = config['updated_at'] %} {% set row_changed_expr -%} ({{ snapshotted_rel }}.{{ updated_at }} < {{ current_rel }}.{{ updated_at }}) {%- endset %} {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %} {% do return({ "unique_key": primary_key, "updated_at": updated_at, "row_changed": row_changed_expr, "scd_id": scd_id_expr }) %} {% endmacro %} """ models__schema_yml = """ snapshots: - name: snapshot_actual data_tests: - mutually_exclusive_ranges config: meta: owner: 'a_owner' """ models__schema_with_target_schema_yml = """ snapshots: - name: snapshot_actual data_tests: - mutually_exclusive_ranges config: meta: owner: 'a_owner' target_schema: schema_from_schema_yml """ models__ref_snapshot_sql = """ select * from {{ ref('snapshot_actual') }} """ macros__test_no_overlaps_sql = """ {% macro get_snapshot_unique_id() -%} {{ return(adapter.dispatch('get_snapshot_unique_id')()) }} {%- endmacro %} {% macro default__get_snapshot_unique_id() -%} {% do return("id || '-' || first_name") %} {%- endmacro %} {# mostly copy+pasted from dbt_utils, but I removed some parameters and added a query that calls get_snapshot_unique_id #} {% test mutually_exclusive_ranges(model) %} with base as ( select {{ get_snapshot_unique_id() }} as dbt_unique_id, * from {{ model }} ), window_functions as ( select dbt_valid_from as lower_bound, coalesce(dbt_valid_to, '2099-1-1T00:00:01') as upper_bound, lead(dbt_valid_from) over ( partition by dbt_unique_id order by dbt_valid_from ) as next_lower_bound, row_number() over ( partition by dbt_unique_id order by dbt_valid_from desc ) = 1 as is_last_record from base ), calc as ( -- We want to return records where one of our assumptions fails, so we'll use -- the `not` function with `and` statements so we can write our assumptions nore cleanly select *, -- For each record: lower_bound should be < upper_bound. -- Coalesce it to return an error on the null case (implicit assumption -- these columns are not_null) coalesce( lower_bound < upper_bound, is_last_record ) as lower_bound_less_than_upper_bound, -- For each record: upper_bound {{ allow_gaps_operator }} the next lower_bound. -- Coalesce it to handle null cases for the last record. coalesce( upper_bound = next_lower_bound, is_last_record, false ) as upper_bound_equal_to_next_lower_bound from window_functions ), validation_errors as ( select * from calc where not( -- THE FOLLOWING SHOULD BE TRUE -- lower_bound_less_than_upper_bound and upper_bound_equal_to_next_lower_bound ) ) select * from validation_errors {% endtest %} """ snapshots_select_noconfig__snapshot_sql = """ {% snapshot snapshot_actual %} {{ config( target_database=var('target_database', database), target_schema=var('target_schema', schema), ) }} select * from {{target.database}}.{{target.schema}}.seed {% endsnapshot %} {% snapshot snapshot_castillo %} {{ config( target_database=var('target_database', database), updated_at='"1-updated_at"', ) }} select id,first_name,last_name,email,gender,ip_address,updated_at as "1-updated_at" from {{target.database}}.{{schema}}.seed where last_name = 'Castillo' {% endsnapshot %} {% snapshot snapshot_alvarez %} {{ config( target_database=var('target_database', database), ) }} select * from {{target.database}}.{{schema}}.seed where last_name = 'Alvarez' {% endsnapshot %} {% snapshot snapshot_kelly %} {# This has no target_database set, which is allowed! #} select * from {{target.database}}.{{schema}}.seed where last_name = 'Kelly' {% endsnapshot %} """ seeds__seed_newcol_csv = """id,first_name,last_name 1,Judith,Kennedy 2,Arthur,Kelly 3,Rachel,Moreno """ seeds__seed_csv = """id,first_name 1,Judith 2,Arthur 3,Rachel """ snapshots_pg_custom_namespaced__snapshot_sql = """ {% snapshot snapshot_actual %} {{ config( target_database=var('target_database', database), target_schema=var('target_schema', schema), unique_key='id || ' ~ "'-'" ~ ' || first_name', strategy='test.custom', updated_at='updated_at', ) }} select * from {{target.database}}.{{target.schema}}.seed {% endsnapshot %} """ snapshots_pg__snapshot_sql = """ {% snapshot snapshot_actual %} {{ config( target_database=var('target_database', database), target_schema=var('target_schema', schema), unique_key='id || ' ~ "'-'" ~ ' || first_name', strategy='timestamp', updated_at='updated_at', ) }} {% if var('invalidate_hard_deletes', 'false') | as_bool %} {{ config(invalidate_hard_deletes=True) }} {% endif %} select * from {{target.database}}.{{target.schema}}.seed {% endsnapshot %} """ snapshots_pg__snapshot_yml = """ snapshots: - name: snapshot_actual relation: "ref('seed')" config: unique_key: "id || '-' || first_name" strategy: timestamp updated_at: updated_at meta: owner: 'a_owner' columns: - name: id data_tests: - not_null """ snapshots_pg__source_snapshot_yml = """ snapshots: - name: snapshot_source relation: "source('information_schema', 'tables')" config: unique_key: "table_schema || '-' || table_name" strategy: check check_cols: all """ snapshots_pg__snapshot_mod_yml = """ snapshots: - name: snapshot_actual relation: "ref('seed')" config: unique_key: "id || '-' || first_name" strategy: timestamp updated_at: updated_at meta: owner: 'b_owner' columns: - name: id data_tests: - not_null """ snapshots_pg__snapshot_no_target_schema_sql = """ {% snapshot snapshot_actual %} {{ config( target_database=var('target_database', database), unique_key='id || ' ~ "'-'" ~ ' || first_name', strategy='timestamp', updated_at='updated_at', ) }} {% if var('invalidate_hard_deletes', 'false') | as_bool %} {{ config(invalidate_hard_deletes=True) }} {% endif %} select * from {{target.database}}.{{target.schema}}.seed {% endsnapshot %} """ models_slow__gen_sql = """ {{ config(materialized='ephemeral') }} /* Generates 50 rows that "appear" to update every second to a query-er. 1 2020-04-21 20:44:00-04 0 2 2020-04-21 20:43:59-04 59 3 2020-04-21 20:43:58-04 58 4 2020-04-21 20:43:57-04 57 .... 1 second later .... 1 2020-04-21 20:44:01-04 1 2 2020-04-21 20:44:00-04 0 3 2020-04-21 20:43:59-04 59 4 2020-04-21 20:43:58-04 58 This view uses pg_sleep(2) to make queries against the view take a non-trivial amount of time Use statement_timestamp() as it changes during a transactions. If we used now() or current_time or similar, then the timestamp of the start of the transaction would be returned instead. */ with gen as ( select id, date_trunc('second', statement_timestamp()) - (interval '1 second' * id) as updated_at from generate_series(1, 10) id ) select id, updated_at, extract(seconds from updated_at)::int as seconds from gen, pg_sleep(2) """ snapshots_longtext__snapshot_sql = """ {% snapshot snapshot_actual %} {{ config( target_database=var('target_database', database), target_schema=schema, unique_key='id', strategy='timestamp', updated_at='updated_at', ) }} select * from {{target.database}}.{{schema}}.super_long {% endsnapshot %} """ snapshots_check_col_noconfig__snapshot_sql = """ {% snapshot snapshot_actual %} select * from {{target.database}}.{{schema}}.seed {% endsnapshot %} {# This should be exactly the same #} {% snapshot snapshot_checkall %} {{ config(check_cols='all') }} select * from {{target.database}}.{{schema}}.seed {% endsnapshot %} """ sources_pg__source_yml = """ sources: - name: information_schema database: dbt schema: information_schema tables: - name: tables """ sources_pg__source_mod_yml = """ sources: - name: information_schema database: dbt schema: information_schema tables: - name: tables description: tables """ ================================================ FILE: tests/functional/snapshots/test_basic_snapshot.py ================================================ import os from datetime import datetime import pytest import pytz from dbt.tests.util import ( check_relations_equal, relation_from_name, run_dbt, update_config_file, write_file, ) from tests.functional.snapshots.fixtures import ( macros__test_no_overlaps_sql, macros_custom_snapshot__custom_sql, models__ref_snapshot_sql, models__schema_with_target_schema_yml, models__schema_yml, seeds__seed_csv, seeds__seed_newcol_csv, snapshots_pg__snapshot_mod_yml, snapshots_pg__snapshot_no_target_schema_sql, snapshots_pg__snapshot_sql, snapshots_pg__snapshot_yml, snapshots_pg__source_snapshot_yml, snapshots_pg_custom__snapshot_sql, snapshots_pg_custom_namespaced__snapshot_sql, sources_pg__source_mod_yml, sources_pg__source_yml, ) snapshots_check_col__snapshot_sql = """ {% snapshot snapshot_actual %} {{ config( target_database=var('target_database', database), target_schema=schema, unique_key='id || ' ~ "'-'" ~ ' || first_name', strategy='check', check_cols=['email'], ) }} select * from {{target.database}}.{{schema}}.seed {% endsnapshot %} {# This should be exactly the same #} {% snapshot snapshot_checkall %} {{ config( target_database=var('target_database', database), target_schema=schema, unique_key='id || ' ~ "'-'" ~ ' || first_name', strategy='check', check_cols='all', ) }} select * from {{target.database}}.{{schema}}.seed {% endsnapshot %} """ snapshots_check_col_noconfig__snapshot_sql = """ {% snapshot snapshot_actual %} select * from {{target.database}}.{{schema}}.seed {% endsnapshot %} {# This should be exactly the same #} {% snapshot snapshot_checkall %} {{ config(check_cols='all') }} select * from {{target.database}}.{{schema}}.seed {% endsnapshot %} """ def snapshot_setup(project, num_snapshot_models=1): path = os.path.join(project.test_data_dir, "seed_pg.sql") project.run_sql_file(path) results = run_dbt(["snapshot"]) assert len(results) == num_snapshot_models run_dbt(["test"]) check_relations_equal(project.adapter, ["snapshot_actual", "snapshot_expected"]) path = os.path.join(project.test_data_dir, "invalidate_postgres.sql") project.run_sql_file(path) path = os.path.join(project.test_data_dir, "update.sql") project.run_sql_file(path) results = run_dbt(["snapshot"]) assert len(results) == num_snapshot_models run_dbt(["test"]) check_relations_equal(project.adapter, ["snapshot_actual", "snapshot_expected"]) def ref_setup(project, num_snapshot_models=1): path = os.path.join(project.test_data_dir, "seed_pg.sql") project.run_sql_file(path) results = run_dbt(["snapshot"]) assert len(results) == num_snapshot_models results = run_dbt(["run"]) assert len(results) == 1 class Basic: @pytest.fixture(scope="class") def snapshots(self): return {"snapshot.sql": snapshots_pg__snapshot_sql} @pytest.fixture(scope="class") def models(self): return { "schema.yml": models__schema_yml, "ref_snapshot.sql": models__ref_snapshot_sql, } @pytest.fixture(scope="class") def macros(self): return {"test_no_overlaps.sql": macros__test_no_overlaps_sql} @pytest.fixture(scope="class") def seeds(self): return {"seed_newcol.csv": seeds__seed_newcol_csv, "seed.csv": seeds__seed_csv} class TestBasicSnapshot(Basic): def test_basic_snapshot(self, project): snapshot_setup(project, num_snapshot_models=1) class TestBasicRef(Basic): def test_basic_ref(self, project): ref_setup(project, num_snapshot_models=1) class TestBasicTargetSchemaConfig(Basic): @pytest.fixture(scope="class") def snapshots(self): return {"snapshot.sql": snapshots_pg__snapshot_no_target_schema_sql} @pytest.fixture(scope="class") def project_config_update(self, unique_schema): return { "snapshots": { "test": { "schema": "alt", } } } def test_target_schema(self, project): manifest = run_dbt(["parse"]) assert len(manifest.nodes) == 5 # ensure that the schema in the snapshot node is the same as target_schema snapshot_id = "snapshot.test.snapshot_actual" snapshot_node = manifest.nodes[snapshot_id] # The schema field be changed by the default "generate_schema_name" # to append an underscore plus the configured schema of "alt". assert snapshot_node.schema == f"{project.test_schema}_alt" assert ( snapshot_node.relation_name == f'"{project.database}"."{project.test_schema}_alt"."snapshot_actual"' ) assert snapshot_node.meta == {"owner": "a_owner"} # write out schema.yml file and check again write_file(models__schema_with_target_schema_yml, "models", "schema.yml") manifest = run_dbt(["parse"]) snapshot_node = manifest.nodes[snapshot_id] assert snapshot_node.schema == "schema_from_schema_yml" class CustomNamespace: @pytest.fixture(scope="class") def snapshots(self): return {"snapshot.sql": snapshots_pg_custom_namespaced__snapshot_sql} @pytest.fixture(scope="class") def macros(self): return { "test_no_overlaps.sql": macros__test_no_overlaps_sql, "custom.sql": macros_custom_snapshot__custom_sql, } @pytest.fixture(scope="class") def models(self): return { "schema.yml": models__schema_yml, "ref_snapshot.sql": models__ref_snapshot_sql, } @pytest.fixture(scope="class") def seeds(self): return {"seed_newcol.csv": seeds__seed_newcol_csv, "seed.csv": seeds__seed_csv} class TestBasicCustomNamespace(CustomNamespace): def test_custom_namespace_snapshot(self, project): snapshot_setup(project, num_snapshot_models=1) class TestRefCustomNamespace(CustomNamespace): def test_custom_namespace_ref(self, project): ref_setup(project, num_snapshot_models=1) class CustomSnapshot: @pytest.fixture(scope="class") def snapshots(self): return {"snapshot.sql": snapshots_pg_custom__snapshot_sql} @pytest.fixture(scope="class") def macros(self): return { "test_no_overlaps.sql": macros__test_no_overlaps_sql, "custom.sql": macros_custom_snapshot__custom_sql, } @pytest.fixture(scope="class") def models(self): return { "schema.yml": models__schema_yml, "ref_snapshot.sql": models__ref_snapshot_sql, } @pytest.fixture(scope="class") def seeds(self): return {"seed_newcol.csv": seeds__seed_newcol_csv, "seed.csv": seeds__seed_csv} class TestBasicCustomSnapshot(CustomSnapshot): def test_custom_snapshot(self, project): snapshot_setup(project, num_snapshot_models=1) class TestRefCustomSnapshot(CustomSnapshot): def test_custom_ref(self, project): ref_setup(project, num_snapshot_models=1) class CheckCols: @pytest.fixture(scope="class") def snapshots(self): return {"snapshot.sql": snapshots_check_col__snapshot_sql} @pytest.fixture(scope="class") def models(self): return { "schema.yml": models__schema_yml, "ref_snapshot.sql": models__ref_snapshot_sql, } @pytest.fixture(scope="class") def macros(self): return {"test_no_overlaps.sql": macros__test_no_overlaps_sql} @pytest.fixture(scope="class") def seeds(self): return {"seed_newcol.csv": seeds__seed_newcol_csv, "seed.csv": seeds__seed_csv} class TestBasicCheckCols(CheckCols): def test_basic_snapshot(self, project): snapshot_setup(project, num_snapshot_models=2) class TestRefCheckCols(CheckCols): def test_check_cols_ref(self, project): ref_setup(project, num_snapshot_models=2) class ConfiguredCheckCols: @pytest.fixture(scope="class") def snapshots(self): return {"snapshot.sql": snapshots_check_col_noconfig__snapshot_sql} @pytest.fixture(scope="class") def models(self): return { "schema.yml": models__schema_yml, "ref_snapshot.sql": models__ref_snapshot_sql, } @pytest.fixture(scope="class") def macros(self): return {"test_no_overlaps.sql": macros__test_no_overlaps_sql} @pytest.fixture(scope="class") def seeds(self): return {"seed_newcol.csv": seeds__seed_newcol_csv, "seed.csv": seeds__seed_csv} @pytest.fixture(scope="class") def project_config_update(self): snapshot_config = { "snapshots": { "test": { "target_schema": "{{ target.schema }}", "unique_key": "id || '-' || first_name", "strategy": "check", "check_cols": ["email"], } } } return snapshot_config class TestBasicConfiguredCheckCols(ConfiguredCheckCols): def test_configured_snapshot(self, project): snapshot_setup(project, num_snapshot_models=2) class TestRefConfiguredCheckCols(ConfiguredCheckCols): def test_configured_ref(self, project): ref_setup(project, num_snapshot_models=2) class UpdatedAtCheckCols: @pytest.fixture(scope="class") def snapshots(self): return {"snapshot.sql": snapshots_check_col_noconfig__snapshot_sql} @pytest.fixture(scope="class") def models(self): return { "schema.yml": models__schema_yml, "ref_snapshot.sql": models__ref_snapshot_sql, } @pytest.fixture(scope="class") def macros(self): return {"test_no_overlaps.sql": macros__test_no_overlaps_sql} @pytest.fixture(scope="class") def seeds(self): return {"seed_newcol.csv": seeds__seed_newcol_csv, "seed.csv": seeds__seed_csv} @pytest.fixture(scope="class") def project_config_update(self): snapshot_config = { "snapshots": { "test": { "target_schema": "{{ target.schema }}", "unique_key": "id || '-' || first_name", "strategy": "check", "check_cols": "all", "updated_at": "updated_at", } } } return snapshot_config class TestBasicUpdatedAtCheckCols(UpdatedAtCheckCols): def test_updated_at_snapshot(self, project): snapshot_setup(project, num_snapshot_models=2) snapshot_expected_relation = relation_from_name(project.adapter, "snapshot_expected") revived_records = project.run_sql( """ select id, updated_at, dbt_valid_from from {} """.format( snapshot_expected_relation ), fetch="all", ) for result in revived_records: # result is a tuple, the updated_at is second and dbt_valid_from is latest assert isinstance(result[1], datetime) assert isinstance(result[2], datetime) assert result[1].replace(tzinfo=pytz.UTC) == result[2].replace(tzinfo=pytz.UTC) class TestRefUpdatedAtCheckCols(UpdatedAtCheckCols): def test_updated_at_ref(self, project): ref_setup(project, num_snapshot_models=2) class BasicYaml(Basic): @pytest.fixture(scope="class") def snapshots(self): """Overrides the same function in Basic to use the YAML method of defining a snapshot.""" return { "snapshot.yml": snapshots_pg__snapshot_yml, "source_snapshot.yml": snapshots_pg__source_snapshot_yml, } @pytest.fixture(scope="class") def models(self): """Overrides the same function in Basic to use a modified version of schema.yml without snapshot config.""" return { "sources.yml": sources_pg__source_yml, "ref_snapshot.sql": models__ref_snapshot_sql, } class TestBasicSnapshotYaml(BasicYaml): def test_basic_snapshot_yaml(self, project): snapshot_setup(project, num_snapshot_models=2) class TestYamlSnapshotCompiledPath(BasicYaml): def test_compiled_path(self, project): """Verify yml-based snapshots get correct compiled paths without doubling.""" manifest = run_dbt(["parse"]) node = manifest.nodes["snapshot.test.snapshot_actual"] assert node.path == "snapshot_actual.sql" assert node.original_file_path == os.path.join("snapshots", "snapshot.yml") run_dbt(["compile"]) expected = os.path.join( project.project_root, "target", "compiled", "test", "snapshots", "snapshot.yml", "snapshot_actual.sql", ) assert os.path.exists(expected), f"Compiled file not found: {expected}" class TestYamlSnapshotPartialParsing(BasicYaml): def test_snapshot_partial_parsing(self, project): manifest = run_dbt(["parse"]) snapshot_id = "snapshot.test.snapshot_actual" assert snapshot_id in manifest.nodes snapshot = manifest.nodes[snapshot_id] assert snapshot.meta["owner"] == "a_owner" # change an unrelated source file write_file(sources_pg__source_mod_yml, "models", "sources.yml") manifest = run_dbt(["parse"]) snapshot = manifest.nodes[snapshot_id] assert snapshot.meta["owner"] == "a_owner" # change snapshot yml file and re-parse write_file(snapshots_pg__snapshot_mod_yml, "snapshots", "snapshot.yml") manifest = run_dbt(["parse"]) snapshot = manifest.nodes[snapshot_id] assert snapshot.meta["owner"] == "b_owner" # modify dbt_project.yml and re-parse config_updates = { "snapshots": { "test": { "+snapshot_meta_column_names": { "dbt_valid_to": "test_valid_to", "dbt_valid_from": "test_valid_from", }, } } } update_config_file(config_updates, "dbt_project.yml") manifest = run_dbt(["parse"]) snapshot = manifest.nodes[snapshot_id] assert snapshot.config.snapshot_meta_column_names.dbt_valid_to == "test_valid_to" ================================================ FILE: tests/functional/snapshots/test_changing_check_cols_snapshot.py ================================================ import pytest from dbt.tests.util import check_relations_equal, run_dbt snapshot_sql = """ {% snapshot snapshot_check_cols_new_column %} {{ config( target_database=database, target_schema=schema, strategy='check', unique_key='id', check_cols=var("check_cols", ['name']), updated_at="'" ~ var("updated_at") ~ "'::timestamp", ) }} {% if var('version') == 1 %} select 1 as id, 'foo' as name {% else %} select 1 as id, 'foo' as name, 'bar' as other {% endif %} {% endsnapshot %} """ expected_csv = """ id,name,other,dbt_scd_id,dbt_updated_at,dbt_valid_from,dbt_valid_to 1,foo,NULL,0d73ad1b216ad884c9f7395d799c912c,2016-07-01 00:00:00.000,2016-07-01 00:00:00.000,2016-07-02 00:00:00.000 1,foo,bar,7df3783934a6a707d51254859260b9ff,2016-07-02 00:00:00.000,2016-07-02 00:00:00.000, """.lstrip() @pytest.fixture(scope="class") def snapshots(): return {"snapshot_check_cols_new_column.sql": snapshot_sql} @pytest.fixture(scope="class") def seeds(): return {"snapshot_check_cols_new_column_expected.csv": expected_csv} @pytest.fixture(scope="class") def project_config_update(): return { "seeds": { "quote_columns": False, "test": { "snapshot_check_cols_new_column_expected": { "+column_types": { "dbt_updated_at": "timestamp without time zone", "dbt_valid_from": "timestamp without time zone", "dbt_valid_to": "timestamp without time zone", }, }, }, }, } def run_check_cols_snapshot_with_schema_change(project, check_cols_override=None): """ Test that snapshots using the "check" strategy and explicit check_cols support adding columns. Approach: 1. Take a snapshot that checks a single non-id column 2. Add a new column to the data 3. Take a snapshot that checks the new non-id column too As long as no error is thrown, then the snapshot was successful """ check_cols = check_cols_override or ["name", "other"] # 1. Create a table that represents the expected data after a series of snapshots vars_dict = {"version": 1, "updated_at": "2016-07-01"} results = run_dbt(["seed", "--show", "--vars", str(vars_dict)]) assert len(results) == 1 # Snapshot 1 # Use only 'name' for check_cols vars_dict = {"version": 1, "check_cols": [check_cols[0]], "updated_at": "2016-07-01"} results = run_dbt(["snapshot", "--vars", str(vars_dict)]) assert len(results) == 1 # Snapshot 2 # Use both 'name' and 'other' for check_cols vars_dict = {"version": 2, "check_cols": check_cols, "updated_at": "2016-07-02"} results = run_dbt(["snapshot", "--vars", str(vars_dict)]) assert len(results) == 1 check_relations_equal( project.adapter, ["snapshot_check_cols_new_column", "snapshot_check_cols_new_column_expected"], compare_snapshot_cols=True, ) # Snapshot 3 # Run it again. Nothing has changed — ensure we don't detect changes vars_dict = {"version": 2, "check_cols": check_cols, "updated_at": "2016-07-02"} results = run_dbt(["snapshot", "--vars", str(vars_dict)]) assert len(results) == 1 check_relations_equal( project.adapter, ["snapshot_check_cols_new_column", "snapshot_check_cols_new_column_expected"], compare_snapshot_cols=True, ) def test_check_cols_snapshot_with_schema_change(project): run_check_cols_snapshot_with_schema_change(project) def test_check_cols_snapshot_with_schema_change_and_mismatched_casing(project): """ Test that this still works if the database-stored version of 'name' + 'other' differs from the user-configured 'NAME' and 'OTHER' """ run_check_cols_snapshot_with_schema_change( project=project, check_cols_override=["NAME", "OTHER"] ) ================================================ FILE: tests/functional/snapshots/test_changing_strategy_snapshot.py ================================================ import pytest from dbt.tests.util import run_dbt from tests.functional.snapshots.fixtures import models_slow__gen_sql test_snapshots_changing_strategy__test_snapshot_sql = """ {# /* Given the repro case for the snapshot build, we'd expect to see both records have color='pink' in their most recent rows. */ #} with expected as ( select 1 as id, 'pink' as color union all select 2 as id, 'pink' as color ), actual as ( select id, color from {{ ref('my_snapshot') }} where color = 'pink' and dbt_valid_to is null ) select * from expected except select * from actual union all select * from actual except select * from expected """ snapshots_changing_strategy__snapshot_sql = """ {# REPRO: 1. Run with check strategy 2. Add a new ts column and run with check strategy 3. Run with timestamp strategy on new ts column Expect: new entry is added for changed rows in (3) #} {% snapshot my_snapshot %} {#--------------- Configuration ------------ #} {{ config( target_schema=schema, unique_key='id' ) }} {% if var('strategy') == 'timestamp' %} {{ config(strategy='timestamp', updated_at='updated_at') }} {% else %} {{ config(strategy='check', check_cols=['color']) }} {% endif %} {#--------------- Test setup ------------ #} {% if var('step') == 1 %} select 1 as id, 'blue' as color union all select 2 as id, 'red' as color {% elif var('step') == 2 %} -- change id=1 color from blue to green -- id=2 is unchanged when using the check strategy select 1 as id, 'green' as color, '2020-01-01'::date as updated_at union all select 2 as id, 'red' as color, '2020-01-01'::date as updated_at {% elif var('step') == 3 %} -- bump timestamp for both records. Expect that after this runs -- using the timestamp strategy, both ids should have the color -- 'pink' in the database. This should be in the future b/c we're -- going to compare to the check timestamp, which will be _now_ select 1 as id, 'pink' as color, (now() + interval '1 day')::date as updated_at union all select 2 as id, 'pink' as color, (now() + interval '1 day')::date as updated_at {% endif %} {% endsnapshot %} """ @pytest.fixture(scope="class") def models(): return {"gen.sql": models_slow__gen_sql} @pytest.fixture(scope="class") def snapshots(): return {"snapshot.sql": snapshots_changing_strategy__snapshot_sql} @pytest.fixture(scope="class") def tests(): return {"test_snapshot.sql": test_snapshots_changing_strategy__test_snapshot_sql} def test_changing_strategy(project): results = run_dbt(["snapshot", "--vars", "{strategy: check, step: 1}"]) assert len(results) == 1 results = run_dbt(["snapshot", "--vars", "{strategy: check, step: 2}"]) assert len(results) == 1 results = run_dbt(["snapshot", "--vars", "{strategy: timestamp, step: 3}"]) assert len(results) == 1 results = run_dbt(["test"]) assert len(results) == 1 ================================================ FILE: tests/functional/snapshots/test_check_cols_snapshot.py ================================================ import pytest from dbt.tests.util import run_dbt snapshot_sql = """ {% snapshot check_cols_cycle %} {{ config( target_database=database, target_schema=schema, unique_key='id', strategy='check', check_cols=['color'] ) }} {% if var('version') == 1 %} select 1 as id, 'red' as color union all select 2 as id, 'green' as color {% elif var('version') == 2 %} select 1 as id, 'blue' as color union all select 2 as id, 'green' as color {% elif var('version') == 3 %} select 1 as id, 'red' as color union all select 2 as id, 'pink' as color {% else %} {% do exceptions.raise_compiler_error("Got bad version: " ~ var('version')) %} {% endif %} {% endsnapshot %} """ snapshot_test_sql = """ with query as ( -- check that the current value for id=1 is red select case when ( select count(*) from {{ ref('check_cols_cycle') }} where id = 1 and color = 'red' and dbt_valid_to is null ) = 1 then 0 else 1 end as failures union all -- check that the previous 'red' value for id=1 is invalidated select case when ( select count(*) from {{ ref('check_cols_cycle') }} where id = 1 and color = 'red' and dbt_valid_to is not null ) = 1 then 0 else 1 end as failures union all -- check that there's only one current record for id=2 select case when ( select count(*) from {{ ref('check_cols_cycle') }} where id = 2 and color = 'pink' and dbt_valid_to is null ) = 1 then 0 else 1 end as failures union all -- check that the previous value for id=2 is represented select case when ( select count(*) from {{ ref('check_cols_cycle') }} where id = 2 and color = 'green' and dbt_valid_to is not null ) = 1 then 0 else 1 end as failures union all -- check that there are 5 records total in the table select case when ( select count(*) from {{ ref('check_cols_cycle') }} ) = 5 then 0 else 1 end as failures ) select * from query where failures = 1 """ @pytest.fixture(scope="class") def snapshots(): return {"my_snapshot.sql": snapshot_sql} @pytest.fixture(scope="class") def tests(): return {"my_test.sql": snapshot_test_sql} def test_snapshots(project): results = run_dbt(["snapshot", "--vars", "version: 1"]) assert len(results) == 1 results = run_dbt(["snapshot", "--vars", "version: 2"]) assert len(results) == 1 results = run_dbt(["snapshot", "--vars", "version: 3"]) assert len(results) == 1 run_dbt(["test", "--select", "test_type:singular", "--vars", "version: 3"]) ================================================ FILE: tests/functional/snapshots/test_check_cols_updated_at_snapshot.py ================================================ import pytest from dbt.tests.util import check_relations_equal, run_dbt snapshot_sql = """ {% snapshot snapshot_check_cols_updated_at_actual %} {{ config( target_database=database, target_schema=schema, unique_key='id', strategy='check', check_cols='all', updated_at="'" ~ var("updated_at") ~ "'::timestamp", ) }} {% if var('version') == 1 %} select 'a' as id, 10 as counter, '2016-01-01T00:00:00Z'::timestamp as timestamp_col union all select 'b' as id, 20 as counter, '2016-01-01T00:00:00Z'::timestamp as timestamp_col {% elif var('version') == 2 %} select 'a' as id, 30 as counter, '2016-01-02T00:00:00Z'::timestamp as timestamp_col union all select 'b' as id, 20 as counter, '2016-01-01T00:00:00Z'::timestamp as timestamp_col union all select 'c' as id, 40 as counter, '2016-01-02T00:00:00Z'::timestamp as timestamp_col {% else %} select 'a' as id, 30 as counter, '2016-01-02T00:00:00Z'::timestamp as timestamp_col union all select 'c' as id, 40 as counter, '2016-01-02T00:00:00Z'::timestamp as timestamp_col {% endif %} {% endsnapshot %} """ expected_csv = """ id,counter,timestamp_col,dbt_scd_id,dbt_updated_at,dbt_valid_from,dbt_valid_to a,10,2016-01-01 00:00:00.000,927354aa091feffd9437ead0bdae7ae1,2016-07-01 00:00:00.000,2016-07-01 00:00:00.000,2016-07-02 00:00:00.000 b,20,2016-01-01 00:00:00.000,40ace4cbf8629f1720ec8a529ed76f8c,2016-07-01 00:00:00.000,2016-07-01 00:00:00.000, a,30,2016-01-02 00:00:00.000,e9133f2b302c50e36f43e770944cec9b,2016-07-02 00:00:00.000,2016-07-02 00:00:00.000, c,40,2016-01-02 00:00:00.000,09d33d35101e788c152f65d0530b6837,2016-07-02 00:00:00.000,2016-07-02 00:00:00.000, """.lstrip() @pytest.fixture(scope="class") def snapshots(): return {"snapshot_check_cols_updated_at_actual.sql": snapshot_sql} @pytest.fixture(scope="class") def seeds(): return {"snapshot_check_cols_updated_at_expected.csv": expected_csv} @pytest.fixture(scope="class") def project_config_update(): return { "seeds": { "quote_columns": False, "test": { "snapshot_check_cols_updated_at_expected": { "+column_types": { "timestamp_col": "timestamp without time zone", "dbt_updated_at": "timestamp without time zone", "dbt_valid_from": "timestamp without time zone", "dbt_valid_to": "timestamp without time zone", }, }, }, }, } def test_snapshots(project): """ Test that the `dbt_updated_at` column reflects the `updated_at` timestamp expression in the config. Approach: 1. Create a table that represents the expected data after a series of snapshots - Use dbt seed to create the expected relation (`snapshot_check_cols_updated_at_expected`) 2. Execute a series of snapshots to create the data - Use a series of (3) dbt snapshot commands to create the actual relation (`snapshot_check_cols_updated_at_actual`) - The logic can switch between 3 different versions of the data (depending on the `version` number) - The `updated_at` value is passed in via `--vars` and cast to a timestamp in the snapshot config 3. Compare the two relations for equality """ # 1. Create a table that represents the expected data after a series of snapshots results = run_dbt(["seed", "--show", "--vars", "{version: 1, updated_at: 2016-07-01}"]) assert len(results) == 1 # 2. Execute a series of snapshots to create the data # Snapshot day 1 results = run_dbt(["snapshot", "--vars", "{version: 1, updated_at: 2016-07-01}"]) assert len(results) == 1 # Snapshot day 2 results = run_dbt(["snapshot", "--vars", "{version: 2, updated_at: 2016-07-02}"]) assert len(results) == 1 # Snapshot day 3 results = run_dbt(["snapshot", "--vars", "{version: 3, updated_at: 2016-07-03}"]) assert len(results) == 1 # 3. Compare the two relations for equality check_relations_equal( project.adapter, ["snapshot_check_cols_updated_at_actual", "snapshot_check_cols_updated_at_expected"], compare_snapshot_cols=True, ) ================================================ FILE: tests/functional/snapshots/test_comment_ending_snapshot.py ================================================ import os import pytest from dbt.tests.util import run_dbt snapshots_with_comment_at_end__snapshot_sql = """ {% snapshot snapshot_actual %} {{ config( target_database=var('target_database', database), target_schema=schema, unique_key='id', strategy='check', check_cols=['email'], ) }} select * from {{target.database}}.{{schema}}.seed -- Test comment to prevent recurrence of https://github.com/dbt-labs/dbt-core/issues/6781 {% endsnapshot %} """ class TestSnapshotsWithCommentAtEnd: @pytest.fixture(scope="class") def snapshots(self): return {"snapshot.sql": snapshots_with_comment_at_end__snapshot_sql} def test_comment_ending(self, project): path = os.path.join(project.test_data_dir, "seed_pg.sql") project.run_sql_file(path) # N.B. Snapshot is run twice to ensure snapshot_check_all_get_existing_columns is fully run # (it exits early if the table doesn't already exist) run_dbt(["snapshot"]) results = run_dbt(["snapshot"]) assert len(results) == 1 ================================================ FILE: tests/functional/snapshots/test_cross_schema_snapshot.py ================================================ import os import pytest from dbt.tests.util import run_dbt from tests.functional.snapshots.fixtures import ( macros__test_no_overlaps_sql, models__ref_snapshot_sql, models__schema_yml, snapshots_pg__snapshot_sql, ) NUM_SNAPSHOT_MODELS = 1 @pytest.fixture(scope="class") def snapshots(): return {"snapshot.sql": snapshots_pg__snapshot_sql} @pytest.fixture(scope="class") def models(): return { "schema.yml": models__schema_yml, "ref_snapshot.sql": models__ref_snapshot_sql, } @pytest.fixture(scope="class") def macros(): return {"test_no_overlaps.sql": macros__test_no_overlaps_sql} def test_cross_schema_snapshot(project): # populate seed and snapshot tables path = os.path.join(project.test_data_dir, "seed_pg.sql") project.run_sql_file(path) target_schema = "{}_snapshotted".format(project.test_schema) # create a snapshot using the new schema results = run_dbt(["snapshot", "--vars", '{{"target_schema": "{}"}}'.format(target_schema)]) assert len(results) == NUM_SNAPSHOT_MODELS # run dbt from test_schema with a ref to to new target_schema results = run_dbt(["run", "--vars", '{{"target_schema": {}}}'.format(target_schema)]) assert len(results) == 1 ================================================ FILE: tests/functional/snapshots/test_hard_delete_snapshot.py ================================================ import os from datetime import datetime, timedelta import pytest import pytz from dbt.tests.util import check_relations_equal, run_dbt from tests.functional.snapshots.fixtures import ( macros__test_no_overlaps_sql, models__ref_snapshot_sql, models__schema_yml, snapshots_pg__snapshot_sql, ) from tests.functional.utils import is_aware # These tests uses the same seed data, containing 20 records of which we hard delete the last 10. # These deleted records set the dbt_valid_to to time the snapshot was ran. def convert_to_aware(d: datetime) -> datetime: # There are two types of datetime objects in Python: naive and aware # Add timezone info for consistent comparison if d is None: return d elif is_aware(d): return d else: # Naive datetime from database - add timezone info for comparison # Note: The timestamp is actually in the database's local timezone, # but we add UTC tzinfo for consistency since both test and snapshot # timestamps are treated the same way return d.replace(tzinfo=pytz.UTC) def is_close_datetime( dt1: datetime, dt2: datetime, atol: timedelta = timedelta(microseconds=1) ) -> bool: # Similar to pytest.approx, math.isclose, and numpy.isclose # Use an absolute tolerance to compare datetimes that may not be perfectly equal. # Two None values will compare as equal. if dt1 is None and dt2 is None: return True elif dt1 is not None and dt2 is not None: return (dt1 > (dt2 - atol)) and (dt1 < (dt2 + atol)) else: return False def datetime_snapshot(project): NUM_SNAPSHOT_MODELS = 1 # Get the timestamp from the database instead of Python to ensure timezone consistency # between local runs and CI runs # Use the same timestamp format that dbt uses for snapshots: timestamp without time zone begin_snapshot_datetime = project.run_sql( "select current_timestamp::timestamp without time zone", fetch="one" )[0] results = run_dbt(["snapshot", "--vars", "{invalidate_hard_deletes: true}"]) assert len(results) == NUM_SNAPSHOT_MODELS return convert_to_aware(begin_snapshot_datetime) @pytest.fixture(scope="class", autouse=True) def setUp(project): path = os.path.join(project.test_data_dir, "seed_pg.sql") project.run_sql_file(path) @pytest.fixture(scope="class") def snapshots(): return {"snapshot.sql": snapshots_pg__snapshot_sql} @pytest.fixture(scope="class") def models(): return { "schema.yml": models__schema_yml, "ref_snapshot.sql": models__ref_snapshot_sql, } @pytest.fixture(scope="class") def macros(): return {"test_no_overlaps.sql": macros__test_no_overlaps_sql} def test_snapshot_hard_delete(project): # run the first snapshot datetime_snapshot(project) check_relations_equal(project.adapter, ["snapshot_expected", "snapshot_actual"]) invalidated_snapshot_datetime = None revived_snapshot_datetime = None # hard delete last 10 records project.run_sql( "delete from {}.{}.seed where id >= 10;".format(project.database, project.test_schema) ) # snapshot and assert invalidated invalidated_snapshot_datetime = datetime_snapshot(project) snapshotted = project.run_sql( """ select id, dbt_valid_to from {}.{}.snapshot_actual order by id """.format( project.database, project.test_schema ), fetch="all", ) assert len(snapshotted) == 20 for result in snapshotted[10:]: # result is a tuple, the dbt_valid_to column is the latest assert isinstance(result[-1], datetime) dbt_valid_to = convert_to_aware(result[-1]) # Plenty of wiggle room if clocks aren't perfectly sync'd, etc assert is_close_datetime( dbt_valid_to, invalidated_snapshot_datetime, timedelta(minutes=1) ), f"SQL timestamp {dbt_valid_to.isoformat()} is not close enough to test timestamp {invalidated_snapshot_datetime.isoformat()}" # revive records # Timestamp must have microseconds for tests below to be meaningful revival_timestamp_dt = project.run_sql( "select current_timestamp::timestamp without time zone", fetch="one" )[0] revival_timestamp = revival_timestamp_dt.strftime("%Y-%m-%d %H:%M:%S.%f") project.run_sql( """ insert into {}.{}.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values (10, 'Rachel', 'Lopez', 'rlopez9@themeforest.net', 'Female', '237.165.82.71', '{}'), (11, 'Donna', 'Welch', 'dwelcha@shutterfly.com', 'Female', '103.33.110.138', '{}') """.format( project.database, project.test_schema, revival_timestamp, revival_timestamp ) ) # snapshot and assert records were revived # Note: the revived_snapshot_datetime here is later than the revival_timestamp above revived_snapshot_datetime = datetime_snapshot(project) # records which weren't revived (id != 10, 11) # dbt_valid_to is not null invalidated_records = project.run_sql( """ select id, dbt_valid_to from {}.{}.snapshot_actual where dbt_valid_to is not null order by id """.format( project.database, project.test_schema ), fetch="all", ) assert len(invalidated_records) == 11 for result in invalidated_records: # result is a tuple, the dbt_valid_to column is the latest assert isinstance(result[1], datetime) dbt_valid_to = convert_to_aware(result[1]) # Plenty of wiggle room if clocks aren't perfectly sync'd, etc assert is_close_datetime( dbt_valid_to, invalidated_snapshot_datetime, timedelta(minutes=1) ), f"SQL timestamp {dbt_valid_to.isoformat()} is not close enough to test timestamp {invalidated_snapshot_datetime.isoformat()}" # records which were revived (id = 10, 11) # dbt_valid_to is null revived_records = project.run_sql( """ select id, dbt_valid_from, dbt_valid_to from {}.{}.snapshot_actual where dbt_valid_to is null and id IN (10, 11) """.format( project.database, project.test_schema ), fetch="all", ) assert len(revived_records) == 2 for result in revived_records: # result is a tuple, the dbt_valid_from is second and dbt_valid_to is latest # dbt_valid_from is the same as the 'updated_at' added in the revived_rows # dbt_valid_to is null assert isinstance(result[1], datetime) dbt_valid_from = convert_to_aware(result[1]) dbt_valid_to = result[2] assert dbt_valid_from <= revived_snapshot_datetime assert dbt_valid_to is None ================================================ FILE: tests/functional/snapshots/test_invalid_namespace_snapshot.py ================================================ import os import pytest from dbt.tests.util import run_dbt from tests.functional.snapshots.fixtures import ( macros__test_no_overlaps_sql, macros_custom_snapshot__custom_sql, models__ref_snapshot_sql, models__schema_yml, seeds__seed_csv, seeds__seed_newcol_csv, ) NUM_SNAPSHOT_MODELS = 1 snapshots_pg_custom_invalid__snapshot_sql = """ {% snapshot snapshot_actual %} {# this custom strategy does not exist in the 'dbt' package #} {{ config( target_database=var('target_database', database), target_schema=var('target_schema', schema), unique_key='id || ' ~ "'-'" ~ ' || first_name', strategy='dbt.custom', updated_at='updated_at', ) }} select * from {{target.database}}.{{target.schema}}.seed {% endsnapshot %} """ @pytest.fixture(scope="class") def snapshots(): return {"snapshots.sql": snapshots_pg_custom_invalid__snapshot_sql} @pytest.fixture(scope="class") def macros(): return { "test_no_overlaps.sql": macros__test_no_overlaps_sql, "custom.sql": macros_custom_snapshot__custom_sql, } @pytest.fixture(scope="class") def models(): return { "schema.yml": models__schema_yml, "ref_snapshot.sql": models__ref_snapshot_sql, } @pytest.fixture(scope="class") def seeds(): return {"seed_newcol.csv": seeds__seed_newcol_csv, "seed.csv": seeds__seed_csv} def test_custom_snapshot_invalid_namespace(project): path = os.path.join(project.test_data_dir, "seed_pg.sql") project.run_sql_file(path) results = run_dbt(["snapshot"], expect_pass=False) assert len(results) == NUM_SNAPSHOT_MODELS ================================================ FILE: tests/functional/snapshots/test_long_text_snapshot.py ================================================ import pytest from dbt.tests.util import run_dbt from tests.functional.snapshots.fixtures import ( macros__test_no_overlaps_sql, models__ref_snapshot_sql, models__schema_yml, ) seed_longtext_sql = """ create table {database}.{schema}.super_long ( id INTEGER, longstring TEXT, updated_at TIMESTAMP WITHOUT TIME ZONE ); insert into {database}.{schema}.super_long (id, longstring, updated_at) VALUES (1, 'short', current_timestamp), (2, repeat('a', 500), current_timestamp); """ snapshots_longtext__snapshot_sql = """ {% snapshot snapshot_actual %} {{ config( target_database=var('target_database', database), target_schema=schema, unique_key='id', strategy='timestamp', updated_at='updated_at', ) }} select * from {{target.database}}.{{schema}}.super_long {% endsnapshot %} """ @pytest.fixture(scope="class") def snapshots(): return {"snapshot.sql": snapshots_longtext__snapshot_sql} @pytest.fixture(scope="class") def models(): return { "schema.yml": models__schema_yml, "ref_snapshot.sql": models__ref_snapshot_sql, } @pytest.fixture(scope="class") def macros(): return {"test_no_overlaps.sql": macros__test_no_overlaps_sql} def test_long_text(project): project.run_sql(seed_longtext_sql) results = run_dbt(["snapshot"]) assert len(results) == 1 with project.adapter.connection_named("test"): status, results = project.adapter.execute( "select * from {}.{}.snapshot_actual".format(project.database, project.test_schema), fetch=True, ) assert len(results) == 2 got_names = set(r.get("longstring") for r in results) assert got_names == {"a" * 500, "short"} ================================================ FILE: tests/functional/snapshots/test_missing_strategy_snapshot.py ================================================ import pytest from dbt.tests.util import run_dbt from dbt_common.dataclass_schema import ValidationError from tests.functional.snapshots.fixtures import ( macros__test_no_overlaps_sql, models__ref_snapshot_sql, models__schema_yml, ) snapshots_invalid__snapshot_sql = """ {% snapshot snapshot_actual %} {# missing the mandatory strategy parameter #} {{ config( unique_key='id || ' ~ "'-'" ~ ' || first_name', updated_at='updated_at', ) }} select * from {{target.database}}.{{schema}}.seed {% endsnapshot %} """ @pytest.fixture(scope="class") def snapshots(): return {"snapshot.sql": snapshots_invalid__snapshot_sql} @pytest.fixture(scope="class") def models(): return { "schema.yml": models__schema_yml, "ref_snapshot.sql": models__ref_snapshot_sql, } @pytest.fixture(scope="class") def macros(): return {"test_no_overlaps.sql": macros__test_no_overlaps_sql} def test_missing_strategy(project): with pytest.raises(ValidationError) as exc: run_dbt(["compile"], expect_pass=False) assert "Snapshots must be configured with a 'strategy' and 'unique_key'" in str(exc.value) ================================================ FILE: tests/functional/snapshots/test_renamed_source_snapshot.py ================================================ import pytest from dbt.tests.util import run_dbt from tests.functional.snapshots.fixtures import ( macros__test_no_overlaps_sql, macros_custom_snapshot__custom_sql, seeds__seed_csv, seeds__seed_newcol_csv, ) snapshots_checkall__snapshot_sql = """ {% snapshot my_snapshot %} {{ config(check_cols='all', unique_key='id', strategy='check', target_database=database, target_schema=schema) }} select * from {{ ref(var('seed_name', 'seed')) }} {% endsnapshot %} """ @pytest.fixture(scope="class") def snapshots(): return {"snapshot.sql": snapshots_checkall__snapshot_sql} @pytest.fixture(scope="class") def macros(): return { "test_no_overlaps.sql": macros__test_no_overlaps_sql, "custom.sql": macros_custom_snapshot__custom_sql, } @pytest.fixture(scope="class") def seeds(): return {"seed_newcol.csv": seeds__seed_newcol_csv, "seed.csv": seeds__seed_csv} def test_renamed_source(project): run_dbt(["seed"]) run_dbt(["snapshot"]) database = project.database results = project.run_sql( "select * from {}.{}.my_snapshot".format(database, project.test_schema), fetch="all", ) assert len(results) == 3 for result in results: assert len(result) == 6 # over ride the ref var in the snapshot definition to use a seed with an additional column, last_name run_dbt(["snapshot", "--vars", "{seed_name: seed_newcol}"]) results = project.run_sql( "select * from {}.{}.my_snapshot where last_name is not NULL".format( database, project.test_schema ), fetch="all", ) assert len(results) == 3 for result in results: # new column assert len(result) == 7 assert result[-1] is not None results = project.run_sql( "select * from {}.{}.my_snapshot where last_name is NULL".format( database, project.test_schema ), fetch="all", ) assert len(results) == 3 for result in results: # new column assert len(result) == 7 ================================================ FILE: tests/functional/snapshots/test_select_exclude_snapshot.py ================================================ import os import pytest from dbt.tests.util import check_relations_equal, check_table_does_not_exist, run_dbt from tests.functional.snapshots.fixtures import ( macros__test_no_overlaps_sql, models__ref_snapshot_sql, models__schema_yml, seeds__seed_csv, seeds__seed_newcol_csv, snapshots_pg__snapshot_sql, snapshots_select__snapshot_sql, snapshots_select_noconfig__snapshot_sql, ) def all_snapshots(project): path = os.path.join(project.test_data_dir, "seed_pg.sql") project.run_sql_file(path) results = run_dbt(["snapshot"]) assert len(results) == 4 check_relations_equal(project.adapter, ["snapshot_castillo", "snapshot_castillo_expected"]) check_relations_equal(project.adapter, ["snapshot_alvarez", "snapshot_alvarez_expected"]) check_relations_equal(project.adapter, ["snapshot_kelly", "snapshot_kelly_expected"]) check_relations_equal(project.adapter, ["snapshot_actual", "snapshot_expected"]) path = os.path.join(project.test_data_dir, "invalidate_postgres.sql") project.run_sql_file(path) path = os.path.join(project.test_data_dir, "update.sql") project.run_sql_file(path) results = run_dbt(["snapshot"]) assert len(results) == 4 check_relations_equal(project.adapter, ["snapshot_castillo", "snapshot_castillo_expected"]) check_relations_equal(project.adapter, ["snapshot_alvarez", "snapshot_alvarez_expected"]) check_relations_equal(project.adapter, ["snapshot_kelly", "snapshot_kelly_expected"]) check_relations_equal(project.adapter, ["snapshot_actual", "snapshot_expected"]) def exclude_snapshots(project): path = os.path.join(project.test_data_dir, "seed_pg.sql") project.run_sql_file(path) results = run_dbt(["snapshot", "--exclude", "snapshot_castillo"]) assert len(results) == 3 check_table_does_not_exist(project.adapter, "snapshot_castillo") check_relations_equal(project.adapter, ["snapshot_alvarez", "snapshot_alvarez_expected"]) check_relations_equal(project.adapter, ["snapshot_kelly", "snapshot_kelly_expected"]) check_relations_equal(project.adapter, ["snapshot_actual", "snapshot_expected"]) def select_snapshots(project): path = os.path.join(project.test_data_dir, "seed_pg.sql") project.run_sql_file(path) results = run_dbt(["snapshot", "--select", "snapshot_castillo"]) assert len(results) == 1 check_relations_equal(project.adapter, ["snapshot_castillo", "snapshot_castillo_expected"]) check_table_does_not_exist(project.adapter, "snapshot_alvarez") check_table_does_not_exist(project.adapter, "snapshot_kelly") check_table_does_not_exist(project.adapter, "snapshot_actual") # all of the tests below use one of both of the above tests with # various combinations of snapshots and macros class SelectBasicSetup: @pytest.fixture(scope="class") def snapshots(self): return { "snapshot.sql": snapshots_pg__snapshot_sql, "snapshot_select.sql": snapshots_select__snapshot_sql, } @pytest.fixture(scope="class") def seeds(self): return {"seed_newcol.csv": seeds__seed_newcol_csv, "seed.csv": seeds__seed_csv} @pytest.fixture(scope="class") def models(self): return { "schema.yml": models__schema_yml, "ref_snapshot.sql": models__ref_snapshot_sql, } @pytest.fixture(scope="class") def macros(self): return {"test_no_overlaps.sql": macros__test_no_overlaps_sql} class TestAllBasic(SelectBasicSetup): def test_all_snapshots(self, project): all_snapshots(project) class TestExcludeBasic(SelectBasicSetup): def test_exclude_snapshots(self, project): exclude_snapshots(project) class TestSelectBasic(SelectBasicSetup): def test_select_snapshots(self, project): select_snapshots(project) class SelectConfiguredSetup: @pytest.fixture(scope="class") def snapshots(self): return {"snapshot.sql": snapshots_select_noconfig__snapshot_sql} @pytest.fixture(scope="class") def seeds(self): return {"seed_newcol.csv": seeds__seed_newcol_csv, "seed.csv": seeds__seed_csv} @pytest.fixture(scope="class") def models(self): return { "schema.yml": models__schema_yml, "ref_snapshot.sql": models__ref_snapshot_sql, } @pytest.fixture(scope="class") def macros(self): return {"test_no_overlaps.sql": macros__test_no_overlaps_sql} # TODO: don't have access to project here so this breaks @pytest.fixture(scope="class") def project_config_update(self): snapshot_config = { "snapshots": { "test": { "target_schema": "{{ target.schema }}", "unique_key": "id || '-' || first_name", "strategy": "timestamp", "updated_at": "updated_at", } } } return snapshot_config class TestConfigured(SelectConfiguredSetup): def test_all_configured_snapshots(self, project): all_snapshots(project) class TestConfiguredExclude(SelectConfiguredSetup): def test_exclude_configured_snapshots(self, project): exclude_snapshots(project) class TestConfiguredSelect(SelectConfiguredSetup): def test_select_configured_snapshots(self, project): select_snapshots(project) ================================================ FILE: tests/functional/snapshots/test_slow_query_snapshot.py ================================================ import pytest from dbt.tests.util import run_dbt from tests.functional.snapshots.fixtures import models_slow__gen_sql snapshots_slow__snapshot_sql = """ {% snapshot my_slow_snapshot %} {{ config( target_database=var('target_database', database), target_schema=schema, unique_key='id', strategy='timestamp', updated_at='updated_at' ) }} select id, updated_at, seconds from {{ ref('gen') }} {% endsnapshot %} """ test_snapshots_slow__test_timestamps_sql = """ /* Assert that the dbt_valid_from of the latest record is equal to the dbt_valid_to of the previous record */ with snapshot as ( select * from {{ ref('my_slow_snapshot') }} ) select snap1.id, snap1.dbt_valid_from as new_valid_from, snap2.dbt_valid_from as old_valid_from, snap2.dbt_valid_to as old_valid_to from snapshot as snap1 join snapshot as snap2 on snap1.id = snap2.id where snap1.dbt_valid_to is null and snap2.dbt_valid_to is not null and snap1.dbt_valid_from != snap2.dbt_valid_to """ @pytest.fixture(scope="class") def models(): return {"gen.sql": models_slow__gen_sql} @pytest.fixture(scope="class") def snapshots(): return {"snapshot.sql": snapshots_slow__snapshot_sql} @pytest.fixture(scope="class") def tests(): return {"test_timestamps.sql": test_snapshots_slow__test_timestamps_sql} def test_slow(project): results = run_dbt(["snapshot"]) assert len(results) == 1 results = run_dbt(["snapshot"]) assert len(results) == 1 results = run_dbt(["test"]) assert len(results) == 1 ================================================ FILE: tests/functional/snapshots/test_snapshot_column_names.py ================================================ import os import pytest from dbt.tests.util import ( check_relations_equal, get_manifest, run_dbt, run_dbt_and_capture, update_config_file, ) snapshot_actual_sql = """ {% snapshot snapshot_actual %} {{ config( unique_key='id || ' ~ "'-'" ~ ' || first_name', ) }} select * from {{target.database}}.{{target.schema}}.seed {% endsnapshot %} """ snapshots_yml = """ snapshots: - name: snapshot_actual config: strategy: timestamp updated_at: updated_at snapshot_meta_column_names: dbt_valid_to: test_valid_to dbt_valid_from: test_valid_from dbt_scd_id: test_scd_id dbt_updated_at: test_updated_at """ snapshots_no_column_names_yml = """ snapshots: - name: snapshot_actual config: strategy: timestamp updated_at: updated_at """ ref_snapshot_sql = """ select * from {{ ref('snapshot_actual') }} """ invalidate_sql = """ -- update records 11 - 21. Change email and updated_at field update {schema}.seed set updated_at = updated_at + interval '1 hour', email = case when id = 20 then 'pfoxj@creativecommons.org' else 'new_' || email end where id >= 10 and id <= 20; -- invalidate records 11 - 21 update {schema}.snapshot_expected set test_valid_to = updated_at + interval '1 hour' where id >= 10 and id <= 20; """ update_sql = """ -- insert v2 of the 11 - 21 records insert into {database}.{schema}.snapshot_expected ( id, first_name, last_name, email, gender, ip_address, updated_at, test_valid_from, test_valid_to, test_updated_at, test_scd_id ) select id, first_name, last_name, email, gender, ip_address, updated_at, -- fields added by snapshotting updated_at as test_valid_from, null::timestamp as test_valid_to, updated_at as test_updated_at, md5(id || '-' || first_name || '|' || updated_at::text) as test_scd_id from {database}.{schema}.seed where id >= 10 and id <= 20; """ class TestSnapshotColumnNames: @pytest.fixture(scope="class") def snapshots(self): return {"snapshot.sql": snapshot_actual_sql} @pytest.fixture(scope="class") def models(self): return { "snapshots.yml": snapshots_yml, "ref_snapshot.sql": ref_snapshot_sql, } def test_snapshot_column_names(self, project): path = os.path.join(project.test_data_dir, "seed_cn.sql") project.run_sql_file(path) results = run_dbt(["snapshot"]) assert len(results) == 1 project.run_sql(invalidate_sql) project.run_sql(update_sql) results = run_dbt(["snapshot"]) assert len(results) == 1 # run_dbt(["test"]) check_relations_equal(project.adapter, ["snapshot_actual", "snapshot_expected"]) class TestSnapshotColumnNamesFromDbtProject: @pytest.fixture(scope="class") def snapshots(self): return {"snapshot.sql": snapshot_actual_sql} @pytest.fixture(scope="class") def models(self): return { "snapshots.yml": snapshots_no_column_names_yml, "ref_snapshot.sql": ref_snapshot_sql, } @pytest.fixture(scope="class") def project_config_update(self): return { "snapshots": { "test": { "+snapshot_meta_column_names": { "dbt_valid_to": "test_valid_to", "dbt_valid_from": "test_valid_from", "dbt_scd_id": "test_scd_id", "dbt_updated_at": "test_updated_at", } } } } def test_snapshot_column_names_from_project(self, project): path = os.path.join(project.test_data_dir, "seed_cn.sql") project.run_sql_file(path) results = run_dbt(["snapshot"]) assert len(results) == 1 project.run_sql(invalidate_sql) project.run_sql(update_sql) results = run_dbt(["snapshot"]) assert len(results) == 1 # run_dbt(["test"]) check_relations_equal(project.adapter, ["snapshot_actual", "snapshot_expected"]) class TestSnapshotInvalidColumnNames: @pytest.fixture(scope="class") def snapshots(self): return {"snapshot.sql": snapshot_actual_sql} @pytest.fixture(scope="class") def models(self): return { "snapshots.yml": snapshots_no_column_names_yml, "ref_snapshot.sql": ref_snapshot_sql, } @pytest.fixture(scope="class") def project_config_update(self): return { "snapshots": { "test": { "+snapshot_meta_column_names": { "dbt_valid_to": "test_valid_to", "dbt_valid_from": "test_valid_from", "dbt_scd_id": "test_scd_id", "dbt_updated_at": "test_updated_at", } } } } def test_snapshot_invalid_column_names(self, project): path = os.path.join(project.test_data_dir, "seed_cn.sql") project.run_sql_file(path) results = run_dbt(["snapshot"]) assert len(results) == 1 manifest = get_manifest(project.project_root) snapshot_node = manifest.nodes["snapshot.test.snapshot_actual"] snapshot_node.config.snapshot_meta_column_names == { "dbt_valid_to": "test_valid_to", "dbt_valid_from": "test_valid_from", "dbt_scd_id": "test_scd_id", "dbt_updated_at": "test_updated_at", } project.run_sql(invalidate_sql) project.run_sql(update_sql) # Change snapshot_meta_columns and look for an error different_columns = { "snapshots": { "test": { "+snapshot_meta_column_names": { "dbt_valid_to": "test_valid_to", "dbt_updated_at": "test_updated_at", } } } } update_config_file(different_columns, "dbt_project.yml") results, log_output = run_dbt_and_capture(["snapshot"], expect_pass=False) assert len(results) == 1 assert "Compilation Error in snapshot snapshot_actual" in log_output assert "Snapshot target is missing configured columns" in log_output ================================================ FILE: tests/functional/snapshots/test_snapshot_config.py ================================================ import pytest from dbt.tests.util import run_dbt, write_file orders_sql = """ select 1 as id, 101 as user_id, 'pending' as status """ snapshot_sql = """ {% snapshot orders_snapshot %} {{ config( target_schema=schema, strategy='check', unique_key='id', check_cols=['status'], ) }} select * from {{ ref('orders') }} {% endsnapshot %} """ snapshot_no_config_sql = """ {% snapshot orders_snapshot %} select * from {{ ref('orders') }} {% endsnapshot %} """ snapshot_schema_yml = """ snapshots: - name: orders_snapshot config: target_schema: test strategy: check unique_key: id check_cols: ['status'] """ class TestSnapshotConfig: @pytest.fixture(scope="class") def models(self): return {"orders.sql": orders_sql} @pytest.fixture(scope="class") def snapshots(self): return {"snapshot_orders.sql": snapshot_sql} def test_config(self, project): run_dbt(["run"]) results = run_dbt(["snapshot"]) assert len(results) == 1 # try to parse with config in schema file write_file( snapshot_no_config_sql, project.project_root, "snapshots", "snapshot_orders.sql" ) write_file(snapshot_schema_yml, project.project_root, "snapshots", "snapshot.yml") results = run_dbt(["parse"]) results = run_dbt(["snapshot"]) assert len(results) == 1 ================================================ FILE: tests/functional/snapshots/test_snapshot_empty.py ================================================ import pytest from dbt.tests.util import run_dbt my_model_sql = """ select 1 as id, {{ dbt.current_timestamp() }} as updated_at """ snapshots_yml = """ snapshots: - name: my_snapshot relation: "ref('my_model')" config: unique_key: id strategy: check check_cols: all dbt_valid_to_current: "date('9999-12-31')" """ class TestSnapshotEmpty: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_model_sql, "snapshots.yml": snapshots_yml, } def test_check(self, project): run_dbt(["run"]) run_dbt(["snapshot", "--empty"]) query = "select id, updated_at, dbt_valid_from, dbt_valid_to from {database}.{schema}.my_snapshot order by updated_at asc" snapshot_out1 = project.run_sql(query, fetch="all") assert snapshot_out1 == [] run_dbt(["run"]) run_dbt(["snapshot", "--empty"]) snapshot_out2 = project.run_sql(query, fetch="all") assert snapshot_out2 == [] ================================================ FILE: tests/functional/snapshots/test_snapshot_timestamps.py ================================================ import pytest from dbt.tests.util import run_dbt, run_dbt_and_capture create_source_sql = """ create table {database}.{schema}.source_users ( id INTEGER, first_name VARCHAR(50), last_name VARCHAR(50), email VARCHAR(50), gender VARCHAR(50), ip_address VARCHAR(20), updated_time TIMESTAMP WITH TIME ZONE ); insert into {database}.{schema}.source_users (id, first_name, last_name, email, gender, ip_address, updated_time) values (1, 'Judith', 'Kennedy', '(not provided)', 'Female', '54.60.24.128', '2015-12-24 12:19:28'), (2, 'Arthur', 'Kelly', '(not provided)', 'Male', '62.56.24.215', '2015-10-28 16:22:15'), (3, 'Rachel', 'Moreno', 'rmoreno2@msu.edu', 'Female', '31.222.249.23', '2016-04-05 02:05:30'); """ model_users_sql = """ select * from {{ source('test_source', 'source_users') }} """ snapshot_sql = """ {% snapshot users_snapshot %} select * from {{ ref('users') }} {% endsnapshot %} """ source_schema_yml = """ sources: - name: test_source loader: custom schema: "{{ target.schema }}" tables: - name: source_users loaded_at_field: updated_time """ snapshot_schema_yml = """ snapshots: - name: users_snapshot config: target_schema: "{{ target.schema }}" strategy: timestamp unique_key: id updated_at: updated_time """ class TestSnapshotConfig: @pytest.fixture(scope="class") def models(self): return { "users.sql": model_users_sql, "source_schema.yml": source_schema_yml, "snapshot_schema.yml": snapshot_schema_yml, } @pytest.fixture(scope="class") def snapshots(self): return {"snapshot.sql": snapshot_sql} def test_timestamp_snapshot(self, project): project.run_sql(create_source_sql) run_dbt(["run"]) results, log_output = run_dbt_and_capture(["snapshot"]) assert len(results) == 1 assert "Please update snapshot config" in log_output ================================================ FILE: tests/functional/source_overrides/fixtures.py ================================================ import pytest dupe_models__schema2_yml = """ version: 2 sources: - name: my_source overrides: localdep schema: "{{ target.schema }}" database: "{{ target.database }}" freshness: error_after: {count: 3, period: day} tables: - name: my_table freshness: null identifier: my_real_seed # on the override, the "color" column is only unique, it can be null! columns: - name: id data_tests: - not_null - unique - name: color data_tests: - unique - name: my_other_table freshness: null identifier: my_real_other_seed - name: snapshot_freshness identifier: snapshot_freshness_base freshness: error_after: {count: 1, period: day} """ dupe_models__schema1_yml = """ version: 2 sources: - name: my_source overrides: localdep schema: "{{ target.schema }}" database: "{{ target.database }}" freshness: error_after: {count: 3, period: day} tables: - name: my_table freshness: null identifier: my_real_seed # on the override, the "color" column is only unique, it can be null! columns: - name: id data_tests: - not_null - unique - name: color data_tests: - unique - name: my_other_table freshness: null identifier: my_real_other_seed - name: snapshot_freshness identifier: snapshot_freshness_base loaded_at_field: updated_at freshness: error_after: {count: 1, period: day} """ local_dependency__dbt_project_yml = """ config-version: 2 name: localdep version: '1.0' profile: 'default' seeds: quote_columns: False seed-paths: ['seeds'] """ local_dependency__models__schema_yml = """ version: 2 sources: - name: my_source schema: invalid_schema database: invalid_database freshness: error_after: {count: 3, period: hour} tables: - name: my_table freshness: null identifier: my_seed columns: - name: id data_tests: - unique - not_null - name: color data_tests: - unique - not_null - name: my_other_table identifier: my_other_seed columns: - name: id data_tests: - unique - not_null - name: letter data_tests: - unique - not_null - name: snapshot_freshness identifier: snapshot_freshness_base loaded_at_field: updated_at freshness: error_after: {count: 1, period: hour} - name: my_other_source schema: "{{ target.schema }}" database: "{{ target.database }}" freshness: error_after: {count: 1, period: day} tables: - name: never_fresh loaded_at_field: updated_at """ local_dependency__models__my_model_sql = """ {{ config(materialized="table") }} with colors as ( select id, color from {{ source('my_source', 'my_table') }} ), letters as ( select id, letter from {{ source('my_source', 'my_other_table') }} ) select letter, color from colors join letters using (id) """ local_dependency__seeds__my_other_seed_csv = """id,letter 1,r 2,g 3,b """ local_dependency__seeds__my_seed_csv = """id,color 1,red 2,green 3,blue """ local_dependency__seeds__keep__never_fresh_csv = """favorite_color,id,first_name,email,ip_address,updated_at blue,1,Larry,lking0@miitbeian.gov.cn,'69.135.206.194',2008-09-12 19:08:31 blue,2,Larry,lperkins1@toplist.cz,'64.210.133.162',1978-05-09 04:15:14 blue,3,Anna,amontgomery2@miitbeian.gov.cn,'168.104.64.114',2011-10-16 04:07:57 blue,4,Sandra,sgeorge3@livejournal.com,'229.235.252.98',1973-07-19 10:52:43 blue,5,Fred,fwoods4@google.cn,'78.229.170.124',2012-09-30 16:38:29 blue,6,Stephen,shanson5@livejournal.com,'182.227.157.105',1995-11-07 21:40:50 blue,7,William,wmartinez6@upenn.edu,'135.139.249.50',1982-09-05 03:11:59 blue,8,Jessica,jlong7@hao123.com,'203.62.178.210',1991-10-16 11:03:15 blue,9,Douglas,dwhite8@tamu.edu,'178.187.247.1',1979-10-01 09:49:48 blue,10,Lisa,lcoleman9@nydailynews.com,'168.234.128.249',2011-05-26 07:45:49 blue,11,Ralph,rfieldsa@home.pl,'55.152.163.149',1972-11-18 19:06:11 blue,12,Louise,lnicholsb@samsung.com,'141.116.153.154',2014-11-25 20:56:14 blue,13,Clarence,cduncanc@sfgate.com,'81.171.31.133',2011-11-17 07:02:36 blue,14,Daniel,dfranklind@omniture.com,'8.204.211.37',1980-09-13 00:09:04 blue,15,Katherine,klanee@auda.org.au,'176.96.134.59',1997-08-22 19:36:56 blue,16,Billy,bwardf@wikia.com,'214.108.78.85',2003-10-19 02:14:47 blue,17,Annie,agarzag@ocn.ne.jp,'190.108.42.70',1988-10-28 15:12:35 blue,18,Shirley,scolemanh@fastcompany.com,'109.251.164.84',1988-08-24 10:50:57 blue,19,Roger,rfrazieri@scribd.com,'38.145.218.108',1985-12-31 15:17:15 blue,20,Lillian,lstanleyj@goodreads.com,'47.57.236.17',1970-06-08 02:09:05 blue,21,Aaron,arodriguezk@nps.gov,'205.245.118.221',1985-10-11 23:07:49 blue,22,Patrick,pparkerl@techcrunch.com,'19.8.100.182',2006-03-29 12:53:56 blue,23,Phillip,pmorenom@intel.com,'41.38.254.103',2011-11-07 15:35:43 blue,24,Henry,hgarcian@newsvine.com,'1.191.216.252',2008-08-28 08:30:44 blue,25,Irene,iturnero@opera.com,'50.17.60.190',1994-04-01 07:15:02 blue,26,Andrew,adunnp@pen.io,'123.52.253.176',2000-11-01 06:03:25 blue,27,David,dgutierrezq@wp.com,'238.23.203.42',1988-01-25 07:29:18 blue,28,Henry,hsanchezr@cyberchimps.com,'248.102.2.185',1983-01-01 13:36:37 blue,29,Evelyn,epetersons@gizmodo.com,'32.80.46.119',1979-07-16 17:24:12 blue,30,Tammy,tmitchellt@purevolume.com,'249.246.167.88',2001-04-03 10:00:23 blue,31,Jacqueline,jlittleu@domainmarket.com,'127.181.97.47',1986-02-11 21:35:50 blue,32,Earl,eortizv@opera.com,'166.47.248.240',1996-07-06 08:16:27 blue,33,Juan,jgordonw@sciencedirect.com,'71.77.2.200',1987-01-31 03:46:44 blue,34,Diane,dhowellx@nyu.edu,'140.94.133.12',1994-06-11 02:30:05 blue,35,Randy,rkennedyy@microsoft.com,'73.255.34.196',2005-05-26 20:28:39 blue,36,Janice,jriveraz@time.com,'22.214.227.32',1990-02-09 04:16:52 blue,37,Laura,lperry10@diigo.com,'159.148.145.73',2015-03-17 05:59:25 blue,38,Gary,gray11@statcounter.com,'40.193.124.56',1970-01-27 10:04:51 blue,39,Jesse,jmcdonald12@typepad.com,'31.7.86.103',2009-03-14 08:14:29 blue,40,Sandra,sgonzalez13@goodreads.com,'223.80.168.239',1993-05-21 14:08:54 blue,41,Scott,smoore14@archive.org,'38.238.46.83',1980-08-30 11:16:56 blue,42,Phillip,pevans15@cisco.com,'158.234.59.34',2011-12-15 23:26:31 blue,43,Steven,sriley16@google.ca,'90.247.57.68',2011-10-29 19:03:28 blue,44,Deborah,dbrown17@hexun.com,'179.125.143.240',1995-04-10 14:36:07 blue,45,Lori,lross18@ow.ly,'64.80.162.180',1980-12-27 16:49:15 blue,46,Sean,sjackson19@tumblr.com,'240.116.183.69',1988-06-12 21:24:45 blue,47,Terry,tbarnes1a@163.com,'118.38.213.137',1997-09-22 16:43:19 blue,48,Dorothy,dross1b@ebay.com,'116.81.76.49',2005-02-28 13:33:24 blue,49,Samuel,swashington1c@house.gov,'38.191.253.40',1989-01-19 21:15:48 blue,50,Ralph,rcarter1d@tinyurl.com,'104.84.60.174',2007-08-11 10:21:49 """ local_dependency__seeds__keep__snapshot_freshness_base_csv = """favorite_color,id,first_name,email,ip_address,updated_at blue,1,Larry,lking0@miitbeian.gov.cn,'69.135.206.194',2008-09-12 19:08:31 blue,2,Larry,lperkins1@toplist.cz,'64.210.133.162',1978-05-09 04:15:14 blue,3,Anna,amontgomery2@miitbeian.gov.cn,'168.104.64.114',2011-10-16 04:07:57 blue,4,Sandra,sgeorge3@livejournal.com,'229.235.252.98',1973-07-19 10:52:43 blue,5,Fred,fwoods4@google.cn,'78.229.170.124',2012-09-30 16:38:29 blue,6,Stephen,shanson5@livejournal.com,'182.227.157.105',1995-11-07 21:40:50 blue,7,William,wmartinez6@upenn.edu,'135.139.249.50',1982-09-05 03:11:59 blue,8,Jessica,jlong7@hao123.com,'203.62.178.210',1991-10-16 11:03:15 blue,9,Douglas,dwhite8@tamu.edu,'178.187.247.1',1979-10-01 09:49:48 blue,10,Lisa,lcoleman9@nydailynews.com,'168.234.128.249',2011-05-26 07:45:49 blue,11,Ralph,rfieldsa@home.pl,'55.152.163.149',1972-11-18 19:06:11 blue,12,Louise,lnicholsb@samsung.com,'141.116.153.154',2014-11-25 20:56:14 blue,13,Clarence,cduncanc@sfgate.com,'81.171.31.133',2011-11-17 07:02:36 blue,14,Daniel,dfranklind@omniture.com,'8.204.211.37',1980-09-13 00:09:04 blue,15,Katherine,klanee@auda.org.au,'176.96.134.59',1997-08-22 19:36:56 blue,16,Billy,bwardf@wikia.com,'214.108.78.85',2003-10-19 02:14:47 blue,17,Annie,agarzag@ocn.ne.jp,'190.108.42.70',1988-10-28 15:12:35 blue,18,Shirley,scolemanh@fastcompany.com,'109.251.164.84',1988-08-24 10:50:57 blue,19,Roger,rfrazieri@scribd.com,'38.145.218.108',1985-12-31 15:17:15 blue,20,Lillian,lstanleyj@goodreads.com,'47.57.236.17',1970-06-08 02:09:05 blue,21,Aaron,arodriguezk@nps.gov,'205.245.118.221',1985-10-11 23:07:49 blue,22,Patrick,pparkerl@techcrunch.com,'19.8.100.182',2006-03-29 12:53:56 blue,23,Phillip,pmorenom@intel.com,'41.38.254.103',2011-11-07 15:35:43 blue,24,Henry,hgarcian@newsvine.com,'1.191.216.252',2008-08-28 08:30:44 blue,25,Irene,iturnero@opera.com,'50.17.60.190',1994-04-01 07:15:02 blue,26,Andrew,adunnp@pen.io,'123.52.253.176',2000-11-01 06:03:25 blue,27,David,dgutierrezq@wp.com,'238.23.203.42',1988-01-25 07:29:18 blue,28,Henry,hsanchezr@cyberchimps.com,'248.102.2.185',1983-01-01 13:36:37 blue,29,Evelyn,epetersons@gizmodo.com,'32.80.46.119',1979-07-16 17:24:12 blue,30,Tammy,tmitchellt@purevolume.com,'249.246.167.88',2001-04-03 10:00:23 blue,31,Jacqueline,jlittleu@domainmarket.com,'127.181.97.47',1986-02-11 21:35:50 blue,32,Earl,eortizv@opera.com,'166.47.248.240',1996-07-06 08:16:27 blue,33,Juan,jgordonw@sciencedirect.com,'71.77.2.200',1987-01-31 03:46:44 blue,34,Diane,dhowellx@nyu.edu,'140.94.133.12',1994-06-11 02:30:05 blue,35,Randy,rkennedyy@microsoft.com,'73.255.34.196',2005-05-26 20:28:39 blue,36,Janice,jriveraz@time.com,'22.214.227.32',1990-02-09 04:16:52 blue,37,Laura,lperry10@diigo.com,'159.148.145.73',2015-03-17 05:59:25 blue,38,Gary,gray11@statcounter.com,'40.193.124.56',1970-01-27 10:04:51 blue,39,Jesse,jmcdonald12@typepad.com,'31.7.86.103',2009-03-14 08:14:29 blue,40,Sandra,sgonzalez13@goodreads.com,'223.80.168.239',1993-05-21 14:08:54 blue,41,Scott,smoore14@archive.org,'38.238.46.83',1980-08-30 11:16:56 blue,42,Phillip,pevans15@cisco.com,'158.234.59.34',2011-12-15 23:26:31 blue,43,Steven,sriley16@google.ca,'90.247.57.68',2011-10-29 19:03:28 blue,44,Deborah,dbrown17@hexun.com,'179.125.143.240',1995-04-10 14:36:07 blue,45,Lori,lross18@ow.ly,'64.80.162.180',1980-12-27 16:49:15 blue,46,Sean,sjackson19@tumblr.com,'240.116.183.69',1988-06-12 21:24:45 blue,47,Terry,tbarnes1a@163.com,'118.38.213.137',1997-09-22 16:43:19 blue,48,Dorothy,dross1b@ebay.com,'116.81.76.49',2005-02-28 13:33:24 blue,49,Samuel,swashington1c@house.gov,'38.191.253.40',1989-01-19 21:15:48 blue,50,Ralph,rcarter1d@tinyurl.com,'104.84.60.174',2007-08-11 10:21:49 green,51,Wayne,whudson1e@princeton.edu,'90.61.24.102',1983-07-03 16:58:12 green,52,Rose,rjames1f@plala.or.jp,'240.83.81.10',1995-06-08 11:46:23 green,53,Louise,lcox1g@theglobeandmail.com,'105.11.82.145',2016-09-19 14:45:51 green,54,Kenneth,kjohnson1h@independent.co.uk,'139.5.45.94',1976-08-17 11:26:19 green,55,Donna,dbrown1i@amazon.co.uk,'19.45.169.45',2006-05-27 16:51:40 green,56,Johnny,jvasquez1j@trellian.com,'118.202.238.23',1975-11-17 08:42:32 green,57,Patrick,pramirez1k@tamu.edu,'231.25.153.198',1997-08-06 11:51:09 green,58,Helen,hlarson1l@prweb.com,'8.40.21.39',1993-08-04 19:53:40 green,59,Patricia,pspencer1m@gmpg.org,'212.198.40.15',1977-08-03 16:37:27 green,60,Joseph,jspencer1n@marriott.com,'13.15.63.238',2005-07-23 20:22:06 green,61,Phillip,pschmidt1o@blogtalkradio.com,'177.98.201.190',1976-05-19 21:47:44 green,62,Joan,jwebb1p@google.ru,'105.229.170.71',1972-09-07 17:53:47 green,63,Phyllis,pkennedy1q@imgur.com,'35.145.8.244',2000-01-01 22:33:37 green,64,Katherine,khunter1r@smh.com.au,'248.168.205.32',1991-01-09 06:40:24 green,65,Laura,lvasquez1s@wiley.com,'128.129.115.152',1997-10-23 12:04:56 green,66,Juan,jdunn1t@state.gov,'44.228.124.51',2004-11-10 05:07:35 green,67,Judith,jholmes1u@wiley.com,'40.227.179.115',1977-08-02 17:01:45 green,68,Beverly,bbaker1v@wufoo.com,'208.34.84.59',2016-03-06 20:07:23 green,69,Lawrence,lcarr1w@flickr.com,'59.158.212.223',1988-09-13 06:07:21 green,70,Gloria,gwilliams1x@mtv.com,'245.231.88.33',1995-03-18 22:32:46 green,71,Steven,ssims1y@cbslocal.com,'104.50.58.255',2001-08-05 21:26:20 green,72,Betty,bmills1z@arstechnica.com,'103.177.214.220',1981-12-14 21:26:54 green,73,Mildred,mfuller20@prnewswire.com,'151.158.8.130',2000-04-19 10:13:55 green,74,Donald,dday21@icq.com,'9.178.102.255',1972-12-03 00:58:24 green,75,Eric,ethomas22@addtoany.com,'85.2.241.227',1992-11-01 05:59:30 green,76,Joyce,jarmstrong23@sitemeter.com,'169.224.20.36',1985-10-24 06:50:01 green,77,Maria,mmartinez24@amazonaws.com,'143.189.167.135',2005-10-05 05:17:42 green,78,Harry,hburton25@youtube.com,'156.47.176.237',1978-03-26 05:53:33 green,79,Kevin,klawrence26@hao123.com,'79.136.183.83',1994-10-12 04:38:52 green,80,David,dhall27@prweb.com,'133.149.172.153',1976-12-15 16:24:24 green,81,Kathy,kperry28@twitter.com,'229.242.72.228',1979-03-04 02:58:56 green,82,Adam,aprice29@elegantthemes.com,'13.145.21.10',1982-11-07 11:46:59 green,83,Brandon,bgriffin2a@va.gov,'73.249.128.212',2013-10-30 05:30:36 green,84,Henry,hnguyen2b@discovery.com,'211.36.214.242',1985-01-09 06:37:27 green,85,Eric,esanchez2c@edublogs.org,'191.166.188.251',2004-05-01 23:21:42 green,86,Jason,jlee2d@jimdo.com,'193.92.16.182',1973-01-08 09:05:39 green,87,Diana,drichards2e@istockphoto.com,'19.130.175.245',1994-10-05 22:50:49 green,88,Andrea,awelch2f@abc.net.au,'94.155.233.96',2002-04-26 08:41:44 green,89,Louis,lwagner2g@miitbeian.gov.cn,'26.217.34.111',2003-08-25 07:56:39 green,90,Jane,jsims2h@seesaa.net,'43.4.220.135',1987-03-20 20:39:04 green,91,Larry,lgrant2i@si.edu,'97.126.79.34',2000-09-07 20:26:19 green,92,Louis,ldean2j@prnewswire.com,'37.148.40.127',2011-09-16 20:12:14 green,93,Jennifer,jcampbell2k@xing.com,'38.106.254.142',1988-07-15 05:06:49 green,94,Wayne,wcunningham2l@google.com.hk,'223.28.26.187',2009-12-15 06:16:54 green,95,Lori,lstevens2m@icq.com,'181.250.181.58',1984-10-28 03:29:19 green,96,Judy,jsimpson2n@marriott.com,'180.121.239.219',1986-02-07 15:18:10 green,97,Phillip,phoward2o@usa.gov,'255.247.0.175',2002-12-26 08:44:45 green,98,Gloria,gwalker2p@usa.gov,'156.140.7.128',1997-10-04 07:58:58 green,99,Paul,pjohnson2q@umn.edu,'183.59.198.197',1991-11-14 12:33:55 green,100,Frank,fgreene2r@blogspot.com,'150.143.68.121',2010-06-12 23:55:39 """ models__schema_yml = """ version: 2 sources: - name: my_source overrides: localdep schema: "{{ target.schema }}" database: "{{ target.database }}" freshness: error_after: {count: 3, period: day} tables: - name: my_table freshness: null identifier: my_real_seed # on the override, the "color" column is only unique, it can be null! columns: - name: id data_tests: - not_null - unique - name: color data_tests: - unique - name: my_other_table freshness: null identifier: my_real_other_seed - name: snapshot_freshness identifier: snapshot_freshness_base loaded_at_field: updated_at freshness: error_after: {count: 1, period: day} """ seeds__expected_result_csv = """letter,color c,cyan m,magenta y,yellow k,key """ seeds__my_real_other_seed_csv = """id,letter 1,c 2,m 3,y 4,k """ seeds__my_real_seed_csv = """id,color 1,cyan 2,magenta 3,yellow 4,key 5,NULL """ @pytest.fixture(scope="class") def local_dependency(): return { "dbt_project.yml": local_dependency__dbt_project_yml, "models": { "schema.yml": local_dependency__models__schema_yml, "my_model.sql": local_dependency__models__my_model_sql, }, "seeds": { "my_other_seed.csv": local_dependency__seeds__my_other_seed_csv, "my_seed.csv": local_dependency__seeds__my_seed_csv, "keep": { "never_fresh.csv": local_dependency__seeds__keep__never_fresh_csv, "snapshot_freshness_base.csv": local_dependency__seeds__keep__snapshot_freshness_base_csv, }, }, } ================================================ FILE: tests/functional/source_overrides/test_simple_source_override.py ================================================ from datetime import datetime, timedelta, timezone from unittest import mock import pytest from dbt.tests.fixtures.project import write_project_files from dbt.tests.util import ( check_relations_equal, run_dbt, run_dbt_and_capture, update_config_file, ) from tests.functional.source_overrides.fixtures import ( # noqa: F401 local_dependency, models__schema_yml, seeds__expected_result_csv, seeds__my_real_other_seed_csv, seeds__my_real_seed_csv, ) class TestSourceOverride: @pytest.fixture(scope="class", autouse=True) def setUp(self, project_root, local_dependency): # noqa: F811 write_project_files(project_root, "local_dependency", local_dependency) @pytest.fixture(scope="class") def models(self): return {"schema.yml": models__schema_yml} @pytest.fixture(scope="class") def seeds(self): return { "expected_result.csv": seeds__expected_result_csv, "my_real_other_seed.csv": seeds__my_real_other_seed_csv, "my_real_seed.csv": seeds__my_real_seed_csv, } @pytest.fixture(scope="class") def packages(self): return { "packages": [ { "local": "local_dependency", }, ] } @pytest.fixture(scope="class") def project_config_update(self): return { "seeds": { "localdep": { "enabled": False, "keep": { "enabled": True, }, }, "quote_columns": False, }, "sources": { "localdep": { "my_other_source": { "enabled": False, } } }, } def _set_updated_at_to(self, insert_id, delta, project): insert_time = datetime.now(timezone.utc).replace(tzinfo=None) + delta timestr = insert_time.strftime("%Y-%m-%d %H:%M:%S") # favorite_color,id,first_name,email,ip_address,updated_at quoted_columns = ",".join( project.adapter.quote(c) for c in ("favorite_color", "id", "first_name", "email", "ip_address", "updated_at") ) kwargs = { "schema": project.test_schema, "time": timestr, "id": insert_id, "source": project.adapter.quote("snapshot_freshness_base"), "quoted_columns": quoted_columns, } raw_code = """INSERT INTO {schema}.{source} ({quoted_columns}) VALUES ( 'blue',{id},'Jake','abc@example.com','192.168.1.1','{time}' )""".format( **kwargs ) project.run_sql(raw_code) return insert_id + 1 @mock.patch("dbt.jsonschemas.jsonschemas._JSONSCHEMA_SUPPORTED_ADAPTERS", {"postgres"}) def test_source_overrides(self, project): insert_id = 101 run_dbt(["deps"]) seed_results = run_dbt(["seed"]) assert len(seed_results) == 5 # There should be 7, as we disabled 1 test of the original 8 test_results = run_dbt(["test"]) assert len(test_results) == 7 results, logs = run_dbt_and_capture(["run"]) assert len(results) == 1 # Since source overrides are now deprecated, shoehorn a check for the # deprecation warning into this existing test. assert "SourceOverrideDeprecation: 1 occurrence" in logs check_relations_equal(project.adapter, ["expected_result", "my_model"]) # set the updated_at field of this seed to last week insert_id = self._set_updated_at_to(insert_id, timedelta(days=-7), project) # if snapshot-freshness fails, freshness just didn't happen! results = run_dbt(["source", "snapshot-freshness"], expect_pass=False) # we disabled my_other_source, so we only run the one freshness check # in assert len(results) == 1 # If snapshot-freshness passes, that means error_after was # applied from the source override but not the source table override insert_id = self._set_updated_at_to(insert_id, timedelta(days=-2), project) results = run_dbt( ["source", "snapshot-freshness"], expect_pass=False, ) assert len(results) == 1 insert_id = self._set_updated_at_to(insert_id, timedelta(hours=-12), project) results = run_dbt(["source", "snapshot-freshness"], expect_pass=True) assert len(results) == 1 # update source to be enabled new_source_config = { "sources": { "localdep": { "my_other_source": { "enabled": True, } } } } update_config_file(new_source_config, project.project_root, "dbt_project.yml") # enable my_other_source, snapshot freshness should fail due to the new # not-fresh source results = run_dbt(["source", "snapshot-freshness"], expect_pass=False) assert len(results) == 2 ================================================ FILE: tests/functional/source_overrides/test_source_overrides_duplicate_model.py ================================================ import os import pytest from dbt.exceptions import CompilationError from dbt.tests.fixtures.project import write_project_files from dbt.tests.util import run_dbt from tests.functional.source_overrides.fixtures import ( # noqa: F401 dupe_models__schema1_yml, dupe_models__schema2_yml, local_dependency, ) class TestSourceOverrideDuplicates: @pytest.fixture(scope="class", autouse=True) def setUp(self, project_root, local_dependency): # noqa: F811 write_project_files(project_root, "local_dependency", local_dependency) @pytest.fixture(scope="class") def models(self): return { "schema2.yml": dupe_models__schema2_yml, "schema1.yml": dupe_models__schema1_yml, } @pytest.fixture(scope="class") def packages(self): return { "packages": [ { "local": "local_dependency", }, ] } @pytest.fixture(scope="class") def project_config_update(self): return { "seeds": { "localdep": { "enabled": False, "keep": { "enabled": True, }, }, "quote_columns": False, }, "sources": { "localdep": { "my_other_source": { "enabled": False, } } }, } def test_source_duplicate_overrides(self, project): run_dbt(["deps"]) with pytest.raises(CompilationError) as exc: run_dbt(["compile"]) assert "dbt found two schema.yml entries for the same source named" in str(exc.value) assert "one of these files" in str(exc.value) schema1_path = os.path.join("models", "schema1.yml") schema2_path = os.path.join("models", "schema2.yml") assert schema1_path in str(exc.value) assert schema2_path in str(exc.value) ================================================ FILE: tests/functional/sources/common_source_setup.py ================================================ import os import pytest import yaml from dbt.tests.util import run_dbt, run_dbt_and_capture from tests.functional.sources.fixtures import ( models_descendant_model_sql, models_ephemeral_model_sql, models_multi_source_model_sql, models_nonsource_descendant_sql, models_schema_yml, models_view_model_sql, seeds_expected_multi_source_csv, seeds_other_source_table_csv, seeds_other_table_csv, seeds_source_csv, ) class BaseSourcesTest: @pytest.fixture(scope="class", autouse=True) def setEnvVars(self): os.environ["DBT_TEST_SCHEMA_NAME_VARIABLE"] = "test_run_schema" yield del os.environ["DBT_TEST_SCHEMA_NAME_VARIABLE"] @pytest.fixture(scope="class") def models(self): return { "schema.yml": models_schema_yml, "view_model.sql": models_view_model_sql, "ephemeral_model.sql": models_ephemeral_model_sql, "descendant_model.sql": models_descendant_model_sql, "multi_source_model.sql": models_multi_source_model_sql, "nonsource_descendant.sql": models_nonsource_descendant_sql, } @pytest.fixture(scope="class") def seeds(self): return { "source.csv": seeds_source_csv, "other_table.csv": seeds_other_table_csv, "expected_multi_source.csv": seeds_expected_multi_source_csv, "other_source_table.csv": seeds_other_source_table_csv, } @pytest.fixture(scope="class") def project_config_update(self): return { "config-version": 2, "seed-paths": ["seeds"], "quoting": {"database": True, "schema": True, "identifier": True}, "seeds": { "quote_columns": True, }, } def _extend_cmd_with_vars(self, project, cmd): vars_dict = { "test_run_schema": project.test_schema, "test_loaded_at": project.adapter.quote("updated_at"), } cmd.extend(["--vars", yaml.safe_dump(vars_dict)]) def run_dbt_with_vars(self, project, cmd, *args, **kwargs): self._extend_cmd_with_vars(project, cmd) return run_dbt(cmd, *args, **kwargs) def run_dbt_and_capture_with_vars(self, project, cmd, *args, **kwargs): self._extend_cmd_with_vars(project, cmd) return run_dbt_and_capture(cmd, *args, **kwargs) ================================================ FILE: tests/functional/sources/data/seed.sql ================================================ create table {schema}.seed_expected ( favorite_color TEXT, id INTEGER, first_name TEXT, email TEXT, ip_address TEXT, updated_at TIMESTAMP WITHOUT TIME ZONE ); INSERT INTO {schema}.seed_expected ("favorite_color","id","first_name","email","ip_address","updated_at") VALUES ('blue',1,'Larry','lking0@miitbeian.gov.cn','''69.135.206.194''','2008-09-12 19:08:31'), ('blue',2,'Larry','lperkins1@toplist.cz','''64.210.133.162''','1978-05-09 04:15:14'), ('blue',3,'Anna','amontgomery2@miitbeian.gov.cn','''168.104.64.114''','2011-10-16 04:07:57'), ('blue',4,'Sandra','sgeorge3@livejournal.com','''229.235.252.98''','1973-07-19 10:52:43'), ('blue',5,'Fred','fwoods4@google.cn','''78.229.170.124''','2012-09-30 16:38:29'), ('blue',6,'Stephen','shanson5@livejournal.com','''182.227.157.105''','1995-11-07 21:40:50'), ('blue',7,'William','wmartinez6@upenn.edu','''135.139.249.50''','1982-09-05 03:11:59'), ('blue',8,'Jessica','jlong7@hao123.com','''203.62.178.210''','1991-10-16 11:03:15'), ('blue',9,'Douglas','dwhite8@tamu.edu','''178.187.247.1''','1979-10-01 09:49:48'), ('blue',10,'Lisa','lcoleman9@nydailynews.com','''168.234.128.249''','2011-05-26 07:45:49'), ('blue',11,'Ralph','rfieldsa@home.pl','''55.152.163.149''','1972-11-18 19:06:11'), ('blue',12,'Louise','lnicholsb@samsung.com','''141.116.153.154''','2014-11-25 20:56:14'), ('blue',13,'Clarence','cduncanc@sfgate.com','''81.171.31.133''','2011-11-17 07:02:36'), ('blue',14,'Daniel','dfranklind@omniture.com','''8.204.211.37''','1980-09-13 00:09:04'), ('blue',15,'Katherine','klanee@auda.org.au','''176.96.134.59''','1997-08-22 19:36:56'), ('blue',16,'Billy','bwardf@wikia.com','''214.108.78.85''','2003-10-19 02:14:47'), ('blue',17,'Annie','agarzag@ocn.ne.jp','''190.108.42.70''','1988-10-28 15:12:35'), ('blue',18,'Shirley','scolemanh@fastcompany.com','''109.251.164.84''','1988-08-24 10:50:57'), ('blue',19,'Roger','rfrazieri@scribd.com','''38.145.218.108''','1985-12-31 15:17:15'), ('blue',20,'Lillian','lstanleyj@goodreads.com','''47.57.236.17''','1970-06-08 02:09:05'), ('blue',21,'Aaron','arodriguezk@nps.gov','''205.245.118.221''','1985-10-11 23:07:49'), ('blue',22,'Patrick','pparkerl@techcrunch.com','''19.8.100.182''','2006-03-29 12:53:56'), ('blue',23,'Phillip','pmorenom@intel.com','''41.38.254.103''','2011-11-07 15:35:43'), ('blue',24,'Henry','hgarcian@newsvine.com','''1.191.216.252''','2008-08-28 08:30:44'), ('blue',25,'Irene','iturnero@opera.com','''50.17.60.190''','1994-04-01 07:15:02'), ('blue',26,'Andrew','adunnp@pen.io','''123.52.253.176''','2000-11-01 06:03:25'), ('blue',27,'David','dgutierrezq@wp.com','''238.23.203.42''','1988-01-25 07:29:18'), ('blue',28,'Henry','hsanchezr@cyberchimps.com','''248.102.2.185''','1983-01-01 13:36:37'), ('blue',29,'Evelyn','epetersons@gizmodo.com','''32.80.46.119''','1979-07-16 17:24:12'), ('blue',30,'Tammy','tmitchellt@purevolume.com','''249.246.167.88''','2001-04-03 10:00:23'), ('blue',31,'Jacqueline','jlittleu@domainmarket.com','''127.181.97.47''','1986-02-11 21:35:50'), ('blue',32,'Earl','eortizv@opera.com','''166.47.248.240''','1996-07-06 08:16:27'), ('blue',33,'Juan','jgordonw@sciencedirect.com','''71.77.2.200''','1987-01-31 03:46:44'), ('blue',34,'Diane','dhowellx@nyu.edu','''140.94.133.12''','1994-06-11 02:30:05'), ('blue',35,'Randy','rkennedyy@microsoft.com','''73.255.34.196''','2005-05-26 20:28:39'), ('blue',36,'Janice','jriveraz@time.com','''22.214.227.32''','1990-02-09 04:16:52'), ('blue',37,'Laura','lperry10@diigo.com','''159.148.145.73''','2015-03-17 05:59:25'), ('blue',38,'Gary','gray11@statcounter.com','''40.193.124.56''','1970-01-27 10:04:51'), ('blue',39,'Jesse','jmcdonald12@typepad.com','''31.7.86.103''','2009-03-14 08:14:29'), ('blue',40,'Sandra','sgonzalez13@goodreads.com','''223.80.168.239''','1993-05-21 14:08:54'), ('blue',41,'Scott','smoore14@archive.org','''38.238.46.83''','1980-08-30 11:16:56'), ('blue',42,'Phillip','pevans15@cisco.com','''158.234.59.34''','2011-12-15 23:26:31'), ('blue',43,'Steven','sriley16@google.ca','''90.247.57.68''','2011-10-29 19:03:28'), ('blue',44,'Deborah','dbrown17@hexun.com','''179.125.143.240''','1995-04-10 14:36:07'), ('blue',45,'Lori','lross18@ow.ly','''64.80.162.180''','1980-12-27 16:49:15'), ('blue',46,'Sean','sjackson19@tumblr.com','''240.116.183.69''','1988-06-12 21:24:45'), ('blue',47,'Terry','tbarnes1a@163.com','''118.38.213.137''','1997-09-22 16:43:19'), ('blue',48,'Dorothy','dross1b@ebay.com','''116.81.76.49''','2005-02-28 13:33:24'), ('blue',49,'Samuel','swashington1c@house.gov','''38.191.253.40''','1989-01-19 21:15:48'), ('blue',50,'Ralph','rcarter1d@tinyurl.com','''104.84.60.174''','2007-08-11 10:21:49'), ('green',51,'Wayne','whudson1e@princeton.edu','''90.61.24.102''','1983-07-03 16:58:12'), ('green',52,'Rose','rjames1f@plala.or.jp','''240.83.81.10''','1995-06-08 11:46:23'), ('green',53,'Louise','lcox1g@theglobeandmail.com','''105.11.82.145''','2016-09-19 14:45:51'), ('green',54,'Kenneth','kjohnson1h@independent.co.uk','''139.5.45.94''','1976-08-17 11:26:19'), ('green',55,'Donna','dbrown1i@amazon.co.uk','''19.45.169.45''','2006-05-27 16:51:40'), ('green',56,'Johnny','jvasquez1j@trellian.com','''118.202.238.23''','1975-11-17 08:42:32'), ('green',57,'Patrick','pramirez1k@tamu.edu','''231.25.153.198''','1997-08-06 11:51:09'), ('green',58,'Helen','hlarson1l@prweb.com','''8.40.21.39''','1993-08-04 19:53:40'), ('green',59,'Patricia','pspencer1m@gmpg.org','''212.198.40.15''','1977-08-03 16:37:27'), ('green',60,'Joseph','jspencer1n@marriott.com','''13.15.63.238''','2005-07-23 20:22:06'), ('green',61,'Phillip','pschmidt1o@blogtalkradio.com','''177.98.201.190''','1976-05-19 21:47:44'), ('green',62,'Joan','jwebb1p@google.ru','''105.229.170.71''','1972-09-07 17:53:47'), ('green',63,'Phyllis','pkennedy1q@imgur.com','''35.145.8.244''','2000-01-01 22:33:37'), ('green',64,'Katherine','khunter1r@smh.com.au','''248.168.205.32''','1991-01-09 06:40:24'), ('green',65,'Laura','lvasquez1s@wiley.com','''128.129.115.152''','1997-10-23 12:04:56'), ('green',66,'Juan','jdunn1t@state.gov','''44.228.124.51''','2004-11-10 05:07:35'), ('green',67,'Judith','jholmes1u@wiley.com','''40.227.179.115''','1977-08-02 17:01:45'), ('green',68,'Beverly','bbaker1v@wufoo.com','''208.34.84.59''','2016-03-06 20:07:23'), ('green',69,'Lawrence','lcarr1w@flickr.com','''59.158.212.223''','1988-09-13 06:07:21'), ('green',70,'Gloria','gwilliams1x@mtv.com','''245.231.88.33''','1995-03-18 22:32:46'), ('green',71,'Steven','ssims1y@cbslocal.com','''104.50.58.255''','2001-08-05 21:26:20'), ('green',72,'Betty','bmills1z@arstechnica.com','''103.177.214.220''','1981-12-14 21:26:54'), ('green',73,'Mildred','mfuller20@prnewswire.com','''151.158.8.130''','2000-04-19 10:13:55'), ('green',74,'Donald','dday21@icq.com','''9.178.102.255''','1972-12-03 00:58:24'), ('green',75,'Eric','ethomas22@addtoany.com','''85.2.241.227''','1992-11-01 05:59:30'), ('green',76,'Joyce','jarmstrong23@sitemeter.com','''169.224.20.36''','1985-10-24 06:50:01'), ('green',77,'Maria','mmartinez24@amazonaws.com','''143.189.167.135''','2005-10-05 05:17:42'), ('green',78,'Harry','hburton25@youtube.com','''156.47.176.237''','1978-03-26 05:53:33'), ('green',79,'Kevin','klawrence26@hao123.com','''79.136.183.83''','1994-10-12 04:38:52'), ('green',80,'David','dhall27@prweb.com','''133.149.172.153''','1976-12-15 16:24:24'), ('green',81,'Kathy','kperry28@twitter.com','''229.242.72.228''','1979-03-04 02:58:56'), ('green',82,'Adam','aprice29@elegantthemes.com','''13.145.21.10''','1982-11-07 11:46:59'), ('green',83,'Brandon','bgriffin2a@va.gov','''73.249.128.212''','2013-10-30 05:30:36'), ('green',84,'Henry','hnguyen2b@discovery.com','''211.36.214.242''','1985-01-09 06:37:27'), ('green',85,'Eric','esanchez2c@edublogs.org','''191.166.188.251''','2004-05-01 23:21:42'), ('green',86,'Jason','jlee2d@jimdo.com','''193.92.16.182''','1973-01-08 09:05:39'), ('green',87,'Diana','drichards2e@istockphoto.com','''19.130.175.245''','1994-10-05 22:50:49'), ('green',88,'Andrea','awelch2f@abc.net.au','''94.155.233.96''','2002-04-26 08:41:44'), ('green',89,'Louis','lwagner2g@miitbeian.gov.cn','''26.217.34.111''','2003-08-25 07:56:39'), ('green',90,'Jane','jsims2h@seesaa.net','''43.4.220.135''','1987-03-20 20:39:04'), ('green',91,'Larry','lgrant2i@si.edu','''97.126.79.34''','2000-09-07 20:26:19'), ('green',92,'Louis','ldean2j@prnewswire.com','''37.148.40.127''','2011-09-16 20:12:14'), ('green',93,'Jennifer','jcampbell2k@xing.com','''38.106.254.142''','1988-07-15 05:06:49'), ('green',94,'Wayne','wcunningham2l@google.com.hk','''223.28.26.187''','2009-12-15 06:16:54'), ('green',95,'Lori','lstevens2m@icq.com','''181.250.181.58''','1984-10-28 03:29:19'), ('green',96,'Judy','jsimpson2n@marriott.com','''180.121.239.219''','1986-02-07 15:18:10'), ('green',97,'Phillip','phoward2o@usa.gov','''255.247.0.175''','2002-12-26 08:44:45'), ('green',98,'Gloria','gwalker2p@usa.gov','''156.140.7.128''','1997-10-04 07:58:58'), ('green',99,'Paul','pjohnson2q@umn.edu','''183.59.198.197''','1991-11-14 12:33:55'), ('green',100,'Frank','fgreene2r@blogspot.com','''150.143.68.121''','2010-06-12 23:55:39'); ================================================ FILE: tests/functional/sources/fixtures.py ================================================ error_models_schema_yml = """version: 2 sources: - name: test_source loader: custom freshness: warn_after: {count: 10, period: hour} error_after: {count: 1, period: day} schema: invalid tables: - name: test_table identifier: source loaded_at_field: updated_at """ error_models_model_sql = """select * from {{ source('test_source', 'test_table') }} """ override_freshness_models_schema_yml = """version: 2 sources: - name: test_source loader: custom freshness: warn_after: {count: 18, period: hour} error_after: {count: 24, period: hour} config: freshness: # default freshness, takes precedence over top-level key above warn_after: {count: 12, period: hour} schema: "{{ var(env_var('DBT_TEST_SCHEMA_NAME_VARIABLE')) }}" loaded_at_field: loaded_at quoting: identifier: True tags: - my_test_source_tag tables: - name: source_a identifier: source loaded_at_field: "{{ var('test_loaded_at') | as_text }}" config: freshness: warn_after: {count: 6, period: hour} # use default error_after, takes precedence over top-level key above freshness: warn_after: {count: 9, period: hour} - name: source_b identifier: source loaded_at_field: "{{ var('test_loaded_at') | as_text }}" freshness: warn_after: {count: 6, period: hour} error_after: {} # use the default error_after defined above - name: source_c identifier: source loaded_at_field: "{{ var('test_loaded_at') | as_text }}" freshness: warn_after: {count: 6, period: hour} error_after: null # override: disable error_after for this table - name: source_d identifier: source loaded_at_field: "{{ var('test_loaded_at') | as_text }}" freshness: warn_after: {count: 6, period: hour} error_after: {count: 144, period: hour} config: freshness: error_after: {count: 72, period: hour} # override: use this new behavior instead of error_after defined above - name: source_e identifier: source loaded_at_field: "{{ var('test_loaded_at') | as_text }}" freshness: null # override: disable freshness for this table """ models_schema_yml = """version: 2 models: - name: descendant_model columns: - name: favorite_color data_tests: - relationships: to: source('test_source', 'test_table') field: favorite_color - name: id data_tests: - unique - not_null sources: - name: test_source loader: custom freshness: warn_after: {count: 10, period: hour} error_after: {count: 1, period: day} schema: "{{ var(env_var('DBT_TEST_SCHEMA_NAME_VARIABLE')) }}" quoting: identifier: True tags: - my_test_source_tag tables: - name: test_table identifier: source loaded_at_field: "{{ var('test_loaded_at') | as_text }}" freshness: error_after: {count: 18, period: hour} tags: - my_test_source_table_tag columns: - name: favorite_color description: The favorite color - name: id description: The user ID data_tests: - unique - not_null tags: - id_column - name: first_name description: The first name of the user data_tests: [] - name: email description: The email address of the user - name: ip_address description: The last IP address the user logged in from - name: updated_at description: The last update time for this user data_tests: - relationships: # do this as a table-level test, just to test out that aspect column_name: favorite_color to: ref('descendant_model') field: favorite_color - name: other_test_table identifier: other_table freshness: null columns: - name: id data_tests: - not_null - unique tags: - id_column - name: disabled_test_table freshness: null loaded_at_field: "{{ var('test_loaded_at') | as_text }}" - name: other_source schema: "{{ var('test_run_schema') }}" quoting: identifier: True tables: - name: test_table identifier: other_source_table - name: external_source schema: "{{ var('test_run_alt_schema', var('test_run_schema')) }}" tables: - name: table """ models_view_model_sql = """{# See here: https://github.com/dbt-labs/dbt-core/pull/1729 #} select * from {{ ref('ephemeral_model') }} """ models_ephemeral_model_sql = """{{ config(materialized='ephemeral') }} select 1 as id """ models_descendant_model_sql = """select * from {{ source('test_source', 'test_table') }} """ models_multi_source_model_sql = """select * from {{ source('test_source', 'other_test_table')}} join {{ source('other_source', 'test_table')}} using (id) """ models_nonsource_descendant_sql = """select * from {{ schema }}.source """ models_newly_added_model_sql = """select 2 as id""" models_newly_added_error_model_sql = """select error from fake_table""" malformed_models_schema_yml = """version: 2 sources: - name: test_source loader: custom schema: "{{ var('test_run_schema') }}" tables: - name: test_table identifier: source data_tests: - relationships: # this is invalid (list of 3 1-key dicts instead of a single 3-key dict) - column_name: favorite_color - to: ref('descendant_model') - field: favorite_color """ malformed_models_descendant_model_sql = """select * from {{ source('test_source', 'test_table') }} """ filtered_models_schema_yml = """version: 2 sources: - name: test_source loader: custom freshness: warn_after: {count: 10, period: hour} error_after: {count: 1, period: day} filter: id > 1 schema: "{{ var(env_var('DBT_TEST_SCHEMA_NAME_VARIABLE')) }}" quoting: identifier: True tables: - name: test_table identifier: source loaded_at_field: updated_at freshness: error_after: {count: 18, period: hour} filter: id > 101 """ macros_macro_sql = """{% macro override_me() -%} {{ exceptions.raise_compiler_error('this is a bad macro') }} {%- endmacro %} {% macro happy_little_macro() -%} {{ override_me() }} {%- endmacro %} {% macro vacuum_source(source_name, table_name) -%} {% call statement('stmt', auto_begin=false, fetch_result=false) %} vacuum {{ source(source_name, table_name) }} {% endcall %} {%- endmacro %} """ seeds_source_csv = """favorite_color,id,first_name,email,ip_address,updated_at blue,1,Larry,lking0@miitbeian.gov.cn,'69.135.206.194',2008-09-12 19:08:31 blue,2,Larry,lperkins1@toplist.cz,'64.210.133.162',1978-05-09 04:15:14 blue,3,Anna,amontgomery2@miitbeian.gov.cn,'168.104.64.114',2011-10-16 04:07:57 blue,4,Sandra,sgeorge3@livejournal.com,'229.235.252.98',1973-07-19 10:52:43 blue,5,Fred,fwoods4@google.cn,'78.229.170.124',2012-09-30 16:38:29 blue,6,Stephen,shanson5@livejournal.com,'182.227.157.105',1995-11-07 21:40:50 blue,7,William,wmartinez6@upenn.edu,'135.139.249.50',1982-09-05 03:11:59 blue,8,Jessica,jlong7@hao123.com,'203.62.178.210',1991-10-16 11:03:15 blue,9,Douglas,dwhite8@tamu.edu,'178.187.247.1',1979-10-01 09:49:48 blue,10,Lisa,lcoleman9@nydailynews.com,'168.234.128.249',2011-05-26 07:45:49 blue,11,Ralph,rfieldsa@home.pl,'55.152.163.149',1972-11-18 19:06:11 blue,12,Louise,lnicholsb@samsung.com,'141.116.153.154',2014-11-25 20:56:14 blue,13,Clarence,cduncanc@sfgate.com,'81.171.31.133',2011-11-17 07:02:36 blue,14,Daniel,dfranklind@omniture.com,'8.204.211.37',1980-09-13 00:09:04 blue,15,Katherine,klanee@auda.org.au,'176.96.134.59',1997-08-22 19:36:56 blue,16,Billy,bwardf@wikia.com,'214.108.78.85',2003-10-19 02:14:47 blue,17,Annie,agarzag@ocn.ne.jp,'190.108.42.70',1988-10-28 15:12:35 blue,18,Shirley,scolemanh@fastcompany.com,'109.251.164.84',1988-08-24 10:50:57 blue,19,Roger,rfrazieri@scribd.com,'38.145.218.108',1985-12-31 15:17:15 blue,20,Lillian,lstanleyj@goodreads.com,'47.57.236.17',1970-06-08 02:09:05 blue,21,Aaron,arodriguezk@nps.gov,'205.245.118.221',1985-10-11 23:07:49 blue,22,Patrick,pparkerl@techcrunch.com,'19.8.100.182',2006-03-29 12:53:56 blue,23,Phillip,pmorenom@intel.com,'41.38.254.103',2011-11-07 15:35:43 blue,24,Henry,hgarcian@newsvine.com,'1.191.216.252',2008-08-28 08:30:44 blue,25,Irene,iturnero@opera.com,'50.17.60.190',1994-04-01 07:15:02 blue,26,Andrew,adunnp@pen.io,'123.52.253.176',2000-11-01 06:03:25 blue,27,David,dgutierrezq@wp.com,'238.23.203.42',1988-01-25 07:29:18 blue,28,Henry,hsanchezr@cyberchimps.com,'248.102.2.185',1983-01-01 13:36:37 blue,29,Evelyn,epetersons@gizmodo.com,'32.80.46.119',1979-07-16 17:24:12 blue,30,Tammy,tmitchellt@purevolume.com,'249.246.167.88',2001-04-03 10:00:23 blue,31,Jacqueline,jlittleu@domainmarket.com,'127.181.97.47',1986-02-11 21:35:50 blue,32,Earl,eortizv@opera.com,'166.47.248.240',1996-07-06 08:16:27 blue,33,Juan,jgordonw@sciencedirect.com,'71.77.2.200',1987-01-31 03:46:44 blue,34,Diane,dhowellx@nyu.edu,'140.94.133.12',1994-06-11 02:30:05 blue,35,Randy,rkennedyy@microsoft.com,'73.255.34.196',2005-05-26 20:28:39 blue,36,Janice,jriveraz@time.com,'22.214.227.32',1990-02-09 04:16:52 blue,37,Laura,lperry10@diigo.com,'159.148.145.73',2015-03-17 05:59:25 blue,38,Gary,gray11@statcounter.com,'40.193.124.56',1970-01-27 10:04:51 blue,39,Jesse,jmcdonald12@typepad.com,'31.7.86.103',2009-03-14 08:14:29 blue,40,Sandra,sgonzalez13@goodreads.com,'223.80.168.239',1993-05-21 14:08:54 blue,41,Scott,smoore14@archive.org,'38.238.46.83',1980-08-30 11:16:56 blue,42,Phillip,pevans15@cisco.com,'158.234.59.34',2011-12-15 23:26:31 blue,43,Steven,sriley16@google.ca,'90.247.57.68',2011-10-29 19:03:28 blue,44,Deborah,dbrown17@hexun.com,'179.125.143.240',1995-04-10 14:36:07 blue,45,Lori,lross18@ow.ly,'64.80.162.180',1980-12-27 16:49:15 blue,46,Sean,sjackson19@tumblr.com,'240.116.183.69',1988-06-12 21:24:45 blue,47,Terry,tbarnes1a@163.com,'118.38.213.137',1997-09-22 16:43:19 blue,48,Dorothy,dross1b@ebay.com,'116.81.76.49',2005-02-28 13:33:24 blue,49,Samuel,swashington1c@house.gov,'38.191.253.40',1989-01-19 21:15:48 blue,50,Ralph,rcarter1d@tinyurl.com,'104.84.60.174',2007-08-11 10:21:49 green,51,Wayne,whudson1e@princeton.edu,'90.61.24.102',1983-07-03 16:58:12 green,52,Rose,rjames1f@plala.or.jp,'240.83.81.10',1995-06-08 11:46:23 green,53,Louise,lcox1g@theglobeandmail.com,'105.11.82.145',2016-09-19 14:45:51 green,54,Kenneth,kjohnson1h@independent.co.uk,'139.5.45.94',1976-08-17 11:26:19 green,55,Donna,dbrown1i@amazon.co.uk,'19.45.169.45',2006-05-27 16:51:40 green,56,Johnny,jvasquez1j@trellian.com,'118.202.238.23',1975-11-17 08:42:32 green,57,Patrick,pramirez1k@tamu.edu,'231.25.153.198',1997-08-06 11:51:09 green,58,Helen,hlarson1l@prweb.com,'8.40.21.39',1993-08-04 19:53:40 green,59,Patricia,pspencer1m@gmpg.org,'212.198.40.15',1977-08-03 16:37:27 green,60,Joseph,jspencer1n@marriott.com,'13.15.63.238',2005-07-23 20:22:06 green,61,Phillip,pschmidt1o@blogtalkradio.com,'177.98.201.190',1976-05-19 21:47:44 green,62,Joan,jwebb1p@google.ru,'105.229.170.71',1972-09-07 17:53:47 green,63,Phyllis,pkennedy1q@imgur.com,'35.145.8.244',2000-01-01 22:33:37 green,64,Katherine,khunter1r@smh.com.au,'248.168.205.32',1991-01-09 06:40:24 green,65,Laura,lvasquez1s@wiley.com,'128.129.115.152',1997-10-23 12:04:56 green,66,Juan,jdunn1t@state.gov,'44.228.124.51',2004-11-10 05:07:35 green,67,Judith,jholmes1u@wiley.com,'40.227.179.115',1977-08-02 17:01:45 green,68,Beverly,bbaker1v@wufoo.com,'208.34.84.59',2016-03-06 20:07:23 green,69,Lawrence,lcarr1w@flickr.com,'59.158.212.223',1988-09-13 06:07:21 green,70,Gloria,gwilliams1x@mtv.com,'245.231.88.33',1995-03-18 22:32:46 green,71,Steven,ssims1y@cbslocal.com,'104.50.58.255',2001-08-05 21:26:20 green,72,Betty,bmills1z@arstechnica.com,'103.177.214.220',1981-12-14 21:26:54 green,73,Mildred,mfuller20@prnewswire.com,'151.158.8.130',2000-04-19 10:13:55 green,74,Donald,dday21@icq.com,'9.178.102.255',1972-12-03 00:58:24 green,75,Eric,ethomas22@addtoany.com,'85.2.241.227',1992-11-01 05:59:30 green,76,Joyce,jarmstrong23@sitemeter.com,'169.224.20.36',1985-10-24 06:50:01 green,77,Maria,mmartinez24@amazonaws.com,'143.189.167.135',2005-10-05 05:17:42 green,78,Harry,hburton25@youtube.com,'156.47.176.237',1978-03-26 05:53:33 green,79,Kevin,klawrence26@hao123.com,'79.136.183.83',1994-10-12 04:38:52 green,80,David,dhall27@prweb.com,'133.149.172.153',1976-12-15 16:24:24 green,81,Kathy,kperry28@twitter.com,'229.242.72.228',1979-03-04 02:58:56 green,82,Adam,aprice29@elegantthemes.com,'13.145.21.10',1982-11-07 11:46:59 green,83,Brandon,bgriffin2a@va.gov,'73.249.128.212',2013-10-30 05:30:36 green,84,Henry,hnguyen2b@discovery.com,'211.36.214.242',1985-01-09 06:37:27 green,85,Eric,esanchez2c@edublogs.org,'191.166.188.251',2004-05-01 23:21:42 green,86,Jason,jlee2d@jimdo.com,'193.92.16.182',1973-01-08 09:05:39 green,87,Diana,drichards2e@istockphoto.com,'19.130.175.245',1994-10-05 22:50:49 green,88,Andrea,awelch2f@abc.net.au,'94.155.233.96',2002-04-26 08:41:44 green,89,Louis,lwagner2g@miitbeian.gov.cn,'26.217.34.111',2003-08-25 07:56:39 green,90,Jane,jsims2h@seesaa.net,'43.4.220.135',1987-03-20 20:39:04 green,91,Larry,lgrant2i@si.edu,'97.126.79.34',2000-09-07 20:26:19 green,92,Louis,ldean2j@prnewswire.com,'37.148.40.127',2011-09-16 20:12:14 green,93,Jennifer,jcampbell2k@xing.com,'38.106.254.142',1988-07-15 05:06:49 green,94,Wayne,wcunningham2l@google.com.hk,'223.28.26.187',2009-12-15 06:16:54 green,95,Lori,lstevens2m@icq.com,'181.250.181.58',1984-10-28 03:29:19 green,96,Judy,jsimpson2n@marriott.com,'180.121.239.219',1986-02-07 15:18:10 green,97,Phillip,phoward2o@usa.gov,'255.247.0.175',2002-12-26 08:44:45 green,98,Gloria,gwalker2p@usa.gov,'156.140.7.128',1997-10-04 07:58:58 green,99,Paul,pjohnson2q@umn.edu,'183.59.198.197',1991-11-14 12:33:55 green,100,Frank,fgreene2r@blogspot.com,'150.143.68.121',2010-06-12 23:55:39 """ seeds_other_table_csv = """id,first_name 1,Larry 2,Curly 3,Moe """ seeds_expected_multi_source_csv = """id,first_name,color 1,Larry,blue 2,Curly,red 3,Moe,green """ seeds_other_source_table_csv = """id,color 1,blue 2,red 3,green """ malformed_schema_tests_schema_yml = """version: 2 sources: - name: test_source schema: "{{ var('test_run_schema') }}" tables: - name: test_table identifier: source columns: - name: favorite_color data_tests: - relationships: to: ref('model') # this will get rendered as its literal field: "{{ 'favorite' ~ 'color' }}" """ malformed_schema_tests_model_sql = """select * from {{ source('test_source', 'test_table') }} """ basic_source_schema_yml = """version: 2 sources: - name: test_source tables: - name: test_table - name: other_source tables: - name: test_table """ disabled_source_level_schema_yml = """version: 2 sources: - name: test_source config: enabled: False tables: - name: test_table - name: disabled_test_table """ disabled_source_table_schema_yml = """version: 2 sources: - name: test_source tables: - name: test_table - name: disabled_test_table config: enabled: False """ all_configs_everywhere_schema_yml = """version: 2 sources: - name: test_source config: enabled: False tables: - name: test_table config: enabled: True - name: other_test_table """ all_configs_not_table_schema_yml = """version: 2 sources: - name: test_source config: enabled: True tables: - name: test_table - name: other_test_table """ all_configs_project_source_schema_yml = """version: 2 sources: - name: test_source tables: - name: test_table config: enabled: True - name: other_test_table """ invalid_config_source_schema_yml = """version: 2 sources: - name: test_source tables: - name: test_table config: enabled: True and False - name: other_test_table """ collect_freshness_macro_override_previous_return_signature = """ {% macro collect_freshness(source, loaded_at_field, filter) %} {% call statement('collect_freshness', fetch_result=True, auto_begin=False) -%} select max({{ loaded_at_field }}) as max_loaded_at, {{ current_timestamp() }} as snapshotted_at from {{ source }} {% if filter %} where {{ filter }} {% endif %} {% endcall %} {{ return(load_result('collect_freshness').table) }} {% endmacro %} """ freshness_via_metadata_schema_yml = """version: 2 sources: - name: test_source loader: custom freshness: warn_after: {count: 10, period: hour} error_after: {count: 1, period: day} schema: my_schema quoting: identifier: True tables: - name: test_table identifier: source """ freshness_via_custom_sql_schema_yml = """version: 2 sources: - name: test_source freshness: warn_after: {count: 10, period: hour} schema: "{{ var(env_var('DBT_TEST_SCHEMA_NAME_VARIABLE')) }}" quoting: identifier: True tags: - my_test_source_tag tables: - name: source_a identifier: source loaded_at_field: "{{ var('test_loaded_at') | as_text }}" - name: source_b identifier: source loaded_at_query: "select max({{ var('test_loaded_at') | as_text }}) from {{this}}" - name: source_c identifier: source loaded_at_query: "select {{current_timestamp()}}" """ freshness_via_custom_sql_config_schema_yml = """version: 2 sources: - name: test_source config: freshness: warn_after: {count: 10, period: hour} schema: "{{ var(env_var('DBT_TEST_SCHEMA_NAME_VARIABLE')) }}" quoting: identifier: True tags: - my_test_source_tag tables: - name: source_a identifier: source loaded_at_field: "{{ var('test_loaded_at') | as_text }}" - name: source_b identifier: source config: loaded_at_query: "select max({{ var('test_loaded_at') | as_text }}) from {{this}}" - name: source_c identifier: source config: loaded_at_query: "select {{current_timestamp()}}" """ freshness_via_custom_sql_source_config_schema_yml = """version: 2 sources: - name: test_source config: freshness: warn_after: {count: 10, period: hour} loaded_at_query: "select {{current_timestamp()}}" schema: "{{ var(env_var('DBT_TEST_SCHEMA_NAME_VARIABLE')) }}" quoting: identifier: True tags: - my_test_source_tag tables: - name: source_c identifier: source """ freshness_with_explicit_null_in_table_schema_yml = """version: 2 sources: - name: test_source schema: "{{ var(env_var('DBT_TEST_SCHEMA_NAME_VARIABLE')) }}" freshness: warn_after: count: 24 period: hour quoting: identifier: True tables: - name: source_a loaded_at_field: "{{ var('test_loaded_at') | as_text }}" config: freshness: null """ freshness_with_explicit_null_in_source_schema_yml = """version: 2 sources: - name: test_source schema: "{{ var(env_var('DBT_TEST_SCHEMA_NAME_VARIABLE')) }}" config: freshness: null quoting: identifier: True tables: - name: source_a loaded_at_field: "{{ var('test_loaded_at') | as_text }}" """ source_config_loaded_at_query_config_level = """ sources: - name: test_source config: loaded_at_query: 'select 1' tables: - name: test_table """ source_config_loaded_at_field_config_level = """ sources: - name: test_source config: loaded_at_field: 'id' tables: - name: test_table """ source_table_config_loaded_at_field_config_level = """ sources: - name: test_source tables: - name: test_table config: loaded_at_field: 'id' """ source_table_config_loaded_at_query_config_level = """ sources: - name: test_source tables: - name: test_table config: loaded_at_query: 'select 1' """ source_table_config_loaded_at_query_not_set_if_field_present = """ sources: - name: test_source config: loaded_at_query: 'select 1' tables: - name: test_table config: loaded_at_field: 'id' """ # Legacy top-level support source_config_loaded_at_field_top_level = """ sources: - name: test_source loaded_at_field: 'id' tables: - name: test_table """ source_config_loaded_at_query_top_level = """ sources: - name: test_source loaded_at_query: 'select 1' tables: - name: test_table """ table_config_loaded_at_field_top_level = """ sources: - name: test_source tables: - name: test_table loaded_at_field: 'id' """ table_config_loaded_at_query_top_level = """ sources: - name: test_source loaded_at_query: 'select 1' tables: - name: test_table loaded_at_query: 'select 1' """ source_table_config_loaded_at_query_not_set_if_field_present_top_level = """ sources: - name: test_source loaded_at_query: 'select 1' tables: - name: test_table loaded_at_field: 'id' """ ================================================ FILE: tests/functional/sources/test_name_chars.py ================================================ from dbt.tests.util import get_manifest, run_dbt, write_file from tests.fixtures.jaffle_shop import JaffleShopProject # Note: in an actual file (as opposed to a string that we write into a files) # there would only be a single backslash. sources_yml = """ sources: - name: something_else database: raw schema: jaffle_shop tables: - name: "\\"/test/orders\\"" - name: customers """ class TestNameChars(JaffleShopProject): def test_quotes_in_table_names(self, project): # Write out a sources definition that includes a table name with quotes and a forward slash # Note: forward slashes are not legal in filenames in Linux (or Windows), # so we won't see forward slashes in model names, because they come from file names. write_file(sources_yml, project.project_root, "models", "sources.yml") manifest = run_dbt(["parse"]) assert len(manifest.sources) == 2 assert 'source.jaffle_shop.something_else."/test/orders"' in manifest.sources.keys() # We've written out the manifest.json artifact, we want to ensure # that it can be read in again (the json is valid). # Note: the key in the json actually looks like: "source.jaffle_shop.something_else.\"/test/orders\"" new_manifest = get_manifest(project.project_root) assert new_manifest assert 'source.jaffle_shop.something_else."/test/orders"' in new_manifest.sources.keys() ================================================ FILE: tests/functional/sources/test_simple_source.py ================================================ import os import pytest import yaml from dbt.exceptions import ParsingError from dbt.tests.util import ( check_relations_equal, check_table_does_not_exist, run_dbt, update_config_file, ) from tests.functional.sources.common_source_setup import BaseSourcesTest from tests.functional.sources.fixtures import ( macros_macro_sql, malformed_models_descendant_model_sql, malformed_models_schema_yml, malformed_schema_tests_model_sql, malformed_schema_tests_schema_yml, ) class SuccessfulSourcesTest(BaseSourcesTest): @pytest.fixture(scope="class", autouse=True) def setUp(self, project): self.run_dbt_with_vars(project, ["seed"]) os.environ["DBT_ENV_CUSTOM_ENV_key"] = "value" yield del os.environ["DBT_ENV_CUSTOM_ENV_key"] @pytest.fixture(scope="class") def macros(self): return {"macro.sql": macros_macro_sql} def _create_schemas(self, project): schema = self.alternative_schema(project.test_schema) project.run_sql(f"drop schema if exists {schema} cascade") project.run_sql(f"create schema {schema}") def alternative_schema(self, test_schema): return test_schema + "_other" @pytest.fixture(scope="class", autouse=True) def createDummyTables(self, project): self._create_schemas(project) project.run_sql("create table {}.dummy_table (id int)".format(project.test_schema)) project.run_sql( "create view {}.external_view as (select * from {}.dummy_table)".format( self.alternative_schema(project.test_schema), project.test_schema ) ) def run_dbt_with_vars(self, project, cmd, *args, **kwargs): vars_dict = { "test_run_schema": project.test_schema, "test_run_alt_schema": self.alternative_schema(project.test_schema), "test_loaded_at": project.adapter.quote("updated_at"), } cmd.extend(["--vars", yaml.safe_dump(vars_dict)]) return run_dbt(cmd, *args, **kwargs) class TestBasicSource(SuccessfulSourcesTest): def test_basic_source_def(self, project): results = self.run_dbt_with_vars(project, ["run"]) assert len(results) == 4 check_relations_equal( project.adapter, ["source", "descendant_model", "nonsource_descendant"] ) check_relations_equal(project.adapter, ["expected_multi_source", "multi_source_model"]) results = self.run_dbt_with_vars(project, ["test"]) assert len(results) == 8 class TestSourceSelector(SuccessfulSourcesTest): def test_source_selector(self, project): # only one of our models explicitly depends upon a source results = self.run_dbt_with_vars( project, ["run", "--models", "source:test_source.test_table+"] ) assert len(results) == 1 check_relations_equal(project.adapter, ["source", "descendant_model"]) check_table_does_not_exist(project.adapter, "nonsource_descendant") check_table_does_not_exist(project.adapter, "multi_source_model") # do the same thing, but with tags results = self.run_dbt_with_vars( project, ["run", "--models", "tag:my_test_source_table_tag+"] ) assert len(results) == 1 results = self.run_dbt_with_vars( project, ["test", "--models", "source:test_source.test_table+"] ) assert len(results) == 6 results = self.run_dbt_with_vars( project, ["test", "--models", "tag:my_test_source_table_tag+"] ) assert len(results) == 6 results = self.run_dbt_with_vars(project, ["test", "--models", "tag:my_test_source_tag+"]) # test_table + other_test_table assert len(results) == 8 results = self.run_dbt_with_vars(project, ["test", "--models", "tag:id_column"]) # all 4 id column tests assert len(results) == 4 class TestEmptySource(SuccessfulSourcesTest): def test_empty_source_def(self, project): # sources themselves can never be selected, so nothing should be run results = self.run_dbt_with_vars( project, ["run", "--models", "source:test_source.test_table"] ) check_table_does_not_exist(project.adapter, "nonsource_descendant") check_table_does_not_exist(project.adapter, "multi_source_model") check_table_does_not_exist(project.adapter, "descendant_model") assert len(results) == 0 class TestSourceDef(SuccessfulSourcesTest): def test_source_only_def(self, project): results = self.run_dbt_with_vars(project, ["run", "--models", "source:other_source+"]) assert len(results) == 1 check_relations_equal(project.adapter, ["expected_multi_source", "multi_source_model"]) check_table_does_not_exist(project.adapter, "nonsource_descendant") check_table_does_not_exist(project.adapter, "descendant_model") results = self.run_dbt_with_vars(project, ["run", "--models", "source:test_source+"]) assert len(results) == 2 check_relations_equal(project.adapter, ["source", "descendant_model"]) check_relations_equal(project.adapter, ["expected_multi_source", "multi_source_model"]) check_table_does_not_exist(project.adapter, "nonsource_descendant") class TestSourceChildrenParents(SuccessfulSourcesTest): def test_source_childrens_parents(self, project): results = self.run_dbt_with_vars(project, ["run", "--models", "@source:test_source"]) assert len(results) == 2 check_relations_equal(project.adapter, ["source", "descendant_model"]) check_relations_equal(project.adapter, ["expected_multi_source", "multi_source_model"]) check_table_does_not_exist(project.adapter, "nonsource_descendant") class TestSourceRunOperation(SuccessfulSourcesTest): def test_run_operation_source(self, project): kwargs = '{"source_name": "test_source", "table_name": "test_table"}' self.run_dbt_with_vars(project, ["run-operation", "vacuum_source", "--args", kwargs]) class TestMalformedSources(BaseSourcesTest): # even seeds should fail, because parsing is what's raising @pytest.fixture(scope="class") def models(self): return { "schema.yml": malformed_models_schema_yml, "descendant_model.sql": malformed_models_descendant_model_sql, } def test_malformed_schema_will_break_run(self, project): with pytest.raises(ParsingError): self.run_dbt_with_vars(project, ["seed"]) class TestRenderingInSourceTests(BaseSourcesTest): @pytest.fixture(scope="class") def models(self): return { "schema.yml": malformed_schema_tests_schema_yml, "model.sql": malformed_schema_tests_model_sql, } def test_render_in_source_tests(self, project): self.run_dbt_with_vars(project, ["seed"]) self.run_dbt_with_vars(project, ["run"]) # syntax error at or near "{", because the test isn't rendered self.run_dbt_with_vars(project, ["test"], expect_pass=False) class TestUnquotedSources(SuccessfulSourcesTest): def test_catalog(self, project): new_quoting_config = { "quoting": { "identifier": False, "schema": False, "database": False, } } update_config_file(new_quoting_config, project.project_root, "dbt_project.yml") self.run_dbt_with_vars(project, ["run"]) self.run_dbt_with_vars(project, ["docs", "generate"]) ================================================ FILE: tests/functional/sources/test_source_configs.py ================================================ import pytest from dbt.artifacts.resources import SourceConfig from dbt.tests.util import get_manifest, run_dbt, update_config_file from dbt_common.dataclass_schema import ValidationError from tests.functional.sources.fixtures import ( all_configs_everywhere_schema_yml, all_configs_not_table_schema_yml, all_configs_project_source_schema_yml, basic_source_schema_yml, disabled_source_level_schema_yml, disabled_source_table_schema_yml, invalid_config_source_schema_yml, source_config_loaded_at_field_config_level, source_config_loaded_at_field_top_level, source_config_loaded_at_query_config_level, source_config_loaded_at_query_top_level, source_table_config_loaded_at_field_config_level, source_table_config_loaded_at_query_config_level, source_table_config_loaded_at_query_not_set_if_field_present, source_table_config_loaded_at_query_not_set_if_field_present_top_level, table_config_loaded_at_field_top_level, table_config_loaded_at_query_top_level, ) class SourceConfigTests: @pytest.fixture(scope="class", autouse=True) def setUp(self): pytest.expected_config = SourceConfig( enabled=True, ) # Test enabled config in dbt_project.yml # expect pass, already implemented class TestSourceEnabledConfigProjectLevel(SourceConfigTests): @pytest.fixture(scope="class") def models(self): return { "schema.yml": basic_source_schema_yml, } @pytest.fixture(scope="class") def project_config_update(self): return { "sources": { "test": { "test_source": { "enabled": True, }, } } } def test_enabled_source_config_dbt_project(self, project): run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "source.test.test_source.test_table" in manifest.sources new_enabled_config = { "sources": { "test": { "test_source": { "enabled": False, }, } } } update_config_file(new_enabled_config, project.project_root, "dbt_project.yml") run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert ( "source.test.test_source.test_table" not in manifest.sources ) # or should it be there with enabled: false?? assert "source.test.other_source.test_table" in manifest.sources # Test enabled config at sources level in yml file class TestConfigYamlSourceLevel(SourceConfigTests): @pytest.fixture(scope="class") def models(self): return { "schema.yml": disabled_source_level_schema_yml, } def test_source_config_yaml_source_level(self, project): run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "source.test.test_source.test_table" not in manifest.sources assert "source.test.test_source.disabled_test_table" not in manifest.sources # Test enabled config at source table level in yaml file class TestConfigYamlSourceTable(SourceConfigTests): @pytest.fixture(scope="class") def models(self): return { "schema.yml": disabled_source_table_schema_yml, } def test_source_config_yaml_source_table(self, project): run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "source.test.test_source.test_table" in manifest.sources assert "source.test.test_source.disabled_test_table" not in manifest.sources # Test inheritence - set configs at project, source, and source-table level - expect source-table level to win class TestSourceConfigsInheritence1(SourceConfigTests): @pytest.fixture(scope="class") def models(self): return {"schema.yml": all_configs_everywhere_schema_yml} @pytest.fixture(scope="class") def project_config_update(self): return {"sources": {"enabled": True}} def test_source_all_configs_source_table(self, project): run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "source.test.test_source.test_table" in manifest.sources assert "source.test.test_source.other_test_table" not in manifest.sources config_test_table = manifest.sources.get("source.test.test_source.test_table").config assert isinstance(config_test_table, SourceConfig) assert config_test_table == pytest.expected_config # Test inheritence - set configs at project and source level - expect source level to win class TestSourceConfigsInheritence2(SourceConfigTests): @pytest.fixture(scope="class") def models(self): return {"schema.yml": all_configs_not_table_schema_yml} @pytest.fixture(scope="class") def project_config_update(self): return {"sources": {"enabled": False}} def test_source_two_configs_source_level(self, project): run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "source.test.test_source.test_table" in manifest.sources assert "source.test.test_source.other_test_table" in manifest.sources config_test_table = manifest.sources.get("source.test.test_source.test_table").config config_other_test_table = manifest.sources.get( "source.test.test_source.other_test_table" ).config assert isinstance(config_test_table, SourceConfig) assert isinstance(config_other_test_table, SourceConfig) assert config_test_table == config_other_test_table assert config_test_table == pytest.expected_config # Test inheritence - set configs at project and source-table level - expect source-table level to win class TestSourceConfigsInheritence3(SourceConfigTests): @pytest.fixture(scope="class") def models(self): return {"schema.yml": all_configs_project_source_schema_yml} @pytest.fixture(scope="class") def project_config_update(self): return {"sources": {"enabled": False}} def test_source_two_configs_source_table(self, project): run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "source.test.test_source.test_table" in manifest.sources assert "source.test.test_source.other_test_table" not in manifest.sources config_test_table = manifest.sources.get("source.test.test_source.test_table").config assert isinstance(config_test_table, SourceConfig) assert config_test_table == pytest.expected_config # Test invalid source configs class TestInvalidSourceConfig(SourceConfigTests): @pytest.fixture(scope="class") def models(self): return { "schema.yml": invalid_config_source_schema_yml, } def test_invalid_config_source(self, project): with pytest.raises(ValidationError) as excinfo: run_dbt(["parse"]) expected_msg = "'True and False' is not of type 'boolean'" assert expected_msg in str(excinfo.value) class TestSourceLoadedAtFieldConfigLevel: @pytest.fixture(scope="class") def models(self): return { "schema.yml": source_config_loaded_at_field_config_level, } def test_loaded_at_field_config(self, project): manifest = run_dbt(["parse"]) source = manifest.sources["source.test.test_source.test_table"] assert source.loaded_at_field == "id" assert source.config.loaded_at_field == "id" class TestSourceLoadedAtQueryConfigLevel: @pytest.fixture(scope="class") def models(self): return { "schema.yml": source_config_loaded_at_query_config_level, } def test_loaded_at_field_config(self, project): manifest = run_dbt(["parse"]) source = manifest.sources["source.test.test_source.test_table"] assert source.loaded_at_query == "select 1" assert source.config.loaded_at_query == "select 1" class TestTableLoadedAtFieldConfigLevel: @pytest.fixture(scope="class") def models(self): return { "schema.yml": source_table_config_loaded_at_field_config_level, } def test_loaded_at_field_config(self, project): manifest = run_dbt(["parse"]) source = manifest.sources["source.test.test_source.test_table"] assert source.loaded_at_field == "id" assert source.config.loaded_at_field == "id" class TestTableLoadedAtQueryConfigLevel: @pytest.fixture(scope="class") def models(self): return { "schema.yml": source_table_config_loaded_at_query_config_level, } def test_loaded_at_field_config(self, project): manifest = run_dbt(["parse"]) source = manifest.sources["source.test.test_source.test_table"] assert source.loaded_at_query == "select 1" assert source.config.loaded_at_query == "select 1" class TestSourceLoadedAtFieldTopLevel: @pytest.fixture(scope="class") def models(self): return { "schema.yml": source_config_loaded_at_field_top_level, } def test_loaded_at_field_config(self, project): manifest = run_dbt(["parse"]) source = manifest.sources["source.test.test_source.test_table"] assert source.loaded_at_field == "id" assert source.config.loaded_at_field == "id" class TestSourceLoadedAtQueryTopLevel: @pytest.fixture(scope="class") def models(self): return { "schema.yml": source_config_loaded_at_query_top_level, } def test_loaded_at_query_config(self, project): manifest = run_dbt(["parse"]) source = manifest.sources["source.test.test_source.test_table"] assert source.loaded_at_query == "select 1" assert source.config.loaded_at_query == "select 1" class TestTableLoadedAtFieldTopLevel: @pytest.fixture(scope="class") def models(self): return { "schema.yml": table_config_loaded_at_field_top_level, } def test_loaded_at_field_config(self, project): manifest = run_dbt(["parse"]) source = manifest.sources["source.test.test_source.test_table"] assert source.loaded_at_field == "id" assert source.config.loaded_at_field == "id" class TestTableLoadedAtQueryTopLevel: @pytest.fixture(scope="class") def models(self): return { "schema.yml": table_config_loaded_at_query_top_level, } def test_loaded_at_query_config(self, project): manifest = run_dbt(["parse"]) source = manifest.sources["source.test.test_source.test_table"] assert source.loaded_at_query == "select 1" assert source.config.loaded_at_query == "select 1" class TestTableLoadedAtQueryNoneWhenFieldSetConfigLevel: @pytest.fixture(scope="class") def models(self): return { "schema.yml": source_table_config_loaded_at_query_not_set_if_field_present, } def test_loaded_at_query_config(self, project): manifest = run_dbt(["parse"]) source = manifest.sources["source.test.test_source.test_table"] assert source.loaded_at_field == "id" assert source.config.loaded_at_field == "id" assert source.loaded_at_query is None assert source.config.loaded_at_query is None class TestTableLoadedAtQueryNoneWhenFieldSetTopLevel: @pytest.fixture(scope="class") def models(self): return { "schema.yml": source_table_config_loaded_at_query_not_set_if_field_present_top_level, } def test_loaded_at_query_config(self, project): manifest = run_dbt(["parse"]) source = manifest.sources["source.test.test_source.test_table"] assert source.loaded_at_field == "id" assert source.config.loaded_at_field == "id" assert source.loaded_at_query is None assert source.config.loaded_at_query is None ================================================ FILE: tests/functional/sources/test_source_fresher_state.py ================================================ import json import os import shutil from datetime import datetime, timedelta, timezone import pytest import dbt.version from dbt.contracts.results import FreshnessExecutionResultArtifact from dbt.tests.util import AnyFloat, AnyStringWith from dbt_common.exceptions import DbtInternalError from tests.functional.sources.common_source_setup import BaseSourcesTest from tests.functional.sources.fixtures import ( error_models_schema_yml, models_newly_added_error_model_sql, models_newly_added_model_sql, ) # TODO: We may create utility classes to handle reusable fixtures. def copy_to_previous_state(): shutil.copyfile("target/manifest.json", "previous_state/manifest.json") shutil.copyfile("target/run_results.json", "previous_state/run_results.json") class SuccessfulSourceFreshnessTest(BaseSourcesTest): @pytest.fixture(scope="class", autouse=True) def setUp(self, project): self.run_dbt_with_vars(project, ["seed"]) pytest._id = 101 pytest.freshness_start_time = datetime.now(timezone.utc).replace(tzinfo=None) # this is the db initial value pytest.last_inserted_time = "2016-09-19T14:45:51+00:00" os.environ["DBT_ENV_CUSTOM_ENV_key"] = "value" yield del os.environ["DBT_ENV_CUSTOM_ENV_key"] def _set_updated_at_to(self, project, delta): insert_time = datetime.now(timezone.utc).replace(tzinfo=None) + delta timestr = insert_time.strftime("%Y-%m-%d %H:%M:%S") # favorite_color,id,first_name,email,ip_address,updated_at insert_id = pytest._id pytest._id += 1 quoted_columns = ",".join( project.adapter.quote(c) for c in ("favorite_color", "id", "first_name", "email", "ip_address", "updated_at") ) kwargs = { "schema": project.test_schema, "time": timestr, "id": insert_id, "source": project.adapter.quote("source"), "quoted_columns": quoted_columns, } raw_code = """INSERT INTO {schema}.{source} ({quoted_columns}) VALUES ( 'blue',{id},'Jake','abc@example.com','192.168.1.1','{time}' )""".format( **kwargs ) project.run_sql(raw_code) pytest.last_inserted_time = insert_time.strftime("%Y-%m-%dT%H:%M:%S+00:00") def assertBetween(self, timestr, start, end=None): datefmt = "%Y-%m-%dT%H:%M:%S.%fZ" if end is None: end = datetime.now(timezone.utc).replace(tzinfo=None) parsed = datetime.strptime(timestr, datefmt) assert start <= parsed assert end >= parsed def _assert_freshness_results(self, path, state): assert os.path.exists(path) with open(path) as fp: data = json.load(fp) try: FreshnessExecutionResultArtifact.validate(data) except Exception: raise pytest.fail("FreshnessExecutionResultArtifact did not validate") assert set(data) == {"metadata", "results", "elapsed_time"} assert "generated_at" in data["metadata"] assert isinstance(data["elapsed_time"], float) self.assertBetween(data["metadata"]["generated_at"], pytest.freshness_start_time) assert ( data["metadata"]["dbt_schema_version"] == "https://schemas.getdbt.com/dbt/sources/v3.json" ) assert data["metadata"]["dbt_version"] == dbt.version.__version__ key = "key" if os.name == "nt": key = key.upper() assert data["metadata"]["env"] == {key: "value"} last_inserted_time = pytest.last_inserted_time assert len(data["results"]) == 1 assert data["results"] == [ { "unique_id": "source.test.test_source.test_table", "max_loaded_at": last_inserted_time, "snapshotted_at": AnyStringWith(), "max_loaded_at_time_ago_in_s": AnyFloat(), "status": state, "criteria": { "filter": None, "warn_after": {"count": 10, "period": "hour"}, "error_after": {"count": 18, "period": "hour"}, }, "adapter_response": {"_message": "SELECT 1", "code": "SELECT", "rows_affected": 1}, "thread_id": AnyStringWith("Thread-"), "execution_time": AnyFloat(), "timing": [ { "name": "compile", "started_at": AnyStringWith(), "completed_at": AnyStringWith(), }, { "name": "execute", "started_at": AnyStringWith(), "completed_at": AnyStringWith(), }, ], } ] class TestSourceFresherNothingToDo(SuccessfulSourceFreshnessTest): def test_source_fresher_nothing_to_do(self, project): self.run_dbt_with_vars(project, ["run"]) self._set_updated_at_to(project, timedelta(hours=-2)) previous_state_results = self.run_dbt_with_vars( project, ["source", "freshness", "-o", "previous_state/sources.json"] ) self._assert_freshness_results("previous_state/sources.json", "pass") copy_to_previous_state() current_state_results = self.run_dbt_with_vars( project, ["source", "freshness", "-o", "target/sources.json"] ) self._assert_freshness_results("target/sources.json", "pass") assert previous_state_results[0].max_loaded_at == current_state_results[0].max_loaded_at source_fresher_results = self.run_dbt_with_vars( project, ["test", "-s", "source_status:fresher+", "--defer", "--state", "./previous_state"], ) assert source_fresher_results.results == [] class TestSourceFresherRun(SuccessfulSourceFreshnessTest): def test_source_fresher_run_error(self, project): self.run_dbt_with_vars(project, ["run"]) previous_state_results = self.run_dbt_with_vars( project, ["source", "freshness", "-o", "previous_state/sources.json"], expect_pass=False, ) self._assert_freshness_results("previous_state/sources.json", "error") copy_to_previous_state() self._set_updated_at_to(project, timedelta(hours=-20)) current_state_results = self.run_dbt_with_vars( project, ["source", "freshness", "-o", "target/sources.json"], expect_pass=False, ) self._assert_freshness_results("target/sources.json", "error") assert previous_state_results[0].max_loaded_at < current_state_results[0].max_loaded_at source_fresher_results = self.run_dbt_with_vars( project, ["run", "-s", "source_status:fresher", "--defer", "--state", "previous_state"], ) assert source_fresher_results.results == [] source_fresher_plus_results = self.run_dbt_with_vars( project, ["run", "-s", "source_status:fresher+", "--defer", "--state", "previous_state"], ) nodes = set([elem.node.name for elem in source_fresher_plus_results]) assert nodes == {"descendant_model"} def test_source_fresher_run_warn(self, project): self.run_dbt_with_vars(project, ["run"]) self._set_updated_at_to(project, timedelta(hours=-17)) previous_state_results = self.run_dbt_with_vars( project, ["source", "freshness", "-o", "previous_state/sources.json"], expect_pass=True, ) self._assert_freshness_results("previous_state/sources.json", "warn") copy_to_previous_state() self._set_updated_at_to(project, timedelta(hours=-11)) current_state_results = self.run_dbt_with_vars( project, ["source", "freshness", "-o", "target/sources.json"] ) self._assert_freshness_results("target/sources.json", "warn") assert previous_state_results[0].max_loaded_at < current_state_results[0].max_loaded_at source_fresher_results = self.run_dbt_with_vars( project, ["run", "-s", "source_status:fresher", "--defer", "--state", "previous_state"], ) assert source_fresher_results.results == [] source_fresher_plus_results = self.run_dbt_with_vars( project, ["run", "-s", "source_status:fresher+", "--defer", "--state", "previous_state"], ) nodes = set([elem.node.name for elem in source_fresher_plus_results]) assert nodes == {"descendant_model"} def test_source_fresher_run_pass(self, project): self.run_dbt_with_vars(project, ["run"]) self._set_updated_at_to(project, timedelta(hours=-2)) previous_state_results = self.run_dbt_with_vars( project, ["source", "freshness", "-o", "previous_state/sources.json"] ) self._assert_freshness_results("previous_state/sources.json", "pass") copy_to_previous_state() self._set_updated_at_to(project, timedelta(hours=-1)) current_state_results = self.run_dbt_with_vars( project, ["source", "freshness", "-o", "target/sources.json"] ) self._assert_freshness_results("target/sources.json", "pass") assert previous_state_results[0].max_loaded_at < current_state_results[0].max_loaded_at source_fresher_results = self.run_dbt_with_vars( project, ["run", "-s", "source_status:fresher", "--defer", "--state", "previous_state"], ) assert source_fresher_results.results == [] source_fresher_plus_results = self.run_dbt_with_vars( project, ["run", "-s", "source_status:fresher+", "--defer", "--state", "previous_state"], ) nodes = set([elem.node.name for elem in source_fresher_plus_results]) assert nodes == {"descendant_model"} class TestSourceFresherBuildStateModified(SuccessfulSourceFreshnessTest): def test_source_fresher_build_state_modified_pass(self, project, project_root): self.run_dbt_with_vars(project, ["run"]) self._set_updated_at_to(project, timedelta(hours=-2)) previous_state_results = self.run_dbt_with_vars( project, ["source", "freshness", "-o", "previous_state/sources.json"] ) self._assert_freshness_results("previous_state/sources.json", "pass") self._set_updated_at_to(project, timedelta(hours=-1)) current_state_results = self.run_dbt_with_vars( project, ["source", "freshness", "-o", "target/sources.json"] ) self._assert_freshness_results("target/sources.json", "pass") assert previous_state_results[0].max_loaded_at < current_state_results[0].max_loaded_at models_path = project_root.join("models/") assert os.path.exists(models_path) with open(f"{models_path}/newly_added_model.sql", "w") as fp: fp.write(models_newly_added_model_sql) copy_to_previous_state() state_modified_results = self.run_dbt_with_vars( project, [ "build", "--select", "source_status:fresher+", "state:modified+", "--defer", "--state", "previous_state", ], ) nodes = set([elem.node.name for elem in state_modified_results]) assert nodes == { "newly_added_model", "source_unique_test_source_test_table_id", "unique_descendant_model_id", "not_null_descendant_model_id", "source_not_null_test_source_test_table_id", "descendant_model", "source_relationships_test_source_test_table_favorite_color__favorite_color__ref_descendant_model_", "relationships_descendant_model_favorite_color__favorite_color__source_test_source_test_table_", } class TestSourceFresherRuntimeError(SuccessfulSourceFreshnessTest): @pytest.fixture(scope="class") def models(self): return { "schema.yml": error_models_schema_yml, } def test_runtime_error_states(self, project): self.run_dbt_with_vars(project, ["run"]) previous_state_results = self.run_dbt_with_vars( project, ["source", "freshness", "-o", "previous_state/sources.json"], expect_pass=False, ) assert len(previous_state_results) == 1 assert previous_state_results[0].status == "runtime error" copy_to_previous_state() self._set_updated_at_to(project, timedelta(hours=-1)) current_state_results = self.run_dbt_with_vars( project, ["source", "freshness", "-o", "target/sources.json"], expect_pass=False, ) assert len(current_state_results) == 1 assert current_state_results[0].status == "runtime error" assert not hasattr(previous_state_results[0], "max_loaded_at") assert not hasattr(current_state_results[0], "max_loaded_at") source_fresher_results = self.run_dbt_with_vars( project, ["test", "-s", "source_status:fresher", "--defer", "--state", "previous_state"], ) assert source_fresher_results.results == [] class TestSourceFresherTest(SuccessfulSourceFreshnessTest): def test_source_fresher_run_error(self, project): self.run_dbt_with_vars(project, ["run"]) previous_state_results = self.run_dbt_with_vars( project, ["source", "freshness", "-o", "previous_state/sources.json"], expect_pass=False, ) self._assert_freshness_results("previous_state/sources.json", "error") copy_to_previous_state() self._set_updated_at_to(project, timedelta(hours=-20)) current_state_results = self.run_dbt_with_vars( project, ["source", "freshness", "-o", "target/sources.json"], expect_pass=False, ) self._assert_freshness_results("target/sources.json", "error") assert previous_state_results[0].max_loaded_at < current_state_results[0].max_loaded_at source_fresher_results = self.run_dbt_with_vars( project, ["test", "-s", "source_status:fresher", "--defer", "--state", "previous_state"], ) nodes = set([elem.node.name for elem in source_fresher_results]) assert nodes == { "relationships_descendant_model_favorite_color__favorite_color__source_test_source_test_table_", "source_not_null_test_source_test_table_id", "source_relationships_test_source_test_table_favorite_color__favorite_color__ref_descendant_model_", "source_unique_test_source_test_table_id", } source_fresher_plus_results = self.run_dbt_with_vars( project, ["test", "-s", "source_status:fresher+", "--defer", "--state", "previous_state"], ) nodes = set([elem.node.name for elem in source_fresher_plus_results]) assert nodes == { "relationships_descendant_model_favorite_color__favorite_color__source_test_source_test_table_", "source_not_null_test_source_test_table_id", "source_relationships_test_source_test_table_favorite_color__favorite_color__ref_descendant_model_", "source_unique_test_source_test_table_id", "unique_descendant_model_id", "not_null_descendant_model_id", } def test_source_fresher_test_warn(self, project): self.run_dbt_with_vars(project, ["run"]) self._set_updated_at_to(project, timedelta(hours=-17)) previous_state_results = self.run_dbt_with_vars( project, ["source", "freshness", "-o", "previous_state/sources.json"], expect_pass=True, ) self._assert_freshness_results("previous_state/sources.json", "warn") copy_to_previous_state() self._set_updated_at_to(project, timedelta(hours=-11)) current_state_results = self.run_dbt_with_vars( project, ["source", "freshness", "-o", "target/sources.json"], expect_pass=True, ) self._assert_freshness_results("target/sources.json", "warn") assert previous_state_results[0].max_loaded_at < current_state_results[0].max_loaded_at source_fresher_results = self.run_dbt_with_vars( project, ["test", "-s", "source_status:fresher", "--defer", "--state", "previous_state"], ) nodes = set([elem.node.name for elem in source_fresher_results]) assert nodes == { "relationships_descendant_model_favorite_color__favorite_color__source_test_source_test_table_", "source_not_null_test_source_test_table_id", "source_relationships_test_source_test_table_favorite_color__favorite_color__ref_descendant_model_", "source_unique_test_source_test_table_id", } source_fresher_plus_results = self.run_dbt_with_vars( project, ["test", "-s", "source_status:fresher+", "--defer", "--state", "previous_state"], ) nodes = set([elem.node.name for elem in source_fresher_plus_results]) assert nodes == { "relationships_descendant_model_favorite_color__favorite_color__source_test_source_test_table_", "source_not_null_test_source_test_table_id", "source_relationships_test_source_test_table_favorite_color__favorite_color__ref_descendant_model_", "source_unique_test_source_test_table_id", "unique_descendant_model_id", "not_null_descendant_model_id", } def test_source_fresher_test_pass(self, project): self.run_dbt_with_vars(project, ["run"]) self._set_updated_at_to(project, timedelta(hours=-2)) previous_state_results = self.run_dbt_with_vars( project, ["source", "freshness", "-o", "previous_state/sources.json"] ) self._assert_freshness_results("previous_state/sources.json", "pass") copy_to_previous_state() self._set_updated_at_to(project, timedelta(hours=-1)) current_state_results = self.run_dbt_with_vars( project, ["source", "freshness", "-o", "target/sources.json"] ) self._assert_freshness_results("target/sources.json", "pass") assert previous_state_results[0].max_loaded_at < current_state_results[0].max_loaded_at source_fresher_results = self.run_dbt_with_vars( project, ["test", "-s", "source_status:fresher", "--defer", "--state", "previous_state"], ) nodes = set([elem.node.name for elem in source_fresher_results]) assert nodes == { "relationships_descendant_model_favorite_color__favorite_color__source_test_source_test_table_", "source_not_null_test_source_test_table_id", "source_relationships_test_source_test_table_favorite_color__favorite_color__ref_descendant_model_", "source_unique_test_source_test_table_id", } source_fresher_plus_results = self.run_dbt_with_vars( project, ["test", "-s", "source_status:fresher+", "--defer", "--state", "previous_state"], ) nodes = set([elem.node.name for elem in source_fresher_plus_results]) assert nodes == { "relationships_descendant_model_favorite_color__favorite_color__source_test_source_test_table_", "source_not_null_test_source_test_table_id", "source_relationships_test_source_test_table_favorite_color__favorite_color__ref_descendant_model_", "source_unique_test_source_test_table_id", "unique_descendant_model_id", "not_null_descendant_model_id", } class TestSourceFresherBuild(SuccessfulSourceFreshnessTest): def test_source_fresher_build_error(self, project): self.run_dbt_with_vars(project, ["build"]) previous_state_results = self.run_dbt_with_vars( project, ["source", "freshness", "-o", "previous_state/sources.json"], expect_pass=False, ) self._assert_freshness_results("previous_state/sources.json", "error") copy_to_previous_state() self._set_updated_at_to(project, timedelta(hours=-20)) current_state_results = self.run_dbt_with_vars( project, ["source", "freshness", "-o", "target/sources.json"], expect_pass=False, ) self._assert_freshness_results("target/sources.json", "error") assert previous_state_results[0].max_loaded_at < current_state_results[0].max_loaded_at source_fresher_results = self.run_dbt_with_vars( project, ["build", "-s", "source_status:fresher", "--defer", "--state", "previous_state"], ) nodes = set([elem.node.name for elem in source_fresher_results]) assert nodes == { "relationships_descendant_model_favorite_color__favorite_color__source_test_source_test_table_", "source_not_null_test_source_test_table_id", "source_relationships_test_source_test_table_favorite_color__favorite_color__ref_descendant_model_", "source_unique_test_source_test_table_id", } source_fresher_plus_results = self.run_dbt_with_vars( project, ["build", "-s", "source_status:fresher+", "--defer", "--state", "previous_state"], ) nodes = set([elem.node.name for elem in source_fresher_plus_results]) assert nodes == { "descendant_model", "relationships_descendant_model_favorite_color__favorite_color__source_test_source_test_table_", "source_not_null_test_source_test_table_id", "source_relationships_test_source_test_table_favorite_color__favorite_color__ref_descendant_model_", "source_unique_test_source_test_table_id", "unique_descendant_model_id", "not_null_descendant_model_id", } def test_source_fresher_build_warn(self, project): self.run_dbt_with_vars(project, ["build"]) self._set_updated_at_to(project, timedelta(hours=-17)) previous_state_results = self.run_dbt_with_vars( project, ["source", "freshness", "-o", "previous_state/sources.json"], expect_pass=True, ) self._assert_freshness_results("previous_state/sources.json", "warn") copy_to_previous_state() self._set_updated_at_to(project, timedelta(hours=-11)) current_state_results = self.run_dbt_with_vars( project, ["source", "freshness", "-o", "target/sources.json"] ) self._assert_freshness_results("target/sources.json", "warn") assert previous_state_results[0].max_loaded_at < current_state_results[0].max_loaded_at source_fresher_results = self.run_dbt_with_vars( project, ["build", "-s", "source_status:fresher", "--defer", "--state", "previous_state"], ) nodes = set([elem.node.name for elem in source_fresher_results]) assert nodes == { "relationships_descendant_model_favorite_color__favorite_color__source_test_source_test_table_", "source_not_null_test_source_test_table_id", "source_relationships_test_source_test_table_favorite_color__favorite_color__ref_descendant_model_", "source_unique_test_source_test_table_id", } source_fresher_plus_results = self.run_dbt_with_vars( project, ["build", "-s", "source_status:fresher+", "--defer", "--state", "previous_state"], ) nodes = set([elem.node.name for elem in source_fresher_plus_results]) assert nodes == { "descendant_model", "relationships_descendant_model_favorite_color__favorite_color__source_test_source_test_table_", "source_not_null_test_source_test_table_id", "source_relationships_test_source_test_table_favorite_color__favorite_color__ref_descendant_model_", "source_unique_test_source_test_table_id", "unique_descendant_model_id", "not_null_descendant_model_id", } def test_source_fresher_build_pass(self, project): self.run_dbt_with_vars(project, ["build"]) self._set_updated_at_to(project, timedelta(hours=-2)) previous_state_results = self.run_dbt_with_vars( project, ["source", "freshness", "-o", "previous_state/sources.json"] ) self._assert_freshness_results("previous_state/sources.json", "pass") copy_to_previous_state() self._set_updated_at_to(project, timedelta(hours=-1)) current_state_results = self.run_dbt_with_vars( project, ["source", "freshness", "-o", "target/sources.json"] ) self._assert_freshness_results("target/sources.json", "pass") assert previous_state_results[0].max_loaded_at < current_state_results[0].max_loaded_at source_fresher_results = self.run_dbt_with_vars( project, ["build", "-s", "source_status:fresher", "--defer", "--state", "previous_state"], ) nodes = set([elem.node.name for elem in source_fresher_results]) assert nodes == { "relationships_descendant_model_favorite_color__favorite_color__source_test_source_test_table_", "source_not_null_test_source_test_table_id", "source_relationships_test_source_test_table_favorite_color__favorite_color__ref_descendant_model_", "source_unique_test_source_test_table_id", } source_fresher_plus_results = self.run_dbt_with_vars( project, ["build", "-s", "source_status:fresher+", "--defer", "--state", "previous_state"], ) nodes = set([elem.node.name for elem in source_fresher_plus_results]) assert nodes == { "descendant_model", "relationships_descendant_model_favorite_color__favorite_color__source_test_source_test_table_", "source_not_null_test_source_test_table_id", "source_relationships_test_source_test_table_favorite_color__favorite_color__ref_descendant_model_", "source_unique_test_source_test_table_id", "unique_descendant_model_id", "not_null_descendant_model_id", } class TestSourceFresherNoPreviousState(SuccessfulSourceFreshnessTest): def test_intentional_failure_no_previous_state(self, project): self.run_dbt_with_vars(project, ["run"]) # TODO add the current and previous but with previous as null with pytest.raises(DbtInternalError) as excinfo: self.run_dbt_with_vars( project, ["run", "-s", "source_status:fresher", "--state", "previous_state"], ) assert "No previous state comparison freshness results in sources.json" in str( excinfo.value ) class TestSourceFresherNoCurrentState(SuccessfulSourceFreshnessTest): def test_intentional_failure_no_previous_state(self, project): self.run_dbt_with_vars(project, ["run"]) previous_state_results = self.run_dbt_with_vars( project, ["source", "freshness", "-o", "previous_state/sources.json"], expect_pass=False, ) self._assert_freshness_results("previous_state/sources.json", "error") copy_to_previous_state() assert previous_state_results[0].max_loaded_at is not None with pytest.raises(DbtInternalError) as excinfo: self.run_dbt_with_vars( project, ["run", "-s", "source_status:fresher", "--defer", "--state", "previous_state"], ) assert "No current state comparison freshness results in sources.json" in str( excinfo.value ) class TestSourceFresherBuildResultSelectors(SuccessfulSourceFreshnessTest): def test_source_fresher_build_state_modified_pass(self, project, project_root): models_path = project_root.join("models/") assert os.path.exists(models_path) with open(f"{models_path}/newly_added_error_model.sql", "w") as fp: fp.write(models_newly_added_error_model_sql) self.run_dbt_with_vars(project, ["run"], expect_pass=False) self._set_updated_at_to(project, timedelta(hours=-2)) previous_state_results = self.run_dbt_with_vars( project, ["source", "freshness", "-o", "previous_state/sources.json"] ) self._assert_freshness_results("previous_state/sources.json", "pass") copy_to_previous_state() self._set_updated_at_to(project, timedelta(hours=-1)) current_state_results = self.run_dbt_with_vars( project, ["source", "freshness", "-o", "target/sources.json"] ) self._assert_freshness_results("target/sources.json", "pass") assert previous_state_results[0].max_loaded_at < current_state_results[0].max_loaded_at state_modified_results = self.run_dbt_with_vars( project, [ "build", "--select", "source_status:fresher+", "result:error+", "--defer", "--state", "previous_state", ], expect_pass=False, ) nodes = set([elem.node.name for elem in state_modified_results]) assert nodes == { "newly_added_error_model", "source_unique_test_source_test_table_id", "unique_descendant_model_id", "not_null_descendant_model_id", "source_not_null_test_source_test_table_id", "descendant_model", "source_relationships_test_source_test_table_favorite_color__favorite_color__ref_descendant_model_", "relationships_descendant_model_favorite_color__favorite_color__source_test_source_test_table_", } ================================================ FILE: tests/functional/sources/test_source_freshness.py ================================================ import json import os from collections import defaultdict from datetime import datetime, timedelta, timezone import pytest import yaml import dbt.version from dbt import deprecations from dbt.artifacts.schemas.freshness import FreshnessResult from dbt.artifacts.schemas.results import FreshnessStatus from dbt.cli.main import dbtRunner from dbt.tests.util import AnyFloat, AnyStringWith from tests.functional.sources.common_source_setup import BaseSourcesTest from tests.functional.sources.fixtures import ( collect_freshness_macro_override_previous_return_signature, error_models_model_sql, error_models_schema_yml, filtered_models_schema_yml, freshness_via_custom_sql_config_schema_yml, freshness_via_custom_sql_schema_yml, freshness_via_custom_sql_source_config_schema_yml, freshness_via_metadata_schema_yml, freshness_with_explicit_null_in_source_schema_yml, freshness_with_explicit_null_in_table_schema_yml, override_freshness_models_schema_yml, ) class SuccessfulSourceFreshnessTest(BaseSourcesTest): @pytest.fixture(scope="class", autouse=True) def setUp(self, project): self.run_dbt_with_vars(project, ["seed"]) pytest._id = 101 pytest.freshness_start_time = datetime.now(timezone.utc).replace(tzinfo=None) # this is the db initial value pytest.last_inserted_time = "2016-09-19T14:45:51+00:00" os.environ["DBT_ENV_CUSTOM_ENV_key"] = "value" yield del os.environ["DBT_ENV_CUSTOM_ENV_key"] def _set_updated_at_to(self, project, delta): insert_time = datetime.now(timezone.utc).replace(tzinfo=None) + delta timestr = insert_time.strftime("%Y-%m-%d %H:%M:%S") # favorite_color,id,first_name,email,ip_address,updated_at insert_id = pytest._id pytest._id += 1 quoted_columns = ",".join( project.adapter.quote(c) for c in ("favorite_color", "id", "first_name", "email", "ip_address", "updated_at") ) kwargs = { "schema": project.test_schema, "time": timestr, "id": insert_id, "source": project.adapter.quote("source"), "quoted_columns": quoted_columns, } raw_code = """INSERT INTO {schema}.{source} ({quoted_columns}) VALUES ( 'blue',{id},'Jake','abc@example.com','192.168.1.1','{time}' )""".format( **kwargs ) project.run_sql(raw_code) pytest.last_inserted_time = insert_time.strftime("%Y-%m-%dT%H:%M:%S+00:00") def assertBetween(self, timestr, start, end=None): datefmt = "%Y-%m-%dT%H:%M:%S.%fZ" if end is None: end = datetime.now(timezone.utc).replace(tzinfo=None) parsed = datetime.strptime(timestr, datefmt) assert start <= parsed assert end >= parsed def _assert_freshness_results(self, path, state): assert os.path.exists(path) with open(path) as fp: data = json.load(fp) assert set(data) == {"metadata", "results", "elapsed_time"} assert "generated_at" in data["metadata"] assert isinstance(data["elapsed_time"], float) self.assertBetween(data["metadata"]["generated_at"], pytest.freshness_start_time) assert ( data["metadata"]["dbt_schema_version"] == "https://schemas.getdbt.com/dbt/sources/v3.json" ) assert data["metadata"]["dbt_version"] == dbt.version.__version__ key = "key" if os.name == "nt": key = key.upper() assert data["metadata"]["env"] == {key: "value"} last_inserted_time = pytest.last_inserted_time assert len(data["results"]) == 1 # TODO: replace below calls - could they be more sane? assert data["results"] == [ { "unique_id": "source.test.test_source.test_table", "max_loaded_at": last_inserted_time, "snapshotted_at": AnyStringWith(), "max_loaded_at_time_ago_in_s": AnyFloat(), "status": state, "criteria": { "filter": None, "warn_after": {"count": 10, "period": "hour"}, "error_after": {"count": 18, "period": "hour"}, }, "adapter_response": {"_message": "SELECT 1", "code": "SELECT", "rows_affected": 1}, "thread_id": AnyStringWith("Thread-"), "execution_time": AnyFloat(), "timing": [ { "name": "compile", "started_at": AnyStringWith(), "completed_at": AnyStringWith(), }, { "name": "execute", "started_at": AnyStringWith(), "completed_at": AnyStringWith(), }, ], } ] def _assert_project_hooks_called(self, logs: str): assert "test.on-run-start.0" in logs assert "test.on-run-start.0" in logs def _assert_project_hooks_not_called(self, logs: str): assert "test.on-run-end.0" not in logs assert "test.on-run-end.0" not in logs class TestSourceFreshness(SuccessfulSourceFreshnessTest): def test_source_freshness(self, project): # test_source.test_table should have a loaded_at field of `updated_at` # and a freshness of warn_after: 10 hours, error_after: 18 hours # by default, our data set is way out of date! results = self.run_dbt_with_vars( project, ["source", "freshness", "-o", "target/error_source.json"], expect_pass=False ) assert len(results) == 1 assert results[0].status == "error" self._assert_freshness_results("target/error_source.json", "error") self._set_updated_at_to(project, timedelta(hours=-12)) results = self.run_dbt_with_vars( project, ["source", "freshness", "-o", "target/warn_source.json"], ) assert len(results) == 1 assert results[0].status == "warn" self._assert_freshness_results("target/warn_source.json", "warn") self._set_updated_at_to(project, timedelta(hours=-2)) results = self.run_dbt_with_vars( project, ["source", "freshness", "-o", "target/pass_source.json"], ) assert len(results) == 1 assert results[0].status == "pass" self._assert_freshness_results("target/pass_source.json", "pass") class TestSourceSnapshotFreshness(SuccessfulSourceFreshnessTest): def test_source_snapshot_freshness(self, project): """Ensures that the deprecated command `source snapshot-freshness` aliases to `source freshness` command. """ results = self.run_dbt_with_vars( project, ["source", "snapshot-freshness", "-o", "target/error_source.json"], expect_pass=False, ) assert len(results) == 1 assert results[0].status == "error" self._assert_freshness_results("target/error_source.json", "error") self._set_updated_at_to(project, timedelta(hours=-12)) results = self.run_dbt_with_vars( project, ["source", "snapshot-freshness", "-o", "target/warn_source.json"], ) assert len(results) == 1 assert results[0].status == "warn" self._assert_freshness_results("target/warn_source.json", "warn") self._set_updated_at_to(project, timedelta(hours=-2)) results = self.run_dbt_with_vars( project, ["source", "snapshot-freshness", "-o", "target/pass_source.json"], ) assert len(results) == 1 assert results[0].status == "pass" self._assert_freshness_results("target/pass_source.json", "pass") class TestSourceFreshnessSelection(SuccessfulSourceFreshnessTest): @pytest.fixture(scope="class") def project_config_update(self, logs_dir): return { "target-path": logs_dir, } def test_source_freshness_selection_select(self, project, logs_dir): """Tests node selection using the --select argument.""" """Also validate that specify a target-path works as expected.""" self._set_updated_at_to(project, timedelta(hours=-2)) # select source directly results = self.run_dbt_with_vars( project, [ "source", "freshness", "--select", "source:test_source.test_table", ], ) assert len(results) == 1 assert results[0].status == "pass" self._assert_freshness_results(f"{logs_dir}/sources.json", "pass") class TestSourceFreshnessExclude(SuccessfulSourceFreshnessTest): def test_source_freshness_selection_exclude(self, project): """Tests node selection using the --select argument. It 'excludes' the only source in the project so it should return no results.""" self._set_updated_at_to(project, timedelta(hours=-2)) # exclude source directly results = self.run_dbt_with_vars( project, [ "source", "freshness", "--exclude", "source:test_source.test_table", "-o", "target/exclude_source.json", ], ) assert len(results) == 0 class TestSourceFreshnessGraph(SuccessfulSourceFreshnessTest): def test_source_freshness_selection_graph_operation(self, project): """Tests node selection using the --select argument with graph operations. `+descendant_model` == select all nodes `descendant_model` depends on. """ self._set_updated_at_to(project, timedelta(hours=-2)) # select model ancestors results = self.run_dbt_with_vars( project, [ "source", "freshness", "--select", "+descendant_model", "-o", "target/ancestor_source.json", ], ) assert len(results) == 1 assert results[0].status == "pass" self._assert_freshness_results("target/ancestor_source.json", "pass") class TestSourceFreshnessErrors(SuccessfulSourceFreshnessTest): @pytest.fixture(scope="class") def models(self): return { "schema.yml": error_models_schema_yml, "model.sql": error_models_model_sql, } def test_source_freshness_error(self, project): results = self.run_dbt_with_vars(project, ["source", "freshness"], expect_pass=False) assert len(results) == 1 assert results[0].status == "runtime error" class TestSourceFreshnessFilter(SuccessfulSourceFreshnessTest): @pytest.fixture(scope="class") def models(self): return {"schema.yml": filtered_models_schema_yml} def test_source_freshness_all_records(self, project): # all records are filtered out self.run_dbt_with_vars(project, ["source", "freshness"], expect_pass=False) # we should insert a record with _id=101 that's fresh, but will still fail # because the filter excludes it self._set_updated_at_to(project, timedelta(hours=-2)) self.run_dbt_with_vars(project, ["source", "freshness"], expect_pass=False) # we should now insert a record with _id=102 that's fresh, and the filter # includes it self._set_updated_at_to(project, timedelta(hours=-2)) self.run_dbt_with_vars(project, ["source", "freshness"], expect_pass=True) class TestOverrideSourceFreshness(SuccessfulSourceFreshnessTest): @pytest.fixture(scope="class") def models(self): return {"schema.yml": override_freshness_models_schema_yml} @pytest.fixture(scope="class") def project_config_update(self): return {"sources": {"+freshness": {"error_after": {"count": 24, "period": "hour"}}}} @staticmethod def get_result_from_unique_id(data, unique_id): try: return list(filter(lambda x: x["unique_id"] == unique_id, data["results"]))[0] except IndexError: raise f"No result for the given unique_id. unique_id={unique_id}" def test_override_source_freshness(self, project): self._set_updated_at_to(project, timedelta(hours=-30)) path = "target/pass_source.json" results, log_output = self.run_dbt_and_capture_with_vars( project, ["source", "freshness", "-o", path], expect_pass=False ) assert len(results) == 4 # freshness disabled for source_e assert "Found `freshness` as a top-level property of `test_source` in file" assert os.path.exists(path) with open(path) as fp: data = json.load(fp) result_source_a = self.get_result_from_unique_id(data, "source.test.test_source.source_a") assert result_source_a["status"] == "error" expected = { "warn_after": {"count": 6, "period": "hour"}, "error_after": {"count": 24, "period": "hour"}, "filter": None, } assert result_source_a["criteria"] == expected result_source_b = self.get_result_from_unique_id(data, "source.test.test_source.source_b") assert result_source_b["status"] == "error" expected = { "warn_after": {"count": 6, "period": "hour"}, "error_after": {"count": 24, "period": "hour"}, "filter": None, } assert result_source_b["criteria"] == expected result_source_c = self.get_result_from_unique_id(data, "source.test.test_source.source_c") assert result_source_c["status"] == "warn" expected = { "warn_after": {"count": 6, "period": "hour"}, "error_after": None, "filter": None, } assert result_source_c["criteria"] == expected result_source_d = self.get_result_from_unique_id(data, "source.test.test_source.source_d") assert result_source_d["status"] == "warn" expected = { "warn_after": {"count": 6, "period": "hour"}, "error_after": {"count": 72, "period": "hour"}, "filter": None, } assert result_source_d["criteria"] == expected class TestSourceFreshnessMacroOverride(SuccessfulSourceFreshnessTest): @pytest.fixture(scope="class") def macros(self): return { "collect_freshness.sql": collect_freshness_macro_override_previous_return_signature } def test_source_freshness(self, project): # ensure that the deprecation warning is raised vars_dict = { "test_run_schema": project.test_schema, "test_loaded_at": project.adapter.quote("updated_at"), } events = [] dbtRunner(callbacks=[events.append]).invoke( ["source", "freshness", "--vars", yaml.safe_dump(vars_dict)] ) matches = list([e for e in events if e.info.name == "CollectFreshnessReturnSignature"]) assert matches class TestMetadataFreshnessFails: @pytest.fixture(scope="class") def models(self): return {"schema.yml": freshness_via_metadata_schema_yml} def test_metadata_freshness_unsupported_parse_warning(self, project): """Since the default test adapter (postgres) does not support metadata based source freshness checks, trying to use that mechanism should result in a parse-time warning.""" got_warning = False def warning_probe(e): nonlocal got_warning if e.info.name == "FreshnessConfigProblem" and e.info.level == "warn": got_warning = True runner = dbtRunner(callbacks=[warning_probe]) runner.invoke(["parse"]) assert got_warning def test_metadata_freshness_unsupported_error_when_run(self, project): runner = dbtRunner() result = runner.invoke(["source", "freshness"]) assert isinstance(result.result, FreshnessResult) assert len(result.result.results) == 1 freshness_result = result.result.results[0] assert freshness_result.status == FreshnessStatus.RuntimeErr assert "Could not compute freshness for source test_table" in freshness_result.message class TestSourceFreshnessProjectHooksNotRun(SuccessfulSourceFreshnessTest): @pytest.fixture(scope="class") def project_config_update(self): return { "config-version": 2, "on-run-start": ["{{ log('on-run-start hooks called') }}"], "on-run-end": ["{{ log('on-run-end hooks called') }}"], "flags": { "source_freshness_run_project_hooks": False, }, } @pytest.fixture(scope="class") def global_deprecations(self): deprecations.reset_deprecations() yield deprecations.reset_deprecations() def test_hooks_do_run_for_source_freshness( self, project, global_deprecations, ): assert deprecations.active_deprecations == defaultdict(int) _, log_output = self.run_dbt_and_capture_with_vars( project, [ "source", "freshness", ], expect_pass=False, ) assert "on-run-start hooks called" not in log_output assert "on-run-end hooks called" not in log_output assert "source-freshness-project-hooks" in deprecations.active_deprecations class TestHooksInSourceFreshness(SuccessfulSourceFreshnessTest): @pytest.fixture(scope="class") def project_config_update(self): return { "config-version": 2, "on-run-start": ["{{ log('on-run-start hooks called') }}"], "on-run-end": ["{{ log('on-run-end hooks called') }}"], "flags": { "source_freshness_run_project_hooks": True, }, } def test_hooks_do_run_for_source_freshness( self, project, ): _, log_output = self.run_dbt_and_capture_with_vars( project, [ "source", "freshness", ], expect_pass=False, ) self._assert_project_hooks_called(log_output) class TestHooksInSourceFreshnessError: @pytest.fixture(scope="class") def models(self): return { "schema.yml": error_models_schema_yml, "model.sql": error_models_model_sql, } @pytest.fixture(scope="class") def project_config_update(self): return { "config-version": 2, "on-run-start": ["select fake_column from table_does_not_exist"], "flags": { "source_freshness_run_project_hooks": True, }, } def test_hooks_do_not_run_for_source_freshness( self, project, ): run_result_error = None def run_result_error_probe(e): nonlocal run_result_error if ( e.info.name == "RunResultError" and e.info.level == "error" and "on-run-start" in e.info.msg ): run_result_error = e.info.msg runner = dbtRunner(callbacks=[run_result_error_probe]) runner.invoke(["source", "freshness"]) assert 'relation "table_does_not_exist" does not exist' in run_result_error class TestHooksInSourceFreshnessDisabled(SuccessfulSourceFreshnessTest): @pytest.fixture(scope="class") def project_config_update(self): return { "config-version": 2, "on-run-start": ["{{ log('on-run-start hooks called') }}"], "on-run-end": ["{{ log('on-run-end hooks called') }}"], "flags": { "source_freshness_run_project_hooks": False, }, } def test_hooks_do_not_run_for_source_freshness( self, project, ): _, log_output = self.run_dbt_and_capture_with_vars( project, [ "source", "freshness", ], expect_pass=False, ) self._assert_project_hooks_not_called(log_output) class TestHooksInSourceFreshnessDefault(SuccessfulSourceFreshnessTest): @pytest.fixture(scope="class") def project_config_update(self): return { "config-version": 2, "on-run-start": ["{{ log('on-run-start hooks called') }}"], "on-run-end": ["{{ log('on-run-end hooks called') }}"], } def test_hooks_do_not_run_for_source_freshness( self, project, ): _, log_output = self.run_dbt_and_capture_with_vars( project, [ "source", "freshness", ], expect_pass=False, ) # default behaviour - hooks are run in source freshness self._assert_project_hooks_called(log_output) class TestSourceFreshnessCustomSQL(SuccessfulSourceFreshnessTest): @pytest.fixture(scope="class") def models(self): return {"schema.yml": freshness_via_custom_sql_schema_yml} def test_source_freshness_custom_sql(self, project): result = self.run_dbt_with_vars(project, ["source", "freshness"], expect_pass=True) # They are the same source but different queries were executed for each assert {r.node.name: r.status for r in result} == { "source_a": "warn", "source_b": "warn", "source_c": "pass", } class TestSourceFreshnessCustomSQLConfig(TestSourceFreshnessCustomSQL): @pytest.fixture(scope="class") def models(self): return {"schema.yml": freshness_via_custom_sql_config_schema_yml} class TestSourceFreshnessCustomSQLSourceConfig(SuccessfulSourceFreshnessTest): @pytest.fixture(scope="class") def models(self): return {"schema.yml": freshness_via_custom_sql_source_config_schema_yml} def test_source_freshness_custom_sql(self, project): result = self.run_dbt_with_vars(project, ["source", "freshness"], expect_pass=True) assert {r.node.name: r.status for r in result} == { "source_c": "pass", } class TestSourceFreshnessExplicitNullInTable(SuccessfulSourceFreshnessTest): @pytest.fixture(scope="class") def models(self): return {"schema.yml": freshness_with_explicit_null_in_table_schema_yml} def test_source_freshness_explicit_null_in_table(self, project): result = self.run_dbt_with_vars(project, ["source", "freshness"], expect_pass=True) assert {r.node.name: r.status for r in result} == {} class TestSourceFreshnessExplicitNullInSource(SuccessfulSourceFreshnessTest): @pytest.fixture(scope="class") def models(self): return {"schema.yml": freshness_with_explicit_null_in_source_schema_yml} def test_source_freshness_explicit_null_in_source(self, project): result = self.run_dbt_with_vars(project, ["source", "freshness"], expect_pass=True) assert {r.node.name: r.status for r in result} == {} ================================================ FILE: tests/functional/sources/test_source_loaded_at_field.py ================================================ import pytest from dbt.exceptions import YamlParseDictError from dbt.tests.util import get_manifest, run_dbt, write_file loaded_at_field_null_schema_yml = """ sources: - name: test_source freshness: warn_after: count: 1 period: day error_after: count: 4 period: day loaded_at_field: updated_at tables: - name: table1 loaded_at_field: null """ loaded_at_field_blank_schema_yml = """ sources: - name: test_source freshness: warn_after: count: 1 period: day error_after: count: 4 period: day loaded_at_field: updated_at tables: - name: table1 loaded_at_field: null """ loaded_at_field_missing_schema_yml = """ sources: - name: test_source freshness: warn_after: count: 1 period: day error_after: count: 4 period: day loaded_at_field: updated_at tables: - name: table1 """ loaded_at_field_defined_schema_yml = """ sources: - name: test_source freshness: warn_after: count: 1 period: day error_after: count: 4 period: day loaded_at_field: updated_at tables: - name: table1 loaded_at_field: updated_at_another_place """ loaded_at_field_empty_string_schema_yml = """ sources: - name: test_source freshness: warn_after: count: 1 period: day error_after: count: 4 period: day loaded_at_field: updated_at tables: - name: table1 loaded_at_field: "" """ class TestParsingLoadedAtField: @pytest.fixture(scope="class") def models(self): return {"schema.yml": loaded_at_field_null_schema_yml} def test_loaded_at_field(self, project): # test setting loaded_at_field to null explicitly at table level run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "source.test.test_source.table1" in manifest.sources assert manifest.sources.get("source.test.test_source.table1").loaded_at_field is None # test setting loaded_at_field at source level, do not set at table level # end up with source level loaded_at_field write_file( loaded_at_field_missing_schema_yml, project.project_root, "models", "schema.yml" ) run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "source.test.test_source.table1" in manifest.sources assert ( manifest.sources.get("source.test.test_source.table1").loaded_at_field == "updated_at" ) # test setting loaded_at_field to nothing, should override Source value for None write_file(loaded_at_field_blank_schema_yml, project.project_root, "models", "schema.yml") run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "source.test.test_source.table1" in manifest.sources assert manifest.sources.get("source.test.test_source.table1").loaded_at_field is None # test setting loaded_at_field at table level to a value - it should override source level write_file( loaded_at_field_defined_schema_yml, project.project_root, "models", "schema.yml" ) run_dbt(["parse"]) manifest = get_manifest(project.project_root) assert "source.test.test_source.table1" in manifest.sources assert ( manifest.sources.get("source.test.test_source.table1").loaded_at_field == "updated_at_another_place" ) # test setting loaded_at_field at table level to an empty string - should error write_file( loaded_at_field_empty_string_schema_yml, project.project_root, "models", "schema.yml" ) with pytest.raises(YamlParseDictError): run_dbt(["parse"]) ================================================ FILE: tests/functional/statements/fixtures.py ================================================ # # Seeds # seeds__statement_expected = """source,value matrix,100 table,100 """ seeds__statement_actual = """id,first_name,last_name,email,gender,ip_address 1,Jack,Hunter,jhunter0@pbs.org,Male,59.80.20.168 2,Kathryn,Walker,kwalker1@ezinearticles.com,Female,194.121.179.35 3,Gerald,Ryan,gryan2@com.com,Male,11.3.212.243 4,Bonnie,Spencer,bspencer3@ameblo.jp,Female,216.32.196.175 5,Harold,Taylor,htaylor4@people.com.cn,Male,253.10.246.136 6,Jacqueline,Griffin,jgriffin5@t.co,Female,16.13.192.220 7,Wanda,Arnold,warnold6@google.nl,Female,232.116.150.64 8,Craig,Ortiz,cortiz7@sciencedaily.com,Male,199.126.106.13 9,Gary,Day,gday8@nih.gov,Male,35.81.68.186 10,Rose,Wright,rwright9@yahoo.co.jp,Female,236.82.178.100 11,Raymond,Kelley,rkelleya@fc2.com,Male,213.65.166.67 12,Gerald,Robinson,grobinsonb@disqus.com,Male,72.232.194.193 13,Mildred,Martinez,mmartinezc@samsung.com,Female,198.29.112.5 14,Dennis,Arnold,darnoldd@google.com,Male,86.96.3.250 15,Judy,Gray,jgraye@opensource.org,Female,79.218.162.245 16,Theresa,Garza,tgarzaf@epa.gov,Female,21.59.100.54 17,Gerald,Robertson,grobertsong@csmonitor.com,Male,131.134.82.96 18,Philip,Hernandez,phernandezh@adobe.com,Male,254.196.137.72 19,Julia,Gonzalez,jgonzalezi@cam.ac.uk,Female,84.240.227.174 20,Andrew,Davis,adavisj@patch.com,Male,9.255.67.25 21,Kimberly,Harper,kharperk@foxnews.com,Female,198.208.120.253 22,Mark,Martin,mmartinl@marketwatch.com,Male,233.138.182.153 23,Cynthia,Ruiz,cruizm@google.fr,Female,18.178.187.201 24,Samuel,Carroll,scarrolln@youtu.be,Male,128.113.96.122 25,Jennifer,Larson,jlarsono@vinaora.com,Female,98.234.85.95 26,Ashley,Perry,aperryp@rakuten.co.jp,Female,247.173.114.52 27,Howard,Rodriguez,hrodriguezq@shutterfly.com,Male,231.188.95.26 28,Amy,Brooks,abrooksr@theatlantic.com,Female,141.199.174.118 29,Louise,Warren,lwarrens@adobe.com,Female,96.105.158.28 30,Tina,Watson,twatsont@myspace.com,Female,251.142.118.177 31,Janice,Kelley,jkelleyu@creativecommons.org,Female,239.167.34.233 32,Terry,Mccoy,tmccoyv@bravesites.com,Male,117.201.183.203 33,Jeffrey,Morgan,jmorganw@surveymonkey.com,Male,78.101.78.149 34,Louis,Harvey,lharveyx@sina.com.cn,Male,51.50.0.167 35,Philip,Miller,pmillery@samsung.com,Male,103.255.222.110 36,Willie,Marshall,wmarshallz@ow.ly,Male,149.219.91.68 37,Patrick,Lopez,plopez10@redcross.org,Male,250.136.229.89 38,Adam,Jenkins,ajenkins11@harvard.edu,Male,7.36.112.81 39,Benjamin,Cruz,bcruz12@linkedin.com,Male,32.38.98.15 40,Ruby,Hawkins,rhawkins13@gmpg.org,Female,135.171.129.255 41,Carlos,Barnes,cbarnes14@a8.net,Male,240.197.85.140 42,Ruby,Griffin,rgriffin15@bravesites.com,Female,19.29.135.24 43,Sean,Mason,smason16@icq.com,Male,159.219.155.249 44,Anthony,Payne,apayne17@utexas.edu,Male,235.168.199.218 45,Steve,Cruz,scruz18@pcworld.com,Male,238.201.81.198 46,Anthony,Garcia,agarcia19@flavors.me,Male,25.85.10.18 47,Doris,Lopez,dlopez1a@sphinn.com,Female,245.218.51.238 48,Susan,Nichols,snichols1b@freewebs.com,Female,199.99.9.61 49,Wanda,Ferguson,wferguson1c@yahoo.co.jp,Female,236.241.135.21 50,Andrea,Pierce,apierce1d@google.co.uk,Female,132.40.10.209 51,Lawrence,Phillips,lphillips1e@jugem.jp,Male,72.226.82.87 52,Judy,Gilbert,jgilbert1f@multiply.com,Female,196.250.15.142 53,Eric,Williams,ewilliams1g@joomla.org,Male,222.202.73.126 54,Ralph,Romero,rromero1h@sogou.com,Male,123.184.125.212 55,Jean,Wilson,jwilson1i@ocn.ne.jp,Female,176.106.32.194 56,Lori,Reynolds,lreynolds1j@illinois.edu,Female,114.181.203.22 57,Donald,Moreno,dmoreno1k@bbc.co.uk,Male,233.249.97.60 58,Steven,Berry,sberry1l@eepurl.com,Male,186.193.50.50 59,Theresa,Shaw,tshaw1m@people.com.cn,Female,120.37.71.222 60,John,Stephens,jstephens1n@nationalgeographic.com,Male,191.87.127.115 61,Richard,Jacobs,rjacobs1o@state.tx.us,Male,66.210.83.155 62,Andrew,Lawson,alawson1p@over-blog.com,Male,54.98.36.94 63,Peter,Morgan,pmorgan1q@rambler.ru,Male,14.77.29.106 64,Nicole,Garrett,ngarrett1r@zimbio.com,Female,21.127.74.68 65,Joshua,Kim,jkim1s@edublogs.org,Male,57.255.207.41 66,Ralph,Roberts,rroberts1t@people.com.cn,Male,222.143.131.109 67,George,Montgomery,gmontgomery1u@smugmug.com,Male,76.75.111.77 68,Gerald,Alvarez,galvarez1v@flavors.me,Male,58.157.186.194 69,Donald,Olson,dolson1w@whitehouse.gov,Male,69.65.74.135 70,Carlos,Morgan,cmorgan1x@pbs.org,Male,96.20.140.87 71,Aaron,Stanley,astanley1y@webnode.com,Male,163.119.217.44 72,Virginia,Long,vlong1z@spiegel.de,Female,204.150.194.182 73,Robert,Berry,rberry20@tripadvisor.com,Male,104.19.48.241 74,Antonio,Brooks,abrooks21@unesco.org,Male,210.31.7.24 75,Ruby,Garcia,rgarcia22@ovh.net,Female,233.218.162.214 76,Jack,Hanson,jhanson23@blogtalkradio.com,Male,31.55.46.199 77,Kathryn,Nelson,knelson24@walmart.com,Female,14.189.146.41 78,Jason,Reed,jreed25@printfriendly.com,Male,141.189.89.255 79,George,Coleman,gcoleman26@people.com.cn,Male,81.189.221.144 80,Rose,King,rking27@ucoz.com,Female,212.123.168.231 81,Johnny,Holmes,jholmes28@boston.com,Male,177.3.93.188 82,Katherine,Gilbert,kgilbert29@altervista.org,Female,199.215.169.61 83,Joshua,Thomas,jthomas2a@ustream.tv,Male,0.8.205.30 84,Julie,Perry,jperry2b@opensource.org,Female,60.116.114.192 85,Richard,Perry,rperry2c@oracle.com,Male,181.125.70.232 86,Kenneth,Ruiz,kruiz2d@wikimedia.org,Male,189.105.137.109 87,Jose,Morgan,jmorgan2e@webnode.com,Male,101.134.215.156 88,Donald,Campbell,dcampbell2f@goo.ne.jp,Male,102.120.215.84 89,Debra,Collins,dcollins2g@uol.com.br,Female,90.13.153.235 90,Jesse,Johnson,jjohnson2h@stumbleupon.com,Male,225.178.125.53 91,Elizabeth,Stone,estone2i@histats.com,Female,123.184.126.221 92,Angela,Rogers,arogers2j@goodreads.com,Female,98.104.132.187 93,Emily,Dixon,edixon2k@mlb.com,Female,39.190.75.57 94,Albert,Scott,ascott2l@tinypic.com,Male,40.209.13.189 95,Barbara,Peterson,bpeterson2m@ow.ly,Female,75.249.136.180 96,Adam,Greene,agreene2n@fastcompany.com,Male,184.173.109.144 97,Earl,Sanders,esanders2o@hc360.com,Male,247.34.90.117 98,Angela,Brooks,abrooks2p@mtv.com,Female,10.63.249.126 99,Harold,Foster,hfoster2q@privacy.gov.au,Male,139.214.40.244 100,Carl,Meyer,cmeyer2r@disqus.com,Male,204.117.7.88 """ # # Models # models__statement_actual = """ -- {{ ref('seed') }} {%- call statement('test_statement', fetch_result=True) -%} select count(*) as "num_records" from {{ ref('seed') }} {%- endcall -%} {% set result = load_result('test_statement') %} {% set res_table = result['table'] %} {% set res_matrix = result['data'] %} {% set matrix_value = res_matrix[0][0] %} {% set table_value = res_table[0]['num_records'] %} select 'matrix' as source, {{ matrix_value }} as value union all select 'table' as source, {{ table_value }} as value """ models__statement_duplicated_load = """ -- {{ ref('seed') }} {%- call statement('test_statement', fetch_result=True) -%} select count(*) as "num_records" from {{ ref('seed') }} {%- endcall -%} {% set result = load_result('test_statement') %} {% set result = load_result('test_statement') %} select 1 """ models__statement_load_main_twice = """ -- {{ ref('seed') }} {%- call statement('main', fetch_result=True) -%} select count(*) as "num_records" from {{ ref('seed') }} {%- endcall -%} {% set result = load_result('main') %} {% set result = load_result('main') %} {% set res_table = result['table'] %} {% set res_matrix = result['data'] %} {% set matrix_value = res_matrix[0][0] %} {% set table_value = res_table[0]['num_records'] %} select 'matrix' as source, {{ matrix_value }} as value union all select 'table' as source, {{ table_value }} as value """ ================================================ FILE: tests/functional/statements/test_statements.py ================================================ import pathlib import pytest from dbt.tests.util import check_relations_equal, run_dbt, write_file from tests.functional.statements.fixtures import ( models__statement_actual, models__statement_duplicated_load, models__statement_load_main_twice, seeds__statement_actual, seeds__statement_expected, ) class TestStatements: @pytest.fixture(scope="class", autouse=True) def setUp(self, project): # put seeds in 'seed' not 'seeds' directory (pathlib.Path(project.project_root) / "seed").mkdir(parents=True, exist_ok=True) write_file(seeds__statement_actual, project.project_root, "seed", "seed.csv") write_file( seeds__statement_expected, project.project_root, "seed", "statement_expected.csv" ) @pytest.fixture(scope="class") def models(self): return { "statement_actual.sql": models__statement_actual, "statement_duplicated_load.sql": models__statement_duplicated_load, "statement_load_main_twice.sql": models__statement_load_main_twice, } @pytest.fixture(scope="class") def project_config_update(self): return { "seeds": { "quote_columns": False, }, "seed-paths": ["seed"], } def test_postgres_statements(self, project): results = run_dbt(["seed"]) assert len(results) == 2 results = run_dbt(["run", "-m", "statement_actual"]) assert len(results) == 1 check_relations_equal(project.adapter, ["statement_actual", "statement_expected"]) def test_duplicated_load_statements(self, project): run_dbt(["seed"]) results = run_dbt(["run", "-m", "statement_duplicated_load"], False) assert len(results) == 1 assert results.results[0].status == "error" assert ( "The 'statement' result named 'test_statement' has already been loaded into a variable" in results.results[0].message ) def test_load_statement_on_main_twice(self, project): run_dbt(["seed"]) results = run_dbt(["run", "-m", "statement_load_main_twice"]) assert len(results) == 1 check_relations_equal(project.adapter, ["statement_load_main_twice", "statement_expected"]) ================================================ FILE: tests/functional/test_empty.py ================================================ import pytest from dbt.tests.util import relation_from_name, run_dbt model_input_sql = """ select 1 as id """ ephemeral_model_input_sql = """ {{ config(materialized='ephemeral') }} select 2 as id """ raw_source_csv = """id 3 """ model_sql = """ select * from {{ ref('model_input') }} union all select * from {{ ref('ephemeral_model_input') }} union all select * from {{ source('seed_sources', 'raw_source') }} """ model_no_ephemeral_ref_sql = """ select * from {{ ref('model_input') }} union all select * from {{ source('seed_sources', 'raw_source') }} """ schema_sources_yml = """ sources: - name: seed_sources schema: "{{ target.schema }}" tables: - name: raw_source """ unit_tests_yml = """ unit_tests: - name: test_my_model model: model_no_ephemeral_ref given: - input: ref('model_input') format: csv rows: | id 1 - input: source('seed_sources', 'raw_source') format: csv rows: | id 2 expect: format: csv rows: | id 1 2 """ class TestEmptyFlag: @pytest.fixture(scope="class") def seeds(self): return { "raw_source.csv": raw_source_csv, } @pytest.fixture(scope="class") def models(self): return { "model_input.sql": model_input_sql, "ephemeral_model_input.sql": ephemeral_model_input_sql, "model.sql": model_sql, "model_no_ephemeral_ref.sql": model_no_ephemeral_ref_sql, "sources.yml": schema_sources_yml, "unit_tests.yml": unit_tests_yml, } def assert_row_count(self, project, relation_name: str, expected_row_count: int): relation = relation_from_name(project.adapter, relation_name) result = project.run_sql(f"select count(*) as num_rows from {relation}", fetch="one") assert result[0] == expected_row_count def test_run_with_empty(self, project): # create source from seed run_dbt(["seed"]) # run without empty - 3 expected rows in output - 1 from each input run_dbt(["run"]) self.assert_row_count(project, "model", 3) # run with empty - 0 expected rows in output run_dbt(["run", "--empty"]) self.assert_row_count(project, "model", 0) # build without empty - 3 expected rows in output - 1 from each input run_dbt(["build"]) self.assert_row_count(project, "model", 3) # build with empty - 0 expected rows in output run_dbt(["build", "--empty"]) self.assert_row_count(project, "model", 0) # ensure dbt compile supports --empty flag run_dbt(["compile", "--empty"]) ================================================ FILE: tests/functional/test_project.py ================================================ from unittest import mock import yaml from pytest_mock import MockerFixture from dbt.deprecations import ( GenericJSONSchemaValidationDeprecation as GenericJSONSchemaValidationDeprecationCore, ) from dbt.events.types import GenericJSONSchemaValidationDeprecation from dbt.tests.util import run_dbt, write_file from dbt_common.events.event_catcher import EventCatcher from dbt_common.events.types import Note class TestProjectJsonschemaValidatedOnlyOnce: """Ensure that the dbt_project.yml file is validated only once, even if it is 'loaded' multiple times""" def test_project(self, project, mocker: MockerFixture) -> None: mocked_jsonschema_validate = mocker.patch( "dbt.jsonschemas.jsonschemas.jsonschema_validate" ) run_dbt(["parse"]) assert mocked_jsonschema_validate.call_count == 1 @mock.patch("dbt.jsonschemas.jsonschemas._JSONSCHEMA_SUPPORTED_ADAPTERS", {"postgres"}) class TestGenericJsonSchemaValidationDeprecation: """Ensure that the generic jsonschema validation deprecation can be fired""" def test_project(self, project, project_root: str) -> None: # `name` was already required prior to this deprecation, so this deprecation doesn't # really add anything. However, this test shows that jsonschema validation issues raise # deprecation warnings via the catchall `GenericJSONSchemaValidationDeprecation` project_missing_name = { "profile": "test", "flags": {"send_anonymous_usage_stats": False}, } write_file(yaml.safe_dump(project_missing_name), project_root, "dbt_project.yml") event_catcher = EventCatcher(GenericJSONSchemaValidationDeprecation) note_catcher = EventCatcher(Note) try: run_dbt( ["parse"], callbacks=[event_catcher.catch, note_catcher.catch], expect_pass=False ) except: # noqa: E722 pass if GenericJSONSchemaValidationDeprecationCore()._is_preview: assert len(note_catcher.caught_events) == 1 assert len(event_catcher.caught_events) == 0 event = note_catcher.caught_events[0] else: assert len(event_catcher.caught_events) == 1 assert len(note_catcher.caught_events) == 0 event = event_catcher.caught_events[0] assert "'name' is a required property at top level" in event.info.msg ================================================ FILE: tests/functional/test_selection/fixtures.py ================================================ import pytest tests__cf_a_b_sql = """ select * from {{ ref('model_a') }} cross join {{ ref('model_b') }} where false """ tests__cf_a_src_sql = """ select * from {{ ref('model_a') }} cross join {{ source('my_src', 'my_tbl') }} where false """ tests__just_a_sql = """ {{ config(tags = ['data_test_tag']) }} select * from {{ ref('model_a') }} where false """ models__schema_yml = """ version: 2 sources: - name: my_src schema: "{{ target.schema }}" tables: - name: my_tbl identifier: model_b columns: - name: fun config: tags: [source_column_config_tag] data_tests: - unique - name: fun_2 config: tags: [source_column_config_tag] data_tests: - unique models: - name: model_a columns: - name: fun tags: [column_level_tag] config: tags: [column_config_level_tag] data_tests: - unique - relationships: to: ref('model_b') field: fun tags: [test_level_tag] - relationships: to: source('my_src', 'my_tbl') field: fun - name: fun_2 tags: [column_level_tag] config: tags: column_config_level_tag data_tests: - unique """ models__model_b_sql = """ {{ config( tags = ['a_or_b'] ) }} select 1 as fun, 2 as fun_2 """ models__model_a_sql = """ {{ config( tags = ['a_or_b'] ) }} select * FROM {{ref('model_b')}} """ @pytest.fixture(scope="class") def tests(): return { "cf_a_b.sql": tests__cf_a_b_sql, "cf_a_src.sql": tests__cf_a_src_sql, "just_a.sql": tests__just_a_sql, } @pytest.fixture(scope="class") def models(): return { "schema.yml": models__schema_yml, "model_b.sql": models__model_b_sql, "model_a.sql": models__model_a_sql, } ================================================ FILE: tests/functional/test_selection/test_selection_expansion.py ================================================ import pytest from dbt.tests.util import run_dbt from tests.functional.test_selection.fixtures import models, tests # noqa: F401 class TestSelectionExpansion: @pytest.fixture(scope="class") def project_config_update(self): return {"config-version": 2, "test-paths": ["tests"]} def list_tests_and_assert( self, include, exclude, expected_tests, indirect_selection="eager", selector_name=None, ): list_args = ["ls", "--resource-type", "test"] if include: list_args.extend(("--select", include)) if exclude: list_args.extend(("--exclude", exclude)) if indirect_selection: list_args.extend(("--indirect-selection", indirect_selection)) if selector_name: list_args.extend(("--selector", selector_name)) listed = run_dbt(list_args) assert len(listed) == len(expected_tests) test_names = [name.split(".")[-1] for name in listed] assert sorted(test_names) == sorted(expected_tests) def run_tests_and_assert( self, include, exclude, expected_tests, indirect_selection="eager", selector_name=None, ): results = run_dbt(["run"]) assert len(results) == 2 test_args = ["test"] if include: test_args.extend(("--models", include)) if exclude: test_args.extend(("--exclude", exclude)) if indirect_selection: test_args.extend(("--indirect-selection", indirect_selection)) if selector_name: test_args.extend(("--selector", selector_name)) results = run_dbt(test_args) tests_run = [r.node.name for r in results] assert len(tests_run) == len(expected_tests) assert sorted(tests_run) == sorted(expected_tests) def test_all_tests_no_specifiers( self, project, ): select = None exclude = None expected = [ "cf_a_b", "cf_a_src", "just_a", "relationships_model_a_fun__fun__ref_model_b_", "relationships_model_a_fun__fun__source_my_src_my_tbl_", "source_unique_my_src_my_tbl_fun", "source_unique_my_src_my_tbl_fun_2", "unique_model_a_fun", "unique_model_a_fun_2", ] self.list_tests_and_assert(select, exclude, expected) self.run_tests_and_assert(select, exclude, expected) def test_model_a_alone( self, project, ): select = "model_a" exclude = None expected = [ "cf_a_b", "cf_a_src", "just_a", "relationships_model_a_fun__fun__ref_model_b_", "relationships_model_a_fun__fun__source_my_src_my_tbl_", "unique_model_a_fun", "unique_model_a_fun_2", ] self.list_tests_and_assert(select, exclude, expected) self.run_tests_and_assert(select, exclude, expected) def test_model_a_model_b( self, project, ): select = "model_a model_b" exclude = None expected = [ "cf_a_b", "cf_a_src", "just_a", "unique_model_a_fun", "unique_model_a_fun_2", "relationships_model_a_fun__fun__ref_model_b_", "relationships_model_a_fun__fun__source_my_src_my_tbl_", ] self.list_tests_and_assert(select, exclude, expected) self.run_tests_and_assert(select, exclude, expected) def test_model_a_sources( self, project, ): select = "model_a source:*" exclude = None expected = [ "cf_a_b", "cf_a_src", "just_a", "unique_model_a_fun", "unique_model_a_fun_2", "source_unique_my_src_my_tbl_fun", "source_unique_my_src_my_tbl_fun_2", "relationships_model_a_fun__fun__ref_model_b_", "relationships_model_a_fun__fun__source_my_src_my_tbl_", ] self.list_tests_and_assert(select, exclude, expected) self.run_tests_and_assert(select, exclude, expected) def test_exclude_model_b( self, project, ): select = None exclude = "model_b" expected = [ "cf_a_src", "just_a", "relationships_model_a_fun__fun__source_my_src_my_tbl_", "source_unique_my_src_my_tbl_fun", "source_unique_my_src_my_tbl_fun_2", "unique_model_a_fun", "unique_model_a_fun_2", ] self.list_tests_and_assert(select, exclude, expected) self.run_tests_and_assert(select, exclude, expected) def test_model_a_exclude_specific_test( self, project, ): select = "model_a" exclude = "unique_model_a_fun" expected = [ "cf_a_b", "cf_a_src", "just_a", "relationships_model_a_fun__fun__ref_model_b_", "relationships_model_a_fun__fun__source_my_src_my_tbl_", "unique_model_a_fun_2", ] self.list_tests_and_assert(select, exclude, expected) self.run_tests_and_assert(select, exclude, expected) def test_model_a_exclude_specific_test_cautious( self, project, ): select = "model_a" exclude = "unique_model_a_fun" expected = ["just_a", "unique_model_a_fun_2"] indirect_selection = "cautious" self.list_tests_and_assert(select, exclude, expected, indirect_selection) self.run_tests_and_assert(select, exclude, expected, indirect_selection) def test_model_a_exclude_specific_test_buildable( self, project, ): select = "model_a" exclude = "unique_model_a_fun" expected = [ "just_a", "cf_a_b", "cf_a_src", "relationships_model_a_fun__fun__ref_model_b_", "relationships_model_a_fun__fun__source_my_src_my_tbl_", "unique_model_a_fun_2", ] indirect_selection = "buildable" self.list_tests_and_assert(select, exclude, expected, indirect_selection) self.run_tests_and_assert(select, exclude, expected, indirect_selection) def test_only_generic( self, project, ): select = "test_type:generic" exclude = None expected = [ "relationships_model_a_fun__fun__ref_model_b_", "relationships_model_a_fun__fun__source_my_src_my_tbl_", "source_unique_my_src_my_tbl_fun", "source_unique_my_src_my_tbl_fun_2", "unique_model_a_fun", "unique_model_a_fun_2", ] self.list_tests_and_assert(select, exclude, expected) self.run_tests_and_assert(select, exclude, expected) def test_model_a_only_singular_unset( self, project, ): select = "model_a,test_type:singular" exclude = None expected = ["cf_a_b", "cf_a_src", "just_a"] self.list_tests_and_assert(select, exclude, expected) self.run_tests_and_assert(select, exclude, expected) def test_model_a_only_singular_eager( self, project, ): select = "model_a,test_type:singular" exclude = None expected = ["cf_a_b", "cf_a_src", "just_a"] self.list_tests_and_assert(select, exclude, expected) self.run_tests_and_assert(select, exclude, expected) def test_model_a_only_singular_cautious( self, project, ): select = "model_a,test_type:singular" exclude = None expected = ["just_a"] indirect_selection = "cautious" self.list_tests_and_assert( select, exclude, expected, indirect_selection=indirect_selection ) self.run_tests_and_assert(select, exclude, expected, indirect_selection=indirect_selection) def test_only_singular( self, project, ): select = "test_type:singular" exclude = None expected = ["cf_a_b", "cf_a_src", "just_a"] self.list_tests_and_assert(select, exclude, expected) self.run_tests_and_assert(select, exclude, expected) def test_model_a_only_singular( self, project, ): select = "model_a,test_type:singular" exclude = None expected = ["cf_a_b", "cf_a_src", "just_a"] self.list_tests_and_assert(select, exclude, expected) self.run_tests_and_assert(select, exclude, expected) def test_test_name_intersection( self, project, ): select = "model_a,test_name:unique" exclude = None expected = ["unique_model_a_fun", "unique_model_a_fun_2"] self.list_tests_and_assert(select, exclude, expected) self.run_tests_and_assert(select, exclude, expected) def test_model_tag_test_name_intersection( self, project, ): select = "tag:a_or_b,test_name:relationships" exclude = None expected = [ "relationships_model_a_fun__fun__ref_model_b_", "relationships_model_a_fun__fun__source_my_src_my_tbl_", ] self.list_tests_and_assert(select, exclude, expected) self.run_tests_and_assert(select, exclude, expected) def test_select_column_level_tag( self, project, ): select = "tag:column_level_tag" exclude = None expected = [ "relationships_model_a_fun__fun__ref_model_b_", "relationships_model_a_fun__fun__source_my_src_my_tbl_", "unique_model_a_fun", "unique_model_a_fun_2", ] self.list_tests_and_assert(select, exclude, expected) self.run_tests_and_assert(select, exclude, expected) def test_exclude_column_level_tag( self, project, ): select = None exclude = "tag:column_level_tag" expected = [ "cf_a_b", "cf_a_src", "just_a", "source_unique_my_src_my_tbl_fun", "source_unique_my_src_my_tbl_fun_2", ] self.list_tests_and_assert(select, exclude, expected) self.run_tests_and_assert(select, exclude, expected) def test_select_column_config_level_tag( self, project, ): select = "tag:column_config_level_tag" exclude = None expected = [ "relationships_model_a_fun__fun__ref_model_b_", "relationships_model_a_fun__fun__source_my_src_my_tbl_", "unique_model_a_fun", "unique_model_a_fun_2", ] self.list_tests_and_assert(select, exclude, expected) self.run_tests_and_assert(select, exclude, expected) def test_select_source_column_config_level_tag( self, project, ): select = "tag:source_column_config_tag" exclude = None expected = [ "source_unique_my_src_my_tbl_fun", "source_unique_my_src_my_tbl_fun_2", ] self.list_tests_and_assert(select, exclude, expected) self.run_tests_and_assert(select, exclude, expected) def test_exclude_column_config_level_tag( self, project, ): select = None exclude = "tag:column_config_level_tag" expected = [ "cf_a_b", "cf_a_src", "just_a", "source_unique_my_src_my_tbl_fun", "source_unique_my_src_my_tbl_fun_2", ] self.list_tests_and_assert(select, exclude, expected) self.run_tests_and_assert(select, exclude, expected) def test_test_level_tag( self, project, ): select = "tag:test_level_tag" exclude = None expected = ["relationships_model_a_fun__fun__ref_model_b_"] self.list_tests_and_assert(select, exclude, expected) self.run_tests_and_assert(select, exclude, expected) def test_exclude_data_test_tag( self, project, ): select = "model_a" exclude = "tag:data_test_tag" expected = [ "cf_a_b", "cf_a_src", "relationships_model_a_fun__fun__ref_model_b_", "relationships_model_a_fun__fun__source_my_src_my_tbl_", "unique_model_a_fun", "unique_model_a_fun_2", ] self.list_tests_and_assert(select, exclude, expected) self.run_tests_and_assert(select, exclude, expected) def test_model_a_indirect_selection( self, project, ): select = "model_a" exclude = None expected = [ "cf_a_b", "cf_a_src", "just_a", "relationships_model_a_fun__fun__ref_model_b_", "relationships_model_a_fun__fun__source_my_src_my_tbl_", "unique_model_a_fun", "unique_model_a_fun_2", ] self.list_tests_and_assert(select, exclude, expected) self.run_tests_and_assert(select, exclude, expected) def test_model_a_indirect_selection_eager( self, project, ): select = "model_a" exclude = None expected = [ "cf_a_b", "cf_a_src", "just_a", "relationships_model_a_fun__fun__ref_model_b_", "relationships_model_a_fun__fun__source_my_src_my_tbl_", "unique_model_a_fun", "unique_model_a_fun_2", ] indirect_selection = "eager" self.list_tests_and_assert(select, exclude, expected, indirect_selection) self.run_tests_and_assert(select, exclude, expected, indirect_selection) def test_model_a_indirect_selection_cautious( self, project, ): select = "model_a" exclude = None expected = [ "just_a", "unique_model_a_fun", "unique_model_a_fun_2", ] indirect_selection = "cautious" self.list_tests_and_assert(select, exclude, expected, indirect_selection) self.run_tests_and_assert(select, exclude, expected, indirect_selection) def test_model_a_indirect_selection_buildable( self, project, ): select = "model_a" exclude = None expected = [ "cf_a_b", "cf_a_src", "just_a", "relationships_model_a_fun__fun__ref_model_b_", "relationships_model_a_fun__fun__source_my_src_my_tbl_", "unique_model_a_fun", "unique_model_a_fun_2", ] indirect_selection = "buildable" self.list_tests_and_assert(select, exclude, expected, indirect_selection) self.run_tests_and_assert(select, exclude, expected, indirect_selection) def test_model_a_indirect_selection_exclude_unique_tests( self, project, ): select = "model_a" exclude = "test_name:unique" indirect_selection = "eager" expected = [ "cf_a_b", "cf_a_src", "just_a", "relationships_model_a_fun__fun__ref_model_b_", "relationships_model_a_fun__fun__source_my_src_my_tbl_", ] self.list_tests_and_assert(select, exclude, expected, indirect_selection) self.run_tests_and_assert(select, exclude, expected, indirect_selection=indirect_selection) def test_model_a_indirect_selection_empty(self, project): results = run_dbt(["ls", "--indirect-selection", "empty", "--select", "model_a"]) assert len(results) == 1 class TestExpansionWithSelectors(TestSelectionExpansion): @pytest.fixture(scope="class") def selectors(self): return """ selectors: - name: model_a_unset_indirect_selection definition: method: fqn value: model_a - name: model_a_cautious_indirect_selection definition: method: fqn value: model_a indirect_selection: "cautious" - name: model_a_eager_indirect_selection definition: method: fqn value: model_a indirect_selection: "eager" - name: model_a_buildable_indirect_selection definition: method: fqn value: model_a indirect_selection: "buildable" """ def test_selector_model_a_unset_indirect_selection( self, project, ): expected = [ "cf_a_b", "cf_a_src", "just_a", "relationships_model_a_fun__fun__ref_model_b_", "relationships_model_a_fun__fun__source_my_src_my_tbl_", "unique_model_a_fun", "unique_model_a_fun_2", ] self.list_tests_and_assert( include=None, exclude=None, expected_tests=expected, selector_name="model_a_unset_indirect_selection", ) self.run_tests_and_assert( include=None, exclude=None, expected_tests=expected, selector_name="model_a_unset_indirect_selection", ) def test_selector_model_a_cautious_indirect_selection( self, project, ): expected = ["just_a", "unique_model_a_fun", "unique_model_a_fun_2"] self.list_tests_and_assert( include=None, exclude=None, expected_tests=expected, selector_name="model_a_cautious_indirect_selection", ) self.run_tests_and_assert( include=None, exclude=None, expected_tests=expected, selector_name="model_a_cautious_indirect_selection", ) def test_selector_model_a_eager_indirect_selection( self, project, ): expected = [ "cf_a_b", "cf_a_src", "just_a", "relationships_model_a_fun__fun__ref_model_b_", "relationships_model_a_fun__fun__source_my_src_my_tbl_", "unique_model_a_fun", "unique_model_a_fun_2", ] self.list_tests_and_assert( include=None, exclude=None, expected_tests=expected, selector_name="model_a_eager_indirect_selection", ) self.run_tests_and_assert( include=None, exclude=None, expected_tests=expected, selector_name="model_a_eager_indirect_selection", ) def test_selector_model_a_buildable_indirect_selection( self, project, ): expected = [ "cf_a_b", "cf_a_src", "just_a", "relationships_model_a_fun__fun__ref_model_b_", "relationships_model_a_fun__fun__source_my_src_my_tbl_", "unique_model_a_fun", "unique_model_a_fun_2", ] self.list_tests_and_assert( include=None, exclude=None, expected_tests=expected, selector_name="model_a_buildable_indirect_selection", ) self.run_tests_and_assert( include=None, exclude=None, expected_tests=expected, selector_name="model_a_buildable_indirect_selection", ) ================================================ FILE: tests/functional/test_singular_tests.py ================================================ import pytest from dbt.tests.util import run_dbt single_test_sql = """ {{ config(warn_if = '>0', error_if ="> 10") }} select 1 as issue """ class TestSingularTestWarnError: @pytest.fixture(scope="class") def tests(self): return {"single_test.sql": single_test_sql} def test_singular_test_warn_error(self, project): results = run_dbt(["--warn-error", "test"], expect_pass=False) assert results.results[0].status == "fail" def test_singular_test_warn_error_options(self, project): results = run_dbt(["--warn-error-options", "{'error': 'all'}", "test"], expect_pass=False) assert results.results[0].status == "fail" def test_singular_test_equals_warn_error(self, project): results = run_dbt(["--warn-error", "test"], expect_pass=False) warn_error_result = results.results[0].status results = run_dbt(["--warn-error-options", "{'error': 'all'}", "test"], expect_pass=False) assert warn_error_result == results.results[0].status ================================================ FILE: tests/functional/threading/test_thread_count.py ================================================ import pytest from dbt.tests.util import run_dbt models__do_nothing__sql = """ with x as (select pg_sleep(1)) select 1 """ class TestThreadCount: @pytest.fixture(scope="class") def models(self): return { "do_nothing_1.sql": models__do_nothing__sql, "do_nothing_2.sql": models__do_nothing__sql, "do_nothing_3.sql": models__do_nothing__sql, "do_nothing_4.sql": models__do_nothing__sql, "do_nothing_5.sql": models__do_nothing__sql, "do_nothing_6.sql": models__do_nothing__sql, "do_nothing_7.sql": models__do_nothing__sql, "do_nothing_8.sql": models__do_nothing__sql, "do_nothing_9.sql": models__do_nothing__sql, "do_nothing_10.sql": models__do_nothing__sql, "do_nothing_11.sql": models__do_nothing__sql, "do_nothing_12.sql": models__do_nothing__sql, "do_nothing_13.sql": models__do_nothing__sql, "do_nothing_14.sql": models__do_nothing__sql, "do_nothing_15.sql": models__do_nothing__sql, "do_nothing_16.sql": models__do_nothing__sql, "do_nothing_17.sql": models__do_nothing__sql, "do_nothing_18.sql": models__do_nothing__sql, "do_nothing_19.sql": models__do_nothing__sql, "do_nothing_20.sql": models__do_nothing__sql, } @pytest.fixture(scope="class") def project_config_update(self): return {"config-version": 2} @pytest.fixture(scope="class") def profiles_config_update(self): return {"threads": 2} def test_threading_8x(self, project): results = run_dbt(args=["run", "--threads", "16"]) assert len(results), 20 ================================================ FILE: tests/functional/time_spines/fixtures.py ================================================ models_people_sql = """ select 1 as id, 'Drew' as first_name, 'Banin' as last_name, 'yellow' as favorite_color, true as loves_dbt, 5 as tenure, current_timestamp as created_at union all select 2 as id, 'Jeremy' as first_name, 'Cohen' as last_name, 'indigo' as favorite_color, true as loves_dbt, 4 as tenure, current_timestamp as created_at union all select 3 as id, 'Callum' as first_name, 'McCann' as last_name, 'emerald' as favorite_color, true as loves_dbt, 0 as tenure, current_timestamp as created_at """ semantic_model_people_yml = """ version: 2 semantic_models: - name: semantic_people model: ref('people') dimensions: - name: favorite_color type: categorical - name: created_at type: TIME type_params: time_granularity: day measures: - name: years_tenure agg: SUM expr: tenure - name: people agg: count expr: id entities: - name: id type: primary defaults: agg_time_dimension: created_at """ metricflow_time_spine_sql = """ SELECT to_date('02/20/2023, 'mm/dd/yyyy') as date_day """ metricflow_time_spine_second_sql = """ SELECT to_datetime('02/20/2023, 'mm/dd/yyyy hh:mm:ss') as ts_second """ valid_time_spines_yml = """ version: 2 models: - name: metricflow_time_spine_second time_spine: standard_granularity_column: ts_second columns: - name: ts_second granularity: second - name: metricflow_time_spine time_spine: standard_granularity_column: date_day custom_granularities: - name: retail_month - name: martian_year column_name: martian__year_xyz columns: - name: date_day granularity: day - name: retail_month - name: martian__year_xyz """ missing_time_spine_yml = """ models: - name: metricflow_time_spine columns: - name: ts_second granularity: second """ time_spine_missing_granularity_yml = """ models: - name: metricflow_time_spine_second time_spine: standard_granularity_column: ts_second columns: - name: ts_second """ time_spine_missing_standard_column_yml = """ models: - name: metricflow_time_spine_second time_spine: standard_granularity_column: ts_second columns: - name: date_day """ time_spine_missing_custom_column_yml = """ models: - name: metricflow_time_spine_second time_spine: standard_granularity_column: date_day custom_granularities: - name: retail_month columns: - name: date_day granularity: day """ time_spine_yml = """ version: 2 models: - name: metricflow_time_spine time_spine: standard_granularity_column: date_day custom_granularities: - name: retail_month - name: martian_day column_name: martian_day columns: - name: date_day granularity: day - name: retail_month - name: martian_day """ ================================================ FILE: tests/functional/time_spines/test_time_spines.py ================================================ from typing import Set import pytest from dbt.cli.main import dbtRunner from dbt.contracts.graph.manifest import Manifest from dbt.contracts.graph.semantic_manifest import SemanticManifest from dbt.exceptions import ParsingError from dbt.tests.util import get_manifest from dbt_semantic_interfaces.type_enums import TimeGranularity from tests.functional.time_spines.fixtures import ( metricflow_time_spine_second_sql, metricflow_time_spine_sql, models_people_sql, semantic_model_people_yml, time_spine_missing_custom_column_yml, time_spine_missing_granularity_yml, time_spine_missing_standard_column_yml, valid_time_spines_yml, ) class TestValidTimeSpines: """Tests that YAML using current time spine configs parses as expected.""" @pytest.fixture(scope="class") def models(self): return { "metricflow_time_spine.sql": metricflow_time_spine_sql, "metricflow_time_spine_second.sql": metricflow_time_spine_second_sql, "time_spines.yml": valid_time_spines_yml, "semantic_model_people.yml": semantic_model_people_yml, "people.sql": models_people_sql, } def test_time_spines(self, project): runner = dbtRunner() result = runner.invoke(["parse"]) assert result.success assert isinstance(result.result, Manifest) manifest = get_manifest(project.project_root) assert manifest # Test that models and columns are set as expected time_spine_models = { id.split(".")[-1]: node for id, node in manifest.nodes.items() if node.time_spine } day_model_name = "metricflow_time_spine" second_model_name = "metricflow_time_spine_second" day_column_name = "date_day" second_column_name = "ts_second" model_names_to_col_names = { day_model_name: day_column_name, second_model_name: second_column_name, } model_names_to_granularities = { day_model_name: TimeGranularity.DAY, second_model_name: TimeGranularity.SECOND, } assert len(time_spine_models) == 2 expected_time_spine_aliases = {second_model_name, day_model_name} assert set(time_spine_models.keys()) == expected_time_spine_aliases for model in time_spine_models.values(): assert ( model.time_spine.standard_granularity_column == model_names_to_col_names[model.name] ) if model.name == day_model_name: assert len(model.time_spine.custom_granularities) == 2 assert { custom_granularity.name for custom_granularity in model.time_spine.custom_granularities } == {"retail_month", "martian_year"} for custom_granularity in model.time_spine.custom_granularities: if custom_granularity.name == "martian_year": assert custom_granularity.column_name == "martian__year_xyz" else: assert len(model.time_spine.custom_granularities) == 0 assert len(model.columns) > 0 assert ( list(model.columns.values())[0].granularity == model_names_to_granularities[model.name] ) # Test that project configs are set as expected in semantic manifest semantic_manifest = SemanticManifest(manifest) assert semantic_manifest.validate() project_config = semantic_manifest._get_pydantic_semantic_manifest().project_configuration # Current configs assert len(project_config.time_spines) == 2 sl_time_spine_aliases: Set[str] = set() for sl_time_spine in project_config.time_spines: alias = sl_time_spine.node_relation.alias sl_time_spine_aliases.add(alias) assert sl_time_spine.primary_column.name == model_names_to_col_names[alias] assert ( sl_time_spine.primary_column.time_granularity == model_names_to_granularities[alias] ) assert sl_time_spine_aliases == expected_time_spine_aliases class TestValidLegacyTimeSpine: """Tests that YAML using only legacy time spine config parses as expected.""" @pytest.fixture(scope="class") def models(self): return { "metricflow_time_spine.sql": metricflow_time_spine_sql, "semantic_model_people.yml": semantic_model_people_yml, "people.sql": models_people_sql, } def test_time_spines(self, project): runner = dbtRunner() result = runner.invoke(["parse"]) assert result.success assert isinstance(result.result, Manifest) manifest = get_manifest(project.project_root) assert manifest # Test that project configs are set as expected in semantic manifest semantic_manifest = SemanticManifest(manifest) assert semantic_manifest.validate() project_config = semantic_manifest._get_pydantic_semantic_manifest().project_configuration # Legacy config assert len(project_config.time_spine_table_configurations) == 1 legacy_time_spine_config = project_config.time_spine_table_configurations[0] assert legacy_time_spine_config.column_name == "date_day" assert ( legacy_time_spine_config.location.replace('"', "").split(".")[-1] == "metricflow_time_spine" ) assert legacy_time_spine_config.grain == TimeGranularity.DAY # Current configs assert len(project_config.time_spines) == 0 class TestMissingTimeSpine: """Tests that YAML with semantic models but no time spines errors.""" @pytest.fixture(scope="class") def models(self): return { "semantic_model_people.yml": semantic_model_people_yml, "people.sql": models_people_sql, } def test_time_spines(self, project): runner = dbtRunner() result = runner.invoke(["parse"]) assert isinstance(result.exception, ParsingError) assert ( "The semantic layer requires a time spine model with granularity DAY or smaller" in result.exception.msg ) class TestTimeSpineStandardColumnMissing: """Tests that YAML with time spine standard granularity column not in model errors.""" @pytest.fixture(scope="class") def models(self): return { "semantic_model_people.yml": semantic_model_people_yml, "people.sql": models_people_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "metricflow_time_spine_second.sql": metricflow_time_spine_second_sql, "time_spines.yml": time_spine_missing_standard_column_yml, } def test_time_spines(self, project): runner = dbtRunner() result = runner.invoke(["parse"]) assert isinstance(result.exception, ParsingError) assert ( "Time spine standard granularity column must be defined on the model." in result.exception.msg ) class TestTimeSpineCustomColumnMissing: """Tests that YAML with time spine custom granularity column not in model errors.""" @pytest.fixture(scope="class") def models(self): return { "semantic_model_people.yml": semantic_model_people_yml, "people.sql": models_people_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "metricflow_time_spine_second.sql": metricflow_time_spine_second_sql, "time_spines.yml": time_spine_missing_custom_column_yml, } def test_time_spines(self, project): runner = dbtRunner() result = runner.invoke(["parse"]) assert isinstance(result.exception, ParsingError) assert ( "Time spine custom granularity columns do not exist in the model." in result.exception.msg ) class TestTimeSpineGranularityMissing: """Tests that YAML with time spine column without granularity errors.""" @pytest.fixture(scope="class") def models(self): return { "semantic_model_people.yml": semantic_model_people_yml, "people.sql": models_people_sql, "metricflow_time_spine.sql": metricflow_time_spine_sql, "metricflow_time_spine_second.sql": metricflow_time_spine_second_sql, "time_spines.yml": time_spine_missing_granularity_yml, } def test_time_spines(self, project): runner = dbtRunner() result = runner.invoke(["parse"]) assert isinstance(result.exception, ParsingError) assert ( "Time spine standard granularity column must have a granularity defined." in result.exception.msg ) ================================================ FILE: tests/functional/timezones/test_timezones.py ================================================ import os import pytest from dbt.tests.util import run_dbt # Canada/Saskatchewan does not observe DST so the time diff won't change depending on when it is in the year model_sql = """ {{ config( materialized='table' ) }} select '{{ run_started_at.astimezone(modules.pytz.timezone("Canada/Saskatchewan")) }}' as run_started_at_saskatchewan, '{{ run_started_at }}' as run_started_at_utc """ class TestTimezones: @pytest.fixture(scope="class") def models(self): return {"timezones.sql": model_sql} @pytest.fixture(scope="class") def dbt_profile_data(self, unique_schema): return { "test": { "outputs": { "dev": { "type": "postgres", "threads": 1, "host": "localhost", "port": int(os.getenv("POSTGRES_TEST_PORT", 5432)), "user": os.getenv("POSTGRES_TEST_USER", "root"), "pass": os.getenv("POSTGRES_TEST_PASS", "password"), "dbname": os.getenv("POSTGRES_TEST_DATABASE", "dbt"), "schema": unique_schema, }, }, "target": "dev", } } @pytest.fixture(scope="class") def query(self, project): return """ select run_started_at_saskatchewan, run_started_at_utc from {schema}.timezones """.format( schema=project.test_schema ) # This test used to use freeze_time, but that doesn't work # with our timestamp fields in proto messages. def test_run_started_at(self, project, query): results = run_dbt(["run"]) assert len(results) == 1 result = project.run_sql(query, fetch="all")[0] saskatchewan, utc = result assert "+00:00" in utc assert "-06:00" in saskatchewan ================================================ FILE: tests/functional/unit_testing/fixtures.py ================================================ import pytest my_model_vars_sql = """ SELECT a+b as c, concat(string_a, string_b) as string_c, not_testing, date_a, {{ dbt.string_literal(type_numeric()) }} as macro_call, {{ dbt.string_literal(var('my_test')) }} as var_call, {{ dbt.string_literal(env_var('MY_TEST', 'default')) }} as env_var_call, {{ dbt.string_literal(invocation_id) }} as invocation_id FROM {{ ref('my_model_a')}} my_model_a JOIN {{ ref('my_model_b' )}} my_model_b ON my_model_a.id = my_model_b.id """ my_model_sql = """ SELECT a+b as c, concat(string_a, string_b) as string_c, not_testing, date_a FROM {{ ref('my_model_a')}} my_model_a JOIN {{ ref('my_model_b' )}} my_model_b ON my_model_a.id = my_model_b.id """ my_model_a_sql = """ SELECT 1 as a, 1 as id, 2 as not_testing, 'a' as string_a, DATE '2020-01-02' as date_a """ my_model_b_sql = """ SELECT 2 as b, 1 as id, 2 as c, 'b' as string_b """ my_model_check_null_sql = """ SELECT CASE WHEN a IS null THEN True ELSE False END a_is_null FROM {{ ref('my_model_a') }} """ test_my_model_a_yml = """ models: - name: my_model_a columns: - name: a tests: - not_null - name: id tests: - not_null """ test_my_model_yml = """ unit_tests: - name: test_my_model model: my_model given: - input: ref('my_model_a') rows: - {id: 1, a: 1} - input: ref('my_model_b') rows: - {id: 1, b: 2} - {id: 2, b: 2} expect: rows: - {c: 2} - name: test_my_model_empty model: my_model given: - input: ref('my_model_a') rows: [] - input: ref('my_model_b') rows: - {id: 1, b: 2} - {id: 2, b: 2} expect: rows: [] - name: test_my_model_overrides model: my_model given: - input: ref('my_model_a') rows: - {id: 1, a: 1} - input: ref('my_model_b') rows: - {id: 1, b: 2} - {id: 2, b: 2} overrides: macros: type_numeric: override invocation_id: 123 vars: my_test: var_override env_vars: MY_TEST: env_var_override expect: rows: - {macro_call: override, var_call: var_override, env_var_call: env_var_override, invocation_id: 123} - name: test_my_model_string_concat model: my_model given: - input: ref('my_model_a') rows: - {id: 1, string_a: a} - input: ref('my_model_b') rows: - {id: 1, string_b: b} expect: rows: - {string_c: ab} config: tags: test_this """ test_my_model_pass_yml = """ unit_tests: - name: test_my_model model: my_model given: - input: ref('my_model_a') rows: - {id: 1, a: 1} - input: ref('my_model_b') rows: - {id: 1, b: 2} - {id: 2, b: 2} expect: rows: - {c: 3} """ test_disabled_unit_test_my_model_yml = """ unit_tests: - name: test_disabled_my_model model: my_model config: enabled: false given: - input: ref('my_model_a') rows: - {id: 1, a: 1} - input: ref('my_model_b') rows: - {id: 1, b: 2} - {id: 2, b: 2} expect: rows: - {c: 3} """ my_model_disabled_sql = """ {{ config(enabled=false) }} select 1 as id """ my_model_disabled_yml = """ models: - name: my_model_disabled_yml config: enabled: false """ disabled_my_model_tests_yml = """ unit_tests: - name: test_disabled_my_model_sql model: my_model_disabled_sql given: [] expect: rows: [{"id": 1}] - name: test_disabled_my_model_yml model: my_model_disabled_yml given: [] expect: rows: [{"id": 1}] """ test_my_model_simple_fixture_yml = """ unit_tests: - name: test_my_model model: my_model given: - input: ref('my_model_a') rows: - {id: 1, a: 1} - input: ref('my_model_b') rows: - {id: 1, b: 2} - {id: 2, b: 2} expect: rows: - {c: 2} - name: test_depends_on_fixture model: my_model given: - input: ref('my_model_a') rows: [] - input: ref('my_model_b') format: csv fixture: test_my_model_fixture expect: rows: [] - name: test_my_model_overrides model: my_model given: - input: ref('my_model_a') rows: - {id: 1, a: 1} - input: ref('my_model_b') rows: - {id: 1, b: 2} - {id: 2, b: 2} overrides: macros: type_numeric: override invocation_id: 123 vars: my_test: var_override env_vars: MY_TEST: env_var_override expect: rows: - {macro_call: override, var_call: var_override, env_var_call: env_var_override, invocation_id: 123} - name: test_has_string_c_ab model: my_model given: - input: ref('my_model_a') rows: - {id: 1, string_a: a} - input: ref('my_model_b') rows: - {id: 1, string_b: b} expect: rows: - {string_c: ab} config: tags: test_this """ datetime_test = """ - name: test_my_model_datetime model: my_model given: - input: ref('my_model_a') rows: - {id: 1, date_a: "2020-01-01"} - input: ref('my_model_b') rows: - {id: 1} expect: rows: - {date_a: "2020-01-01"} """ event_sql = """ select DATE '2020-01-01' as event_time, 1 as event union all select DATE '2020-01-02' as event_time, 2 as event union all select DATE '2020-01-03' as event_time, 3 as event """ datetime_test_invalid_format_key = """ - name: test_my_model_datetime model: my_model given: - input: ref('my_model_a') format: xxxx rows: - {id: 1, date_a: "2020-01-01"} - input: ref('my_model_b') rows: - {id: 1} expect: rows: - {date_a: "2020-01-01"} """ datetime_test_invalid_csv_values = """ - name: test_my_model_datetime model: my_model given: - input: ref('my_model_a') format: csv rows: - {id: 1, date_a: "2020-01-01"} - input: ref('my_model_b') rows: - {id: 1} expect: rows: - {date_a: "2020-01-01"} """ datetime_test_invalid_csv_file_values = """ - name: test_my_model_datetime model: my_model given: - input: ref('my_model_a') format: csv rows: - {id: 1, date_a: "2020-01-01"} - input: ref('my_model_b') rows: - {id: 1} expect: rows: - {date_a: "2020-01-01"} """ event_sql = """ select DATE '2020-01-01' as event_time, 1 as event union all select DATE '2020-01-02' as event_time, 2 as event union all select DATE '2020-01-03' as event_time, 3 as event """ my_incremental_model_sql = """ {{ config( materialized='incremental' ) }} select * from {{ ref('events') }} {% if is_incremental() %} where event_time > (select max(event_time) from {{ this }}) {% endif %} """ my_incremental_model_with_alias_sql = """ {{ config( materialized='incremental', alias='alias_name' ) }} select * from {{ ref('events') }} {% if is_incremental() %} where event_time > (select max(event_time) from {{ this }}) {% endif %} """ my_incremental_model_versioned_yml = """ models: - name: my_incremental_model latest_version: 1 versions: - v: 1 """ test_my_model_incremental_yml_basic = """ unit_tests: - name: incremental_false model: my_incremental_model overrides: macros: is_incremental: false given: - input: ref('events') rows: - {event_time: "2020-01-01", event: 1} expect: rows: - {event_time: "2020-01-01", event: 1} - name: incremental_true model: my_incremental_model overrides: macros: is_incremental: true given: - input: ref('events') rows: - {event_time: "2020-01-01", event: 1} - {event_time: "2020-01-02", event: 2} - {event_time: "2020-01-03", event: 3} - input: this rows: - {event_time: "2020-01-01", event: 1} expect: rows: - {event_time: "2020-01-02", event: 2} - {event_time: "2020-01-03", event: 3} """ test_my_model_incremental_yml_no_override = """ unit_tests: - name: incremental_false model: my_incremental_model given: - input: ref('events') rows: - {event_time: "2020-01-01", event: 1} expect: rows: - {event_time: "2020-01-01", event: 1} """ test_my_model_incremental_yml_wrong_override = """ unit_tests: - name: incremental_false model: my_incremental_model overrides: macros: is_incremental: foobar given: - input: ref('events') rows: - {event_time: "2020-01-01", event: 1} expect: rows: - {event_time: "2020-01-01", event: 1} """ test_my_model_incremental_yml_no_this_input = """ unit_tests: - name: incremental_true model: my_incremental_model overrides: macros: is_incremental: true given: - input: ref('events') rows: - {event_time: "2020-01-01", event: 1} - {event_time: "2020-01-02", event: 2} - {event_time: "2020-01-03", event: 3} expect: rows: - {event_time: "2020-01-02", event: 2} - {event_time: "2020-01-03", event: 3} """ # -- inline csv tests test_my_model_csv_yml = """ unit_tests: - name: test_my_model model: my_model given: - input: ref('my_model_a') format: csv rows: | id,a 1,1 - input: ref('my_model_b') format: csv rows: | id,b 1,2 2,2 expect: format: csv rows: | c 2 - name: test_my_model_empty model: my_model given: - input: ref('my_model_a') rows: [] - input: ref('my_model_b') format: csv rows: | id,b 1,2 2,2 expect: rows: [] - name: test_my_model_overrides model: my_model given: - input: ref('my_model_a') format: csv rows: | id,a 1,1 - input: ref('my_model_b') format: csv rows: | id,b 1,2 2,2 overrides: macros: type_numeric: override invocation_id: 123 vars: my_test: var_override env_vars: MY_TEST: env_var_override expect: rows: - {macro_call: override, var_call: var_override, env_var_call: env_var_override, invocation_id: 123} - name: test_my_model_string_concat model: my_model given: - input: ref('my_model_a') format: csv rows: | id,string_a 1,a - input: ref('my_model_b') format: csv rows: | id,string_b 1,b expect: format: csv rows: | string_c ab config: tags: test_this """ # -- csv file tests test_my_model_file_csv_yml = """ unit_tests: - name: test_my_model model: my_model given: - input: ref('my_model_a') format: csv fixture: test_my_model_a_numeric_fixture - input: ref('my_model_b') format: csv fixture: test_my_model_fixture expect: format: csv fixture: test_my_model_basic_fixture - name: test_my_model_empty model: my_model given: - input: ref('my_model_a') format: csv fixture: test_my_model_a_empty_fixture - input: ref('my_model_b') format: csv fixture: test_my_model_fixture expect: format: csv fixture: test_my_model_a_empty_fixture - name: test_my_model_overrides model: my_model given: - input: ref('my_model_a') format: csv fixture: test_my_model_a_numeric_fixture - input: ref('my_model_b') format: csv fixture: test_my_model_fixture overrides: macros: type_numeric: override invocation_id: 123 vars: my_test: var_override env_vars: MY_TEST: env_var_override expect: rows: - {macro_call: override, var_call: var_override, env_var_call: env_var_override, invocation_id: 123} - name: test_my_model_string_concat model: my_model given: - input: ref('my_model_a') format: csv fixture: test_my_model_a_fixture - input: ref('my_model_b') format: csv fixture: test_my_model_b_fixture expect: format: csv fixture: test_my_model_concat_fixture config: tags: test_this """ test_my_model_fixture_csv = """id,b 1,2 2,2 """ test_my_model_a_fixture_csv = """id,string_a 1,a """ test_my_model_a_with_null_fixture_csv = """id,a 1, 2,3 """ test_my_model_a_empty_fixture_csv = """ """ test_my_model_a_numeric_fixture_csv = """id,a 1,1 """ test_my_model_b_fixture_csv = """id,string_b 1,b """ test_my_model_basic_fixture_csv = """c 2 """ test_my_model_concat_fixture_csv = """string_c ab """ # -- mixed inline and file csv test_my_model_mixed_csv_yml = """ unit_tests: - name: test_my_model model: my_model given: - input: ref('my_model_a') format: csv rows: | id,a 1,1 - input: ref('my_model_b') format: csv rows: | id,b 1,2 2,2 expect: format: csv fixture: test_my_model_basic_fixture - name: test_my_model_empty model: my_model given: - input: ref('my_model_a') format: csv fixture: test_my_model_a_empty_fixture - input: ref('my_model_b') format: csv rows: | id,b 1,2 2,2 expect: format: csv fixture: test_my_model_a_empty_fixture - name: test_my_model_overrides model: my_model given: - input: ref('my_model_a') format: csv rows: | id,a 1,1 - input: ref('my_model_b') format: csv fixture: test_my_model_fixture overrides: macros: type_numeric: override invocation_id: 123 vars: my_test: var_override env_vars: MY_TEST: env_var_override expect: rows: - {macro_call: override, var_call: var_override, env_var_call: env_var_override, invocation_id: 123} - name: test_my_model_string_concat model: my_model given: - input: ref('my_model_a') format: csv fixture: test_my_model_a_fixture - input: ref('my_model_b') format: csv fixture: test_my_model_b_fixture expect: format: csv rows: | string_c ab config: tags: test_this """ # unit tests with errors # -- fixture file doesn't exist test_my_model_missing_csv_yml = """ unit_tests: - name: test_missing_csv_file model: my_model given: - input: ref('my_model_a') format: csv rows: | id,a 1,1 - input: ref('my_model_b') format: csv rows: | id,b 1,2 2,2 expect: format: csv fixture: fake_fixture """ test_my_model_duplicate_csv_yml = """ unit_tests: - name: test_missing_csv_file model: my_model given: - input: ref('my_model_a') format: csv rows: | id,a 1,1 - input: ref('my_model_b') format: csv rows: | id,b 1,2 2,2 expect: format: csv fixture: test_my_model_basic_fixture """ test_model_a_b_yml = """ unit_tests: - name: my_test_name model: my_model_a given: [] expect: rows: - {a: 1, id: 1, not_testing: 2, string_a: "a", date_a: "2020-01-02"} - name: my_test_name model: my_model_b given: [] expect: rows: - {b: 2, id: 1, c: 2, string_b: "b"} """ test_model_a_with_duplicate_test_name_yml = """ unit_tests: - name: my_test_name model: my_model_a given: [] expect: rows: - {a: 1, id: 1, not_testing: 2, string_a: "a", date_a: "2020-01-02"} - name: my_test_name model: my_model_a given: [] expect: rows: - {a: 1, id: 1, not_testing: 2, string_a: "a", date_a: "2020-01-02"} """ test_my_model_yml_invalid = """ unit_tests: - name: test_my_model model: my_model given: - input: ref('my_model_a') rows: - {id: 1, a: "a"} - input: ref('my_model_b') rows: - {id: 1, b: 2} - {id: 2, b: 2} expect: rows: - {c: 3} """ test_my_model_yml_invalid_ref = """ unit_tests: - name: test_my_model model: my_model given: - input: ref('my_model_x') rows: - {id: 1, a: 1} - input: ref('my_model_b') rows: - {id: 1, b: 2} - {id: 2, b: 2} expect: rows: - {c: 3} """ # -- unit testing versioned models my_model_v1_sql = """ SELECT a, b, a+b as c, concat(string_a, string_b) as string_c, not_testing, date_a FROM {{ ref('my_model_a')}} my_model_a JOIN {{ ref('my_model_b' )}} my_model_b ON my_model_a.id = my_model_b.id """ my_model_v2_sql = """ SELECT a, b, a+b as c, concat(string_a, string_b) as string_c, date_a FROM {{ ref('my_model_a')}} my_model_a JOIN {{ ref('my_model_b' )}} my_model_b ON my_model_a.id = my_model_b.id """ my_model_v3_sql = """ SELECT a, b, a+b as c, concat(string_a, string_b) as string_c FROM {{ ref('my_model_a')}} my_model_a JOIN {{ ref('my_model_b' )}} my_model_b ON my_model_a.id = my_model_b.id """ my_model_versioned_yml = """ models: - name: my_model latest_version: 1 access: public config: contract: enforced: true columns: - name: a data_type: integer - name: b data_type: integer - name: c data_type: integer - name: string_c data_type: string - name: not_testing data_type: integer - name: date_a data_type: date versions: - v: 1 - v: 2 columns: # This means: use the 'columns' list from above, but exclude not_testing - include: "all" exclude: - not_testing - v: 3 # now exclude another column columns: - include: all exclude: - not_testing - date_a """ my_model_versioned_no_2_yml = """ models: - name: my_model latest_version: 1 access: public config: contract: enforced: true columns: - name: a data_type: integer - name: b data_type: integer - name: c data_type: integer - name: string_c data_type: string - name: not_testing data_type: integer - name: date_a data_type: date versions: - v: 1 - v: 3 # now exclude another column columns: - include: all exclude: - not_testing - date_a """ test_my_model_all_versions_yml = """ unit_tests: - name: test_my_model model: my_model given: - input: ref('my_model_a') format: csv rows: | id,a 1,1 2,3 - input: ref('my_model_b') format: csv rows: | id,b 1,2 2,2 expect: format: csv rows: | a,b,c 1,2,3 3,2,5 """ test_my_model_exclude_versions_yml = """ unit_tests: - name: test_my_model model: my_model versions: exclude: - 2 given: - input: ref('my_model_a') format: csv rows: | id,a 1,1 2,3 - input: ref('my_model_b') format: csv rows: | id,b 1,2 2,2 expect: format: csv rows: | a,b,c 1,2,3 3,2,5 """ test_my_model_include_versions_yml = """ unit_tests: - name: test_my_model model: my_model versions: include: - 2 given: - input: ref('my_model_a') format: csv rows: | id,a 1,1 2,3 - input: ref('my_model_b') format: csv rows: | id,b 1,2 2,2 expect: format: csv rows: | a,b,c 1,2,3 3,2,5 """ test_my_model_include_exclude_versions_yml = """ unit_tests: - name: test_my_model model: my_model versions: include: - 2 exclude: - 3 given: - input: ref('my_model_a') format: csv rows: | id,a 1,1 2,3 - input: ref('my_model_b') format: csv rows: | id,b 1,2 2,2 expect: format: csv rows: | a,b,c 1,2,3 3,2,5 """ test_my_model_include_unversioned_yml = """ unit_tests: - name: test_my_model model: my_model versions: include: - 2 given: - input: ref('my_model_a') rows: - {id: 1, a: 1} - input: ref('my_model_b') rows: - {id: 1, b: 2} - {id: 2, b: 2} expect: rows: - {c: 2} """ my_model_version_ref_sql = """ select * from {{ ref('my_model', version=2) }} """ test_my_model_version_ref_yml = """ unit_tests: - name: test_my_model_version_ref model: my_model_version_ref given: - input: ref('my_model', version=2) rows: - {c: 2} expect: rows: - {c: 2} """ # -- unit testing external models top_level_domains_sql = """ SELECT 'example.com' AS tld UNION ALL SELECT 'gmail.com' AS tld """ valid_emails_sql = """ WITH accounts AS ( SELECT user_id, email, email_top_level_domain FROM {{ ref('external_package', 'external_model')}} ), top_level_domains AS ( SELECT tld FROM {{ ref('top_level_domains')}} ), joined AS ( SELECT accounts.user_id as user_id, top_level_domains.tld as tld FROM accounts LEFT OUTER JOIN top_level_domains ON accounts.email_top_level_domain = top_level_domains.tld ) SELECT joined.user_id as user_id, CASE WHEN joined.tld IS NULL THEN FALSE ELSE TRUE END AS is_valid_email_address from joined """ external_package__accounts_seed_csv = """user_id,email,email_top_level_domain 1,"example@example.com","example.com" """ external_package__external_model_sql = """ SELECT user_id, email, email_top_level_domain FROM {{ ref('accounts_seed') }} """ external_package_project_yml = """ name: external_package version: '1.0' config-version: 2 model-paths: ["models"] # paths to models analysis-paths: ["analyses"] # path with analysis files which are compiled, but not run target-path: "target" # path for compiled code clean-targets: ["target"] # directories removed by the clean task test-paths: ["tests"] # where to store test results seed-paths: ["seeds"] # load CSVs from this directory with `dbt seed` macro-paths: ["macros"] # where to find macros profile: user models: external_package: """ @pytest.fixture(scope="class") def external_package(): return { "dbt_project.yml": external_package_project_yml, "seeds": {"accounts_seed.csv": external_package__accounts_seed_csv}, "models": { "external_model.sql": external_package__external_model_sql, }, } model_select_1_sql = """ select 1 as id """ model_select_2_sql = """ select 2 as id """ test_expect_2_yml = """ unit_tests: - name: test_my_model model: my_model given: [] expect: rows: - {id: 2} """ test_my_model_csv_null_yml = """ unit_tests: - name: test_my_model_check_null model: my_model_check_null given: - input: ref('my_model_a') format: csv rows: | id,a 1, 2,3 expect: format: csv rows: | a_is_null True False """ test_my_model_file_csv_null_yml = """ unit_tests: - name: test_my_model_check_null model: my_model_check_null given: - input: ref('my_model_a') format: csv fixture: test_my_model_a_with_null_fixture expect: format: csv rows: | a_is_null True False """ ================================================ FILE: tests/functional/unit_testing/test_csv_fixtures.py ================================================ import pytest from fixtures import ( datetime_test, datetime_test_invalid_csv_values, datetime_test_invalid_format_key, my_model_a_sql, my_model_b_sql, my_model_check_null_sql, my_model_sql, test_my_model_a_empty_fixture_csv, test_my_model_a_fixture_csv, test_my_model_a_numeric_fixture_csv, test_my_model_a_with_null_fixture_csv, test_my_model_b_fixture_csv, test_my_model_basic_fixture_csv, test_my_model_concat_fixture_csv, test_my_model_csv_null_yml, test_my_model_csv_yml, test_my_model_duplicate_csv_yml, test_my_model_file_csv_null_yml, test_my_model_file_csv_yml, test_my_model_fixture_csv, test_my_model_missing_csv_yml, test_my_model_mixed_csv_yml, ) from dbt.exceptions import DuplicateResourceNameError, ParsingError, YamlParseDictError from dbt.tests.util import rm_file, run_dbt, write_file class TestUnitTestsWithInlineCSV: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_model_sql, "my_model_a.sql": my_model_a_sql, "my_model_b.sql": my_model_b_sql, "test_my_model.yml": test_my_model_csv_yml + datetime_test, } def test_unit_test(self, project): results = run_dbt(["run"]) assert len(results) == 3 # Select by model name results = run_dbt(["test", "--select", "my_model"], expect_pass=False) assert len(results) == 5 # Check error with invalid format key write_file( test_my_model_csv_yml + datetime_test_invalid_format_key, project.project_root, "models", "test_my_model.yml", ) with pytest.raises(YamlParseDictError): results = run_dbt(["test", "--select", "my_model"], expect_pass=False) # Check error with csv format defined but dict on rows write_file( test_my_model_csv_yml + datetime_test_invalid_csv_values, project.project_root, "models", "test_my_model.yml", ) with pytest.raises(ParsingError): results = run_dbt(["test", "--select", "my_model"], expect_pass=False) class TestUnitTestsWithFileCSV: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_model_sql, "my_model_a.sql": my_model_a_sql, "my_model_b.sql": my_model_b_sql, "test_my_model.yml": test_my_model_file_csv_yml + datetime_test, } @pytest.fixture(scope="class") def tests(self): return { "fixtures": { "test_my_model_fixture.csv": test_my_model_fixture_csv, "test_my_model_a_fixture.csv": test_my_model_a_fixture_csv, "test_my_model_b_fixture.csv": test_my_model_b_fixture_csv, "test_my_model_basic_fixture.csv": test_my_model_basic_fixture_csv, "test_my_model_a_numeric_fixture.csv": test_my_model_a_numeric_fixture_csv, "test_my_model_a_empty_fixture.csv": test_my_model_a_empty_fixture_csv, "test_my_model_concat_fixture.csv": test_my_model_concat_fixture_csv, } } def test_unit_test(self, project): results = run_dbt(["run"]) assert len(results) == 3 manifest = run_dbt(["parse"]) # Note: this manifest is deserialized from msgpack fixture = manifest.fixtures["fixture.test.test_my_model_a_fixture"] fixture_source_file = manifest.files[fixture.file_id] assert fixture_source_file.fixture == "fixture.test.test_my_model_a_fixture" assert fixture_source_file.unit_tests == [ "unit_test.test.my_model.test_my_model_string_concat" ] # Select by model name results = run_dbt(["test", "--select", "my_model"], expect_pass=False) assert len(results) == 5 # Check partial parsing remove fixture file rm_file(project.project_root, "tests", "fixtures", "test_my_model_a_fixture.csv") with pytest.raises( ParsingError, match="File not found for fixture 'test_my_model_a_fixture' in unit tests", ): run_dbt(["test", "--select", "my_model"], expect_pass=False) # put back file and check that it works write_file( test_my_model_a_fixture_csv, project.project_root, "tests", "fixtures", "test_my_model_a_fixture.csv", ) results = run_dbt(["test", "--select", "my_model"], expect_pass=False) assert len(results) == 5 # Now update file write_file( test_my_model_a_fixture_csv + "2,2", project.project_root, "tests", "fixtures", "test_my_model_a_fixture.csv", ) manifest = run_dbt(["parse"]) fixture = manifest.fixtures["fixture.test.test_my_model_a_fixture"] fixture_source_file = manifest.files[fixture.file_id] assert "2,2" in fixture_source_file.contents assert fixture.rows == [{"id": "1", "string_a": "a"}, {"id": "2", "string_a": "2"}] # Check error with invalid format key write_file( test_my_model_file_csv_yml + datetime_test_invalid_format_key, project.project_root, "models", "test_my_model.yml", ) with pytest.raises(YamlParseDictError): results = run_dbt(["test", "--select", "my_model"], expect_pass=False) # Check error with csv format defined but dict on rows write_file( test_my_model_file_csv_yml + datetime_test_invalid_csv_values, project.project_root, "models", "test_my_model.yml", ) with pytest.raises(ParsingError): results = run_dbt(["test", "--select", "my_model"], expect_pass=False) class TestUnitTestsWithMixedCSV: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_model_sql, "my_model_a.sql": my_model_a_sql, "my_model_b.sql": my_model_b_sql, "test_my_model.yml": test_my_model_mixed_csv_yml + datetime_test, } @pytest.fixture(scope="class") def tests(self): return { "fixtures": { "test_my_model_fixture.csv": test_my_model_fixture_csv, "test_my_model_a_fixture.csv": test_my_model_a_fixture_csv, "test_my_model_b_fixture.csv": test_my_model_b_fixture_csv, "test_my_model_basic_fixture.csv": test_my_model_basic_fixture_csv, "test_my_model_a_numeric_fixture.csv": test_my_model_a_numeric_fixture_csv, "test_my_model_a_empty_fixture.csv": test_my_model_a_empty_fixture_csv, "test_my_model_concat_fixture.csv": test_my_model_concat_fixture_csv, } } def test_unit_test(self, project): results = run_dbt(["run"]) assert len(results) == 3 # Select by model name results = run_dbt(["test", "--select", "my_model"], expect_pass=False) assert len(results) == 5 # Check error with invalid format key write_file( test_my_model_mixed_csv_yml + datetime_test_invalid_format_key, project.project_root, "models", "test_my_model.yml", ) with pytest.raises(YamlParseDictError): results = run_dbt(["test", "--select", "my_model"], expect_pass=False) # Check error with csv format defined but dict on rows write_file( test_my_model_mixed_csv_yml + datetime_test_invalid_csv_values, project.project_root, "models", "test_my_model.yml", ) with pytest.raises(ParsingError): results = run_dbt(["test", "--select", "my_model"], expect_pass=False) class TestUnitTestsInlineCSVEmptyValueIsNull: @pytest.fixture(scope="class") def models(self): return { "my_model_a.sql": my_model_a_sql, "my_model_check_null.sql": my_model_check_null_sql, "test_my_model_csv_null.yml": test_my_model_csv_null_yml, } def test_unit_test(self, project): results = run_dbt(["run"]) assert len(results) == 2 # Select by model name results = run_dbt(["test", "--select", "my_model_check_null"], expect_pass=True) assert len(results) == 1 class TestUnitTestsFileCSVEmptyValueIsNull: @pytest.fixture(scope="class") def models(self): return { "my_model_a.sql": my_model_a_sql, "my_model_check_null.sql": my_model_check_null_sql, "test_my_model_file_csv_null.yml": test_my_model_file_csv_null_yml, } @pytest.fixture(scope="class") def tests(self): return { "fixtures": { "test_my_model_a_with_null_fixture.csv": test_my_model_a_with_null_fixture_csv, } } def test_unit_test(self, project): results = run_dbt(["run"]) assert len(results) == 2 # Select by model name results = run_dbt(["test", "--select", "my_model_check_null"], expect_pass=True) assert len(results) == 1 class TestUnitTestsMissingCSVFile: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_model_sql, "my_model_a.sql": my_model_a_sql, "my_model_b.sql": my_model_b_sql, "test_my_model.yml": test_my_model_missing_csv_yml, } def test_missing(self, project): with pytest.raises(ParsingError): run_dbt(["run"]) class TestUnitTestsDuplicateCSVFile: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_model_sql, "my_model_a.sql": my_model_a_sql, "my_model_b.sql": my_model_b_sql, "test_my_model.yml": test_my_model_duplicate_csv_yml, } @pytest.fixture(scope="class") def tests(self): return { "fixtures": { "one-folder": { "test_my_model_basic_fixture.csv": test_my_model_basic_fixture_csv, }, "another-folder": { "test_my_model_basic_fixture.csv": test_my_model_basic_fixture_csv, }, } } def test_duplicate(self, project): with pytest.raises(DuplicateResourceNameError): run_dbt(["run"]) ================================================ FILE: tests/functional/unit_testing/test_sql_format.py ================================================ import pytest from dbt.tests.util import run_dbt wizards_csv = """id,w_name,email,email_tld,phone,world 1,Albus Dumbledore,a.dumbledore@gmail.com,gmail.com,813-456-9087,1 2,Gandalf,gandy811@yahoo.com,yahoo.com,551-329-8367,2 3,Winifred Sanderson,winnie@hocuspocus.com,hocuspocus.com,,6 4,Marnie Piper,cromwellwitch@gmail.com,gmail.com,,5 5,Grace Goheen,grace.goheen@dbtlabs.com,dbtlabs.com,,3 6,Glinda,glinda_good@hotmail.com,hotmail.com,912-458-3289,4 """ top_level_email_domains_csv = """tld gmail.com yahoo.com hocuspocus.com dbtlabs.com hotmail.com """ worlds_csv = """id,name 1,The Wizarding World 2,Middle-earth 3,dbt Labs 4,Oz 5,Halloweentown 6,Salem """ stg_wizards_sql = """ select id as wizard_id, w_name as wizard_name, email, email_tld as email_top_level_domain, phone as phone_number, world as world_id from {{ ref('wizards') }} """ stg_worlds_sql = """ select id as world_id, name as world_name from {{ ref('worlds') }} """ dim_wizards_sql = """ with wizards as ( select * from {{ ref('stg_wizards') }} ), worlds as ( select * from {{ ref('stg_worlds') }} ), accepted_email_domains as ( select * from {{ ref('top_level_email_domains') }} ), check_valid_emails as ( select wizards.wizard_id, wizards.wizard_name, wizards.email, wizards.phone_number, wizards.world_id, coalesce ( wizards.email ~ '^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,}$' = true and accepted_email_domains.tld is not null, false) as is_valid_email_address from wizards left join accepted_email_domains on wizards.email_top_level_domain = lower(accepted_email_domains.tld) ) select check_valid_emails.wizard_id, check_valid_emails.wizard_name, check_valid_emails.email, check_valid_emails.is_valid_email_address, check_valid_emails.phone_number, worlds.world_name from check_valid_emails left join worlds on check_valid_emails.world_id = worlds.world_id """ orig_schema_yml = """ unit_tests: - name: test_valid_email_address model: dim_wizards given: - input: ref('stg_wizards') rows: - {email: cool@example.com, email_top_level_domain: example.com} - {email: cool@unknown.com, email_top_level_domain: unknown.com} - {email: badgmail.com, email_top_level_domain: gmail.com} - {email: missingdot@gmailcom, email_top_level_domain: gmail.com} - input: ref('top_level_email_domains') rows: - {tld: example.com} - {tld: gmail.com} - input: ref('stg_worlds') rows: [] expect: rows: - {email: cool@example.com, is_valid_email_address: true} - {email: cool@unknown.com, is_valid_email_address: false} - {email: badgmail.com, is_valid_email_address: false} - {email: missingdot@gmailcom, is_valid_email_address: false} """ schema_yml = """ unit_tests: - name: test_valid_email_address model: dim_wizards given: - input: ref('stg_wizards') format: sql rows: | select 1 as wizard_id, 'joe' as wizard_name, 'cool@example.com' as email, 'example.com' as email_top_level_domain, '123' as phone_number, 1 as world_id union all select 2 as wizard_id, 'don' as wizard_name, 'cool@unknown.com' as email, 'unknown.com' as email_top_level_domain, '456' as phone_number, 2 as world_id union all select 3 as wizard_id, 'mary' as wizard_name, 'badgmail.com' as email, 'gmail.com' as email_top_level_domain, '789' as phone_number, 3 as world_id union all select 4 as wizard_id, 'jane' as wizard_name, 'missingdot@gmailcom' as email, 'gmail.com' as email_top_level_domain, '102' as phone_number, 4 as world_id - input: ref('top_level_email_domains') format: sql rows: | select 'example.com' as tld union all select 'gmail.com' as tld - input: ref('stg_worlds') rows: [] expect: format: sql rows: | select 1 as wizard_id, 'joe' as wizard_name, 'cool@example.com' as email, true as is_valid_email_address, '123' as phone_number, null as world_name union all select 2 as wizard_id, 'don' as wizard_name, 'cool@unknown.com' as email, false as is_valid_email_address, '456' as phone_number, null as world_name union all select 3 as wizard_id, 'mary' as wizard_name, 'badgmail.com' as email, false as is_valid_email_address, '789' as phone_number, null as world_name union all select 4 as wizard_id, 'jane' as wizard_name, 'missingdot@gmailcom' as email, false as is_valid_email_address, '102' as phone_number, null as world_name """ class TestSQLFormat: @pytest.fixture(scope="class") def seeds(self): return { "wizards.csv": wizards_csv, "top_level_email_domains.csv": top_level_email_domains_csv, "worlds.csv": worlds_csv, } @pytest.fixture(scope="class") def models(self): return { "stg_wizards.sql": stg_wizards_sql, "stg_worlds.sql": stg_worlds_sql, "dim_wizards.sql": dim_wizards_sql, "schema.yml": schema_yml, } def test_sql_format(self, project): results = run_dbt(["build"]) assert len(results) == 7 stg_wizards_fixture_sql = """ select 1 as wizard_id, 'joe' as wizard_name, 'cool@example.com' as email, 'example.com' as email_top_level_domain, '123' as phone_number, 1 as world_id union all select 2 as wizard_id, 'don' as wizard_name, 'cool@unknown.com' as email, 'unknown.com' as email_top_level_domain, '456' as phone_number, 2 as world_id union all select 3 as wizard_id, 'mary' as wizard_name, 'badgmail.com' as email, 'gmail.com' as email_top_level_domain, '789' as phone_number, 3 as world_id union all select 4 as wizard_id, 'jane' as wizard_name, 'missingdot@gmailcom' as email, 'gmail.com' as email_top_level_domain, '102' as phone_number, 4 as world_id """ top_level_email_domains_fixture_sql = """ select 'example.com' as tld union all select 'gmail.com' as tld """ test_valid_email_address_fixture_sql = """ select 1 as wizard_id, 'joe' as wizard_name, 'cool@example.com' as email, true as is_valid_email_address, '123' as phone_number, null as world_name union all select 2 as wizard_id, 'don' as wizard_name, 'cool@unknown.com' as email, false as is_valid_email_address, '456' as phone_number, null as world_name union all select 3 as wizard_id, 'mary' as wizard_name, 'badgmail.com' as email, false as is_valid_email_address, '789' as phone_number, null as world_name union all select 4 as wizard_id, 'jane' as wizard_name, 'missingdot@gmailcom' as email, false as is_valid_email_address, '102' as phone_number, null as world_name """ fixture_schema_yml = """ unit_tests: - name: test_valid_email_address model: dim_wizards given: - input: ref('stg_wizards') format: sql fixture: stg_wizards_fixture - input: ref('top_level_email_domains') format: sql fixture: top_level_email_domains_fixture - input: ref('stg_worlds') rows: [] expect: format: sql fixture: test_valid_email_address_fixture """ class TestSQLFormatFixtures: @pytest.fixture(scope="class") def tests(self): return { "fixtures": { "test_valid_email_address_fixture.sql": test_valid_email_address_fixture_sql, "top_level_email_domains_fixture.sql": top_level_email_domains_fixture_sql, "stg_wizards_fixture.sql": stg_wizards_fixture_sql, } } @pytest.fixture(scope="class") def seeds(self): return { "wizards.csv": wizards_csv, "top_level_email_domains.csv": top_level_email_domains_csv, "worlds.csv": worlds_csv, } @pytest.fixture(scope="class") def models(self): return { "stg_wizards.sql": stg_wizards_sql, "stg_worlds.sql": stg_worlds_sql, "dim_wizards.sql": dim_wizards_sql, "schema.yml": fixture_schema_yml, } def test_sql_format_fixtures(self, project): results = run_dbt(["build"]) assert len(results) == 7 ================================================ FILE: tests/functional/unit_testing/test_state.py ================================================ import os import shutil from copy import deepcopy import pytest from fixtures import ( model_select_1_sql, model_select_2_sql, my_model_a_sql, my_model_b_sql, my_model_vars_sql, test_expect_2_yml, ) from fixtures import test_my_model_b_fixture_csv as test_my_model_fixture_csv_modified from fixtures import test_my_model_fixture_csv, test_my_model_simple_fixture_yml from dbt.tests.util import run_dbt, write_config_file, write_file class UnitTestState: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_model_vars_sql, "my_model_a.sql": my_model_a_sql, "my_model_b.sql": my_model_b_sql, "test_my_model.yml": test_my_model_simple_fixture_yml, } @pytest.fixture(scope="class") def tests(self): return { "fixtures": { "test_my_model_fixture.csv": test_my_model_fixture_csv, } } @pytest.fixture(scope="class") def project_config_update(self): return {"vars": {"my_test": "my_test_var"}} def copy_state(self, project_root): state_path = os.path.join(project_root, "state") if not os.path.exists(state_path): os.makedirs(state_path) shutil.copyfile( f"{project_root}/target/manifest.json", f"{project_root}/state/manifest.json" ) shutil.copyfile( f"{project_root}/target/run_results.json", f"{project_root}/state/run_results.json" ) class TestUnitTestStateModified(UnitTestState): def test_state_modified(self, project): run_dbt(["run"]) run_dbt(["test"], expect_pass=False) self.copy_state(project.project_root) # no changes results = run_dbt(["test", "--select", "state:modified", "--state", "state"]) assert len(results) == 0 # change underlying fixture file write_file( test_my_model_fixture_csv_modified, project.project_root, "tests", "fixtures", "test_my_model_fixture.csv", ) results = run_dbt( ["test", "--select", "state:modified", "--state", "state"], expect_pass=True ) assert len(results) == 1 assert results[0].node.name.endswith("test_depends_on_fixture") # reset changes self.copy_state(project.project_root) # change unit test definition of a single unit test with_changes = test_my_model_simple_fixture_yml.replace("{string_c: ab}", "{string_c: bc}") write_config_file(with_changes, project.project_root, "models", "test_my_model.yml") results = run_dbt( ["test", "--select", "state:modified", "--state", "state"], expect_pass=False ) assert len(results) == 1 assert results[0].node.name.endswith("test_has_string_c_ab") # change underlying model logic write_config_file( test_my_model_simple_fixture_yml, project.project_root, "models", "test_my_model.yml" ) write_file( my_model_vars_sql.replace("a+b as c,", "a + b as c,"), project.project_root, "models", "my_model.sql", ) results = run_dbt( ["test", "--select", "state:modified", "--state", "state"], expect_pass=False ) assert len(results) == 4 class TestUnitTestRetry(UnitTestState): def test_unit_test_retry(self, project): run_dbt(["run"]) run_dbt(["test"], expect_pass=False) self.copy_state(project.project_root) results = run_dbt(["retry"], expect_pass=False) assert len(results) == 1 class TestUnitTestDeferState(UnitTestState): @pytest.fixture(scope="class") def other_schema(self, unique_schema): return unique_schema + "_other" @pytest.fixture(scope="class") def profiles_config_update(self, dbt_profile_target, unique_schema, other_schema): outputs = {"default": dbt_profile_target, "otherschema": deepcopy(dbt_profile_target)} outputs["default"]["schema"] = unique_schema outputs["otherschema"]["schema"] = other_schema return {"test": {"outputs": outputs, "target": "default"}} def test_unit_test_defer_state(self, project): run_dbt(["run", "--target", "otherschema"]) self.copy_state(project.project_root) results = run_dbt(["test", "--defer", "--state", "state"], expect_pass=False) assert len(results) == 4 assert sorted([r.status for r in results]) == ["fail", "pass", "pass", "pass"] class TestUnitTestDeferDoesntOverwrite(UnitTestState): @pytest.fixture(scope="class") def models(self): return {"my_model.sql": model_select_1_sql, "test_my_model.yml": test_expect_2_yml} def test_unit_test_defer_state(self, project): run_dbt(["test"], expect_pass=False) self.copy_state(project.project_root) write_file( model_select_2_sql, project.project_root, "models", "my_model.sql", ) results = run_dbt(["test", "--defer", "--state", "state"]) assert len(results) == 1 assert sorted([r.status for r in results]) == ["pass"] ================================================ FILE: tests/functional/unit_testing/test_unit_testing.py ================================================ import os from unittest import mock import pytest from fixtures import ( # noqa: F401 datetime_test, event_sql, external_package, external_package__accounts_seed_csv, my_incremental_model_sql, my_incremental_model_versioned_yml, my_incremental_model_with_alias_sql, my_model_a_sql, my_model_b_sql, my_model_sql, my_model_vars_sql, test_my_model_incremental_yml_basic, test_my_model_incremental_yml_no_override, test_my_model_incremental_yml_no_this_input, test_my_model_incremental_yml_wrong_override, test_my_model_yml, test_my_model_yml_invalid, test_my_model_yml_invalid_ref, top_level_domains_sql, valid_emails_sql, ) from dbt.contracts.results import NodeStatus from dbt.exceptions import DuplicateResourceNameError, ParsingError from dbt.plugins.manifest import ModelNodeArgs, PluginNodes from dbt.tests.fixtures.project import write_project_files from dbt.tests.util import ( file_exists, get_manifest, read_file, run_dbt, run_dbt_and_capture, write_file, ) from tests.unit.utils import normalize class TestUnitTests: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_model_vars_sql, "my_model_a.sql": my_model_a_sql, "my_model_b.sql": my_model_b_sql, "test_my_model.yml": test_my_model_yml + datetime_test, } @pytest.fixture(scope="class") def project_config_update(self): return {"vars": {"my_test": "my_test_var"}} def test_basic(self, project): results = run_dbt(["run"]) assert len(results) == 3 # Select by model name results = run_dbt(["test", "--select", "my_model"], expect_pass=False) assert len(results) == 5 results = run_dbt( ["build", "--select", "my_model", "--resource-types", "model unit_test"], expect_pass=False, ) assert len(results) == 6 for result in results: if result.node.unique_id == "model.test.my_model": result.status == NodeStatus.Skipped # Run build command but specify no unit tests results = run_dbt( ["build", "--select", "my_model", "--exclude-resource-types", "unit_test"], expect_pass=True, ) assert len(results) == 1 # Exclude unit tests with environment variable for build command os.environ["DBT_EXCLUDE_RESOURCE_TYPES"] = "unit_test" results = run_dbt(["build", "--select", "my_model"], expect_pass=True) assert len(results) == 1 # Exclude unit tests with environment variable for test command results = run_dbt(["test", "--select", "my_model"], expect_pass=True) assert len(results) == 0 # Exclude unit tests with environment variable for list command results = run_dbt(["list", "--select", "my_model"], expect_pass=True) assert len(results) == 1 del os.environ["DBT_EXCLUDE_RESOURCE_TYPES"] # Test select by test name results = run_dbt(["test", "--select", "test_name:test_my_model_string_concat"]) assert len(results) == 1 # Select, method not specified results = run_dbt(["test", "--select", "test_my_model_overrides"]) assert len(results) == 1 # Select using tag results = run_dbt(["test", "--select", "tag:test_this"]) assert len(results) == 1 # Partial parsing... remove test write_file(test_my_model_yml, project.project_root, "models", "test_my_model.yml") results = run_dbt(["test", "--select", "my_model"], expect_pass=False) assert len(results) == 4 # Partial parsing... put back removed test write_file( test_my_model_yml + datetime_test, project.project_root, "models", "test_my_model.yml" ) results = run_dbt(["test", "--select", "my_model"], expect_pass=False) assert len(results) == 5 manifest = get_manifest(project.project_root) assert len(manifest.unit_tests) == 5 # Every unit test has a depends_on to the model it tests for unit_test_definition in manifest.unit_tests.values(): assert unit_test_definition.depends_on.nodes[0] == "model.test.my_model" # Check for duplicate unit test name # this doesn't currently pass with partial parsing because of the root problem # described in https://github.com/dbt-labs/dbt-core/issues/8982 write_file( test_my_model_yml + datetime_test + datetime_test, project.project_root, "models", "test_my_model.yml", ) with pytest.raises(DuplicateResourceNameError): run_dbt(["run", "--no-partial-parse", "--select", "my_model"]) class TestUnitTestIncrementalModelBasic: @pytest.fixture(scope="class") def models(self): return { "my_incremental_model.sql": my_incremental_model_sql, "events.sql": event_sql, "schema.yml": test_my_model_incremental_yml_basic, } def test_basic(self, project): results = run_dbt(["run"]) assert len(results) == 2 # Select by model name results = run_dbt(["test", "--select", "my_incremental_model"], expect_pass=True) assert len(results) == 2 class TestUnitTestIncrementalModelNoOverride: @pytest.fixture(scope="class") def models(self): return { "my_incremental_model.sql": my_incremental_model_sql, "events.sql": event_sql, "schema.yml": test_my_model_incremental_yml_no_override, } def test_no_override(self, project): with pytest.raises( ParsingError, match="Boolean override for 'is_incremental' must be provided for unit test 'incremental_false' in model 'my_incremental_model'", ): run_dbt(["parse"]) class TestUnitTestIncrementalModelWrongOverride: @pytest.fixture(scope="class") def models(self): return { "my_incremental_model.sql": my_incremental_model_sql, "events.sql": event_sql, "schema.yml": test_my_model_incremental_yml_wrong_override, } def test_str_override(self, project): with pytest.raises( ParsingError, match="Boolean override for 'is_incremental' must be provided for unit test 'incremental_false' in model 'my_incremental_model'", ): run_dbt(["parse"]) class TestUnitTestIncrementalModelNoThisInput: @pytest.fixture(scope="class") def models(self): return { "my_incremental_model.sql": my_incremental_model_sql, "events.sql": event_sql, "schema.yml": test_my_model_incremental_yml_no_this_input, } def test_no_this_input(self, project): with pytest.raises( ParsingError, match="Unit test 'incremental_true' for incremental model 'my_incremental_model' must have a 'this' input", ): run_dbt(["parse"]) my_new_model = """ select my_favorite_seed.id, a + b as c from {{ ref('my_favorite_seed') }} as my_favorite_seed inner join {{ ref('my_favorite_model') }} as my_favorite_model on my_favorite_seed.id = my_favorite_model.id """ my_favorite_model = """ select 2 as id, 3 as b """ seed_my_favorite_seed = """id,a 1,5 2,4 3,3 4,2 5,1 """ schema_yml_explicit_seed = """ unit_tests: - name: t model: my_new_model given: - input: ref('my_favorite_seed') rows: - {id: 1, a: 10} - input: ref('my_favorite_model') rows: - {id: 1, b: 2} expect: rows: - {id: 1, c: 12} """ schema_yml_implicit_seed = """ unit_tests: - name: t model: my_new_model given: - input: ref('my_favorite_seed') - input: ref('my_favorite_model') rows: - {id: 1, b: 2} expect: rows: - {id: 1, c: 7} """ schema_yml_nonexistent_seed = """ unit_tests: - name: t model: my_new_model given: - input: ref('my_second_favorite_seed') - input: ref('my_favorite_model') rows: - {id: 1, b: 2} expect: rows: - {id: 1, c: 7} """ class TestUnitTestIncrementalModelWithAlias: @pytest.fixture(scope="class") def models(self): return { "my_incremental_model.sql": my_incremental_model_with_alias_sql, "events.sql": event_sql, "schema.yml": test_my_model_incremental_yml_basic, } def test_basic(self, project): results = run_dbt(["run"]) assert len(results) == 2 # Select by model name results = run_dbt(["test", "--select", "my_incremental_model"], expect_pass=True) assert len(results) == 2 class TestUnitTestIncrementalModelWithVersion: @pytest.fixture(scope="class") def models(self): return { "my_incremental_model.sql": my_incremental_model_sql, "events.sql": event_sql, "schema.yml": my_incremental_model_versioned_yml + test_my_model_incremental_yml_basic, } def test_basic(self, project): results = run_dbt(["run"]) assert len(results) == 2 # Select by model name results = run_dbt(["test", "--select", "my_incremental_model"], expect_pass=True) assert len(results) == 2 schema_ref_with_versioned_model = """ models: - name: source latest_version: {latest_version} versions: - v: 1 - v: 2 - name: model_to_test unit_tests: - name: ref_versioned model: 'model_to_test' given: - input: {input} rows: - {{result: 3}} expect: rows: - {{result: 3}} """ class TestUnitTestRefWithVersion: @pytest.fixture(scope="class") def models(self): return { "model_to_test.sql": "select result from {{ ref('source')}}", "source.sql": "select 2 as result", "source_v2.sql": "select 2 as result", "schema.yml": schema_ref_with_versioned_model.format( **{"latest_version": 1, "input": "ref('source')"} ), } def test_basic(self, project): results = run_dbt(["run"]) results = run_dbt(["test", "--select", "model_to_test"], expect_pass=True) assert len(results) == 1 class TestUnitTestRefMissingVersionModel: @pytest.fixture(scope="class") def models(self): return { "model_to_test.sql": "select result from {{ ref('source')}}", "source_v1.sql": "select 2 as result", "source_v2.sql": "select 2 as result", "schema.yml": schema_ref_with_versioned_model.format( **{"latest_version": 1, "input": "ref('source', v=1)"} ), } def test_basic(self, project): results = run_dbt(["run"]) results = run_dbt(["test", "--select", "model_to_test"], expect_pass=True) assert len(results) == 1 class TestUnitTestRefWithMissingVersionRef: @pytest.fixture(scope="class") def models(self): return { "model_to_test.sql": "select result from {{ ref('source', v=1)}}", "source_v1.sql": "select 2 as result", "source_v2.sql": "select 2 as result", "schema.yml": schema_ref_with_versioned_model.format( **{"latest_version": 1, "input": "ref('source')"} ), } def test_basic(self, project): results = run_dbt(["run"]) results = run_dbt(["test", "--select", "model_to_test"], expect_pass=True) assert len(results) == 1 class TestUnitTestRefWithVersionLatestSecond: @pytest.fixture(scope="class") def models(self): return { "model_to_test.sql": "select result from {{ ref('source')}}", "source_v1.sql": "select 2 as result", "source_v2.sql": "select 2 as result", "schema.yml": schema_ref_with_versioned_model.format( **{"latest_version": 2, "input": "ref('source')"} ), } def test_basic(self, project): results = run_dbt(["run"]) results = run_dbt(["test", "--select", "model_to_test"], expect_pass=True) assert len(results) == 1 class TestUnitTestRefWithVersionMissingRefTest: @pytest.fixture(scope="class") def models(self): return { "model_to_test.sql": "select result from {{ ref('source', v=2)}}", "source_v1.sql": "select 2 as result", "source_v2.sql": "select 2 as result", "schema.yml": schema_ref_with_versioned_model.format( **{"latest_version": 1, "input": "ref('source')"} ), } def test_basic(self, project): results = run_dbt(["run"]) assert len(results) == 3 # run_dbt(["test", "--select", "model_to_test"], expect_pass=False) exec_result, _ = run_dbt_and_capture( ["test", "--select", "model_to_test"], expect_pass=False ) msg_error = exec_result[0].message assert msg_error.lower().lstrip().startswith("compilation error") class TestUnitTestRefWithVersionDiffLatest: @pytest.fixture(scope="class") def models(self): return { "model_to_test.sql": "select result from {{ ref('source', v=2)}}", "source_v1.sql": "select 2 as result", "source_v2.sql": "select 2 as result", "schema.yml": schema_ref_with_versioned_model.format( **{"latest_version": 1, "input": "ref('source', v=2)"} ), } def test_basic(self, project): results = run_dbt(["run"]) assert len(results) == 3 run_dbt(["test", "--select", "model_to_test"], expect_pass=True) class TestUnitTestExplicitSeed: @pytest.fixture(scope="class") def seeds(self): return {"my_favorite_seed.csv": seed_my_favorite_seed} @pytest.fixture(scope="class") def models(self): return { "my_new_model.sql": my_new_model, "my_favorite_model.sql": my_favorite_model, "schema.yml": schema_yml_explicit_seed, } def test_explicit_seed(self, project): run_dbt(["seed"]) run_dbt(["run"]) # Select by model name results = run_dbt(["test", "--select", "my_new_model"], expect_pass=True) assert len(results) == 1 class TestUnitTestImplicitSeed: @pytest.fixture(scope="class") def seeds(self): return {"my_favorite_seed.csv": seed_my_favorite_seed} @pytest.fixture(scope="class") def models(self): return { "my_new_model.sql": my_new_model, "my_favorite_model.sql": my_favorite_model, "schema.yml": schema_yml_implicit_seed, } def test_implicit_seed(self, project): run_dbt(["seed"]) run_dbt(["run"]) # Select by model name results = run_dbt(["test", "--select", "my_new_model"], expect_pass=True) assert len(results) == 1 class TestUnitTestNonexistentSeed: @pytest.fixture(scope="class") def seeds(self): return {"my_favorite_seed.csv": seed_my_favorite_seed} @pytest.fixture(scope="class") def models(self): return { "my_new_model.sql": my_new_model, "my_favorite_model.sql": my_favorite_model, "schema.yml": schema_yml_nonexistent_seed, } def test_nonexistent_seed(self, project): with pytest.raises( ParsingError, match="Unable to find seed 'test.my_second_favorite_seed' for unit tests" ): run_dbt(["test", "--select", "my_new_model"], expect_pass=False) class TestUnitTestInvalidInputConfiguration: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_model_sql, "my_model_a.sql": my_model_a_sql, "my_model_b.sql": my_model_b_sql, "test_my_model.yml": test_my_model_yml_invalid, } def test_invalid_input_configuration(self, project): results = run_dbt(["run"]) assert len(results) == 3 # A data type in a given row is incorrect, and we'll get a runtime error run_dbt(["test"], expect_pass=False) # Test invalid model ref. Parsing error InvalidUnitTestGivenInput write_file( test_my_model_yml_invalid_ref, project.project_root, "models", "test_my_model.yml" ) results = run_dbt(["test"], expect_pass=False) result = results.results[0] assert "not found in the manifest" in result.message unit_test_ext_node_yml = """ unit_tests: - name: unit_test_ext_node model: valid_emails given: - input: ref('external_package', 'external_model') rows: - {user_id: 1, email: cool@example.com, email_top_level_domain: example.com} - {user_id: 2, email: cool@unknown.com, email_top_level_domain: unknown.com} - {user_id: 3, email: badgmail.com, email_top_level_domain: gmail.com} - {user_id: 4, email: missingdot@gmailcom, email_top_level_domain: gmail.com} - input: ref('top_level_domains') rows: - {tld: example.com} - {tld: gmail.com} expect: rows: - {user_id: 1, is_valid_email_address: true} - {user_id: 2, is_valid_email_address: false} - {user_id: 3, is_valid_email_address: true} - {user_id: 4, is_valid_email_address: true} """ class TestUnitTestExternalPackageNode: @pytest.fixture(scope="class", autouse=True) def setUp(self, project_root, external_package): # noqa: F811 write_project_files(project_root, "external_package", external_package) @pytest.fixture(scope="class") def packages(self): return {"packages": [{"local": "external_package"}]} @pytest.fixture(scope="class") def models(self): return { "top_level_domains.sql": top_level_domains_sql, "valid_emails.sql": valid_emails_sql, "unit_test_ext_node.yml": unit_test_ext_node_yml, } def test_unit_test_ext_nodes( self, project, ): # `deps` to install the external package run_dbt(["deps"], expect_pass=True) # `seed` need so a table exists for `external_model` to point to run_dbt(["seed"], expect_pass=True) # `run` needed to ensure `top_level_domains` exists in database for column getting step run_dbt(["run"], expect_pass=True) results = run_dbt(["test", "--select", "valid_emails"], expect_pass=True) assert len(results) == 1 class TestUnitTestExternalProjectNode: @pytest.fixture(scope="class") def external_model_node(self, unique_schema): return ModelNodeArgs( name="external_model", package_name="external_package", identifier="external_node_seed", schema=unique_schema, ) @pytest.fixture(scope="class") def seeds(self): return {"external_node_seed.csv": external_package__accounts_seed_csv} @pytest.fixture(scope="class") def models(self): return { "top_level_domains.sql": top_level_domains_sql, "valid_emails.sql": valid_emails_sql, "unit_test_ext_node.yml": unit_test_ext_node_yml, } @mock.patch("dbt.plugins.get_plugin_manager") def test_unit_test_ext_nodes( self, get_plugin_manager, project, external_model_node, ): # initial plugin - one external model external_nodes = PluginNodes() external_nodes.add_model(external_model_node) get_plugin_manager.return_value.get_nodes.return_value = external_nodes # `seed` need so a table exists for `external_model` to point to run_dbt(["seed"], expect_pass=True) # `run` needed to ensure `top_level_domains` exists in database for column getting step run_dbt(["run"], expect_pass=True) results = run_dbt(["test", "--select", "valid_emails"], expect_pass=True) assert len(results) == 1 subfolder_model_a_sql = """select 1 as id, 'blue' as color""" subfolder_model_b_sql = """ select id, color from {{ ref('model_a') }} """ subfolder_my_model_yml = """ unit_tests: - name: my_unit_test model: model_b given: - input: ref('model_a') rows: - { id: 1, color: 'blue' } expect: rows: - { id: 1, color: 'red' } """ class TestUnitTestSubfolderPath: @pytest.fixture(scope="class") def models(self): return { "subfolder": { "model_a.sql": subfolder_model_a_sql, "model_b.sql": subfolder_model_b_sql, "my_model.yml": subfolder_my_model_yml, } } def test_subfolder_unit_test(self, project): results, output = run_dbt_and_capture(["build"], expect_pass=False) # Test that input fixture doesn't overwrite the original model assert ( read_file("target/compiled/test/models/subfolder/model_a.sql").strip() == subfolder_model_a_sql.strip() ) # Test that correct path is written in logs assert ( normalize( "target/compiled/test/models/subfolder/my_model.yml/models/subfolder/my_unit_test.sql" ) in output ) assert file_exists( normalize( "target/compiled/test/models/subfolder/my_model.yml/models/subfolder/my_unit_test.sql" ) ) my_model_with_function_sql = """ select {{ function('double_it') }}(1) as doubled_value """ test_my_model_with_function_yml = """ unit_tests: - name: test_my_model model: my_model given: [] expect: rows: - {doubled_value: 2} """ double_it_sql = """ select value * 2 """ double_it_yml = """ functions: - name: double_it description: Doubles whatever number is passed in arguments: - name: value data_type: float description: A number to be doubled returns: data_type: float """ class TestUnitTestModelWithFunction: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_model_with_function_sql, "test_my_model.yml": test_my_model_with_function_yml, } @pytest.fixture(scope="class") def functions(self): return { "double_it.sql": double_it_sql, "double_it.yml": double_it_yml, } def test_model_with_function(self, project): run_dbt(["build", "--empty"]) run_dbt(["test"]) ================================================ FILE: tests/functional/unit_testing/test_ut_adapter_hooks.py ================================================ from unittest import mock import pytest from dbt.tests.util import run_dbt, run_dbt_and_capture from dbt_common.exceptions import CompilationError from tests.functional.unit_testing.fixtures import ( my_model_a_sql, my_model_b_sql, my_model_sql, test_my_model_pass_yml, ) class BaseUnitTestAdapterHook: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_model_sql, "my_model_a.sql": my_model_a_sql, "my_model_b.sql": my_model_b_sql, "test_my_model.yml": test_my_model_pass_yml, } class TestUnitTestAdapterPreHook(BaseUnitTestAdapterHook): def test_unit_test_runs_adapter_pre_hook_passes(self, project): results = run_dbt(["run"]) assert len(results) == 3 mock_pre_model_hook = mock.Mock() with mock.patch.object(type(project.adapter), "pre_model_hook", mock_pre_model_hook): results = run_dbt(["test", "--select", "test_name:test_my_model"], expect_pass=True) assert len(results) == 1 mock_pre_model_hook.assert_called_once() def test_unit_test_runs_adapter_pre_hook_fails(self, project): results = run_dbt(["run"]) assert len(results) == 3 mock_pre_model_hook = mock.Mock() mock_pre_model_hook.side_effect = CompilationError("exception from adapter.pre_model_hook") with mock.patch.object(type(project.adapter), "pre_model_hook", mock_pre_model_hook): (_, log_output) = run_dbt_and_capture( ["test", "--select", "test_name:test_my_model"], expect_pass=False ) assert "exception from adapter.pre_model_hook" in log_output class TestUnitTestAdapterPostHook(BaseUnitTestAdapterHook): def test_unit_test_runs_adapter_post_hook_pass(self, project): results = run_dbt(["run"]) assert len(results) == 3 mock_post_model_hook = mock.Mock() with mock.patch.object(type(project.adapter), "post_model_hook", mock_post_model_hook): results = run_dbt(["test", "--select", "test_name:test_my_model"], expect_pass=True) assert len(results) == 1 mock_post_model_hook.assert_called_once() def test_unit_test_runs_adapter_post_hook_fails(self, project): results = run_dbt(["run"]) assert len(results) == 3 mock_post_model_hook = mock.Mock() mock_post_model_hook.side_effect = CompilationError( "exception from adapter.post_model_hook" ) with mock.patch.object(type(project.adapter), "post_model_hook", mock_post_model_hook): (_, log_output) = run_dbt_and_capture( ["test", "--select", "test_name:test_my_model"], expect_pass=False ) assert "exception from adapter.post_model_hook" in log_output ================================================ FILE: tests/functional/unit_testing/test_ut_aliases.py ================================================ import pytest from dbt.tests.util import run_dbt model_with_alias_sql = """ {{ config( alias='beautiful_alias', schema='events', materialized='view' ) }} select 'foo' as foo """ model_tested = """ select * from {{ ref('model_with_alias') }} """ unit_test_yml = """ unit_tests: - name: test_model_with_alias_input model: model_tested given: - input: ref('model_with_alias') rows: - {foo: bar } - {foo: foo } expect: rows: - {foo: bar } - {foo: foo } """ class TestUnitTestInputWithAlias: @pytest.fixture(scope="class") def models(self): return { "model_with_alias.sql": model_with_alias_sql, "model_tested.sql": model_tested, "unit_test.yml": unit_test_yml, } def test_input_with_alias(self, project): results = run_dbt(["run"]) assert len(results) == 2 results = run_dbt(["test"]) assert len(results) == 1 ================================================ FILE: tests/functional/unit_testing/test_ut_dependency.py ================================================ import pytest from dbt.tests.fixtures.project import write_project_files from dbt.tests.util import get_unique_ids_in_results, run_dbt local_dependency__dbt_project_yml = """ name: 'local_dep' version: '1.0' seeds: quote_columns: False """ local_dependency__schema_yml = """ sources: - name: seed_source schema: "{{ var('schema_override', target.schema) }}" tables: - name: "seed" columns: - name: id data_tests: - unique unit_tests: - name: test_dep_model_id model: dep_model given: - input: ref('seed') rows: - {id: 1, name: Joe} expect: rows: - {name_id: Joe_1} """ local_dependency__dep_model_sql = """ select name || '_' || id as name_id from {{ ref('seed') }} """ local_dependency__seed_csv = """id,name 1,Mary 2,Sam 3,John """ my_model_sql = """ select * from {{ ref('dep_model') }} """ my_model_schema_yml = """ unit_tests: - name: test_my_model_name_id model: my_model given: - input: ref('dep_model') rows: - {name_id: Joe_1} expect: rows: - {name_id: Joe_1} """ class TestUnitTestingInDependency: @pytest.fixture(scope="class", autouse=True) def setUp(self, project_root): local_dependency_files = { "dbt_project.yml": local_dependency__dbt_project_yml, "models": { "schema.yml": local_dependency__schema_yml, "dep_model.sql": local_dependency__dep_model_sql, }, "seeds": {"seed.csv": local_dependency__seed_csv}, } write_project_files(project_root, "local_dependency", local_dependency_files) @pytest.fixture(scope="class") def packages(self): return {"packages": [{"local": "local_dependency"}]} @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_model_sql, "schema.yml": my_model_schema_yml, } def test_unit_test_in_dependency(self, project): run_dbt(["deps"]) run_dbt(["seed"]) results = run_dbt(["run"]) assert len(results) == 2 results = run_dbt(["test"]) assert len(results) == 3 unique_ids = get_unique_ids_in_results(results) assert "unit_test.local_dep.dep_model.test_dep_model_id" in unique_ids results = run_dbt(["test", "--select", "test_type:unit"]) # two unit tests, 1 in root package, one in local_dep package assert len(results) == 2 results = run_dbt(["test", "--select", "local_dep"]) # 2 tests in local_dep package assert len(results) == 2 results = run_dbt(["test", "--select", "test"]) # 1 test in root package assert len(results) == 1 ================================================ FILE: tests/functional/unit_testing/test_ut_diffing.py ================================================ import pytest from dbt.tests.util import run_dbt my_input_model = """ SELECT 1 as id, 'some string' as status """ my_model = """ SELECT * FROM {{ ref("my_input_model") }} """ test_my_model_order_insensitive = """ unit_tests: - name: unordered_no_nulls model: my_model given: - input: ref("my_input_model") rows: - {"id": 1, "status": 'B'} - {"id": 2, "status": 'B'} - {"id": 3, "status": 'A'} expect: rows: - {"id": 3, "status": 'A'} - {"id": 2, "status": 'B'} - {"id": 1, "status": 'B'} - name: unordered_with_nulls model: my_model given: - input: ref("my_input_model") rows: - {"id": , "status": 'B'} - {"id": , "status": 'B'} - {"id": 3, "status": 'A'} expect: rows: - {"id": 3, "status": 'A'} - {"id": , "status": 'B'} - {"id": , "status": 'B'} - name: unordered_with_nulls_2 model: my_model given: - input: ref("my_input_model") rows: - {"id": 3, "status": 'A'} - {"id": , "status": 'B'} - {"id": , "status": 'B'} expect: rows: - {"id": , "status": 'B'} - {"id": , "status": 'B'} - {"id": 3, "status": 'A'} - name: unordered_with_nulls_mixed_columns model: my_model given: - input: ref("my_input_model") rows: - {"id": 3, "status": 'A'} - {"id": , "status": 'B'} - {"id": 1, "status": } expect: rows: - {"id": 1, "status": } - {"id": , "status": 'B'} - {"id": 3, "status": 'A'} - name: unordered_with_null model: my_model given: - input: ref("my_input_model") rows: - {"id": 3, "status": 'A'} - {"id": , "status": 'B'} expect: rows: - {"id": , "status": 'B'} - {"id": 3, "status": 'A'} - name: ordered_with_nulls model: my_model given: - input: ref("my_input_model") rows: - {"id": 3, "status": 'A'} - {"id": , "status": 'B'} - {"id": , "status": 'B'} expect: rows: - {"id": 3, "status": 'A'} - {"id": , "status": 'B'} - {"id": , "status": 'B'} """ class TestUnitTestingDiffIsOrderAgnostic: @pytest.fixture(scope="class") def models(self): return { "my_input_model.sql": my_input_model, "my_model.sql": my_model, "test_my_model.yml": test_my_model_order_insensitive, } def test_unit_testing_diff_is_order_insensitive(self, project): run_dbt(["run"]) # Select by model name results = run_dbt(["test", "--select", "my_model"], expect_pass=True) assert len(results) == 6 ================================================ FILE: tests/functional/unit_testing/test_ut_ephemeral.py ================================================ import pytest from dbt.contracts.results import RunStatus, TestStatus from dbt.tests.util import run_dbt, write_file ephemeral_model_sql = """ {{ config(materialized="ephemeral") }} select 1 as id, 'Emily' as first_name """ nested_ephemeral_model_sql = """ {{ config(materialized="ephemeral") }} select * from {{ ref('ephemeral_model') }} """ customers_sql = """ select * from {{ ref('nested_ephemeral_model') }} """ test_sql_format_yml = """ unit_tests: - name: test_customers model: customers given: - input: ref('nested_ephemeral_model') format: sql rows: | select 1 as id, 'Emily' as first_name expect: rows: - {id: 1, first_name: Emily} """ failing_test_sql_format_yml = """ - name: fail_test_customers model: customers given: - input: ref('nested_ephemeral_model') format: sql rows: | select 1 as id, 'Emily' as first_name expect: rows: - {id: 1, first_name: Joan} """ class TestUnitTestEphemeralInput: @pytest.fixture(scope="class") def models(self): return { "customers.sql": customers_sql, "ephemeral_model.sql": ephemeral_model_sql, "nested_ephemeral_model.sql": nested_ephemeral_model_sql, "tests.yml": test_sql_format_yml, } def test_ephemeral_input(self, project): results = run_dbt(["run"]) len(results) == 1 results = run_dbt(["test", "--select", "test_type:unit"]) assert len(results) == 1 results = run_dbt(["build"]) assert len(results) == 2 result_unique_ids = [result.node.unique_id for result in results] assert len(result_unique_ids) == 2 assert "unit_test.test.customers.test_customers" in result_unique_ids # write failing unit test write_file( test_sql_format_yml + failing_test_sql_format_yml, project.project_root, "models", "tests.yml", ) results = run_dbt(["build"], expect_pass=False) for result in results: if result.node.unique_id == "model.test.customers": assert result.status == RunStatus.Skipped elif result.node.unique_id == "unit_test.test.customers.fail_test_customers": assert result.status == TestStatus.Fail assert len(results) == 3 ================================================ FILE: tests/functional/unit_testing/test_ut_list.py ================================================ import json import os import pytest from fixtures import ( # noqa: F401 datetime_test, disabled_my_model_tests_yml, my_model_a_sql, my_model_b_sql, my_model_disabled_sql, my_model_disabled_yml, my_model_vars_sql, test_disabled_unit_test_my_model_yml, test_my_model_yml, ) from dbt.tests.util import run_dbt class TestUnitTestList: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_model_vars_sql, "my_model_a.sql": my_model_a_sql, "my_model_b.sql": my_model_b_sql, "test_my_model.yml": test_my_model_yml + datetime_test, "test_disabled_my_model.yml": test_disabled_unit_test_my_model_yml, } @pytest.fixture(scope="class") def project_config_update(self): return {"vars": {"my_test": "my_test_var"}} def test_unit_test_list(self, project): # make sure things are working results = run_dbt(["run"]) assert len(results) == 3 results = run_dbt(["test"], expect_pass=False) assert len(results) == 5 results = run_dbt(["list"]) expected = [ "test.my_model", "test.my_model_a", "test.my_model_b", "unit_test:test.test_my_model", "unit_test:test.test_my_model_datetime", "unit_test:test.test_my_model_empty", "unit_test:test.test_my_model_overrides", "unit_test:test.test_my_model_string_concat", ] assert sorted(results) == sorted(expected) results = run_dbt(["list", "--select", "test_type:unit"]) assert len(results) == 5 # Check json result results = run_dbt(["list", "--select", "test_type:unit", "--output", "json"]) expected_test_my_model = { "name": "test_my_model", "resource_type": "unit_test", "package_name": "test", "original_file_path": os.path.join("models", "test_my_model.yml"), "unique_id": "unit_test.test.my_model.test_my_model", "depends_on": {"macros": [], "nodes": ["model.test.my_model"]}, "config": {"tags": [], "meta": {}, "enabled": True}, } for result in results: json_result = json.loads(result) if "name" in json_result and json_result["name"] == "test_my_model": assert json_result == expected_test_my_model results = run_dbt( [ "list", "--select", "test_type:unit", "--output", "json", "--output-keys", "unique_id", "model", ] ) for result in results: json_result = json.loads(result) assert json_result["model"] == "my_model" class TestUnitTestListDisabled: @pytest.fixture(scope="class") def models(self): return { "my_model_disabled_yml.sql": "select 1 as id", "my_model_disabled_sql.sql": my_model_disabled_sql, "my_model_disabled_yml.yml": my_model_disabled_yml, "disabled_my_model_tests.yml": disabled_my_model_tests_yml, } def test_disabled_unit_tests(self, project): results = run_dbt(["test", "--select", "test_type:unit"]) assert len(results) == 0 ================================================ FILE: tests/functional/unit_testing/test_ut_macros.py ================================================ import pytest from dbt.tests.util import run_dbt my_model_without_composition_sql = """ {{ config(materialized='table') }} {% set one = macro_one() %} {% set two = macro_two() %} select 1 as id """ my_model_with_composition_sql = """ {{ config(materialized='table') }} {% set one = macro_one() %} {% set two = macro_two() %} {% set one_plus_two = one + two %} select 1 as id """ my_macro_sql = """ {% macro macro_one() -%} {{ return(1) }} {%- endmacro %} {% macro macro_two() -%} {{ return(2) }} {%- endmacro %} """ my_unit_test_yml = """ unit_tests: - name: my_unit_test model: my_model given: [] expect: rows: - {id: 1} """ class TestMacroWithoutComposition: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_model_without_composition_sql, "my_unit_test.yml": my_unit_test_yml, } @pytest.fixture(scope="class") def macros(self): return {"my_macros.sql": my_macro_sql} def test_macro_in_unit_test(self, project): # Test that a model without macro composition properly resolves macro names in unit tests run_dbt(["test"]) class TestMacroComposition: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_model_with_composition_sql, "my_unit_test.yml": my_unit_test_yml, } @pytest.fixture(scope="class") def macros(self): return {"my_macros.sql": my_macro_sql} def test_macro_composition_in_unit_test(self, project): # Verify model works fine outside of unit testing results = run_dbt(["run"]) assert len(results) == 1 # Test that a model with macro composition properly resolves macro names in unit tests run_dbt(["test"]) ================================================ FILE: tests/functional/unit_testing/test_ut_names.py ================================================ import pytest from fixtures import ( my_model_a_sql, my_model_b_sql, test_model_a_b_yml, test_model_a_with_duplicate_test_name_yml, ) from dbt.exceptions import DuplicateResourceNameError from dbt.tests.util import run_dbt, run_dbt_and_capture class TestUnitTestDuplicateTestNamesAcrossModels: @pytest.fixture(scope="class") def models(self): return { "my_model_a.sql": my_model_a_sql, "my_model_b.sql": my_model_b_sql, "test_model_a_b.yml": test_model_a_b_yml, } def test_duplicate_test_names_across_models(self, project): results = run_dbt(["run"]) assert len(results) == 2 # Select duplicate tests results, log_output = run_dbt_and_capture(["test"], expect_pass=True) assert len(results) == 2 assert {"model.test.my_model_a", "model.test.my_model_b"} == { result.node.tested_node_unique_id for result in results } assert "my_model_a::my_test_name" in log_output assert "my_model_b::my_test_name" in log_output # Test select duplicates by by test name results = run_dbt(["test", "--select", "test_name:my_test_name"]) assert len(results) == 2 assert {"model.test.my_model_a", "model.test.my_model_b"} == { result.node.tested_node_unique_id for result in results } assert "my_model_a::my_test_name" in log_output assert "my_model_b::my_test_name" in log_output results = run_dbt(["test", "--select", "my_model_a,test_name:my_test_name"]) assert len(results) == 1 assert results[0].node.tested_node_unique_id == "model.test.my_model_a" results = run_dbt(["test", "--select", "my_model_b,test_name:my_test_name"]) assert len(results) == 1 assert results[0].node.tested_node_unique_id == "model.test.my_model_b" # Test select by model name results = run_dbt(["test", "--select", "my_model_a"]) assert len(results) == 1 assert results[0].node.tested_node_unique_id == "model.test.my_model_a" results = run_dbt(["test", "--select", "my_model_b"]) assert len(results) == 1 assert results[0].node.tested_node_unique_id == "model.test.my_model_b" class TestUnitTestDuplicateTestNamesWithinModel: @pytest.fixture(scope="class") def models(self): return { "my_model_a.sql": my_model_a_sql, "test_model_a.yml": test_model_a_with_duplicate_test_name_yml, } def test_duplicate_test_names_within_model(self, project): with pytest.raises(DuplicateResourceNameError): run_dbt(["run"]) ================================================ FILE: tests/functional/unit_testing/test_ut_overrides.py ================================================ import pytest from dbt.tests.util import run_dbt my_model_with_macros = """ SELECT {{ current_timestamp() }} as global_current_timestamp, {{ dbt.current_timestamp() }} as dbt_current_timestamp, {{ dbt.type_int() }} as dbt_type_int, {{ my_macro() }} as user_defined_my_macro, {{ dbt_utils.generate_surrogate_key() }} as package_defined_macro """ test_my_model_with_macros = """ unit_tests: - name: test_macro_overrides model: my_model_with_macros overrides: macros: current_timestamp: "'current_timestamp_override'" dbt.type_int: "'dbt_macro_override'" my_macro: "'global_user_defined_macro_override'" dbt_utils.generate_surrogate_key: "'package_macro_override'" given: [] expect: rows: - global_current_timestamp: "current_timestamp_override" dbt_current_timestamp: "current_timestamp_override" dbt_type_int: "dbt_macro_override" user_defined_my_macro: "global_user_defined_macro_override" package_defined_macro: "package_macro_override" """ MY_MACRO_SQL = """ {% macro my_macro() -%} {{ test }} {%- endmacro %} """ class TestUnitTestingMacroOverrides: @pytest.fixture(scope="class") def packages(self): return { "packages": [ { "package": "dbt-labs/dbt_utils", "version": "1.1.1", }, ] } @pytest.fixture(scope="class") def models(self): return { "my_model_with_macros.sql": my_model_with_macros, "test_my_model_with_macros.yml": test_my_model_with_macros, } @pytest.fixture(scope="class") def macros(self): return {"my_macro.sql": MY_MACRO_SQL} def test_macro_overrides(self, project): run_dbt(["deps"]) # Select by model name results = run_dbt(["test", "--select", "my_model_with_macros"], expect_pass=True) assert len(results) == 1 ================================================ FILE: tests/functional/unit_testing/test_ut_resource_types.py ================================================ import pytest from fixtures import ( # noqa: F401 my_model_a_sql, my_model_b_sql, my_model_sql, test_my_model_a_yml, test_my_model_pass_yml, ) from dbt.tests.util import run_dbt EXPECTED_MODELS = [ "test.my_model", "test.my_model_a", "test.my_model_b", ] EXPECTED_DATA_TESTS = [ "test.not_null_my_model_a_a", "test.not_null_my_model_a_id", ] EXPECTED_UNIT_TESTS = [ "unit_test:test.test_my_model", ] class TestUnitTestResourceTypes: @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_model_sql, "my_model_a.sql": my_model_a_sql, "my_model_b.sql": my_model_b_sql, "test_my_model.yml": test_my_model_pass_yml, "test_my_model_a.yml": test_my_model_a_yml, } def test_unit_test_list(self, project): results = run_dbt(["run"]) # unit tests results = run_dbt(["list", "--resource-type", "unit_test"]) assert sorted(results) == EXPECTED_UNIT_TESTS results = run_dbt(["list", "--exclude-resource-types", "model", "test"]) assert sorted(results) == EXPECTED_UNIT_TESTS results = run_dbt(["test", "--resource-type", "unit_test"]) assert len(results) == len(EXPECTED_UNIT_TESTS) results = run_dbt(["test", "--exclude-resource-types", "model", "test"]) assert len(results) == len(EXPECTED_UNIT_TESTS) # data tests results = run_dbt(["list", "--resource-type", "test"]) assert sorted(results) == EXPECTED_DATA_TESTS results = run_dbt(["list", "--exclude-resource-types", "unit_test", "model"]) assert sorted(results) == EXPECTED_DATA_TESTS results = run_dbt(["test", "--resource-type", "test"]) assert len(results) == len(EXPECTED_DATA_TESTS) results = run_dbt(["test", "--exclude-resource-types", "unit_test", "model"]) assert len(results) == len(EXPECTED_DATA_TESTS) results = run_dbt(["build", "--resource-type", "test"]) assert len(results) == len(EXPECTED_DATA_TESTS) results = run_dbt(["build", "--exclude-resource-types", "unit_test", "model"]) assert len(results) == len(EXPECTED_DATA_TESTS) # models results = run_dbt(["list", "--resource-type", "model"]) assert sorted(results) == EXPECTED_MODELS results = run_dbt(["list", "--exclude-resource-type", "unit_test", "test"]) assert sorted(results) == EXPECTED_MODELS results = run_dbt(["test", "--resource-type", "model"]) assert len(results) == 0 results = run_dbt(["test", "--exclude-resource-types", "unit_test", "test"]) assert len(results) == 0 results = run_dbt(["build", "--resource-type", "model"]) assert len(results) == len(EXPECTED_MODELS) results = run_dbt(["build", "--exclude-resource-type", "unit_test", "test"]) assert len(results) == len(EXPECTED_MODELS) ================================================ FILE: tests/functional/unit_testing/test_ut_snapshot_dependency.py ================================================ import pytest from dbt.contracts.results import RunStatus, TestStatus from dbt.tests.util import run_dbt raw_customers_csv = """id,first_name,last_name,email,gender,ip_address,updated_at 1,'Judith','Kennedy','(not provided)','Female','54.60.24.128','2015-12-24 12:19:28' 2,'Arthur','Kelly','(not provided)','Male','62.56.24.215','2015-10-28 16:22:15' 3,'Rachel','Moreno','rmoreno2@msu.edu','Female','31.222.249.23','2016-04-05 02:05:30' 4,'Ralph','Turner','rturner3@hp.com','Male','157.83.76.114','2016-08-08 00:06:51' 5,'Laura','Gonzales','lgonzales4@howstuffworks.com','Female','30.54.105.168','2016-09-01 08:25:38' 6,'Katherine','Lopez','klopez5@yahoo.co.jp','Female','169.138.46.89','2016-08-30 18:52:11' 7,'Jeremy','Hamilton','jhamilton6@mozilla.org','Male','231.189.13.133','2016-07-17 02:09:46' """ top_level_domains_csv = """id,domain 3,'msu.edu' 4,'hp.com' 5,'howstuffworks.com' 6,'yahoo.co.jp' 7,'mozilla.org' """ snapshots_users__snapshot_sql = """ {% snapshot snapshot_users %} {{ config( target_database=var('target_database', database), target_schema=schema, unique_key='id || ' ~ "'-'" ~ ' || first_name', strategy='check', check_cols=['email'], ) }} select *, split_part(email, '@', 2) as domain from {{target.database}}.{{schema}}.raw_customers {% endsnapshot %} """ unit_test_yml = """ sources: - name: seed_sources schema: "{{ target.schema }}" tables: - name: top_level_domains columns: - name: id - name: domain unit_tests: - name: test_is_valid_email_address model: customers given: - input: ref('snapshot_users') rows: - {id: 1, email: cool@example.com, domain: example.com} - {id: 2, email: cool@unknown.com, domain: unknown.com} - {id: 3, email: badgmail.com, domain: gmailcom} - {id: 4, email: missingdot@gmailcom, domain: gmailcom} - input: source('seed_sources', 'top_level_domains') rows: - {domain: example.com} - {domain: gmail.com} expect: rows: - {id: 1, is_valid_email_address: true} - {id: 2, is_valid_email_address: false} - {id: 3, is_valid_email_address: false} - {id: 4, is_valid_email_address: false} - name: fail_is_valid_email_address model: customers given: - input: ref('snapshot_users') rows: - {id: 1, email: cool@example.com, domain: example.com} - input: source('seed_sources', 'top_level_domains') rows: - {domain: example.com} - {domain: gmail.com} expect: rows: - {id: 1, is_valid_email_address: false} """ customers_sql = """ with snapshot_users as ( select * from {{ ref('snapshot_users') }} ), top_level_domains as ( select * from {{ source('seed_sources', 'top_level_domains') }} ), matched_values as ( select snapshot_users.*, case when exists ( select 1 from top_level_domains where top_level_domains.domain = snapshot_users.domain ) then true else false end as is_valid_email_address from snapshot_users ) select * from matched_values """ class TestUnitTestSnapshotDependency: @pytest.fixture(scope="class") def seeds(self): return { "raw_customers.csv": raw_customers_csv, "top_level_domains.csv": top_level_domains_csv, } @pytest.fixture(scope="class") def models(self): return { "customers.sql": customers_sql, "unit_tests.yml": unit_test_yml, } @pytest.fixture(scope="class") def snapshots(self): return { "snapshot_users.sql": snapshots_users__snapshot_sql, } def test_snapshot_dependency(self, project): seed_results = run_dbt(["seed"]) len(seed_results) == 2 snapshot_results = run_dbt(["snapshot"]) len(snapshot_results) == 1 model_results = run_dbt(["run"]) len(model_results) == 1 # test passing unit test results = run_dbt(["test", "--select", "test_name:test_is_valid_email_address"]) assert len(results) == 1 # test failing unit test results = run_dbt( ["test", "--select", "test_name:fail_is_valid_email_address"], expect_pass=False ) assert len(results) == 1 assert results[0].status == TestStatus.Fail # test all with build results = run_dbt(["build"], expect_pass=False) for result in results: if result.node.unique_id == "unit_test.test.customers.fail_is_valid_email_address": # This will always fail, regarless of order executed assert result.status == TestStatus.Fail elif result.node.unique_id == "unit_test.test.customers.test_is_valid_email_address": # there's no guarantee that the order of the results will be the same. If the # failed test runs first this one gets skipped. If this runs first it passes. assert result.status in [TestStatus.Pass, TestStatus.Skipped] elif result.node.unique_id == "model.test.customers": # This is always skipped because one test always fails assert result.status == RunStatus.Skipped assert len(results) == 6 ================================================ FILE: tests/functional/unit_testing/test_ut_sources.py ================================================ from copy import deepcopy import pytest from dbt.contracts.results import RunStatus, TestStatus from dbt.tests.util import run_dbt, write_file raw_customers_csv = """id,first_name,last_name,email 1,Michael,Perez,mperez0@chronoengine.com 2,Shawn,Mccoy,smccoy1@reddit.com 3,Kathleen,Payne,kpayne2@cargocollective.com 4,Jimmy,Cooper,jcooper3@cargocollective.com 5,Katherine,Rice,krice4@typepad.com 6,Sarah,Ryan,sryan5@gnu.org 7,Martin,Mcdonald,mmcdonald6@opera.com 8,Frank,Robinson,frobinson7@wunderground.com 9,Jennifer,Franklin,jfranklin8@mail.ru 10,Henry,Welch,hwelch9@list-manage.com """ schema_sources_yml = """ sources: - name: seed_sources schema: "{{ target.schema }}" tables: - name: raw_customers columns: - name: id data_tests: - not_null: severity: "{{ 'error' if target.name == 'prod' else 'warn' }}" - unique - name: first_name - name: last_name - name: email unit_tests: - name: test_customers model: customers given: - input: source('seed_sources', 'raw_customers') rows: - {id: 1, first_name: Emily} expect: rows: - {id: 1, first_name: Emily} """ customers_sql = """ select * from {{ source('seed_sources', 'raw_customers') }} """ failing_test_schema_yml = """ - name: fail_test_customers model: customers given: - input: source('seed_sources', 'raw_customers') rows: - {id: 1, first_name: Emily} expect: rows: - {id: 1, first_name: Joan} """ schema_duplicate_source_names_yml = """ sources: - name: seed_sources schema: "{{ target.schema }}" tables: - name: raw_customers - name: seed_sources_2 schema: "{{ target.schema }}_other" tables: - name: raw_customers unit_tests: - name: test_customers model: customers_duplicate_source_names given: - input: source('seed_sources', 'raw_customers') rows: - {id: 1, first_name: Emily} - input: source('seed_sources_2', 'raw_customers') rows: - {id: 2, first_name: Michelle} expect: rows: - {id: 1, first_name: Emily} - {id: 2, first_name: Michelle} """ customers_duplicate_source_names_sql = """ select * from {{ source('seed_sources', 'raw_customers') }} union all select * from {{ source('seed_sources_2', 'raw_customers') }} """ class TestUnitTestSourceInput: @pytest.fixture(scope="class") def seeds(self): return { "raw_customers.csv": raw_customers_csv, } @pytest.fixture(scope="class") def models(self): return { "customers.sql": customers_sql, "sources.yml": schema_sources_yml, } def test_source_input(self, project): results = run_dbt(["seed"]) results = run_dbt(["run"]) len(results) == 1 results = run_dbt(["test", "--select", "test_type:unit"]) assert len(results) == 1 results = run_dbt(["build"]) assert len(results) == 5 result_unique_ids = [result.node.unique_id for result in results] assert len(result_unique_ids) == 5 assert "unit_test.test.customers.test_customers" in result_unique_ids # write failing unit test write_file( schema_sources_yml + failing_test_schema_yml, project.project_root, "models", "sources.yml", ) results = run_dbt(["build"], expect_pass=False) for result in results: if result.node.unique_id == "model.test.customers": assert result.status == RunStatus.Skipped elif result.node.unique_id == "unit_test.test.customers.fail_test_customers": assert result.status == TestStatus.Fail assert len(results) == 6 class TestUnitTestSourceInputSameNames: @pytest.fixture(scope="class") def other_schema(self, unique_schema): return unique_schema + "_other" @pytest.fixture(scope="class") def profiles_config_update(self, dbt_profile_target, unique_schema, other_schema): outputs = {"default": dbt_profile_target, "otherschema": deepcopy(dbt_profile_target)} outputs["default"]["schema"] = unique_schema outputs["otherschema"]["schema"] = other_schema return {"test": {"outputs": outputs, "target": "default"}} @pytest.fixture(scope="class") def seeds(self): return { "raw_customers.csv": raw_customers_csv, } @pytest.fixture(scope="class") def models(self): return { "customers_duplicate_source_names.sql": customers_duplicate_source_names_sql, "sources.yml": schema_duplicate_source_names_yml, } def test_source_input_same_names(self, project, other_schema): results = run_dbt(["seed"]) project.create_test_schema(schema_name=other_schema) results = run_dbt(["seed", "--target", "otherschema"]) results = run_dbt(["test", "--select", "test_type:unit"]) assert len(results) == 1 ================================================ FILE: tests/functional/unit_testing/test_ut_variables.py ================================================ import pytest from dbt.tests.util import run_dbt, run_dbt_and_capture dbt_project_yml = """ vars: columns_list_one: - column_a - column_b columns_list_two: - column_c """ my_model_one_variable_sql = """ {{ config(materialized='table') }} -- {{ get_columns(include=var('columns_list_one'))}} select 1 as id """ my_model_two_variables_sql = """ {{ config(materialized='table') }} -- {{ get_columns(include=var('columns_list_one') + var('columns_list_two'))}} select 1 as id """ my_macro_sql = """ {%- macro get_columns(include=[]) -%} {%- for col in include -%} {{ col }}{% if not loop.last %}, {% endif %} {%- endfor -%} {%- endmacro -%} """ my_unit_test_yml = """ unit_tests: - name: my_unit_test model: my_model given: [] expect: rows: - {id: 1} """ class TestUnitTestOneVariables: @pytest.fixture(scope="class") def project_config_update(self): return dbt_project_yml @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_model_one_variable_sql, "my_unit_test.yml": my_unit_test_yml, } @pytest.fixture(scope="class") def macros(self): return {"my_macros.sql": my_macro_sql} def test_one_variable_as_input_to_macro(self, project): run_dbt_and_capture(["test"], expect_pass=True) class TestUnitTestTwoVariables: @pytest.fixture(scope="class") def project_config_update(self): return dbt_project_yml @pytest.fixture(scope="class") def models(self): return { "my_model.sql": my_model_two_variables_sql, "my_unit_test.yml": my_unit_test_yml, } @pytest.fixture(scope="class") def macros(self): return {"my_macros.sql": my_macro_sql} def test_two_variables_as_input_to_macro(self, project): # Verify model works fine outside of unit testing results = run_dbt(["run"]) assert len(results) == 1 run_dbt(["test"]) ================================================ FILE: tests/functional/unit_testing/test_ut_versions.py ================================================ import pytest from dbt.exceptions import ParsingError, YamlParseDictError from dbt.tests.util import get_unique_ids_in_results, run_dbt, write_file from tests.functional.unit_testing.fixtures import ( my_model_a_sql, my_model_b_sql, my_model_sql, my_model_v1_sql, my_model_v2_sql, my_model_v3_sql, my_model_version_ref_sql, my_model_versioned_no_2_yml, my_model_versioned_yml, test_my_model_all_versions_yml, test_my_model_exclude_versions_yml, test_my_model_include_exclude_versions_yml, test_my_model_include_unversioned_yml, test_my_model_include_versions_yml, test_my_model_version_ref_yml, ) # test with no version specified, then add an exclude version, then switch # to include version and make sure the right unit tests are generated for each class TestVersions: @pytest.fixture(scope="class") def models(self): return { "my_model_a.sql": my_model_a_sql, "my_model_b.sql": my_model_b_sql, "my_model_v1.sql": my_model_v1_sql, "my_model_v2.sql": my_model_v2_sql, "my_model_v3.sql": my_model_v3_sql, "schema.yml": my_model_versioned_yml, "unit_tests.yml": test_my_model_all_versions_yml, } def test_versions(self, project): results = run_dbt(["run"]) assert len(results) == 5 # "my_model" has three versions: 1, 2, 3 # There is a single unit_test which doesn't specify a version, # so it should run for all versions. results = run_dbt(["test"]) assert len(results) == 3 unique_ids = get_unique_ids_in_results(results) expected_ids = [ "unit_test.test.my_model.test_my_model_v1", "unit_test.test.my_model.test_my_model_v2", "unit_test.test.my_model.test_my_model_v3", ] assert sorted(expected_ids) == sorted(unique_ids) # Select tests for a single versioned model results = run_dbt(["test", "--select", "my_model.v2"]) assert len(results) == 1 unique_ids = get_unique_ids_in_results(results) assert unique_ids == ["unit_test.test.my_model.test_my_model_v2"] # select tests for all my_models results = run_dbt(["test", "--select", "my_model"]) assert len(results) == 3 unique_ids = get_unique_ids_in_results(results) assert sorted(expected_ids) == sorted(unique_ids) # with an exclude version specified, should create a separate unit test # for each version except the excluded version (v2) write_file( test_my_model_exclude_versions_yml, project.project_root, "models", "unit_tests.yml" ) results = run_dbt(["test"]) assert len(results) == 2 unique_ids = get_unique_ids_in_results(results) # v2 model should be excluded expected_ids = [ "unit_test.test.my_model.test_my_model_v1", "unit_test.test.my_model.test_my_model_v3", ] assert sorted(expected_ids) == sorted(unique_ids) # test with an include version specified, should create a single unit test for # only the version specified (2) write_file( test_my_model_include_versions_yml, project.project_root, "models", "unit_tests.yml" ) results = run_dbt(["test"]) assert len(results) == 1 unique_ids = get_unique_ids_in_results(results) # v2 model should be only one included expected_ids = [ "unit_test.test.my_model.test_my_model_v2", ] assert sorted(expected_ids) == sorted(unique_ids) # Change to remove version 2 of model and get an error write_file(my_model_versioned_no_2_yml, project.project_root, "models", "schema.yml") with pytest.raises(ParsingError): run_dbt(["test"]) # test with an include and exclude version specified, should raise an error class TestIncludeExcludeSpecified: @pytest.fixture(scope="class") def models(self): return { "my_model_a.sql": my_model_a_sql, "my_model_b.sql": my_model_b_sql, "my_model_v1.sql": my_model_v1_sql, "my_model_v2.sql": my_model_v2_sql, "my_model_v3.sql": my_model_v3_sql, "schema.yml": my_model_versioned_yml, "unit_tests.yml": test_my_model_include_exclude_versions_yml, } def test_include_exclude_specified(self, project): with pytest.raises(YamlParseDictError): run_dbt(["parse"]) # test with an include for an unversioned model, should error class TestIncludeUnversioned: @pytest.fixture(scope="class") def models(self): return { "my_model_a.sql": my_model_a_sql, "my_model_b.sql": my_model_b_sql, "my_model.sql": my_model_sql, "unit_tests.yml": test_my_model_include_unversioned_yml, } def test_include_unversioned(self, project): with pytest.raises(ParsingError): run_dbt(["parse"]) # test specifying the fixture version with {{ ref(name, version) }} class TestVersionedFixture: @pytest.fixture(scope="class") def models(self): return { "my_model_a.sql": my_model_a_sql, "my_model_b.sql": my_model_b_sql, "my_model_v1.sql": my_model_v1_sql, "my_model_v2.sql": my_model_v2_sql, "my_model_v3.sql": my_model_v3_sql, "my_model_version_ref.sql": my_model_version_ref_sql, "schema.yml": my_model_versioned_yml, "unit_tests.yml": test_my_model_version_ref_yml, } def test_versioned_fixture(self, project): results = run_dbt(["run"]) assert len(results) == 6 results = run_dbt(["test"]) assert len(results) == 1 unique_ids = get_unique_ids_in_results(results) # v2 model should be only one included expected_ids = ["unit_test.test.my_model_version_ref.test_my_model_version_ref"] assert expected_ids == unique_ids ================================================ FILE: tests/functional/utils.py ================================================ import os from contextlib import contextmanager from datetime import datetime from pathlib import Path from typing import Optional @contextmanager def up_one(return_path: Optional[Path] = None): current_path = Path.cwd() os.chdir("../") try: yield finally: os.chdir(return_path or current_path) def is_aware(dt: datetime) -> bool: return dt.tzinfo is not None and dt.tzinfo.utcoffset(dt) is not None ================================================ FILE: tests/unit/README.md ================================================ # Unit test README ### The Why We need to ensure that we can go from objects to dictionaries and back without any changes. If some property or property value of an object gets dropped, added, or modified while transitioning between its different possible representations, that is problematic. ### The How The easiest way to ensure things don't get droped, added, or modified is by starting with an object, dictifying it, moving back to an object, and then asserting that everything is equivalent. There are many potential edge cases though: what about optional fields, what about lists of things, and etc. To address this we use hypothesis, which will build multiple versions of the object we're interested in testing, and run the different generated versions of the object through the test. This gives us confidence that for any allowable configuration of an object, state is not changed when moving back and forth betweeen the python object version and the seralized version. ### The What - We test concrete classes in the codebase and do not test abstract classes as they are implementation details. [reference](https://enterprisecraftsmanship.com/posts/how-to-unit-test-an-abstract-class/) ================================================ FILE: tests/unit/__init__.py ================================================ # Unit testing directory ================================================ FILE: tests/unit/artifacts/test_base_resource.py ================================================ from dataclasses import dataclass import pytest from dbt.artifacts.resources.base import BaseResource from dbt.artifacts.resources.types import NodeType @dataclass class BaseResourceWithDefaultField(BaseResource): field_with_default: bool = True class TestMinorSchemaChange: @pytest.fixture def base_resource(self): return BaseResource( name="test", resource_type=NodeType.Model, package_name="test_package", path="test_path", original_file_path="test_original_file_path", unique_id="test_unique_id", ) @pytest.fixture def base_resource_new_default_field(self): return BaseResourceWithDefaultField( name="test", resource_type=NodeType.Model, package_name="test_package", path="test_path", original_file_path="test_original_file_path", unique_id="test_unique_id", field_with_default=False, ) def test_serializing_new_default_field_is_backward_compatabile( self, base_resource_new_default_field ): # old code (using old class) can create an instance of itself given new data (new class) BaseResource.from_dict(base_resource_new_default_field.to_dict()) def test_serializing_new_default_field_is_forward_compatible(self, base_resource): # new code (using new class) can create an instance of itself given old data (old class) BaseResourceWithDefaultField.from_dict(base_resource.to_dict()) def test_serializing_removed_default_field_is_backward_compatabile(self, base_resource): # old code (using old class with default field) can create an instance of itself given new data (class w/o default field) old_resource = BaseResourceWithDefaultField.from_dict(base_resource.to_dict()) # set to the default value when not provided in data assert old_resource.field_with_default is True def test_serializing_removed_default_field_is_forward_compatible( self, base_resource_new_default_field ): # new code (using class without default field) can create an instance of itself given old data (class with old field) BaseResource.from_dict(base_resource_new_default_field.to_dict()) ================================================ FILE: tests/unit/artifacts/test_run_execution_result.py ================================================ from dateutil.tz import tzutc from hypothesis import given from hypothesis.strategies import ( builds, composite, dictionaries, floats, lists, sampled_from, text, ) from dbt.contracts.results import RunExecutionResult, RunResult, RunStatus, TimingInfo from tests.unit.fixtures import model_node @composite def run_result_strategy(draw): node = model_node() status = draw(sampled_from(list(RunStatus))) message = draw(text(min_size=0, max_size=30) | sampled_from([None])) result = RunResult.from_node(node=node, status=status, message=message) result.execution_time = draw(floats(min_value=0.0, max_value=30.0)) result.timing = draw(lists(builds(TimingInfo), max_size=2)) return result @given( args=dictionaries(text(min_size=1, max_size=10), text(min_size=1, max_size=10)), elapsed_time=floats(min_value=0.0, max_value=30.0), results=lists(run_result_strategy(), min_size=1, max_size=3), ) def test_run_execution_result_serialization(args, elapsed_time, results): obj = RunExecutionResult(results=results, elapsed_time=elapsed_time, args=args) obj_from_dict = RunExecutionResult.from_dict(obj.to_dict()) assert obj_from_dict.args == obj.args assert len(obj_from_dict.results) == len(obj.results) assert obj.generated_at.tzinfo is None assert obj_from_dict.generated_at.tzinfo == tzutc() assert obj_from_dict.generated_at.replace(tzinfo=None) == obj.generated_at for original, deserialized in zip(obj.results, obj_from_dict.results): assert original.node.created_at == deserialized.node.created_at ================================================ FILE: tests/unit/cli/test_flags.py ================================================ from pathlib import Path from typing import List, Optional import click import pytest from dbt.cli.exceptions import DbtUsageException from dbt.cli.flags import Flags from dbt.cli.main import cli from dbt.cli.types import Command from dbt.contracts.project import ProjectFlags from dbt.tests.util import rm_file, write_file from dbt_common.exceptions import DbtInternalError from dbt_common.helper_types import WarnErrorOptionsV2 class TestFlags: def make_dbt_context( self, context_name: str, args: List[str], parent: Optional[click.Context] = None ) -> click.Context: ctx = cli.make_context(context_name, args.copy(), parent) return ctx @pytest.fixture(scope="class") def run_context(self) -> click.Context: return self.make_dbt_context("run", ["run"]) @pytest.fixture def project_flags(self) -> ProjectFlags: return ProjectFlags() def test_cli_args_unmodified(self): args = ["--target", "my_target"] args_before = args.copy() self.make_dbt_context("context", args) assert args == args_before def test_which(self, run_context): flags = Flags(run_context) assert flags.WHICH == "run" @pytest.mark.parametrize("param", cli.params) def test_cli_group_flags_from_params(self, run_context, param): flags = Flags(run_context) if "DEPRECATED_" in param.name.upper(): assert not hasattr(flags, param.name.upper()) return if param.name.upper() in ("VERSION", "LOG_PATH"): return assert hasattr(flags, param.name.upper()) assert getattr(flags, param.name.upper()) == run_context.params[param.name.lower()] def test_log_path_default(self, run_context): flags = Flags(run_context) assert hasattr(flags, "LOG_PATH") assert getattr(flags, "LOG_PATH") == Path("logs") def test_log_file_max_size_default(self, run_context): flags = Flags(run_context) assert hasattr(flags, "LOG_FILE_MAX_BYTES") assert getattr(flags, "LOG_FILE_MAX_BYTES") == 10 * 1024 * 1024 @pytest.mark.parametrize( "set_stats_param,do_not_track,expected_anonymous_usage_stats", [ # set_stats_param = default, DNT = True, expected = False ("default", "1", False), ("default", "t", False), ("default", "true", False), ("default", "y", False), ("default", "yes", False), # set_stats_param = default, DNT = false, expected = True ("default", "false", True), ("default", "anything", True), # set_stats_param = True, DNT = True, expected = False (True, "1", False), (True, "t", False), (True, "true", False), (True, "y", False), (True, "yes", False), # set_stats_param = True, DNT = false, expected = True (True, "false", True), (True, "anything", True), (True, "2", True), # set_stats_param = False, DNT = True, expected = False (False, "1", False), (False, "t", False), (False, "true", False), (False, "y", False), (False, "yes", False), # set_stats_param = False, DNT = False, expected = False (False, "false", False), (False, "anything", False), (False, "2", False), ], ) def test_anonymous_usage_state( self, monkeypatch, run_context, set_stats_param, do_not_track, expected_anonymous_usage_stats, ): monkeypatch.setenv("DO_NOT_TRACK", do_not_track) if set_stats_param != "default": run_context.params["send_anonymous_usage_stats"] = set_stats_param flags = Flags(run_context) assert flags.SEND_ANONYMOUS_USAGE_STATS == expected_anonymous_usage_stats def test_resource_types(self, monkeypatch): monkeypatch.setenv("DBT_RESOURCE_TYPES", "model") build_context = self.make_dbt_context("build", ["build"]) build_context.params["resource_types"] = ("unit_test",) flags = Flags(build_context) assert flags.resource_types == ("unit_test",) def test_empty_project_flags_uses_default(self, run_context, project_flags): flags = Flags(run_context, project_flags) assert flags.USE_COLORS == run_context.params["use_colors"] def test_none_project_flags_uses_default(self, run_context): flags = Flags(run_context, None) assert flags.USE_COLORS == run_context.params["use_colors"] def test_prefer_project_flags_to_default(self, run_context, project_flags): project_flags.use_colors = False # ensure default value is not the same as user config assert run_context.params["use_colors"] is not project_flags.use_colors flags = Flags(run_context, project_flags) assert flags.USE_COLORS == project_flags.use_colors def test_prefer_param_value_to_project_flags(self): project_flags = ProjectFlags(use_colors=False) context = self.make_dbt_context("run", ["--use-colors", "True", "run"]) flags = Flags(context, project_flags) assert flags.USE_COLORS def test_prefer_env_to_project_flags(self, monkeypatch, project_flags): project_flags.use_colors = False monkeypatch.setenv("DBT_USE_COLORS", "True") context = self.make_dbt_context("run", ["run"]) flags = Flags(context, project_flags) assert flags.USE_COLORS def test_mutually_exclusive_options_passed_separately(self): """Assert options that are mutually exclusive can be passed separately without error""" warn_error_context = self.make_dbt_context("run", ["--warn-error", "run"]) flags = Flags(warn_error_context) assert flags.WARN_ERROR warn_error_options_context = self.make_dbt_context( "run", ["--warn-error-options", '{"error": "all"}', "run"] ) flags = Flags(warn_error_options_context) assert flags.WARN_ERROR_OPTIONS == WarnErrorOptionsV2(error="all") def test_mutually_exclusive_options_from_cli(self): context = self.make_dbt_context( "run", ["--warn-error", "--warn-error-options", '{"error": "all"}', "run"] ) with pytest.raises(DbtUsageException): Flags(context) @pytest.mark.parametrize("warn_error", [True, False]) def test_mutually_exclusive_options_from_project_flags(self, warn_error, project_flags): project_flags.warn_error = warn_error context = self.make_dbt_context("run", ["--warn-error-options", '{"error": "all"}', "run"]) with pytest.raises(DbtUsageException): Flags(context, project_flags) @pytest.mark.parametrize("warn_error", ["True", "False"]) def test_mutually_exclusive_options_from_envvar(self, warn_error, monkeypatch): monkeypatch.setenv("DBT_WARN_ERROR", warn_error) monkeypatch.setenv("DBT_WARN_ERROR_OPTIONS", '{"error":"all"}') context = self.make_dbt_context("run", ["run"]) with pytest.raises(DbtUsageException): Flags(context) @pytest.mark.parametrize("warn_error", [True, False]) def test_mutually_exclusive_options_from_cli_and_project_flags( self, warn_error, project_flags ): project_flags.warn_error = warn_error context = self.make_dbt_context("run", ["--warn-error-options", '{"error": "all"}', "run"]) with pytest.raises(DbtUsageException): Flags(context, project_flags) @pytest.mark.parametrize("warn_error", ["True", "False"]) def test_mutually_exclusive_options_from_cli_and_envvar(self, warn_error, monkeypatch): monkeypatch.setenv("DBT_WARN_ERROR", warn_error) context = self.make_dbt_context("run", ["--warn-error-options", '{"error": "all"}', "run"]) with pytest.raises(DbtUsageException): Flags(context) @pytest.mark.parametrize("warn_error", ["True", "False"]) def test_mutually_exclusive_options_from_project_flags_and_envvar( self, project_flags, warn_error, monkeypatch ): project_flags.warn_error = warn_error monkeypatch.setenv("DBT_WARN_ERROR_OPTIONS", '{"error": "all"}') context = self.make_dbt_context("run", ["run"]) with pytest.raises(DbtUsageException): Flags(context, project_flags) @pytest.mark.parametrize( "cli_colors,cli_colors_file,flag_colors,flag_colors_file", [ (None, None, True, True), (True, None, True, True), (None, True, True, True), (False, None, False, False), (None, False, True, False), (True, True, True, True), (False, False, False, False), (True, False, True, False), (False, True, False, True), ], ) def test_no_color_interaction( self, cli_colors, cli_colors_file, flag_colors, flag_colors_file ): cli_params = [] if cli_colors is not None: cli_params.append("--use-colors" if cli_colors else "--no-use-colors") if cli_colors_file is not None: cli_params.append("--use-colors-file" if cli_colors_file else "--no-use-colors-file") cli_params.append("run") context = self.make_dbt_context("run", cli_params) flags = Flags(context, None) assert flags.USE_COLORS == flag_colors assert flags.USE_COLORS_FILE == flag_colors_file @pytest.mark.parametrize( "cli_log_level,cli_log_level_file,flag_log_level,flag_log_level_file", [ (None, None, "info", "debug"), ("error", None, "error", "error"), # explicit level overrides file level... ("info", None, "info", "info"), # ...but file level doesn't change console level ( "debug", "warn", "debug", "warn", ), # still, two separate explicit levels are applied independently ], ) def test_log_level_interaction( self, cli_log_level, cli_log_level_file, flag_log_level, flag_log_level_file ): cli_params = [] if cli_log_level is not None: cli_params.append("--log-level") cli_params.append(cli_log_level) if cli_log_level_file is not None: cli_params.append("--log-level-file") cli_params.append(cli_log_level_file) cli_params.append("run") context = self.make_dbt_context("run", cli_params) flags = Flags(context, None) assert flags.LOG_LEVEL == flag_log_level assert flags.LOG_LEVEL_FILE == flag_log_level_file @pytest.mark.parametrize( "cli_log_format,cli_log_format_file,flag_log_format,flag_log_format_file", [ (None, None, "default", "debug"), ("json", None, "json", "json"), # explicit format overrides file format... (None, "json", "default", "json"), # ...but file format doesn't change console format ( "debug", "text", "debug", "text", ), # still, two separate explicit formats are applied independently ], ) def test_log_format_interaction( self, cli_log_format, cli_log_format_file, flag_log_format, flag_log_format_file ): cli_params = [] if cli_log_format is not None: cli_params.append("--log-format") cli_params.append(cli_log_format) if cli_log_format_file is not None: cli_params.append("--log-format-file") cli_params.append(cli_log_format_file) cli_params.append("run") context = self.make_dbt_context("run", cli_params) flags = Flags(context, None) assert flags.LOG_FORMAT == flag_log_format assert flags.LOG_FORMAT_FILE == flag_log_format_file def test_log_settings_from_config(self): """Test that values set in ProjectFlags for log settings will set flags as expected""" context = self.make_dbt_context("run", ["run"]) config = ProjectFlags(log_format="json", log_level="warn", use_colors=False) flags = Flags(context, config) assert flags.LOG_FORMAT == "json" assert flags.LOG_FORMAT_FILE == "json" assert flags.LOG_LEVEL == "warn" assert flags.LOG_LEVEL_FILE == "warn" assert flags.USE_COLORS is False assert flags.USE_COLORS_FILE is False def test_log_file_settings_from_config(self): """Test that values set in ProjectFlags for log *file* settings will set flags as expected, leaving the console logging flags with their default values""" context = self.make_dbt_context("run", ["run"]) config = ProjectFlags(log_format_file="json", log_level_file="warn", use_colors_file=False) flags = Flags(context, config) assert flags.LOG_FORMAT == "default" assert flags.LOG_FORMAT_FILE == "json" assert flags.LOG_LEVEL == "info" assert flags.LOG_LEVEL_FILE == "warn" assert flags.USE_COLORS is True assert flags.USE_COLORS_FILE is False def test_duplicate_flags_raises_error(self): parent_context = self.make_dbt_context("parent", ["--version-check"]) context = self.make_dbt_context("child", ["--version-check"], parent_context) with pytest.raises(DbtUsageException): Flags(context) def test_global_flag_at_child_context(self): parent_context_a = self.make_dbt_context("parent_context_a", ["--no-use-colors"]) child_context_a = self.make_dbt_context("child_context_a", ["run"], parent_context_a) flags_a = Flags(child_context_a) parent_context_b = self.make_dbt_context("parent_context_b", ["run"]) child_context_b = self.make_dbt_context( "child_context_b", ["--no-use-colors"], parent_context_b ) flags_b = Flags(child_context_b) assert flags_a.USE_COLORS == flags_b.USE_COLORS def test_global_flag_with_env_var(self, monkeypatch): # The environment variable is used for whichever parent or child # does not have a cli command. # Test that "child" global flag overrides env var monkeypatch.setenv("DBT_QUIET", "0") parent_context = self.make_dbt_context("parent", ["--no-use-colors"]) child_context = self.make_dbt_context("child", ["--quiet"], parent_context) flags = Flags(child_context) assert flags.QUIET is True # Test that "parent" global flag overrides env var parent_context = self.make_dbt_context("parent", ["--quiet"]) child_context = self.make_dbt_context("child", ["--no-use-colors"], parent_context) flags = Flags(child_context) assert flags.QUIET is True def test_set_project_only_flags(self, project_flags, run_context): flags = Flags(run_context, project_flags) for project_only_flag, project_only_flag_value in project_flags.project_only_flags.items(): assert getattr(flags, project_only_flag) == project_only_flag_value # sanity check: ensure project_only_flag is not part of the click context assert project_only_flag not in run_context.params def _create_flags_from_dict(self, cmd, d): write_file("", "profiles.yml") result = Flags.from_dict(cmd, d) assert result.which is cmd.value rm_file("profiles.yml") return result def test_from_dict__run(self): args_dict = { "print": False, "select": ["model_one", "model_two"], } result = self._create_flags_from_dict(Command.RUN, args_dict) assert "model_one" in result.select[0] assert "model_two" in result.select[1] def test_from_dict__build(self): args_dict = { "print": True, "state": "some/path", "defer_state": None, } result = self._create_flags_from_dict(Command.BUILD, args_dict) assert result.print is True assert "some/path" in str(result.state) assert result.defer_state is None def test_from_dict__seed(self): args_dict = {"use_colors": False, "exclude": ["model_three"]} result = self._create_flags_from_dict(Command.SEED, args_dict) assert result.use_colors is False assert "model_three" in result.exclude[0] def test_from_dict__which_fails(self): args_dict = {"which": "some bad command"} with pytest.raises(DbtInternalError, match=r"does not match value of which"): self._create_flags_from_dict(Command.RUN, args_dict) def test_from_dict_0_value(self): args_dict = {"log_file_max_bytes": 0} flags = Flags.from_dict(Command.RUN, args_dict) assert flags.LOG_FILE_MAX_BYTES == 0 def test_project_flag_defaults(): flags = ProjectFlags() # From # 9183: Let's add a unit test that ensures that: # every attribute of ProjectFlags that has a corresponding click option # in params.py should be set to None by default (except for anon user # tracking). Going forward, flags can have non-None defaults if they # do not have a corresponding CLI option/env var. These will be used # to control backwards incompatible interface or behaviour changes. # List of all flags except send_anonymous_usage_stats project_flags = [ "cache_selected_only", "debug", "fail_fast", "indirect_selection", "log_format", "log_format_file", "log_level", "log_level_file", "partial_parse", "populate_cache", "printer_width", "static_parser", "use_colors", "use_colors_file", "use_experimental_parser", "version_check", "warn_error", "warn_error_options", "write_json", ] for flag in project_flags: assert getattr(flags, flag) is None ================================================ FILE: tests/unit/cli/test_main.py ================================================ import click from dbt.cli.flags import command_args from dbt.cli.main import cli from dbt.cli.types import Command class TestCLI: def _all_commands(self, group=cli, result=set()): for command in group.commands.values(): result.add(command) if isinstance(command, click.Group): self._all_commands(command, result) continue return result def test_commands_have_docstrings(self): for command in self._all_commands(): assert command.__doc__ is not None # TODO: This isn't the ideal way to test params as # they will be tested as many times as they are used as decorators. # This is inefficent (obvs) def test_unhidden_params_have_help_texts(self): def run_test(command): for param in command.params: # arguments can't have help text if not isinstance(param, click.Argument) and not param.hidden: assert param.help is not None if type(command) is click.Group: for command in command.commands.values(): run_test(command) run_test(cli) def test_param_names_match_envvars(self): def run_test(command): for param in command.params: # deprecated params are named "deprecated_x" and do not need to have # a parallel name like "DBT_" if isinstance(param.envvar, list): envvar = param.envvar[0] else: envvar = param.envvar if envvar is not None and "deprecated_" not in param.name: assert ("DBT_" + param.name.upper() == envvar) or ( "DBT_ENGINE_" + param.name.upper() == envvar ) if type(command) is click.Group: for command in command.commands.values(): run_test(command) run_test(cli) def test_commands_in_enum_and_dict(self): for command in self._all_commands(cli): if isinstance(command, click.Group): continue cmd = Command.from_str(command.name) command_args(cmd) ================================================ FILE: tests/unit/cli/test_option_types.py ================================================ from datetime import datetime from typing import Union import freezegun import pytest import pytz from click import BadParameter, Option from dbt.cli.option_types import YAML, SampleType from dbt.event_time.sample_window import SampleWindow class TestYAML: @pytest.mark.parametrize( "raw_value,expected_converted_value", [ ("{}", {}), ("{'test_var_key': 'test_var_value'}", {"test_var_key": "test_var_value"}), ], ) def test_yaml_init(self, raw_value, expected_converted_value): converted_value = YAML().convert(raw_value, Option(["--vars"]), None) assert converted_value == expected_converted_value @pytest.mark.parametrize( "invalid_yaml_str", ["{", ""], ) def test_yaml_init_invalid_yaml_str(self, invalid_yaml_str): with pytest.raises(BadParameter) as e: YAML().convert(invalid_yaml_str, Option(["--vars"]), None) assert "--vars" in e.value.format_message() class TestSampleType: @pytest.mark.parametrize( "input,expected_result", [ ( "{'start': '2025-01-24', 'end': '2025-01-27'}", SampleWindow( start=datetime(2025, 1, 24, 0, 0, 0, 0, pytz.UTC), end=datetime(2025, 1, 27, 0, 0, 0, 0, pytz.UTC), ), ), ( "{'tart': '2025-01-24', 'bend': '2025-01-27'}", BadParameter('Field "start" of type datetime is missing in SampleWindow instance'), ), ( "{}", BadParameter('Field "start" of type datetime is missing in SampleWindow instance'), ), ( "cats", BadParameter( "Runtime Error\n Cannot load SAMPLE_WINDOW from 'cats'. Must be of form 'DAYS_INT GRAIN_SIZE'." ), ), ], ) def test_convert(self, input: str, expected_result: Union[SampleWindow, Exception]): try: result = SampleType().convert(input, Option(["--sample"]), None) assert result == expected_result except Exception as e: assert str(e) == str(expected_result) # this had to be a seprate test case because the @freezegun.freeze_time # was screwing up the instantiation of SampleWindow.from_dict calls for the # other test cases @freezegun.freeze_time("2025-01-28T02:03:0Z") def test_convert_relative(self): input = "3 days" expected_result = SampleWindow( start=datetime(2025, 1, 25, 2, 3, 0, 0, pytz.UTC), end=datetime(2025, 1, 28, 2, 3, 0, 0, pytz.UTC), ) result = SampleType().convert(input, Option(["--sample"]), None) assert result == expected_result ================================================ FILE: tests/unit/clients/__init__.py ================================================ ================================================ FILE: tests/unit/clients/test_jinja.py ================================================ from contextlib import contextmanager import pytest import yaml from dbt.clients.jinja import get_rendered, get_template from dbt_common.exceptions import JinjaRenderingError @contextmanager def returns(value): yield value @contextmanager def raises(value): with pytest.raises(value) as exc: yield exc def expected_id(arg): if isinstance(arg, list): return "_".join(arg) jinja_tests = [ # strings ( """foo: bar""", returns("bar"), returns("bar"), ), ( '''foo: "bar"''', returns("bar"), returns("bar"), ), ( '''foo: "'bar'"''', returns("'bar'"), returns("'bar'"), ), ( """foo: '"bar"'""", returns('"bar"'), returns('"bar"'), ), ( '''foo: "{{ 'bar' | as_text }}"''', returns("bar"), returns("bar"), ), ( '''foo: "{{ 'bar' | as_bool }}"''', returns("bar"), raises(JinjaRenderingError), ), ( '''foo: "{{ 'bar' | as_number }}"''', returns("bar"), raises(JinjaRenderingError), ), ( '''foo: "{{ 'bar' | as_native }}"''', returns("bar"), returns("bar"), ), # ints ( """foo: 1""", returns("1"), returns("1"), ), ( '''foo: "1"''', returns("1"), returns("1"), ), ( '''foo: "'1'"''', returns("'1'"), returns("'1'"), ), ( """foo: '"1"'""", returns('"1"'), returns('"1"'), ), ( '''foo: "{{ 1 }}"''', returns("1"), returns("1"), ), ( '''foo: "{{ '1' }}"''', returns("1"), returns("1"), ), ( '''foo: "'{{ 1 }}'"''', returns("'1'"), returns("'1'"), ), ( '''foo: "'{{ '1' }}'"''', returns("'1'"), returns("'1'"), ), ( '''foo: "{{ 1 | as_text }}"''', returns("1"), returns("1"), ), ( '''foo: "{{ 1 | as_bool }}"''', returns("1"), raises(JinjaRenderingError), ), ( '''foo: "{{ 1 | as_number }}"''', returns("1"), returns(1), ), ( '''foo: "{{ 1 | as_native }}"''', returns("1"), returns(1), ), ( '''foo: "{{ '1' | as_text }}"''', returns("1"), returns("1"), ), ( '''foo: "{{ '1' | as_bool }}"''', returns("1"), raises(JinjaRenderingError), ), ( '''foo: "{{ '1' | as_number }}"''', returns("1"), returns(1), ), ( '''foo: "{{ '1' | as_native }}"''', returns("1"), returns(1), ), # booleans. # Note the discrepancy with true vs True: `true` is recognized by jinja but # not literal_eval, but `True` is recognized by ast.literal_eval. # For extra fun, yaml recognizes both. # unquoted true ( '''foo: "{{ True }}"''', returns("True"), returns("True"), ), ( '''foo: "{{ True | as_text }}"''', returns("True"), returns("True"), ), ( '''foo: "{{ True | as_bool }}"''', returns("True"), returns(True), ), ( '''foo: "{{ True | as_number }}"''', returns("True"), raises(JinjaRenderingError), ), ( '''foo: "{{ True | as_native }}"''', returns("True"), returns(True), ), # unquoted true ( '''foo: "{{ true }}"''', returns("True"), returns("True"), ), ( '''foo: "{{ true | as_text }}"''', returns("True"), returns("True"), ), ( '''foo: "{{ true | as_bool }}"''', returns("True"), returns(True), ), ( '''foo: "{{ true | as_number }}"''', returns("True"), raises(JinjaRenderingError), ), ( '''foo: "{{ true | as_native }}"''', returns("True"), returns(True), ), ( '''foo: "{{ 'true' | as_text }}"''', returns("true"), returns("true"), ), # quoted 'true' ( '''foo: "'{{ true }}'"''', returns("'True'"), returns("'True'"), ), # jinja true -> python True -> str(True) -> "True" -> quoted ( '''foo: "'{{ true | as_text }}'"''', returns("'True'"), returns("'True'"), ), ( '''foo: "'{{ true | as_bool }}'"''', returns("'True'"), returns("'True'"), ), ( '''foo: "'{{ true | as_number }}'"''', returns("'True'"), returns("'True'"), ), ( '''foo: "'{{ true | as_native }}'"''', returns("'True'"), returns("'True'"), ), # unquoted True ( '''foo: "{{ True }}"''', returns("True"), returns("True"), ), ( '''foo: "{{ True | as_text }}"''', returns("True"), returns("True"), ), # True -> string 'True' -> text -> str('True') -> 'True' ( '''foo: "{{ True | as_bool }}"''', returns("True"), returns(True), ), ( '''foo: "{{ True | as_number }}"''', returns("True"), raises(JinjaRenderingError), ), ( '''foo: "{{ True | as_native }}"''', returns("True"), returns(True), ), # quoted 'True' within rendering ( '''foo: "{{ 'True' | as_text }}"''', returns("True"), returns("True"), ), # 'True' -> string 'True' -> text -> str('True') -> 'True' ( '''foo: "{{ 'True' | as_bool }}"''', returns("True"), returns(True), ), # quoted 'True' outside rendering ( '''foo: "'{{ True }}'"''', returns("'True'"), returns("'True'"), ), ( '''foo: "'{{ True | as_bool }}'"''', returns("'True'"), returns("'True'"), ), # yaml turns 'yes' into a boolean true ( """foo: yes""", returns("True"), returns("True"), ), ( '''foo: "yes"''', returns("yes"), returns("yes"), ), # concatenation ( '''foo: "{{ (a_int + 100) | as_native }}"''', returns("200"), returns(200), ), ( '''foo: "{{ (a_str ~ 100) | as_native }}"''', returns("100100"), returns(100100), ), ( '''foo: "{{( a_int ~ 100) | as_native }}"''', returns("100100"), returns(100100), ), # multiple nodes -> always str ( '''foo: "{{ a_str | as_native }}{{ a_str | as_native }}"''', returns("100100"), returns("100100"), ), ( '''foo: "{{ a_int | as_native }}{{ a_int | as_native }}"''', returns("100100"), returns("100100"), ), ( '''foo: "'{{ a_int | as_native }}{{ a_int | as_native }}'"''', returns("'100100'"), returns("'100100'"), ), ( """foo:""", returns("None"), returns("None"), ), ( """foo: null""", returns("None"), returns("None"), ), ( '''foo: ""''', returns(""), returns(""), ), ( '''foo: "{{ '' | as_native }}"''', returns(""), returns(""), ), # very annoying, but jinja 'none' is yaml 'null'. ( '''foo: "{{ none | as_native }}"''', returns("None"), returns(None), ), # make sure we don't include comments in the output (see #2707) ( '''foo: "{# #}hello"''', returns("hello"), returns("hello"), ), ( '''foo: "{% if false %}{% endif %}hello"''', returns("hello"), returns("hello"), ), ] @pytest.mark.parametrize("value,text_expectation,native_expectation", jinja_tests, ids=expected_id) def test_jinja_rendering_string(value, text_expectation, native_expectation): foo_value = yaml.safe_load(value)["foo"] ctx = {"a_str": "100", "a_int": 100, "b_str": "hello"} with text_expectation as text_result: assert text_result == get_rendered(foo_value, ctx, native=False) with native_expectation as native_result: assert native_result == get_rendered(foo_value, ctx, native=True) def test_do(): s = "{% set my_dict = {} %}\n{% do my_dict.update(a=1) %}" template = get_template(s, {}) mod = template.make_module() assert mod.my_dict == {"a": 1} def test_regular_render(): s = '{{ "some_value" | as_native }}' value = get_rendered(s, {}, native=False) assert value == "some_value" s = "{{ 1991 | as_native }}" value = get_rendered(s, {}, native=False) assert value == "1991" s = '{{ "some_value" | as_text }}' value = get_rendered(s, {}, native=False) assert value == "some_value" s = "{{ 1991 | as_text }}" value = get_rendered(s, {}, native=False) assert value == "1991" def test_native_render(): s = '{{ "some_value" | as_native }}' value = get_rendered(s, {}, native=True) assert value == "some_value" s = "{{ 1991 | as_native }}" value = get_rendered(s, {}, native=True) assert value == 1991 s = '{{ "some_value" | as_text }}' value = get_rendered(s, {}, native=True) assert value == "some_value" s = "{{ 1991 | as_text }}" value = get_rendered(s, {}, native=True) assert value == "1991" ================================================ FILE: tests/unit/clients/test_jinja_static.py ================================================ import pytest from dbt.artifacts.resources import RefArgs from dbt.clients.jinja_static import ( statically_extract_has_name_this, statically_extract_macro_calls, statically_parse_ref_or_source, statically_parse_unrendered_config, ) from dbt.context.base import generate_base_context from dbt.exceptions import ParsingError @pytest.mark.parametrize( "macro_string,expected_possible_macro_calls", [ ( "{% macro parent_macro() %} {% do return(nested_macro()) %} {% endmacro %}", ["nested_macro"], ), ( "{% macro lr_macro() %} {{ return(load_result('relations').table) }} {% endmacro %}", ["load_result"], ), ( "{% macro get_snapshot_unique_id() -%} {{ return(adapter.dispatch('get_snapshot_unique_id')()) }} {%- endmacro %}", ["get_snapshot_unique_id"], ), ( "{% macro get_columns_in_query(select_sql) -%} {{ return(adapter.dispatch('get_columns_in_query')(select_sql)) }} {% endmacro %}", ["get_columns_in_query"], ), ( """{% macro test_mutually_exclusive_ranges(model) %} with base as ( select {{ get_snapshot_unique_id() }} as dbt_unique_id, * from {{ model }} ) {% endmacro %}""", ["get_snapshot_unique_id"], ), ( "{% macro test_my_test(model) %} select {{ current_timestamp_backcompat() }} {% endmacro %}", ["current_timestamp_backcompat"], ), ( "{% macro some_test(model) -%} {{ return(adapter.dispatch('test_some_kind4', 'foo_utils4')) }} {%- endmacro %}", ["test_some_kind4", "foo_utils4.test_some_kind4"], ), ( "{% macro some_test(model) -%} {{ return(adapter.dispatch('test_some_kind5', macro_namespace = 'foo_utils5')) }} {%- endmacro %}", ["test_some_kind5", "foo_utils5.test_some_kind5"], ), ], ) def test_extract_macro_calls(macro_string, expected_possible_macro_calls): cli_vars = {"local_utils_dispatch_list": ["foo_utils4"]} ctx = generate_base_context(cli_vars) possible_macro_calls = statically_extract_macro_calls(macro_string, ctx) assert possible_macro_calls == expected_possible_macro_calls class TestStaticallyParseRefOrSource: def test_invalid_expression(self): with pytest.raises(ParsingError): statically_parse_ref_or_source("invalid") @pytest.mark.parametrize( "expression,expected_ref_or_source", [ ("ref('model')", RefArgs(name="model")), ("ref('package','model')", RefArgs(name="model", package="package")), ("ref('model',v=3)", RefArgs(name="model", version=3)), ("ref('package','model',v=3)", RefArgs(name="model", package="package", version=3)), ("source('schema', 'table')", ["schema", "table"]), ], ) def test_valid_ref_expression(self, expression, expected_ref_or_source): ref_or_source = statically_parse_ref_or_source(expression) assert ref_or_source == expected_ref_or_source class TestStaticallyParseUnrenderedConfig: @pytest.mark.parametrize( "expression,expected_unrendered_config", [ ( "{{ config(materialized='view') }}", {"materialized": "Keyword(key='materialized', value=Const(value='view'))"}, ), ( "{{ config(materialized='view', enabled=True) }}", { "materialized": "Keyword(key='materialized', value=Const(value='view'))", "enabled": "Keyword(key='enabled', value=Const(value=True))", }, ), ( "{{ config(materialized=env_var('test')) }}", { "materialized": "Keyword(key='materialized', value=Call(node=Name(name='env_var', ctx='load'), args=[Const(value='test')], kwargs=[], dyn_args=None, dyn_kwargs=None))" }, ), ( "{{ config(materialized=env_var('test', default='default')) }}", { "materialized": "Keyword(key='materialized', value=Call(node=Name(name='env_var', ctx='load'), args=[Const(value='test')], kwargs=[Keyword(key='default', value=Const(value='default'))], dyn_args=None, dyn_kwargs=None))" }, ), ( "{{ config(materialized=env_var('test', default=env_var('default'))) }}", { "materialized": "Keyword(key='materialized', value=Call(node=Name(name='env_var', ctx='load'), args=[Const(value='test')], kwargs=[Keyword(key='default', value=Call(node=Name(name='env_var', ctx='load'), args=[Const(value='default')], kwargs=[], dyn_args=None, dyn_kwargs=None))], dyn_args=None, dyn_kwargs=None))" }, ), ], ) def test_statically_parse_unrendered_config(self, expression, expected_unrendered_config): unrendered_config = statically_parse_unrendered_config(expression) assert unrendered_config == expected_unrendered_config @pytest.mark.parametrize( "raw_code,expected_result", [ ("{{ this }}", True), ("{{ this.variable }}", True), ("{{ log(this) }}", True), ("{{ some_other_this }}", False), ("this", False), ("{{ some_object.this }}", False), ], ) def test_statically_extract_has_name_this(raw_code: str, expected_result: bool) -> None: assert statically_extract_has_name_this(raw_code) == expected_result ================================================ FILE: tests/unit/clients/test_registry.py ================================================ import unittest from dbt.clients.registry import _get_package_with_retries from dbt_common.exceptions import ConnectionError class testRegistryGetRequestException(unittest.TestCase): def test_registry_request_error_catching(self): # using non routable IP to test connection error logic in the _get_package_with_retries function self.assertRaises(ConnectionError, _get_package_with_retries, "", "http://0.0.0.0") ================================================ FILE: tests/unit/clients/test_yaml_helper.py ================================================ from dbt.clients.checked_load import checked_load no_dupe__yml = """ a: b: 1 b: a: 1 """ top_level_dupe__yml = """ a: b: 1 a: c: 1 d: 2 e: 3 """ nested_dupe__yml = """ a: b: 1 c: d: 1 e: 2 d: 3 """ multiple_dupes__yml = """ a: b: c: 1 d: e: f: 1 g: 2 f: 3 h: 4 f: 5 """ # Overrides should not cause duplicate key warnings on the data_tests key below. override__yml = """ version: 2 models: - name: my_first_dbt_model description: &description "A starter dbt model" columns: - ©_me name: id description: The ID. data_tests: - not_null - name: my_second_dbt_model description: *description columns: - <<: *copy_me data_tests: - unique """ override_with_issue__yml = """ version: 2 models: - name: my_first_dbt_model description: &description "A starter dbt model" columns: - ©_me name: id description: The ID. data_tests: - not_null - name: my_second_dbt_model description: *description columns: - <<: *copy_me data_tests: - unique data_tests: - unique """ def test_checked_load(): no_dupe_issues = checked_load(no_dupe__yml)[1] assert no_dupe_issues == [] top_level_dupe_issues = checked_load(top_level_dupe__yml)[1] assert len(top_level_dupe_issues) == 1 nested_dupe_issues = checked_load(nested_dupe__yml)[1] assert len(nested_dupe_issues) == 1 multiple_dupes_issues = checked_load(multiple_dupes__yml)[1] assert len(multiple_dupes_issues) == 2 override_dupes_issues = checked_load(override__yml)[1] assert len(override_dupes_issues) == 0 # This currently fails. We are not checking for genuine duplicate keys # in override anchors. # real_override_dupes_issues = checked_load(override__yml)[1] # assert len(real_override_dupes_issues) == 1 ================================================ FILE: tests/unit/config/__init__.py ================================================ import os import shutil import tempfile import unittest from argparse import Namespace from contextlib import contextmanager import yaml import dbt.config import dbt.exceptions from dbt import flags from dbt.constants import PACKAGES_FILE_NAME from dbt.flags import set_from_args from tests.unit.utils import normalize INITIAL_ROOT = os.getcwd() @contextmanager def temp_cd(path): current_path = os.getcwd() os.chdir(path) try: yield finally: os.chdir(current_path) @contextmanager def raises_nothing(): yield def empty_profile_renderer(): return dbt.config.renderer.ProfileRenderer({}) def empty_project_renderer(): return dbt.config.renderer.DbtProjectYamlRenderer() model_config = { "my_package_name": { "enabled": True, "adwords": { "adwords_ads": {"materialized": "table", "enabled": True, "schema": "analytics"} }, "snowplow": { "snowplow_sessions": { "sort": "timestamp", "materialized": "incremental", "dist": "user_id", "unique_key": "id", }, "base": { "snowplow_events": { "sort": ["timestamp", "userid"], "materialized": "table", "sort_type": "interleaved", "dist": "userid", } }, }, } } model_fqns = frozenset( ( ("my_package_name", "snowplow", "snowplow_sessions"), ("my_package_name", "snowplow", "base", "snowplow_events"), ("my_package_name", "adwords", "adwords_ads"), ) ) class Args: def __init__( self, profiles_dir=None, threads=None, profile=None, cli_vars=None, version_check=None, project_dir=None, target=None, ): self.profile = profile self.threads = threads self.target = target if profiles_dir is not None: self.profiles_dir = profiles_dir flags.PROFILES_DIR = profiles_dir if cli_vars is not None: self.vars = cli_vars if version_check is not None: self.version_check = version_check if project_dir is not None: self.project_dir = project_dir class BaseConfigTest(unittest.TestCase): """Subclass this, and before calling the superclass setUp, set self.profiles_dir and self.project_dir. """ def setUp(self): # Write project self.project_dir = normalize(tempfile.mkdtemp()) self.default_project_data = { "version": "0.0.1", "name": "my_test_project", "profile": "default", } self.write_project(self.default_project_data) # Write profile self.profiles_dir = normalize(tempfile.mkdtemp()) self.default_profile_data = { "default": { "outputs": { "postgres": { "type": "postgres", "host": "postgres-db-hostname", "port": 5555, "user": "db_user", "pass": "db_pass", "dbname": "postgres-db-name", "schema": "postgres-schema", "threads": 7, }, "with-vars": { "type": "{{ env_var('env_value_type') }}", "host": "{{ env_var('env_value_host') }}", "port": "{{ env_var('env_value_port') | as_number }}", "user": "{{ env_var('env_value_user') }}", "pass": "{{ env_var('env_value_pass') }}", "dbname": "{{ env_var('env_value_dbname') }}", "schema": "{{ env_var('env_value_schema') }}", }, "cli-and-env-vars": { "type": "{{ env_var('env_value_type') }}", "host": "{{ var('cli_value_host') }}", "port": "{{ env_var('env_value_port') | as_number }}", "user": "{{ env_var('env_value_user') }}", "pass": "{{ env_var('env_value_pass') }}", "dbname": "{{ env_var('env_value_dbname') }}", "schema": "{{ env_var('env_value_schema') }}", }, }, "target": "postgres", }, "other": { "outputs": { "other-postgres": { "type": "postgres", "host": "other-postgres-db-hostname", "port": 4444, "user": "other_db_user", "pass": "other_db_pass", "dbname": "other-postgres-db-name", "schema": "other-postgres-schema", "threads": 2, } }, "target": "other-postgres", }, "empty_profile_data": {}, } self.write_profile(self.default_profile_data) self.args = Namespace( profiles_dir=self.profiles_dir, cli_vars={}, version_check=True, project_dir=self.project_dir, target=None, threads=None, profile=None, ) set_from_args(self.args, None) self.env_override = { "env_value_type": "postgres", "env_value_host": "env-postgres-host", "env_value_port": "6543", "env_value_user": "env-postgres-user", "env_value_pass": "env-postgres-pass", "env_value_dbname": "env-postgres-dbname", "env_value_schema": "env-postgres-schema", "env_value_profile": "default", } def assertRaisesOrReturns(self, exc): if exc is None: return raises_nothing() else: return self.assertRaises(exc) def tearDown(self): try: shutil.rmtree(self.project_dir) except EnvironmentError: pass try: shutil.rmtree(self.profiles_dir) except EnvironmentError: pass def project_path(self, name): return os.path.join(self.project_dir, name) def profile_path(self, name): return os.path.join(self.profiles_dir, name) def write_project(self, project_data=None): if project_data is None: project_data = self.project_data with open(self.project_path("dbt_project.yml"), "w") as fp: yaml.dump(project_data, fp) def write_packages(self, package_data): with open(self.project_path("packages.yml"), "w") as fp: yaml.dump(package_data, fp) def write_profile(self, profile_data=None): if profile_data is None: profile_data = self.profile_data with open(self.profile_path("profiles.yml"), "w") as fp: yaml.dump(profile_data, fp) def write_empty_profile(self): with open(self.profile_path("profiles.yml"), "w") as fp: yaml.dump("", fp) def project_from_config_norender( cfg, packages=None, project_root="/invalid-root-path", verify_version=False ): if packages is None: packages = {} partial = dbt.config.project.PartialProject.from_dicts( project_root, project_dict=cfg, packages_dict=packages, selectors_dict={}, verify_version=verify_version, ) # no rendering ... Why? partial.project_dict["project-root"] = project_root rendered = dbt.config.project.RenderComponents( project_dict=partial.project_dict, packages_dict=partial.packages_dict, selectors_dict=partial.selectors_dict, ) return partial.create_project(rendered) def project_from_config_rendered( cfg, packages=None, project_root="/invalid-root-path", verify_version=False, packages_specified_path=PACKAGES_FILE_NAME, ): if packages is None: packages = {} partial = dbt.config.project.PartialProject.from_dicts( project_root, project_dict=cfg, packages_dict=packages, selectors_dict={}, verify_version=verify_version, packages_specified_path=packages_specified_path, ) return partial.render(empty_project_renderer()) ================================================ FILE: tests/unit/config/test_profile.py ================================================ import os from copy import deepcopy from unittest import mock import dbt.config import dbt.exceptions from dbt.adapters.postgres import PostgresCredentials from dbt.flags import set_from_args from dbt.tests.util import safe_set_invocation_context from tests.unit.config import ( BaseConfigTest, empty_profile_renderer, project_from_config_norender, ) class TestProfile(BaseConfigTest): def from_raw_profiles(self): renderer = empty_profile_renderer() return dbt.config.Profile.from_raw_profiles(self.default_profile_data, "default", renderer) def test_from_raw_profiles(self): profile = self.from_raw_profiles() self.assertEqual(profile.profile_name, "default") self.assertEqual(profile.target_name, "postgres") self.assertEqual(profile.threads, 7) self.assertTrue(isinstance(profile.credentials, PostgresCredentials)) self.assertEqual(profile.credentials.type, "postgres") self.assertEqual(profile.credentials.host, "postgres-db-hostname") self.assertEqual(profile.credentials.port, 5555) self.assertEqual(profile.credentials.user, "db_user") self.assertEqual(profile.credentials.password, "db_pass") self.assertEqual(profile.credentials.schema, "postgres-schema") self.assertEqual(profile.credentials.database, "postgres-db-name") def test_missing_type(self): del self.default_profile_data["default"]["outputs"]["postgres"]["type"] with self.assertRaises(dbt.exceptions.DbtProfileError) as exc: self.from_raw_profiles() self.assertIn("type", str(exc.exception)) self.assertIn("postgres", str(exc.exception)) self.assertIn("default", str(exc.exception)) def test_bad_type(self): self.default_profile_data["default"]["outputs"]["postgres"]["type"] = "invalid" with self.assertRaises(dbt.exceptions.DbtProfileError) as exc: self.from_raw_profiles() self.assertIn("Credentials", str(exc.exception)) self.assertIn("postgres", str(exc.exception)) self.assertIn("default", str(exc.exception)) def test_invalid_credentials(self): del self.default_profile_data["default"]["outputs"]["postgres"]["host"] with self.assertRaises(dbt.exceptions.DbtProfileError) as exc: self.from_raw_profiles() self.assertIn("Credentials", str(exc.exception)) self.assertIn("postgres", str(exc.exception)) self.assertIn("default", str(exc.exception)) def test_missing_target(self): profile = self.default_profile_data["default"] del profile["target"] profile["outputs"]["default"] = profile["outputs"]["postgres"] profile = self.from_raw_profiles() self.assertEqual(profile.profile_name, "default") self.assertEqual(profile.target_name, "default") self.assertEqual(profile.credentials.type, "postgres") def test_extra_path(self): self.default_project_data.update( { "model-paths": ["models"], "source-paths": ["other-models"], } ) with self.assertRaises(dbt.exceptions.DbtProjectError) as exc: project_from_config_norender(self.default_project_data, project_root=self.project_dir) self.assertIn("source-paths and model-paths", str(exc.exception)) self.assertIn("cannot both be defined.", str(exc.exception)) def test_profile_invalid_project(self): renderer = empty_profile_renderer() with self.assertRaises(dbt.exceptions.DbtProjectError) as exc: dbt.config.Profile.from_raw_profiles( self.default_profile_data, "invalid-profile", renderer ) self.assertEqual(exc.exception.result_type, "invalid_project") self.assertIn("Could not find", str(exc.exception)) self.assertIn("invalid-profile", str(exc.exception)) def test_profile_invalid_target(self): renderer = empty_profile_renderer() with self.assertRaises(dbt.exceptions.DbtProfileError) as exc: dbt.config.Profile.from_raw_profiles( self.default_profile_data, "default", renderer, target_override="nope" ) self.assertIn("nope", str(exc.exception)) self.assertIn("- postgres", str(exc.exception)) self.assertIn("- with-vars", str(exc.exception)) def test_no_outputs(self): renderer = empty_profile_renderer() with self.assertRaises(dbt.exceptions.DbtProfileError) as exc: dbt.config.Profile.from_raw_profiles( {"some-profile": {"target": "blah"}}, "some-profile", renderer ) self.assertIn("outputs not specified", str(exc.exception)) self.assertIn("some-profile", str(exc.exception)) def test_neq(self): profile = self.from_raw_profiles() self.assertNotEqual(profile, object()) def test_eq(self): renderer = empty_profile_renderer() profile = dbt.config.Profile.from_raw_profiles( deepcopy(self.default_profile_data), "default", renderer ) other = dbt.config.Profile.from_raw_profiles( deepcopy(self.default_profile_data), "default", renderer ) self.assertEqual(profile, other) def test_invalid_env_vars(self): self.env_override["env_value_port"] = "hello" with mock.patch.dict(os.environ, self.env_override): with self.assertRaises(dbt.exceptions.DbtProfileError) as exc: safe_set_invocation_context() renderer = empty_profile_renderer() dbt.config.Profile.from_raw_profile_info( self.default_profile_data["default"], "default", renderer, target_override="with-vars", ) self.assertIn("Could not convert value 'hello' into type 'number'", str(exc.exception)) class TestProfileFile(BaseConfigTest): def from_raw_profile_info(self, raw_profile=None, profile_name="default", **kwargs): if raw_profile is None: raw_profile = self.default_profile_data["default"] renderer = empty_profile_renderer() kw = { "raw_profile": raw_profile, "profile_name": profile_name, "renderer": renderer, } kw.update(kwargs) return dbt.config.Profile.from_raw_profile_info(**kw) def from_args(self, project_profile_name="default", **kwargs): kw = { "project_profile_name": project_profile_name, "renderer": empty_profile_renderer(), "threads_override": self.args.threads, "target_override": self.args.target, "profile_name_override": self.args.profile, } kw.update(kwargs) return dbt.config.Profile.render(**kw) def test_profile_simple(self): profile = self.from_args() from_raw = self.from_raw_profile_info() self.assertEqual(profile.profile_name, "default") self.assertEqual(profile.target_name, "postgres") self.assertEqual(profile.threads, 7) self.assertTrue(isinstance(profile.credentials, PostgresCredentials)) self.assertEqual(profile.credentials.type, "postgres") self.assertEqual(profile.credentials.host, "postgres-db-hostname") self.assertEqual(profile.credentials.port, 5555) self.assertEqual(profile.credentials.user, "db_user") self.assertEqual(profile.credentials.password, "db_pass") self.assertEqual(profile.credentials.schema, "postgres-schema") self.assertEqual(profile.credentials.database, "postgres-db-name") self.assertEqual(profile, from_raw) def test_profile_override(self): self.args.profile = "other" self.args.threads = 3 set_from_args(self.args, None) profile = self.from_args() from_raw = self.from_raw_profile_info( self.default_profile_data["other"], "other", threads_override=3, ) self.assertEqual(profile.profile_name, "other") self.assertEqual(profile.target_name, "other-postgres") self.assertEqual(profile.threads, 3) self.assertTrue(isinstance(profile.credentials, PostgresCredentials)) self.assertEqual(profile.credentials.type, "postgres") self.assertEqual(profile.credentials.host, "other-postgres-db-hostname") self.assertEqual(profile.credentials.port, 4444) self.assertEqual(profile.credentials.user, "other_db_user") self.assertEqual(profile.credentials.password, "other_db_pass") self.assertEqual(profile.credentials.schema, "other-postgres-schema") self.assertEqual(profile.credentials.database, "other-postgres-db-name") self.assertEqual(profile, from_raw) def test_env_vars(self): self.args.target = "with-vars" with mock.patch.dict(os.environ, self.env_override): safe_set_invocation_context() # reset invocation context with new env profile = self.from_args() from_raw = self.from_raw_profile_info(target_override="with-vars") self.assertEqual(profile.profile_name, "default") self.assertEqual(profile.target_name, "with-vars") self.assertEqual(profile.threads, 1) self.assertEqual(profile.credentials.type, "postgres") self.assertEqual(profile.credentials.host, "env-postgres-host") self.assertEqual(profile.credentials.port, 6543) self.assertEqual(profile.credentials.user, "env-postgres-user") self.assertEqual(profile.credentials.password, "env-postgres-pass") self.assertEqual(profile, from_raw) def test_env_vars_env_target(self): self.default_profile_data["default"]["target"] = "{{ env_var('env_value_target') }}" self.write_profile(self.default_profile_data) self.env_override["env_value_target"] = "with-vars" with mock.patch.dict(os.environ, self.env_override): safe_set_invocation_context() # reset invocation context with new env profile = self.from_args() from_raw = self.from_raw_profile_info(target_override="with-vars") self.assertEqual(profile.profile_name, "default") self.assertEqual(profile.target_name, "with-vars") self.assertEqual(profile.threads, 1) self.assertEqual(profile.credentials.type, "postgres") self.assertEqual(profile.credentials.host, "env-postgres-host") self.assertEqual(profile.credentials.port, 6543) self.assertEqual(profile.credentials.user, "env-postgres-user") self.assertEqual(profile.credentials.password, "env-postgres-pass") self.assertEqual(profile, from_raw) def test_invalid_env_vars(self): self.env_override["env_value_port"] = "hello" self.args.target = "with-vars" with mock.patch.dict(os.environ, self.env_override): with self.assertRaises(dbt.exceptions.DbtProfileError) as exc: safe_set_invocation_context() # reset invocation context with new env self.from_args() self.assertIn("Could not convert value 'hello' into type 'number'", str(exc.exception)) def test_cli_and_env_vars(self): self.args.target = "cli-and-env-vars" self.args.vars = {"cli_value_host": "cli-postgres-host"} renderer = dbt.config.renderer.ProfileRenderer({"cli_value_host": "cli-postgres-host"}) with mock.patch.dict(os.environ, self.env_override): safe_set_invocation_context() # reset invocation context with new env profile = self.from_args(renderer=renderer) from_raw = self.from_raw_profile_info( target_override="cli-and-env-vars", renderer=renderer, ) self.assertEqual(profile.profile_name, "default") self.assertEqual(profile.target_name, "cli-and-env-vars") self.assertEqual(profile.threads, 1) self.assertEqual(profile.credentials.type, "postgres") self.assertEqual(profile.credentials.host, "cli-postgres-host") self.assertEqual(profile.credentials.port, 6543) self.assertEqual(profile.credentials.user, "env-postgres-user") self.assertEqual(profile.credentials.password, "env-postgres-pass") self.assertEqual(profile, from_raw) def test_no_profile(self): with self.assertRaises(dbt.exceptions.DbtProjectError) as exc: self.from_args(project_profile_name=None) self.assertIn("no profile was specified", str(exc.exception)) def test_empty_profile(self): self.write_empty_profile() with self.assertRaises(dbt.exceptions.DbtProfileError) as exc: self.from_args() self.assertIn("profiles.yml is empty", str(exc.exception)) def test_profile_with_empty_profile_data(self): renderer = empty_profile_renderer() with self.assertRaises(dbt.exceptions.DbtProfileError) as exc: dbt.config.Profile.from_raw_profiles( self.default_profile_data, "empty_profile_data", renderer ) self.assertIn("Profile empty_profile_data in profiles.yml is empty", str(exc.exception)) ================================================ FILE: tests/unit/config/test_project.py ================================================ import json import os import unittest from copy import deepcopy from typing import Any, Dict from unittest import mock import pytest import dbt.config import dbt.exceptions from dbt.adapters.contracts.connection import DEFAULT_QUERY_COMMENT, QueryComment from dbt.adapters.factory import load_plugin from dbt.config.project import Project, _get_required_version from dbt.constants import DEPENDENCIES_FILE_NAME from dbt.contracts.project import GitPackage, LocalPackage, PackageConfig from dbt.deprecations import ( GenericJSONSchemaValidationDeprecation as GenericJSONSchemaValidationDeprecationCore, ) from dbt.events.types import GenericJSONSchemaValidationDeprecation from dbt.flags import set_from_args from dbt.jsonschemas.jsonschemas import project_schema from dbt.node_types import NodeType from dbt.tests.util import safe_set_invocation_context from dbt_common.events.event_catcher import EventCatcher from dbt_common.events.event_manager_client import get_event_manager from dbt_common.events.types import Note from dbt_common.exceptions import DbtRuntimeError from dbt_common.semver import VersionSpecifier from tests.unit.config import ( BaseConfigTest, empty_project_renderer, project_from_config_norender, project_from_config_rendered, ) class TestProjectMethods: def test_all_source_paths(self, project: Project): assert ( project.all_source_paths.sort() == ["models", "seeds", "snapshots", "analyses", "macros", "tests"].sort() ) def test_generic_test_paths(self, project: Project): assert project.generic_test_paths == ["tests/generic"] def test_fixture_paths(self, project: Project): assert project.fixture_paths == ["tests/fixtures"] def test__str__(self, project: Project): assert ( str(project) == "{'name': 'test_project', 'version': 1.0, 'project-root': 'doesnt/actually/exist', 'profile': 'test_profile', 'model-paths': ['models'], 'macro-paths': ['macros'], 'seed-paths': ['seeds'], 'test-paths': ['tests'], 'analysis-paths': ['analyses'], 'docs-paths': ['docs'], 'asset-paths': ['assets'], 'target-path': 'target', 'snapshot-paths': ['snapshots'], 'clean-targets': ['target'], 'log-path': 'path/to/project/logs', 'quoting': {}, 'models': {}, 'on-run-start': [], 'on-run-end': [], 'dispatch': [{'macro_namespace': 'dbt_utils', 'search_order': ['test_project', 'dbt_utils']}], 'seeds': {}, 'snapshots': {}, 'sources': {}, 'data_tests': {}, 'unit_tests': {}, 'metrics': {}, 'semantic-models': {}, 'saved-queries': {}, 'exposures': {}, 'functions': {}, 'vars': {}, 'require-dbt-version': ['=0.0.0'], 'restrict-access': False, 'dbt-cloud': {}, 'flags': {}, 'query-comment': {'comment': \"\\n{%- set comment_dict = {} -%}\\n{%- do comment_dict.update(\\n app='dbt',\\n dbt_version=dbt_version,\\n profile_name=target.get('profile_name'),\\n target_name=target.get('target_name'),\\n) -%}\\n{%- if node is not none -%}\\n {%- do comment_dict.update(\\n node_id=node.unique_id,\\n ) -%}\\n{% else %}\\n {# in the node context, the connection name is the node_id #}\\n {%- do comment_dict.update(connection_name=connection_name) -%}\\n{%- endif -%}\\n{{ return(tojson(comment_dict)) }}\\n\", 'append': False, 'job-label': False}, 'packages': []}" ) def test_get_selector(self, project: Project): selector = project.get_selector("my_selector") assert selector.raw == "give me cats" with pytest.raises(DbtRuntimeError): project.get_selector("doesnt_exist") def test_get_default_selector_name(self, project: Project): default_selector_name = project.get_default_selector_name() assert default_selector_name == "my_selector" project.selectors["my_selector"]["default"] = False default_selector_name = project.get_default_selector_name() assert default_selector_name is None def test_get_macro_search_order(self, project: Project): search_order = project.get_macro_search_order("dbt_utils") assert search_order == ["test_project", "dbt_utils"] search_order = project.get_macro_search_order("doesnt_exist") assert search_order is None def test_project_target_path(self, project: Project): assert project.project_target_path == "doesnt/actually/exist/target" def test_eq(self, project: Project): other = deepcopy(project) assert project == other def test_neq(self, project: Project): other = deepcopy(project) other.project_name = "other project" assert project != other def test_hashed_name(self, project: Project): assert project.hashed_name() == "6e72a69d5c5cca8f0400338441c022e4" class TestProjectInitialization(BaseConfigTest): def test_defaults(self): project = project_from_config_norender( self.default_project_data, project_root=self.project_dir ) self.assertEqual(project.project_name, "my_test_project") self.assertEqual(project.version, "0.0.1") self.assertEqual(project.profile_name, "default") self.assertEqual(project.project_root, self.project_dir) self.assertEqual(project.model_paths, ["models"]) self.assertEqual(project.macro_paths, ["macros"]) self.assertEqual(project.seed_paths, ["seeds"]) self.assertEqual(project.test_paths, ["tests"]) self.assertEqual(project.analysis_paths, ["analyses"]) self.assertEqual( set(project.docs_paths), {"models", "seeds", "snapshots", "analyses", "macros", "tests", "functions"}, ) self.assertEqual(project.asset_paths, []) self.assertEqual(project.target_path, "target") self.assertEqual(project.clean_targets, ["target"]) self.assertEqual(project.log_path, "logs") self.assertEqual(project.packages_install_path, "dbt_packages") self.assertEqual(project.quoting, {}) self.assertEqual(project.models, {}) self.assertEqual(project.on_run_start, []) self.assertEqual(project.on_run_end, []) self.assertEqual(project.seeds, {}) self.assertEqual(project.dbt_version, [VersionSpecifier.from_version_string(">=0.0.0")]) self.assertEqual(project.packages, PackageConfig(packages=[])) # just make sure str() doesn't crash anything, that's always # embarrassing str(project) def test_implicit_overrides(self): self.default_project_data.update( { "model-paths": ["other-models"], } ) project = project_from_config_norender( self.default_project_data, project_root=self.project_dir ) self.assertEqual( set(project.docs_paths), {"other-models", "seeds", "snapshots", "analyses", "macros", "tests", "functions"}, ) def test_all_overrides(self): # log-path is not tested because it is set exclusively from flags, not cfg self.default_project_data.update( { "model-paths": ["other-models"], "macro-paths": ["other-macros"], "seed-paths": ["other-seeds"], "test-paths": ["other-tests"], "analysis-paths": ["other-analyses"], "docs-paths": ["docs"], "asset-paths": ["other-assets"], "clean-targets": ["another-target"], "packages-install-path": "other-dbt_packages", "quoting": {"identifier": False, "snowflake_ignore_case": True}, "models": { "pre-hook": ["{{ logging.log_model_start_event() }}"], "post-hook": ["{{ logging.log_model_end_event() }}"], "my_test_project": { "first": { "enabled": False, "sub": { "enabled": True, }, }, "second": { "materialized": "table", }, }, "third_party": { "third": { "materialized": "view", }, }, }, "on-run-start": [ "{{ logging.log_run_start_event() }}", ], "on-run-end": [ "{{ logging.log_run_end_event() }}", ], "seeds": { "my_test_project": { "enabled": True, "schema": "seed_data", "post-hook": "grant select on {{ this }} to bi_user", }, }, "data_tests": {"my_test_project": {"fail_calc": "sum(failures)"}}, "require-dbt-version": ">=0.1.0", } ) packages = { "packages": [ { "local": "foo", }, {"git": "git@example.com:dbt-labs/dbt-utils.git", "revision": "test-rev"}, ], } project = project_from_config_norender( self.default_project_data, project_root=self.project_dir, packages=packages ) self.assertEqual(project.project_name, "my_test_project") self.assertEqual(project.version, "0.0.1") self.assertEqual(project.profile_name, "default") self.assertEqual(project.model_paths, ["other-models"]) self.assertEqual(project.macro_paths, ["other-macros"]) self.assertEqual(project.seed_paths, ["other-seeds"]) self.assertEqual(project.test_paths, ["other-tests"]) self.assertEqual(project.analysis_paths, ["other-analyses"]) self.assertEqual(project.docs_paths, ["docs"]) self.assertEqual(project.asset_paths, ["other-assets"]) self.assertEqual(project.clean_targets, ["another-target"]) self.assertEqual(project.packages_install_path, "other-dbt_packages") self.assertEqual(project.quoting, {"identifier": False, "snowflake_ignore_case": True}) self.assertEqual( project.models, { "pre-hook": ["{{ logging.log_model_start_event() }}"], "post-hook": ["{{ logging.log_model_end_event() }}"], "my_test_project": { "first": { "enabled": False, "sub": { "enabled": True, }, }, "second": { "materialized": "table", }, }, "third_party": { "third": { "materialized": "view", }, }, }, ) self.assertEqual(project.on_run_start, ["{{ logging.log_run_start_event() }}"]) self.assertEqual(project.on_run_end, ["{{ logging.log_run_end_event() }}"]) self.assertEqual( project.seeds, { "my_test_project": { "enabled": True, "schema": "seed_data", "post-hook": "grant select on {{ this }} to bi_user", }, }, ) self.assertEqual( project.data_tests, { "my_test_project": {"fail_calc": "sum(failures)"}, }, ) self.assertEqual(project.dbt_version, [VersionSpecifier.from_version_string(">=0.1.0")]) self.assertEqual( project.packages, PackageConfig( packages=[ LocalPackage(local="foo", unrendered={"local": "foo"}), GitPackage( git="git@example.com:dbt-labs/dbt-utils.git", revision="test-rev", unrendered={ "git": "git@example.com:dbt-labs/dbt-utils.git", "revision": "test-rev", }, ), ] ), ) str(project) # this does the equivalent of project.to_project_config(with_packages=True) json.dumps(project.to_project_config()) def test_string_run_hooks(self): self.default_project_data.update( { "on-run-start": "{{ logging.log_run_start_event() }}", "on-run-end": "{{ logging.log_run_end_event() }}", } ) project = project_from_config_rendered(self.default_project_data) self.assertEqual(project.on_run_start, ["{{ logging.log_run_start_event() }}"]) self.assertEqual(project.on_run_end, ["{{ logging.log_run_end_event() }}"]) def test_invalid_project_name(self): self.default_project_data["name"] = "invalid-project-name" with self.assertRaises(dbt.exceptions.DbtProjectError) as exc: project_from_config_norender(self.default_project_data, project_root=self.project_dir) self.assertIn("invalid-project-name", str(exc.exception)) def test_no_project(self): os.remove(os.path.join(self.project_dir, "dbt_project.yml")) renderer = empty_project_renderer() with self.assertRaises(dbt.exceptions.DbtProjectError) as exc: dbt.config.Project.from_project_root(self.project_dir, renderer) self.assertIn("No dbt_project.yml", str(exc.exception)) def test_invalid_version(self): self.default_project_data["require-dbt-version"] = "hello!" with self.assertRaises(dbt.exceptions.DbtProjectError): project_from_config_norender(self.default_project_data, project_root=self.project_dir) def test_unsupported_version(self): self.default_project_data["require-dbt-version"] = ">99999.0.0" # allowed, because the RuntimeConfig checks, not the Project itself project_from_config_norender(self.default_project_data, project_root=self.project_dir) def test_none_values(self): self.default_project_data.update( { "models": None, "seeds": None, "on-run-end": None, "on-run-start": None, } ) project = project_from_config_rendered(self.default_project_data) self.assertEqual(project.models, {}) self.assertEqual(project.on_run_start, []) self.assertEqual(project.on_run_end, []) self.assertEqual(project.seeds, {}) def test_nested_none_values(self): self.default_project_data.update( { "models": {"vars": None, "pre-hook": None, "post-hook": None}, "seeds": {"vars": None, "pre-hook": None, "post-hook": None, "column_types": None}, } ) project = project_from_config_rendered(self.default_project_data) self.assertEqual(project.models, {"vars": {}, "pre-hook": [], "post-hook": []}) self.assertEqual( project.seeds, {"vars": {}, "pre-hook": [], "post-hook": [], "column_types": {}} ) @pytest.mark.skipif(os.name == "nt", reason="crashes CI for Windows") def test_cycle(self): models = {} models["models"] = models self.default_project_data.update( { "models": models, } ) with self.assertRaises(dbt.exceptions.DbtProjectError) as exc: project_from_config_rendered(self.default_project_data) assert "Cycle detected" in str(exc.exception) def test_query_comment_empty(self): self.default_project_data.update( { "query-comment": None, } ) project = project_from_config_norender( self.default_project_data, project_root=self.project_dir ) self.assertEqual(project.query_comment.comment, "") self.assertEqual(project.query_comment.append, QueryComment().append) self.default_project_data.update( { "query-comment": "", } ) project = project_from_config_norender( self.default_project_data, project_root=self.project_dir ) self.assertEqual(project.query_comment.comment, "") self.assertEqual(project.query_comment.append, QueryComment().append) def test_default_query_comment(self): project = project_from_config_norender( self.default_project_data, project_root=self.project_dir ) self.assertEqual(project.query_comment, QueryComment()) def test_default_query_comment_append(self): self.default_project_data.update( { "query-comment": {"append": True}, } ) project = project_from_config_norender( self.default_project_data, project_root=self.project_dir ) self.assertEqual(project.query_comment.comment, DEFAULT_QUERY_COMMENT) self.assertEqual(project.query_comment.append, True) def test_custom_query_comment_append(self): self.default_project_data.update( { "query-comment": {"comment": "run by user test", "append": True}, } ) project = project_from_config_norender( self.default_project_data, project_root=self.project_dir ) self.assertEqual(project.query_comment.comment, "run by user test") self.assertEqual(project.query_comment.append, True) def test_default_query_comment_append_False(self): self.default_project_data.update( { "query-comment": {"append": False}, } ) project = project_from_config_norender( self.default_project_data, project_root=self.project_dir ) self.assertEqual(project.query_comment.comment, DEFAULT_QUERY_COMMENT) self.assertEqual(project.query_comment.append, False) def test_custom_query_comment_append_false(self): self.default_project_data.update( { "query-comment": {"comment": "run by user test", "append": False}, } ) project = project_from_config_norender( self.default_project_data, project_root=self.project_dir ) self.assertEqual(project.query_comment.comment, "run by user test") self.assertEqual(project.query_comment.append, False) def test_packages_from_dependencies(self): packages = { "packages": [ { "git": "{{ env_var('some_package') }}", "warn-unpinned": True, } ], } project = project_from_config_rendered( self.default_project_data, packages, packages_specified_path=DEPENDENCIES_FILE_NAME ) git_package = project.packages.packages[0] # packages did not render because packages_specified_path=DEPENDENCIES_FILE_NAME assert git_package.git == "{{ env_var('some_package') }}" class TestProjectFile(BaseConfigTest): def test_from_project_root(self): renderer = empty_project_renderer() project = dbt.config.Project.from_project_root(self.project_dir, renderer) from_config = project_from_config_norender( self.default_project_data, project_root=self.project_dir ) self.assertEqual(project, from_config) self.assertEqual(project.version, "0.0.1") self.assertEqual(project.project_name, "my_test_project") def test_with_invalid_package(self): renderer = empty_project_renderer() self.write_packages({"invalid": ["not a package of any kind"]}) with self.assertRaises(dbt.exceptions.DbtProjectError): dbt.config.Project.from_project_root(self.project_dir, renderer) class TestVariableProjectFile(BaseConfigTest): def setUp(self): super().setUp() self.default_project_data["version"] = "{{ var('cli_version') }}" self.default_project_data["name"] = "blah" self.default_project_data["profile"] = "{{ env_var('env_value_profile') }}" self.write_project(self.default_project_data) def test_cli_and_env_vars(self): renderer = dbt.config.renderer.DbtProjectYamlRenderer(None, {"cli_version": "0.1.2"}) with mock.patch.dict(os.environ, self.env_override): safe_set_invocation_context() # reset invocation context with new env project = dbt.config.Project.from_project_root( self.project_dir, renderer, ) self.assertEqual(renderer.ctx_obj.env_vars, {"env_value_profile": "default"}) self.assertEqual(project.version, "0.1.2") self.assertEqual(project.project_name, "blah") self.assertEqual(project.profile_name, "default") class TestVarLookups(unittest.TestCase): def setUp(self): self.initial_src_vars = { # globals "foo": 123, "bar": "hello", # project-scoped "my_project": { "bar": "goodbye", "baz": True, }, "other_project": { "foo": 456, }, } self.src_vars = deepcopy(self.initial_src_vars) self.dst = {"vars": deepcopy(self.initial_src_vars)} self.projects = ["my_project", "other_project", "third_project"] load_plugin("postgres") self.local_var_search = mock.MagicMock( fqn=["my_project", "my_model"], resource_type=NodeType.Model, package_name="my_project" ) self.other_var_search = mock.MagicMock( fqn=["other_project", "model"], resource_type=NodeType.Model, package_name="other_project", ) self.third_var_search = mock.MagicMock( fqn=["third_project", "third_model"], resource_type=NodeType.Model, package_name="third_project", ) def test_lookups(self): vars_provider = dbt.config.project.VarProvider(self.initial_src_vars) expected = [ (self.local_var_search, "foo", 123), (self.other_var_search, "foo", 456), (self.third_var_search, "foo", 123), (self.local_var_search, "bar", "goodbye"), (self.other_var_search, "bar", "hello"), (self.third_var_search, "bar", "hello"), (self.local_var_search, "baz", True), (self.other_var_search, "baz", None), (self.third_var_search, "baz", None), ] for node, key, expected_value in expected: value = vars_provider.vars_for(node, "postgres").get(key) assert value == expected_value class TestMultipleProjectFlags(BaseConfigTest): def setUp(self): super().setUp() self.default_project_data.update( { "flags": { "send_anonymous_usage_data": False, } } ) self.write_project(self.default_project_data) self.default_profile_data.update( { "config": { "send_anonymous_usage_data": False, } } ) self.write_profile(self.default_profile_data) def test_setting_multiple_flags(self): with pytest.raises(dbt.exceptions.DbtProjectError): set_from_args(self.args, None) class TestGetRequiredVersion: @pytest.fixture def project_dict(self) -> Dict[str, Any]: return { "name": "test_project", "require-dbt-version": ">0.0.0", } def test_supported_version(self, project_dict: Dict[str, Any]) -> None: specifiers = _get_required_version(project_dict=project_dict, verify_version=True) assert set(x.to_version_string() for x in specifiers) == {">0.0.0"} def test_unsupported_version(self, project_dict: Dict[str, Any]) -> None: project_dict["require-dbt-version"] = ">99999.0.0" with pytest.raises( dbt.exceptions.DbtProjectError, match="This version of dbt is not supported" ): _get_required_version(project_dict=project_dict, verify_version=True) def test_unsupported_version_no_check(self, project_dict: Dict[str, Any]) -> None: project_dict["require-dbt-version"] = ">99999.0.0" specifiers = _get_required_version(project_dict=project_dict, verify_version=False) assert set(x.to_version_string() for x in specifiers) == {">99999.0.0"} def test_supported_version_range(self, project_dict: Dict[str, Any]) -> None: project_dict["require-dbt-version"] = [">0.0.0", "<=99999.0.0"] specifiers = _get_required_version(project_dict=project_dict, verify_version=True) assert set(x.to_version_string() for x in specifiers) == {">0.0.0", "<=99999.0.0"} def test_unsupported_version_range(self, project_dict: Dict[str, Any]) -> None: project_dict["require-dbt-version"] = [">0.0.0", "<=0.0.1"] with pytest.raises( dbt.exceptions.DbtProjectError, match="This version of dbt is not supported" ): _get_required_version(project_dict=project_dict, verify_version=True) def test_unsupported_version_range_no_check(self, project_dict: Dict[str, Any]) -> None: project_dict["require-dbt-version"] = [">0.0.0", "<=0.0.1"] specifiers = _get_required_version(project_dict=project_dict, verify_version=False) assert set(x.to_version_string() for x in specifiers) == {">0.0.0", "<=0.0.1"} def test_impossible_version_range(self, project_dict: Dict[str, Any]) -> None: project_dict["require-dbt-version"] = [">99999.0.0", "<=0.0.1"] with pytest.raises( dbt.exceptions.DbtProjectError, match="The package version requirement can never be satisfied", ): _get_required_version(project_dict=project_dict, verify_version=True) class TestDeprecations: def test_jsonschema_validate(self) -> None: from dbt.jsonschemas.jsonschemas import jsonschema_validate project_dict: Dict[str, Any] = {} event_catcher = EventCatcher(GenericJSONSchemaValidationDeprecation) note_catcher = EventCatcher(Note) get_event_manager().add_callback(event_catcher.catch) get_event_manager().add_callback(note_catcher.catch) jsonschema_validate( schema=project_schema(), json=project_dict, file_path="dbt_project.yml" ) if GenericJSONSchemaValidationDeprecationCore()._is_preview: assert len(note_catcher.caught_events) == 1 assert len(event_catcher.caught_events) == 0 event = note_catcher.caught_events[0] else: assert len(event_catcher.caught_events) == 1 assert len(note_catcher.caught_events) == 0 event = event_catcher.caught_events[0] assert "'name' is a required property at top level" in event.info.msg ================================================ FILE: tests/unit/config/test_renderer_with_vars.py ================================================ """Unit tests for rendering dbt_project.yml with vars without defaults.""" import pytest from dbt.config.renderer import DbtProjectYamlRenderer from dbt.context.base import BaseContext class TestRendererWithRequiredVars: """Test that DbtProjectYamlRenderer doesn't raise errors for missing vars""" def test_base_context_with_require_vars_false(self): """Test that BaseContext with require_vars=False returns None for missing vars""" context = BaseContext(cli_vars={}, require_vars=False) var_func = context.var # Missing var should return None when require_vars=False assert var_func("missing_var") is None # Missing var with default should return the default assert var_func("missing_var", "default_value") == "default_value" # Existing var should return the value context2 = BaseContext(cli_vars={"existing_var": "value"}, require_vars=False) var_func2 = context2.var assert var_func2("existing_var") == "value" def test_base_context_with_require_vars_true_raises_error(self): """Test that BaseContext with require_vars=True raises error for missing vars""" from dbt.exceptions import RequiredVarNotFoundError context = BaseContext(cli_vars={}, require_vars=True) var_func = context.var # Missing var should raise error when require_vars=True with pytest.raises(RequiredVarNotFoundError): var_func("missing_var") def test_dbt_project_yaml_renderer_doesnt_fail_on_missing_vars(self): """Test that DbtProjectYamlRenderer with require_vars=False can render configs with missing vars""" # Pass require_vars=False to enable lenient mode (used by dbt deps) renderer = DbtProjectYamlRenderer(profile=None, cli_vars={}, require_vars=False) # This project config uses a var without a default value project_dict = { "name": "test_project", "version": "1.0", "models": {"test_project": {"+dataset": "dqm_{{ var('my_dataset') }}"}}, } # This should not raise an error in lenient mode rendered = renderer.render_data(project_dict) # The var should be rendered as None (which becomes "dqm_None" in the string) assert "models" in rendered assert "test_project" in rendered["models"] # When var returns None, it gets stringified in the template assert rendered["models"]["test_project"]["+dataset"] == "dqm_None" def test_dbt_project_yaml_renderer_with_provided_var(self): """Test that DbtProjectYamlRenderer works correctly when var is provided""" renderer = DbtProjectYamlRenderer(profile=None, cli_vars={"my_dataset": "prod"}) project_dict = { "name": "test_project", "version": "1.0", "models": {"test_project": {"+dataset": "dqm_{{ var('my_dataset') }}"}}, } # This should render correctly with the provided var rendered = renderer.render_data(project_dict) # The var should be properly rendered assert rendered["models"]["test_project"]["+dataset"] == "dqm_prod" ================================================ FILE: tests/unit/config/test_runtime.py ================================================ import os import tempfile from argparse import Namespace from typing import Any, Dict from unittest import mock import pytest from pytest_mock import MockerFixture import dbt.config import dbt.exceptions from dbt import tracking from dbt.config.profile import Profile from dbt.config.project import Project from dbt.config.runtime import RuntimeConfig from dbt.contracts.project import PackageConfig from dbt.events.types import UnusedResourceConfigPath from dbt.flags import set_from_args from dbt.tests.util import safe_set_invocation_context from dbt_common.events.event_catcher import EventCatcher from dbt_common.events.event_manager_client import add_callback_to_manager from tests.unit.config import BaseConfigTest, temp_cd class TestRuntimeConfig: @pytest.fixture def args(self) -> Namespace: return Namespace( profiles_dir=tempfile.mkdtemp(), cli_vars={}, version_check=True, project_dir=tempfile.mkdtemp(), target=None, threads=None, profile=None, ) def test_str(self, profile: Profile, project: Project) -> None: config = dbt.config.RuntimeConfig.from_parts(project, profile, {}) # to make sure nothing terrible happens str(config) def test_from_parts(self, args: Namespace, profile: Profile, project: Project): config = dbt.config.RuntimeConfig.from_parts(project, profile, args) assert config.cli_vars == {} assert config.to_profile_info() == profile.to_profile_info() # we should have the default quoting set in the full config, but not in # the project # TODO(jeb): Adapters must assert that quoting is populated? expected_project = project.to_project_config() assert expected_project["quoting"] == {} expected_project["quoting"] = { "database": True, "identifier": True, "schema": True, } assert config.to_project_config() == expected_project def test_get_metadata(self, mocker: MockerFixture, runtime_config: RuntimeConfig) -> None: mock_user = mocker.patch.object(tracking, "active_user") mock_user.id = "cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf" set_from_args(Namespace(SEND_ANONYMOUS_USAGE_STATS=False), None) metadata = runtime_config.get_metadata() # ensure user_id and send_anonymous_usage_stats are set correctly assert metadata.user_id == mock_user.id assert not metadata.send_anonymous_usage_stats @pytest.fixture def used_fqns(self) -> Dict[str, Any]: return {"models": frozenset((("my_test_project", "foo", "bar"),))} def test_warn_for_unused_resource_config_paths( self, runtime_config: RuntimeConfig, used_fqns: Dict[str, Any], ): catcher = EventCatcher(event_to_catch=UnusedResourceConfigPath) add_callback_to_manager(catcher.catch) runtime_config.models = { "my_test_project": { "foo": { "materialized": "view", "bar": { "materialized": "table", }, "baz": { "materialized": "table", }, } } } runtime_config.warn_for_unused_resource_config_paths(used_fqns, []) len(catcher.caught_events) == 1 expected_msg = "models.my_test_project.foo.baz" assert expected_msg in str(catcher.caught_events[0].data) def test_warn_for_unused_resource_config_paths_empty_models( self, runtime_config: RuntimeConfig, used_fqns: Dict[str, Any], ) -> None: catcher = EventCatcher(event_to_catch=UnusedResourceConfigPath) add_callback_to_manager(catcher.catch) # models should already be empty, but lets ensure it runtime_config.models = {} runtime_config.warn_for_unused_resource_config_paths(used_fqns, ()) assert len(catcher.caught_events) == 0 class TestRuntimeConfigFiles(BaseConfigTest): def test_from_args(self): with temp_cd(self.project_dir): config = dbt.config.RuntimeConfig.from_args(self.args) self.assertEqual(config.version, "0.0.1") self.assertEqual(config.profile_name, "default") # on osx, for example, these are not necessarily equal due to /private self.assertTrue(os.path.samefile(config.project_root, self.project_dir)) self.assertEqual(config.model_paths, ["models"]) self.assertEqual(config.macro_paths, ["macros"]) self.assertEqual(config.seed_paths, ["seeds"]) self.assertEqual(config.test_paths, ["tests"]) self.assertEqual(config.analysis_paths, ["analyses"]) self.assertEqual( set(config.docs_paths), {"models", "seeds", "snapshots", "analyses", "macros", "tests", "functions"}, ) self.assertEqual(config.asset_paths, []) self.assertEqual(config.target_path, "target") self.assertEqual(config.clean_targets, ["target"]) self.assertEqual(config.log_path, "logs") self.assertEqual(config.packages_install_path, "dbt_packages") self.assertEqual(config.quoting, {"database": True, "identifier": True, "schema": True}) self.assertEqual(config.models, {}) self.assertEqual(config.on_run_start, []) self.assertEqual(config.on_run_end, []) self.assertEqual(config.seeds, {}) self.assertEqual(config.packages, PackageConfig(packages=[])) self.assertEqual(config.project_name, "my_test_project") class TestVariableRuntimeConfigFiles(BaseConfigTest): def setUp(self): super().setUp() self.default_project_data.update( { "version": "{{ var('cli_version') }}", "name": "blah", "profile": "{{ env_var('env_value_profile') }}", "on-run-end": [ "{{ env_var('env_value_profile') }}", ], "models": { "foo": { "post-hook": "{{ env_var('env_value_profile') }}", }, "bar": { # just gibberish, make sure it gets interpreted "materialized": "{{ env_var('env_value_profile') }}", }, }, "seeds": { "foo": { "post-hook": "{{ env_var('env_value_profile') }}", }, "bar": { # just gibberish, make sure it gets interpreted "materialized": "{{ env_var('env_value_profile') }}", }, }, } ) self.write_project(self.default_project_data) def test_cli_and_env_vars(self): self.args.target = "cli-and-env-vars" self.args.vars = {"cli_value_host": "cli-postgres-host", "cli_version": "0.1.2"} self.args.project_dir = self.project_dir set_from_args(self.args, None) with mock.patch.dict(os.environ, self.env_override): safe_set_invocation_context() # reset invocation context with new env config = dbt.config.RuntimeConfig.from_args(self.args) self.assertEqual(config.version, "0.1.2") self.assertEqual(config.project_name, "blah") self.assertEqual(config.profile_name, "default") self.assertEqual(config.credentials.host, "cli-postgres-host") self.assertEqual(config.credentials.user, "env-postgres-user") # make sure hooks are not interpreted self.assertEqual(config.on_run_end, ["{{ env_var('env_value_profile') }}"]) self.assertEqual(config.models["foo"]["post-hook"], "{{ env_var('env_value_profile') }}") self.assertEqual(config.models["bar"]["materialized"], "default") # rendered! self.assertEqual(config.seeds["foo"]["post-hook"], "{{ env_var('env_value_profile') }}") self.assertEqual(config.seeds["bar"]["materialized"], "default") # rendered! ================================================ FILE: tests/unit/config/test_selectors.py ================================================ import textwrap import unittest import yaml import dbt.exceptions from dbt.config.selectors import SelectorConfig, SelectorDict, selector_config_from_data def get_selector_dict(txt: str) -> dict: txt = textwrap.dedent(txt) dct = yaml.safe_load(txt) return dct class SelectorUnitTest(unittest.TestCase): def test_parse_multiple_excludes(self): dct = get_selector_dict( """\ selectors: - name: mult_excl definition: union: - method: tag value: nightly - exclude: - method: tag value: hourly - exclude: - method: tag value: daily """ ) with self.assertRaisesRegex( dbt.exceptions.DbtSelectorsError, "cannot provide multiple exclude arguments" ): selector_config_from_data(dct) def test_parse_set_op_plus(self): dct = get_selector_dict( """\ selectors: - name: union_plus definition: - union: - method: tag value: nightly - exclude: - method: tag value: hourly - method: tag value: foo """ ) with self.assertRaisesRegex( dbt.exceptions.DbtSelectorsError, "Valid root-level selector definitions" ): selector_config_from_data(dct) def test_parse_multiple_methods(self): dct = get_selector_dict( """\ selectors: - name: mult_methods definition: - tag:hourly - tag:nightly - fqn:start """ ) with self.assertRaisesRegex( dbt.exceptions.DbtSelectorsError, "Valid root-level selector definitions" ): selector_config_from_data(dct) def test_parse_set_with_method(self): dct = get_selector_dict( """\ selectors: - name: mixed_syntaxes definition: key: value method: tag value: foo union: - method: tag value: m1234 - exclude: - method: tag value: m5678 """ ) with self.assertRaisesRegex( dbt.exceptions.DbtSelectorsError, "Only a single 'union' or 'intersection' key is allowed", ): selector_config_from_data(dct) def test_complex_sector(self): dct = get_selector_dict( """\ selectors: - name: nightly_diet_snowplow definition: union: - intersection: - method: source value: snowplow childrens_parents: true - method: tag value: nightly - method: path value: models/export - exclude: - intersection: - method: package value: snowplow - method: config.materialized value: incremental - method: fqn value: export_performance_timing """ ) selectors = selector_config_from_data(dct) assert isinstance(selectors, SelectorConfig) def test_exclude_not_list(self): dct = get_selector_dict( """\ selectors: - name: summa_exclude definition: union: - method: tag value: nightly - exclude: method: tag value: daily """ ) with self.assertRaisesRegex(dbt.exceptions.DbtSelectorsError, "Expected a list"): selector_config_from_data(dct) def test_invalid_key(self): dct = get_selector_dict( """\ selectors: - name: summa_nothing definition: method: tag key: nightly """ ) with self.assertRaisesRegex(dbt.exceptions.DbtSelectorsError, "Expected either 1 key"): selector_config_from_data(dct) def test_invalid_single_def(self): dct = get_selector_dict( """\ selectors: - name: summa_nothing definition: fubar: tag """ ) with self.assertRaisesRegex(dbt.exceptions.DbtSelectorsError, "not a valid method name"): selector_config_from_data(dct) def test_method_no_value(self): dct = get_selector_dict( """\ selectors: - name: summa_nothing definition: method: tag """ ) with self.assertRaisesRegex(dbt.exceptions.DbtSelectorsError, "not a valid method name"): selector_config_from_data(dct) def test_multiple_default_true(self): """Test selector_config_from_data returns the correct error when multiple default values are set """ dct = get_selector_dict( """\ selectors: - name: summa_nothing definition: method: tag value: nightly default: true - name: summa_something definition: method: tag value: daily default: true """ ) with self.assertRaisesRegex( dbt.exceptions.DbtSelectorsError, "Found multiple selectors with `default: true`:" ): selector_config_from_data(dct) def test_compare_cli_non_cli(self): dct = get_selector_dict( """\ selectors: - name: nightly_diet_snowplow description: "This uses more CLI-style syntax" definition: union: - intersection: - '@source:snowplow' - 'tag:nightly' - 'models/export' - exclude: - intersection: - 'package:snowplow' - 'config.materialized:incremental' - export_performance_timing - name: nightly_diet_snowplow_full description: "This is a fuller YAML specification" definition: union: - intersection: - method: source value: snowplow childrens_parents: true - method: tag value: nightly - method: path value: models/export - exclude: - intersection: - method: package value: snowplow - method: config.materialized value: incremental - method: fqn value: export_performance_timing """ ) sel_dict = SelectorDict.parse_from_selectors_list(dct["selectors"]) assert sel_dict with_strings = sel_dict["nightly_diet_snowplow"]["definition"] no_strings = sel_dict["nightly_diet_snowplow_full"]["definition"] self.assertEqual(with_strings, no_strings) def test_single_string_definition(self): dct = get_selector_dict( """\ selectors: - name: nightly_selector definition: 'tag:nightly' """ ) sel_dict = SelectorDict.parse_from_selectors_list(dct["selectors"]) assert sel_dict expected = {"method": "tag", "value": "nightly"} definition = sel_dict["nightly_selector"]["definition"] self.assertEqual(expected, definition) def test_single_key_value_definition(self): dct = get_selector_dict( """\ selectors: - name: nightly_selector definition: tag: nightly """ ) sel_dict = SelectorDict.parse_from_selectors_list(dct["selectors"]) assert sel_dict expected = {"method": "tag", "value": "nightly"} definition = sel_dict["nightly_selector"]["definition"] self.assertEqual(expected, definition) def test_parent_definition(self): dct = get_selector_dict( """\ selectors: - name: kpi_nightly_selector definition: '+exposure:kpi_nightly' """ ) sel_dict = SelectorDict.parse_from_selectors_list(dct["selectors"]) assert sel_dict expected = {"method": "exposure", "value": "kpi_nightly", "parents": True} definition = sel_dict["kpi_nightly_selector"]["definition"] self.assertEqual(expected, definition) def test_plus_definition(self): dct = get_selector_dict( """\ selectors: - name: my_model_children_selector definition: 'my_model+2' """ ) sel_dict = SelectorDict.parse_from_selectors_list(dct["selectors"]) assert sel_dict expected = {"method": "fqn", "value": "my_model", "children": True, "children_depth": "2"} definition = sel_dict["my_model_children_selector"]["definition"] self.assertEqual(expected, definition) ================================================ FILE: tests/unit/config/test_utils.py ================================================ import pytest from dbt.config.utils import ( exclusive_primary_alt_value_setting, normalize_warn_error_options, ) from dbt.exceptions import DbtExclusivePropertyUseError class TestExclusivePrimaryAltValueSetting: @pytest.fixture(scope="class") def primary_key(self) -> str: return "key_a" @pytest.fixture(scope="class") def alt_key(self) -> str: return "key_b" @pytest.fixture(scope="class") def value(self) -> str: return "I LIKE CATS" def test_primary_set(self, primary_key: str, alt_key: str, value: str): test_dict = {primary_key: value} exclusive_primary_alt_value_setting(test_dict, primary_key, alt_key) assert test_dict.get(primary_key) == value assert test_dict.get(alt_key) is None def test_alt_set(self, primary_key: str, alt_key: str, value: str): test_dict = {alt_key: value} exclusive_primary_alt_value_setting(test_dict, primary_key, alt_key) assert test_dict.get(primary_key) == value def test_primary_and_alt_set(self, primary_key: str, alt_key: str, value: str): test_dict = {primary_key: value, alt_key: value} with pytest.raises(DbtExclusivePropertyUseError): exclusive_primary_alt_value_setting(test_dict, primary_key, alt_key) def test_neither_primary_nor_alt_set(self, primary_key: str, alt_key: str): test_dict = {} exclusive_primary_alt_value_setting(test_dict, primary_key, alt_key) assert test_dict.get(primary_key) is None assert test_dict.get(alt_key) is None class TestNormalizeWarnErrorOptions: def test_primary_set(self): test_dict = { "include": ["SomeWarning"], } normalize_warn_error_options(test_dict) assert len(test_dict) == 1 assert test_dict["error"] == ["SomeWarning"] def test_convert(self): test_dict = {"exclude": None, "silence": None, "error": ["SomeWarning"]} normalize_warn_error_options(test_dict) assert test_dict["warn"] == [] assert test_dict["error"] == ["SomeWarning"] assert test_dict["silence"] == [] def test_both_keys_set(self): test_dict = { "warn": ["SomeWarning"], "exclude": ["SomeWarning"], } with pytest.raises(DbtExclusivePropertyUseError): normalize_warn_error_options(test_dict) def test_empty_dict(self): test_dict = {} normalize_warn_error_options(test_dict) assert test_dict.get("error") is None assert test_dict.get("warn") is None ================================================ FILE: tests/unit/config/test_vars_file.py ================================================ from typing import Any, Dict import pytest import yaml from dbt.config.project import ( VarProvider, validate_vars_not_in_both, vars_data_from_root, ) from dbt.constants import VARS_FILE_NAME from dbt.exceptions import DbtProjectError class TestVarsDataFromRoot: """Tests for vars_data_from_root function.""" def test_returns_empty_dict_when_file_missing(self, tmp_path) -> None: """Should return empty dict when vars.yml doesn't exist.""" result = vars_data_from_root(str(tmp_path)) assert result == {} def test_returns_empty_dict_when_file_empty(self, tmp_path) -> None: """Should return empty dict when vars.yml is empty.""" vars_path = tmp_path / VARS_FILE_NAME vars_path.write_text("") result = vars_data_from_root(str(tmp_path)) assert result == {} def test_returns_empty_dict_when_no_vars_key(self, tmp_path) -> None: """Should return empty dict when vars.yml has no 'vars' key.""" vars_path = tmp_path / VARS_FILE_NAME vars_path.write_text("other_key: value\n") result = vars_data_from_root(str(tmp_path)) assert result == {} def test_returns_vars_from_file(self, tmp_path) -> None: """Should return contents of 'vars' key from vars.yml.""" vars_path = tmp_path / VARS_FILE_NAME vars_data = { "vars": { "my_var": "my_value", "another_var": 123, } } with open(vars_path, "w") as f: yaml.dump(vars_data, f) result = vars_data_from_root(str(tmp_path)) assert result == {"my_var": "my_value", "another_var": 123} def test_returns_package_scoped_vars(self, tmp_path) -> None: """Should support package-scoped vars.""" vars_path = tmp_path / VARS_FILE_NAME vars_data = { "vars": { "global_var": "global_value", "my_package": { "package_var": "package_value", }, } } with open(vars_path, "w") as f: yaml.dump(vars_data, f) result = vars_data_from_root(str(tmp_path)) assert result == { "global_var": "global_value", "my_package": {"package_var": "package_value"}, } class TestValidateVarsNotInBoth: """Tests for validate_vars_not_in_both function.""" def test_no_error_when_no_vars_file_and_no_project_vars(self) -> None: """Should not raise when neither vars.yml nor dbt_project.yml have vars.""" project_dict: Dict[str, Any] = {"name": "test_project"} validate_vars_not_in_both(project_dict, has_vars_file=False) def test_no_error_when_only_vars_file_has_vars(self) -> None: """Should not raise when only vars.yml has vars.""" project_dict: Dict[str, Any] = {"name": "test_project"} validate_vars_not_in_both(project_dict, has_vars_file=True) def test_no_error_when_only_project_has_vars(self) -> None: """Should not raise when only dbt_project.yml has vars.""" project_dict: Dict[str, Any] = { "name": "test_project", "vars": {"my_var": "my_value"}, } validate_vars_not_in_both(project_dict, has_vars_file=False) def test_error_when_both_have_vars(self) -> None: """Should raise DbtProjectError when both sources have vars.""" project_dict: Dict[str, Any] = { "name": "test_project", "vars": {"my_var": "my_value"}, } with pytest.raises(DbtProjectError) as exc_info: validate_vars_not_in_both(project_dict, has_vars_file=True) assert "vars.yml" in str(exc_info.value) assert "dbt_project.yml" in str(exc_info.value) def test_no_error_when_project_vars_is_empty(self) -> None: """Should not raise when dbt_project.yml vars is empty dict.""" project_dict: Dict[str, Any] = { "name": "test_project", "vars": {}, } # Empty vars dict is falsy, so this should not raise validate_vars_not_in_both(project_dict, has_vars_file=True) def test_no_error_when_project_vars_is_none(self) -> None: """Should not raise when dbt_project.yml vars is None.""" project_dict: Dict[str, Any] = { "name": "test_project", "vars": None, } validate_vars_not_in_both(project_dict, has_vars_file=True) class TestVarProviderWithVarsFromFile: """Tests for VarProvider with vars from file.""" def test_vars_from_file_only(self) -> None: """VarProvider should work with vars from file.""" vars_dict = {"my_var": "my_value", "another_var": 123} provider = VarProvider(vars_dict) assert provider.to_dict() == vars_dict def test_vars_from_file_with_package_scoped(self) -> None: """VarProvider should handle package-scoped vars from file.""" vars_dict = { "global_var": "global_value", "my_package": {"package_var": "package_value"}, } provider = VarProvider(vars_dict) assert provider.to_dict() == vars_dict ================================================ FILE: tests/unit/conftest.py ================================================ import pytest from dbt.artifacts.resources import Quoting, SourceConfig from dbt.artifacts.resources.types import NodeType from dbt.contracts.graph.nodes import SourceDefinition # All manifest related fixtures. from tests.unit.utils.adapter import * # noqa from tests.unit.utils.config import * # noqa from tests.unit.utils.event_manager import * # noqa from tests.unit.utils.flags import * # noqa from tests.unit.utils.manifest import * # noqa from tests.unit.utils.project import * # noqa @pytest.fixture def basic_parsed_source_definition_object(): return SourceDefinition( columns={}, database="some_db", description="", fqn=["test", "source", "my_source", "my_source_table"], identifier="my_source_table", loader="stitch", name="my_source_table", original_file_path="/root/models/sources.yml", package_name="test", path="/root/models/sources.yml", quoting=Quoting(), resource_type=NodeType.Source, schema="some_schema", source_description="my source description", source_name="my_source", unique_id="test.source.my_source.my_source_table", tags=[], config=SourceConfig(), ) ================================================ FILE: tests/unit/context/__init__.py ================================================ ================================================ FILE: tests/unit/context/test_base.py ================================================ import os from jinja2.runtime import Undefined from dbt.context.base import BaseContext class TestBaseContext: def test_log_jinja_undefined(self): # regression test for CT-2259 try: os.environ["DBT_ENV_SECRET_LOG_TEST"] = "cats_are_cool" BaseContext.log(msg=Undefined(), info=True) except Exception as e: assert False, f"Logging an jinja2.Undefined object raises an exception: {e}" def test_log_with_dbt_env_secret(self): # regression test for CT-1783 try: os.environ["DBT_ENV_SECRET_LOG_TEST"] = "cats_are_cool" BaseContext.log({"fact1": "I like cats"}, info=True) except Exception as e: assert False, f"Logging while a `DBT_ENV_SECRET` was set raised an exception: {e}" def test_flags(self): expected_context_flags = { "use_experimental_parser", "static_parser", "warn_error", "warn_error_options", "write_json", "partial_parse", "use_colors", "profiles_dir", "debug", "log_format", "version_check", "fail_fast", "send_anonymous_usage_stats", "printer_width", "indirect_selection", "log_cache_events", "quiet", "no_print", "cache_selected_only", "introspect", "target_path", "log_path", "invocation_command", "empty", } flags = BaseContext(cli_vars={}).flags for expected_flag in expected_context_flags: assert hasattr(flags, expected_flag.upper()) ================================================ FILE: tests/unit/context/test_context.py ================================================ import os from typing import Any, Dict, Set from unittest import mock import pytest import dbt_common.exceptions from dbt.adapters import factory, postgres from dbt.clients.jinja import MacroStack from dbt.config.project import VarProvider from dbt.context import base, docs, macros, providers, query_header from dbt.contracts.files import FileHash from dbt.contracts.graph.nodes import ( DependsOn, Macro, ModelNode, NodeConfig, UnitTestNode, UnitTestOverrides, ) from dbt.node_types import NodeType from dbt_common.events.functions import reset_metadata_vars from tests.unit.mock_adapter import adapter_factory from tests.unit.utils import clear_plugin, config_from_parts_or_dicts, inject_adapter class TestVar: @pytest.fixture def model(self): return ModelNode( alias="model_one", name="model_one", database="dbt", schema="analytics", resource_type=NodeType.Model, unique_id="model.root.model_one", fqn=["root", "model_one"], package_name="root", original_file_path="model_one.sql", refs=[], sources=[], depends_on=DependsOn(), config=NodeConfig.from_dict( { "enabled": True, "materialized": "view", "persist_docs": {}, "post-hook": [], "pre-hook": [], "vars": {}, "quoting": {}, "column_types": {}, "tags": [], } ), tags=[], path="model_one.sql", language="sql", raw_code="", description="", columns={}, checksum=FileHash.from_contents(""), ) @pytest.fixture def context(self): return mock.MagicMock() @pytest.fixture def provider(self): return VarProvider({}) @pytest.fixture def config(self, provider): return mock.MagicMock(config_version=2, vars=provider, cli_vars={}, project_name="root") def test_var_default_something(self, model, config, context): config.cli_vars = {"foo": "baz"} var = providers.RuntimeVar(context, config, model) assert var("foo") == "baz" assert var("foo", "bar") == "baz" def test_var_default_none(self, model, config, context): config.cli_vars = {"foo": None} var = providers.RuntimeVar(context, config, model) assert var("foo") is None assert var("foo", "bar") is None def test_var_not_defined(self, model, config, context): var = providers.RuntimeVar(self.context, config, model) assert var("foo", "bar") == "bar" with pytest.raises(dbt_common.exceptions.CompilationError): var("foo") def test_parser_var_default_something(self, model, config, context): config.cli_vars = {"foo": "baz"} var = providers.ParseVar(context, config, model) assert var("foo") == "baz" assert var("foo", "bar") == "baz" def test_parser_var_default_none(self, model, config, context): config.cli_vars = {"foo": None} var = providers.ParseVar(context, config, model) assert var("foo") is None assert var("foo", "bar") is None def test_parser_var_not_defined(self, model, config, context): # at parse-time, we should not raise if we encounter a missing var # that way disabled models don't get parse errors var = providers.ParseVar(context, config, model) assert var("foo", "bar") == "bar" assert var("foo") is None class TestParseWrapper: @pytest.fixture def mock_adapter(self): mock_config = mock.MagicMock() mock_mp_context = mock.MagicMock() adapter_class = adapter_factory() return adapter_class(mock_config, mock_mp_context) @pytest.fixture def wrapper(self, mock_adapter): namespace = mock.MagicMock() return providers.ParseDatabaseWrapper(mock_adapter, namespace) @pytest.fixture def responder(self, mock_adapter): return mock_adapter.responder def test_unwrapped_method(self, wrapper, responder): assert wrapper.quote("test_value") == '"test_value"' responder.quote.assert_called_once_with("test_value") def test_wrapped_method(self, wrapper, responder): found = wrapper.get_relation("database", "schema", "identifier") assert found is None responder.get_relation.assert_not_called() class TestRuntimeWrapper: @pytest.fixture def mock_adapter(self): mock_config = mock.MagicMock() mock_config.quoting = { "database": True, "schema": True, "identifier": True, } mock_mp_context = mock.MagicMock() adapter_class = adapter_factory() return adapter_class(mock_config, mock_mp_context) @pytest.fixture def wrapper(self, mock_adapter): namespace = mock.MagicMock() return providers.RuntimeDatabaseWrapper(mock_adapter, namespace) @pytest.fixture def responder(self, mock_adapter): return mock_adapter.responder def test_unwrapped_method(self, wrapper, responder): # the 'quote' method isn't wrapped, we should get our expected inputs assert wrapper.quote("test_value") == '"test_value"' responder.quote.assert_called_once_with("test_value") def assert_has_keys(required_keys: Set[str], maybe_keys: Set[str], ctx: Dict[str, Any]): keys = set(ctx) for key in required_keys: assert key in keys, f"{key} in required keys but not in context" keys.remove(key) extras = keys.difference(maybe_keys) assert not extras, f"got extra keys in context: {extras}" REQUIRED_BASE_KEYS = frozenset( { "context", "builtins", "dbt_version", "var", "env_var", "return", "fromjson", "tojson", "fromyaml", "toyaml", "set", "set_strict", "zip", "zip_strict", "log", "run_started_at", "invocation_id", "thread_id", "modules", "flags", "print", "diff_of_two_dicts", "local_md5", } ) REQUIRED_TARGET_KEYS = REQUIRED_BASE_KEYS | {"target"} REQUIRED_DOCS_KEYS = REQUIRED_TARGET_KEYS | {"project_name"} | {"doc"} MACROS = frozenset({"macro_a", "macro_b", "root", "dbt"}) REQUIRED_QUERY_HEADER_KEYS = ( REQUIRED_TARGET_KEYS | {"project_name", "context_macro_stack"} | MACROS ) REQUIRED_MACRO_KEYS = REQUIRED_QUERY_HEADER_KEYS | { "_sql_results", "load_result", "store_result", "store_raw_result", "validation", "write", "render", "try_or_compiler_error", "load_agate_table", "ref", "source", "metric", "config", "execute", "exceptions", "database", "schema", "adapter", "api", "column", "env", "graph", "model", "pre_hooks", "post_hooks", "sql", "sql_now", "adapter_macro", "selected_resources", "invocation_args_dict", "submit_python_job", "dbt_metadata_envs", "function", } REQUIRED_MODEL_KEYS = REQUIRED_MACRO_KEYS | {"this", "compiled_code"} MAYBE_KEYS = frozenset({"debug", "defer_relation"}) POSTGRES_PROFILE_DATA = { "target": "test", "quoting": {}, "outputs": { "test": { "type": "postgres", "host": "localhost", "schema": "analytics", "user": "test", "pass": "test", "dbname": "test", "port": 1, } }, } PROJECT_DATA = { "name": "root", "version": "0.1", "profile": "test", "project-root": os.getcwd(), "config-version": 2, } def model(): return ModelNode( alias="model_one", name="model_one", database="dbt", schema="analytics", resource_type=NodeType.Model, unique_id="model.root.model_one", fqn=["root", "model_one"], package_name="root", original_file_path="model_one.sql", refs=[], sources=[], depends_on=DependsOn(), config=NodeConfig.from_dict( { "enabled": True, "materialized": "view", "persist_docs": {}, "post-hook": [], "pre-hook": [], "vars": {}, "quoting": {}, "column_types": {}, "tags": [], } ), tags=[], path="model_one.sql", language="sql", raw_code="", description="", columns={}, ) def test_base_context(): ctx = base.generate_base_context({}) assert_has_keys(REQUIRED_BASE_KEYS, MAYBE_KEYS, ctx) def mock_macro(name, package_name): macro = mock.MagicMock( __class__=Macro, package_name=package_name, resource_type="macro", unique_id=f"macro.{package_name}.{name}", ) # Mock(name=...) does not set the `name` attribute, this does. macro.name = name return macro def mock_manifest(config, additional_macros=None): default_macro_names = ["macro_a", "macro_b"] default_macros = [mock_macro(name, config.project_name) for name in default_macro_names] additional_macros = additional_macros or [] all_macros = default_macros + additional_macros manifest_macros = {} macros_by_package = {} for macro in all_macros: manifest_macros[macro.unique_id] = macro if macro.package_name not in macros_by_package: macros_by_package[macro.package_name] = {} macro_package = macros_by_package[macro.package_name] macro_package[macro.name] = macro def gmbp(): return macros_by_package m = mock.MagicMock(macros=manifest_macros) m.get_macros_by_package = gmbp return m def mock_model(): return mock.MagicMock( __class__=ModelNode, alias="model_one", name="model_one", database="dbt", schema="analytics", resource_type=NodeType.Model, unique_id="model.root.model_one", fqn=["root", "model_one"], package_name="root", original_file_path="model_one.sql", refs=[], sources=[], depends_on=DependsOn(), config=NodeConfig.from_dict( { "enabled": True, "materialized": "view", "persist_docs": {}, "post-hook": [], "pre-hook": [], "vars": {}, "quoting": {}, "column_types": {}, "tags": [], } ), tags=[], path="model_one.sql", language="sql", raw_code="", description="", columns={}, ) def mock_unit_test_node(): return mock.MagicMock( __class__=UnitTestNode, resource_type=NodeType.Unit, tested_node_unique_id="model.root.model_one", ) @pytest.fixture def get_adapter(): with mock.patch.object(providers, "get_adapter") as patch: yield patch @pytest.fixture def get_include_paths(): with mock.patch.object(factory, "get_include_paths") as patch: patch.return_value = [] yield patch @pytest.fixture def config_postgres(): return config_from_parts_or_dicts(PROJECT_DATA, POSTGRES_PROFILE_DATA) @pytest.fixture def manifest_fx(config_postgres): return mock_manifest(config_postgres) @pytest.fixture def postgres_adapter(config_postgres, get_adapter): adapter = postgres.PostgresAdapter(config_postgres) inject_adapter(adapter, postgres.Plugin) get_adapter.return_value = adapter yield adapter clear_plugin(postgres.Plugin) def test_query_header_context(config_postgres, manifest_fx): ctx = query_header.generate_query_header_context( config=config_postgres, manifest=manifest_fx, ) assert_has_keys(REQUIRED_QUERY_HEADER_KEYS, MAYBE_KEYS, ctx) def test_macro_runtime_context(config_postgres, manifest_fx, get_adapter, get_include_paths): ctx = providers.generate_runtime_macro_context( macro=manifest_fx.macros["macro.root.macro_a"], config=config_postgres, manifest=manifest_fx, package_name="root", ) assert_has_keys(REQUIRED_MACRO_KEYS, MAYBE_KEYS, ctx) def test_invocation_args_to_dict_in_macro_runtime_context( config_postgres, manifest_fx, get_adapter, get_include_paths ): ctx = providers.generate_runtime_macro_context( macro=manifest_fx.macros["macro.root.macro_a"], config=config_postgres, manifest=manifest_fx, package_name="root", ) # Comes from dbt/flags.py as they are the only values set that aren't None at default assert ctx["invocation_args_dict"]["printer_width"] == 80 # Comes from unit/utils.py config_from_parts_or_dicts method assert ctx["invocation_args_dict"]["profile_dir"] == "/dev/null" assert isinstance(ctx["invocation_args_dict"]["warn_error_options"], Dict) assert ctx["invocation_args_dict"]["warn_error_options"] == { "error": [], "warn": [], "silence": [], } def test_model_parse_context(config_postgres, manifest_fx, get_adapter, get_include_paths): ctx = providers.generate_parser_model_context( model=mock_model(), config=config_postgres, manifest=manifest_fx, context_config=mock.MagicMock(), ) assert_has_keys(REQUIRED_MODEL_KEYS, MAYBE_KEYS, ctx) def test_model_runtime_context(config_postgres, manifest_fx, get_adapter, get_include_paths): ctx = providers.generate_runtime_model_context( model=mock_model(), config=config_postgres, manifest=manifest_fx, ) assert_has_keys(REQUIRED_MODEL_KEYS, MAYBE_KEYS, ctx) def test_docs_runtime_context(config_postgres): ctx = docs.generate_runtime_docs_context(config_postgres, mock_model(), [], "root") assert_has_keys(REQUIRED_DOCS_KEYS, MAYBE_KEYS, ctx) def test_macro_namespace_duplicates(config_postgres, manifest_fx): mn = macros.MacroNamespaceBuilder("root", "search", MacroStack(), ["dbt_postgres", "dbt"]) mn.add_macros(manifest_fx.macros.values(), {}) # same pkg, same name: error with pytest.raises(dbt_common.exceptions.CompilationError): mn.add_macro(mock_macro("macro_a", "root"), {}) # different pkg, same name: no error mn.add_macros(mock_macro("macro_a", "dbt"), {}) def test_macro_namespace(config_postgres, manifest_fx): mn = macros.MacroNamespaceBuilder("root", "search", MacroStack(), ["dbt_postgres", "dbt"]) mbp = manifest_fx.get_macros_by_package() dbt_macro = mock_macro("some_macro", "dbt") mbp["dbt"] = {"some_macro": dbt_macro} # same namespace, same name, different pkg! pg_macro = mock_macro("some_macro", "dbt_postgres") mbp["dbt_postgres"] = {"some_macro": pg_macro} # same name, different package package_macro = mock_macro("some_macro", "root") mbp["root"]["some_macro"] = package_macro namespace = mn.build_namespace(mbp, {}) dct = dict(namespace) for result in [dct, namespace]: assert "dbt" in result assert "root" in result assert "some_macro" in result assert "dbt_postgres" not in result # tests __len__ assert len(result) == 5 # tests __iter__ assert set(result) == {"dbt", "root", "some_macro", "macro_a", "macro_b"} assert len(result["dbt"]) == 1 # from the regular manifest + some_macro assert len(result["root"]) == 3 assert result["dbt"]["some_macro"].macro is pg_macro assert result["root"]["some_macro"].macro is package_macro assert result["some_macro"].macro is package_macro def test_dbt_metadata_envs( monkeypatch, config_postgres, manifest_fx, get_adapter, get_include_paths ): reset_metadata_vars() envs = { "DBT_ENV_CUSTOM_ENV_RUN_ID": 1234, "DBT_ENV_CUSTOM_ENV_JOB_ID": 5678, "DBT_ENV_RUN_ID": 91011, "RANDOM_ENV": 121314, } monkeypatch.setattr(os, "environ", envs) ctx = providers.generate_runtime_macro_context( macro=manifest_fx.macros["macro.root.macro_a"], config=config_postgres, manifest=manifest_fx, package_name="root", ) assert ctx["dbt_metadata_envs"] == {"JOB_ID": 5678, "RUN_ID": 1234} # cleanup reset_metadata_vars() def test_unit_test_runtime_context(config_postgres, manifest_fx, get_adapter, get_include_paths): ctx = providers.generate_runtime_unit_test_context( unit_test=mock_unit_test_node(), config=config_postgres, manifest=manifest_fx, ) assert_has_keys(REQUIRED_MODEL_KEYS, MAYBE_KEYS, ctx) def test_unit_test_runtime_context_macro_overrides_global( config_postgres, manifest_fx, get_adapter, get_include_paths ): unit_test = mock_unit_test_node() unit_test.overrides = UnitTestOverrides(macros={"macro_a": "override"}) ctx = providers.generate_runtime_unit_test_context( unit_test=unit_test, config=config_postgres, manifest=manifest_fx, ) assert ctx["macro_a"]() == "override" def test_unit_test_runtime_context_macro_overrides_package( config_postgres, manifest_fx, get_adapter, get_include_paths ): unit_test = mock_unit_test_node() unit_test.overrides = UnitTestOverrides(macros={"some_package.some_macro": "override"}) dbt_macro = mock_macro("some_macro", "some_package") manifest_with_dbt_macro = mock_manifest(config_postgres, additional_macros=[dbt_macro]) ctx = providers.generate_runtime_unit_test_context( unit_test=unit_test, config=config_postgres, manifest=manifest_with_dbt_macro, ) assert ctx["some_package"]["some_macro"]() == "override" @pytest.mark.parametrize( "overrides,expected_override_value", [ # override dbt macro at global level ({"some_macro": "override"}, "override"), # # override dbt macro at dbt-namespaced level level ({"dbt.some_macro": "override"}, "override"), # override dbt macro at both levels - global override should win ( {"some_macro": "dbt_global_override", "dbt.some_macro": "dbt_namespaced_override"}, "dbt_global_override", ), # override dbt macro at both levels - global override should win, regardless of order ( {"dbt.some_macro": "dbt_namespaced_override", "some_macro": "dbt_global_override"}, "dbt_global_override", ), ], ) def test_unit_test_runtime_context_macro_overrides_dbt_macro( overrides, expected_override_value, config_postgres, manifest_fx, get_adapter, get_include_paths, ): unit_test = mock_unit_test_node() unit_test.overrides = UnitTestOverrides(macros=overrides) dbt_macro = mock_macro("some_macro", "dbt") manifest_with_dbt_macro = mock_manifest(config_postgres, additional_macros=[dbt_macro]) ctx = providers.generate_runtime_unit_test_context( unit_test=unit_test, config=config_postgres, manifest=manifest_with_dbt_macro, ) assert ctx["some_macro"]() == expected_override_value assert ctx["dbt"]["some_macro"]() == expected_override_value ================================================ FILE: tests/unit/context/test_macro_resolver.py ================================================ import unittest from unittest import mock from dbt.context.macro_resolver import MacroResolver from dbt.contracts.graph.nodes import Macro def mock_macro(name, package_name): macro = mock.MagicMock( __class__=Macro, package_name=package_name, resource_type="macro", unique_id=f"macro.{package_name}.{name}", ) # Mock(name=...) does not set the `name` attribute, this does. macro.name = name return macro class TestMacroResolver(unittest.TestCase): def test_resolver(self): data = [ {"package_name": "my_test", "name": "unique"}, {"package_name": "my_test", "name": "macro_xx"}, {"package_name": "one", "name": "unique"}, {"package_name": "one", "name": "not_null"}, {"package_name": "two", "name": "macro_a"}, {"package_name": "two", "name": "macro_b"}, ] macros = {} for mdata in data: macro = mock_macro(mdata["name"], mdata["package_name"]) macros[macro.unique_id] = macro resolver = MacroResolver(macros, "my_test", ["one"]) assert resolver self.assertEqual(resolver.get_macro_id("one", "not_null"), "macro.one.not_null") ================================================ FILE: tests/unit/context/test_providers.py ================================================ from argparse import Namespace from datetime import datetime from typing import Any, Optional, Type, Union from unittest import mock import pytest import pytz from dbt.adapters.base import BaseRelation from dbt.artifacts.resources import NodeConfig, Quoting, SeedConfig, SnapshotConfig from dbt.artifacts.resources.types import BatchSize from dbt.context.providers import ( BaseResolver, EventTimeFilter, RuntimeRefResolver, RuntimeSourceResolver, ) from dbt.contracts.graph.nodes import BatchContext, ModelNode, SnapshotNode from dbt.event_time.sample_window import SampleWindow from dbt.flags import set_from_args class TestBaseResolver: class ResolverSubclass(BaseResolver): def __call__(self, *args: str): pass @pytest.fixture def resolver(self): return self.ResolverSubclass( db_wrapper=mock.Mock(), model=mock.Mock(), config=mock.Mock(), manifest=mock.Mock(), ) @pytest.mark.parametrize( "empty,expected_resolve_limit", [(False, None), (True, 0)], ) def test_resolve_limit(self, resolver, empty, expected_resolve_limit): resolver.config.args.EMPTY = empty assert resolver.resolve_limit == expected_resolve_limit @pytest.mark.parametrize( "event_time_column,column_quote,source_quote,expected_field_name", [ # Simple column name, no quoting needed ("simple_column", None, None, "simple_column"), ("no_quote_column", False, None, "no_quote_column"), ("source_no_quote", None, False, "source_no_quote"), ("both_no_quote", False, False, "both_no_quote"), # Column-level quote configuration (takes precedence) ("column_quoted", True, None, '"column_quoted"'), ("column_quoted_source_no", True, False, '"column_quoted_source_no"'), ("column_quoted_source_yes", True, True, '"column_quoted_source_yes"'), # Source-level quote configuration (fallback when column-level is None) ("source_quoted", None, True, '"source_quoted"'), ( "source_quoted_override", False, True, "source_quoted_override", ), # False overrides True # Camel case and spaced column names ("camelCaseColumn", None, None, "camelCaseColumn"), ("camelCaseQuoted", True, None, '"camelCaseQuoted"'), ("snake_case_column", None, None, "snake_case_column"), ("snake_case_quoted", True, None, '"snake_case_quoted"'), ("Spaced Column Name", None, None, "Spaced Column Name"), ("Spaced Column Quoted", True, None, '"Spaced Column Quoted"'), ("Spaced Column Source Quoted", None, True, '"Spaced Column Source Quoted"'), # Edge cases ("", None, None, ""), ("", True, None, '""'), ("edge_case_column", None, None, "edge_case_column"), ("edge_case_quoted", True, None, '"edge_case_quoted"'), ], ) def test_resolve_event_time_field_name( self, resolver, event_time_column, column_quote, source_quote, expected_field_name ): """Test the _resolve_event_time_field_name method with various quoting configurations.""" # Create a mock target with columns target = mock.Mock() target.config = mock.Mock() target.config.event_time = event_time_column # Mock columns dictionary mock_column = mock.Mock() mock_column.name = event_time_column mock_column.data_type = "timestamp" # Set column-level quote configuration if column_quote is not None: mock_column.quote = column_quote else: # Explicitly set to None to avoid Mock object being returned mock_column.quote = None target.columns = {event_time_column: mock_column} # Set source-level quote configuration if source_quote is not None: target.quoting = mock.Mock() target.quoting.column = source_quote else: # Explicitly set to None to avoid Mock object being returned target.quoting = mock.Mock() target.quoting.column = None # Call the method result = resolver._resolve_event_time_field_name(target) # Assert the result assert result == expected_field_name @pytest.mark.parametrize( "event_time_column,column_quote,source_quote,expected_field_name", [ # Column not found in columns dict - should fall back to source-level quoting ("missing_column", None, None, "missing_column"), ("missing_column_source_quoted", None, True, '"missing_column_source_quoted"'), ("missing_column_source_no_quote", None, False, "missing_column_source_no_quote"), # Column found but no quote attribute - should fall back to source-level quoting ("found_no_quote_attr", None, None, "found_no_quote_attr"), ( "found_no_quote_attr_source_quoted", None, True, '"found_no_quote_attr_source_quoted"', ), ( "found_no_quote_attr_source_no_quote", None, False, "found_no_quote_attr_source_no_quote", ), ], ) def test_resolve_event_time_field_name_column_not_found( self, resolver, event_time_column, column_quote, source_quote, expected_field_name ): """Test _resolve_event_time_field_name when column is not found or has no quote attribute.""" # Create a mock target with different columns target = mock.Mock() target.config = mock.Mock() target.config.event_time = event_time_column # Mock columns dictionary with different column mock_column = mock.Mock() mock_column.name = "different_column_name" mock_column.data_type = "timestamp" # Set column-level quote configuration (but for different column) if column_quote is not None: mock_column.quote = column_quote else: # Explicitly set to None to avoid Mock object being returned mock_column.quote = None target.columns = {"different_column_name": mock_column} # Set source-level quote configuration if source_quote is not None: target.quoting = mock.Mock() target.quoting.column = source_quote else: # Explicitly set to None to avoid Mock object being returned target.quoting = mock.Mock() target.quoting.column = None # Call the method result = resolver._resolve_event_time_field_name(target) # Assert the result assert result == expected_field_name def test_resolve_event_time_field_name_no_columns(self, resolver): """Test _resolve_event_time_field_name when target has no columns attribute.""" # Create a mock target without columns target = mock.Mock() target.config = mock.Mock() target.config.event_time = "no_columns_column" # No columns attribute target.columns = {} # Set source-level quote configuration target.quoting = mock.Mock() target.quoting.column = True # Call the method result = resolver._resolve_event_time_field_name(target) # Should return quoted column name when source-level quoting is True assert result == '"no_columns_column"' def test_resolve_event_time_field_name_no_quoting_attribute(self, resolver): """Test _resolve_event_time_field_name when target has no quoting attribute.""" # Create a mock target without quoting target = mock.Mock() target.config = mock.Mock() target.config.event_time = "no_quoting_attr_column" # Mock columns dictionary mock_column = mock.Mock() mock_column.name = "no_quoting_attr_column" mock_column.data_type = "timestamp" # No quote attribute - explicitly set to None mock_column.quote = None target.columns = {"no_quoting_attr_column": mock_column} # No quoting attribute - explicitly set to None target.quoting = mock.Mock() target.quoting.column = None # Call the method result = resolver._resolve_event_time_field_name(target) # Should return unquoted column name assert result == "no_quoting_attr_column" @pytest.mark.parametrize( "use_microbatch_batches,materialized,incremental_strategy,sample,resolver_model_node,target_type,resolver_model_type,expect_filter", [ # Microbatch model without sample ( True, "incremental", "microbatch", None, True, NodeConfig, ModelNode, True, ), # Microbatch model with sample ( True, "incremental", "microbatch", SampleWindow( start=datetime(2024, 1, 1, tzinfo=pytz.UTC), end=datetime(2025, 1, 1, tzinfo=pytz.UTC), ), True, NodeConfig, ModelNode, True, ), # Normal model with sample ( False, "table", None, SampleWindow( start=datetime(2024, 1, 1, tzinfo=pytz.UTC), end=datetime(2025, 1, 1, tzinfo=pytz.UTC), ), True, NodeConfig, ModelNode, True, ), # Incremental merge model with sample ( True, "incremental", "merge", SampleWindow( start=datetime(2024, 1, 1, tzinfo=pytz.UTC), end=datetime(2025, 1, 1, tzinfo=pytz.UTC), ), True, NodeConfig, ModelNode, True, ), # Sample, but not model node ( False, "table", None, SampleWindow( start=datetime(2024, 1, 1, tzinfo=pytz.UTC), end=datetime(2025, 1, 1, tzinfo=pytz.UTC), ), False, NodeConfig, ModelNode, False, ), # Microbatch, but not model node ( True, "incremental", "microbatch", None, False, NodeConfig, ModelNode, False, ), # Mircrobatch model, but not using batches ( False, "incremental", "microbatch", None, True, NodeConfig, ModelNode, False, ), # Non microbatch model, but supposed to use batches ( True, "table", "microbatch", None, True, NodeConfig, ModelNode, False, ), # Incremental merge (True, "incremental", "merge", None, True, NodeConfig, ModelNode, False), # Target seed node, with sample ( False, "table", None, SampleWindow.from_relative_string("2 days"), True, SeedConfig, ModelNode, True, ), # Target seed node, without sample (False, "table", None, None, True, SeedConfig, ModelNode, False), # Sample model from snapshot node ( False, "table", None, SampleWindow.from_relative_string("2 days"), True, NodeConfig, SnapshotNode, True, ), # Target model from snapshot, without sample (False, "table", None, None, True, NodeConfig, SnapshotNode, False), # Target snapshot from model, with sample ( False, "table", None, SampleWindow.from_relative_string("2 days"), True, SnapshotConfig, ModelNode, True, ), ], ) def test_resolve_event_time_filter( self, resolver: ResolverSubclass, use_microbatch_batches: bool, materialized: str, incremental_strategy: Optional[str], sample: Optional[SampleWindow], resolver_model_node: bool, target_type: Any, resolver_model_type: Union[Type[ModelNode], Type[SnapshotNode]], expect_filter: bool, ) -> None: # Target mocking target = mock.Mock() target.config = mock.MagicMock(target_type) target.config.event_time = "created_at" # Resolver mocking resolver.config.args.EVENT_TIME_END = None resolver.config.args.EVENT_TIME_START = None resolver.config.args.sample = sample if resolver_model_node: resolver.model = mock.MagicMock(spec=resolver_model_type) resolver.model.batch = BatchContext( id="1", event_time_start=datetime(2024, 1, 1, tzinfo=pytz.UTC), event_time_end=datetime(2025, 1, 1, tzinfo=pytz.UTC), ) resolver.model.config = mock.MagicMock(NodeConfig) resolver.model.config.materialized = materialized resolver.model.config.incremental_strategy = incremental_strategy resolver.model.config.batch_size = BatchSize.day resolver.model.config.lookback = 1 resolver.manifest.use_microbatch_batches = mock.Mock() resolver.manifest.use_microbatch_batches.return_value = use_microbatch_batches # Try to get an EventTimeFilter event_time_filter = resolver.resolve_event_time_filter(target=target) if expect_filter: assert isinstance(event_time_filter, EventTimeFilter) else: assert event_time_filter is None class TestRuntimeRefResolver: @pytest.fixture def resolver(self): mock_db_wrapper = mock.Mock() mock_db_wrapper.Relation = BaseRelation return RuntimeRefResolver( db_wrapper=mock_db_wrapper, model=mock.Mock(), config=mock.Mock(), manifest=mock.Mock(), ) @pytest.mark.parametrize( "empty,is_ephemeral_model,expected_limit", [ (False, False, None), (True, False, 0), (False, True, None), (True, True, 0), ], ) def test_create_relation_with_empty(self, resolver, empty, is_ephemeral_model, expected_limit): # setup resolver and input node resolver.config.args.EMPTY = empty resolver.config.quoting = {} mock_node = mock.Mock() mock_node.database = "test" mock_node.schema = "test" mock_node.identifier = "test" mock_node.quoting_dict = {} mock_node.alias = "test" mock_node.is_ephemeral_model = is_ephemeral_model mock_node.defer_relation = None set_from_args( Namespace(require_batched_execution_for_custom_microbatch_strategy=False), None ) # create limited relation with mock.patch("dbt.contracts.graph.nodes.ParsedNode", new=mock.Mock): relation = resolver.create_relation(mock_node) assert relation.limit == expected_limit class TestRuntimeSourceResolver: @pytest.fixture def resolver(self): mock_db_wrapper = mock.Mock() mock_db_wrapper.Relation = BaseRelation return RuntimeSourceResolver( db_wrapper=mock_db_wrapper, model=mock.Mock(), config=mock.Mock(), manifest=mock.Mock(), ) @pytest.mark.parametrize( "empty,expected_limit", [ (False, None), (True, 0), ], ) def test_create_relation_with_empty(self, resolver, empty, expected_limit): # setup resolver and input source resolver.config.args.EMPTY = empty resolver.config.quoting = {} mock_source = mock.Mock() mock_source.database = "test" mock_source.schema = "test" mock_source.identifier = "test" mock_source.quoting = Quoting() mock_source.quoting_dict = {} resolver.manifest.resolve_source.return_value = mock_source set_from_args( Namespace(require_batched_execution_for_custom_microbatch_strategy=False), None ) # create limited relation relation = resolver.resolve("test", "test") assert relation.limit == expected_limit ================================================ FILE: tests/unit/context/test_query_header.py ================================================ import re from unittest import mock import pytest from dbt.adapters.base.query_headers import MacroQueryStringSetter from dbt.context.query_header import generate_query_header_context from tests.unit.utils import config_from_parts_or_dicts class TestQueryHeaderContext: @pytest.fixture def profile_cfg(self): return { "outputs": { "test": { "type": "postgres", "dbname": "postgres", "user": "test", "host": "test", "pass": "test", "port": 5432, "schema": "test", }, }, "target": "test", } @pytest.fixture def project_cfg(self): return { "name": "query_headers", "version": "0.1", "profile": "test", "config-version": 2, } @pytest.fixture def query(self): return "SELECT 1;" def test_comment_should_prepend_query_by_default(self, profile_cfg, project_cfg, query): config = config_from_parts_or_dicts(project_cfg, profile_cfg) query_header_context = generate_query_header_context(config, mock.MagicMock(macros={})) query_header = MacroQueryStringSetter(config, query_header_context) sql = query_header.add(query) assert re.match(f"^\/\*.*\*\/\n{query}$", sql) # noqa: [W605] def test_append_comment(self, profile_cfg, project_cfg, query): project_cfg.update({"query-comment": {"comment": "executed by dbt", "append": True}}) config = config_from_parts_or_dicts(project_cfg, profile_cfg) query_header_context = generate_query_header_context(config, mock.MagicMock(macros={})) query_header = MacroQueryStringSetter(config, query_header_context) sql = query_header.add(query) assert sql == f"{query[:-1]}\n/* executed by dbt */;" def test_disable_query_comment(self, profile_cfg, project_cfg, query): project_cfg.update({"query-comment": ""}) config = config_from_parts_or_dicts(project_cfg, profile_cfg) query_header = MacroQueryStringSetter(config, mock.MagicMock(macros={})) assert query_header.add(query) == query ================================================ FILE: tests/unit/contracts/__init__.py ================================================ ================================================ FILE: tests/unit/contracts/files/test_schema_source_file.py ================================================ from dbt.contracts.files import SchemaSourceFile def test_fix_metrics_from_measure(): # This is a test for converting "generated_metrics" to "metrics_from_measures" schema_source_file = { "path": { "searched_path": "models", "relative_path": "schema.yml", "modification_time": 1721228094.7544806, "project_root": "/Users/a_user/sample_project", }, "checksum": { "name": "sha256", "checksum": "63130d480a44a481aa0adc0a8469dccbb72ea36cc09f06683a584a31339f362e", }, "project_name": "test", "parse_file_type": "schema", "dfy": { "models": [{"name": "fct_revenue", "description": "This is the model fct_revenue."}], "semantic_models": [ { "name": "revenue", "description": "This is the FIRST semantic model.", "model": "ref('fct_revenue')", "defaults": {"agg_time_dimension": "ds"}, "measures": [ { "name": "txn_revenue", "expr": "revenue", "agg": "sum", "agg_time_dimension": "ds", "create_metric": True, }, { "name": "sum_of_things", "expr": 2, "agg": "sum", "agg_time_dimension": "ds", }, ], "dimensions": [ { "name": "ds", "type": "time", "expr": "created_at", "type_params": {"time_granularity": "day"}, } ], "entities": [ {"name": "user", "type": "foreign", "expr": "user_id"}, {"name": "id", "type": "primary"}, ], }, { "name": "alt_revenue", "description": "This is the second revenue semantic model.", "model": "ref('fct_revenue')", "defaults": {"agg_time_dimension": "ads"}, "measures": [ { "name": "alt_txn_revenue", "expr": "revenue", "agg": "sum", "agg_time_dimension": "ads", "create_metric": True, }, { "name": "alt_sum_of_things", "expr": 2, "agg": "sum", "agg_time_dimension": "ads", }, ], "dimensions": [ { "name": "ads", "type": "time", "expr": "created_at", "type_params": {"time_granularity": "day"}, } ], "entities": [ {"name": "user", "type": "foreign", "expr": "user_id"}, {"name": "id", "type": "primary"}, ], }, ], "metrics": [ { "name": "simple_metric", "label": "Simple Metric", "type": "simple", "type_params": {"measure": "sum_of_things"}, } ], }, "data_tests": {}, "metrics": ["metric.test.simple_metric"], "generated_metrics": ["metric.test.txn_revenue", "metric.test.alt_txn_revenue"], "metrics_from_measures": {}, "ndp": ["model.test.fct_revenue"], "semantic_models": ["semantic_model.test.revenue", "semantic_model.test.alt_revenue"], "mcp": {}, "env_vars": {}, } expected_metrics_from_measures = { "revenue": ["metric.test.txn_revenue"], "alt_revenue": ["metric.test.alt_txn_revenue"], } ssf = SchemaSourceFile.from_dict(schema_source_file) assert ssf ssf.fix_metrics_from_measures() assert ssf.generated_metrics == [] assert ssf.metrics_from_measures == expected_metrics_from_measures ================================================ FILE: tests/unit/contracts/graph/__init__.py ================================================ ================================================ FILE: tests/unit/contracts/graph/test_manifest.py ================================================ import os import unittest from argparse import Namespace from collections import namedtuple from copy import deepcopy from datetime import datetime, timezone from itertools import product from unittest import mock import freezegun import pytest import dbt.version import dbt_common.invocation from dbt import tracking from dbt.adapters.base.plugin import AdapterPlugin from dbt.artifacts.resources import ( ExposureType, MaturityType, MetricInputMeasure, MetricTypeParams, Owner, RefArgs, UnitTestOutputFixture, WhereFilter, WhereFilterIntersection, ) from dbt.contracts.files import FileHash from dbt.contracts.graph.manifest import DisabledLookup, Manifest, ManifestMetadata from dbt.contracts.graph.nodes import ( DependsOn, Exposure, Group, Metric, ModelConfig, ModelNode, SeedNode, SourceDefinition, UnitTestDefinition, ) from dbt.exceptions import AmbiguousResourceNameRefError, ParsingError from dbt.flags import set_from_args from dbt.node_types import NodeType from dbt_common.events.functions import reset_metadata_vars from dbt_semantic_interfaces.type_enums import MetricType from tests.unit.utils import ( MockDocumentation, MockGenerateMacro, MockMacro, MockMaterialization, MockNode, MockSource, inject_plugin, make_manifest, ) REQUIRED_PARSED_NODE_KEYS = frozenset( { "alias", "tags", "config", "unique_id", "refs", "sources", "metrics", "functions", "meta", "depends_on", "database", "schema", "name", "resource_type", "group", "package_name", "path", "original_file_path", "raw_code", "language", "description", "primary_key", "columns", "fqn", "build_path", "compiled_path", "patch_path", "docs", "doc_blocks", "checksum", "unrendered_config", "unrendered_config_call_dict", "created_at", "config_call_dict", "relation_name", "contract", "access", "version", "latest_version", "constraints", "deprecation_date", "defer_relation", "time_spine", "batch", } ) REQUIRED_COMPILED_NODE_KEYS = frozenset( REQUIRED_PARSED_NODE_KEYS | {"compiled", "extra_ctes_injected", "extra_ctes", "compiled_code", "relation_name"} ) ENV_KEY_NAME = "KEY" if os.name == "nt" else "key" class ManifestTest(unittest.TestCase): def setUp(self): reset_metadata_vars() # TODO: why is this needed for tests in this module to pass? tracking.active_user = None self.maxDiff = None self.model_config = ModelConfig.from_dict( { "enabled": True, "materialized": "view", "persist_docs": {}, "post-hook": [], "pre-hook": [], "vars": {}, "quoting": {}, "column_types": {}, "tags": [], } ) self.exposures = { "exposure.root.my_exposure": Exposure( name="my_exposure", type=ExposureType.Dashboard, owner=Owner(email="some@email.com"), resource_type=NodeType.Exposure, description="Test description", maturity=MaturityType.High, url="hhtp://mydashboard.com", depends_on=DependsOn(nodes=["model.root.multi"]), refs=[RefArgs(name="multi")], sources=[], fqn=["root", "my_exposure"], unique_id="exposure.root.my_exposure", package_name="root", path="my_exposure.sql", original_file_path="my_exposure.sql", ) } self.metrics = { "metric.root.my_metric": Metric( name="new_customers", label="New Customers", description="New customers", meta={"is_okr": True}, tags=["okrs"], type=MetricType.SIMPLE, type_params=MetricTypeParams( measure=MetricInputMeasure( name="customers", filter=WhereFilterIntersection( [WhereFilter(where_sql_template="is_new = True")] ), ) ), resource_type=NodeType.Metric, depends_on=DependsOn(nodes=["semantic_model.root.customers"]), refs=[RefArgs(name="customers")], fqn=["root", "my_metric"], unique_id="metric.root.my_metric", package_name="root", path="my_metric.yml", original_file_path="my_metric.yml", ) } self.groups = { "group.root.my_group": Group( name="my_group", owner=Owner(email="some@email.com"), resource_type=NodeType.Group, unique_id="group.root.my_group", package_name="root", path="my_metric.yml", original_file_path="my_metric.yml", ) } self.nested_nodes = { "model.snowplow.events": ModelNode( name="events", database="dbt", schema="analytics", alias="events", resource_type=NodeType.Model, unique_id="model.snowplow.events", fqn=["snowplow", "events"], package_name="snowplow", refs=[], sources=[], metrics=[], depends_on=DependsOn(), config=self.model_config, tags=[], path="events.sql", original_file_path="events.sql", meta={}, language="sql", raw_code="does not matter", checksum=FileHash.empty(), ), "model.root.events": ModelNode( name="events", database="dbt", schema="analytics", alias="events", resource_type=NodeType.Model, unique_id="model.root.events", fqn=["root", "events"], package_name="root", refs=[], sources=[], metrics=[], depends_on=DependsOn(), config=self.model_config, tags=[], path="events.sql", original_file_path="events.sql", meta={}, language="sql", raw_code="does not matter", checksum=FileHash.empty(), ), "model.root.dep": ModelNode( name="dep", database="dbt", schema="analytics", alias="dep", resource_type=NodeType.Model, unique_id="model.root.dep", fqn=["root", "dep"], package_name="root", refs=[RefArgs(name="events")], sources=[], metrics=[], depends_on=DependsOn(nodes=["model.root.events"]), config=self.model_config, tags=[], path="multi.sql", original_file_path="multi.sql", meta={}, language="sql", raw_code="does not matter", checksum=FileHash.empty(), ), "model.root.nested": ModelNode( name="nested", database="dbt", schema="analytics", alias="nested", resource_type=NodeType.Model, unique_id="model.root.nested", fqn=["root", "nested"], package_name="root", refs=[RefArgs(name="events")], sources=[], metrics=[], depends_on=DependsOn(nodes=["model.root.dep"]), config=self.model_config, tags=[], path="multi.sql", original_file_path="multi.sql", meta={}, language="sql", raw_code="does not matter", checksum=FileHash.empty(), ), "model.root.sibling": ModelNode( name="sibling", database="dbt", schema="analytics", alias="sibling", resource_type=NodeType.Model, unique_id="model.root.sibling", fqn=["root", "sibling"], package_name="root", refs=[RefArgs(name="events")], sources=[], metrics=[], depends_on=DependsOn(nodes=["model.root.events"]), config=self.model_config, tags=[], path="multi.sql", original_file_path="multi.sql", meta={}, language="sql", raw_code="does not matter", checksum=FileHash.empty(), ), "model.root.multi": ModelNode( name="multi", database="dbt", schema="analytics", alias="multi", resource_type=NodeType.Model, unique_id="model.root.multi", fqn=["root", "multi"], package_name="root", refs=[RefArgs(name="events")], sources=[], metrics=[], depends_on=DependsOn(nodes=["model.root.nested", "model.root.sibling"]), config=self.model_config, tags=[], path="multi.sql", original_file_path="multi.sql", meta={}, language="sql", raw_code="does not matter", checksum=FileHash.empty(), ), } self.sources = { "source.root.my_source.my_table": SourceDefinition( database="raw", schema="analytics", resource_type=NodeType.Source, identifier="some_source", name="my_table", source_name="my_source", source_description="My source description", description="Table description", loader="a_loader", unique_id="source.test.my_source.my_table", fqn=["test", "my_source", "my_table"], package_name="root", path="schema.yml", original_file_path="schema.yml", ), } self.semantic_models = {} self.saved_queries = {} for exposure in self.exposures.values(): exposure.validate(exposure.to_dict(omit_none=True)) for metric in self.metrics.values(): metric.validate(metric.to_dict(omit_none=True)) for node in self.nested_nodes.values(): node.validate(node.to_dict(omit_none=True)) for source in self.sources.values(): source.validate(source.to_dict(omit_none=True)) os.environ["DBT_ENV_CUSTOM_ENV_key"] = "value" def tearDown(self): del os.environ["DBT_ENV_CUSTOM_ENV_key"] reset_metadata_vars() @mock.patch.object(tracking, "active_user") @freezegun.freeze_time("2018-02-14T09:15:13Z") def test_no_nodes(self, mock_user): manifest = Manifest( nodes={}, sources={}, macros={}, docs={}, disabled={}, files={}, exposures={}, functions={}, metrics={}, selectors={}, metadata=ManifestMetadata( generated_at=datetime.now(timezone.utc).replace(tzinfo=None) ), semantic_models={}, saved_queries={}, ) invocation_id = dbt_common.invocation._INVOCATION_ID invocation_started_at = dbt_common.invocation._INVOCATION_STARTED_AT mock_user.id = "cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf" set_from_args(Namespace(SEND_ANONYMOUS_USAGE_STATS=False), None) self.assertEqual( manifest.writable_manifest().to_dict(omit_none=True), { "nodes": {}, "sources": {}, "macros": {}, "exposures": {}, "functions": {}, "metrics": {}, "groups": {}, "selectors": {}, "parent_map": {}, "child_map": {}, "group_map": {}, "metadata": { "generated_at": "2018-02-14T09:15:13Z", "dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v12.json", "dbt_version": dbt.version.__version__, "env": {ENV_KEY_NAME: "value"}, "invocation_id": invocation_id, "invocation_started_at": str(invocation_started_at).replace(" ", "T") + "Z", "send_anonymous_usage_stats": False, "user_id": "cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf", "quoting": {}, }, "docs": {}, "disabled": {}, "semantic_models": {}, "unit_tests": {}, "saved_queries": {}, }, ) @freezegun.freeze_time("2018-02-14T09:15:13Z") @mock.patch.object(tracking, "active_user") def test_nested_nodes(self, mock_user): set_from_args(Namespace(SEND_ANONYMOUS_USAGE_STATS=False), None) mock_user.id = "cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf" nodes = deepcopy(self.nested_nodes) manifest = Manifest( nodes=nodes, sources={}, macros={}, docs={}, disabled={}, files={}, exposures={}, metrics={}, selectors={}, metadata=ManifestMetadata( generated_at=datetime.now(timezone.utc).replace(tzinfo=None) ), ) serialized = manifest.writable_manifest().to_dict(omit_none=True) self.assertEqual(serialized["metadata"]["generated_at"], "2018-02-14T09:15:13Z") self.assertEqual(serialized["metadata"]["user_id"], mock_user.id) self.assertFalse(serialized["metadata"]["send_anonymous_usage_stats"]) self.assertEqual(serialized["docs"], {}) self.assertEqual(serialized["disabled"], {}) parent_map = serialized["parent_map"] child_map = serialized["child_map"] # make sure there aren't any extra/missing keys. self.assertEqual(set(parent_map), set(nodes)) self.assertEqual(set(child_map), set(nodes)) self.assertEqual(parent_map["model.root.sibling"], ["model.root.events"]) self.assertEqual(parent_map["model.root.nested"], ["model.root.dep"]) self.assertEqual(parent_map["model.root.dep"], ["model.root.events"]) # order doesn't matter. self.assertEqual( set(parent_map["model.root.multi"]), set(["model.root.nested", "model.root.sibling"]) ) self.assertEqual( parent_map["model.root.events"], [], ) self.assertEqual( parent_map["model.snowplow.events"], [], ) self.assertEqual( child_map["model.root.sibling"], ["model.root.multi"], ) self.assertEqual( child_map["model.root.nested"], ["model.root.multi"], ) self.assertEqual(child_map["model.root.dep"], ["model.root.nested"]) self.assertEqual(child_map["model.root.multi"], []) self.assertEqual( set(child_map["model.root.events"]), set(["model.root.dep", "model.root.sibling"]) ) self.assertEqual(child_map["model.snowplow.events"], []) def test_build_flat_graph(self): exposures = deepcopy(self.exposures) metrics = deepcopy(self.metrics) groups = deepcopy(self.groups) nodes = deepcopy(self.nested_nodes) sources = deepcopy(self.sources) manifest = Manifest( nodes=nodes, sources=sources, macros={}, docs={}, disabled={}, files={}, exposures=exposures, metrics=metrics, groups=groups, selectors={}, ) manifest.build_flat_graph() flat_graph = manifest.flat_graph flat_exposures = flat_graph["exposures"] flat_groups = flat_graph["groups"] flat_metrics = flat_graph["metrics"] flat_nodes = flat_graph["nodes"] flat_sources = flat_graph["sources"] flat_semantic_models = flat_graph["semantic_models"] flat_saved_queries = flat_graph["saved_queries"] flat_unit_tests = flat_graph["unit_tests"] self.assertEqual( set(flat_graph), set( [ "exposures", "functions", "groups", "nodes", "sources", "metrics", "semantic_models", "saved_queries", "unit_tests", ] ), ) self.assertEqual(set(flat_exposures), set(self.exposures)) self.assertEqual(set(flat_groups), set(self.groups)) self.assertEqual(set(flat_metrics), set(self.metrics)) self.assertEqual(set(flat_nodes), set(self.nested_nodes)) self.assertEqual(set(flat_sources), set(self.sources)) self.assertEqual(set(flat_semantic_models), set(self.semantic_models)) self.assertEqual(set(flat_saved_queries), set(self.saved_queries)) self.assertEqual(set(flat_unit_tests), set()) for node in flat_nodes.values(): self.assertEqual(frozenset(node), REQUIRED_PARSED_NODE_KEYS) def test_build_flat_graph_with_unit_tests(self): """Test that unit tests are included in flat_graph.""" nodes = deepcopy(self.nested_nodes) # Create a unit test for the events model unit_test = UnitTestDefinition( name="test_events", model="events", package_name="root", resource_type=NodeType.Unit, path="unit_tests.yml", original_file_path="models/unit_tests.yml", unique_id="unit_test.root.events.test_events", given=[], expect=UnitTestOutputFixture(rows=[{"id": 1}]), fqn=["root", "events", "test_events"], depends_on=DependsOn(nodes=["model.root.events"]), ) unit_tests = {unit_test.unique_id: unit_test} manifest = Manifest( nodes=nodes, sources=deepcopy(self.sources), macros={}, docs={}, disabled={}, files={}, exposures=deepcopy(self.exposures), metrics=deepcopy(self.metrics), groups=deepcopy(self.groups), selectors={}, unit_tests=unit_tests, ) manifest.build_flat_graph() flat_graph = manifest.flat_graph # Verify unit_tests key exists self.assertIn("unit_tests", flat_graph) flat_unit_tests = flat_graph["unit_tests"] # Verify the unit test is in flat_graph self.assertEqual(set(flat_unit_tests), set(unit_tests)) self.assertIn("unit_test.root.events.test_events", flat_unit_tests) # Verify the unit test data is serialized correctly flat_unit_test = flat_unit_tests["unit_test.root.events.test_events"] self.assertEqual(flat_unit_test["name"], "test_events") self.assertEqual(flat_unit_test["model"], "events") self.assertEqual(flat_unit_test["package_name"], "root") @mock.patch.object(tracking, "active_user") @freezegun.freeze_time("2018-02-14T09:15:13Z") def test_no_nodes_with_metadata(self, mock_user): mock_user.id = "cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf" dbt_common.invocation._INVOCATION_ID = "01234567-0123-0123-0123-0123456789ab" dbt_common.invocation._INVOCATION_STARTED_AT = datetime.now(timezone.utc).replace( tzinfo=None ) set_from_args(Namespace(SEND_ANONYMOUS_USAGE_STATS=False), None) metadata = ManifestMetadata( project_id="098f6bcd4621d373cade4e832627b4f6", adapter_type="postgres", generated_at=datetime.now(timezone.utc).replace(tzinfo=None), invocation_started_at=dbt_common.invocation._INVOCATION_STARTED_AT, user_id="cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf", send_anonymous_usage_stats=False, ) manifest = Manifest( nodes={}, sources={}, macros={}, docs={}, disabled={}, selectors={}, metadata=metadata, files={}, exposures={}, functions={}, semantic_models={}, saved_queries={}, ) invocation_started_at = ( str(dbt_common.invocation._INVOCATION_STARTED_AT).replace(" ", "T") + "Z" ) self.assertEqual( manifest.writable_manifest().to_dict(omit_none=True), { "nodes": {}, "sources": {}, "macros": {}, "exposures": {}, "functions": {}, "metrics": {}, "groups": {}, "selectors": {}, "parent_map": {}, "child_map": {}, "group_map": {}, "docs": {}, "metadata": { "generated_at": "2018-02-14T09:15:13Z", "dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v12.json", "dbt_version": dbt.version.__version__, "project_id": "098f6bcd4621d373cade4e832627b4f6", "user_id": "cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf", "send_anonymous_usage_stats": False, "adapter_type": "postgres", "invocation_id": "01234567-0123-0123-0123-0123456789ab", "invocation_started_at": invocation_started_at, "env": {ENV_KEY_NAME: "value"}, "quoting": {}, }, "disabled": {}, "semantic_models": {}, "unit_tests": {}, "saved_queries": {}, }, ) def test_get_resource_fqns_empty(self): manifest = Manifest( nodes={}, sources={}, macros={}, docs={}, disabled={}, files={}, exposures={}, selectors={}, ) self.assertEqual(manifest.get_resource_fqns(), {}) def test_get_resource_fqns(self): nodes = deepcopy(self.nested_nodes) nodes["seed.root.seed"] = SeedNode( name="seed", database="dbt", schema="analytics", alias="seed", resource_type=NodeType.Seed, unique_id="seed.root.seed", fqn=["root", "seed"], package_name="root", config=self.model_config, tags=[], path="seed.csv", original_file_path="seed.csv", checksum=FileHash.empty(), ) manifest = Manifest( nodes=nodes, sources=self.sources, macros={}, docs={}, disabled={}, files={}, exposures=self.exposures, metrics=self.metrics, selectors={}, ) expect = { "metrics": frozenset([("root", "my_metric")]), "exposures": frozenset([("root", "my_exposure")]), "models": frozenset( [ ("snowplow", "events"), ("root", "events"), ("root", "dep"), ("root", "nested"), ("root", "sibling"), ("root", "multi"), ] ), "seeds": frozenset([("root", "seed")]), "sources": frozenset([("test", "my_source", "my_table")]), } resource_fqns = manifest.get_resource_fqns() self.assertEqual(resource_fqns, expect) def test_deepcopy_copies_flat_graph(self): test_node = ModelNode( name="events", database="dbt", schema="analytics", alias="events", resource_type=NodeType.Model, unique_id="model.snowplow.events", fqn=["snowplow", "events"], package_name="snowplow", refs=[], sources=[], metrics=[], depends_on=DependsOn(), config=self.model_config, tags=[], path="events.sql", original_file_path="events.sql", meta={}, language="sql", raw_code="does not matter", checksum=FileHash.empty(), ) original = make_manifest(nodes=[test_node]) original.build_flat_graph() copy = original.deepcopy() self.assertEqual(original.flat_graph, copy.flat_graph) class MixedManifestTest(unittest.TestCase): def setUp(self): self.maxDiff = None self.model_config = ModelConfig.from_dict( { "enabled": True, "materialized": "view", "persist_docs": {}, "post-hook": [], "pre-hook": [], "vars": {}, "quoting": {}, "column_types": {}, "tags": [], } ) self.nested_nodes = { "model.snowplow.events": ModelNode( name="events", database="dbt", schema="analytics", alias="events", resource_type=NodeType.Model, unique_id="model.snowplow.events", fqn=["snowplow", "events"], package_name="snowplow", refs=[], sources=[], depends_on=DependsOn(), config=self.model_config, tags=[], path="events.sql", original_file_path="events.sql", language="sql", raw_code="does not matter", meta={}, compiled=True, compiled_code="also does not matter", extra_ctes_injected=True, relation_name='"dbt"."analytics"."events"', extra_ctes=[], checksum=FileHash.empty(), ), "model.root.events": ModelNode( name="events", database="dbt", schema="analytics", alias="events", resource_type=NodeType.Model, unique_id="model.root.events", fqn=["root", "events"], package_name="root", refs=[], sources=[], depends_on=DependsOn(), config=self.model_config, tags=[], path="events.sql", original_file_path="events.sql", raw_code="does not matter", meta={}, compiled=True, compiled_code="also does not matter", language="sql", extra_ctes_injected=True, relation_name='"dbt"."analytics"."events"', extra_ctes=[], checksum=FileHash.empty(), ), "model.root.dep": ModelNode( name="dep", database="dbt", schema="analytics", alias="dep", resource_type=NodeType.Model, unique_id="model.root.dep", fqn=["root", "dep"], package_name="root", refs=[RefArgs(name="events")], sources=[], depends_on=DependsOn(nodes=["model.root.events"]), config=self.model_config, tags=[], path="multi.sql", original_file_path="multi.sql", meta={}, language="sql", raw_code="does not matter", checksum=FileHash.empty(), ), "model.root.versioned.v1": ModelNode( name="versioned", database="dbt", schema="analytics", alias="dep", resource_type=NodeType.Model, unique_id="model.root.versioned.v1", fqn=["root", "dep"], package_name="root", refs=[], sources=[], depends_on=DependsOn(), config=self.model_config, tags=[], path="versioned.sql", original_file_path="versioned.sql", meta={}, language="sql", raw_code="does not matter", checksum=FileHash.empty(), version=1, ), "model.root.dep_version": ModelNode( name="dep_version", database="dbt", schema="analytics", alias="dep", resource_type=NodeType.Model, unique_id="model.root.dep_version", fqn=["root", "dep"], package_name="root", refs=[RefArgs(name="versioned", version=1)], sources=[], depends_on=DependsOn(nodes=["model.root.versioned.v1"]), config=self.model_config, tags=[], path="dep_version.sql", original_file_path="dep_version.sql", meta={}, language="sql", raw_code="does not matter", checksum=FileHash.empty(), ), "model.root.nested": ModelNode( name="nested", database="dbt", schema="analytics", alias="nested", resource_type=NodeType.Model, unique_id="model.root.nested", fqn=["root", "nested"], package_name="root", refs=[RefArgs(name="events")], sources=[], depends_on=DependsOn(nodes=["model.root.dep"]), config=self.model_config, tags=[], path="multi.sql", original_file_path="multi.sql", meta={}, language="sql", raw_code="does not matter", checksum=FileHash.empty(), ), "model.root.sibling": ModelNode( name="sibling", database="dbt", schema="analytics", alias="sibling", resource_type=NodeType.Model, unique_id="model.root.sibling", fqn=["root", "sibling"], package_name="root", refs=[RefArgs(name="events")], sources=[], depends_on=DependsOn(nodes=["model.root.events"]), config=self.model_config, tags=[], path="multi.sql", original_file_path="multi.sql", meta={}, language="sql", raw_code="does not matter", checksum=FileHash.empty(), ), "model.root.multi": ModelNode( name="multi", database="dbt", schema="analytics", alias="multi", resource_type=NodeType.Model, unique_id="model.root.multi", fqn=["root", "multi"], package_name="root", refs=[RefArgs(name="events")], sources=[], depends_on=DependsOn(nodes=["model.root.nested", "model.root.sibling"]), config=self.model_config, tags=[], path="multi.sql", original_file_path="multi.sql", meta={}, language="sql", raw_code="does not matter", checksum=FileHash.empty(), ), } os.environ["DBT_ENV_CUSTOM_ENV_key"] = "value" def tearDown(self): del os.environ["DBT_ENV_CUSTOM_ENV_key"] @mock.patch.object(tracking, "active_user") @freezegun.freeze_time("2018-02-14T09:15:13Z") def test_no_nodes(self, mock_user): mock_user.id = "cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf" set_from_args(Namespace(SEND_ANONYMOUS_USAGE_STATS=False), None) invocation_started_at = datetime.now(timezone.utc).replace(tzinfo=None) metadata = ManifestMetadata( generated_at=datetime.now(timezone.utc).replace(tzinfo=None), invocation_id="01234567-0123-0123-0123-0123456789ab", invocation_started_at=invocation_started_at, ) manifest = Manifest( nodes={}, sources={}, macros={}, docs={}, selectors={}, disabled={}, metadata=metadata, files={}, exposures={}, semantic_models={}, saved_queries={}, ) self.assertEqual( manifest.writable_manifest().to_dict(omit_none=True), { "nodes": {}, "macros": {}, "sources": {}, "exposures": {}, "functions": {}, "metrics": {}, "groups": {}, "selectors": {}, "parent_map": {}, "child_map": {}, "group_map": {}, "metadata": { "generated_at": "2018-02-14T09:15:13Z", "dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v12.json", "dbt_version": dbt.version.__version__, "invocation_id": "01234567-0123-0123-0123-0123456789ab", "invocation_started_at": str(invocation_started_at).replace(" ", "T") + "Z", "env": {ENV_KEY_NAME: "value"}, "send_anonymous_usage_stats": False, "user_id": "cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf", "quoting": {}, }, "docs": {}, "disabled": {}, "semantic_models": {}, "unit_tests": {}, "saved_queries": {}, }, ) @freezegun.freeze_time("2018-02-14T09:15:13Z") def test_nested_nodes(self): nodes = deepcopy(self.nested_nodes) manifest = Manifest( nodes=nodes, sources={}, macros={}, docs={}, disabled={}, selectors={}, metadata=ManifestMetadata( generated_at=datetime.now(timezone.utc).replace(tzinfo=None) ), files={}, exposures={}, ) serialized = manifest.writable_manifest().to_dict(omit_none=True) self.assertEqual(serialized["metadata"]["generated_at"], "2018-02-14T09:15:13Z") self.assertEqual(serialized["disabled"], {}) parent_map = serialized["parent_map"] child_map = serialized["child_map"] # make sure there aren't any extra/missing keys. self.assertEqual(set(parent_map), set(nodes)) self.assertEqual(set(child_map), set(nodes)) self.assertEqual(parent_map["model.root.sibling"], ["model.root.events"]) self.assertEqual(parent_map["model.root.nested"], ["model.root.dep"]) self.assertEqual(parent_map["model.root.dep"], ["model.root.events"]) # order doesn't matter. self.assertEqual( set(parent_map["model.root.multi"]), set(["model.root.nested", "model.root.sibling"]) ) self.assertEqual( parent_map["model.root.events"], [], ) self.assertEqual( parent_map["model.snowplow.events"], [], ) self.assertEqual( child_map["model.root.sibling"], ["model.root.multi"], ) self.assertEqual( child_map["model.root.nested"], ["model.root.multi"], ) self.assertEqual(child_map["model.root.dep"], ["model.root.nested"]) self.assertEqual(child_map["model.root.multi"], []) self.assertEqual( set(child_map["model.root.events"]), set(["model.root.dep", "model.root.sibling"]) ) self.assertEqual(child_map["model.snowplow.events"], []) def test_build_flat_graph(self): nodes = deepcopy(self.nested_nodes) manifest = Manifest( nodes=nodes, sources={}, functions={}, macros={}, docs={}, disabled={}, selectors={}, files={}, exposures={}, semantic_models={}, saved_queries={}, ) manifest.build_flat_graph() flat_graph = manifest.flat_graph flat_nodes = flat_graph["nodes"] self.assertEqual( set(flat_graph), set( [ "exposures", "functions", "groups", "metrics", "nodes", "sources", "semantic_models", "saved_queries", "unit_tests", ] ), ) self.assertEqual(set(flat_nodes), set(self.nested_nodes)) self.assertEqual(set(flat_graph["unit_tests"]), set()) compiled_count = 0 for node in flat_nodes.values(): if node.get("compiled"): self.assertEqual(frozenset(node), REQUIRED_COMPILED_NODE_KEYS) compiled_count += 1 else: self.assertEqual(frozenset(node), REQUIRED_PARSED_NODE_KEYS) self.assertEqual(compiled_count, 2) def test_merge_from_artifact(self): original_nodes = deepcopy(self.nested_nodes) other_nodes = deepcopy(self.nested_nodes) nested2 = other_nodes.pop("model.root.nested") nested2.name = "nested2" nested2.alias = "nested2" nested2.fqn = ["root", "nested2"] other_nodes["model.root.nested2"] = nested2 for k, v in other_nodes.items(): v.database = "other_" + v.database v.schema = "other_" + v.schema v.alias = "other_" + v.alias if v.relation_name: v.relation_name = "other_" + v.relation_name other_nodes[k] = v original_manifest = Manifest(nodes=original_nodes) other_manifest = Manifest(nodes=other_nodes) original_manifest.merge_from_artifact(other_manifest) # new node added should not be in original manifest assert "model.root.nested2" not in original_manifest.nodes # old node removed should not have defer_relation in original manifest assert original_manifest.nodes["model.root.nested"].defer_relation is None # for all other nodes, check that defer_relation is updated for k, v in original_manifest.nodes.items(): if v.defer_relation: self.assertEqual("other_" + v.database, v.defer_relation.database) self.assertEqual("other_" + v.schema, v.defer_relation.schema) self.assertEqual("other_" + v.alias, v.defer_relation.alias) if v.relation_name: self.assertEqual("other_" + v.relation_name, v.defer_relation.relation_name) # Tests of the manifest search code (find_X_by_Y) class TestManifestSearch(unittest.TestCase): _macros: list = [] _nodes: list = [] _docs: list = [] @property def macros(self): return self._macros @property def nodes(self): return self._nodes @property def docs(self): return self._docs def setUp(self): self.manifest = Manifest( nodes={n.unique_id: n for n in self.nodes}, macros={m.unique_id: m for m in self.macros}, docs={d.unique_id: d for d in self.docs}, disabled={}, files={}, exposures={}, metrics={}, selectors={}, ) FindMacroSpec = namedtuple("FindMacroSpec", "macros,expected") macro_parameter_sets = [ # empty FindMacroSpec( macros=[], expected={None: None, "root": None, "dep": None, "dbt": None}, ), # just root FindMacroSpec( macros=[MockMacro("root")], expected={None: "root", "root": "root", "dep": None, "dbt": None}, ), # just dep FindMacroSpec( macros=[MockMacro("dep")], expected={None: "dep", "root": None, "dep": "dep", "dbt": None}, ), # just dbt FindMacroSpec( macros=[MockMacro("dbt")], expected={None: "dbt", "root": None, "dep": None, "dbt": "dbt"}, ), # root overrides dep FindMacroSpec( macros=[MockMacro("root"), MockMacro("dep")], expected={None: "root", "root": "root", "dep": "dep", "dbt": None}, ), # root overrides core FindMacroSpec( macros=[MockMacro("root"), MockMacro("dbt")], expected={None: "root", "root": "root", "dep": None, "dbt": "dbt"}, ), # dep overrides core FindMacroSpec( macros=[MockMacro("dep"), MockMacro("dbt")], expected={None: "dep", "root": None, "dep": "dep", "dbt": "dbt"}, ), # root overrides dep overrides core FindMacroSpec( macros=[MockMacro("root"), MockMacro("dep"), MockMacro("dbt")], expected={None: "root", "root": "root", "dep": "dep", "dbt": "dbt"}, ), ] def id_macro(arg): if isinstance(arg, list): macro_names = "__".join(f"{m.package_name}" for m in arg) return f"m_[{macro_names}]" if isinstance(arg, dict): arg_names = "__".join(f"{k}_{v}" for k, v in arg.items()) return f"exp_{{{arg_names}}}" @pytest.mark.parametrize("macros,expectations", macro_parameter_sets, ids=id_macro) def test_find_macro_by_name(macros, expectations): manifest = make_manifest(macros=macros) for package, expected in expectations.items(): result = manifest.find_macro_by_name( name="my_macro", root_project_name="root", package=package ) if expected is None: assert result is expected else: assert result.package_name == expected generate_name_parameter_sets = [ # empty FindMacroSpec( macros=[], expected={None: None, "root": None, "dep": None, "dbt": None}, ), # just root FindMacroSpec( macros=[MockGenerateMacro("root")], # "root" is not imported expected={None: "root", "root": None, "dep": None, "dbt": None}, ), # just dep FindMacroSpec( macros=[MockGenerateMacro("dep")], expected={None: None, "root": None, "dep": "dep", "dbt": None}, ), # just dbt FindMacroSpec( macros=[MockGenerateMacro("dbt")], # "dbt" is not imported expected={None: "dbt", "root": None, "dep": None, "dbt": None}, ), # root overrides dep FindMacroSpec( macros=[MockGenerateMacro("root"), MockGenerateMacro("dep")], expected={None: "root", "root": None, "dep": "dep", "dbt": None}, ), # root overrides core FindMacroSpec( macros=[MockGenerateMacro("root"), MockGenerateMacro("dbt")], expected={None: "root", "root": None, "dep": None, "dbt": None}, ), # dep overrides core FindMacroSpec( macros=[MockGenerateMacro("dep"), MockGenerateMacro("dbt")], expected={None: "dbt", "root": None, "dep": "dep", "dbt": None}, ), # root overrides dep overrides core FindMacroSpec( macros=[MockGenerateMacro("root"), MockGenerateMacro("dep"), MockGenerateMacro("dbt")], expected={None: "root", "root": None, "dep": "dep", "dbt": None}, ), ] @pytest.mark.parametrize("macros,expectations", generate_name_parameter_sets, ids=id_macro) def test_find_generate_macros_by_name(macros, expectations): manifest = make_manifest(macros=macros) result = manifest.find_generate_macro_by_name( component="some_component", root_project_name="root" ) for package, expected in expectations.items(): result = manifest.find_generate_macro_by_name( component="some_component", root_project_name="root", imported_package=package ) if expected is None: assert result is expected else: assert result.package_name == expected FindMaterializationSpec = namedtuple("FindMaterializationSpec", "macros,adapter_type,expected") def _materialization_parameter_sets_legacy(): # inject the plugins used for materialization parameter tests FooPlugin = AdapterPlugin( adapter=mock.MagicMock(), credentials=mock.MagicMock(), include_path="/path/to/root/plugin", project_name="foo", ) FooPlugin.adapter.type.return_value = "foo" inject_plugin(FooPlugin) BarPlugin = AdapterPlugin( adapter=mock.MagicMock(), credentials=mock.MagicMock(), include_path="/path/to/root/plugin", dependencies=["foo"], project_name="bar", ) BarPlugin.adapter.type.return_value = "bar" inject_plugin(BarPlugin) sets = [ FindMaterializationSpec(macros=[], adapter_type="foo", expected=None), ] # default only, each project sets.extend( FindMaterializationSpec( macros=[MockMaterialization(project, adapter_type=None)], adapter_type="foo", expected=(project, "default"), ) for project in ["root", "dep", "dbt"] ) # other type only, each project sets.extend( FindMaterializationSpec( macros=[MockMaterialization(project, adapter_type="bar")], adapter_type="foo", expected=None, ) for project in ["root", "dep", "dbt"] ) # matching type only, each project sets.extend( FindMaterializationSpec( macros=[MockMaterialization(project, adapter_type="foo")], adapter_type="foo", expected=(project, "foo"), ) for project in ["root", "dep", "dbt"] ) sets.extend( [ # matching type and default everywhere FindMaterializationSpec( macros=[ MockMaterialization(project, adapter_type=atype) for (project, atype) in product(["root", "dep", "dbt"], ["foo", None]) ], adapter_type="foo", expected=("root", "foo"), ), # default in core, override is in dep, and root has unrelated override # should find the dep override. FindMaterializationSpec( macros=[ MockMaterialization("root", adapter_type="bar"), MockMaterialization("dep", adapter_type="foo"), MockMaterialization("dbt", adapter_type=None), ], adapter_type="foo", expected=("dep", "foo"), ), # default in core, unrelated override is in dep, and root has an override # should find the root override. FindMaterializationSpec( macros=[ MockMaterialization("root", adapter_type="foo"), MockMaterialization("dep", adapter_type="bar"), MockMaterialization("dbt", adapter_type=None), ], adapter_type="foo", expected=("root", "foo"), ), # default in core, override is in dep, and root has an override too. # should find the root override. FindMaterializationSpec( macros=[ MockMaterialization("root", adapter_type="foo"), MockMaterialization("dep", adapter_type="foo"), MockMaterialization("dbt", adapter_type=None), ], adapter_type="foo", expected=("root", "foo"), ), # core has default + adapter, dep has adapter, root has default # should find the dependency implementation, because it's the most specific FindMaterializationSpec( macros=[ MockMaterialization("root", adapter_type=None), MockMaterialization("dep", adapter_type="foo"), MockMaterialization("dbt", adapter_type=None), MockMaterialization("dbt", adapter_type="foo"), ], adapter_type="foo", expected=("dep", "foo"), ), ] ) # inherit from parent adapter sets.extend( FindMaterializationSpec( macros=[MockMaterialization(project, adapter_type="foo")], adapter_type="bar", expected=(project, "foo"), ) for project in ["root", "dep", "dbt"] ) sets.extend( FindMaterializationSpec( macros=[ MockMaterialization(project, adapter_type="foo"), MockMaterialization(project, adapter_type="bar"), ], adapter_type="bar", expected=(project, "bar"), ) for project in ["root", "dep", "dbt"] ) return sets def id_mat(arg): if isinstance(arg, list): macro_names = "__".join(f"{m.package_name}_{m.adapter_type}" for m in arg) return f"m_[{macro_names}]" elif isinstance(arg, tuple): return "_".join(arg) @pytest.mark.parametrize( "macros,adapter_type,expected", _materialization_parameter_sets_legacy(), ids=id_mat, ) def test_find_materialization_by_name_legacy(macros, adapter_type, expected): set_from_args( Namespace( SEND_ANONYMOUS_USAGE_STATS=False, REQUIRE_EXPLICIT_PACKAGE_OVERRIDES_FOR_BUILTIN_MATERIALIZATIONS=False, ), None, ) manifest = make_manifest(macros=macros) result = manifest.find_materialization_macro_by_name( project_name="root", materialization_name="my_materialization", adapter_type=adapter_type, ) if expected is None: assert result is expected else: expected_package, expected_adapter_type = expected assert result.adapter_type == expected_adapter_type assert result.package_name == expected_package def _materialization_parameter_sets(): # inject the plugins used for materialization parameter tests FooPlugin = AdapterPlugin( adapter=mock.MagicMock(), credentials=mock.MagicMock(), include_path="/path/to/root/plugin", project_name="foo", ) FooPlugin.adapter.type.return_value = "foo" inject_plugin(FooPlugin) BarPlugin = AdapterPlugin( adapter=mock.MagicMock(), credentials=mock.MagicMock(), include_path="/path/to/root/plugin", dependencies=["foo"], project_name="bar", ) BarPlugin.adapter.type.return_value = "bar" inject_plugin(BarPlugin) sets = [ FindMaterializationSpec(macros=[], adapter_type="foo", expected=None), ] # default only, each project sets.extend( FindMaterializationSpec( macros=[MockMaterialization(project, adapter_type=None)], adapter_type="foo", expected=(project, "default"), ) for project in ["root", "dep", "dbt"] ) # other type only, each project sets.extend( FindMaterializationSpec( macros=[MockMaterialization(project, adapter_type="bar")], adapter_type="foo", expected=None, ) for project in ["root", "dep", "dbt"] ) # matching type only, each project sets.extend( FindMaterializationSpec( macros=[MockMaterialization(project, adapter_type="foo")], adapter_type="foo", expected=(project, "foo"), ) for project in ["root", "dep", "dbt"] ) sets.extend( [ # matching type and default everywhere FindMaterializationSpec( macros=[ MockMaterialization(project, adapter_type=atype) for (project, atype) in product(["root", "dep", "dbt"], ["foo", None]) ], adapter_type="foo", expected=("root", "foo"), ), # default in core, override is in dep, and root has unrelated override # should find the dbt default because default materializations cannot be overwritten by packages. FindMaterializationSpec( macros=[ MockMaterialization("root", adapter_type="bar"), MockMaterialization("dep", adapter_type="foo"), MockMaterialization("dbt", adapter_type=None), ], adapter_type="foo", expected=("dbt", "default"), ), # default in core, unrelated override is in dep, and root has an override # should find the root override. FindMaterializationSpec( macros=[ MockMaterialization("root", adapter_type="foo"), MockMaterialization("dep", adapter_type="bar"), MockMaterialization("dbt", adapter_type=None), ], adapter_type="foo", expected=("root", "foo"), ), # default in core, override is in dep, and root has an override too. # should find the root override. FindMaterializationSpec( macros=[ MockMaterialization("root", adapter_type="foo"), MockMaterialization("dep", adapter_type="foo"), MockMaterialization("dbt", adapter_type=None), ], adapter_type="foo", expected=("root", "foo"), ), # core has default + adapter, dep has adapter, root has default # should find the default adapter implementation, because it's the most specific # and default materializations cannot be overwritten by packages FindMaterializationSpec( macros=[ MockMaterialization("root", adapter_type=None), MockMaterialization("dep", adapter_type="foo"), MockMaterialization("dbt", adapter_type=None), MockMaterialization("dbt", adapter_type="foo"), ], adapter_type="foo", expected=("dbt", "foo"), ), ] ) # inherit from parent adapter sets.extend( FindMaterializationSpec( macros=[MockMaterialization(project, adapter_type="foo")], adapter_type="bar", expected=(project, "foo"), ) for project in ["root", "dep", "dbt"] ) sets.extend( FindMaterializationSpec( macros=[ MockMaterialization(project, adapter_type="foo"), MockMaterialization(project, adapter_type="bar"), ], adapter_type="bar", expected=(project, "bar"), ) for project in ["root", "dep", "dbt"] ) return sets @pytest.mark.parametrize( "macros,adapter_type,expected", _materialization_parameter_sets(), ids=id_mat, ) def test_find_materialization_by_name(macros, adapter_type, expected): set_from_args( Namespace( SEND_ANONYMOUS_USAGE_STATS=False, REQUIRE_EXPLICIT_PACKAGE_OVERRIDES_FOR_BUILTIN_MATERIALIZATIONS=True, ), None, ) manifest = make_manifest(macros=macros) result = manifest.find_materialization_macro_by_name( project_name="root", materialization_name="my_materialization", adapter_type=adapter_type, ) if expected is None: assert result is expected else: expected_package, expected_adapter_type = expected assert result.adapter_type == expected_adapter_type assert result.package_name == expected_package FindNodeSpec = namedtuple("FindNodeSpec", "nodes,sources,package,version,expected") def _refable_parameter_sets(): sets = [ # empties FindNodeSpec(nodes=[], sources=[], package=None, version=None, expected=None), FindNodeSpec(nodes=[], sources=[], package="root", version=None, expected=None), ] sets.extend( # only one model, no package specified -> find it in any package FindNodeSpec( nodes=[MockNode(project, "my_model")], sources=[], package=None, version=None, expected=(project, "my_model"), ) for project in ["root", "dep"] ) # only one model, no package specified -> find it in any package sets.extend( [ FindNodeSpec( nodes=[MockNode("root", "my_model")], sources=[], package="root", version=None, expected=("root", "my_model"), ), FindNodeSpec( nodes=[MockNode("dep", "my_model")], sources=[], package="root", version=None, expected=None, ), # versioned model lookups FindNodeSpec( nodes=[MockNode("root", "my_model", version="2")], sources=[], package="root", version="2", expected=("root", "my_model", "2"), ), FindNodeSpec( nodes=[MockNode("root", "my_model", version="2")], sources=[], package="root", version=2, expected=("root", "my_model", "2"), ), FindNodeSpec( nodes=[MockNode("root", "my_model", version="3")], sources=[], package="root", version="2", expected=None, ), FindNodeSpec( nodes=[MockNode("root", "my_model", version="3", is_latest_version=True)], sources=[], package="root", version=None, expected=("root", "my_model", "3"), ), FindNodeSpec( nodes=[MockNode("root", "my_model", version="3", is_latest_version=False)], sources=[], package="root", version=None, expected=None, ), FindNodeSpec( nodes=[MockNode("root", "my_model", version="0", is_latest_version=False)], sources=[], package="root", version=None, expected=None, ), FindNodeSpec( nodes=[MockNode("root", "my_model", version="0", is_latest_version=True)], sources=[], package="root", version=None, expected=("root", "my_model", "0"), ), # a source with that name exists, but not a refable FindNodeSpec( nodes=[], sources=[MockSource("root", "my_source", "my_model")], package=None, version=None, expected=None, ), # a source with that name exists, and a refable FindNodeSpec( nodes=[MockNode("root", "my_model")], sources=[MockSource("root", "my_source", "my_model")], package=None, version=None, expected=("root", "my_model"), ), FindNodeSpec( nodes=[MockNode("root", "my_model")], sources=[MockSource("root", "my_source", "my_model")], package="root", version=None, expected=("root", "my_model"), ), FindNodeSpec( nodes=[MockNode("root", "my_model")], sources=[MockSource("root", "my_source", "my_model")], package="dep", version=None, expected=None, ), # duplicate node name across package FindNodeSpec( nodes=[MockNode("project_a", "my_model"), MockNode("project_b", "my_model")], sources=[], package="project_a", version=None, expected=("project_a", "my_model"), ), # duplicate node name across package: root node preferred to package node FindNodeSpec( nodes=[MockNode("root", "my_model"), MockNode("project_a", "my_model")], sources=[], package=None, version=None, expected=("root", "my_model"), ), FindNodeSpec( nodes=[MockNode("root", "my_model"), MockNode("project_a", "my_model")], sources=[], package="root", version=None, expected=("root", "my_model"), ), FindNodeSpec( nodes=[MockNode("root", "my_model"), MockNode("project_a", "my_model")], sources=[], package="project_a", version=None, expected=("project_a", "my_model"), ), # duplicate node name across package: resolved by version FindNodeSpec( nodes=[ MockNode("project_a", "my_model", version="1"), MockNode("project_b", "my_model", version="2"), ], sources=[], package=None, version="1", expected=("project_a", "my_model", "1"), ), ] ) return sets def _ambiguous_ref_parameter_sets(): sets = [ FindNodeSpec( nodes=[MockNode("project_a", "my_model"), MockNode("project_b", "my_model")], sources=[], package=None, version=None, expected=None, ), FindNodeSpec( nodes=[ MockNode("project_a", "my_model", version="2", is_latest_version=True), MockNode("project_b", "my_model", version="1", is_latest_version=True), ], sources=[], package=None, version=None, expected=None, ), ] return sets def _duplicate_node_name_across_packages_ref_parameter_sets(): sets = [ FindNodeSpec( nodes=[MockNode("project_a", "my_model"), MockNode("root", "my_model")], sources=[], package=None, version=None, expected=("project_a", "my_model"), ), ] return sets def id_nodes(arg): if isinstance(arg, list): node_names = "__".join(f"{n.package_name}_{n.search_name}" for n in arg) return f"m_[{node_names}]" elif isinstance(arg, tuple): return "_".join(arg) @pytest.mark.parametrize( "nodes,sources,package,version,expected", _refable_parameter_sets(), ids=id_nodes, ) def test_resolve_ref(nodes, sources, package, version, expected): manifest = make_manifest(nodes=nodes, sources=sources) result = manifest.resolve_ref( source_node=None, target_model_name="my_model", target_model_package=package, target_model_version=version, current_project="root", node_package="root", ) if expected is None: assert result is expected else: assert result is not None assert len(expected) in (2, 3) if len(expected) == 2: expected_package, expected_name = expected elif len(expected) == 3: expected_package, expected_name, expected_version = expected assert result.version == expected_version assert result.name == expected_name assert result.package_name == expected_package @pytest.mark.parametrize( "nodes,sources,package,version,expected", _ambiguous_ref_parameter_sets(), ids=id_nodes, ) def test_resolve_ref_ambiguous_resource_name_across_packages( nodes, sources, package, version, expected ): manifest = make_manifest( nodes=nodes, sources=sources, ) with pytest.raises(AmbiguousResourceNameRefError): manifest.resolve_ref( source_node=None, target_model_name="my_model", target_model_package=None, target_model_version=version, current_project="root", node_package="root", ) @pytest.mark.parametrize( "nodes,sources,package,version,expected", _duplicate_node_name_across_packages_ref_parameter_sets(), ids=id_nodes, ) def test_resolve_ref_with_node_package_legacy(nodes, sources, package, version, expected): set_from_args( Namespace( SEND_ANONYMOUS_USAGE_STATS=False, REQUIRE_REF_SEARCHES_NODE_PACKAGE_BEFORE_ROOT=False, ), None, ) manifest = make_manifest(nodes=nodes, sources=sources) result = manifest.resolve_ref( source_node=None, target_model_name="my_model", target_model_package=package, target_model_version=version, current_project="root", node_package="project_a", ) assert result.name == "my_model" assert result.package_name == "root" @pytest.mark.parametrize( "nodes,sources,package,version,expected", _duplicate_node_name_across_packages_ref_parameter_sets(), ids=id_nodes, ) def test_resolve_ref_with_node_package(nodes, sources, package, version, expected): set_from_args( Namespace( SEND_ANONYMOUS_USAGE_STATS=False, REQUIRE_REF_SEARCHES_NODE_PACKAGE_BEFORE_ROOT=True, ), None, ) manifest = make_manifest(nodes=nodes, sources=sources) result = manifest.resolve_ref( source_node=None, target_model_name="my_model", target_model_package=package, target_model_version=version, current_project="root", node_package="project_a", ) assert result.name == "my_model" assert result.package_name == "project_a" def _source_parameter_sets(): sets = [ # empties FindNodeSpec(nodes=[], sources=[], package="dep", version=None, expected=None), FindNodeSpec(nodes=[], sources=[], package="root", version=None, expected=None), ] sets.extend( # models with the name, but not sources FindNodeSpec( nodes=[MockNode("root", name)], sources=[], package=project, version=None, expected=None, ) for project in ("root", "dep") for name in ("my_source", "my_table") ) # exists in root alongside nodes with name parts sets.extend( FindNodeSpec( nodes=[MockNode("root", "my_source"), MockNode("root", "my_table")], sources=[MockSource("root", "my_source", "my_table")], package=project, version=None, expected=("root", "my_source", "my_table"), ) for project in ("root", "dep") ) sets.extend( # wrong source name FindNodeSpec( nodes=[], sources=[MockSource("root", "my_other_source", "my_table")], package=project, version=None, expected=None, ) for project in ("root", "dep") ) sets.extend( # wrong table name FindNodeSpec( nodes=[], sources=[MockSource("root", "my_source", "my_other_table")], package=project, version=None, expected=None, ) for project in ("root", "dep") ) sets.append( # should be found by the package=None search FindNodeSpec( nodes=[], sources=[MockSource("other", "my_source", "my_table")], package="root", version=None, expected=("other", "my_source", "my_table"), ) ) sets.extend( # exists in root check various projects (other project -> not found) FindNodeSpec( nodes=[], sources=[MockSource("root", "my_source", "my_table")], package=project, version=None, expected=("root", "my_source", "my_table"), ) for project in ("root", "dep") ) return sets @pytest.mark.parametrize( "nodes,sources,package,version,expected", _source_parameter_sets(), ids=id_nodes, ) def test_resolve_source(nodes, sources, package, version, expected): set_from_args( Namespace( SEND_ANONYMOUS_USAGE_STATS=False, REQUIRE_REF_SEARCHES_NODE_PACKAGE_BEFORE_ROOT=False, ), None, ) manifest = make_manifest(nodes=nodes, sources=sources) result = manifest.resolve_source( target_source_name="my_source", target_table_name="my_table", current_project=package, node_package="dep", ) if expected is None: assert result is expected else: assert result is not None assert len(expected) == 3 expected_package, expected_source_name, expected_name = expected assert result.source_name == expected_source_name assert result.name == expected_name assert result.package_name == expected_package FindDocSpec = namedtuple("FindDocSpec", "docs,package,expected") def _docs_parameter_sets(): sets = [] sets.extend( # empty FindDocSpec(docs=[], package=project, expected=None) for project in ("root", None) ) sets.extend( # basic: exists in root FindDocSpec( docs=[MockDocumentation("root", "my_doc")], package=project, expected=("root", "my_doc"), ) for project in ("root", None) ) sets.extend( [ # exists in other FindDocSpec(docs=[MockDocumentation("dep", "my_doc")], package="root", expected=None), FindDocSpec( docs=[MockDocumentation("dep", "my_doc")], package=None, expected=("dep", "my_doc") ), ] ) return sets @pytest.mark.parametrize( "docs,package,expected", _docs_parameter_sets(), ids=id_nodes, ) def test_resolve_doc(docs, package, expected): manifest = make_manifest(docs=docs) result = manifest.resolve_doc( name="my_doc", package=package, current_project="root", node_package="root" ) if expected is None: assert result is expected else: assert result is not None assert len(expected) == 2 expected_package, expected_name = expected assert result.name == expected_name assert result.package_name == expected_package class TestManifestFindNodeFromRefOrSource: @pytest.fixture def mock_node(self): return MockNode("my_package", "my_model") @pytest.fixture def mock_disabled_node(self): return MockNode("my_package", "disabled_node", config={"enabled": False}) @pytest.fixture def mock_source(self): return MockSource("root", "my_source", "source_table") @pytest.fixture def mock_disabled_source(self): return MockSource("root", "my_source", "disabled_source_table", config={"enabled": False}) @pytest.fixture def mock_manifest(self, mock_node, mock_source, mock_disabled_node, mock_disabled_source): return make_manifest( nodes=[mock_node, mock_disabled_node], sources=[mock_source, mock_disabled_source] ) @pytest.mark.parametrize( "expression,expected_node", [ ("ref('my_package', 'my_model')", "mock_node"), ("ref('my_package', 'doesnt_exist')", None), ("ref('my_package', 'disabled_node')", "mock_disabled_node"), ("source('my_source', 'source_table')", "mock_source"), ("source('my_source', 'doesnt_exist')", None), ("source('my_source', 'disabled_source_table')", "mock_disabled_source"), ], ) def test_find_node_from_ref_or_source(self, expression, expected_node, mock_manifest, request): node = mock_manifest.find_node_from_ref_or_source(expression) if expected_node is None: assert node is None else: assert node == request.getfixturevalue(expected_node) @pytest.mark.parametrize("invalid_expression", ["invalid", "ref(')"]) def test_find_node_from_ref_or_source_invalid_expression( self, invalid_expression, mock_manifest ): with pytest.raises(ParsingError): mock_manifest.find_node_from_ref_or_source(invalid_expression) class TestDisabledLookup: @pytest.fixture(scope="class") def manifest(self): return Manifest( nodes={}, sources={}, macros={}, docs={}, disabled={}, files={}, exposures={}, selectors={}, ) @pytest.fixture(scope="class") def mock_model(self): return MockNode("package", "name", NodeType.Model) @pytest.fixture(scope="class") def mock_model_with_version(self): return MockNode("package", "name", NodeType.Model, version=3) @pytest.fixture(scope="class") def mock_seed(self): return MockNode("package", "name", NodeType.Seed) def test_find(self, manifest, mock_model): manifest.disabled = {"model.package.name": [mock_model]} lookup = DisabledLookup(manifest) assert lookup.find("name", "package") == [mock_model] def test_find_wrong_name(self, manifest, mock_model): manifest.disabled = {"model.package.name": [mock_model]} lookup = DisabledLookup(manifest) assert lookup.find("missing_name", "package") is None def test_find_wrong_package(self, manifest, mock_model): manifest.disabled = {"model.package.name": [mock_model]} lookup = DisabledLookup(manifest) assert lookup.find("name", "missing_package") is None def test_find_wrong_version(self, manifest, mock_model): manifest.disabled = {"model.package.name": [mock_model]} lookup = DisabledLookup(manifest) assert lookup.find("name", "package", version=3) is None def test_find_wrong_resource_types(self, manifest, mock_model): manifest.disabled = {"model.package.name": [mock_model]} lookup = DisabledLookup(manifest) assert lookup.find("name", "package", resource_types=[NodeType.Analysis]) is None def test_find_no_package(self, manifest, mock_model): manifest.disabled = {"model.package.name": [mock_model]} lookup = DisabledLookup(manifest) assert lookup.find("name", None) == [mock_model] def test_find_versioned_node(self, manifest, mock_model_with_version): manifest.disabled = {"model.package.name": [mock_model_with_version]} lookup = DisabledLookup(manifest) assert lookup.find("name", "package", version=3) == [mock_model_with_version] def test_find_versioned_node_no_package(self, manifest, mock_model_with_version): manifest.disabled = {"model.package.name": [mock_model_with_version]} lookup = DisabledLookup(manifest) assert lookup.find("name", None, version=3) == [mock_model_with_version] def test_find_versioned_node_no_version(self, manifest, mock_model_with_version): manifest.disabled = {"model.package.name": [mock_model_with_version]} lookup = DisabledLookup(manifest) assert lookup.find("name", "package") is None def test_find_versioned_node_wrong_version(self, manifest, mock_model_with_version): manifest.disabled = {"model.package.name": [mock_model_with_version]} lookup = DisabledLookup(manifest) assert lookup.find("name", "package", version=2) is None def test_find_versioned_node_wrong_name(self, manifest, mock_model_with_version): manifest.disabled = {"model.package.name": [mock_model_with_version]} lookup = DisabledLookup(manifest) assert lookup.find("wrong_name", "package", version=3) is None def test_find_versioned_node_wrong_package(self, manifest, mock_model_with_version): manifest.disabled = {"model.package.name": [mock_model_with_version]} lookup = DisabledLookup(manifest) assert lookup.find("name", "wrong_package", version=3) is None def test_find_multiple_nodes(self, manifest, mock_model, mock_seed): manifest.disabled = {"model.package.name": [mock_model, mock_seed]} lookup = DisabledLookup(manifest) assert lookup.find("name", "package") == [mock_model, mock_seed] def test_find_multiple_nodes_with_resource_types(self, manifest, mock_model, mock_seed): manifest.disabled = {"model.package.name": [mock_model, mock_seed]} lookup = DisabledLookup(manifest) assert lookup.find("name", "package", resource_types=[NodeType.Model]) == [mock_model] def test_find_multiple_nodes_with_wrong_resource_types(self, manifest, mock_model, mock_seed): manifest.disabled = {"model.package.name": [mock_model, mock_seed]} lookup = DisabledLookup(manifest) assert lookup.find("name", "package", resource_types=[NodeType.Analysis]) is None def test_find_multiple_nodes_with_resource_types_empty(self, manifest, mock_model, mock_seed): manifest.disabled = {"model.package.name": [mock_model, mock_seed]} lookup = DisabledLookup(manifest) assert lookup.find("name", "package", resource_types=[]) is None ================================================ FILE: tests/unit/contracts/graph/test_node_args.py ================================================ from dbt.contracts.graph.node_args import ModelNodeArgs class TestModelNodeArgs: def test_model_node_args_unique_id(self) -> None: model_node_args = ModelNodeArgs( name="name", package_name="package", identifier="identifier", schema="schema" ) assert model_node_args.unique_id == "model.package.name" def test_model_node_args_unique_id_with_version(self) -> None: model_node_args = ModelNodeArgs( name="name", package_name="package", identifier="identifier", schema="schema", version=1, ) assert model_node_args.unique_id == "model.package.name.v1" def test_model_node_args_fqn(self) -> None: model_node_args = ModelNodeArgs( name="name", package_name="package", identifier="identifier", schema="schema", ) assert model_node_args.fqn == ["package", "name"] def test_model_node_args_fqn_with_version(self) -> None: model_node_args = ModelNodeArgs( name="name", package_name="package", identifier="identifier", schema="schema", version=1, ) assert model_node_args.fqn == ["package", "name", "v1"] def test_model_node_args_fqn_with_version_zero(self) -> None: model_node_args = ModelNodeArgs( name="name", package_name="package", identifier="identifier", schema="schema", version=0, ) assert model_node_args.fqn == ["package", "name", "v0"] ================================================ FILE: tests/unit/contracts/graph/test_nodes.py ================================================ import pickle import re from argparse import Namespace from dataclasses import replace import pytest from dbt.artifacts.resources import ColumnInfo, TestConfig, TestMetadata from dbt.compilation import inject_ctes_into_sql from dbt.contracts.files import FileHash from dbt.contracts.graph.nodes import ( DependsOn, GenericTestNode, InjectedCTE, ModelConfig, ModelNode, ) from dbt.node_types import NodeType from tests.unit.fixtures import generic_test_node, model_node from tests.unit.utils import ( assert_fails_validation, assert_from_dict, assert_symmetric, replace_config, ) @pytest.fixture def args_for_flags() -> Namespace: return Namespace(state_modified_compare_vars=False) def norm_whitespace(string): _RE_COMBINE_WHITESPACE = re.compile(r"\s+") string = _RE_COMBINE_WHITESPACE.sub(" ", string).strip() return string @pytest.fixture def basic_uncompiled_model(): return ModelNode( package_name="test", path="/root/models/foo.sql", original_file_path="models/foo.sql", language="sql", raw_code='select * from {{ ref("other") }}', name="foo", resource_type=NodeType.Model, unique_id="model.test.foo", fqn=["test", "models", "foo"], refs=[], sources=[], metrics=[], depends_on=DependsOn(), description="", database="test_db", schema="test_schema", alias="bar", tags=[], config=ModelConfig(), meta={}, compiled=False, extra_ctes=[], extra_ctes_injected=False, checksum=FileHash.from_contents(""), unrendered_config={}, ) @pytest.fixture def basic_compiled_model(): return model_node() @pytest.fixture def minimal_uncompiled_dict(): return { "name": "foo", "created_at": 1, "resource_type": str(NodeType.Model), "path": "/root/models/foo.sql", "original_file_path": "models/foo.sql", "package_name": "test", "language": "sql", "raw_code": 'select * from {{ ref("other") }}', "unique_id": "model.test.foo", "fqn": ["test", "models", "foo"], "database": "test_db", "schema": "test_schema", "alias": "bar", "compiled": False, "checksum": { "name": "sha256", "checksum": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", }, "unrendered_config": {}, } @pytest.fixture def basic_uncompiled_dict(): return { "name": "foo", "created_at": 1, "resource_type": str(NodeType.Model), "path": "/root/models/foo.sql", "original_file_path": "models/foo.sql", "package_name": "test", "language": "sql", "raw_code": 'select * from {{ ref("other") }}', "unique_id": "model.test.foo", "fqn": ["test", "models", "foo"], "refs": [], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": []}, "database": "test_db", "description": "", "schema": "test_schema", "alias": "bar", "tags": [], "config": { "column_types": {}, "enabled": True, "materialized": "view", "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "tags": [], "on_schema_change": "ignore", "on_configuration_change": "apply", "meta": {}, "grants": {}, "packages": [], }, "docs": {"show": True}, "columns": {}, "meta": {}, "compiled": False, "extra_ctes": [], "extra_ctes_injected": False, "checksum": { "name": "sha256", "checksum": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", }, "unrendered_config": {}, "unrendered_config_call_dict": {}, "config_call_dict": {}, } @pytest.fixture def basic_compiled_dict(): return { "name": "foo", "created_at": 1, "resource_type": str(NodeType.Model), "path": "/root/models/foo.sql", "original_file_path": "models/foo.sql", "package_name": "test", "language": "sql", "raw_code": 'select * from {{ ref("other") }}', "unique_id": "model.test.foo", "fqn": ["test", "models", "foo"], "refs": [], "sources": [], "metrics": [], "functions": [], "depends_on": {"macros": [], "nodes": []}, "database": "test_db", "description": "", "primary_key": [], "schema": "test_schema", "alias": "bar", "tags": [], "config": { "column_types": {}, "enabled": True, "materialized": "view", "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "tags": [], "on_schema_change": "ignore", "on_configuration_change": "apply", "meta": {}, "grants": {}, "packages": [], "contract": {"enforced": False, "alias_types": True}, "docs": {"show": True}, "access": "protected", "lookback": 1, }, "docs": {"show": True}, "columns": {}, "contract": {"enforced": False, "alias_types": True}, "meta": {}, "compiled": True, "extra_ctes": [{"id": "whatever", "sql": "select * from other"}], "extra_ctes_injected": True, "compiled_code": "with whatever as (select * from other) select * from whatever", "checksum": { "name": "sha256", "checksum": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", }, "unrendered_config": {}, "unrendered_config_call_dict": {}, "config_call_dict": {}, "access": "protected", "constraints": [], "doc_blocks": [], } @pytest.mark.skip("Haven't found where we would use uncompiled node") def test_basic_uncompiled_model( minimal_uncompiled_dict, basic_uncompiled_dict, basic_uncompiled_model ): node_dict = basic_uncompiled_dict node = basic_uncompiled_model assert_symmetric(node, node_dict, ModelNode) assert node.empty is False assert node.is_refable is True assert node.is_ephemeral is False assert_from_dict(node, minimal_uncompiled_dict, ModelNode) pickle.loads(pickle.dumps(node)) def test_basic_compiled_model(basic_compiled_dict, basic_compiled_model): node_dict = basic_compiled_dict node = basic_compiled_model assert_symmetric(node, node_dict, ModelNode) assert node.empty is False assert node.is_refable is True assert node.is_ephemeral is False def test_extra_fields_model_okay(minimal_uncompiled_dict): extra = minimal_uncompiled_dict extra["notvalid"] = "nope" # Model still load fine with extra fields loaded_model = ModelNode.from_dict(extra) assert not hasattr(loaded_model, "notvalid") def test_invalid_bad_type_model(minimal_uncompiled_dict): bad_type = minimal_uncompiled_dict bad_type["resource_type"] = str(NodeType.Macro) assert_fails_validation(bad_type, ModelNode) unchanged_compiled_models = [ lambda u: (u, replace(u, description="a description")), lambda u: (u, replace(u, tags=["mytag"])), lambda u: (u, replace(u, meta={"cool_key": "cool value"})), # changing the final alias/schema/datbase isn't a change - could just be target changing! lambda u: (u, replace(u, database="nope")), lambda u: (u, replace(u, schema="nope")), lambda u: (u, replace(u, alias="nope")), # None -> False is a config change even though it's pretty much the same lambda u: ( replace(u, config=replace(u.config, persist_docs={"relation": False})), replace(u, config=replace(u.config, persist_docs={"relation": False})), ), lambda u: ( replace(u, config=replace(u.config, persist_docs={"columns": False})), replace(u, config=replace(u.config, persist_docs={"columns": False})), ), # True -> True lambda u: ( replace(u, config=replace(u.config, persist_docs={"relation": True})), replace(u, config=replace(u.config, persist_docs={"relation": True})), ), lambda u: ( replace(u, config=replace(u.config, persist_docs={"columns": True})), replace(u, config=replace(u.config, persist_docs={"columns": True})), ), # only columns docs enabled, but description changed lambda u: ( replace(u, config=replace(u.config, persist_docs={"columns": True})), replace( u, config=replace(u.config, persist_docs={"columns": True}), description="a model description", ), ), # only relation docs eanbled, but columns changed lambda u: ( replace(u, config=replace(u.config, persist_docs={"relation": True})), replace( u, config=replace(u.config, persist_docs={"relation": True}), columns={"a": ColumnInfo(name="a", description="a column description")}, ), ), ] changed_compiled_models = [ lambda u: (u, None), lambda u: (u, replace(u, raw_code="select * from wherever")), lambda u: ( u, replace( u, fqn=["test", "models", "subdir", "foo"], original_file_path="models/subdir/foo.sql", path="/root/models/subdir/foo.sql", ), ), lambda u: (u, replace_config(u, full_refresh=True)), lambda u: (u, replace_config(u, post_hook=["select 1 as id"])), lambda u: (u, replace_config(u, pre_hook=["select 1 as id"])), lambda u: ( u, replace_config(u, quoting={"database": True, "schema": False, "identifier": False}), ), # we changed persist_docs values lambda u: (u, replace_config(u, persist_docs={"relation": True})), lambda u: (u, replace_config(u, persist_docs={"columns": True})), lambda u: (u, replace_config(u, persist_docs={"columns": True, "relation": True})), # None -> False is a config change even though it's pretty much the same lambda u: (u, replace_config(u, persist_docs={"relation": False})), lambda u: (u, replace_config(u, persist_docs={"columns": False})), # persist docs was true for the relation and we changed the model description lambda u: ( replace_config(u, persist_docs={"relation": True}), replace_config(u, persist_docs={"relation": True}, description="a model description"), ), # persist docs was true for columns and we changed the model description lambda u: ( replace_config(u, persist_docs={"columns": True}), replace_config( u, persist_docs={"columns": True}, columns={"a": ColumnInfo(name="a", description="a column description")}, ), ), # changing alias/schema/database on the config level is a change lambda u: (u, replace_config(u, database="nope")), lambda u: (u, replace_config(u, schema="nope")), lambda u: (u, replace_config(u, alias="nope")), ] @pytest.mark.parametrize("func", unchanged_compiled_models) def test_compare_unchanged_model(func, basic_uncompiled_model): node, compare = func(basic_uncompiled_model) assert node.same_contents(compare, "postgres") @pytest.mark.parametrize("func", changed_compiled_models) def test_compare_changed_model(func, basic_uncompiled_model): node, compare = func(basic_uncompiled_model) assert not node.same_contents(compare, "postgres") @pytest.fixture def minimal_schema_test_dict(): return { "name": "foo", "created_at": 1, "resource_type": str(NodeType.Test), "path": "/root/x/path.sql", "original_file_path": "/root/path.sql", "package_name": "test", "language": "sql", "raw_code": 'select * from {{ ref("other") }}', "unique_id": "model.test.foo", "fqn": ["test", "models", "foo"], "database": "test_db", "schema": "dbt_test__audit", "alias": "bar", "test_metadata": { "name": "foo", "kwargs": {}, }, "compiled": False, "checksum": { "name": "sha256", "checksum": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", }, } @pytest.fixture def basic_uncompiled_schema_test_node(): return GenericTestNode( package_name="test", path="/root/x/path.sql", original_file_path="/root/path.sql", language="sql", raw_code='select * from {{ ref("other") }}', name="foo", resource_type=NodeType.Test, unique_id="model.test.foo", fqn=["test", "models", "foo"], refs=[], sources=[], metrics=[], depends_on=DependsOn(), description="", database="test_db", schema="dbt_test__audit", alias="bar", tags=[], config=TestConfig(), meta={}, compiled=False, extra_ctes=[], extra_ctes_injected=False, test_metadata=TestMetadata(namespace=None, name="foo", kwargs={}), checksum=FileHash.from_contents(""), unrendered_config={}, ) @pytest.fixture def basic_compiled_schema_test_node(): return generic_test_node() @pytest.fixture def basic_uncompiled_schema_test_dict(): return { "name": "foo", "created_at": 1, "resource_type": str(NodeType.Test), "path": "/root/x/path.sql", "original_file_path": "/root/path.sql", "package_name": "test", "language": "sql", "raw_code": 'select * from {{ ref("other") }}', "unique_id": "model.test.foo", "fqn": ["test", "models", "foo"], "refs": [], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": []}, "database": "test_db", "description": "", "schema": "dbt_test__audit", "alias": "bar", "tags": [], "config": { "enabled": True, "materialized": "test", "tags": [], "severity": "ERROR", "schema": "dbt_test__audit", "warn_if": "!= 0", "error_if": "!= 0", "fail_calc": "count(*)", "meta": {}, }, "docs": {"show": True}, "columns": {}, "meta": {}, "compiled": False, "extra_ctes": [], "extra_ctes_injected": False, "test_metadata": { "name": "foo", "kwargs": {}, }, "checksum": { "name": "sha256", "checksum": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", }, "unrendered_config": {}, "unrendered_config_call_dict": {}, "config_call_dict": {}, } @pytest.fixture def basic_compiled_schema_test_dict(): return { "name": "foo", "created_at": 1, "resource_type": str(NodeType.Test), "path": "/root/x/path.sql", "original_file_path": "/root/path.sql", "package_name": "test", "language": "sql", "raw_code": 'select * from {{ ref("other") }}', "unique_id": "model.test.foo", "fqn": ["test", "models", "foo"], "refs": [], "sources": [], "metrics": [], "functions": [], "depends_on": {"macros": [], "nodes": []}, "database": "test_db", "description": "", "schema": "dbt_test__audit", "alias": "bar", "tags": [], "config": { "enabled": True, "materialized": "test", "tags": [], "severity": "warn", "schema": "dbt_test__audit", "warn_if": "!= 0", "error_if": "!= 0", "fail_calc": "count(*)", "meta": {}, }, "docs": {"show": True}, "columns": {}, "contract": {"enforced": False, "alias_types": True}, "meta": {}, "compiled": True, "extra_ctes": [{"id": "whatever", "sql": "select * from other"}], "extra_ctes_injected": True, "compiled_code": "with whatever as (select * from other) select * from whatever", "column_name": "id", "test_metadata": { "name": "foo", "kwargs": {}, }, "checksum": { "name": "sha256", "checksum": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", }, "unrendered_config": { "severity": "warn", }, "unrendered_config_call_dict": {}, "config_call_dict": {}, "doc_blocks": [], } @pytest.mark.skip("Haven't found where we would use uncompiled node") def test_basic_uncompiled_schema_test( basic_uncompiled_schema_test_node, basic_uncompiled_schema_test_dict, minimal_schema_test_dict ): node = basic_uncompiled_schema_test_node node_dict = basic_uncompiled_schema_test_dict minimum = minimal_schema_test_dict assert_symmetric(node, node_dict, GenericTestNode) assert node.empty is False assert node.is_refable is False assert node.is_ephemeral is False assert_from_dict(node, minimum, GenericTestNode) def test_basic_compiled_schema_test( basic_compiled_schema_test_node, basic_compiled_schema_test_dict ): node = basic_compiled_schema_test_node node_dict = basic_compiled_schema_test_dict assert_symmetric(node, node_dict, GenericTestNode) assert node.empty is False assert node.is_refable is False assert node.is_ephemeral is False def test_invalid_extra_schema_test_fields(minimal_schema_test_dict): bad_extra = minimal_schema_test_dict bad_extra["extra"] = "extra value" assert_fails_validation(bad_extra, GenericTestNode) def test_invalid_resource_type_schema_test(minimal_schema_test_dict): bad_type = minimal_schema_test_dict bad_type["resource_type"] = str(NodeType.Model) assert_fails_validation(bad_type, GenericTestNode) unchanged_schema_tests = [ # for tests, raw_code isn't a change (because it's always the same for a given test macro) lambda u: replace(u, raw_code="select * from wherever"), lambda u: replace(u, description="a description"), lambda u: replace(u, tags=["mytag"]), lambda u: replace(u, meta={"cool_key": "cool value"}), # these values don't even mean anything on schema tests! lambda u: replace_config(u, alias="nope"), lambda u: replace_config(u, database="nope"), lambda u: replace_config(u, schema="nope"), lambda u: replace(u, database="other_db"), lambda u: replace(u, schema="other_schema"), lambda u: replace(u, alias="foo"), lambda u: replace_config(u, full_refresh=True), lambda u: replace_config(u, post_hook=["select 1 as id"]), lambda u: replace_config(u, pre_hook=["select 1 as id"]), lambda u: replace_config(u, quoting={"database": True, "schema": False, "identifier": False}), ] changed_schema_tests = [ lambda u: None, lambda u: replace( u, fqn=["test", "models", "subdir", "foo"], original_file_path="models/subdir/foo.sql", path="/root/models/subdir/foo.sql", ), lambda u: replace_config(u, severity="warn"), # If we checked test metadata, these would caount. But we don't, because these changes would all change the unique ID, so it's irrelevant. # lambda u: replace(u, test_metadata=replace(u.test_metadata, namespace='something')), # lambda u: replace(u, test_metadata=replace(u.test_metadata, name='bar')), # lambda u: replace(u, test_metadata=replace(u.test_metadata, kwargs={'arg': 'value'})), ] @pytest.mark.parametrize("func", unchanged_schema_tests) def test_compare_unchanged_schema_test(func, basic_uncompiled_schema_test_node): value = func(basic_uncompiled_schema_test_node) assert basic_uncompiled_schema_test_node.same_contents(value, "postgres") @pytest.mark.parametrize("func", changed_schema_tests) def test_compare_changed_schema_test(func, basic_uncompiled_schema_test_node): value = func(basic_uncompiled_schema_test_node) assert not basic_uncompiled_schema_test_node.same_contents(value, "postgres") def test_compare_to_compiled(basic_uncompiled_schema_test_node, basic_compiled_schema_test_node): # if you fix the severity, they should be the "same". uncompiled = basic_uncompiled_schema_test_node compiled = basic_compiled_schema_test_node assert not uncompiled.same_contents(compiled, "postgres") fixed_config = replace(compiled.config, severity=uncompiled.config.severity) fixed_compiled = replace( compiled, config=fixed_config, unrendered_config=uncompiled.unrendered_config ) assert uncompiled.same_contents(fixed_compiled, "postgres") def test_inject_ctes_simple1(): starting_sql = "select * from __dbt__cte__base" ctes = [ InjectedCTE( id="model.test.base", sql=" __dbt__cte__base as (\n\n\nselect * from test16873767336887004702_test_ephemeral.seed\n)", ) ] expected_sql = """with __dbt__cte__base as ( select * from test16873767336887004702_test_ephemeral.seed ) select * from __dbt__cte__base""" generated_sql = inject_ctes_into_sql(starting_sql, ctes) assert norm_whitespace(generated_sql) == norm_whitespace(expected_sql) def test_inject_ctes_simple2(): starting_sql = "select * from __dbt__cte__ephemeral_level_two" ctes = [ InjectedCTE( id="model.test.ephemeral_level_two", sql=' __dbt__cte__ephemeral_level_two as (\n\nselect * from "dbt"."test16873757769710148165_test_ephemeral"."source_table"\n)', ) ] expected_sql = """with __dbt__cte__ephemeral_level_two as ( select * from "dbt"."test16873757769710148165_test_ephemeral"."source_table" ) select * from __dbt__cte__ephemeral_level_two""" generated_sql = inject_ctes_into_sql(starting_sql, ctes) assert norm_whitespace(generated_sql) == norm_whitespace(expected_sql) def test_inject_ctes_multiple_ctes(): starting_sql = "select * from __dbt__cte__ephemeral" ctes = [ InjectedCTE( id="model.test.ephemeral_level_two", sql=' __dbt__cte__ephemeral_level_two as (\n\nselect * from "dbt"."test16873735573223965828_test_ephemeral"."source_table"\n)', ), InjectedCTE( id="model.test.ephemeral", sql=" __dbt__cte__ephemeral as (\n\nselect * from __dbt__cte__ephemeral_level_two\n)", ), ] expected_sql = """with __dbt__cte__ephemeral_level_two as ( select * from "dbt"."test16873735573223965828_test_ephemeral"."source_table" ), __dbt__cte__ephemeral as ( select * from __dbt__cte__ephemeral_level_two ) select * from __dbt__cte__ephemeral""" generated_sql = inject_ctes_into_sql(starting_sql, ctes) assert norm_whitespace(generated_sql) == norm_whitespace(expected_sql) def test_inject_ctes_multiple_ctes_more_complex(): starting_sql = """select * from __dbt__cte__female_only union all select * from "dbt"."test16873757723266827902_test_ephemeral"."double_dependent" where gender = 'Male'""" ctes = [ InjectedCTE( id="model.test.base", sql=" __dbt__cte__base as (\n\n\nselect * from test16873757723266827902_test_ephemeral.seed\n)", ), InjectedCTE( id="model.test.base_copy", sql=" __dbt__cte__base_copy as (\n\n\nselect * from __dbt__cte__base\n)", ), InjectedCTE( id="model.test.female_only", sql=" __dbt__cte__female_only as (\n\n\nselect * from __dbt__cte__base_copy where gender = 'Female'\n)", ), ] expected_sql = """with __dbt__cte__base as ( select * from test16873757723266827902_test_ephemeral.seed ), __dbt__cte__base_copy as ( select * from __dbt__cte__base ), __dbt__cte__female_only as ( select * from __dbt__cte__base_copy where gender = 'Female' ) select * from __dbt__cte__female_only union all select * from "dbt"."test16873757723266827902_test_ephemeral"."double_dependent" where gender = 'Male'""" generated_sql = inject_ctes_into_sql(starting_sql, ctes) assert norm_whitespace(generated_sql) == norm_whitespace(expected_sql) def test_inject_ctes_starting_with1(): starting_sql = """ with internal_cte as (select * from sessions) select * from internal_cte """ ctes = [ InjectedCTE( id="cte_id_1", sql="__dbt__cte__ephemeral as (select * from table)", ), InjectedCTE( id="cte_id_2", sql="__dbt__cte__events as (select id, type from events)", ), ] expected_sql = """with __dbt__cte__ephemeral as (select * from table), __dbt__cte__events as (select id, type from events), internal_cte as (select * from sessions) select * from internal_cte""" generated_sql = inject_ctes_into_sql(starting_sql, ctes) assert norm_whitespace(generated_sql) == norm_whitespace(expected_sql) def test_inject_ctes_starting_with2(): starting_sql = """with my_other_cool_cte as ( select id, name from __dbt__cte__ephemeral where id > 1000 ) select name, id from my_other_cool_cte""" ctes = [ InjectedCTE( id="model.singular_tests_ephemeral.ephemeral", sql=' __dbt__cte__ephemeral as (\n\n\nwith my_cool_cte as (\n select name, id from "dbt"."test16873917221900185954_test_singular_tests_ephemeral"."base"\n)\nselect id, name from my_cool_cte where id is not null\n)', ) ] expected_sql = """with __dbt__cte__ephemeral as ( with my_cool_cte as ( select name, id from "dbt"."test16873917221900185954_test_singular_tests_ephemeral"."base" ) select id, name from my_cool_cte where id is not null ), my_other_cool_cte as ( select id, name from __dbt__cte__ephemeral where id > 1000 ) select name, id from my_other_cool_cte""" generated_sql = inject_ctes_into_sql(starting_sql, ctes) assert norm_whitespace(generated_sql) == norm_whitespace(expected_sql) def test_inject_ctes_comment_with(): # Test injection with a comment containing "with" starting_sql = """ --- This is sql with a comment select * from __dbt__cte__base """ ctes = [ InjectedCTE( id="model.test.base", sql=" __dbt__cte__base as (\n\n\nselect * from test16873767336887004702_test_ephemeral.seed\n)", ) ] expected_sql = """with __dbt__cte__base as ( select * from test16873767336887004702_test_ephemeral.seed ) --- This is sql with a comment select * from __dbt__cte__base""" generated_sql = inject_ctes_into_sql(starting_sql, ctes) assert norm_whitespace(generated_sql) == norm_whitespace(expected_sql) def test_inject_ctes_with_recursive(): # Test injection with "recursive" keyword starting_sql = """ with recursive t(n) as ( select * from __dbt__cte__first_ephemeral_model union all select n+1 from t where n < 100 ) select sum(n) from t """ ctes = [ InjectedCTE( id="model.test.first_ephemeral_model", sql=" __dbt__cte__first_ephemeral_model as (\n\nselect 1 as fun\n)", ) ] expected_sql = """with recursive __dbt__cte__first_ephemeral_model as ( select 1 as fun ), t(n) as ( select * from __dbt__cte__first_ephemeral_model union all select n+1 from t where n < 100 ) select sum(n) from t """ generated_sql = inject_ctes_into_sql(starting_sql, ctes) assert norm_whitespace(generated_sql) == norm_whitespace(expected_sql) ================================================ FILE: tests/unit/contracts/graph/test_nodes_parsed.py ================================================ import pickle from argparse import Namespace from dataclasses import replace import pytest from hypothesis import given from hypothesis.strategies import builds, lists from dbt.artifacts.resources import ( ColumnInfo, Dimension, Entity, ExposureConfig, ExposureType, FreshnessThreshold, Hook, MacroConfig, MacroDependsOn, MaturityType, Measure, MetricInputMeasure, MetricTypeParams, Owner, Quoting, RefArgs, SourceConfig, ) from dbt.artifacts.resources import SourceDefinition as SourceDefinitionResource from dbt.artifacts.resources import TestMetadata, Time from dbt.artifacts.resources.types import TimePeriod from dbt.contracts.files import FileHash from dbt.contracts.graph.model_config import ( ModelConfig, NodeConfig, SeedConfig, SnapshotConfig, TestConfig, ) from dbt.contracts.graph.nodes import ( DependsOn, Docs, Documentation, Exposure, GenericTestNode, HookNode, Macro, Metric, ModelNode, SeedNode, SemanticModel, SnapshotNode, SourceDefinition, ) from dbt.node_types import AccessType, NodeType from dbt_common.dataclass_schema import ValidationError from dbt_semantic_interfaces.type_enums import MetricType from tests.unit.utils import ( ContractTestCase, assert_fails_validation, assert_from_dict, assert_symmetric, compare_dicts, dict_replace, replace_config, ) @pytest.fixture def args_for_flags() -> Namespace: return Namespace( send_anonymous_usage_stats=False, state_modified_compare_more_unrendered_values=False, state_modified_compare_vars=False, ) @pytest.fixture def populated_node_config_object(): result = ModelConfig( column_types={"a": "text"}, materialized="table", post_hook=[Hook(sql='insert into blah(a, b) select "1", 1')], ) result._extra["extra"] = "even more" return result @pytest.fixture def populated_node_config_dict(): return { "column_types": {"a": "text"}, "enabled": True, "materialized": "table", "persist_docs": {}, "post-hook": [{"sql": 'insert into blah(a, b) select "1", 1', "transaction": True}], "pre-hook": [], "quoting": {}, "tags": [], "extra": "even more", "on_schema_change": "ignore", "on_configuration_change": "apply", "meta": {}, "grants": {}, "packages": [], "docs": {"show": True}, "contract": {"enforced": False, "alias_types": True}, "access": "protected", "lookback": 1, } def test_config_populated(populated_node_config_object, populated_node_config_dict): assert_symmetric(populated_node_config_object, populated_node_config_dict, ModelConfig) pickle.loads(pickle.dumps(populated_node_config_object)) @pytest.fixture def unrendered_node_config_dict(): return { "column_types": {"a": "text"}, "materialized": "table", "post_hook": 'insert into blah(a, b) select "1", 1', } different_node_configs = [ lambda c: dict_replace(c, post_hook=[]), lambda c: dict_replace(c, materialized="view"), lambda c: dict_replace(c, quoting={"database": True}), lambda c: dict_replace(c, extra="different extra"), lambda c: dict_replace(c, column_types={"a": "varchar(256)"}), ] same_node_configs = [ lambda c: dict_replace(c, tags=["mytag"]), lambda c: dict_replace(c, alias="changed"), lambda c: dict_replace(c, schema="changed"), lambda c: dict_replace(c, database="changed"), ] @pytest.mark.parametrize("func", different_node_configs) def test_config_different(unrendered_node_config_dict, func): value = func(unrendered_node_config_dict) assert not ModelConfig.same_contents(unrendered_node_config_dict, value) @pytest.mark.parametrize("func", same_node_configs) def test_config_same(unrendered_node_config_dict, func): value = func(unrendered_node_config_dict) assert unrendered_node_config_dict != value assert ModelConfig.same_contents(unrendered_node_config_dict, value) @pytest.fixture def base_parsed_model_dict(): return { "name": "foo", "created_at": 1.0, "resource_type": str(NodeType.Model), "path": "/root/x/path.sql", "original_file_path": "/root/path.sql", "package_name": "test", "language": "sql", "raw_code": "select * from wherever", "unique_id": "model.test.foo", "fqn": ["test", "models", "foo"], "refs": [], "sources": [], "metrics": [], "functions": [], "depends_on": {"macros": [], "nodes": []}, "database": "test_db", "description": "", "primary_key": [], "schema": "test_schema", "alias": "bar", "tags": [], "config": { "column_types": {}, "enabled": True, "materialized": "view", "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "tags": [], "on_schema_change": "ignore", "on_configuration_change": "apply", "meta": {}, "grants": {}, "docs": {"show": True}, "contract": {"enforced": False, "alias_types": True}, "packages": [], "access": "protected", "lookback": 1, }, "docs": {"show": True}, "contract": {"enforced": False, "alias_types": True}, "columns": {}, "meta": {}, "checksum": { "name": "sha256", "checksum": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", }, "unrendered_config": {}, "unrendered_config_call_dict": {}, "config_call_dict": {}, "access": AccessType.Protected.value, "constraints": [], "doc_blocks": [], } @pytest.fixture def basic_parsed_model_object(): return ModelNode( package_name="test", path="/root/x/path.sql", original_file_path="/root/path.sql", language="sql", raw_code="select * from wherever", name="foo", resource_type=NodeType.Model, unique_id="model.test.foo", fqn=["test", "models", "foo"], refs=[], sources=[], metrics=[], depends_on=DependsOn(), description="", primary_key=[], database="test_db", schema="test_schema", alias="bar", tags=[], config=ModelConfig(), meta={}, checksum=FileHash.from_contents(""), created_at=1.0, ) @pytest.fixture def minimal_parsed_model_dict(): return { "name": "foo", "created_at": 1.0, "resource_type": str(NodeType.Model), "path": "/root/x/path.sql", "original_file_path": "/root/path.sql", "package_name": "test", "language": "sql", "raw_code": "select * from wherever", "unique_id": "model.test.foo", "fqn": ["test", "models", "foo"], "database": "test_db", "schema": "test_schema", "alias": "bar", "checksum": { "name": "sha256", "checksum": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", }, "unrendered_config": {}, } @pytest.fixture def complex_parsed_model_dict(): return { "name": "foo", "created_at": 1.0, "resource_type": str(NodeType.Model), "path": "/root/x/path.sql", "original_file_path": "/root/path.sql", "package_name": "test", "language": "sql", "raw_code": 'select * from {{ ref("bar") }}', "unique_id": "model.test.foo", "fqn": ["test", "models", "foo"], "refs": [], "sources": [], "metrics": [], "functions": [], "depends_on": {"macros": [], "nodes": ["model.test.bar"]}, "database": "test_db", "description": "My parsed node", "primary_key": [], "schema": "test_schema", "alias": "bar", "tags": ["tag"], "meta": {}, "config": { "column_types": {"a": "text"}, "enabled": True, "materialized": "ephemeral", "persist_docs": {}, "post-hook": [{"sql": 'insert into blah(a, b) select "1", 1', "transaction": True}], "pre-hook": [], "quoting": {}, "tags": [], "on_schema_change": "ignore", "on_configuration_change": "apply", "meta": {}, "grants": {}, "docs": {"show": True}, "contract": {"enforced": False, "alias_types": True}, "packages": [], "access": "protected", "lookback": 1, }, "docs": {"show": True}, "contract": {"enforced": False, "alias_types": True}, "columns": { "a": { "name": "a", "description": "a text field", "meta": {}, "tags": [], "constraints": [], "doc_blocks": [], "config": {"meta": {}, "tags": []}, }, }, "checksum": { "name": "sha256", "checksum": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", }, "unrendered_config": { "column_types": {"a": "text"}, "materialized": "ephemeral", "post_hook": ['insert into blah(a, b) select "1", 1'], }, "unrendered_config_call_dict": {}, "config_call_dict": {}, "access": AccessType.Protected.value, "constraints": [], "doc_blocks": [], } @pytest.fixture def complex_parsed_model_object(): return ModelNode( package_name="test", path="/root/x/path.sql", original_file_path="/root/path.sql", language="sql", raw_code='select * from {{ ref("bar") }}', name="foo", resource_type=NodeType.Model, unique_id="model.test.foo", fqn=["test", "models", "foo"], refs=[], sources=[], metrics=[], depends_on=DependsOn(nodes=["model.test.bar"]), description="My parsed node", database="test_db", schema="test_schema", alias="bar", tags=["tag"], meta={}, config=ModelConfig( column_types={"a": "text"}, materialized="ephemeral", post_hook=[Hook(sql='insert into blah(a, b) select "1", 1')], ), columns={"a": ColumnInfo("a", "a text field", {})}, checksum=FileHash.from_contents(""), unrendered_config={ "column_types": {"a": "text"}, "materialized": "ephemeral", "post_hook": ['insert into blah(a, b) select "1", 1'], }, ) def test_model_basic(basic_parsed_model_object, base_parsed_model_dict, minimal_parsed_model_dict): node = basic_parsed_model_object node_dict = base_parsed_model_dict compare_dicts(node.to_dict(), node_dict) assert_symmetric(node, node_dict) assert node.empty is False assert node.is_refable is True assert node.is_ephemeral is False minimum = minimal_parsed_model_dict assert_from_dict(node, minimum) pickle.loads(pickle.dumps(node)) def test_model_complex(complex_parsed_model_object, complex_parsed_model_dict): node = complex_parsed_model_object node_dict = complex_parsed_model_dict assert_symmetric(node, node_dict) assert node.empty is False assert node.is_refable is True assert node.is_ephemeral is True def test_invalid_bad_tags(base_parsed_model_dict): # bad top-level field bad_tags = base_parsed_model_dict bad_tags["tags"] = 100 assert_fails_validation(bad_tags, ModelNode) def test_invalid_bad_materialized(base_parsed_model_dict): # bad nested field bad_materialized = base_parsed_model_dict bad_materialized["config"]["materialized"] = None assert_fails_validation(bad_materialized, ModelNode) unchanged_nodes = [ lambda u: (u, replace(u, tags=["mytag"])), lambda u: (u, replace(u, meta={"something": 1000})), # True -> True lambda u: ( replace_config(u, persist_docs={"relation": True}), replace_config(u, persist_docs={"relation": True}), ), lambda u: ( replace_config(u, persist_docs={"columns": True}), replace_config(u, persist_docs={"columns": True}), ), # only columns docs enabled, but description changed lambda u: ( replace_config(u, persist_docs={"columns": True}), replace( replace_config(u, persist_docs={"columns": True}), description="a model description" ), ), # only relation docs eanbled, but columns changed lambda u: ( replace_config(u, persist_docs={"relation": True}), replace( replace_config(u, persist_docs={"relation": True}), columns={"a": ColumnInfo(name="a", description="a column description")}, ), ), # not tracked, we track config.alias/config.schema/config.database lambda u: (u, replace(u, alias="other")), lambda u: (u, replace(u, schema="other")), lambda u: (u, replace(u, database="other")), # unchanged ref representations - protected is default lambda u: (u, replace(u, access=AccessType.Protected)), ] changed_nodes = [ lambda u: ( u, replace( u, fqn=["test", "models", "subdir", "foo"], original_file_path="models/subdir/foo.sql", path="/root/models/subdir/foo.sql", ), ), # None -> False is a config change even though it's pretty much the same lambda u: (u, replace_config(u, persist_docs={"relation": False})), lambda u: (u, replace_config(u, persist_docs={"columns": False})), # persist docs was true for the relation and we changed the model description lambda u: ( replace_config(u, persist_docs={"relation": True}), replace( replace_config(u, persist_docs={"relation": True}), description="a model description" ), ), # persist docs was true for columns and we changed the model description lambda u: ( replace_config(u, persist_docs={"columns": True}), replace( replace_config(u, persist_docs={"columns": True}), columns={"a": ColumnInfo(name="a", description="a column description")}, ), ), # not tracked, we track config.alias/config.schema/config.database lambda u: (u, replace_config(u, alias="other")), lambda u: (u, replace_config(u, schema="other")), lambda u: (u, replace_config(u, database="other")), # changed ref representations lambda u: (u, replace_config(u, access=AccessType.Public)), lambda u: (u, replace_config(u, latest_version=2)), lambda u: (u, replace_config(u, version=2)), ] @pytest.mark.parametrize("func", unchanged_nodes) def test_compare_unchanged_parsed_model(func, basic_parsed_model_object): node, compare = func(basic_parsed_model_object) assert node.same_contents(compare, "postgres") @pytest.mark.parametrize("func", changed_nodes) def test_compare_changed_model(func, basic_parsed_model_object): node, compare = func(basic_parsed_model_object) assert not node.same_contents(compare, "postgres") @pytest.fixture def basic_parsed_seed_dict(): return { "name": "foo", "created_at": 1.0, "resource_type": str(NodeType.Seed), "path": "/root/seeds/seed.csv", "original_file_path": "seeds/seed.csv", "package_name": "test", "raw_code": "", "unique_id": "seed.test.foo", "fqn": ["test", "seeds", "foo"], "database": "test_db", "depends_on": {"macros": []}, "description": "", "schema": "test_schema", "tags": [], "alias": "foo", "config": { "column_types": {}, "delimiter": ",", "enabled": True, "materialized": "seed", "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "tags": [], "on_schema_change": "ignore", "on_configuration_change": "apply", "meta": {}, "grants": {}, "docs": {"show": True}, "contract": {"enforced": False, "alias_types": True}, "packages": [], "lookback": 1, }, "docs": {"show": True}, "columns": {}, "meta": {}, "checksum": {"name": "path", "checksum": "seeds/seed.csv"}, "unrendered_config": {}, "unrendered_config_call_dict": {}, "config_call_dict": {}, "doc_blocks": [], } @pytest.fixture def basic_parsed_seed_object(): return SeedNode( name="foo", resource_type=NodeType.Seed, path="/root/seeds/seed.csv", original_file_path="seeds/seed.csv", package_name="test", raw_code="", unique_id="seed.test.foo", fqn=["test", "seeds", "foo"], database="test_db", description="", schema="test_schema", tags=[], alias="foo", config=SeedConfig(), # config=SeedConfig(quote_columns=True), docs=Docs(show=True), columns={}, meta={}, checksum=FileHash(name="path", checksum="seeds/seed.csv"), unrendered_config={}, ) @pytest.fixture def minimal_parsed_seed_dict(): return { "name": "foo", "created_at": 1.0, "resource_type": str(NodeType.Seed), "path": "/root/seeds/seed.csv", "original_file_path": "seeds/seed.csv", "package_name": "test", "raw_code": "", "unique_id": "seed.test.foo", "fqn": ["test", "seeds", "foo"], "database": "test_db", "schema": "test_schema", "alias": "foo", "checksum": {"name": "path", "checksum": "seeds/seed.csv"}, } @pytest.fixture def complex_parsed_seed_dict(): return { "name": "foo", "created_at": 1.0, "resource_type": str(NodeType.Seed), "path": "/root/seeds/seed.csv", "original_file_path": "seeds/seed.csv", "package_name": "test", "raw_code": "", "unique_id": "seed.test.foo", "fqn": ["test", "seeds", "foo"], "database": "test_db", "depends_on": {"macros": []}, "description": "a description", "schema": "test_schema", "tags": ["mytag"], "alias": "foo", "config": { "column_types": {}, "delimiter": ",", "enabled": True, "materialized": "seed", "persist_docs": {"relation": True, "columns": True}, "post-hook": [], "pre-hook": [], "quoting": {}, "tags": [], "quote_columns": True, "on_schema_change": "ignore", "on_configuration_change": "apply", "meta": {}, "grants": {}, "docs": {"show": True}, "contract": {"enforced": False, "alias_types": True}, "packages": [], "lookback": 1, }, "docs": {"show": True}, "columns": { "a": { "name": "a", "description": "a column description", "meta": {}, "tags": [], "constraints": [], "doc_blocks": [], "config": {"meta": {}, "tags": []}, } }, "meta": {"foo": 1000}, "checksum": { "name": "sha256", "checksum": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", }, "unrendered_config": { "persist_docs": {"relation": True, "columns": True}, }, "unrendered_config_call_dict": {}, "config_call_dict": {}, "doc_blocks": [], } @pytest.fixture def complex_parsed_seed_object(): return SeedNode( name="foo", resource_type=NodeType.Seed, path="/root/seeds/seed.csv", original_file_path="seeds/seed.csv", package_name="test", raw_code="", unique_id="seed.test.foo", fqn=["test", "seeds", "foo"], database="test_db", depends_on=MacroDependsOn(), description="a description", schema="test_schema", tags=["mytag"], alias="foo", config=SeedConfig( quote_columns=True, delimiter=",", persist_docs={"relation": True, "columns": True}, ), docs=Docs(show=True), columns={"a": ColumnInfo(name="a", description="a column description")}, meta={"foo": 1000}, checksum=FileHash.from_contents(""), unrendered_config={ "persist_docs": {"relation": True, "columns": True}, }, ) def test_seed_basic(basic_parsed_seed_dict, basic_parsed_seed_object, minimal_parsed_seed_dict): assert_symmetric(basic_parsed_seed_object, basic_parsed_seed_dict) assert basic_parsed_seed_object.get_materialization() == "seed" assert_from_dict(basic_parsed_seed_object, minimal_parsed_seed_dict, SeedNode) def test_seed_complex(complex_parsed_seed_dict, complex_parsed_seed_object): assert_symmetric(complex_parsed_seed_object, complex_parsed_seed_dict) assert complex_parsed_seed_object.get_materialization() == "seed" unchanged_seeds = [ lambda u: (u, replace(u, tags=["mytag"])), lambda u: (u, replace(u, meta={"something": 1000})), # True -> True lambda u: ( replace_config(u, persist_docs={"relation": True}), replace_config(u, persist_docs={"relation": True}), ), lambda u: ( replace_config(u, persist_docs={"columns": True}), replace_config(u, persist_docs={"columns": True}), ), # only columns docs enabled, but description changed lambda u: ( replace_config(u, persist_docs={"columns": True}), replace( replace_config(u, persist_docs={"columns": True}), description="a model description" ), ), # only relation docs eanbled, but columns changed lambda u: ( replace_config(u, persist_docs={"relation": True}), replace( replace_config(u, persist_docs={"relation": True}), columns={"a": ColumnInfo(name="a", description="a column description")}, ), ), lambda u: (u, replace(u, alias="other")), lambda u: (u, replace(u, schema="other")), lambda u: (u, replace(u, database="other")), ] changed_seeds = [ lambda u: ( u, replace( u, fqn=["test", "models", "subdir", "foo"], original_file_path="models/subdir/foo.sql", path="/root/models/subdir/foo.sql", ), ), # None -> False is a config change even though it's pretty much the same lambda u: (u, replace_config(u, persist_docs={"relation": False})), lambda u: (u, replace_config(u, persist_docs={"columns": False})), # persist docs was true for the relation and we changed the model description lambda u: ( replace_config(u, persist_docs={"relation": True}), replace( replace_config(u, persist_docs={"relation": True}), description="a model description" ), ), # persist docs was true for columns and we changed the model description lambda u: ( replace_config(u, persist_docs={"columns": True}), replace( replace_config(u, persist_docs={"columns": True}), columns={"a": ColumnInfo(name="a", description="a column description")}, ), ), lambda u: (u, replace_config(u, alias="other")), lambda u: (u, replace_config(u, schema="other")), lambda u: (u, replace_config(u, database="other")), ] @pytest.mark.parametrize("func", unchanged_seeds) def test_compare_unchanged_parsed_seed(func, basic_parsed_seed_object): node, compare = func(basic_parsed_seed_object) assert node.same_contents(compare, "postgres") @pytest.mark.parametrize("func", changed_seeds) def test_compare_changed_seed(func, basic_parsed_seed_object): node, compare = func(basic_parsed_seed_object) assert not node.same_contents(compare, "postgres") @pytest.fixture def minimal_parsed_hook_dict(): return { "name": "foo", "resource_type": str(NodeType.Operation), "path": "/root/x/path.sql", "original_file_path": "/root/path.sql", "package_name": "test", "language": "sql", "raw_code": "select * from wherever", "unique_id": "model.test.foo", "fqn": ["test", "models", "foo"], "database": "test_db", "schema": "test_schema", "alias": "bar", "checksum": { "name": "sha256", "checksum": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", }, } @pytest.fixture def base_parsed_hook_dict(): return { "name": "foo", "created_at": 1.0, "resource_type": str(NodeType.Operation), "path": "/root/x/path.sql", "original_file_path": "/root/path.sql", "package_name": "test", "language": "sql", "raw_code": "select * from wherever", "unique_id": "model.test.foo", "fqn": ["test", "models", "foo"], "refs": [], "sources": [], "metrics": [], "functions": [], "depends_on": {"macros": [], "nodes": []}, "database": "test_db", "description": "", "schema": "test_schema", "alias": "bar", "tags": [], "config": { "column_types": {}, "enabled": True, "materialized": "view", "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "tags": [], "on_schema_change": "ignore", "on_configuration_change": "apply", "meta": {}, "grants": {}, "docs": {"show": True}, "contract": {"enforced": False, "alias_types": True}, "packages": [], "lookback": 1, }, "docs": {"show": True}, "contract": {"enforced": False, "alias_types": True}, "columns": {}, "meta": {}, "checksum": { "name": "sha256", "checksum": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", }, "unrendered_config": {}, "unrendered_config_call_dict": {}, "config_call_dict": {}, "doc_blocks": [], } @pytest.fixture def base_parsed_hook_object(): return HookNode( package_name="test", path="/root/x/path.sql", original_file_path="/root/path.sql", language="sql", raw_code="select * from wherever", name="foo", resource_type=NodeType.Operation, unique_id="model.test.foo", fqn=["test", "models", "foo"], refs=[], sources=[], metrics=[], depends_on=DependsOn(), description="", database="test_db", schema="test_schema", alias="bar", tags=[], config=NodeConfig(), index=None, checksum=FileHash.from_contents(""), unrendered_config={}, ) @pytest.fixture def complex_parsed_hook_dict(): return { "name": "foo", "created_at": 1.0, "resource_type": str(NodeType.Operation), "path": "/root/x/path.sql", "original_file_path": "/root/path.sql", "package_name": "test", "language": "sql", "raw_code": 'select * from {{ ref("bar") }}', "unique_id": "model.test.foo", "fqn": ["test", "models", "foo"], "refs": [], "sources": [], "metrics": [], "functions": [], "depends_on": {"macros": [], "nodes": ["model.test.bar"]}, "database": "test_db", "description": "My parsed node", "schema": "test_schema", "alias": "bar", "tags": ["tag"], "meta": {}, "config": { "column_types": {"a": "text"}, "enabled": True, "materialized": "table", "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "tags": [], "on_schema_change": "ignore", "on_configuration_change": "apply", "meta": {}, "grants": {}, "docs": {"show": True}, "contract": {"enforced": False, "alias_types": True}, "packages": [], "lookback": 1, }, "docs": {"show": True}, "contract": {"enforced": False, "alias_types": True}, "columns": { "a": { "name": "a", "description": "a text field", "meta": {}, "tags": [], "constraints": [], "doc_blocks": [], "config": { "meta": {}, "tags": [], }, }, }, "index": 13, "checksum": { "name": "sha256", "checksum": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", }, "unrendered_config": { "column_types": {"a": "text"}, "materialized": "table", }, "unrendered_config_call_dict": {}, "config_call_dict": {}, "doc_blocks": [], } @pytest.fixture def complex_parsed_hook_object(): return HookNode( package_name="test", path="/root/x/path.sql", original_file_path="/root/path.sql", language="sql", raw_code='select * from {{ ref("bar") }}', name="foo", resource_type=NodeType.Operation, unique_id="model.test.foo", fqn=["test", "models", "foo"], refs=[], sources=[], metrics=[], depends_on=DependsOn(nodes=["model.test.bar"]), description="My parsed node", database="test_db", schema="test_schema", alias="bar", tags=["tag"], meta={}, config=NodeConfig(column_types={"a": "text"}, materialized="table", post_hook=[]), columns={"a": ColumnInfo("a", "a text field", {})}, index=13, checksum=FileHash.from_contents(""), unrendered_config={ "column_types": {"a": "text"}, "materialized": "table", }, ) def test_basic_parsed_hook( minimal_parsed_hook_dict, base_parsed_hook_dict, base_parsed_hook_object ): node = base_parsed_hook_object node_dict = base_parsed_hook_dict minimum = minimal_parsed_hook_dict assert_symmetric(node, node_dict, HookNode) assert node.empty is False assert node.is_refable is False assert node.get_materialization() == "view" assert_from_dict(node, minimum, HookNode) pickle.loads(pickle.dumps(node)) def test_complex_parsed_hook(complex_parsed_hook_dict, complex_parsed_hook_object): node = complex_parsed_hook_object node_dict = complex_parsed_hook_dict # what's different? assert_symmetric(node, node_dict) assert node.empty is False assert node.is_refable is False assert node.get_materialization() == "table" def test_invalid_hook_index_type(base_parsed_hook_dict): bad_index = base_parsed_hook_dict bad_index["index"] = "a string!?" assert_fails_validation(bad_index, HookNode) @pytest.fixture def minimal_parsed_schema_test_dict(): return { "name": "foo", "created_at": 1.0, "resource_type": str(NodeType.Test), "path": "/root/x/path.sql", "original_file_path": "/root/path.sql", "package_name": "test", "language": "sql", "raw_code": "select * from wherever", "unique_id": "test.test.foo", "fqn": ["test", "models", "foo"], "database": "test_db", "schema": "test_schema", "alias": "bar", "meta": {}, "test_metadata": { "name": "foo", "kwargs": {}, }, "checksum": { "name": "sha256", "checksum": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", }, "unrendered_config_call_dict": {}, "config_call_dict": {}, } @pytest.fixture def basic_parsed_schema_test_dict(): return { "name": "foo", "created_at": 1.0, "resource_type": str(NodeType.Test), "path": "/root/x/path.sql", "original_file_path": "/root/path.sql", "package_name": "test", "language": "sql", "raw_code": "select * from wherever", "unique_id": "test.test.foo", "fqn": ["test", "models", "foo"], "refs": [], "sources": [], "metrics": [], "functions": [], "depends_on": {"macros": [], "nodes": []}, "database": "test_db", "description": "", "schema": "test_schema", "alias": "bar", "tags": [], "meta": {}, "config": { "enabled": True, "materialized": "test", "tags": [], "severity": "ERROR", "warn_if": "!= 0", "error_if": "!= 0", "fail_calc": "count(*)", "meta": {}, "schema": "dbt_test__audit", }, "docs": {"show": True}, "contract": {"enforced": False, "alias_types": True}, "columns": {}, "test_metadata": { "name": "foo", "kwargs": {}, }, "checksum": { "name": "sha256", "checksum": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", }, "unrendered_config": {}, "unrendered_config_call_dict": {}, "config_call_dict": {}, "doc_blocks": [], } @pytest.fixture def basic_parsed_schema_test_object(): return GenericTestNode( package_name="test", path="/root/x/path.sql", original_file_path="/root/path.sql", language="sql", raw_code="select * from wherever", name="foo", resource_type=NodeType.Test, unique_id="test.test.foo", fqn=["test", "models", "foo"], refs=[], sources=[], metrics=[], depends_on=DependsOn(), description="", database="test_db", schema="test_schema", alias="bar", tags=[], meta={}, config=TestConfig(), test_metadata=TestMetadata(namespace=None, name="foo", kwargs={}), checksum=FileHash.from_contents(""), ) @pytest.fixture def complex_parsed_schema_test_dict(): return { "name": "foo", "created_at": 1.0, "resource_type": str(NodeType.Test), "path": "/root/x/path.sql", "original_file_path": "/root/path.sql", "package_name": "test", "language": "sql", "raw_code": 'select * from {{ ref("bar") }}', "unique_id": "test.test.foo", "fqn": ["test", "models", "foo"], "refs": [], "sources": [], "metrics": [], "functions": [], "depends_on": {"macros": [], "nodes": ["model.test.bar"]}, "database": "test_db", "description": "My parsed node", "schema": "test_schema", "alias": "bar", "tags": ["tag"], "meta": {}, "config": { "enabled": True, "materialized": "table", "tags": [], "severity": "WARN", "warn_if": "!= 0", "error_if": "!= 0", "fail_calc": "count(*)", "extra_key": "extra value", "meta": {}, "schema": "dbt_test__audit", }, "docs": {"show": False}, "contract": {"enforced": False, "alias_types": True}, "columns": { "a": { "name": "a", "description": "a text field", "meta": {}, "tags": [], "constraints": [], "doc_blocks": [], "config": {"meta": {}, "tags": []}, }, }, "column_name": "id", "test_metadata": { "name": "foo", "kwargs": {}, }, "checksum": { "name": "sha256", "checksum": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", }, "unrendered_config": {"materialized": "table", "severity": "WARN"}, "unrendered_config_call_dict": {}, "config_call_dict": {}, "doc_blocks": [], } @pytest.fixture def complex_parsed_schema_test_object(): cfg = TestConfig(materialized="table", severity="WARN") cfg._extra.update({"extra_key": "extra value"}) return GenericTestNode( package_name="test", path="/root/x/path.sql", original_file_path="/root/path.sql", language="sql", raw_code='select * from {{ ref("bar") }}', name="foo", resource_type=NodeType.Test, unique_id="test.test.foo", fqn=["test", "models", "foo"], refs=[], sources=[], metrics=[], depends_on=DependsOn(nodes=["model.test.bar"]), description="My parsed node", database="test_db", schema="test_schema", alias="bar", tags=["tag"], meta={}, config=cfg, columns={"a": ColumnInfo("a", "a text field", {})}, column_name="id", docs=Docs(show=False), test_metadata=TestMetadata(namespace=None, name="foo", kwargs={}), checksum=FileHash.from_contents(""), unrendered_config={"materialized": "table", "severity": "WARN"}, ) def test_basic_schema_test_node( minimal_parsed_schema_test_dict, basic_parsed_schema_test_dict, basic_parsed_schema_test_object ): node = basic_parsed_schema_test_object node_dict = basic_parsed_schema_test_dict minimum = minimal_parsed_schema_test_dict assert_symmetric(node, node_dict, GenericTestNode) assert node.empty is False assert node.is_ephemeral is False assert node.is_refable is False assert node.get_materialization() == "test" assert_from_dict(node, minimum, GenericTestNode) pickle.loads(pickle.dumps(node)) def test_complex_schema_test_node( complex_parsed_schema_test_dict, complex_parsed_schema_test_object ): # this tests for the presence of _extra keys node = complex_parsed_schema_test_object # GenericTestNode assert node.config._extra["extra_key"] node_dict = complex_parsed_schema_test_dict assert_symmetric(node, node_dict) assert node.empty is False def test_invalid_column_name_type(complex_parsed_schema_test_dict): # bad top-level field bad_column_name = complex_parsed_schema_test_dict bad_column_name["column_name"] = {} assert_fails_validation(bad_column_name, GenericTestNode) def test_invalid_severity(complex_parsed_schema_test_dict): invalid_config_value = complex_parsed_schema_test_dict invalid_config_value["config"]["severity"] = "WERROR" assert_fails_validation(invalid_config_value, GenericTestNode) @pytest.fixture def basic_timestamp_snapshot_config_dict(): return { "column_types": {}, "enabled": True, "materialized": "snapshot", "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "tags": [], "unique_key": "id", "snapshot_meta_column_names": {}, "strategy": "timestamp", "updated_at": "last_update", "target_database": "some_snapshot_db", "target_schema": "some_snapshot_schema", "on_schema_change": "ignore", "on_configuration_change": "apply", "meta": {}, "grants": {}, "packages": [], "docs": {"show": True}, "contract": {"enforced": False, "alias_types": True}, "lookback": 1, } @pytest.fixture def basic_timestamp_snapshot_config_object(): return SnapshotConfig( strategy="timestamp", updated_at="last_update", unique_key="id", target_database="some_snapshot_db", target_schema="some_snapshot_schema", ) @pytest.fixture def complex_timestamp_snapshot_config_dict(): return { "column_types": {"a": "text"}, "enabled": True, "materialized": "snapshot", "persist_docs": {}, "post-hook": [{"sql": 'insert into blah(a, b) select "1", 1', "transaction": True}], "pre-hook": [], "quoting": {}, "snapshot_meta_column_names": {}, "tags": [], "target_database": "some_snapshot_db", "target_schema": "some_snapshot_schema", "unique_key": "id", "extra": "even more", "strategy": "timestamp", "updated_at": "last_update", "on_schema_change": "ignore", "on_configuration_change": "apply", "meta": {}, "grants": {}, "packages": [], "docs": {"show": True}, "contract": {"enforced": False, "alias_types": True}, "lookback": 1, } @pytest.fixture def complex_timestamp_snapshot_config_object(): cfg = SnapshotConfig( column_types={"a": "text"}, materialized="snapshot", post_hook=[Hook(sql='insert into blah(a, b) select "1", 1')], strategy="timestamp", target_database="some_snapshot_db", target_schema="some_snapshot_schema", updated_at="last_update", unique_key="id", ) cfg._extra["extra"] = "even more" return cfg def test_basic_timestamp_snapshot_config( basic_timestamp_snapshot_config_dict, basic_timestamp_snapshot_config_object ): cfg = basic_timestamp_snapshot_config_object cfg_dict = basic_timestamp_snapshot_config_dict assert_symmetric(cfg, cfg_dict) pickle.loads(pickle.dumps(cfg)) def test_complex_timestamp_snapshot_config( complex_timestamp_snapshot_config_dict, complex_timestamp_snapshot_config_object ): cfg = complex_timestamp_snapshot_config_object cfg_dict = complex_timestamp_snapshot_config_dict assert_symmetric(cfg, cfg_dict, SnapshotConfig) def test_invalid_missing_updated_at(basic_timestamp_snapshot_config_dict): bad_fields = basic_timestamp_snapshot_config_dict del bad_fields["updated_at"] bad_fields["check_cols"] = "all" assert_snapshot_config_fails_validation(bad_fields) @pytest.fixture def basic_check_snapshot_config_dict(): return { "column_types": {}, "enabled": True, "materialized": "snapshot", "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "snapshot_meta_column_names": {}, "tags": [], "target_database": "some_snapshot_db", "target_schema": "some_snapshot_schema", "unique_key": "id", "strategy": "check", "check_cols": "all", "on_schema_change": "ignore", "on_configuration_change": "apply", "meta": {}, "grants": {}, "packages": [], "docs": {"show": True}, "contract": {"enforced": False, "alias_types": True}, "lookback": 1, } @pytest.fixture def basic_check_snapshot_config_object(): return SnapshotConfig( strategy="check", check_cols="all", unique_key="id", target_database="some_snapshot_db", target_schema="some_snapshot_schema", ) @pytest.fixture def complex_set_snapshot_config_dict(): return { "column_types": {"a": "text"}, "enabled": True, "materialized": "snapshot", "persist_docs": {}, "post-hook": [{"sql": 'insert into blah(a, b) select "1", 1', "transaction": True}], "pre-hook": [], "quoting": {}, "snapshot_meta_column_names": {}, "tags": [], "target_database": "some_snapshot_db", "target_schema": "some_snapshot_schema", "unique_key": "id", "extra": "even more", "strategy": "check", "check_cols": ["a", "b"], "on_schema_change": "ignore", "on_configuration_change": "apply", "meta": {}, "grants": {}, "packages": [], "docs": {"show": True}, "contract": {"enforced": False, "alias_types": True}, "lookback": 1, } @pytest.fixture def complex_set_snapshot_config_object(): cfg = SnapshotConfig( column_types={"a": "text"}, materialized="snapshot", post_hook=[Hook(sql='insert into blah(a, b) select "1", 1')], strategy="check", check_cols=["a", "b"], target_database="some_snapshot_db", target_schema="some_snapshot_schema", unique_key="id", ) cfg._extra["extra"] = "even more" return cfg def test_basic_snapshot_config( basic_check_snapshot_config_dict, basic_check_snapshot_config_object ): cfg_dict = basic_check_snapshot_config_dict cfg = basic_check_snapshot_config_object assert_symmetric(cfg, cfg_dict, SnapshotConfig) pickle.loads(pickle.dumps(cfg)) def test_complex_snapshot_config( complex_set_snapshot_config_dict, complex_set_snapshot_config_object ): cfg_dict = complex_set_snapshot_config_dict cfg = complex_set_snapshot_config_object assert_symmetric(cfg, cfg_dict) pickle.loads(pickle.dumps(cfg)) def test_invalid_check_wrong_strategy(basic_check_snapshot_config_dict): wrong_strategy = basic_check_snapshot_config_dict wrong_strategy["strategy"] = "timestamp" assert_snapshot_config_fails_validation(wrong_strategy) def test_invalid_missing_check_cols(basic_check_snapshot_config_dict): wrong_fields = basic_check_snapshot_config_dict del wrong_fields["check_cols"] with pytest.raises(ValidationError, match=r"A snapshot configured with the check strategy"): SnapshotConfig.validate(wrong_fields) cfg = SnapshotConfig.from_dict(wrong_fields) cfg.final_validate() def test_missing_snapshot_configs(basic_check_snapshot_config_dict): wrong_fields = basic_check_snapshot_config_dict del wrong_fields["strategy"] with pytest.raises(ValidationError, match=r"Snapshots must be configured with a 'strategy'"): SnapshotConfig.validate(wrong_fields) cfg = SnapshotConfig.from_dict(wrong_fields) cfg.final_validate() wrong_fields["strategy"] = "timestamp" del wrong_fields["unique_key"] with pytest.raises(ValidationError, match=r"Snapshots must be configured with a 'strategy'"): SnapshotConfig.validate(wrong_fields) cfg = SnapshotConfig.from_dict(wrong_fields) cfg.final_validate() def assert_snapshot_config_fails_validation(dct): with pytest.raises(ValidationError): SnapshotConfig.validate(dct) obj = SnapshotConfig.from_dict(dct) obj.final_validate() def test_invalid_check_value(basic_check_snapshot_config_dict): invalid_check_type = basic_check_snapshot_config_dict invalid_check_type["check_cols"] = "some" assert_snapshot_config_fails_validation(invalid_check_type) @pytest.fixture def basic_timestamp_snapshot_dict(): return { "name": "foo", "created_at": 1.0, "resource_type": str(NodeType.Snapshot), "path": "/root/x/path.sql", "original_file_path": "/root/path.sql", "package_name": "test", "language": "sql", "raw_code": "select * from wherever", "unique_id": "model.test.foo", "fqn": ["test", "models", "foo"], "refs": [], "sources": [], "metrics": [], "functions": [], "depends_on": {"macros": [], "nodes": []}, "database": "test_db", "description": "", "schema": "test_schema", "alias": "bar", "tags": [], "config": { "column_types": {}, "enabled": True, "materialized": "snapshot", "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "snapshot_meta_column_names": {}, "tags": [], "target_database": "some_snapshot_db", "target_schema": "some_snapshot_schema", "unique_key": "id", "strategy": "timestamp", "updated_at": "last_update", "on_schema_change": "ignore", "on_configuration_change": "apply", "meta": {}, "grants": {}, "docs": {"show": True}, "contract": {"enforced": False, "alias_types": True}, "packages": [], "lookback": 1, }, "docs": {"show": True}, "contract": {"enforced": False, "alias_types": True}, "columns": {}, "meta": {}, "checksum": { "name": "sha256", "checksum": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", }, "unrendered_config": { "strategy": "timestamp", "unique_key": "id", "updated_at": "last_update", "target_database": "some_snapshot_db", "target_schema": "some_snapshot_schema", }, "unrendered_config_call_dict": {}, "config_call_dict": {}, "doc_blocks": [], } @pytest.fixture def basic_timestamp_snapshot_object(): return SnapshotNode( package_name="test", path="/root/x/path.sql", original_file_path="/root/path.sql", language="sql", raw_code="select * from wherever", name="foo", resource_type=NodeType.Snapshot, unique_id="model.test.foo", fqn=["test", "models", "foo"], refs=[], sources=[], metrics=[], depends_on=DependsOn(), description="", database="test_db", schema="test_schema", alias="bar", tags=[], config=SnapshotConfig( strategy="timestamp", unique_key="id", updated_at="last_update", target_database="some_snapshot_db", target_schema="some_snapshot_schema", ), checksum=FileHash.from_contents(""), unrendered_config={ "strategy": "timestamp", "unique_key": "id", "updated_at": "last_update", "target_database": "some_snapshot_db", "target_schema": "some_snapshot_schema", }, ) @pytest.fixture def basic_check_snapshot_dict(): return { "name": "foo", "created_at": 1.0, "resource_type": str(NodeType.Snapshot), "path": "/root/x/path.sql", "original_file_path": "/root/path.sql", "package_name": "test", "language": "sql", "raw_code": "select * from wherever", "unique_id": "model.test.foo", "fqn": ["test", "models", "foo"], "refs": [], "sources": [], "metrics": [], "functions": [], "depends_on": {"macros": [], "nodes": []}, "database": "test_db", "description": "", "schema": "test_schema", "alias": "bar", "tags": [], "config": { "column_types": {}, "enabled": True, "materialized": "snapshot", "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "snapshot_meta_column_names": {}, "tags": [], "target_database": "some_snapshot_db", "target_schema": "some_snapshot_schema", "unique_key": "id", "strategy": "check", "check_cols": "all", "on_schema_change": "ignore", "on_configuration_change": "apply", "meta": {}, "grants": {}, "docs": {"show": True}, "contract": {"enforced": False, "alias_types": True}, "packages": [], "lookback": 1, }, "docs": {"show": True}, "contract": {"enforced": False, "alias_types": True}, "columns": {}, "meta": {}, "checksum": { "name": "sha256", "checksum": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", }, "unrendered_config": { "target_database": "some_snapshot_db", "target_schema": "some_snapshot_schema", "unique_key": "id", "strategy": "check", "check_cols": "all", }, "unrendered_config_call_dict": {}, "config_call_dict": {}, "doc_blocks": [], } @pytest.fixture def basic_check_snapshot_object(): return SnapshotNode( package_name="test", path="/root/x/path.sql", original_file_path="/root/path.sql", language="sql", raw_code="select * from wherever", name="foo", resource_type=NodeType.Snapshot, unique_id="model.test.foo", fqn=["test", "models", "foo"], refs=[], sources=[], metrics=[], depends_on=DependsOn(), description="", database="test_db", schema="test_schema", alias="bar", tags=[], config=SnapshotConfig( strategy="check", unique_key="id", check_cols="all", target_database="some_snapshot_db", target_schema="some_snapshot_schema", ), checksum=FileHash.from_contents(""), unrendered_config={ "target_database": "some_snapshot_db", "target_schema": "some_snapshot_schema", "unique_key": "id", "strategy": "check", "check_cols": "all", }, ) def test_timestamp_snapshot_ok( basic_timestamp_snapshot_dict, basic_timestamp_snapshot_object, ): node_dict = basic_timestamp_snapshot_dict node = basic_timestamp_snapshot_object assert_symmetric(node, node_dict, SnapshotNode) assert node.is_refable is True assert node.is_ephemeral is False pickle.loads(pickle.dumps(node)) def test_check_snapshot_ok( basic_check_snapshot_dict, basic_check_snapshot_object, ): node_dict = basic_check_snapshot_dict node = basic_check_snapshot_object assert_symmetric(node, node_dict, SnapshotNode) assert node.is_refable is True assert node.is_ephemeral is False pickle.loads(pickle.dumps(node)) def test_invalid_snapshot_bad_resource_type(basic_timestamp_snapshot_dict): bad_resource_type = basic_timestamp_snapshot_dict bad_resource_type["resource_type"] = str(NodeType.Model) assert_fails_validation(bad_resource_type, SnapshotNode) class TestParsedMacro(ContractTestCase): ContractType = Macro def _ok_dict(self): return { "name": "foo", "path": "/root/path.sql", "original_file_path": "/root/path.sql", "created_at": 1.0, "package_name": "test", "macro_sql": "{% macro foo() %}select 1 as id{% endmacro %}", "resource_type": "macro", "unique_id": "macro.test.foo", "depends_on": {"macros": []}, "meta": {}, "config": { "meta": {}, "docs": {"show": True}, }, "description": "my macro description", "docs": {"show": True}, "arguments": [], } def test_ok(self): macro_dict = self._ok_dict() macro = self.ContractType( name="foo", path="/root/path.sql", original_file_path="/root/path.sql", package_name="test", macro_sql="{% macro foo() %}select 1 as id{% endmacro %}", resource_type=NodeType.Macro, unique_id="macro.test.foo", depends_on=MacroDependsOn(), meta={}, config=MacroConfig(), description="my macro description", arguments=[], ) assert_symmetric(macro, macro_dict) pickle.loads(pickle.dumps(macro)) def test_invalid_missing_unique_id(self): bad_missing_uid = self._ok_dict() del bad_missing_uid["unique_id"] self.assert_fails_validation(bad_missing_uid) def test_invalid_extra_field(self): bad_extra_field = self._ok_dict() bad_extra_field["extra"] = "too many fields" self.assert_fails_validation(bad_extra_field) class TestParsedDocumentation(ContractTestCase): ContractType = Documentation def _ok_dict(self): return { "block_contents": "some doc contents", "name": "foo", "resource_type": "doc", "original_file_path": "/root/docs/doc.md", "package_name": "test", "path": "/root/docs", "unique_id": "test.foo", } def test_ok(self): doc_dict = self._ok_dict() doc = self.ContractType( package_name="test", path="/root/docs", original_file_path="/root/docs/doc.md", name="foo", unique_id="test.foo", block_contents="some doc contents", resource_type=NodeType.Documentation, ) self.assert_symmetric(doc, doc_dict) pickle.loads(pickle.dumps(doc)) def test_invalid_missing(self): bad_missing_contents = self._ok_dict() del bad_missing_contents["block_contents"] self.assert_fails_validation(bad_missing_contents) def test_invalid_extra(self): bad_extra_field = self._ok_dict() bad_extra_field["extra"] = "more" self.assert_fails_validation(bad_extra_field) @pytest.fixture def minimum_parsed_source_definition_dict(): return { "package_name": "test", "path": "/root/models/sources.yml", "original_file_path": "/root/models/sources.yml", "created_at": 1.0, "database": "some_db", "schema": "some_schema", "fqn": ["test", "source", "my_source", "my_source_table"], "source_name": "my_source", "name": "my_source_table", "source_description": "my source description", "loader": "stitch", "identifier": "my_source_table", "resource_type": str(NodeType.Source), "unique_id": "test.source.my_source.my_source_table", } @pytest.fixture def basic_parsed_source_definition_dict(): return { "package_name": "test", "path": "/root/models/sources.yml", "original_file_path": "/root/models/sources.yml", "created_at": 1.0, "database": "some_db", "schema": "some_schema", "fqn": ["test", "source", "my_source", "my_source_table"], "source_name": "my_source", "name": "my_source_table", "source_description": "my source description", "loader": "stitch", "identifier": "my_source_table", "resource_type": str(NodeType.Source), "description": "", "columns": {}, "quoting": {}, "unique_id": "test.source.my_source.my_source_table", "meta": {}, "source_meta": {}, "tags": [], "config": { "enabled": True, "freshness": { "warn_after": {}, "error_after": {}, }, "tags": [], "meta": {}, }, "unrendered_config": {}, "doc_blocks": [], } @pytest.fixture def complex_parsed_source_definition_dict(): return { "package_name": "test", "path": "/root/models/sources.yml", "original_file_path": "/root/models/sources.yml", "created_at": 1.0, "database": "some_db", "schema": "some_schema", "fqn": ["test", "source", "my_source", "my_source_table"], "source_name": "my_source", "name": "my_source_table", "source_description": "my source description", "loader": "stitch", "identifier": "my_source_table", "resource_type": str(NodeType.Source), "description": "", "columns": {}, "quoting": {}, "unique_id": "test.source.my_source.my_source_table", "meta": {}, "source_meta": {}, "tags": ["my_tag"], "config": { "enabled": True, "freshness": { "warn_after": {}, "error_after": {}, }, "tags": [], "meta": {}, }, "freshness": {"warn_after": {"period": "hour", "count": 1}, "error_after": {}}, "loaded_at_field": "loaded_at", "unrendered_config": {}, "doc_blocks": [], } @pytest.fixture def complex_parsed_source_definition_object(): return SourceDefinition( columns={}, database="some_db", description="", fqn=["test", "source", "my_source", "my_source_table"], identifier="my_source_table", loader="stitch", name="my_source_table", original_file_path="/root/models/sources.yml", package_name="test", path="/root/models/sources.yml", quoting=Quoting(), resource_type=NodeType.Source, schema="some_schema", source_description="my source description", source_name="my_source", unique_id="test.source.my_source.my_source_table", tags=["my_tag"], config=SourceConfig(), freshness=FreshnessThreshold(warn_after=Time(period=TimePeriod.hour, count=1)), loaded_at_field="loaded_at", ) def test_basic_source_definition( minimum_parsed_source_definition_dict, basic_parsed_source_definition_dict, basic_parsed_source_definition_object, ): node = basic_parsed_source_definition_object node_dict = basic_parsed_source_definition_dict minimum = minimum_parsed_source_definition_dict assert_symmetric(node.to_resource(), node_dict, SourceDefinitionResource) assert node.is_ephemeral is False assert node.is_refable is False assert node.has_freshness is False assert_from_dict(node.to_resource(), minimum, SourceDefinitionResource) pickle.loads(pickle.dumps(node)) def test_extra_fields_source_definition_okay(minimum_parsed_source_definition_dict): extra = minimum_parsed_source_definition_dict extra["notvalid"] = "nope" # Model still load fine with extra fields loaded_source = SourceDefinition.from_dict(extra) assert not hasattr(loaded_source, "notvalid") def test_invalid_missing(minimum_parsed_source_definition_dict): bad_missing_name = minimum_parsed_source_definition_dict del bad_missing_name["name"] assert_fails_validation(bad_missing_name, SourceDefinition) def test_invalid_bad_resource_type(minimum_parsed_source_definition_dict): bad_resource_type = minimum_parsed_source_definition_dict bad_resource_type["resource_type"] = str(NodeType.Model) assert_fails_validation(bad_resource_type, SourceDefinition) def test_complex_source_definition( complex_parsed_source_definition_dict, complex_parsed_source_definition_object ): node = complex_parsed_source_definition_object node_dict = complex_parsed_source_definition_dict assert_symmetric(node.to_resource(), node_dict, SourceDefinitionResource) assert node.is_ephemeral is False assert node.is_refable is False assert node.has_freshness is True pickle.loads(pickle.dumps(node)) def test_source_no_freshness(complex_parsed_source_definition_object): node = complex_parsed_source_definition_object assert node.has_freshness is True node.freshness = None assert node.has_freshness is False unchanged_source_definitions = [ lambda u: (u, replace(u, tags=["mytag"])), lambda u: (u, replace(u, meta={"a": 1000})), ] changed_source_definitions = [ lambda u: ( u, replace( u, freshness=FreshnessThreshold(warn_after=Time(period=TimePeriod.hour, count=1)), loaded_at_field="loaded_at", ), ), lambda u: (u, replace(u, loaded_at_field="loaded_at")), lambda u: ( u, replace( u, freshness=FreshnessThreshold(error_after=Time(period=TimePeriod.hour, count=1)) ), ), lambda u: (u, replace(u, quoting=Quoting(identifier=True))), lambda u: (u, replace(u, database="other_database")), lambda u: (u, replace(u, schema="other_schema")), lambda u: (u, replace(u, identifier="identifier")), ] @pytest.mark.parametrize("func", unchanged_source_definitions) def test_compare_unchanged_parsed_source_definition(func, basic_parsed_source_definition_object): node, compare = func(basic_parsed_source_definition_object) assert node.same_contents(compare) @pytest.mark.parametrize("func", changed_source_definitions) def test_compare_changed_source_definition(func, basic_parsed_source_definition_object): node, compare = func(basic_parsed_source_definition_object) assert not node.same_contents(compare) @pytest.fixture def minimal_parsed_exposure_dict(): return { "name": "my_exposure", "type": "notebook", "owner": { "email": "test@example.com", }, "fqn": ["test", "exposures", "my_exposure"], "unique_id": "exposure.test.my_exposure", "package_name": "test", "meta": {}, "tags": [], "path": "models/something.yml", "original_file_path": "models/something.yml", "description": "", "created_at": 1.0, "resource_type": "exposure", } @pytest.fixture def basic_parsed_exposure_dict(): return { "name": "my_exposure", "type": "notebook", "owner": { "email": "test@example.com", }, "resource_type": "exposure", "depends_on": { "nodes": [], "macros": [], }, "refs": [], "sources": [], "metrics": [], "fqn": ["test", "exposures", "my_exposure"], "unique_id": "exposure.test.my_exposure", "package_name": "test", "path": "models/something.yml", "original_file_path": "models/something.yml", "description": "", "meta": {}, "tags": [], "created_at": 1.0, "config": { "enabled": True, "tags": [], "meta": {}, }, "unrendered_config": {}, } @pytest.fixture def basic_parsed_exposure_object(): return Exposure( name="my_exposure", resource_type=NodeType.Exposure, type=ExposureType.Notebook, fqn=["test", "exposures", "my_exposure"], unique_id="exposure.test.my_exposure", package_name="test", path="models/something.yml", original_file_path="models/something.yml", owner=Owner(email="test@example.com"), description="", meta={}, tags=[], config=ExposureConfig(), unrendered_config={}, ) @pytest.fixture def complex_parsed_exposure_dict(): return { "name": "my_exposure", "type": "analysis", "created_at": 1.0, "owner": { "email": "test@example.com", "name": "A Name", }, "resource_type": "exposure", "maturity": "low", "url": "https://example.com/analyses/1", "description": "my description", "meta": {"tool": "my_tool", "is_something": False}, "tags": ["my_department"], "depends_on": { "nodes": ["models.test.my_model"], "macros": [], }, "refs": [], "sources": [], "metrics": [], "fqn": ["test", "exposures", "my_exposure"], "unique_id": "exposure.test.my_exposure", "package_name": "test", "path": "models/something.yml", "original_file_path": "models/something.yml", "config": { "enabled": True, "tags": [], "meta": {}, }, "unrendered_config": {}, } @pytest.fixture def complex_parsed_exposure_object(): return Exposure( name="my_exposure", resource_type=NodeType.Exposure, type=ExposureType.Analysis, owner=Owner(email="test@example.com", name="A Name"), maturity=MaturityType.Low, url="https://example.com/analyses/1", description="my description", meta={"tool": "my_tool", "is_something": False}, tags=["my_department"], depends_on=DependsOn(nodes=["models.test.my_model"]), fqn=["test", "exposures", "my_exposure"], unique_id="exposure.test.my_exposure", package_name="test", path="models/something.yml", original_file_path="models/something.yml", config=ExposureConfig(), unrendered_config={}, ) def test_basic_parsed_exposure( minimal_parsed_exposure_dict, basic_parsed_exposure_dict, basic_parsed_exposure_object ): assert_symmetric(basic_parsed_exposure_object, basic_parsed_exposure_dict, Exposure) assert_from_dict(basic_parsed_exposure_object, minimal_parsed_exposure_dict, Exposure) pickle.loads(pickle.dumps(basic_parsed_exposure_object)) def test_complex_parsed_exposure(complex_parsed_exposure_dict, complex_parsed_exposure_object): assert_symmetric(complex_parsed_exposure_object, complex_parsed_exposure_dict, Exposure) unchanged_parsed_exposures = [ lambda u: (u, u), ] changed_parsed_exposures = [ lambda u: (u, replace(u, fqn=u.fqn[:-1] + ["something", u.fqn[-1]])), lambda u: (u, replace(u, type=ExposureType.ML)), lambda u: (u, replace(u, owner=replace(u.owner, name="My Name"))), lambda u: (u, replace(u, maturity=MaturityType.Medium)), lambda u: (u, replace(u, url="https://example.com/dashboard/1")), lambda u: (u, replace(u, description="My description")), lambda u: (u, replace(u, depends_on=DependsOn(nodes=["model.test.blah"]))), ] @pytest.mark.parametrize("func", unchanged_parsed_exposures) def test_compare_unchanged_parsed_exposure(func, basic_parsed_exposure_object): node, compare = func(basic_parsed_exposure_object) assert node.same_contents(compare) @pytest.mark.parametrize("func", changed_parsed_exposures) def test_compare_changed_exposure(func, basic_parsed_exposure_object): node, compare = func(basic_parsed_exposure_object) assert not node.same_contents(compare) # METRICS @pytest.fixture def minimal_parsed_metric_dict(): return { "name": "my_metric", "type": "simple", "type_params": {"measure": {"name": "my_measure"}}, "timestamp": "created_at", "time_grains": ["day"], "fqn": ["test", "metrics", "my_metric"], "unique_id": "metric.test.my_metric", "package_name": "test", "meta": {}, "tags": [], "path": "models/something.yml", "original_file_path": "models/something.yml", "description": "", "created_at": 1.0, } @pytest.fixture def basic_parsed_metric_dict(): return { "name": "new_customers", "label": "New Customers", "type": "simple", "type_params": { "measure": {"name": "customers", "filter": {"where_sql_template": "is_new = true"}}, }, "resource_type": "metric", "refs": [["dim_customers"]], "sources": [], "metrics": [], "fqn": ["test", "metrics", "my_metric"], "unique_id": "metric.test.my_metric", "package_name": "test", "path": "models/something.yml", "original_file_path": "models/something.yml", "description": "New Customers", "meta": {}, "tags": [], "created_at": 1.0, "depends_on": { "nodes": [], "macros": [], }, } @pytest.fixture def basic_parsed_metric_object(): return Metric( name="my_metric", resource_type=NodeType.Metric, type=MetricType.SIMPLE, type_params=MetricTypeParams(measure=MetricInputMeasure(name="a_measure")), fqn=["test", "metrics", "myq_metric"], unique_id="metric.test.my_metric", package_name="test", path="models/something.yml", original_file_path="models/something.yml", description="", meta={}, tags=[], ) @given( builds( SemanticModel, depends_on=builds(DependsOn), dimensions=lists(builds(Dimension)), entities=lists(builds(Entity)), measures=lists(builds(Measure)), refs=lists(builds(RefArgs)), ) ) def test_semantic_model_symmetry(semantic_model: SemanticModel): assert semantic_model == SemanticModel.from_dict(semantic_model.to_dict()) assert semantic_model == pickle.loads(pickle.dumps(semantic_model)) ================================================ FILE: tests/unit/contracts/graph/test_semantic_manifest.py ================================================ from unittest.mock import patch import pytest from core.dbt.contracts.graph.manifest import Manifest from dbt.artifacts.resources.types import NodeType from dbt.artifacts.resources.v1.metric import ( CumulativeTypeParams, MetricTimeWindow, MetricTypeParams, ) from dbt.artifacts.resources.v1.model import ModelConfig, TimeSpine from dbt.artifacts.resources.v1.semantic_model import DimensionType from dbt.constants import LEGACY_TIME_SPINE_MODEL_NAME from dbt.contracts.files import FileHash from dbt.contracts.graph.nodes import ( ColumnInfo, DependsOn, Metric, ModelNode, SemanticModel, ) from dbt.contracts.graph.semantic_manifest import SemanticManifest from dbt.events.types import TimeDimensionsRequireGranularityDeprecation from dbt_common.events.event_catcher import EventCatcher from dbt_common.events.event_manager_client import add_callback_to_manager from dbt_semantic_interfaces.type_enums import TimeGranularity from dbt_semantic_interfaces.type_enums.metric_type import MetricType # Overwrite the default nods to construct the manifest @pytest.fixture def nodes(metricflow_time_spine_model): return [metricflow_time_spine_model] @pytest.fixture def semantic_models( semantic_model, ) -> list: return [semantic_model] @pytest.fixture def metrics( metric, ) -> list: return [metric] class TestSemanticManifest: def test_validate(self, manifest): with patch("dbt.contracts.graph.semantic_manifest.get_flags") as patched_get_flags: patched_get_flags.return_value.require_yaml_configuration_for_mf_time_spines = True sm_manifest = SemanticManifest(manifest) assert sm_manifest.validate() def test_require_yaml_configuration_for_mf_time_spines( self, manifest: Manifest, metricflow_time_spine_model: ModelNode ): with patch("dbt.contracts.graph.semantic_manifest.get_flags") as patched_get_flags, patch( "dbt.contracts.graph.semantic_manifest.deprecations" ) as patched_deprecations: patched_get_flags.return_value.require_yaml_configuration_for_mf_time_spines = False manifest.nodes[metricflow_time_spine_model.unique_id] = metricflow_time_spine_model sm_manifest = SemanticManifest(manifest) assert sm_manifest.validate() assert patched_deprecations.warn.call_count == 1 def test_metricflow_time_spine_non_day_grain_deprecation_warning( self, manifest: Manifest, metricflow_time_spine_model: ModelNode ): """Test that a metricflow_time_spine with non-day grain does not trigger deprecation warning.""" # Create a metricflow_time_spine model with HOUR granularity metricflow_time_spine_hour = ModelNode( name=LEGACY_TIME_SPINE_MODEL_NAME, database="dbt", schema="analytics", alias=LEGACY_TIME_SPINE_MODEL_NAME, resource_type=NodeType.Model, unique_id="model.test.metricflow_time_spine", fqn=["test", "metricflow_time_spine"], package_name="test", refs=[], sources=[], metrics=[], depends_on=DependsOn(), config=ModelConfig(), tags=[], path="metricflow_time_spine.sql", original_file_path="metricflow_time_spine.sql", meta={}, language="sql", raw_code="SELECT DATEADD(hour, ROW_NUMBER() OVER (ORDER BY 1), '2020-01-01'::timestamp) as ts_hour", checksum=FileHash.empty(), relation_name="", columns={ "ts_hour": ColumnInfo( name="ts_hour", description="", meta={}, data_type="timestamp", constraints=[], quote=None, tags=[], granularity=TimeGranularity.HOUR, ) }, time_spine=TimeSpine(standard_granularity_column="ts_hour"), ) with patch("dbt.contracts.graph.semantic_manifest.get_flags") as patched_get_flags, patch( "dbt.contracts.graph.semantic_manifest.deprecations" ) as patched_deprecations: patched_get_flags.return_value.require_yaml_configuration_for_mf_time_spines = False manifest.nodes[metricflow_time_spine_hour.unique_id] = metricflow_time_spine_hour sm_manifest = SemanticManifest(manifest) assert sm_manifest.validate() assert patched_deprecations.warn.call_count == 0 @pytest.mark.parametrize( "metric_type_params, num_warns, should_error, flag_value", [ ( MetricTypeParams(grain_to_date=TimeGranularity.MONTH), 1, False, False, ), ( MetricTypeParams( window=MetricTimeWindow(count=1, granularity=TimeGranularity.MONTH.value) ), 1, False, False, ), ( MetricTypeParams( cumulative_type_params=CumulativeTypeParams( grain_to_date=TimeGranularity.MONTH.value, ) ), 0, False, False, ), ( MetricTypeParams( cumulative_type_params=CumulativeTypeParams( window=MetricTimeWindow(count=1, granularity=TimeGranularity.MONTH.value), ) ), 0, False, False, ), ( MetricTypeParams(grain_to_date=TimeGranularity.MONTH), 0, True, True, ), ( MetricTypeParams( window=MetricTimeWindow(count=1, granularity=TimeGranularity.MONTH.value) ), 0, True, True, ), ( MetricTypeParams( cumulative_type_params=CumulativeTypeParams( grain_to_date=TimeGranularity.MONTH.value, ) ), 0, False, True, ), ( MetricTypeParams( cumulative_type_params=CumulativeTypeParams( window=MetricTimeWindow(count=1, granularity=TimeGranularity.MONTH.value), ) ), 0, False, True, ), ], ) def test_deprecate_cumulative_type_params( self, manifest: Manifest, metric_type_params: MetricTypeParams, num_warns: int, should_error: bool, flag_value: bool, ): with patch("dbt.contracts.graph.semantic_manifest.get_flags") as patched_get_flags, patch( "dbt.contracts.graph.semantic_manifest.deprecations" ) as patched_deprecations: patched_get_flags.return_value.require_nested_cumulative_type_params = flag_value manifest.metrics["metric.test.my_metric"] = Metric( name="my_metric", type=MetricType.CUMULATIVE, type_params=metric_type_params, resource_type=NodeType.Metric, package_name="test", path="models/test/my_metric.yml", original_file_path="models/test/my_metric.yml", unique_id="metric.test.my_metric", fqn=["test", "my_metric"], description="My metric", label="My Metric", ) sm_manifest = SemanticManifest(manifest) assert sm_manifest.validate() != should_error assert patched_deprecations.warn.call_count == num_warns def test_time_dimensions_require_granularity_deprecation( self, manifest: Manifest, semantic_model: SemanticModel ): # we only do this patch because the semantic manifest validation requires the flag to be set with patch("dbt.contracts.graph.semantic_manifest.get_flags") as patched_get_flags: patched_get_flags.return_value.require_yaml_configuration_for_mf_time_spines = False event_catcher = EventCatcher(TimeDimensionsRequireGranularityDeprecation) add_callback_to_manager(event_catcher.catch) sm_manifest = SemanticManifest(manifest) assert sm_manifest.validate() is True, "Semantic manifest should validate successfully" assert len(event_catcher.caught_events) == 0 # Set the time dimension granularity to None dimension = semantic_model.dimensions[0] assert dimension.type == DimensionType.TIME dimension.type_params = None # This also removes the time granularity sm_manifest = SemanticManifest(manifest) assert sm_manifest.validate() is True, "Semantic manifest should validate successfully" assert len(event_catcher.caught_events) == 1 ================================================ FILE: tests/unit/contracts/graph/test_udfs.py ================================================ from dataclasses import dataclass from typing import Optional import pytest from dbt.contracts.files import FileHash from dbt.contracts.graph.nodes import FunctionNode, FunctionReturns, NodeType from dbt.task.function import FunctionRunner @dataclass class StubRelation: database: Optional[str] = None schema: Optional[str] = None name: Optional[str] = None def __str__(self): parts = [self.database, self.schema, self.name] return ".".join(part for part in parts if part is not None) def include(self, database: bool = True, schema: bool = True, name: bool = True): if not database: self.database = None if not schema: self.schema = None if not name: self.name = None return self @pytest.fixture def function_node(): return FunctionNode( resource_type=NodeType.Function, name="name", returns=FunctionReturns(data_type="int"), database="db", schema="schema", package_name="pkg", path="path/to/file.sql", original_file_path="path/to/original/file.sql", unique_id="pkg.schema.name", fqn=["pkg", "schema", "name"], alias="alias", checksum=FileHash.from_contents("test"), ) def test_function_node_description(mock_adapter, runtime_config, function_node): mock_adapter.Relation.create_from.return_value = StubRelation( database="db", schema="schema", name="name" ) runner = FunctionRunner( config=runtime_config, adapter=mock_adapter, node=function_node, node_index=0, num_nodes=1, ) assert runner.describe_node() == "function db.schema.name" def test_function_node_description_with_default_database( mock_adapter, runtime_config, function_node ): mock_adapter.Relation.create_from.return_value = StubRelation( database=None, schema="schema", name="name" ) function_node.database = runtime_config.credentials.database runner = FunctionRunner( config=runtime_config, adapter=mock_adapter, node=function_node, node_index=0, num_nodes=1, ) assert runner.describe_node() == "function schema.name" ================================================ FILE: tests/unit/contracts/graph/test_unparsed.py ================================================ import pickle from abc import abstractmethod from datetime import timedelta from typing import Any, Dict import pytest from typing_extensions import override from dbt.artifacts.resources import ( ExposureType, FreshnessThreshold, MaturityType, Owner, Quoting, Time, ) from dbt.artifacts.resources.types import TimePeriod from dbt.artifacts.schemas.results import FreshnessStatus from dbt.contracts.graph.unparsed import ( Docs, HasColumnTests, UnparsedColumn, UnparsedConversionTypeParams, UnparsedDerivedDimensionV2, UnparsedDocumentationFile, UnparsedExposure, UnparsedMacro, UnparsedMetric, UnparsedMetricInputMeasure, UnparsedMetricTypeParams, UnparsedMetricV2, UnparsedModelUpdate, UnparsedNode, UnparsedNodeUpdate, UnparsedRunHook, UnparsedSourceDefinition, UnparsedSourceTableDefinition, UnparsedVersion, ) from dbt.exceptions import ParsingError from dbt.node_types import NodeType from dbt.parser.schemas import ParserRef from dbt_semantic_interfaces.type_enums.conversion_calculation_type import ( ConversionCalculationType, ) from tests.unit.utils import ContractTestCase class TestUnparsedMacro(ContractTestCase): ContractType = UnparsedMacro def test_ok(self): macro_dict = { "path": "/root/path.sql", "original_file_path": "/root/path.sql", "package_name": "test", "language": "sql", "raw_code": "{% macro foo() %}select 1 as id{% endmacro %}", "resource_type": "macro", } macro = self.ContractType( path="/root/path.sql", original_file_path="/root/path.sql", package_name="test", language="sql", raw_code="{% macro foo() %}select 1 as id{% endmacro %}", resource_type=NodeType.Macro, ) self.assert_symmetric(macro, macro_dict) pickle.loads(pickle.dumps(macro)) def test_invalid_missing_field(self): macro_dict = { "path": "/root/path.sql", "original_file_path": "/root/path.sql", # 'package_name': 'test', "language": "sql", "raw_code": "{% macro foo() %}select 1 as id{% endmacro %}", "resource_type": "macro", } self.assert_fails_validation(macro_dict) def test_invalid_extra_field(self): macro_dict = { "path": "/root/path.sql", "original_file_path": "/root/path.sql", "package_name": "test", "language": "sql", "raw_code": "{% macro foo() %}select 1 as id{% endmacro %}", "extra": "extra", "resource_type": "macro", } self.assert_fails_validation(macro_dict) class TestUnparsedNode(ContractTestCase): ContractType = UnparsedNode def test_ok(self): node_dict = { "name": "foo", "resource_type": NodeType.Model, "path": "/root/x/path.sql", "original_file_path": "/root/path.sql", "package_name": "test", "language": "sql", "raw_code": 'select * from {{ ref("thing") }}', } node = self.ContractType( package_name="test", path="/root/x/path.sql", original_file_path="/root/path.sql", language="sql", raw_code='select * from {{ ref("thing") }}', name="foo", resource_type=NodeType.Model, ) self.assert_symmetric(node, node_dict) self.assertFalse(node.empty) self.assert_fails_validation(node_dict, cls=UnparsedRunHook) self.assert_fails_validation(node_dict, cls=UnparsedMacro) pickle.loads(pickle.dumps(node)) def test_empty(self): node_dict = { "name": "foo", "resource_type": NodeType.Model, "path": "/root/x/path.sql", "original_file_path": "/root/path.sql", "package_name": "test", "language": "sql", "raw_code": " \n", } node = UnparsedNode( package_name="test", path="/root/x/path.sql", original_file_path="/root/path.sql", language="sql", raw_code=" \n", name="foo", resource_type=NodeType.Model, ) self.assert_symmetric(node, node_dict) self.assertTrue(node.empty) self.assert_fails_validation(node_dict, cls=UnparsedRunHook) self.assert_fails_validation(node_dict, cls=UnparsedMacro) class TestUnparsedRunHook(ContractTestCase): ContractType = UnparsedRunHook def test_ok(self): node_dict = { "name": "foo", "resource_type": NodeType.Operation, "path": "/root/dbt_project.yml", "original_file_path": "/root/dbt_project.yml", "package_name": "test", "language": "sql", "raw_code": "GRANT select on dbt_postgres", "index": 4, } node = self.ContractType( package_name="test", path="/root/dbt_project.yml", original_file_path="/root/dbt_project.yml", language="sql", raw_code="GRANT select on dbt_postgres", name="foo", resource_type=NodeType.Operation, index=4, ) self.assert_symmetric(node, node_dict) self.assert_fails_validation(node_dict, cls=UnparsedNode) pickle.loads(pickle.dumps(node)) def test_bad_type(self): node_dict = { "name": "foo", "resource_type": NodeType.Model, # invalid "path": "/root/dbt_project.yml", "original_file_path": "/root/dbt_project.yml", "package_name": "test", "language": "sql", "raw_code": "GRANT select on dbt_postgres", "index": 4, } self.assert_fails_validation(node_dict) class TestFreshnessThreshold(ContractTestCase): ContractType = FreshnessThreshold def test_empty(self): empty = self.ContractType() self.assert_symmetric(empty, {"error_after": {}, "warn_after": {}}) self.assertEqual(empty.status(float("Inf")), FreshnessStatus.Pass) self.assertEqual(empty.status(0), FreshnessStatus.Pass) def test_both(self): threshold = self.ContractType( warn_after=Time(count=18, period=TimePeriod.hour), error_after=Time(count=2, period=TimePeriod.day), ) dct = { "error_after": {"count": 2, "period": "day"}, "warn_after": {"count": 18, "period": "hour"}, } self.assert_symmetric(threshold, dct) error_seconds = timedelta(days=3).total_seconds() warn_seconds = timedelta(days=1).total_seconds() pass_seconds = timedelta(hours=3).total_seconds() self.assertEqual(threshold.status(error_seconds), FreshnessStatus.Error) self.assertEqual(threshold.status(warn_seconds), FreshnessStatus.Warn) self.assertEqual(threshold.status(pass_seconds), FreshnessStatus.Pass) pickle.loads(pickle.dumps(threshold)) def test_merged(self): t1 = self.ContractType( warn_after=Time(count=36, period=TimePeriod.hour), error_after=Time(count=2, period=TimePeriod.day), ) t2 = self.ContractType( warn_after=Time(count=18, period=TimePeriod.hour), ) threshold = self.ContractType( warn_after=Time(count=18, period=TimePeriod.hour), error_after=Time(count=None, period=None), ) self.assertEqual(threshold, t1.merged(t2)) warn_seconds = timedelta(days=1).total_seconds() pass_seconds = timedelta(hours=3).total_seconds() self.assertEqual(threshold.status(warn_seconds), FreshnessStatus.Warn) self.assertEqual(threshold.status(pass_seconds), FreshnessStatus.Pass) class TestQuoting(ContractTestCase): ContractType = Quoting def test_empty(self): empty = self.ContractType() self.assert_symmetric(empty, {}) def test_partial(self): a = self.ContractType(None, True, False) b = self.ContractType(True, False, None) self.assert_symmetric(a, {"schema": True, "identifier": False}) self.assert_symmetric(b, {"database": True, "schema": False}) c = a.merged(b) self.assertEqual(c, self.ContractType(True, False, False)) self.assert_symmetric(c, {"database": True, "schema": False, "identifier": False}) pickle.loads(pickle.dumps(c)) class TestUnparsedSourceDefinition(ContractTestCase): ContractType = UnparsedSourceDefinition def test_defaults(self): minimum = self.ContractType(name="foo") from_dict = {"name": "foo"} to_dict = { "name": "foo", "description": "", "freshness": {"error_after": {}, "warn_after": {}}, "quoting": {}, "tables": [], "loader": "", "meta": {}, "tags": [], "config": {}, } self.assert_from_dict(minimum, from_dict) self.assert_to_dict(minimum, to_dict) def test_contents(self): empty = self.ContractType( name="foo", description="a description", quoting=Quoting(database=False), loader="some_loader", freshness=FreshnessThreshold(), tables=[], meta={}, ) dct = { "name": "foo", "description": "a description", "quoting": {"database": False}, "loader": "some_loader", "freshness": {"error_after": {}, "warn_after": {}}, "tables": [], "meta": {}, "tags": [], "config": {}, } self.assert_symmetric(empty, dct) def test_table_defaults(self): table_1 = UnparsedSourceTableDefinition(name="table1") table_2 = UnparsedSourceTableDefinition( name="table2", description="table 2", quoting=Quoting(database=True), ) source = self.ContractType(name="foo", tables=[table_1, table_2]) from_dict = { "name": "foo", "tables": [ {"name": "table1"}, { "name": "table2", "description": "table 2", "quoting": {"database": True}, }, ], } to_dict = { "name": "foo", "description": "", "config": {}, "loader": "", "freshness": {"error_after": {}, "warn_after": {}}, "quoting": {}, "meta": {}, "tables": [ { "name": "table1", "description": "", "config": {}, "docs": {"show": True}, "data_tests": [], "tests": [], "columns": [], "constraints": [], "quoting": {}, "freshness": {"error_after": {}, "warn_after": {}}, "meta": {}, "tags": [], }, { "name": "table2", "description": "table 2", "config": {}, "docs": {"show": True}, "data_tests": [], "tests": [], "columns": [], "constraints": [], "quoting": {"database": True}, "freshness": {"error_after": {}, "warn_after": {}}, "meta": {}, "tags": [], }, ], "tags": [], } self.assert_from_dict(source, from_dict) self.assert_symmetric(source, to_dict) pickle.loads(pickle.dumps(source)) class TestUnparsedDocumentationFile(ContractTestCase): ContractType = UnparsedDocumentationFile def test_ok(self): doc = self.ContractType( package_name="test", path="/root/docs", original_file_path="/root/docs/doc.md", file_contents="blah blah blah", ) doc_dict = { "package_name": "test", "path": "/root/docs", "original_file_path": "/root/docs/doc.md", "file_contents": "blah blah blah", } self.assert_symmetric(doc, doc_dict) self.assertEqual(doc.resource_type, NodeType.Documentation) self.assert_fails_validation(doc_dict, UnparsedNode) pickle.loads(pickle.dumps(doc)) def test_extra_field(self): self.assert_fails_validation({}) doc_dict = { "package_name": "test", "path": "/root/docs", "original_file_path": "/root/docs/doc.md", "file_contents": "blah blah blah", "resource_type": "docs", } self.assert_fails_validation(doc_dict) class TestUnparsedNodeUpdate(ContractTestCase): ContractType = UnparsedNodeUpdate def test_defaults(self): minimum = self.ContractType( name="foo", yaml_key="models", original_file_path="/some/fake/path", package_name="test", ) from_dict = { "name": "foo", "yaml_key": "models", "original_file_path": "/some/fake/path", "package_name": "test", } to_dict = { "name": "foo", "yaml_key": "models", "original_file_path": "/some/fake/path", "package_name": "test", "columns": [], "description": "", "docs": {"show": True}, "data_tests": [], "tests": [], "meta": {}, "config": {}, "constraints": [], } self.assert_from_dict(minimum, from_dict) self.assert_to_dict(minimum, to_dict) def test_contents(self): update = self.ContractType( name="foo", yaml_key="models", original_file_path="/some/fake/path", package_name="test", description="a description", data_tests=["table_test"], meta={"key": ["value1", "value2"]}, columns=[ UnparsedColumn( name="x", description="x description", meta={"key2": "value3"}, ), UnparsedColumn( name="y", description="y description", data_tests=["unique", {"accepted_values": {"values": ["blue", "green"]}}], meta={}, tags=["a", "b"], ), ], docs=Docs(show=False), ) dct = { "name": "foo", "yaml_key": "models", "original_file_path": "/some/fake/path", "package_name": "test", "description": "a description", "data_tests": ["table_test"], "tests": [], "meta": {"key": ["value1", "value2"]}, "constraints": [], "columns": [ { "name": "x", "description": "x description", "docs": {"show": True}, "data_tests": [], "tests": [], "meta": {"key2": "value3"}, "tags": [], "constraints": [], "config": {}, }, { "name": "y", "description": "y description", "docs": {"show": True}, "data_tests": ["unique", {"accepted_values": {"values": ["blue", "green"]}}], "tests": [], "meta": {}, "tags": ["a", "b"], "constraints": [], "config": {}, }, ], "docs": {"show": False}, "config": {}, } self.assert_symmetric(update, dct) pickle.loads(pickle.dumps(update)) def test_bad_test_type(self): dct = { "name": "foo", "yaml_key": "models", "original_file_path": "/some/fake/path", "package_name": "test", "description": "a description", "data_tests": ["table_test"], "tests": [], "meta": {"key": ["value1", "value2"]}, "columns": [ { "name": "x", "description": "x description", "docs": {"show": True}, "data_tests": [], "tests": [], "meta": {"key2": "value3"}, }, { "name": "y", "description": "y description", "docs": {"show": True}, "data_tests": [100, {"accepted_values": {"values": ["blue", "green"]}}], "tests": [], "meta": {}, "yaml_key": "models", "original_file_path": "/some/fake/path", }, ], "docs": {"show": True}, } self.assert_fails_validation(dct) dct = { "name": "foo", "yaml_key": "models", "original_file_path": "/some/fake/path", "package_name": "test", "description": "a description", "data_tests": ["table_test"], "tests": [], "meta": {"key": ["value1", "value2"]}, "columns": [ # column missing a name { "description": "x description", "docs": {"show": True}, "data_tests": [], "tests": [], "meta": {"key2": "value3"}, }, { "name": "y", "description": "y description", "docs": {"show": True}, "data_tests": ["unique", {"accepted_values": {"values": ["blue", "green"]}}], "tests": [], "meta": {}, "yaml_key": "models", "original_file_path": "/some/fake/path", }, ], "docs": {"show": True}, } self.assert_fails_validation(dct) # missing a name dct = { "yaml_key": "models", "original_file_path": "/some/fake/path", "package_name": "test", "description": "a description", "data_tests": ["table_test"], "tests": [], "meta": {"key": ["value1", "value2"]}, "columns": [ { "name": "x", "description": "x description", "docs": {"show": True}, "data_tests": [], "tests": [], "meta": {"key2": "value3"}, }, { "name": "y", "description": "y description", "docs": {"show": True}, "data_tests": ["unique", {"accepted_values": {"values": ["blue", "green"]}}], "tests": [], "meta": {}, "yaml_key": "models", "original_file_path": "/some/fake/path", }, ], "docs": {"show": True}, } self.assert_fails_validation(dct) class TestUnparsedModelUpdate(ContractTestCase): ContractType = UnparsedModelUpdate def test_defaults(self): minimum = self.ContractType( name="foo", yaml_key="models", original_file_path="/some/fake/path", package_name="test", ) from_dict = { "name": "foo", "yaml_key": "models", "original_file_path": "/some/fake/path", "package_name": "test", } to_dict = { "name": "foo", "yaml_key": "models", "original_file_path": "/some/fake/path", "package_name": "test", "columns": [], "description": "", "docs": {"show": True}, "data_tests": [], "tests": [], "meta": {}, "config": {}, "constraints": [], "versions": [], } self.assert_from_dict(minimum, from_dict) self.assert_to_dict(minimum, to_dict) def test_contents(self): update = self.ContractType( name="foo", yaml_key="models", original_file_path="/some/fake/path", package_name="test", description="a description", data_tests=["table_test"], meta={"key": ["value1", "value2"]}, columns=[ UnparsedColumn( name="x", description="x description", meta={"key2": "value3"}, ), UnparsedColumn( name="y", description="y description", data_tests=["unique", {"accepted_values": {"values": ["blue", "green"]}}], meta={}, tags=["a", "b"], ), ], docs=Docs(show=False), versions=[UnparsedVersion(v=2)], ) dct = { "name": "foo", "yaml_key": "models", "original_file_path": "/some/fake/path", "package_name": "test", "description": "a description", "data_tests": ["table_test"], "tests": [], "meta": {"key": ["value1", "value2"]}, "constraints": [], "versions": [ { "v": 2, "description": "", "columns": [], "config": {}, "constraints": [], "docs": {"show": True}, } ], "columns": [ { "name": "x", "description": "x description", "docs": {"show": True}, "data_tests": [], "tests": [], "meta": {"key2": "value3"}, "tags": [], "constraints": [], "config": {}, }, { "name": "y", "description": "y description", "docs": {"show": True}, "data_tests": ["unique", {"accepted_values": {"values": ["blue", "green"]}}], "tests": [], "meta": {}, "tags": ["a", "b"], "constraints": [], "config": {}, }, ], "docs": {"show": False}, "config": {}, } self.assert_symmetric(update, dct) pickle.loads(pickle.dumps(update)) def test_bad_test_type(self): dct = { "name": "foo", "yaml_key": "models", "original_file_path": "/some/fake/path", "package_name": "test", "description": "a description", "data_tests": ["table_test"], "tests": [], "meta": {"key": ["value1", "value2"]}, "columns": [ { "name": "x", "description": "x description", "docs": {"show": True}, "data_tests": [], "tests": [], "meta": {"key2": "value3"}, }, { "name": "y", "description": "y description", "docs": {"show": True}, "data_tests": [100, {"accepted_values": {"values": ["blue", "green"]}}], "tests": [], "meta": {}, "yaml_key": "models", "original_file_path": "/some/fake/path", }, ], "docs": {"show": True}, } self.assert_fails_validation(dct) dct = { "name": "foo", "yaml_key": "models", "original_file_path": "/some/fake/path", "package_name": "test", "description": "a description", "data_tests": ["table_test"], "tests": [], "meta": {"key": ["value1", "value2"]}, "columns": [ # column missing a name { "description": "x description", "docs": {"show": True}, "data_tests": [], "tests": [], "meta": {"key2": "value3"}, }, { "name": "y", "description": "y description", "docs": {"show": True}, "data_tests": ["unique", {"accepted_values": {"values": ["blue", "green"]}}], "tests": [], "meta": {}, "yaml_key": "models", "original_file_path": "/some/fake/path", }, ], "docs": {"show": True}, } self.assert_fails_validation(dct) # missing a name dct = { "yaml_key": "models", "original_file_path": "/some/fake/path", "package_name": "test", "description": "a description", "data_tests": ["table_test"], "tests": [], "meta": {"key": ["value1", "value2"]}, "columns": [ { "name": "x", "description": "x description", "docs": {"show": True}, "data_tests": [], "tests": [], "meta": {"key2": "value3"}, }, { "name": "y", "description": "y description", "docs": {"show": True}, "data_tests": ["unique", {"accepted_values": {"values": ["blue", "green"]}}], "tests": [], "meta": {}, "yaml_key": "models", "original_file_path": "/some/fake/path", }, ], "docs": {"show": True}, } self.assert_fails_validation(dct) class TestUnparsedExposure(ContractTestCase): ContractType = UnparsedExposure def get_ok_dict(self): return { "name": "my_exposure", "type": "dashboard", "owner": {"name": "example", "email": "name@example.com", "slack": "#channel"}, "maturity": "medium", "meta": {"tool": "my_tool"}, "tags": ["my_department"], "url": "https://example.com/dashboards/1", "description": "A exposure", "config": {}, "depends_on": [ 'ref("my_model")', 'source("raw", "source_table")', ], } def test_ok(self): exposure = self.ContractType( name="my_exposure", type=ExposureType.Dashboard, owner=Owner(name="example", email="name@example.com", _extra={"slack": "#channel"}), maturity=MaturityType.Medium, url="https://example.com/dashboards/1", description="A exposure", config={}, meta={"tool": "my_tool"}, tags=["my_department"], depends_on=['ref("my_model")', 'source("raw", "source_table")'], ) dct = self.get_ok_dict() self.assert_symmetric(exposure, dct) pickle.loads(pickle.dumps(exposure)) def test_ok_exposures(self): for exposure_allowed in ("dashboard", "notebook", "analysis", "ml", "application"): tst = self.get_ok_dict() tst["type"] = exposure_allowed assert self.ContractType.from_dict(tst).type == exposure_allowed def test_bad_exposure(self): # bad exposure: None isn't allowed for exposure_not_allowed in (None, "not an exposure"): tst = self.get_ok_dict() tst["type"] = exposure_not_allowed self.assert_fails_validation(tst) def test_no_exposure(self): tst = self.get_ok_dict() del tst["type"] self.assert_fails_validation(tst) def test_ok_maturities(self): for maturity_allowed in (None, "low", "medium", "high"): tst = self.get_ok_dict() tst["maturity"] = maturity_allowed assert self.ContractType.from_dict(tst).maturity == maturity_allowed tst = self.get_ok_dict() del tst["maturity"] assert self.ContractType.from_dict(tst).maturity is None def test_bad_maturity(self): tst = self.get_ok_dict() tst["maturity"] = "invalid maturity" self.assert_fails_validation(tst) def test_bad_owner_missing_things(self): tst = self.get_ok_dict() del tst["owner"]["email"] del tst["owner"]["name"] self.assert_fails_validation(tst) del tst["owner"] self.assert_fails_validation(tst) def test_bad_tags(self): tst = self.get_ok_dict() tst["tags"] = [123] self.assert_fails_validation(tst) class TestUnparsedConversionTypeParams(ContractTestCase): """Only Applies to v1 Semantic Metrics.""" ContractType = UnparsedConversionTypeParams def get_old_style_ok_dict(self): return { "entity": "customers", "base_measure": { "name": "customers", "filter": "is_new = true", "join_to_timespine": False, }, "conversion_measure": "orders", "calculation": "conversion_rate", "window": "7d", } def test_old_style_ok(self): params = self.ContractType.from_dict(self.get_old_style_ok_dict()) assert params.base_measure is not None assert params.conversion_measure is not None assert params.calculation == ConversionCalculationType.CONVERSION_RATE.value assert params.window == "7d" def test_old_style_bad_no_base_measure(self): tst = self.get_old_style_ok_dict() del tst["base_measure"] self.assert_fails_validation(tst) def test_old_style_bad_no_conversion_measure(self): tst = self.get_old_style_ok_dict() del tst["conversion_measure"] self.assert_fails_validation(tst) class BaseTestUnparsedMetric: @abstractmethod def get_ok_dict(self) -> Dict[str, Any]: raise NotImplementedError() def test_bad_tags(self): tst = self.get_ok_dict() tst["tags"] = [123] self.assert_fails_validation(tst) def test_bad_metric_name_with_spaces(self): tst = self.get_ok_dict() tst["name"] = "metric name with spaces" self.assert_fails_validation(tst) def test_bad_metric_name_too_long(self): tst = self.get_ok_dict() tst["name"] = "a" * 251 self.assert_fails_validation(tst) def test_bad_metric_name_does_not_start_with_letter(self): tst = self.get_ok_dict() tst["name"] = "123metric" self.assert_fails_validation(tst) tst["name"] = "_metric" self.assert_fails_validation(tst) def test_bad_metric_name_contains_special_characters(self): tst = self.get_ok_dict() tst["name"] = "metric!name" self.assert_fails_validation(tst) tst["name"] = "metric@name" self.assert_fails_validation(tst) tst["name"] = "metric#name" self.assert_fails_validation(tst) tst["name"] = "metric$name" self.assert_fails_validation(tst) tst["name"] = "metric-name" self.assert_fails_validation(tst) class TestUnparsedMetric(BaseTestUnparsedMetric, ContractTestCase): ContractType = UnparsedMetric @override def get_ok_dict(self): return { "name": "new_customers", "label": "New Customers", "description": "New customers", "type": "simple", "type_params": { "measure": { "name": "customers", "filter": "is_new = true", "join_to_timespine": False, }, }, "config": {}, "tags": [], "meta": {"is_okr": True}, } def test_ok(self): metric = self.ContractType( name="new_customers", label="New Customers", description="New customers", type="simple", type_params=UnparsedMetricTypeParams( measure=UnparsedMetricInputMeasure( name="customers", filter="is_new = true", ) ), config={}, meta={"is_okr": True}, ) dct = self.get_ok_dict() self.assert_symmetric(metric, dct) pickle.loads(pickle.dumps(metric)) def test_bad_metric_no_type_params(self): tst = self.get_ok_dict() del tst["type_params"] self.assert_fails_validation(tst) class TestUnparsedMetricV2(BaseTestUnparsedMetric, ContractTestCase): ContractType = UnparsedMetricV2 @override def get_ok_dict(self): return { "name": "new_customers", "label": "New Customers", "description": "New customers", "type": "simple", "agg": "sum", "filter": "is_new = true", "join_to_timespine": False, "config": { "tags": [], "meta": {"is_okr": True}, }, } def get_ok_dict_with_defaults(self): dct = self.get_ok_dict() dct["hidden"] = False dct["period_agg"] = "first" return dct def test_ok(self): metric = self.ContractType( name="new_customers", label="New Customers", description="New customers", agg="sum", filter="is_new = true", join_to_timespine=False, config={ "tags": [], "meta": {"is_okr": True}, }, ) dct = self.get_ok_dict() # add defaults: dct["hidden"] = False dct["period_agg"] = "first" self.assert_symmetric(metric, dct) pickle.loads(pickle.dumps(metric)) def test_simple_metric_with_no_agg_fails_validation(self): tst = self.get_ok_dict_with_defaults() del tst["agg"] self.assert_fails_validation(tst) class TestUnparsedVersion(ContractTestCase): ContractType = UnparsedVersion def get_ok_dict(self): return { "v": 2, "defined_in": "test_defined_in", "description": "A version", "config": {}, "constraints": [], "docs": {"show": False}, "data_tests": [], "columns": [], } def test_ok(self): version = self.ContractType( v=2, defined_in="test_defined_in", description="A version", config={}, constraints=[], docs=Docs(show=False), data_tests=[], columns=[], ) dct = self.get_ok_dict() self.assert_symmetric(version, dct) pickle.loads(pickle.dumps(version)) def test_bad_version_no_v(self): version = self.get_ok_dict() del version["v"] self.assert_fails_validation(version) @pytest.mark.parametrize( "left,right,expected_lt", [ # same types (2, 12, True), (12, 2, False), ("a", "b", True), ("b", "a", False), # mismatched types - numeric (2, 12.0, True), (12.0, 2, False), (2, "12", True), ("12", 2, False), # mismatched types (1, "test", True), ("test", 1, False), ], ) def test_unparsed_version_lt(left, right, expected_lt): assert (UnparsedVersion(left) < UnparsedVersion(right)) == expected_lt def test_column_parse(): unparsed_col = HasColumnTests( columns=[UnparsedColumn(name="TestCol", constraints=[{"type": "!INVALID!"}])] ) with pytest.raises(ParsingError): ParserRef.from_target(unparsed_col) class TestUnparsedColumnTimeDimensionGranularityValidation(ContractTestCase): """Test validation that SL YAML V2 column with time dimension must specify granularity, and that dimension validity_params require granularity.""" ContractType = UnparsedColumn def test_time_dimension_without_granularity_fails_validation(self): column_dict = { "name": "created_at", "dimension": {"type": "time", "name": "created_at_dim"}, } self.assert_fails_validation(column_dict) def test_time_dimension_with_granularity_passes_validation(self): column_dict = { "name": "created_at", "granularity": "day", "dimension": {"type": "time", "name": "created_at_dim"}, } col = self.ContractType.from_dict(column_dict) self.assertEqual(col.granularity, "day") self.assertEqual(col.name, "created_at") def test_time_dimension_string_without_granularity_fails_validation(self): """Dimension as string 'time' without granularity must fail.""" column_dict = { "name": "created_at", "dimension": "time", } self.assert_fails_validation(column_dict) def test_time_dimension_string_with_granularity_passes_validation(self): """Dimension as string 'time' with granularity must pass.""" column_dict = { "name": "created_at", "granularity": "day", "dimension": "time", } col = self.ContractType.from_dict(column_dict) self.assertEqual(col.granularity, "day") self.assertEqual(col.name, "created_at") def test_non_time_dimension_string_passes_without_granularity(self): """Dimension as string (e.g. categorical) does not require granularity.""" column_dict = { "name": "category", "dimension": "categorical", } col = self.ContractType.from_dict(column_dict) self.assertEqual(col.name, "category") self.assertIsNone(col.granularity) def test_dimension_with_validity_params_without_granularity_fails_validation(self): """Dimension (dict) with validity_params must have column granularity.""" column_dict = { "name": "valid_from", "dimension": { "type": "time", "name": "valid_from_dim", "validity_params": {"is_start": True, "is_end": False}, }, } self.assert_fails_validation(column_dict) def test_dimension_with_validity_params_with_granularity_passes_validation(self): """Dimension (dict) with validity_params and granularity passes.""" column_dict = { "name": "valid_from", "granularity": "day", "dimension": { "type": "time", "name": "valid_from_dim", "validity_params": {"is_start": True, "is_end": True}, }, } col = self.ContractType.from_dict(column_dict) self.assertEqual(col.granularity, "day") self.assertEqual(col.name, "valid_from") def test_dimension_without_validity_params_passes_without_granularity_when_not_time(self): """Dimension (dict) without validity_params and not time does not require granularity.""" column_dict = { "name": "category", "dimension": {"type": "categorical", "name": "category_dim"}, } col = self.ContractType.from_dict(column_dict) self.assertEqual(col.name, "category") self.assertIsNone(col.granularity) class TestUnparsedDerivedDimensionV2ValidityParamsValidation(ContractTestCase): """Test validation that UnparsedDerivedDimensionV2 only has validity_params when it has granularity.""" ContractType = UnparsedDerivedDimensionV2 def test_derived_dimension_with_validity_params_without_granularity_fails_validation(self): """Derived dimension with validity_params must have granularity.""" dimension_dict = { "type": "time", "name": "valid_from_dim", "expr": "valid_from", "validity_params": {"is_start": True, "is_end": False}, } self.assert_fails_validation(dimension_dict) def test_derived_dimension_with_validity_params_with_granularity_passes_validation(self): """Derived dimension with validity_params and granularity passes.""" dimension_dict = { "type": "time", "name": "valid_from_dim", "expr": "valid_from", "granularity": "day", "validity_params": {"is_start": True, "is_end": True}, } dim = self.ContractType.from_dict(dimension_dict) self.assertEqual(dim.granularity, "day") self.assertEqual(dim.name, "valid_from_dim") def test_derived_dimension_without_validity_params_passes_without_granularity(self): """Derived dimension without validity_params does not require granularity.""" dimension_dict = { "type": "time", "name": "some_dim", "expr": "some_column", } dim = self.ContractType.from_dict(dimension_dict) self.assertEqual(dim.name, "some_dim") self.assertIsNone(dim.granularity) ================================================ FILE: tests/unit/contracts/test_project.py ================================================ from dbt.contracts.project import Project from dbt_common.dataclass_schema import ValidationError from tests.unit.utils import ContractTestCase class TestProject(ContractTestCase): ContractType = Project def test_minimal(self): dct = { "name": "test", "version": "1.0", "profile": "test", "project-root": "/usr/src/app", "config-version": 2, } project = self.ContractType( name="test", version="1.0", profile="test", project_root="/usr/src/app", config_version=2, ) self.assert_from_dict(project, dct) def test_invalid_name(self): dct = { "name": "log", "version": "1.0", "profile": "test", "project-root": "/usr/src/app", "config-version": 2, } with self.assertRaises(ValidationError): self.ContractType.validate(dct) ================================================ FILE: tests/unit/deps/__init__.py ================================================ ================================================ FILE: tests/unit/deps/test_deps.py ================================================ import unittest from argparse import Namespace from copy import deepcopy from unittest import mock import dbt.deps import dbt.exceptions from dbt.clients.registry import is_compatible_version from dbt.config.project import PartialProject from dbt.config.renderer import DbtProjectYamlRenderer from dbt.contracts.project import ( GitPackage, LocalPackage, PackageConfig, RegistryPackage, TarballPackage, ) from dbt.deps.git import GitUnpinnedPackage from dbt.deps.local import LocalPinnedPackage, LocalUnpinnedPackage from dbt.deps.registry import RegistryUnpinnedPackage from dbt.deps.resolver import resolve_packages from dbt.deps.tarball import TarballUnpinnedPackage from dbt.flags import set_from_args from dbt.task.deps import DepsTask from dbt.version import get_installed_version from dbt_common.dataclass_schema import ValidationError from dbt_common.semver import VersionSpecifier set_from_args(Namespace(WARN_ERROR=False), None) class TestLocalPackage(unittest.TestCase): def test_init(self): a_contract = LocalPackage.from_dict({"local": "/path/to/package"}) self.assertEqual(a_contract.local, "/path/to/package") a = LocalUnpinnedPackage.from_contract(a_contract) self.assertEqual(a.local, "/path/to/package") a_pinned = a.resolved() self.assertEqual(a_pinned.local, "/path/to/package") self.assertEqual(str(a_pinned), "/path/to/package") class TestTarballPackage(unittest.TestCase): class MockMetadata: name = "mock_metadata_name" @mock.patch("dbt.config.project.PartialProject.from_project_root") @mock.patch("os.listdir") @mock.patch("dbt.deps.tarball.get_downloads_path") @mock.patch("dbt_common.clients.system.untar_package") @mock.patch("dbt_common.clients.system.download") def test_fetch_metadata( self, mock_download, mock_untar_package, mock_get_downloads_path, mock_listdir, mock_from_project_root, ): mock_listdir.return_value = ["one_directory/"] mock_get_downloads_path.return_value = "downloads_path" mock_from_project_root.return_value = object() mock_from_project_root.return_value dict_well_formed_contract = { "tarball": "http://example.com/invalid_url@/package.tar.gz", "name": "my_package", } a_contract = TarballPackage.from_dict(dict_well_formed_contract) a = TarballUnpinnedPackage.from_contract(a_contract) a_pinned = a.resolved() with mock.patch.object(PartialProject, "from_project_root", return_value=PartialProject): with mock.patch.object( PartialProject, "render_package_metadata", return_value=self.MockMetadata ): metadata = a_pinned.fetch_metadata("", DbtProjectYamlRenderer()) assert metadata == self.MockMetadata mock_download.assert_called_once_with( "http://example.com/invalid_url@/package.tar.gz", "downloads_path/my_package" ) mock_untar_package.assert_called_once_with( "downloads_path/my_package", "downloads_path/my_package_untarred", "my_package" ) @mock.patch("dbt.config.project.PartialProject.from_project_root") @mock.patch("os.listdir") @mock.patch("dbt.deps.tarball.get_downloads_path") @mock.patch("dbt_common.clients.system.untar_package") @mock.patch("dbt_common.clients.system.download") def test_fetch_metadata_fails_on_incorrect_tar_folder_structure( self, mock_download, mock_untar_package, mock_get_downloads_path, mock_listdir, mock_from_project_root, ): mock_listdir.return_value = ["one_directory/", "another_directory/"] mock_get_downloads_path.return_value = "downloads_path" mock_from_project_root.return_value = object() mock_from_project_root.return_value dict_well_formed_contract = { "tarball": "http://example.com/invalid_url@/package.tar.gz", "name": "my_package", } a_contract = TarballPackage.from_dict(dict_well_formed_contract) a = TarballUnpinnedPackage.from_contract(a_contract) a_pinned = a.resolved() with mock.patch.object(PartialProject, "from_project_root", return_value=PartialProject): with mock.patch.object( PartialProject, "render_package_metadata", return_value=self.MockMetadata ): with self.assertRaises(dbt.exceptions.DependencyError): a_pinned.fetch_metadata("", DbtProjectYamlRenderer()) @mock.patch("dbt.deps.tarball.get_downloads_path") def test_tarball_package_contract(self, mock_get_downloads_path): dict_well_formed_contract = { "tarball": "http://example.com/invalid_url@/package.tar.gz", "name": "my_cool_package", } a_contract = TarballPackage.from_dict(dict_well_formed_contract) # check contract and resolver self.assertEqual(a_contract.tarball, "http://example.com/invalid_url@/package.tar.gz") self.assertEqual(a_contract.name, "my_cool_package") a = TarballUnpinnedPackage.from_contract(a_contract) self.assertEqual(a.tarball, "http://example.com/invalid_url@/package.tar.gz") self.assertEqual(a.package, "my_cool_package") a_pinned = a.resolved() self.assertEqual(a_pinned.source_type(), "tarball") a_pinned_dict = a_pinned.to_dict() self.assertEqual( a_pinned_dict, { "tarball": "http://example.com/invalid_url@/package.tar.gz", "name": "my_cool_package", }, ) @mock.patch("dbt.deps.tarball.get_downloads_path") def test_tarball_pinned_package_contract_with_unrendered(self, mock_get_downloads_path): contract = TarballPackage( tarball="http://example.com/invalid_url@/package.tar.gz", name="my_cool_package", unrendered={"tarball": "tarball_unrendered"}, ) tarball_unpinned_package = TarballUnpinnedPackage.from_contract(contract) self.assertEqual( tarball_unpinned_package.tarball, "http://example.com/invalid_url@/package.tar.gz" ) self.assertEqual(tarball_unpinned_package.package, "my_cool_package") self.assertEqual(tarball_unpinned_package.tarball_unrendered, "tarball_unrendered") tarball_pinned_package = tarball_unpinned_package.resolved() tarball_unpinned_package_dict = tarball_pinned_package.to_dict() self.assertEqual( tarball_unpinned_package_dict, {"tarball": "tarball_unrendered", "name": "my_cool_package"}, ) def test_tarball_package_contract_fails_on_no_name(self): from mashumaro.exceptions import MissingField # check bad contract (no name) fails a_contract = {"tarball": "http://example.com"} with self.assertRaises(MissingField): TarballPackage.from_dict(a_contract) class TestGitPackage(unittest.TestCase): def test_init(self): a_contract = GitPackage.from_dict( {"git": "http://example.com", "revision": "0.0.1"}, ) self.assertEqual(a_contract.git, "http://example.com") self.assertEqual(a_contract.revision, "0.0.1") self.assertIs(a_contract.warn_unpinned, None) a = GitUnpinnedPackage.from_contract(a_contract) self.assertEqual(a.git, "http://example.com") self.assertEqual(a.revisions, ["0.0.1"]) self.assertIs(a.warn_unpinned, True) a_pinned = a.resolved() self.assertEqual(a_pinned.name, "http://example.com") self.assertEqual(a_pinned.get_version(), "0.0.1") self.assertEqual(a_pinned.source_type(), "git") self.assertIs(a_pinned.warn_unpinned, True) a_pinned_dict = a_pinned.to_dict() self.assertEqual(a_pinned_dict, {"git": "http://example.com", "revision": "0.0.1"}) def test_init_with_unrendered(self): contract = GitPackage( git="http://example.com", revision="0.0.1", unrendered={"git": "git_unrendered"} ) git_unpinned_package = GitUnpinnedPackage.from_contract(contract) self.assertEqual(git_unpinned_package.git, "http://example.com") self.assertEqual(git_unpinned_package.revisions, ["0.0.1"]) self.assertIs(git_unpinned_package.git_unrendered, "git_unrendered") git_pinned_package = git_unpinned_package.resolved() git_pinned_package_dict = git_pinned_package.to_dict() self.assertEqual(git_pinned_package_dict, {"git": "git_unrendered", "revision": "0.0.1"}) @mock.patch("shutil.copytree") @mock.patch("dbt.deps.local.system.make_symlink") @mock.patch("dbt.deps.local.LocalPinnedPackage.get_installation_path") @mock.patch("dbt.deps.local.LocalPinnedPackage.resolve_path") def test_deps_install( self, mock_resolve_path, mock_get_installation_path, mock_symlink, mock_shutil ): mock_resolve_path.return_value = "/tmp/source" mock_get_installation_path.return_value = "/tmp/dest" mock_symlink.side_effect = OSError("Install deps symlink error") LocalPinnedPackage("local").install("dummy", "dummy") self.assertEqual(mock_shutil.call_count, 1) mock_shutil.assert_called_once_with("/tmp/source", "/tmp/dest") def test_invalid(self): with self.assertRaises(ValidationError): GitPackage.validate( {"git": "http://example.com", "version": "0.0.1"}, ) def test_resolve_ok(self): a_contract = GitPackage.from_dict( {"git": "http://example.com", "revision": "0.0.1"}, ) b_contract = GitPackage.from_dict( {"git": "http://example.com", "revision": "0.0.1", "warn-unpinned": False}, ) d_contract = GitPackage.from_dict( {"git": "http://example.com", "revision": "0.0.1", "subdirectory": "foo-bar"}, ) a = GitUnpinnedPackage.from_contract(a_contract) b = GitUnpinnedPackage.from_contract(b_contract) c = a.incorporate(b) d = GitUnpinnedPackage.from_contract(d_contract) self.assertTrue(a.warn_unpinned) self.assertFalse(b.warn_unpinned) self.assertTrue(d.warn_unpinned) c_pinned = c.resolved() self.assertEqual(c_pinned.name, "http://example.com") self.assertEqual(c_pinned.get_version(), "0.0.1") self.assertEqual(c_pinned.source_type(), "git") self.assertFalse(c_pinned.warn_unpinned) d_pinned = d.resolved() self.assertEqual(d_pinned.name, "http://example.com/foo-bar") self.assertEqual(d_pinned.get_version(), "0.0.1") self.assertEqual(d_pinned.source_type(), "git") self.assertEqual(d_pinned.subdirectory, "foo-bar") def test_resolve_fail(self): a_contract = GitPackage.from_dict( {"git": "http://example.com", "revision": "0.0.1"}, ) b_contract = GitPackage.from_dict( {"git": "http://example.com", "revision": "0.0.2"}, ) a = GitUnpinnedPackage.from_contract(a_contract) b = GitUnpinnedPackage.from_contract(b_contract) c = a.incorporate(b) self.assertEqual(c.git, "http://example.com") self.assertEqual(c.revisions, ["0.0.1", "0.0.2"]) with self.assertRaises(dbt.exceptions.DependencyError): c.resolved() def test_default_revision(self): a_contract = GitPackage.from_dict({"git": "http://example.com"}) self.assertEqual(a_contract.revision, None) self.assertIs(a_contract.warn_unpinned, None) a = GitUnpinnedPackage.from_contract(a_contract) self.assertEqual(a.git, "http://example.com") self.assertEqual(a.revisions, []) self.assertIs(a.warn_unpinned, True) a_pinned = a.resolved() self.assertEqual(a_pinned.name, "http://example.com") self.assertEqual(a_pinned.get_version(), "HEAD") self.assertEqual(a_pinned.source_type(), "git") self.assertIs(a_pinned.warn_unpinned, True) class TestHubPackage(unittest.TestCase): def setUp(self): self.patcher = mock.patch("dbt.deps.registry.registry") self.registry = self.patcher.start() self.index_cached = self.registry.index_cached self.get_compatible_versions = self.registry.get_compatible_versions self.package_version = self.registry.package_version self.index_cached.return_value = [ "dbt-labs-test/a", ] self.get_compatible_versions.return_value = ["0.1.2", "0.1.3", "0.1.4a1"] self.package_version.return_value = { "id": "dbt-labs-test/a/0.1.2", "name": "a", "version": "0.1.2", "packages": [], "_source": { "blahblah": "asdfas", }, "downloads": { "tarball": "https://example.com/invalid-url!", "extra": "field", }, "newfield": ["another", "value"], } def tearDown(self): self.patcher.stop() def test_init(self): a_contract = RegistryPackage( package="dbt-labs-test/a", version="0.1.2", ) self.assertEqual(a_contract.package, "dbt-labs-test/a") self.assertEqual(a_contract.version, "0.1.2") a = RegistryUnpinnedPackage.from_contract(a_contract) self.assertEqual(a.package, "dbt-labs-test/a") self.assertEqual( a.versions, [ VersionSpecifier( build=None, major="0", matcher="=", minor="1", patch="2", prerelease=None ) ], ) a_pinned = a.resolved() self.assertEqual(a_contract.package, "dbt-labs-test/a") self.assertEqual(a_contract.version, "0.1.2") self.assertEqual(a_pinned.source_type(), "hub") def test_invalid(self): with self.assertRaises(ValidationError): RegistryPackage.validate( {"package": "namespace/name", "key": "invalid"}, ) def test_resolve_ok(self): a_contract = RegistryPackage(package="dbt-labs-test/a", version="0.1.2") b_contract = RegistryPackage(package="dbt-labs-test/a", version="0.1.2") a = RegistryUnpinnedPackage.from_contract(a_contract) b = RegistryUnpinnedPackage.from_contract(b_contract) c = a.incorporate(b) self.assertEqual(c.package, "dbt-labs-test/a") self.assertEqual( c.versions, [ VersionSpecifier( build=None, major="0", matcher="=", minor="1", patch="2", prerelease=None, ), VersionSpecifier( build=None, major="0", matcher="=", minor="1", patch="2", prerelease=None, ), ], ) c_pinned = c.resolved() self.assertEqual(c_pinned.package, "dbt-labs-test/a") self.assertEqual(c_pinned.version, "0.1.2") self.assertEqual(c_pinned.source_type(), "hub") def test_resolve_missing_package(self): a = RegistryUnpinnedPackage.from_contract( RegistryPackage(package="dbt-labs-test/b", version="0.1.2") ) with self.assertRaises(dbt.exceptions.DependencyError) as exc: a.resolved() msg = "Package dbt-labs-test/b was not found in the package index" self.assertIn(msg, str(exc.exception)) def test_resolve_missing_version(self): a = RegistryUnpinnedPackage.from_contract( RegistryPackage(package="dbt-labs-test/a", version="0.1.4") ) with self.assertRaises(dbt.exceptions.DependencyError) as exc: a.resolved() # Check that key parts of the error message are present and avoid spacing failures error_msg = str(exc.exception) assert ( "Could not find a matching compatible version for package dbt-labs-test/a" in error_msg ) assert "Requested range: =0.1.4, =0.1.4" in error_msg assert "Compatible versions: ['0.1.2', '0.1.3']" in error_msg def test_resolve_conflict(self): a_contract = RegistryPackage(package="dbt-labs-test/a", version="0.1.2") b_contract = RegistryPackage(package="dbt-labs-test/a", version="0.1.3") a = RegistryUnpinnedPackage.from_contract(a_contract) b = RegistryUnpinnedPackage.from_contract(b_contract) c = a.incorporate(b) with self.assertRaises(dbt.exceptions.DependencyError) as exc: c.resolved() msg = ( "Version error for package dbt-labs-test/a: Could not " "find a satisfactory version from options: ['=0.1.2', '=0.1.3']" ) self.assertIn(msg, str(exc.exception)) def test_dependency_error_inherits_from_dbt_runtime_error(self): """ Test that DependencyError is a DbtRuntimeError (not plain Exception). Related to issue #12049 - DependencyError MUST inherit from DbtRuntimeError so that the CLI's exception handler (core/dbt/cli/requires.py line 186) catches it WITHOUT printing Python stack traces. The CLI has two exception handlers: - Line 186-188: Catches DbtBaseException -> NO stack trace (desired) - Line 189-192: Catches BaseException -> SHOWS stack trace (undesired) If DependencyError inherits from Exception, it's caught by the BaseException handler which prints full stack traces. If it inherits from DbtRuntimeError, it's caught by the DbtBaseException handler which shows clean error messages. This is an implementation test (checking inheritance) because the actual behavior (stack trace vs no stack trace) happens in the CLI layer, not here. """ from dbt_common.exceptions import DbtRuntimeError # This is the critical assertion - DependencyError MUST be a DbtRuntimeError self.assertTrue( issubclass(dbt.exceptions.DependencyError, DbtRuntimeError), "DependencyError must inherit from DbtRuntimeError (not plain Exception) " "to avoid stack traces in CLI output. See issue #12049.", ) # Verify this applies to actual raised exceptions from version conflicts a_contract = RegistryPackage(package="dbt-labs-test/a", version="0.1.2") b_contract = RegistryPackage(package="dbt-labs-test/a", version="0.1.3") a = RegistryUnpinnedPackage.from_contract(a_contract) b = RegistryUnpinnedPackage.from_contract(b_contract) c = a.incorporate(b) with self.assertRaises(dbt.exceptions.DependencyError) as exc: c.resolved() # The raised exception instance must be a DbtRuntimeError self.assertIsInstance( exc.exception, DbtRuntimeError, "Raised DependencyError must be instance of DbtRuntimeError", ) # Verify the error message is still clean and user-friendly error_msg = str(exc.exception) self.assertIn("Version error for package dbt-labs-test/a", error_msg) self.assertIn("Could not find a satisfactory version", error_msg) def test_resolve_ranges(self): a_contract = RegistryPackage(package="dbt-labs-test/a", version="0.1.2") b_contract = RegistryPackage(package="dbt-labs-test/a", version="<0.1.4") a = RegistryUnpinnedPackage.from_contract(a_contract) b = RegistryUnpinnedPackage.from_contract(b_contract) c = a.incorporate(b) self.assertEqual(c.package, "dbt-labs-test/a") self.assertEqual( c.versions, [ VersionSpecifier( build=None, major="0", matcher="=", minor="1", patch="2", prerelease=None, ), VersionSpecifier( build=None, major="0", matcher="<", minor="1", patch="4", prerelease=None, ), ], ) c_pinned = c.resolved() self.assertEqual(c_pinned.package, "dbt-labs-test/a") self.assertEqual(c_pinned.version, "0.1.2") self.assertEqual(c_pinned.source_type(), "hub") def test_resolve_ranges_install_prerelease_default_false(self): a_contract = RegistryPackage(package="dbt-labs-test/a", version=">0.1.2") b_contract = RegistryPackage(package="dbt-labs-test/a", version="<0.1.5") a = RegistryUnpinnedPackage.from_contract(a_contract) b = RegistryUnpinnedPackage.from_contract(b_contract) c = a.incorporate(b) self.assertEqual(c.package, "dbt-labs-test/a") self.assertEqual( c.versions, [ VersionSpecifier( build=None, major="0", matcher=">", minor="1", patch="2", prerelease=None, ), VersionSpecifier( build=None, major="0", matcher="<", minor="1", patch="5", prerelease=None, ), ], ) c_pinned = c.resolved() self.assertEqual(c_pinned.package, "dbt-labs-test/a") self.assertEqual(c_pinned.version, "0.1.3") self.assertEqual(c_pinned.source_type(), "hub") def test_resolve_ranges_install_prerelease_true(self): a_contract = RegistryPackage( package="dbt-labs-test/a", version=">0.1.2", install_prerelease=True ) b_contract = RegistryPackage(package="dbt-labs-test/a", version="<0.1.5") a = RegistryUnpinnedPackage.from_contract(a_contract) b = RegistryUnpinnedPackage.from_contract(b_contract) c = a.incorporate(b) self.assertEqual(c.package, "dbt-labs-test/a") self.assertEqual( c.versions, [ VersionSpecifier( build=None, major="0", matcher=">", minor="1", patch="2", prerelease=None, ), VersionSpecifier( build=None, major="0", matcher="<", minor="1", patch="5", prerelease=None, ), ], ) c_pinned = c.resolved() self.assertEqual(c_pinned.package, "dbt-labs-test/a") self.assertEqual(c_pinned.version, "0.1.4a1") self.assertEqual(c_pinned.source_type(), "hub") def test_get_version_latest_prelease_true(self): a_contract = RegistryPackage( package="dbt-labs-test/a", version=">0.1.0", install_prerelease=True ) b_contract = RegistryPackage(package="dbt-labs-test/a", version="<0.1.4") a = RegistryUnpinnedPackage.from_contract(a_contract) b = RegistryUnpinnedPackage.from_contract(b_contract) c = a.incorporate(b) self.assertEqual(c.package, "dbt-labs-test/a") self.assertEqual( c.versions, [ VersionSpecifier( build=None, major="0", matcher=">", minor="1", patch="0", prerelease=None, ), VersionSpecifier( build=None, major="0", matcher="<", minor="1", patch="4", prerelease=None, ), ], ) c_pinned = c.resolved() self.assertEqual(c_pinned.package, "dbt-labs-test/a") self.assertEqual(c_pinned.version, "0.1.3") self.assertEqual(c_pinned.get_version_latest(), "0.1.4a1") self.assertEqual(c_pinned.source_type(), "hub") def test_get_version_latest_prelease_false(self): a_contract = RegistryPackage( package="dbt-labs-test/a", version=">0.1.0", install_prerelease=False ) b_contract = RegistryPackage(package="dbt-labs-test/a", version="<0.1.4") a = RegistryUnpinnedPackage.from_contract(a_contract) b = RegistryUnpinnedPackage.from_contract(b_contract) c = a.incorporate(b) self.assertEqual(c.package, "dbt-labs-test/a") self.assertEqual( c.versions, [ VersionSpecifier( build=None, major="0", matcher=">", minor="1", patch="0", prerelease=None, ), VersionSpecifier( build=None, major="0", matcher="<", minor="1", patch="4", prerelease=None, ), ], ) c_pinned = c.resolved() self.assertEqual(c_pinned.package, "dbt-labs-test/a") self.assertEqual(c_pinned.version, "0.1.3") self.assertEqual(c_pinned.get_version_latest(), "0.1.3") self.assertEqual(c_pinned.source_type(), "hub") def test_get_version_prerelease_explicitly_requested(self): a_contract = RegistryPackage( package="dbt-labs-test/a", version="0.1.4a1", install_prerelease=None ) a = RegistryUnpinnedPackage.from_contract(a_contract) self.assertEqual(a.package, "dbt-labs-test/a") self.assertEqual( a.versions, [ VersionSpecifier( build=None, major="0", matcher="=", minor="1", patch="4", prerelease="a1", ), ], ) a_pinned = a.resolved() self.assertEqual(a_pinned.package, "dbt-labs-test/a") self.assertEqual(a_pinned.version, "0.1.4a1") self.assertEqual(a_pinned.get_version_latest(), "0.1.4a1") self.assertEqual(a_pinned.source_type(), "hub") class MockRegistry: def __init__(self, packages): self.packages = packages def index_cached(self, registry_base_url=None): return sorted(self.packages) def package(self, package_name, registry_base_url=None): try: pkg = self.packages[package_name] except KeyError: return [] return pkg def get_compatible_versions(self, package_name, dbt_version, should_version_check): packages = self.package(package_name) return [ pkg_version for pkg_version, info in packages.items() if is_compatible_version(info, dbt_version) ] def package_version(self, name, version): try: return self.packages[name][version] except KeyError: return None class TestPackageSpec(unittest.TestCase): def setUp(self): dbt_version = get_installed_version() next_version = deepcopy(dbt_version) next_version.minor = str(int(next_version.minor) + 1) next_version.prerelease = None require_next_version = ">" + next_version.to_version_string() self.patcher = mock.patch("dbt.deps.registry.registry") self.registry = self.patcher.start() self.mock_registry = MockRegistry( packages={ "dbt-labs-test/a": { "0.1.2": { "id": "dbt-labs-test/a/0.1.2", "name": "a", "version": "0.1.2", "packages": [], "_source": { "blahblah": "asdfas", }, "downloads": { "tarball": "https://example.com/invalid-url!", "extra": "field", }, "newfield": ["another", "value"], }, "0.1.3": { "id": "dbt-labs-test/a/0.1.3", "name": "a", "version": "0.1.3", "packages": [], "_source": { "blahblah": "asdfas", }, "downloads": { "tarball": "https://example.com/invalid-url!", "extra": "field", }, "newfield": ["another", "value"], }, "0.1.4a1": { "id": "dbt-labs-test/a/0.1.3a1", "name": "a", "version": "0.1.4a1", "packages": [], "_source": { "blahblah": "asdfas", }, "downloads": { "tarball": "https://example.com/invalid-url!", "extra": "field", }, "newfield": ["another", "value"], }, "0.2.0": { "id": "dbt-labs-test/a/0.2.0", "name": "a", "version": "0.2.0", "packages": [], "_source": { "blahblah": "asdfas", }, # this one shouldn't be picked! "require_dbt_version": require_next_version, "downloads": { "tarball": "https://example.com/invalid-url!", "extra": "field", }, "newfield": ["another", "value"], }, }, "dbt-labs-test/b": { "0.2.1": { "id": "dbt-labs-test/b/0.2.1", "name": "b", "version": "0.2.1", "packages": [{"package": "dbt-labs-test/a", "version": ">=0.1.3"}], "_source": { "blahblah": "asdfas", }, "downloads": { "tarball": "https://example.com/invalid-url!", "extra": "field", }, "newfield": ["another", "value"], }, }, } ) self.registry.index_cached.side_effect = self.mock_registry.index_cached self.registry.get_compatible_versions.side_effect = ( self.mock_registry.get_compatible_versions ) self.registry.package_version.side_effect = self.mock_registry.package_version def tearDown(self): self.patcher.stop() def test_dependency_resolution(self): package_config = PackageConfig.from_dict( { "packages": [ {"package": "dbt-labs-test/a", "version": ">0.1.2"}, {"package": "dbt-labs-test/b", "version": "0.2.1"}, ], } ) resolved = resolve_packages( package_config.packages, mock.MagicMock(project_name="test"), {} ) self.assertEqual(len(resolved), 2) self.assertEqual(resolved[0].name, "dbt-labs-test/a") self.assertEqual(resolved[0].version, "0.1.3") self.assertEqual(resolved[1].name, "dbt-labs-test/b") self.assertEqual(resolved[1].version, "0.2.1") def test_private_package_raise_error(self): package_config = PackageConfig.from_dict( { "packages": [ {"private": "dbt-labs-test/a", "subdirectory": "foo-bar"}, ], } ) with self.assertRaisesRegex( dbt.exceptions.DependencyError, "Cannot resolve private package" ): resolve_packages(package_config.packages, mock.MagicMock(project_name="test"), {}) def test_dependency_resolution_allow_prerelease(self): package_config = PackageConfig.from_dict( { "packages": [ { "package": "dbt-labs-test/a", "version": ">0.1.2", "install_prerelease": True, }, {"package": "dbt-labs-test/b", "version": "0.2.1"}, ], } ) resolved = resolve_packages( package_config.packages, mock.MagicMock(project_name="test"), {} ) self.assertEqual(resolved[0].name, "dbt-labs-test/a") self.assertEqual(resolved[0].version, "0.1.4a1") def test_validation_error_when_version_is_missing_from_package_config(self): packages_data = {"packages": [{"package": "dbt-labs-test/b", "version": None}]} with self.assertRaises(ValidationError) as exc: PackageConfig.validate(data=packages_data) msg = "dbt-labs-test/b is missing the version. When installing from the Hub package index, version is a required property" assert msg in str(exc.exception) def test_validation_error_when_namespace_is_missing_from_package_config(self): packages_data = {"packages": [{"package": "dbt-labs", "version": "1.0.0"}]} with self.assertRaises(ValidationError) as exc: PackageConfig.validate(data=packages_data) msg = "dbt-labs was not found in the package index. Packages on the index require a namespace, e.g dbt-labs/dbt_utils" assert msg in str(exc.exception) class TestCheckForDuplicatePackagesWithBooleans(unittest.TestCase): """Unit test for check_for_duplicate_packages method with boolean values. This is a regression test for issue #9104 where the method would fail with "TypeError: argument of type 'bool' is not iterable" when package entries contained boolean fields like warn-unpinned: false. """ def test_check_duplicate_with_warn_unpinned_false(self): """Test that check_for_duplicate_packages handles warn-unpinned: false""" # Create a mock DepsTask with minimal setup mock_args = Namespace( add_package={"name": "audit_helper", "version": "0.9.0"}, source="hub" ) # Mock project - we don't need a real one for this test with mock.patch("dbt.task.deps.BaseTask.__init__"): task = DepsTask.__new__(DepsTask) task.args = mock_args # Test data: packages.yml with warn-unpinned: false packages_yml = { "packages": [ { "git": "https://github.com/dbt-labs/dbt-utils.git", "revision": "1.0.0", "warn-unpinned": False, # This is the problematic boolean }, ] } # This should not raise TypeError result = task.check_for_duplicate_packages(packages_yml) # Verify the result is returned (not None) self.assertIsNotNone(result) self.assertIn("packages", result) # Original package should still be there (no duplicate) self.assertEqual(len(result["packages"]), 1) def test_check_duplicate_with_warn_unpinned_true(self): """Test that check_for_duplicate_packages handles warn-unpinned: true""" mock_args = Namespace( add_package={"name": "audit_helper", "version": "0.9.0"}, source="hub" ) with mock.patch("dbt.task.deps.BaseTask.__init__"): task = DepsTask.__new__(DepsTask) task.args = mock_args packages_yml = { "packages": [ { "git": "https://github.com/dbt-labs/dbt-utils.git", "revision": "1.0.0", "warn-unpinned": True, # Another boolean value }, ] } # This should not raise TypeError result = task.check_for_duplicate_packages(packages_yml) self.assertIsNotNone(result) self.assertEqual(len(result["packages"]), 1) def test_check_duplicate_with_subdirectory_and_warn_unpinned(self): """Test with multiple non-string values (subdirectory string + warn-unpinned bool)""" mock_args = Namespace( add_package={"name": "audit_helper", "version": "0.9.0"}, source="hub" ) with mock.patch("dbt.task.deps.BaseTask.__init__"): task = DepsTask.__new__(DepsTask) task.args = mock_args packages_yml = { "packages": [ { "git": "https://github.com/dbt-labs/dbt-utils.git", "revision": "1.0.0", "subdirectory": "some_dir", "warn-unpinned": False, }, ] } # This should not raise TypeError result = task.check_for_duplicate_packages(packages_yml) self.assertIsNotNone(result) self.assertEqual(len(result["packages"]), 1) def test_check_duplicate_detects_git_match(self): """Test that duplicate detection still works for git packages""" mock_args = Namespace(add_package={"name": "dbt-utils", "version": "1.1.0"}, source="git") with mock.patch("dbt.task.deps.BaseTask.__init__"): task = DepsTask.__new__(DepsTask) task.args = mock_args packages_yml = { "packages": [ { "git": "https://github.com/dbt-labs/dbt-utils.git", "revision": "1.0.0", "warn-unpinned": False, }, ] } # The package name "dbt-utils" should match in the git URL with mock.patch("dbt_common.events.functions.fire_event"): result = task.check_for_duplicate_packages(packages_yml) # The duplicate should be removed self.assertIsNotNone(result) self.assertEqual(len(result["packages"]), 0) def test_check_duplicate_with_hub_package(self): """Test with hub package (which don't have warn-unpinned)""" mock_args = Namespace( add_package={"name": "another_package", "version": "1.0.0"}, source="hub" ) with mock.patch("dbt.task.deps.BaseTask.__init__"): task = DepsTask.__new__(DepsTask) task.args = mock_args packages_yml = { "packages": [ { "package": "dbt-labs/dbt_utils", "version": "1.0.0", }, ] } # This should work fine result = task.check_for_duplicate_packages(packages_yml) self.assertIsNotNone(result) self.assertEqual(len(result["packages"]), 1) def test_check_duplicate_with_mixed_package_types(self): """Test with mixed package types (hub + git with warn-unpinned)""" mock_args = Namespace( add_package={"name": "audit_helper", "version": "0.9.0"}, source="hub" ) with mock.patch("dbt.task.deps.BaseTask.__init__"): task = DepsTask.__new__(DepsTask) task.args = mock_args packages_yml = { "packages": [ { "package": "dbt-labs/dbt_utils", "version": "1.0.0", }, { "git": "https://github.com/example/some-package.git", "revision": "0.5.0", "warn-unpinned": False, }, ] } # This should not raise TypeError result = task.check_for_duplicate_packages(packages_yml) self.assertIsNotNone(result) # Both packages should remain (no duplicates) self.assertEqual(len(result["packages"]), 2) def test_check_duplicate_cross_source_hub_removes_git(self): """Test that adding a hub package removes a git package with the same name""" # When adding "dbt-labs/dbt_utils", it should match git URLs containing "dbt_utils" # Note: The full package name "dbt-labs/dbt_utils" is what gets checked mock_args = Namespace( add_package={"name": "dbt-labs/dbt_utils", "version": "1.0.0"}, source="hub" ) with mock.patch("dbt.task.deps.BaseTask.__init__"): task = DepsTask.__new__(DepsTask) task.args = mock_args packages_yml = { "packages": [ { "git": "https://github.com/dbt-labs/dbt_utils.git", "revision": "1.0.0", "warn-unpinned": False, }, ] } # The hub package name "dbt-labs/dbt_utils" should match in the git URL # because "dbt_utils" appears in both with mock.patch("dbt_common.events.functions.fire_event"): result = task.check_for_duplicate_packages(packages_yml) # The git package should be removed (cross-source duplicate) self.assertIsNotNone(result) self.assertEqual(len(result["packages"]), 0) def test_check_duplicate_multiple_matches(self): """Test that all duplicate packages are removed, not just the first one""" # When adding "dbt-labs/dbt_utils", it should match any identifier containing "dbt_utils" mock_args = Namespace( add_package={"name": "dbt-labs/dbt_utils", "version": "1.0.0"}, source="hub" ) with mock.patch("dbt.task.deps.BaseTask.__init__"): task = DepsTask.__new__(DepsTask) task.args = mock_args packages_yml = { "packages": [ { "git": "https://github.com/dbt-labs/dbt_utils.git", "revision": "0.9.0", }, { "git": "https://github.com/fivetran/dbt_amplitude", "revision": "1.0.0", }, { "package": "dbt-labs/dbt_utils", # Exact match "version": "0.8.0", }, ] } # Should remove both packages that contain "dbt_utils" with mock.patch("dbt_common.events.functions.fire_event"): result = task.check_for_duplicate_packages(packages_yml) # Only the dbt_amplitude package should remain self.assertIsNotNone(result) self.assertEqual(len(result["packages"]), 1) self.assertIn("dbt_amplitude", result["packages"][0]["git"]) def test_check_duplicate_underscore_hyphen_matching(self): """Test that underscore and hyphen variants match (dbt_utils matches dbt-utils)""" # Adding hub package with underscore should match git package with hyphen mock_args = Namespace( add_package={"name": "dbt-labs/dbt_utils", "version": "1.0.0"}, source="hub" ) with mock.patch("dbt.task.deps.BaseTask.__init__"): task = DepsTask.__new__(DepsTask) task.args = mock_args packages_yml = { "packages": [ { "git": "https://github.com/dbt-labs/dbt-utils.git", # hyphen in URL "revision": "1.0.0", }, ] } # Should match because "dbt-utils" variant matches the git URL with mock.patch("dbt_common.events.functions.fire_event"): result = task.check_for_duplicate_packages(packages_yml) self.assertIsNotNone(result) self.assertEqual(len(result["packages"]), 0) # Git package removed def test_check_duplicate_no_partial_word_match(self): """Test that partial word matches are rejected (dbt-core shouldn't match dbt-core-utils)""" mock_args = Namespace( add_package={"name": "dbt-labs/dbt-core", "version": "1.0.0"}, source="hub" ) with mock.patch("dbt.task.deps.BaseTask.__init__"): task = DepsTask.__new__(DepsTask) task.args = mock_args packages_yml = { "packages": [ { "git": "https://github.com/dbt-labs/dbt-core-utils.git", "revision": "1.0.0", }, { "package": "other-org/my-dbt-core-fork", "version": "2.0.0", }, ] } # Should NOT match because "dbt-core" is part of a larger word with mock.patch("dbt_common.events.functions.fire_event"): result = task.check_for_duplicate_packages(packages_yml) # Both packages should remain (no matches) self.assertIsNotNone(result) self.assertEqual(len(result["packages"]), 2) def test_check_duplicate_exact_word_boundary_match(self): """Test that exact matches with word boundaries work correctly""" mock_args = Namespace( add_package={"name": "dbt-labs/dbt-utils", "version": "1.0.0"}, source="hub" ) with mock.patch("dbt.task.deps.BaseTask.__init__"): task = DepsTask.__new__(DepsTask) task.args = mock_args packages_yml = { "packages": [ { "git": "https://github.com/dbt-labs/dbt-utils.git", # Should match "revision": "1.0.0", }, { "git": "https://github.com/other/dbt-utils-extra.git", # Should NOT match "revision": "2.0.0", }, { "package": "dbt-labs/dbt_utils", # Should match (underscore variant) "version": "0.9.0", }, ] } with mock.patch("dbt_common.events.functions.fire_event"): result = task.check_for_duplicate_packages(packages_yml) # Only dbt-utils-extra should remain self.assertIsNotNone(result) self.assertEqual(len(result["packages"]), 1) self.assertIn("dbt-utils-extra", result["packages"][0]["git"]) ================================================ FILE: tests/unit/event_time/test_event_time.py ================================================ from datetime import datetime import pytest import pytz from dbt.artifacts.resources.types import BatchSize from dbt.event_time.event_time import offset_timestamp class TestEventTime: @pytest.mark.parametrize( "timestamp,batch_size,offset,expected_timestamp", [ ( datetime(2024, 9, 5, 3, 56, 1, 1, pytz.UTC), BatchSize.year, 1, datetime(2025, 9, 5, 3, 56, 1, 1, pytz.UTC), ), ( datetime(2024, 9, 5, 3, 56, 1, 1, pytz.UTC), BatchSize.year, -1, datetime(2023, 9, 5, 3, 56, 1, 1, pytz.UTC), ), ( datetime(2024, 9, 5, 3, 56, 1, 1, pytz.UTC), BatchSize.month, 1, datetime(2024, 10, 5, 3, 56, 1, 1, pytz.UTC), ), ( datetime(2024, 9, 5, 3, 56, 1, 1, pytz.UTC), BatchSize.month, -1, datetime(2024, 8, 5, 3, 56, 1, 1, pytz.UTC), ), ( datetime(2024, 9, 5, 3, 56, 1, 1, pytz.UTC), BatchSize.day, 1, datetime(2024, 9, 6, 3, 56, 1, 1, pytz.UTC), ), ( datetime(2024, 9, 5, 3, 56, 1, 1, pytz.UTC), BatchSize.day, -1, datetime(2024, 9, 4, 3, 56, 1, 1, pytz.UTC), ), ( datetime(2024, 9, 5, 3, 56, 1, 1, pytz.UTC), BatchSize.hour, 1, datetime(2024, 9, 5, 4, 56, 1, 1, pytz.UTC), ), ( datetime(2024, 9, 5, 3, 56, 1, 1, pytz.UTC), BatchSize.hour, -1, datetime(2024, 9, 5, 2, 56, 1, 1, pytz.UTC), ), ( datetime(2024, 1, 31, 16, 6, 0, 0, pytz.UTC), BatchSize.month, 1, datetime(2024, 2, 29, 16, 6, 0, 0, pytz.UTC), ), ( datetime(2024, 2, 29, 16, 6, 0, 0, pytz.UTC), BatchSize.year, 1, datetime(2025, 2, 28, 16, 6, 0, 0, pytz.UTC), ), ], ) def test_offset_timestamp(self, timestamp, batch_size, offset, expected_timestamp): assert offset_timestamp(timestamp, batch_size, offset) == expected_timestamp ================================================ FILE: tests/unit/event_time/test_sample_mode.py ================================================ from datetime import datetime from typing import Union import freezegun import pytest import pytz from dbt.event_time.sample_window import SampleWindow from dbt_common.exceptions import DbtRuntimeError @pytest.mark.parametrize( "relative_string,expected_result", [ ( "4 years", SampleWindow( start=datetime(2021, 1, 28, 18, 4, 0, 0, pytz.UTC), end=datetime(2025, 1, 28, 18, 4, 0, 0, pytz.UTC), ), ), ( "1 year", SampleWindow( start=datetime(2024, 1, 28, 18, 4, 0, 0, pytz.UTC), end=datetime(2025, 1, 28, 18, 4, 0, 0, pytz.UTC), ), ), ( "4 YEARS", SampleWindow( start=datetime(2021, 1, 28, 18, 4, 0, 0, pytz.UTC), end=datetime(2025, 1, 28, 18, 4, 0, 0, pytz.UTC), ), ), ( "1 YEAR", SampleWindow( start=datetime(2024, 1, 28, 18, 4, 0, 0, pytz.UTC), end=datetime(2025, 1, 28, 18, 4, 0, 0, pytz.UTC), ), ), ( "4 months", SampleWindow( start=datetime(2024, 9, 28, 18, 4, 0, 0, pytz.UTC), end=datetime(2025, 1, 28, 18, 4, 0, 0, pytz.UTC), ), ), ( "1 month", SampleWindow( start=datetime(2024, 12, 28, 18, 4, 0, 0, pytz.UTC), end=datetime(2025, 1, 28, 18, 4, 0, 0, pytz.UTC), ), ), ( "4 MONTHS", SampleWindow( start=datetime(2024, 9, 28, 18, 4, 0, 0, pytz.UTC), end=datetime(2025, 1, 28, 18, 4, 0, 0, pytz.UTC), ), ), ( "1 MONTH", SampleWindow( start=datetime(2024, 12, 28, 18, 4, 0, 0, pytz.UTC), end=datetime(2025, 1, 28, 18, 4, 0, 0, pytz.UTC), ), ), ( "4 days", SampleWindow( start=datetime(2025, 1, 24, 18, 4, 0, 0, pytz.UTC), end=datetime(2025, 1, 28, 18, 4, 0, 0, pytz.UTC), ), ), ( "1 day", SampleWindow( start=datetime(2025, 1, 27, 18, 4, 0, 0, pytz.UTC), end=datetime(2025, 1, 28, 18, 4, 0, 0, pytz.UTC), ), ), ( "4 DAYS", SampleWindow( start=datetime(2025, 1, 24, 18, 4, 0, 0, pytz.UTC), end=datetime(2025, 1, 28, 18, 4, 0, 0, pytz.UTC), ), ), ( "1 DAY", SampleWindow( start=datetime(2025, 1, 27, 18, 4, 0, 0, pytz.UTC), end=datetime(2025, 1, 28, 18, 4, 0, 0, pytz.UTC), ), ), ( "4 hours", SampleWindow( start=datetime(2025, 1, 28, 14, 4, 0, 0, pytz.UTC), end=datetime(2025, 1, 28, 18, 4, 0, 0, pytz.UTC), ), ), ( "1 hour", SampleWindow( start=datetime(2025, 1, 28, 17, 4, 0, 0, pytz.UTC), end=datetime(2025, 1, 28, 18, 4, 0, 0, pytz.UTC), ), ), ( "4 HOURS", SampleWindow( start=datetime(2025, 1, 28, 14, 4, 0, 0, pytz.UTC), end=datetime(2025, 1, 28, 18, 4, 0, 0, pytz.UTC), ), ), ( "1 HOUR", SampleWindow( start=datetime(2025, 1, 28, 17, 4, 0, 0, pytz.UTC), end=datetime(2025, 1, 28, 18, 4, 0, 0, pytz.UTC), ), ), ( "1 week", DbtRuntimeError( "Invalid grain size 'week'. Must be one of ['hour', 'day', 'month', 'year', 'hours', 'days', 'months', 'years']." ), ), ("an hour", DbtRuntimeError("Unable to convert 'an' to an integer.")), ( "3", DbtRuntimeError( "Cannot load SAMPLE_WINDOW from '3'. Must be of form 'DAYS_INT GRAIN_SIZE'." ), ), ( "True", DbtRuntimeError( "Cannot load SAMPLE_WINDOW from 'True'. Must be of form 'DAYS_INT GRAIN_SIZE'." ), ), ("days 3", DbtRuntimeError("Unable to convert 'days' to an integer.")), ( "{}", DbtRuntimeError( "Cannot load SAMPLE_WINDOW from '{}'. Must be of form 'DAYS_INT GRAIN_SIZE'." ), ), ], ) @freezegun.freeze_time("2025-01-28T18:04:0Z") def test_from_relative_string( relative_string: str, expected_result: Union[SampleWindow, Exception] ): try: result = SampleWindow.from_relative_string(relative_string) assert result == expected_result except Exception as e: assert str(e) == str(expected_result) ================================================ FILE: tests/unit/events/__init__.py ================================================ ================================================ FILE: tests/unit/events/test_logging.py ================================================ from argparse import Namespace from pytest_mock import MockerFixture from dbt.events.logging import setup_event_logger from dbt.flags import get_flags, set_from_args from dbt_common.events.base_types import BaseEvent from dbt_common.events.event_catcher import EventCatcher from dbt_common.events.event_manager_client import get_event_manager from dbt_common.events.logger import LoggerConfig class TestSetupEventLogger: def test_clears_preexisting_event_manager_state(self) -> None: manager = get_event_manager() manager.add_logger(LoggerConfig(name="test_logger")) manager.callbacks.append(EventCatcher(BaseEvent).catch) assert len(manager.loggers) == 1 assert len(manager.callbacks) == 1 args = Namespace(log_level="none", log_level_file="none") set_from_args(args, {}) setup_event_logger(get_flags()) assert len(manager.loggers) == 0 assert len(manager.callbacks) == 1 # snowplow tracker for behavior flags def test_specify_max_bytes( self, mocker: MockerFixture, ) -> None: patched_file_handler = mocker.patch("dbt_common.events.logger.RotatingFileHandler") args = Namespace(log_file_max_bytes=1234567) set_from_args(args, {}) setup_event_logger(get_flags()) patched_file_handler.assert_called_once_with( filename="logs/dbt.log", encoding="utf8", maxBytes=1234567, backupCount=5 ) ================================================ FILE: tests/unit/events/test_types.py ================================================ from dbtlabs.proto.public.v1.fields import core_types_pb2 from google.protobuf.json_format import MessageToDict from dbt.adapters.events.types import PluginLoadError, RollbackFailed from dbt.events.types import ( LogNodeResult, LogStartLine, LogTestResult, MainEncounteredError, MainReportArgs, MainReportVersion, ) from dbt.version import installed from dbt_common.events import types_pb2 from dbt_common.events.base_types import EventLevel, msg_from_base_event from dbt_common.events.functions import ( LOG_VERSION, msg_to_dict, msg_to_json, reset_metadata_vars, ) info_keys = { "name", "code", "msg", "level", "invocation_id", "pid", "thread", "ts", "extra", "category", } def test_events(): # A001 event event = MainReportVersion(version=str(installed), log_version=LOG_VERSION) msg = msg_from_base_event(event) msg_dict = msg_to_dict(msg) msg_json = msg_to_json(msg) serialized = msg.SerializeToString() assert "Running with dbt=" in str(serialized) assert set(msg_dict.keys()) == {"info", "data"} assert set(msg_dict["data"].keys()) == {"version", "log_version"} assert set(msg_dict["info"].keys()) == info_keys assert msg_json assert msg.info.code == "A001" # Extract EventInfo from serialized message generic_msg = types_pb2.GenericMessage() generic_msg.ParseFromString(serialized) assert generic_msg.info.code == "A001" # get the message class for the real message from the generic message message_class = getattr(core_types_pb2, f"{generic_msg.info.name}Msg") new_msg = message_class() new_msg.ParseFromString(serialized) assert new_msg.info.code == msg.info.code assert new_msg.data.version == msg.data.version # A002 event event = MainReportArgs(args={"one": "1", "two": "2"}) msg = msg_from_base_event(event) msg_dict = msg_to_dict(msg) msg_json = msg_to_json(msg) assert set(msg_dict.keys()) == {"info", "data"} assert set(msg_dict["data"].keys()) == {"args"} assert set(msg_dict["info"].keys()) == info_keys assert msg_json assert msg.info.code == "A002" def test_exception_events(): event = RollbackFailed(conn_name="test", exc_info="something failed") msg = msg_from_base_event(event) msg_dict = msg_to_dict(msg) msg_json = msg_to_json(msg) assert set(msg_dict.keys()) == {"info", "data"} assert set(msg_dict["data"].keys()) == {"conn_name", "exc_info"} assert set(msg_dict["info"].keys()) == info_keys assert msg_json assert msg.info.code == "E009" event = PluginLoadError(exc_info="something failed") msg = msg_from_base_event(event) msg_dict = msg_to_dict(msg) msg_json = msg_to_json(msg) assert set(msg_dict["data"].keys()) == {"exc_info"} assert set(msg_dict["info"].keys()) == info_keys assert msg_json assert msg.info.code == "E036" assert msg.info.msg == "something failed" # Z002 event event = MainEncounteredError(exc="Rollback failed") msg = msg_from_base_event(event) msg_dict = msg_to_dict(msg) msg_json = msg_to_json(msg) assert set(msg_dict["data"].keys()) == {"exc"} assert set(msg_dict["info"].keys()) == info_keys assert msg_json assert msg.info.code == "Z002" def test_node_info_events(): meta_dict = { "key1": ["value1", 2], "key2": {"nested-dict-key": "value2"}, "key3": 1, "key4": ["string1", 1, "string2", 2], } node_info = { "node_path": "some_path", "node_name": "some_name", "unique_id": "some_id", "resource_type": "model", "materialized": "table", "node_status": "started", "node_started_at": "some_time", "node_finished_at": "another_time", "meta": meta_dict, "node_relation": { "database": "some_database", "schema": "some_schema", "alias": "some_alias", "relation_name": "some.relation.name", }, "node_checksum": "some_checksum", } event = LogStartLine( description="some description", index=123, total=111, node_info=node_info, ) assert event assert event.node_info.node_path == "some_path" assert event.to_dict()["node_info"]["meta"] == meta_dict node_info["node_relation"]["database"] = None node_info["node_relation"]["relation_name"] = "some_schema.some_alias" event = LogStartLine( description="some description", index=123, total=111, node_info=node_info, ) assert event.node_info.node_relation.database == "" assert event.node_info.node_relation.relation_name == "some_schema.some_alias" assert event.node_info.node_checksum == "some_checksum" def test_extra_dict_on_event(monkeypatch): monkeypatch.setenv("DBT_ENV_CUSTOM_ENV_env_key", "env_value") reset_metadata_vars() event = MainReportVersion(version=str(installed), log_version=LOG_VERSION) msg = msg_from_base_event(event) msg_dict = msg_to_dict(msg) assert set(msg_dict["info"].keys()) == info_keys extra_dict = {"env_key": "env_value"} assert msg.info.extra == extra_dict serialized = msg.SerializeToString() # Extract EventInfo from serialized message generic_msg = types_pb2.GenericMessage() generic_msg.ParseFromString(serialized) assert generic_msg.info.code == "A001" # get the message class for the real message from the generic message message_class = getattr(core_types_pb2, f"{generic_msg.info.name}Msg") new_msg = message_class() new_msg.ParseFromString(serialized) new_msg_dict = MessageToDict(new_msg) assert new_msg_dict["info"]["extra"] == msg.info.extra # clean up reset_metadata_vars() def test_dynamic_level_events(): event = LogTestResult(name="model_name", status="pass", index=1, num_models=3, num_failures=0) msg = msg_from_base_event(event, level=EventLevel.INFO) assert msg assert msg.info.level == "info" def test_log_node_result(): event = LogNodeResult( node_info={}, status="error", index=1, total=0, msg="some message", ) msg = msg_from_base_event(event, level=EventLevel.ERROR) assert msg assert msg.info.msg == "some message" assert msg.info.level == "error" ================================================ FILE: tests/unit/fixtures.py ================================================ from dbt.artifacts.resources import Contract, TestConfig, TestMetadata from dbt.contracts.files import FileHash from dbt.contracts.graph.nodes import ( DependsOn, GenericTestNode, InjectedCTE, ModelConfig, ModelNode, ) from dbt.node_types import NodeType def model_node(): return ModelNode( package_name="test", path="/root/models/foo.sql", original_file_path="models/foo.sql", language="sql", raw_code='select * from {{ ref("other") }}', name="foo", resource_type=NodeType.Model, unique_id="model.test.foo", fqn=["test", "models", "foo"], refs=[], sources=[], metrics=[], depends_on=DependsOn(), description="", primary_key=[], database="test_db", schema="test_schema", alias="bar", tags=[], config=ModelConfig(), contract=Contract(), meta={}, compiled=True, extra_ctes=[InjectedCTE("whatever", "select * from other")], extra_ctes_injected=True, compiled_code="with whatever as (select * from other) select * from whatever", checksum=FileHash.from_contents(""), unrendered_config={}, ) def generic_test_node(): return GenericTestNode( package_name="test", path="/root/x/path.sql", original_file_path="/root/path.sql", language="sql", raw_code='select * from {{ ref("other") }}', name="foo", resource_type=NodeType.Test, unique_id="model.test.foo", fqn=["test", "models", "foo"], refs=[], sources=[], metrics=[], depends_on=DependsOn(), description="", database="test_db", schema="dbt_test__audit", alias="bar", tags=[], config=TestConfig(severity="warn"), contract=Contract(), meta={}, compiled=True, extra_ctes=[InjectedCTE("whatever", "select * from other")], extra_ctes_injected=True, compiled_code="with whatever as (select * from other) select * from whatever", column_name="id", test_metadata=TestMetadata(namespace=None, name="foo", kwargs={}), checksum=FileHash.from_contents(""), unrendered_config={ "severity": "warn", }, ) ================================================ FILE: tests/unit/graph/__init__.py ================================================ ================================================ FILE: tests/unit/graph/test_cli.py ================================================ import textwrap import yaml from dbt.contracts.selection import SelectorFile from dbt.graph import ( SelectionCriteria, SelectionDifference, SelectionIntersection, SelectionUnion, cli, ) from dbt.graph.selector_methods import MethodName def parse_file(txt: str) -> SelectorFile: txt = textwrap.dedent(txt) dct = yaml.safe_load(txt) sf = SelectorFile.from_dict(dct) return sf class Union: def __init__(self, *args): self.components = args def __str__(self): return f"Union(components={self.components})" def __repr__(self): return f"Union(components={self.components!r})" def __eq__(self, other): if not isinstance(other, SelectionUnion): return False return all(mine == theirs for mine, theirs in zip(self.components, other.components)) class Intersection: def __init__(self, *args): self.components = args def __str__(self): return f"Intersection(components={self.components})" def __repr__(self): return f"Intersection(components={self.components!r})" def __eq__(self, other): if not isinstance(other, SelectionIntersection): return False return all(mine == theirs for mine, theirs in zip(self.components, other.components)) class Difference: def __init__(self, *args): self.components = args def __str__(self): return f"Difference(components={self.components})" def __repr__(self): return f"Difference(components={self.components!r})" def __eq__(self, other): if not isinstance(other, SelectionDifference): return False return all(mine == theirs for mine, theirs in zip(self.components, other.components)) class Criteria: def __init__(self, method, value, **kwargs): self.method = method self.value = value self.kwargs = kwargs def __str__(self): return f"Criteria(method={self.method}, value={self.value}, **{self.kwargs})" def __repr__(self): return f"Criteria(method={self.method!r}, value={self.value!r}, **{self.kwargs!r})" def __eq__(self, other): if not isinstance(other, SelectionCriteria): return False return ( self.method == other.method and self.value == other.value and all(getattr(other, k) == v for k, v in self.kwargs.items()) ) def test_parse_simple(): sf = parse_file( """\ selectors: - name: tagged_foo description: Selector for foo-tagged models definition: tag: foo """ ) assert len(sf.selectors) == 1 assert sf.selectors[0].description == "Selector for foo-tagged models" parsed = cli.parse_from_selectors_definition(sf) assert len(parsed) == 1 assert "tagged_foo" in parsed assert ( Criteria( method=MethodName.Tag, method_arguments=[], value="foo", children=False, parents=False, childrens_parents=False, children_depth=None, parents_depth=None, ) == parsed["tagged_foo"]["definition"] ) def test_parse_simple_childrens_parents(): sf = parse_file( """\ selectors: - name: tagged_foo definition: method: tag value: foo childrens_parents: True """ ) assert len(sf.selectors) == 1 parsed = cli.parse_from_selectors_definition(sf) assert len(parsed) == 1 assert "tagged_foo" in parsed assert ( Criteria( method=MethodName.Tag, method_arguments=[], value="foo", children=False, parents=False, childrens_parents=True, children_depth=None, parents_depth=None, ) == parsed["tagged_foo"]["definition"] ) def test_parse_simple_arguments_with_modifiers(): sf = parse_file( """\ selectors: - name: configured_view definition: method: config.materialized value: view parents: True children: True children_depth: 2 """ ) assert len(sf.selectors) == 1 parsed = cli.parse_from_selectors_definition(sf) assert len(parsed) == 1 assert "configured_view" in parsed assert ( Criteria( method=MethodName.Config, method_arguments=["materialized"], value="view", children=True, parents=True, childrens_parents=False, children_depth=2, parents_depth=None, ) == parsed["configured_view"]["definition"] ) def test_parse_union(): sf = parse_file( """\ selectors: - name: views-or-foos definition: union: - method: config.materialized value: view - tag: foo """ ) assert len(sf.selectors) == 1 parsed = cli.parse_from_selectors_definition(sf) assert "views-or-foos" in parsed assert ( Union( Criteria(method=MethodName.Config, value="view", method_arguments=["materialized"]), Criteria(method=MethodName.Tag, value="foo", method_arguments=[]), ) == parsed["views-or-foos"]["definition"] ) def test_parse_intersection(): sf = parse_file( """\ selectors: - name: views-and-foos definition: intersection: - method: config.materialized value: view - tag: foo """ ) assert len(sf.selectors) == 1 parsed = cli.parse_from_selectors_definition(sf) assert "views-and-foos" in parsed assert ( Intersection( Criteria(method=MethodName.Config, value="view", method_arguments=["materialized"]), Criteria(method=MethodName.Tag, value="foo", method_arguments=[]), ) == parsed["views-and-foos"]["definition"] ) def test_parse_union_excluding(): sf = parse_file( """\ selectors: - name: views-or-foos-not-bars definition: union: - method: config.materialized value: view - tag: foo - exclude: - tag: bar """ ) assert len(sf.selectors) == 1 parsed = cli.parse_from_selectors_definition(sf) assert "views-or-foos-not-bars" in parsed assert ( Difference( Union( Criteria( method=MethodName.Config, value="view", method_arguments=["materialized"] ), Criteria(method=MethodName.Tag, value="foo", method_arguments=[]), ), Criteria(method=MethodName.Tag, value="bar", method_arguments=[]), ) == parsed["views-or-foos-not-bars"]["definition"] ) def test_parse_yaml_complex(): sf = parse_file( """\ selectors: - name: test_name definition: union: - intersection: - tag: foo - tag: bar - union: - package: snowplow - config.materialized: incremental - union: - path: "models/snowplow/marketing/custom_events.sql" - fqn: "snowplow.marketing" - intersection: - resource_type: seed - package: snowplow - exclude: - country_codes - intersection: - tag: baz - config.materialized: ephemeral - name: weeknights definition: union: - tag: nightly - tag:weeknights_only """ ) assert len(sf.selectors) == 2 parsed = cli.parse_from_selectors_definition(sf) assert "test_name" in parsed assert "weeknights" in parsed assert ( Union( Criteria(method=MethodName.Tag, value="nightly"), Criteria(method=MethodName.Tag, value="weeknights_only"), ) == parsed["weeknights"]["definition"] ) assert ( Union( Intersection( Criteria(method=MethodName.Tag, value="foo"), Criteria(method=MethodName.Tag, value="bar"), Union( Criteria(method=MethodName.Package, value="snowplow"), Criteria( method=MethodName.Config, value="incremental", method_arguments=["materialized"], ), ), ), Union( Criteria( method=MethodName.Path, value="models/snowplow/marketing/custom_events.sql" ), Criteria(method=MethodName.FQN, value="snowplow.marketing"), ), Difference( Intersection( Criteria(method=MethodName.ResourceType, value="seed"), Criteria(method=MethodName.Package, value="snowplow"), ), Union( Criteria(method=MethodName.FQN, value="country_codes"), Intersection( Criteria(method=MethodName.Tag, value="baz"), Criteria( method=MethodName.Config, value="ephemeral", method_arguments=["materialized"], ), ), ), ), ) == parsed["test_name"]["definition"] ) def test_parse_selection(): sf = parse_file( """\ selectors: - name: default definition: union: - tag: foo - tag: bar - name: inherited definition: method: selector value: default """ ) assert len(sf.selectors) == 2 parsed = cli.parse_from_selectors_definition(sf) assert "default" in parsed assert "inherited" in parsed assert ( Union( Criteria(method=MethodName.Tag, value="foo"), Criteria(method=MethodName.Tag, value="bar"), ) == parsed["default"]["definition"] ) assert ( Criteria(method=MethodName.Selector, value="default") == parsed["inherited"]["definition"] ) def test_parse_selection_with_exclusion(): sf = parse_file( """\ selectors: - name: default definition: union: - tag: foo - tag: bar - name: inherited definition: union: - method: selector value: default - exclude: - tag: bar """ ) assert len(sf.selectors) == 2 parsed = cli.parse_from_selectors_definition(sf) assert "default" in parsed assert "inherited" in parsed assert ( Union( Criteria(method=MethodName.Tag, value="foo"), Criteria(method=MethodName.Tag, value="bar"), ) == parsed["default"]["definition"] ) assert ( Difference( Union( Criteria(method=MethodName.Selector, value="default"), ), Criteria(method=MethodName.Tag, value="bar"), ) == parsed["inherited"]["definition"] ) ================================================ FILE: tests/unit/graph/test_graph.py ================================================ import pytest from dbt.compilation import Linker from dbt.contracts.graph.manifest import Manifest from dbt.contracts.graph.nodes import ModelNode from dbt.graph.graph import Graph from tests.unit.utils.manifest import make_model class TestGraph: @pytest.fixture def extra_parent_model(self) -> ModelNode: return make_model(pkg="pkg", name="extra_parent_model", code="SELECT 'cats' as interests") @pytest.fixture def non_shared_child_of_extra(self, extra_parent_model: ModelNode) -> ModelNode: return make_model( pkg="pkg", name="non_shared_child_of_extra", code='SELECT * FROM {{ ref("extra_parent_model") }}', refs=[extra_parent_model], ) @pytest.fixture def model_with_two_direct_parents( self, extra_parent_model: ModelNode, ephemeral_model: ModelNode ) -> ModelNode: return make_model( pkg="pkg", name="model_with_two_direct_parents", code='SELECT * FROM {{ ref("ephemeral_model") }} UNION ALL SELECT * FROM {{ ref("extra_parent_model") }}', refs=[extra_parent_model, ephemeral_model], ) @pytest.fixture(autouse=True) def local_manifest_extensions( self, manifest: Manifest, model_with_two_direct_parents: ModelNode, non_shared_child_of_extra: ModelNode, extra_parent_model: ModelNode, ) -> Manifest: manifest.add_node_nofile(extra_parent_model) manifest.add_node_nofile(non_shared_child_of_extra) manifest.add_node_nofile(model_with_two_direct_parents) @pytest.fixture def graph(self, manifest: Manifest, local_manifest_extensions) -> Graph: # We include the `local_manifest_extensions` in the arguments to ensure # that fixture adds our extra node sbefore creating the graph linker = Linker() linker.link_graph(manifest=manifest) return Graph(graph=linker.graph) def test_nodes(self, graph: Graph, manifest: Manifest): graph_nodes = graph.nodes() all_manifest_nodes = [] for resources in manifest.get_resource_fqns().values(): all_manifest_nodes.extend(list(resources)) # Assert that it is a set, thus no duplicates assert isinstance(graph_nodes, set) assert len(graph_nodes) == len(all_manifest_nodes) def test_descendantcs(self, graph: Graph, manifest: Manifest) -> None: model: ModelNode = manifest.nodes["model.pkg.ephemeral_model"] # check result when not limiting the depth descendants = graph.descendants(node=model.unique_id) assert descendants == { "model.pkg.model_with_two_direct_parents", "test.pkg.view_test_nothing", "model.pkg.view_model", "model.pkg.table_model", } # check that result excludes nodes that are out of depth descendants = graph.descendants(node=model.unique_id, max_depth=1) assert descendants == { "model.pkg.model_with_two_direct_parents", "model.pkg.table_model", "model.pkg.view_model", } def test_ancestors(self, graph: Graph, manifest: Manifest) -> None: model: ModelNode = manifest.nodes["model.pkg.table_model"] # check result when not limiting the depth ancestors = graph.ancestors(node=model.unique_id) assert ancestors == { "model.pkg.ephemeral_model", "source.pkg.raw.seed", } # check that result excludes nodes that are out of depth ancestors = graph.ancestors(node=model.unique_id, max_depth=1) assert ancestors == {"model.pkg.ephemeral_model"} @pytest.mark.skip(reason="I haven't figured out how to add edge types to nodes") def test_exclude_edge_type(self) -> None: # I though something like the following would produce # linker = Linker() # linker.link_graph(manifest=manifest) # linker.add_test_edges(manifest=manifest) # graph = Graph(graph=linker.graph) pass def test_select_childrens_parents( self, graph: Graph, model_with_two_direct_parents: ModelNode, extra_parent_model: ModelNode, ephemeral_model: ModelNode, ) -> None: # `select_childrens_parents` should return # * all children of the selected node (without depth limit) # * all parents of the children of the selected node (without depth limit) childrens_parents = graph.select_childrens_parents( selected={ extra_parent_model.unique_id, } ) assert model_with_two_direct_parents.unique_id in childrens_parents assert extra_parent_model.unique_id in childrens_parents assert ephemeral_model.unique_id in childrens_parents assert len(childrens_parents) == 5 def test_select_children( self, graph: Graph, ephemeral_model: ModelNode, extra_parent_model: ModelNode, ) -> None: ephemerals_children = graph.select_children(selected={ephemeral_model.unique_id}) extras_children = graph.select_children(selected={extra_parent_model.unique_id}) joint_children = graph.select_children( selected={extra_parent_model.unique_id, ephemeral_model.unique_id} ) assert joint_children == ephemerals_children.union(extras_children) # These additional assertions are because we intentionally setup the test such that # neither nodes children set is a subset of the other assert not ephemerals_children.issubset(extras_children) assert not extras_children.issubset(ephemerals_children) def test_select_parents( self, graph: Graph, non_shared_child_of_extra: ModelNode, table_model: ModelNode, ) -> None: non_shareds_parents = graph.select_parents(selected={non_shared_child_of_extra.unique_id}) tables_parents = graph.select_parents(selected={table_model.unique_id}) joint_parents = graph.select_parents( selected={table_model.unique_id, non_shared_child_of_extra.unique_id} ) assert joint_parents == tables_parents.union(non_shareds_parents) # These additional assertions are because we intentionally setup the test such that # neither nodes parents set is a subset of the other assert not non_shareds_parents.issubset(tables_parents) assert not tables_parents.issubset(non_shareds_parents) ================================================ FILE: tests/unit/graph/test_nodes.py ================================================ from copy import deepcopy from datetime import datetime from typing import List import pytest from freezegun import freeze_time from dbt.artifacts.resources import ( Defaults, Dimension, Entity, FileHash, Measure, Metric, MetricAggregationParams, MetricTypeParams, TestMetadata, ) from dbt.artifacts.resources.v1.semantic_model import NodeRelation from dbt.contracts.graph.model_config import TestConfig from dbt.contracts.graph.nodes import ColumnInfo, ModelNode, ParsedNode, SemanticModel from dbt.node_types import NodeType from dbt_common.contracts.constraints import ( ColumnLevelConstraint, ConstraintType, ModelLevelConstraint, ) from dbt_semantic_interfaces.references import MeasureReference, TimeDimensionReference from dbt_semantic_interfaces.type_enums import ( AggregationType, DimensionType, EntityType, MetricType, ) from tests.unit.fixtures import generic_test_node, model_node class TestModelNode: @pytest.fixture(scope="class") def default_model_node(self): return ModelNode( resource_type=NodeType.Model, unique_id="model.test_package.test_name", name="test_name", package_name="test_package", schema="test_schema", alias="test_alias", fqn=["models", "test_name"], original_file_path="test_original_file_path", checksum=FileHash.from_contents("checksum"), path="test_path", database=None, ) @pytest.mark.parametrize( "deprecation_date,current_date,expected_is_past_deprecation_date", [ (None, "2024-05-02", False), ("2024-05-01", "2024-05-02", True), ("2024-05-01", "2024-05-01", False), ("2024-05-01", "2024-04-30", False), ], ) def test_is_past_deprecation_date( self, default_model_node, deprecation_date, current_date, expected_is_past_deprecation_date ): with freeze_time(current_date): if deprecation_date is not None: default_model_node.deprecation_date = datetime.strptime( deprecation_date, "%Y-%m-%d" ).astimezone() assert default_model_node.is_past_deprecation_date is expected_is_past_deprecation_date @pytest.mark.parametrize( "model_constraints,columns,expected_all_constraints", [ ([], {}, []), ( [ModelLevelConstraint(type=ConstraintType.foreign_key)], {}, [ModelLevelConstraint(type=ConstraintType.foreign_key)], ), ( [], { "id": ColumnInfo( name="id", constraints=[ColumnLevelConstraint(type=ConstraintType.foreign_key)], ) }, [ColumnLevelConstraint(type=ConstraintType.foreign_key)], ), ( [ModelLevelConstraint(type=ConstraintType.foreign_key)], { "id": ColumnInfo( name="id", constraints=[ColumnLevelConstraint(type=ConstraintType.foreign_key)], ) }, [ ModelLevelConstraint(type=ConstraintType.foreign_key), ColumnLevelConstraint(type=ConstraintType.foreign_key), ], ), ], ) def test_all_constraints( self, default_model_node, model_constraints, columns, expected_all_constraints ): default_model_node.constraints = model_constraints default_model_node.columns = columns assert default_model_node.all_constraints == expected_all_constraints class TestSemanticModel: @pytest.fixture(scope="function") def dimensions(self) -> List[Dimension]: return [Dimension(name="ds", type=DimensionType)] @pytest.fixture(scope="function") def entities(self) -> List[Entity]: return [Entity(name="test_entity", type=EntityType.PRIMARY, expr="id")] @pytest.fixture(scope="function") def measures(self) -> List[Measure]: return [Measure(name="test_measure", agg=AggregationType.COUNT, expr="id")] @pytest.fixture(scope="function") def default_semantic_model( self, dimensions: List[Dimension], entities: List[Entity], measures: List[Measure] ) -> SemanticModel: return SemanticModel( name="test_semantic_model", resource_type=NodeType.SemanticModel, model="ref('test_model')", package_name="test", path="test_path", original_file_path="test_fixture", unique_id=f"{NodeType.SemanticModel}.test.test_semantic_model", fqn=[], defaults=Defaults(agg_time_dimension="ds"), dimensions=dimensions, entities=entities, measures=measures, node_relation=NodeRelation( alias="test_alias", schema_name="test_schema", database="test_database" ), ) @pytest.fixture(scope="function") def default_simple_metric(self) -> Metric: return Metric( name="test_metric", resource_type=NodeType.Metric, package_name="test", path="test_path", original_file_path="test_fixture", unique_id=f"{NodeType.Metric}.test.test_metric", fqn=[], description="", label="", type=MetricType.SIMPLE, type_params=MetricTypeParams( metric_aggregation_params=MetricAggregationParams( semantic_model="test_semantic_model", agg=AggregationType.COUNT, ) ), ) def test_checked_agg_time_dimension_for_measure_via_defaults( self, default_semantic_model: SemanticModel, ): assert default_semantic_model.defaults.agg_time_dimension is not None measure = default_semantic_model.measures[0] measure.agg_time_dimension = None default_semantic_model.checked_agg_time_dimension_for_measure( MeasureReference(element_name=measure.name) ) def test_checked_agg_time_dimension_for_measure_via_measure( self, default_semantic_model: SemanticModel ): default_semantic_model.defaults = None measure = default_semantic_model.measures[0] measure.agg_time_dimension = default_semantic_model.dimensions[0].name default_semantic_model.checked_agg_time_dimension_for_measure( MeasureReference(element_name=measure.name) ) def test_checked_agg_time_dimension_for_measure_exception( self, default_semantic_model: SemanticModel ): default_semantic_model.defaults = None measure = default_semantic_model.measures[0] measure.agg_time_dimension = None with pytest.raises(AssertionError) as execinfo: default_semantic_model.checked_agg_time_dimension_for_measure( MeasureReference(measure.name) ) assert ( f"Aggregation time dimension for measure {measure.name} on semantic model {default_semantic_model.name}" in str(execinfo.value) ) def test_checked_agg_time_dimension_for_simple_metric( self, default_semantic_model: SemanticModel, default_simple_metric: Metric ): time_dimension_reference = ( default_semantic_model.checked_agg_time_dimension_for_simple_metric( default_simple_metric ) ) assert time_dimension_reference == TimeDimensionReference(element_name="ds") def test_checked_agg_time_dimension_for_simple_metric_missing_metric_aggregation_params( self, default_semantic_model: SemanticModel, default_simple_metric: Metric ): default_simple_metric.type_params.metric_aggregation_params = None with pytest.raises(AssertionError) as execinfo: default_semantic_model.checked_agg_time_dimension_for_simple_metric( default_simple_metric ) assert "Simple metrics must have metric_aggregation_params." in str(execinfo.value) def test_check_agg_time_dimension_for_simple_metric_missing_agg_time_dimension( self, default_semantic_model: SemanticModel, default_simple_metric: Metric ): default_semantic_model.defaults.agg_time_dimension = None with pytest.raises(AssertionError) as execinfo: default_semantic_model.checked_agg_time_dimension_for_simple_metric( default_simple_metric ) assert ( f"Aggregation time dimension for metric {default_simple_metric.name} is not set!" in str(execinfo.value) ) def test_semantic_model_same_contents(self, default_semantic_model: SemanticModel): default_semantic_model_copy = deepcopy(default_semantic_model) assert default_semantic_model.same_contents(default_semantic_model_copy) def test_semantic_model_same_contents_update_model( self, default_semantic_model: SemanticModel ): default_semantic_model_copy = deepcopy(default_semantic_model) default_semantic_model_copy.model = "ref('test_another_model')" assert not default_semantic_model.same_contents(default_semantic_model_copy) def test_semantic_model_same_contents_different_node_relation( self, default_semantic_model: SemanticModel, ): default_semantic_model_copy = deepcopy(default_semantic_model) default_semantic_model_copy.node_relation.alias = "test_another_alias" # Relation should not be consided in same_contents assert default_semantic_model.same_contents(default_semantic_model_copy) # Infer primary key def test_no_primary_key(): model = model_node() assert model.infer_primary_key([]) == [] def test_primary_key_model_constraint(): model = model_node() model.constraints = [ModelLevelConstraint(type=ConstraintType.primary_key, columns=["pk"])] assertSameContents(model.infer_primary_key([]), ["pk"]) model.constraints = [ ModelLevelConstraint(type=ConstraintType.primary_key, columns=["pk1", "pk2"]) ] assertSameContents(model.infer_primary_key([]), ["pk1", "pk2"]) def test_primary_key_column_constraint(): model = model_node() model.columns = { "column1": ColumnInfo( "column1", constraints=[ColumnLevelConstraint(type=ConstraintType.primary_key)] ), "column2": ColumnInfo("column2"), } assertSameContents(model.infer_primary_key([]), ["column1"]) def test_unique_non_null_single(): model = model_node() test1 = generic_test_node() test1.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column1"}) test2 = generic_test_node() test2.test_metadata = TestMetadata(name="not_null", kwargs={"column_name": "column1"}) test3 = generic_test_node() test3.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column2"}) tests = [test1, test2] assertSameContents(model.infer_primary_key(tests), ["column1"]) def test_unique_non_null_multiple(): model = model_node() tests = [] for i in range(2): for enabled in [True, False]: test1 = generic_test_node() test1.test_metadata = TestMetadata( name="unique", kwargs={"column_name": "column" + str(i) + str(enabled)} ) test1.config = TestConfig(enabled=enabled) test2 = generic_test_node() test2.test_metadata = TestMetadata( name="not_null", kwargs={"column_name": "column" + str(i) + str(enabled)} ) test2.config = TestConfig(enabled=enabled) tests.extend([test1, test2]) assertSameContents( model.infer_primary_key(tests), ["column0True", "column1True", "column0False", "column1False"], ) def test_enabled_unique_single(): model = model_node() test1 = generic_test_node() test1.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column1"}) test2 = generic_test_node() test2.config = TestConfig(enabled=False) test2.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column3"}) tests = [test1, test2] assertSameContents(model.infer_primary_key(tests), ["column1"]) def test_enabled_unique_multiple(): model = model_node() test1 = generic_test_node() test1.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column1"}) test2 = generic_test_node() test2.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column2 || column3"}) tests = [test1, test2] assertSameContents(model.infer_primary_key(tests), ["column1", "column2 || column3"]) def test_enabled_unique_combo_single(): model = model_node() test1 = generic_test_node() test1.test_metadata = TestMetadata( name="unique_combination_of_columns", kwargs={"combination_of_columns": ["column1", "column2"]}, ) test2 = generic_test_node() test2.config = TestConfig(enabled=False) test2.test_metadata = TestMetadata( name="unique_combination_of_columns", kwargs={"combination_of_columns": ["column3", "column4"]}, ) tests = [test1, test2] assertSameContents(model.infer_primary_key(tests), ["column1", "column2"]) def test_enabled_unique_combo_multiple(): model = model_node() test1 = generic_test_node() test1.test_metadata = TestMetadata( name="unique", kwargs={"combination_of_columns": ["column1", "column2"]} ) test2 = generic_test_node() test2.test_metadata = TestMetadata( name="unique", kwargs={"combination_of_columns": ["column3", "column4"]} ) tests = [test1, test2] assertSameContents( model.infer_primary_key(tests), ["column1", "column2", "column3", "column4"] ) def test_disabled_unique_single(): model = model_node() test1 = generic_test_node() test1.config = TestConfig(enabled=False) test1.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column1"}) test2 = generic_test_node() test2.test_metadata = TestMetadata(name="not_null", kwargs={"column_name": "column2"}) tests = [test1, test2] assertSameContents(model.infer_primary_key(tests), ["column1"]) def test_disabled_unique_multiple(): model = model_node() test1 = generic_test_node() test1.config = TestConfig(enabled=False) test1.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column1"}) test2 = generic_test_node() test2.config = TestConfig(enabled=False) test2.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column2 || column3"}) tests = [test1, test2] assertSameContents(model.infer_primary_key(tests), ["column1", "column2 || column3"]) def test_disabled_unique_combo_single(): model = model_node() test1 = generic_test_node() test1.config = TestConfig(enabled=False) test1.test_metadata = TestMetadata( name="unique", kwargs={"combination_of_columns": ["column1", "column2"]} ) test2 = generic_test_node() test2.config = TestConfig(enabled=False) test2.test_metadata = TestMetadata( name="random", kwargs={"combination_of_columns": ["column3", "column4"]} ) tests = [test1, test2] assertSameContents(model.infer_primary_key(tests), ["column1", "column2"]) def test_disabled_unique_combo_multiple(): model = model_node() test1 = generic_test_node() test1.config = TestConfig(enabled=False) test1.test_metadata = TestMetadata( name="unique", kwargs={"combination_of_columns": ["column1", "column2"]} ) test2 = generic_test_node() test2.config = TestConfig(enabled=False) test2.test_metadata = TestMetadata( name="unique", kwargs={"combination_of_columns": ["column3", "column4"]} ) tests = [test1, test2] assertSameContents( model.infer_primary_key(tests), ["column1", "column2", "column3", "column4"] ) def assertSameContents(list1, list2): assert sorted(list1) == sorted(list2) class TestParsedNode: @pytest.fixture(scope="class") def parsed_node(self) -> ParsedNode: return ParsedNode( resource_type=NodeType.Model, unique_id="model.test_package.test_name", name="test_name", package_name="test_package", schema="test_schema", alias="test_alias", fqn=["models", "test_name"], original_file_path="test_original_file_path", checksum=FileHash.from_contents("checksum"), path="test_path.sql", database=None, ) def test_get_target_write_path(self, parsed_node): write_path = parsed_node.get_target_write_path("target_path", "subdirectory") assert ( write_path == "target_path/subdirectory/test_package/test_original_file_path/test_path.sql" ) def test_get_target_write_path_split(self, parsed_node): write_path = parsed_node.get_target_write_path("target_path", "subdirectory", "split") assert ( write_path == "target_path/subdirectory/test_package/test_original_file_path/test_path/test_path_split.sql" ) ================================================ FILE: tests/unit/graph/test_queue.py ================================================ import networkx as nx import pytest from dbt.contracts.graph.manifest import Manifest from dbt.graph.queue import GraphQueue from tests.unit.utils import MockNode, make_manifest class TestGraphQueue: @pytest.fixture(scope="class") def manifest(self) -> Manifest: return make_manifest( nodes=[ MockNode(package="test_package", name="upstream_model"), MockNode(package="test_package", name="downstream_model"), ] ) @pytest.fixture(scope="class") def graph(self) -> nx.DiGraph: graph = nx.DiGraph() graph.add_edge("model.test_package.upstream_model", "model.test_package.downstream_model") return graph def test_init_graph_queue(self, manifest, graph): graph_queue = GraphQueue(graph=graph, manifest=manifest, selected={}) assert graph_queue.manifest == manifest assert graph_queue.graph == graph assert graph_queue.inner.queue == [(0, "model.test_package.upstream_model")] assert graph_queue.in_progress == set() assert graph_queue.queued == {"model.test_package.upstream_model"} assert graph_queue.lock def test_init_graph_queue_preserve_edges_false(self, manifest, graph): graph_queue = GraphQueue(graph=graph, manifest=manifest, selected={}, preserve_edges=False) # when preserve_edges is set to false, dependencies between nodes are no longer tracked in the priority queue assert list(graph_queue.graph.edges) == [] assert graph_queue.inner.queue == [ (0, "model.test_package.downstream_model"), (0, "model.test_package.upstream_model"), ] assert graph_queue.queued == { "model.test_package.upstream_model", "model.test_package.downstream_model", } ================================================ FILE: tests/unit/graph/test_selector.py ================================================ import string from argparse import Namespace from queue import Empty from typing import List from unittest.mock import MagicMock import networkx as nx import pytest import dbt.compilation import dbt.config import dbt.exceptions import dbt.graph.cli as graph_cli import dbt.graph.selector as graph_selector import dbt.parser import dbt.parser.manifest import dbt.utils import dbt_common.exceptions from dbt.config.runtime import RuntimeConfig from dbt.flags import set_from_args from dbt.graph import NodeSelector, parse_difference from dbt.node_types import NodeType from tests.unit.utils.manifest import make_manifest, make_model set_from_args(Namespace(WARN_ERROR=False), None) def _get_graph(): integer_graph = nx.balanced_tree(2, 2, nx.DiGraph()) package_mapping = { i: "m." + ("X" if i % 2 == 0 else "Y") + "." + letter for (i, letter) in enumerate(string.ascii_lowercase) } # Edges: [(X.a, Y.b), (X.a, X.c), (Y.b, Y.d), (Y.b, X.e), (X.c, Y.f), (X.c, X.g)] return graph_selector.Graph(nx.relabel_nodes(integer_graph, package_mapping)) def _get_manifest(graph): nodes = {} for unique_id in graph: fqn = unique_id.split(".") node = MagicMock( unique_id=unique_id, fqn=fqn, package_name=fqn[0], tags=[], resource_type=NodeType.Model, empty=False, config=MagicMock(enabled=True), is_versioned=False, ) nodes[unique_id] = node nodes["m.X.a"].tags = ["abc"] nodes["m.Y.b"].tags = ["abc", "bcef"] nodes["m.X.c"].tags = ["abc", "bcef"] nodes["m.Y.d"].tags = [] nodes["m.X.e"].tags = ["efg", "bcef"] nodes["m.Y.f"].tags = ["efg", "bcef"] nodes["m.X.g"].tags = ["efg"] return MagicMock(nodes=nodes) @pytest.fixture def graph(): return _get_graph() @pytest.fixture def mock_manifest_with_mock_graph(graph): return _get_manifest(graph) def id_macro(arg): if isinstance(arg, str): return arg try: return "_".join(arg) except TypeError: return arg run_specs = [ # include by fqn (["X.a"], [], {"m.X.a"}), # include by tag (["tag:abc"], [], {"m.X.a", "m.Y.b", "m.X.c"}), # exclude by tag (["*"], ["tag:abc"], {"m.Y.d", "m.X.e", "m.Y.f", "m.X.g"}), # tag + fqn (["tag:abc", "a"], [], {"m.X.a", "m.Y.b", "m.X.c"}), (["tag:abc", "d"], [], {"m.X.a", "m.Y.b", "m.X.c", "m.Y.d"}), # multiple node selection across packages (["X.a", "b"], [], {"m.X.a", "m.Y.b"}), (["X.a+"], ["b"], {"m.X.a", "m.X.c", "m.Y.d", "m.X.e", "m.Y.f", "m.X.g"}), # children (["X.c+"], [], {"m.X.c", "m.Y.f", "m.X.g"}), (["X.a+1"], [], {"m.X.a", "m.Y.b", "m.X.c"}), (["X.a+"], ["tag:efg"], {"m.X.a", "m.Y.b", "m.X.c", "m.Y.d"}), # parents (["+Y.f"], [], {"m.X.c", "m.Y.f", "m.X.a"}), (["1+Y.f"], [], {"m.X.c", "m.Y.f"}), # childrens parents (["@X.c"], [], {"m.X.a", "m.X.c", "m.Y.f", "m.X.g"}), # multiple selection/exclusion (["tag:abc", "tag:bcef"], [], {"m.X.a", "m.Y.b", "m.X.c", "m.X.e", "m.Y.f"}), (["tag:abc", "tag:bcef"], ["tag:efg"], {"m.X.a", "m.Y.b", "m.X.c"}), (["tag:abc", "tag:bcef"], ["tag:efg", "a"], {"m.Y.b", "m.X.c"}), # intersections (["a,a"], [], {"m.X.a"}), (["+c,c+"], [], {"m.X.c"}), (["a,b"], [], set()), (["tag:abc,tag:bcef"], [], {"m.Y.b", "m.X.c"}), (["*,tag:abc,a"], [], {"m.X.a"}), (["a,tag:abc,*"], [], {"m.X.a"}), (["tag:abc,tag:bcef"], ["c"], {"m.Y.b"}), (["tag:bcef,tag:efg"], ["tag:bcef,@b"], {"m.Y.f"}), (["tag:bcef,tag:efg"], ["tag:bcef,@a"], set()), (["*,@a,+b"], ["*,tag:abc,tag:bcef"], {"m.X.a"}), (["tag:bcef,tag:efg", "*,tag:abc"], [], {"m.X.a", "m.Y.b", "m.X.c", "m.X.e", "m.Y.f"}), (["tag:bcef,tag:efg", "*,tag:abc"], ["e"], {"m.X.a", "m.Y.b", "m.X.c", "m.Y.f"}), (["tag:bcef,tag:efg", "*,tag:abc"], ["e"], {"m.X.a", "m.Y.b", "m.X.c", "m.Y.f"}), (["tag:bcef,tag:efg", "*,tag:abc"], ["e", "f"], {"m.X.a", "m.Y.b", "m.X.c"}), (["tag:bcef,tag:efg", "*,tag:abc"], ["tag:abc,tag:bcef"], {"m.X.a", "m.X.e", "m.Y.f"}), (["tag:bcef,tag:efg", "*,tag:abc"], ["tag:abc,tag:bcef", "tag:abc,a"], {"m.X.e", "m.Y.f"}), ] @pytest.mark.parametrize("include,exclude,expected", run_specs, ids=id_macro) def test_run_specs(include, exclude, expected, graph, mock_manifest_with_mock_graph): selector = graph_selector.NodeSelector(graph, mock_manifest_with_mock_graph) spec = graph_cli.parse_difference(include, exclude) selected, _ = selector.select_nodes(spec) assert selected == expected param_specs = [ ("a", False, None, False, None, "fqn", "a", False), ("+a", True, None, False, None, "fqn", "a", False), ("256+a", True, 256, False, None, "fqn", "a", False), ("a+", False, None, True, None, "fqn", "a", False), ("a+256", False, None, True, 256, "fqn", "a", False), ("+a+", True, None, True, None, "fqn", "a", False), ("16+a+32", True, 16, True, 32, "fqn", "a", False), ("@a", False, None, False, None, "fqn", "a", True), ("a.b", False, None, False, None, "fqn", "a.b", False), ("+a.b", True, None, False, None, "fqn", "a.b", False), ("256+a.b", True, 256, False, None, "fqn", "a.b", False), ("a.b+", False, None, True, None, "fqn", "a.b", False), ("a.b+256", False, None, True, 256, "fqn", "a.b", False), ("+a.b+", True, None, True, None, "fqn", "a.b", False), ("16+a.b+32", True, 16, True, 32, "fqn", "a.b", False), ("@a.b", False, None, False, None, "fqn", "a.b", True), ("a.b.*", False, None, False, None, "fqn", "a.b.*", False), ("+a.b.*", True, None, False, None, "fqn", "a.b.*", False), ("256+a.b.*", True, 256, False, None, "fqn", "a.b.*", False), ("a.b.*+", False, None, True, None, "fqn", "a.b.*", False), ("a.b.*+256", False, None, True, 256, "fqn", "a.b.*", False), ("+a.b.*+", True, None, True, None, "fqn", "a.b.*", False), ("16+a.b.*+32", True, 16, True, 32, "fqn", "a.b.*", False), ("@a.b.*", False, None, False, None, "fqn", "a.b.*", True), ("tag:a", False, None, False, None, "tag", "a", False), ("+tag:a", True, None, False, None, "tag", "a", False), ("256+tag:a", True, 256, False, None, "tag", "a", False), ("tag:a+", False, None, True, None, "tag", "a", False), ("tag:a+256", False, None, True, 256, "tag", "a", False), ("+tag:a+", True, None, True, None, "tag", "a", False), ("16+tag:a+32", True, 16, True, 32, "tag", "a", False), ("@tag:a", False, None, False, None, "tag", "a", True), ("source:a", False, None, False, None, "source", "a", False), ("source:a+", False, None, True, None, "source", "a", False), ("source:a+1", False, None, True, 1, "source", "a", False), ("source:a+32", False, None, True, 32, "source", "a", False), ("@source:a", False, None, False, None, "source", "a", True), ] @pytest.mark.parametrize( "spec,parents,parents_depth,children,children_depth,filter_type,filter_value,childrens_parents", param_specs, ids=id_macro, ) def test_parse_specs( spec, parents, parents_depth, children, children_depth, filter_type, filter_value, childrens_parents, ): parsed = graph_selector.SelectionCriteria.from_single_spec(spec) assert parsed.parents == parents assert parsed.parents_depth == parents_depth assert parsed.children == children assert parsed.children_depth == children_depth assert parsed.method == filter_type assert parsed.value == filter_value assert parsed.childrens_parents == childrens_parents invalid_specs = [ "@a+", "@a.b+", "@a.b*+", "@tag:a+", "@source:a+", ] @pytest.mark.parametrize("invalid", invalid_specs, ids=lambda k: str(k)) def test_invalid_specs(invalid): with pytest.raises(dbt_common.exceptions.DbtRuntimeError): graph_selector.SelectionCriteria.from_single_spec(invalid) class TestCompiler: def test_single_model(self, runtime_config: RuntimeConfig): model = make_model(pkg="pkg", name="model_one", code="SELECT * FROM events") manifest = make_manifest(nodes=[model]) compiler = dbt.compilation.Compiler(config=runtime_config) linker = compiler.compile(manifest) assert linker.nodes() == {model.unique_id} assert linker.edges() == set() def test_two_models_simple_ref(self, runtime_config: RuntimeConfig): model_one = make_model(pkg="pkg", name="model_one", code="SELECT * FROM events") model_two = make_model( pkg="pkg", name="model_two", code="SELECT * FROM {{ref('model_one')}}", refs=[model_one], ) models = [model_one, model_two] manifest = make_manifest(nodes=models) compiler = dbt.compilation.Compiler(config=runtime_config) linker = compiler.compile(manifest) expected_nodes: List[str] = [model.unique_id for model in models] assert linker.nodes() == set(expected_nodes) assert list(linker.edges()) == [tuple(expected_nodes)] class TestNodeSelector: def test_dependency_list(self, runtime_config: RuntimeConfig): model_one = make_model(pkg="pkg", name="model_one", code="SELECT * FROM events") model_two = make_model( pkg="pkg", name="model_two", code="SELECT * FROM {{ref('model_one')}}", refs=[model_one], ) model_three = make_model( pkg="pkg", name="model_three", code=""" SELECT * FROM {{ ref("model_1") }} union all SELECT * FROM {{ ref("model_2") }} """, refs=[model_one, model_two], ) model_four = make_model( pkg="pkg", name="model_four", code="SELECT * FROM {{ref('model_three')}}", refs=[model_three], ) models = [model_one, model_two, model_three, model_four] manifest = make_manifest(nodes=models) # Get the graph compiler = dbt.compilation.Compiler(runtime_config) graph = compiler.compile(manifest) # Create the selector and get the queue selector = NodeSelector(graph, manifest) queue = selector.get_graph_queue( parse_difference( None, None, ) ) for model in models: assert not queue.empty() got = queue.get(block=False) assert got.unique_id == model.unique_id with pytest.raises(Empty): queue.get(block=False) queue.mark_done(got.unique_id) assert queue.empty() def test_select_downstream_of_empty_model(self, runtime_config: RuntimeConfig): # empty model model_one = make_model(pkg="other", name="model_one", code="") # non-empty model model_two = make_model( pkg="pkg", name="model_two", code="""select * from {{ref('model_one')}}""", refs=[model_one], ) models = [model_one, model_two] manifest = make_manifest(nodes=models) # Get the graph compiler = dbt.compilation.Compiler(runtime_config) graph = compiler.compile(manifest) # Ensure that model_two is selected as downstream of model_one selector = NodeSelector(graph, manifest) spec = graph_selector.SelectionCriteria.from_single_spec("model_one+") assert selector.get_selected(spec) == {"model.pkg.model_two"} # Ensure that --indirect-selection empty returns the same result spec.indirect_selection = graph_selector.IndirectSelection.Empty assert selector.get_selected(spec) == {"model.pkg.model_two"} ================================================ FILE: tests/unit/graph/test_selector_methods.py ================================================ import copy from argparse import Namespace from dataclasses import replace from pathlib import Path from unittest import mock import pytest import dbt.compilation import dbt_common.exceptions from dbt.artifacts.resources import ColumnInfo, FileHash from dbt.contracts.graph.manifest import Manifest from dbt.contracts.state import PreviousState from dbt.graph import NodeSelector from dbt.graph.selector_methods import ( AccessSelectorMethod, ConfigSelectorMethod, ExposureSelectorMethod, FileSelectorMethod, GroupSelectorMethod, MethodManager, MetricSelectorMethod, PackageSelectorMethod, PathSelectorMethod, QualifiedNameSelectorMethod, SavedQuerySelectorMethod, SelectorSelectorMethod, SemanticModelSelectorMethod, SourceSelectorMethod, StateSelectorMethod, TagSelectorMethod, TestNameSelectorMethod, TestTypeSelectorMethod, UnitTestSelectorMethod, VersionSelectorMethod, ) from dbt.graph.selector_spec import ( SelectionCriteria, SelectionIntersection, SelectionUnion, ) from dbt_common.ui import warning_tag from tests.unit.utils import replace_config from tests.unit.utils.manifest import ( make_exposure, make_group, make_macro, make_metric, make_model, make_saved_query, make_seed, make_semantic_model, make_unit_test, ) def search_manifest_using_method(manifest, method, selection): selected = method.search( set(manifest.nodes) | set(manifest.sources) | set(manifest.exposures) | set(manifest.metrics) | set(manifest.semantic_models) | set(manifest.saved_queries) | set(manifest.unit_tests), selection, ) results = {manifest.expect(uid).search_name for uid in selected} return results def test_select_fqn(manifest): methods = MethodManager(manifest, None) method = methods.get_method("fqn", []) assert isinstance(method, QualifiedNameSelectorMethod) assert method.arguments == [] assert search_manifest_using_method(manifest, method, "pkg.unions") == { "union_model", "mynamespace.union_model", } assert not search_manifest_using_method(manifest, method, "ext.unions") # sources don't show up, because selection pretends they have no FQN. Should it? assert search_manifest_using_method(manifest, method, "pkg") == { "union_model", "versioned_model.v1", "versioned_model.v2", "versioned_model.v3", "versioned_model.v4", "versioned_model.v12", "table_model", "table_model_py", "table_model_csv", "view_model", "ephemeral_model", "seed", "mynamespace.union_model", "mynamespace.ephemeral_model", "mynamespace.seed", "unit_test_table_model", } assert search_manifest_using_method(manifest, method, "ext") == {"ext_model"} # versions assert search_manifest_using_method(manifest, method, "versioned_model") == { "versioned_model.v1", "versioned_model.v2", "versioned_model.v3", "versioned_model.v4", "versioned_model.v12", } assert search_manifest_using_method(manifest, method, "versioned_model.v1") == { "versioned_model.v1" } # version selection with _ instead of '.' assert search_manifest_using_method(manifest, method, "versioned_model_v1") == { "versioned_model.v1" } # version selection with _ instead of '.' - latest version assert search_manifest_using_method(manifest, method, "versioned_model_v2") == { "versioned_model.v2" } # wildcards assert search_manifest_using_method(manifest, method, "*.*.*_model") == { "mynamespace.union_model", "mynamespace.ephemeral_model", "test_semantic_model", "union_model", "unit_test_table_model", } # multiple wildcards assert search_manifest_using_method(manifest, method, "*unions*") == { "union_model", "mynamespace.union_model", } # negation assert not search_manifest_using_method(manifest, method, "!pkg*") # single wildcard assert search_manifest_using_method(manifest, method, "pkg.t*") == { "table_model", "table_model_py", "table_model_csv", "unit_test_table_model", } # wildcard and ? (matches exactly one character) assert search_manifest_using_method(manifest, method, "*ext_m?del") == {"ext_model"} # multiple ? assert search_manifest_using_method(manifest, method, "*.?????_model") == { "union_model", "table_model", "mynamespace.union_model", } # multiple ranges assert search_manifest_using_method(manifest, method, "*.[t-u][a-n][b-i][l-o][e-n]_model") == { "union_model", "table_model", "mynamespace.union_model", } def test_select_tag(manifest): methods = MethodManager(manifest, None) method = methods.get_method("tag", []) assert isinstance(method, TagSelectorMethod) assert method.arguments == [] assert search_manifest_using_method(manifest, method, "uses_ephemeral") == { "view_model", "table_model", } assert not search_manifest_using_method(manifest, method, "missing") assert search_manifest_using_method(manifest, method, "uses_eph*") == { "view_model", "table_model", } def test_select_selector(manifest, runtime_config): compiler = dbt.compilation.Compiler(runtime_config) graph = compiler.compile(manifest) selectors = { "union_selector": { "definition": SelectionUnion( components=[ SelectionCriteria.from_single_spec("fqn:table_model"), SelectionCriteria.from_single_spec("fqn:view_model"), ] ), }, "intersection_selector": { "definition": SelectionIntersection( components=[ SelectionCriteria.from_single_spec("selector:union_selector"), SelectionCriteria.from_single_spec("fqn:table_model"), ] ), }, } selector = NodeSelector(graph, manifest, selectors=selectors) methods = MethodManager(manifest, None) method = methods.get_method( "selector", [], selectors=selectors, get_selected_callback=selector.get_selected ) assert isinstance(method, SelectorSelectorMethod) assert search_manifest_using_method(manifest, method, "union_selector").issuperset( { "table_model", "view_model", } ) assert search_manifest_using_method(manifest, method, "intersection_selector").issuperset( { "table_model", } ) def test_select_group(manifest, view_model): group_name = "my_group" group = make_group("test", group_name) manifest.groups[group.unique_id] = group change_node( manifest, replace(view_model, config={"materialized": "view", "group": group_name}), ) methods = MethodManager(manifest, None) method = methods.get_method("group", []) assert isinstance(method, GroupSelectorMethod) assert method.arguments == [] assert search_manifest_using_method(manifest, method, group_name) == {"view_model"} assert search_manifest_using_method(manifest, method, "my?group") == {"view_model"} assert not search_manifest_using_method(manifest, method, "not_my_group") def test_select_access(manifest, view_model): change_node( manifest, replace(view_model, access="public"), ) methods = MethodManager(manifest, None) method = methods.get_method("access", []) assert isinstance(method, AccessSelectorMethod) assert method.arguments == [] assert search_manifest_using_method(manifest, method, "public") == {"view_model"} assert not search_manifest_using_method(manifest, method, "private") def test_select_source(manifest): methods = MethodManager(manifest, None) method = methods.get_method("source", []) assert isinstance(method, SourceSelectorMethod) assert method.arguments == [] # the lookup is based on how many components you provide: source, source.table, package.source.table assert search_manifest_using_method(manifest, method, "raw") == { "raw.seed", "raw.ext_source", "raw.ext_source_2", } assert search_manifest_using_method(manifest, method, "raw.seed") == {"raw.seed"} assert search_manifest_using_method(manifest, method, "pkg.raw.seed") == {"raw.seed"} assert search_manifest_using_method(manifest, method, "pkg.*.*") == {"raw.seed"} assert search_manifest_using_method(manifest, method, "raw.*") == { "raw.seed", "raw.ext_source", "raw.ext_source_2", } assert search_manifest_using_method(manifest, method, "ext.raw.*") == { "raw.ext_source", "raw.ext_source_2", } assert not search_manifest_using_method(manifest, method, "missing") assert not search_manifest_using_method(manifest, method, "raw.missing") assert not search_manifest_using_method(manifest, method, "missing.raw.seed") assert search_manifest_using_method(manifest, method, "ext.*.*") == { "ext_raw.ext_source", "ext_raw.ext_source_2", "raw.ext_source", "raw.ext_source_2", } assert search_manifest_using_method(manifest, method, "ext_raw") == { "ext_raw.ext_source", "ext_raw.ext_source_2", } assert search_manifest_using_method(manifest, method, "ext.ext_raw.*") == { "ext_raw.ext_source", "ext_raw.ext_source_2", } assert not search_manifest_using_method(manifest, method, "pkg.ext_raw.*") assert search_manifest_using_method(manifest, method, "*.ext_[s]ourc?") == { "ext_raw.ext_source", "raw.ext_source", } # TODO: this requires writing out files @pytest.mark.skip("TODO: write manifest files to disk") def test_select_path(manifest): methods = MethodManager(manifest, None) method = methods.get_method("path", []) assert isinstance(method, PathSelectorMethod) assert method.arguments == [] assert search_manifest_using_method(manifest, method, "subdirectory/*.sql") == { "union_model", "table_model", } assert search_manifest_using_method(manifest, method, "subdirectory/union_model.sql") == { "union_model" } assert search_manifest_using_method(manifest, method, "models/*.sql") == { "view_model", "ephemeral_model", } assert not search_manifest_using_method(manifest, method, "missing") assert not search_manifest_using_method(manifest, method, "models/missing.sql") assert not search_manifest_using_method(manifest, method, "models/missing*") def test_select_file(manifest): methods = MethodManager(manifest, None) method = methods.get_method("file", []) assert isinstance(method, FileSelectorMethod) assert method.arguments == [] assert search_manifest_using_method(manifest, method, "table_model.sql") == {"table_model"} assert search_manifest_using_method(manifest, method, "table_model.py") == {"table_model_py"} assert search_manifest_using_method(manifest, method, "table_model.csv") == {"table_model_csv"} assert search_manifest_using_method(manifest, method, "union_model.sql") == { "union_model", "mynamespace.union_model", } assert not search_manifest_using_method(manifest, method, "missing.sql") assert not search_manifest_using_method(manifest, method, "missing.py") assert search_manifest_using_method(manifest, method, "table_*.csv") == {"table_model_csv"} # stem selector match assert search_manifest_using_method(manifest, method, "union_model") == { "union_model", "mynamespace.union_model", } assert search_manifest_using_method(manifest, method, "versioned_model_v1") == { "versioned_model.v1" } def test_select_package(manifest): methods = MethodManager(manifest, None) method = methods.get_method("package", []) assert isinstance(method, PackageSelectorMethod) assert method.arguments == [] assert search_manifest_using_method(manifest, method, "pkg") == { "union_model", "versioned_model.v1", "versioned_model.v2", "versioned_model.v3", "versioned_model.v4", "versioned_model.v12", "table_model", "table_model_py", "table_model_csv", "view_model", "ephemeral_model", "seed", "raw.seed", "unique_table_model_id", "not_null_table_model_id", "unique_view_model_id", "view_test_nothing", "mynamespace.seed", "mynamespace.ephemeral_model", "mynamespace.union_model", "unit_test_table_model", } assert search_manifest_using_method(manifest, method, "ext") == { "ext_model", "ext_raw.ext_source", "ext_raw.ext_source_2", "raw.ext_source", "raw.ext_source_2", "unique_ext_raw_ext_source_id", } assert not search_manifest_using_method(manifest, method, "missing") assert search_manifest_using_method(manifest, method, "ex*") == { "ext_model", "ext_raw.ext_source", "ext_raw.ext_source_2", "raw.ext_source", "raw.ext_source_2", "unique_ext_raw_ext_source_id", } def test_select_package_this(manifest): new_manifest = copy.deepcopy(manifest) # change the package name for all nodes except ones where the unique_id contains "table_model" for id, node in new_manifest.nodes.items(): if "table_model" not in id: node.package_name = "foo" for source in new_manifest.sources.values(): if "table_model" not in source.unique_id: source.package_name = "foo" methods = MethodManager(new_manifest, None) method = methods.get_method("package", []) assert isinstance(method, PackageSelectorMethod) assert method.arguments == [] assert search_manifest_using_method(new_manifest, method, "this") == { "not_null_table_model_id", "table_model", "table_model_csv", "table_model_py", "unique_table_model_id", "unit_test_table_model", } def test_select_config_materialized(manifest): methods = MethodManager(manifest, None) method = methods.get_method("config", ["materialized"]) assert isinstance(method, ConfigSelectorMethod) assert method.arguments == ["materialized"] assert search_manifest_using_method(manifest, method, "view") == {"view_model", "ext_model"} assert search_manifest_using_method(manifest, method, "table") == { "table_model", "table_model_py", "table_model_csv", "union_model", "versioned_model.v1", "versioned_model.v2", "versioned_model.v3", "versioned_model.v4", "versioned_model.v12", "mynamespace.union_model", } def test_select_config_meta(manifest): methods = MethodManager(manifest, None) string_method = methods.get_method("config", ["meta", "string_property"]) assert search_manifest_using_method(manifest, string_method, "some_string") == {"table_model"} assert not search_manifest_using_method(manifest, string_method, "other_string") == { "table_model" } truthy_bool_method = methods.get_method("config", ["meta", "truthy_bool_property"]) assert search_manifest_using_method(manifest, truthy_bool_method, "true") == {"table_model"} assert not search_manifest_using_method(manifest, truthy_bool_method, "false") == { "table_model" } assert not search_manifest_using_method(manifest, truthy_bool_method, "other") == { "table_model" } falsy_bool_method = methods.get_method("config", ["meta", "falsy_bool_property"]) assert search_manifest_using_method(manifest, falsy_bool_method, "false") == {"table_model"} assert not search_manifest_using_method(manifest, falsy_bool_method, "true") == {"table_model"} assert not search_manifest_using_method(manifest, falsy_bool_method, "other") == { "table_model" } list_method = methods.get_method("config", ["meta", "list_property"]) assert search_manifest_using_method(manifest, list_method, "some_value") == {"table_model"} assert search_manifest_using_method(manifest, list_method, "true") == {"table_model"} assert search_manifest_using_method(manifest, list_method, "false") == {"table_model"} assert not search_manifest_using_method(manifest, list_method, "other") == {"table_model"} def test_select_test_name(manifest): methods = MethodManager(manifest, None) method = methods.get_method("test_name", []) assert isinstance(method, TestNameSelectorMethod) assert method.arguments == [] assert search_manifest_using_method(manifest, method, "unique") == { "unique_table_model_id", "unique_view_model_id", "unique_ext_raw_ext_source_id", } assert search_manifest_using_method(manifest, method, "not_null") == { "not_null_table_model_id" } assert not search_manifest_using_method(manifest, method, "notatest") assert search_manifest_using_method(manifest, method, "not_*") == {"not_null_table_model_id"} def test_select_test_type(manifest): methods = MethodManager(manifest, None) method = methods.get_method("test_type", []) assert isinstance(method, TestTypeSelectorMethod) assert method.arguments == [] assert search_manifest_using_method(manifest, method, "generic") == { "unique_table_model_id", "not_null_table_model_id", "unique_view_model_id", "unique_ext_raw_ext_source_id", } assert search_manifest_using_method(manifest, method, "singular") == {"view_test_nothing"} # test backwards compatibility assert search_manifest_using_method(manifest, method, "schema") == { "unique_table_model_id", "not_null_table_model_id", "unique_view_model_id", "unique_ext_raw_ext_source_id", } assert search_manifest_using_method(manifest, method, "data") == { "view_test_nothing", "unique_table_model_id", "not_null_table_model_id", "unique_view_model_id", "unique_ext_raw_ext_source_id", } assert search_manifest_using_method(manifest, method, "unit") == { "unit_test_table_model", } def test_select_version(manifest): methods = MethodManager(manifest, None) method = methods.get_method("version", []) assert isinstance(method, VersionSelectorMethod) assert method.arguments == [] assert search_manifest_using_method(manifest, method, "latest") == {"versioned_model.v2"} assert search_manifest_using_method(manifest, method, "old") == {"versioned_model.v1"} assert search_manifest_using_method(manifest, method, "prerelease") == { "versioned_model.v3", "versioned_model.v4", "versioned_model.v12", } assert search_manifest_using_method(manifest, method, "none") == { "table_model_py", "union_model", "view_model", "mynamespace.ephemeral_model", "table_model_csv", "ephemeral_model", "mynamespace.union_model", "table_model", "ext_model", } def test_select_exposure(manifest): exposure = make_exposure("test", "my_exposure") manifest.exposures[exposure.unique_id] = exposure methods = MethodManager(manifest, None) method = methods.get_method("exposure", []) assert isinstance(method, ExposureSelectorMethod) assert search_manifest_using_method(manifest, method, "my_exposure") == {"my_exposure"} assert not search_manifest_using_method(manifest, method, "not_my_exposure") assert search_manifest_using_method(manifest, method, "my_e*e") == {"my_exposure"} def test_select_metric(manifest): metric = make_metric("test", "my_metric") manifest.metrics[metric.unique_id] = metric methods = MethodManager(manifest, None) method = methods.get_method("metric", []) assert isinstance(method, MetricSelectorMethod) assert search_manifest_using_method(manifest, method, "my_metric") == {"my_metric"} assert not search_manifest_using_method(manifest, method, "not_my_metric") assert search_manifest_using_method(manifest, method, "*_metric") == {"my_metric"} def test_select_semantic_model(manifest, table_model): semantic_model = make_semantic_model( "pkg", "customer", model=table_model, path="_semantic_models.yml", ) manifest.semantic_models[semantic_model.unique_id] = semantic_model methods = MethodManager(manifest, None) method = methods.get_method("semantic_model", []) assert isinstance(method, SemanticModelSelectorMethod) assert search_manifest_using_method(manifest, method, "customer") == {"customer"} assert not search_manifest_using_method(manifest, method, "not_customer") assert search_manifest_using_method(manifest, method, "*omer") == {"customer"} def test_select_semantic_model_by_tag(manifest, table_model): semantic_model = make_semantic_model( "pkg", "customer", model=table_model, path="_semantic_models.yml", ) manifest.semantic_models[semantic_model.unique_id] = semantic_model methods = MethodManager(manifest, None) method = methods.get_method("tag", []) assert isinstance(method, TagSelectorMethod) assert method.arguments == [] search_manifest_using_method(manifest, method, "any_tag") def test_select_saved_query(manifest: Manifest) -> None: metric = make_metric("test", "my_metric") saved_query = make_saved_query( "pkg", "test_saved_query", "my_metric", ) manifest.metrics[metric.unique_id] = metric manifest.saved_queries[saved_query.unique_id] = saved_query methods = MethodManager(manifest, None) method = methods.get_method("saved_query", []) assert isinstance(method, SavedQuerySelectorMethod) assert search_manifest_using_method(manifest, method, "test_saved_query") == { "test_saved_query" } assert not search_manifest_using_method(manifest, method, "not_test_saved_query") assert search_manifest_using_method(manifest, method, "*uery") == {"test_saved_query"} def test_select_saved_query_by_tag(manifest: Manifest) -> None: metric = make_metric("test", "my_metric") saved_query = make_saved_query( "pkg", "test_saved_query", "my_metric", ) manifest.metrics[metric.unique_id] = metric manifest.saved_queries[saved_query.unique_id] = saved_query methods = MethodManager(manifest, None) method = methods.get_method("tag", []) assert isinstance(method, TagSelectorMethod) assert method.arguments == [] search_manifest_using_method(manifest, method, "any_tag") def test_modified_saved_query(manifest: Manifest) -> None: metric = make_metric("test", "my_metric") saved_query = make_saved_query( "pkg", "test_saved_query", "my_metric", ) manifest.metrics[metric.unique_id] = metric manifest.saved_queries[saved_query.unique_id] = saved_query # Create PreviousState with a saved query, this deepcopies manifest previous_state = create_previous_state(manifest) method = statemethod(manifest, previous_state) # create another metric and add to saved query alt_metric = make_metric("test", "alt_metric") manifest.metrics[alt_metric.unique_id] = alt_metric saved_query.query_params.metrics.append("alt_metric") assert search_manifest_using_method(manifest, method, "modified") == {"test_saved_query"} def test_select_unit_test(manifest: Manifest) -> None: test_model = make_model("test", "my_model", "select 1 as id") unit_test = make_unit_test("test", "my_unit_test", test_model) manifest.unit_tests[unit_test.unique_id] = unit_test methods = MethodManager(manifest, None) method = methods.get_method("unit_test", []) assert isinstance(method, UnitTestSelectorMethod) assert not search_manifest_using_method(manifest, method, "not_test_unit_test") assert search_manifest_using_method(manifest, method, "*nit_test") == {unit_test.search_name} assert search_manifest_using_method(manifest, method, "test.my_unit_test") == { unit_test.search_name } assert search_manifest_using_method(manifest, method, "my_unit_test") == { unit_test.search_name } def create_previous_state(manifest): writable = copy.deepcopy(manifest).writable_manifest() state = PreviousState( state_path=Path("/path/does/not/exist"), target_path=Path("/path/does/not/exist"), project_root=Path("/path/does/not/exist"), ) state.manifest = Manifest.from_writable_manifest(writable) return state @pytest.fixture def previous_state(manifest): return create_previous_state(manifest) @pytest.fixture def args_for_flags(): return Namespace( state_modified_compare_more_unrendered_values=False, state_modified_compare_vars=False ) def add_node(manifest, node): manifest.nodes[node.unique_id] = node def add_macro(manifest, macro): manifest.macros[macro.unique_id] = macro def change_node(manifest, node, change=None): if change is not None: node = change(node) manifest.nodes[node.unique_id] = node def statemethod(manifest, previous_state): methods = MethodManager(manifest, previous_state) method = methods.get_method("state", []) assert isinstance(method, StateSelectorMethod) assert method.arguments == [] return method def test_select_state_no_change(manifest, previous_state): method = statemethod(manifest, previous_state) assert not search_manifest_using_method(manifest, method, "modified") assert not search_manifest_using_method(manifest, method, "new") assert not search_manifest_using_method(manifest, method, "modified.configs") assert not search_manifest_using_method(manifest, method, "modified.persisted_descriptions") assert not search_manifest_using_method(manifest, method, "modified.relation") assert not search_manifest_using_method(manifest, method, "modified.macros") def test_select_state_nothing(manifest, previous_state): previous_state.manifest = None method = statemethod(manifest, previous_state) with pytest.raises(dbt_common.exceptions.DbtRuntimeError) as exc: search_manifest_using_method(manifest, method, "modified") assert "no comparison manifest" in str(exc.value) with pytest.raises(dbt_common.exceptions.DbtRuntimeError) as exc: search_manifest_using_method(manifest, method, "new") assert "no comparison manifest" in str(exc.value) with pytest.raises(dbt_common.exceptions.DbtRuntimeError) as exc: search_manifest_using_method(manifest, method, "unmodified") assert "no comparison manifest" in str(exc.value) with pytest.raises(dbt_common.exceptions.DbtRuntimeError) as exc: search_manifest_using_method(manifest, method, "old") assert "no comparison manifest" in str(exc.value) def test_select_state_added_model(manifest, previous_state): add_node(manifest, make_model("pkg", "another_model", "select 1 as id")) method = statemethod(manifest, previous_state) assert search_manifest_using_method(manifest, method, "modified") == {"another_model"} assert search_manifest_using_method(manifest, method, "new") == {"another_model"} assert search_manifest_using_method(manifest, method, "modified.body") == {"another_model"} # none of these assert not {"another_model"} in search_manifest_using_method(manifest, method, "old") assert not {"another_model"} in search_manifest_using_method(manifest, method, "unmodified") def test_select_state_changed_model_sql(manifest, previous_state, view_model): change_node(manifest, replace(view_model, raw_code="select 1 as id")) method = statemethod(manifest, previous_state) # both of these assert search_manifest_using_method(manifest, method, "modified") == {"view_model"} assert search_manifest_using_method(manifest, method, "modified.body") == {"view_model"} # none of these assert not search_manifest_using_method(manifest, method, "new") assert not {"view_model"} in search_manifest_using_method(manifest, method, "old") assert not {"view_model"} in search_manifest_using_method(manifest, method, "unmodified") assert not search_manifest_using_method(manifest, method, "modified.configs") assert not search_manifest_using_method(manifest, method, "modified.persisted_descriptions") assert not search_manifest_using_method(manifest, method, "modified.relation") assert not search_manifest_using_method(manifest, method, "modified.macros") def test_select_state_changed_model_fqn(manifest, previous_state, view_model): change_node( manifest, replace(view_model, fqn=view_model.fqn[:-1] + ["nested"] + view_model.fqn[-1:]) ) method = statemethod(manifest, previous_state) assert search_manifest_using_method(manifest, method, "modified") == {"view_model"} assert not search_manifest_using_method(manifest, method, "new") # none of these assert not {"view_model"} in search_manifest_using_method(manifest, method, "old") assert not {"view_model"} in search_manifest_using_method(manifest, method, "unmodified") def test_select_state_added_seed(manifest, previous_state): add_node(manifest, make_seed("pkg", "another_seed")) method = statemethod(manifest, previous_state) assert search_manifest_using_method(manifest, method, "modified") == {"another_seed"} assert search_manifest_using_method(manifest, method, "new") == {"another_seed"} # none of these assert not {"another_seed"} in search_manifest_using_method(manifest, method, "old") assert not {"another_seed"} in search_manifest_using_method(manifest, method, "unmodified") def test_select_state_changed_seed_checksum_sha_to_sha(manifest, previous_state, seed): change_node(manifest, replace(seed, checksum=FileHash.from_contents("changed"))) method = statemethod(manifest, previous_state) assert search_manifest_using_method(manifest, method, "modified") == {"seed"} assert not search_manifest_using_method(manifest, method, "new") assert "seed" not in search_manifest_using_method(manifest, method, "unmodified") def test_select_state_changed_seed_checksum_path_to_path(manifest, previous_state, seed): change_node( previous_state.manifest, replace(seed, checksum=FileHash(name="path", checksum=seed.original_file_path)), ) change_node( manifest, replace(seed, checksum=FileHash(name="path", checksum=seed.original_file_path)) ) method = statemethod(manifest, previous_state) with mock.patch("dbt.contracts.graph.nodes.warn_or_error") as warn_or_error_patch: assert not search_manifest_using_method(manifest, method, "modified") warn_or_error_patch.assert_called_once() event = warn_or_error_patch.call_args[0][0] assert type(event).__name__ == "SeedExceedsLimitSamePath" msg = event.message() assert msg.startswith(warning_tag("Found a seed (pkg.seed) >1MB in size")) with mock.patch("dbt.contracts.graph.nodes.warn_or_error") as warn_or_error_patch: assert not search_manifest_using_method(manifest, method, "new") warn_or_error_patch.assert_not_called() with mock.patch("dbt.contracts.graph.nodes.warn_or_error") as warn_or_error_patch: assert search_manifest_using_method(manifest, method, "old") warn_or_error_patch.assert_not_called() with mock.patch("dbt.contracts.graph.nodes.warn_or_error") as warn_or_error_patch: assert search_manifest_using_method(manifest, method, "unmodified") warn_or_error_patch.assert_called_once() event = warn_or_error_patch.call_args[0][0] assert type(event).__name__ == "SeedExceedsLimitSamePath" msg = event.message() assert msg.startswith(warning_tag("Found a seed (pkg.seed) >1MB in size")) def test_select_state_changed_seed_checksum_sha_to_path(manifest, previous_state, seed): change_node( manifest, replace(seed, checksum=FileHash(name="path", checksum=seed.original_file_path)) ) method = statemethod(manifest, previous_state) with mock.patch("dbt.contracts.graph.nodes.warn_or_error") as warn_or_error_patch: assert search_manifest_using_method(manifest, method, "modified") == {"seed"} warn_or_error_patch.assert_called_once() event = warn_or_error_patch.call_args[0][0] assert type(event).__name__ == "SeedIncreased" msg = event.message() assert msg.startswith(warning_tag("Found a seed (pkg.seed) >1MB in size")) with mock.patch("dbt.contracts.graph.nodes.warn_or_error") as warn_or_error_patch: assert not search_manifest_using_method(manifest, method, "new") warn_or_error_patch.assert_not_called() with mock.patch("dbt.contracts.graph.nodes.warn_or_error") as warn_or_error_patch: assert search_manifest_using_method(manifest, method, "old") warn_or_error_patch.assert_not_called() with mock.patch("dbt.contracts.graph.nodes.warn_or_error") as warn_or_error_patch: assert search_manifest_using_method(manifest, method, "unmodified") warn_or_error_patch.assert_called_once() event = warn_or_error_patch.call_args[0][0] assert type(event).__name__ == "SeedIncreased" msg = event.message() assert msg.startswith(warning_tag("Found a seed (pkg.seed) >1MB in size")) def test_select_state_changed_seed_checksum_path_to_sha(manifest, previous_state, seed): change_node( previous_state.manifest, replace(seed, checksum=FileHash(name="path", checksum=seed.original_file_path)), ) method = statemethod(manifest, previous_state) with mock.patch("dbt.contracts.graph.nodes.warn_or_error") as warn_or_error_patch: assert search_manifest_using_method(manifest, method, "modified") == {"seed"} warn_or_error_patch.assert_not_called() with mock.patch("dbt.contracts.graph.nodes.warn_or_error") as warn_or_error_patch: assert not search_manifest_using_method(manifest, method, "new") warn_or_error_patch.assert_not_called() with mock.patch("dbt.contracts.graph.nodes.warn_or_error") as warn_or_error_patch: assert "seed" not in search_manifest_using_method(manifest, method, "unmodified") warn_or_error_patch.assert_not_called() with mock.patch("dbt.contracts.graph.nodes.warn_or_error") as warn_or_error_patch: assert "seed" in search_manifest_using_method(manifest, method, "old") warn_or_error_patch.assert_not_called() def test_select_state_changed_seed_fqn(manifest, previous_state, seed): change_node(manifest, replace(seed, fqn=seed.fqn[:-1] + ["nested"] + seed.fqn[-1:])) method = statemethod(manifest, previous_state) assert search_manifest_using_method(manifest, method, "modified") == {"seed"} assert not search_manifest_using_method(manifest, method, "new") assert "seed" not in search_manifest_using_method(manifest, method, "unmodified") assert "seed" in search_manifest_using_method(manifest, method, "old") def test_select_state_changed_seed_relation_documented(manifest, previous_state, seed): seed_doc_relation = replace_config(seed, persist_docs={"relation": True}) change_node(manifest, seed_doc_relation) method = statemethod(manifest, previous_state) assert search_manifest_using_method(manifest, method, "modified") == {"seed"} assert search_manifest_using_method(manifest, method, "modified.configs") == {"seed"} assert "seed" in search_manifest_using_method(manifest, method, "old") assert not search_manifest_using_method(manifest, method, "new") assert "seed" not in search_manifest_using_method(manifest, method, "unmodified") assert not search_manifest_using_method(manifest, method, "modified.body") assert not search_manifest_using_method(manifest, method, "modified.persisted_descriptions") def test_select_state_changed_seed_relation_documented_nodocs(manifest, previous_state, seed): seed_doc_relation = replace_config(seed, persist_docs={"relation": True}) seed_doc_relation_documented = replace(seed_doc_relation, description="a description") change_node(previous_state.manifest, seed_doc_relation) change_node(manifest, seed_doc_relation_documented) method = statemethod(manifest, previous_state) assert search_manifest_using_method(manifest, method, "modified") == {"seed"} assert search_manifest_using_method(manifest, method, "modified.persisted_descriptions") == { "seed" } assert "seed" in search_manifest_using_method(manifest, method, "old") assert not search_manifest_using_method(manifest, method, "new") assert "seed" not in search_manifest_using_method(manifest, method, "unmodified") assert not search_manifest_using_method(manifest, method, "modified.configs") def test_select_state_changed_seed_relation_documented_withdocs(manifest, previous_state, seed): seed_doc_relation = replace_config(seed, persist_docs={"relation": True}) seed_doc_relation_documented = replace(seed_doc_relation, description="a description") change_node(previous_state.manifest, seed_doc_relation_documented) change_node(manifest, seed_doc_relation) method = statemethod(manifest, previous_state) assert search_manifest_using_method(manifest, method, "modified") == {"seed"} assert search_manifest_using_method(manifest, method, "modified.persisted_descriptions") == { "seed" } assert "seed" in search_manifest_using_method(manifest, method, "old") assert not search_manifest_using_method(manifest, method, "new") assert "seed" not in search_manifest_using_method(manifest, method, "unmodified") def test_select_state_changed_seed_columns_documented(manifest, previous_state, seed): # changing persist_docs, even without changing the description -> changed seed_doc_columns = replace_config(seed, persist_docs={"columns": True}) change_node(manifest, seed_doc_columns) method = statemethod(manifest, previous_state) assert search_manifest_using_method(manifest, method, "modified") == {"seed"} assert search_manifest_using_method(manifest, method, "modified.configs") == {"seed"} assert "seed" in search_manifest_using_method(manifest, method, "old") assert not search_manifest_using_method(manifest, method, "new") assert "seed" not in search_manifest_using_method(manifest, method, "unmodified") assert not search_manifest_using_method(manifest, method, "modified.persisted_descriptions") def test_select_state_changed_seed_columns_documented_nodocs(manifest, previous_state, seed): seed_doc_columns = replace_config(seed, persist_docs={"columns": True}) seed_doc_columns_documented_columns = replace( seed_doc_columns, columns={"a": ColumnInfo(name="a", description="a description")}, ) change_node(previous_state.manifest, seed_doc_columns) change_node(manifest, seed_doc_columns_documented_columns) method = statemethod(manifest, previous_state) assert search_manifest_using_method(manifest, method, "modified") == {"seed"} assert search_manifest_using_method(manifest, method, "modified.persisted_descriptions") == { "seed" } assert "seed" in search_manifest_using_method(manifest, method, "old") assert not search_manifest_using_method(manifest, method, "new") assert "seed" not in search_manifest_using_method(manifest, method, "unmodified") assert not search_manifest_using_method(manifest, method, "modified.configs") def test_select_state_changed_seed_columns_documented_withdocs(manifest, previous_state, seed): seed_doc_columns = replace_config(seed, persist_docs={"columns": True}) seed_doc_columns_documented_columns = replace( seed_doc_columns, columns={"a": ColumnInfo(name="a", description="a description")}, ) change_node(manifest, seed_doc_columns) change_node(previous_state.manifest, seed_doc_columns_documented_columns) method = statemethod(manifest, previous_state) assert search_manifest_using_method(manifest, method, "modified") == {"seed"} assert search_manifest_using_method(manifest, method, "modified.persisted_descriptions") == { "seed" } assert "seed" in search_manifest_using_method(manifest, method, "old") assert not search_manifest_using_method(manifest, method, "new") assert "seed" not in search_manifest_using_method(manifest, method, "unmodified") assert not search_manifest_using_method(manifest, method, "modified.configs") def test_select_state_changed_test_macro_sql( manifest, previous_state, macro_default_test_not_null ): manifest.macros[macro_default_test_not_null.unique_id] = replace( macro_default_test_not_null, macro_sql="lalala" ) method = statemethod(manifest, previous_state) assert search_manifest_using_method(manifest, method, "modified") == { "not_null_table_model_id" } assert search_manifest_using_method(manifest, method, "modified.macros") == { "not_null_table_model_id" } assert "not_null_table_model_id" in search_manifest_using_method(manifest, method, "old") assert not search_manifest_using_method(manifest, method, "new") assert "not_null_table_model_id" not in search_manifest_using_method( manifest, method, "unmodified" ) def test_select_state_changed_test_macros(manifest, previous_state): changed_macro = make_macro("dbt", "changed_macro", "blablabla") add_macro(manifest, changed_macro) add_macro(previous_state.manifest, replace(changed_macro, macro_sql="something different")) unchanged_macro = make_macro("dbt", "unchanged_macro", "blablabla") add_macro(manifest, unchanged_macro) add_macro(previous_state.manifest, unchanged_macro) model1 = make_model( "dbt", "model1", "blablabla", depends_on_macros=[changed_macro.unique_id, unchanged_macro.unique_id], ) add_node(manifest, model1) add_node(previous_state.manifest, model1) model2 = make_model( "dbt", "model2", "blablabla", depends_on_macros=[unchanged_macro.unique_id, changed_macro.unique_id], ) add_node(manifest, model2) add_node(previous_state.manifest, model2) method = statemethod(manifest, previous_state) assert search_manifest_using_method(manifest, method, "modified") == {"model1", "model2"} assert search_manifest_using_method(manifest, method, "modified.macros") == { "model1", "model2", } assert "model1" and "model2" in search_manifest_using_method(manifest, method, "old") assert not search_manifest_using_method(manifest, method, "new") assert "model1" and "model2" not in search_manifest_using_method( manifest, method, "unmodified" ) def test_select_state_changed_test_macros_with_upstream_change(manifest, previous_state): changed_macro = make_macro("dbt", "changed_macro", "blablabla") add_macro(manifest, changed_macro) add_macro(previous_state.manifest, replace(changed_macro, macro_sql="something different")) unchanged_macro1 = make_macro("dbt", "unchanged_macro", "blablabla") add_macro(manifest, unchanged_macro1) add_macro(previous_state.manifest, unchanged_macro1) unchanged_macro2 = make_macro( "dbt", "unchanged_macro", "blablabla", depends_on_macros=[unchanged_macro1.unique_id, changed_macro.unique_id], ) add_macro(manifest, unchanged_macro2) add_macro(previous_state.manifest, unchanged_macro2) unchanged_macro3 = make_macro( "dbt", "unchanged_macro", "blablabla", depends_on_macros=[changed_macro.unique_id, unchanged_macro1.unique_id], ) add_macro(manifest, unchanged_macro3) add_macro(previous_state.manifest, unchanged_macro3) model1 = make_model( "dbt", "model1", "blablabla", depends_on_macros=[unchanged_macro1.unique_id] ) add_node(manifest, model1) add_node(previous_state.manifest, model1) model2 = make_model( "dbt", "model2", "blablabla", depends_on_macros=[unchanged_macro3.unique_id] ) add_node(manifest, model2) add_node(previous_state.manifest, model2) method = statemethod(manifest, previous_state) assert search_manifest_using_method(manifest, method, "modified") == {"model1", "model2"} assert search_manifest_using_method(manifest, method, "modified.macros") == { "model1", "model2", } assert "model1" and "model2" in search_manifest_using_method(manifest, method, "old") assert not search_manifest_using_method(manifest, method, "new") assert "model1" and "model2" not in search_manifest_using_method( manifest, method, "unmodified" ) ================================================ FILE: tests/unit/graph/test_selector_spec.py ================================================ import os from unittest.mock import patch import pytest from dbt.exceptions import DbtRuntimeError from dbt.graph.selector_methods import MethodName from dbt.graph.selector_spec import ( IndirectSelection, SelectionCriteria, SelectionDifference, SelectionIntersection, SelectionUnion, ) @pytest.mark.parametrize( "indirect_selection_value,expected_value", [(v, v) for v in IndirectSelection], ) def test_selection_criteria_default_indirect_value(indirect_selection_value, expected_value): # Check selection criteria with indirect selection value would follow the resolved value in flags # if indirect selection is not specified in the selection criteria. with patch("dbt.graph.selector_spec.get_flags") as patched_get_flags: patched_get_flags.return_value.INDIRECT_SELECTION = indirect_selection_value patched_get_flags.INDIRECT_SELECTION = indirect_selection_value selection_dict_without_indirect_selection_specified = { "method": "path", "value": "models/marts/orders.sql", "children": False, "parents": False, } selection_criteria_without_indirect_selection_specified = ( SelectionCriteria.selection_criteria_from_dict( selection_dict_without_indirect_selection_specified, selection_dict_without_indirect_selection_specified, ) ) assert ( selection_criteria_without_indirect_selection_specified.indirect_selection == expected_value ) selection_dict_without_indirect_selection_specified = { "method": "path", "value": "models/marts/orders.sql", "children": False, "parents": False, "indirect_selection": "buildable", } selection_criteria_with_indirect_selection_specified = ( SelectionCriteria.selection_criteria_from_dict( selection_dict_without_indirect_selection_specified, selection_dict_without_indirect_selection_specified, ) ) assert ( selection_criteria_with_indirect_selection_specified.indirect_selection == "buildable" ) def test_raw_parse_simple(): raw = "asdf" result = SelectionCriteria.from_single_spec(raw) assert result.raw == raw assert result.method == MethodName.FQN assert result.method_arguments == [] assert result.value == raw assert not result.childrens_parents assert not result.children assert not result.parents assert result.parents_depth is None assert result.children_depth is None def test_raw_parse_simple_infer_path(): raw = os.path.join("asdf", "*") result = SelectionCriteria.from_single_spec(raw) assert result.raw == raw assert result.method == MethodName.Path assert result.method_arguments == [] assert result.value == raw assert not result.childrens_parents assert not result.children assert not result.parents assert result.parents_depth is None assert result.children_depth is None def test_raw_parse_simple_infer_path_modified(): raw = "@" + os.path.join("asdf", "*") result = SelectionCriteria.from_single_spec(raw) assert result.raw == raw assert result.method == MethodName.Path assert result.method_arguments == [] assert result.value == raw[1:] assert result.childrens_parents assert not result.children assert not result.parents assert result.parents_depth is None assert result.children_depth is None def test_raw_parse_simple_infer_fqn_parents(): raw = "+asdf" result = SelectionCriteria.from_single_spec(raw) assert result.raw == raw assert result.method == MethodName.FQN assert result.method_arguments == [] assert result.value == "asdf" assert not result.childrens_parents assert not result.children assert result.parents assert result.parents_depth is None assert result.children_depth is None def test_raw_parse_simple_infer_fqn_children(): raw = "asdf+" result = SelectionCriteria.from_single_spec(raw) assert result.raw == raw assert result.method == MethodName.FQN assert result.method_arguments == [] assert result.value == "asdf" assert not result.childrens_parents assert result.children assert not result.parents assert result.parents_depth is None assert result.children_depth is None def test_raw_parse_complex(): raw = "2+config.arg.secondarg:argument_value+4" result = SelectionCriteria.from_single_spec(raw) assert result.raw == raw assert result.method == MethodName.Config assert result.method_arguments == ["arg", "secondarg"] assert result.value == "argument_value" assert not result.childrens_parents assert result.children assert result.parents assert result.parents_depth == 2 assert result.children_depth == 4 def test_raw_parse_weird(): # you can have an empty method name (defaults to FQN/path) and you can have # an empty value, so you can also have this... result = SelectionCriteria.from_single_spec("") assert result.raw == "" assert result.method == MethodName.FQN assert result.method_arguments == [] assert result.value == "" assert not result.childrens_parents assert not result.children assert not result.parents assert result.parents_depth is None assert result.children_depth is None def test_raw_parse_selector_method(): """selector:foo parses as method=Selector, value=foo (for combining YAML selectors with --select/--exclude).""" raw = "1+selector:staging+2" result = SelectionCriteria.from_single_spec(raw) assert result.raw == raw assert result.method == MethodName.Selector assert not result.childrens_parents assert result.children assert result.parents assert result.method_arguments == [] assert result.value == "staging" assert result.children_depth == 2 assert result.parents_depth == 1 def test_raw_parse_invalid(): with pytest.raises(DbtRuntimeError): SelectionCriteria.from_single_spec("invalid_method:something") with pytest.raises(DbtRuntimeError): SelectionCriteria.from_single_spec("@foo+") def test_intersection(): fqn_a = SelectionCriteria.from_single_spec("fqn:model_a") fqn_b = SelectionCriteria.from_single_spec("fqn:model_b") intersection = SelectionIntersection(components=[fqn_a, fqn_b]) assert list(intersection) == [fqn_a, fqn_b] combined = intersection.combine_selections( [{"model_a", "model_b", "model_c"}, {"model_c", "model_d"}] ) assert combined == {"model_c"} def test_difference(): fqn_a = SelectionCriteria.from_single_spec("fqn:model_a") fqn_b = SelectionCriteria.from_single_spec("fqn:model_b") difference = SelectionDifference(components=[fqn_a, fqn_b]) assert list(difference) == [fqn_a, fqn_b] combined = difference.combine_selections( [{"model_a", "model_b", "model_c"}, {"model_c", "model_d"}] ) assert combined == {"model_a", "model_b"} fqn_c = SelectionCriteria.from_single_spec("fqn:model_c") difference = SelectionDifference(components=[fqn_a, fqn_b, fqn_c]) assert list(difference) == [fqn_a, fqn_b, fqn_c] combined = difference.combine_selections( [{"model_a", "model_b", "model_c"}, {"model_c", "model_d"}, {"model_a"}] ) assert combined == {"model_b"} def test_union(): fqn_a = SelectionCriteria.from_single_spec("fqn:model_a") fqn_b = SelectionCriteria.from_single_spec("fqn:model_b") fqn_c = SelectionCriteria.from_single_spec("fqn:model_c") difference = SelectionUnion(components=[fqn_a, fqn_b, fqn_c]) combined = difference.combine_selections( [{"model_a", "model_b"}, {"model_b", "model_c"}, {"model_d"}] ) assert combined == {"model_a", "model_b", "model_c", "model_d"} ================================================ FILE: tests/unit/materializations/incremental/test_microbatch.py ================================================ from datetime import datetime from unittest import mock import pytest import pytz from freezegun import freeze_time from dbt.artifacts.resources import NodeConfig from dbt.artifacts.resources.types import BatchSize from dbt.materializations.incremental.microbatch import MicrobatchBuilder MODEL_CONFIG_BEGIN = datetime(2024, 1, 1, 0, 0, 0, 0, pytz.UTC) class TestMicrobatchBuilder: @pytest.fixture(scope="class") def microbatch_model(self): model = mock.Mock() model.config = mock.MagicMock(NodeConfig) model.config.materialized = "incremental" model.config.incremental_strategy = "microbatch" model.config.begin = MODEL_CONFIG_BEGIN model.config.batch_size = BatchSize.day return model @freeze_time("2024-09-05 08:56:00") @pytest.mark.parametrize( "is_incremental,event_time_end,expected_end_time", [ ( False, None, datetime(2024, 9, 6, 0, 0, 0, 0, pytz.UTC), ), ( True, None, datetime(2024, 9, 6, 0, 0, 0, 0, pytz.UTC), ), ( False, datetime(2024, 10, 1, 0, 0, 0, 0, pytz.UTC), datetime(2024, 10, 1, 0, 0, 0, 0, pytz.UTC), ), ( True, datetime(2024, 10, 1, 0, 0, 0, 0, pytz.UTC), datetime(2024, 10, 1, 0, 0, 0, 0, pytz.UTC), ), ], ) def test_build_end_time( self, microbatch_model, is_incremental, event_time_end, expected_end_time ): microbatch_builder = MicrobatchBuilder( model=microbatch_model, is_incremental=is_incremental, event_time_start=None, event_time_end=event_time_end, ) assert microbatch_builder.build_end_time() == expected_end_time @pytest.mark.parametrize( "is_incremental,event_time_start,checkpoint,batch_size,lookback,expected_start_time", [ ( False, None, datetime(2024, 9, 5, 8, 56, 0, 0, pytz.UTC), BatchSize.day, 0, # is_incremental: False => model.config.begin MODEL_CONFIG_BEGIN, ), # BatchSize.year ( False, datetime(2024, 9, 5, 8, 56, 0, 0, pytz.UTC), datetime(2024, 9, 5, 8, 56, 0, 0, pytz.UTC), BatchSize.year, 0, datetime(2024, 1, 1, 0, 0, 0, 0, pytz.UTC), ), ( False, datetime(2024, 9, 5, 8, 56, 0, 0, pytz.UTC), datetime(2024, 9, 5, 8, 56, 0, 0, pytz.UTC), BatchSize.year, # Offset not applied when event_time_start provided 1, datetime(2024, 1, 1, 0, 0, 0, 0, pytz.UTC), ), ( False, None, datetime(2024, 9, 5, 8, 56, 0, 0, pytz.UTC), BatchSize.year, 0, # is_incremental=False + no start_time -> model.config.begin MODEL_CONFIG_BEGIN, ), ( True, None, datetime(2024, 9, 5, 8, 56, 0, 0, pytz.UTC), BatchSize.year, 0, datetime(2024, 1, 1, 0, 0, 0, 0, pytz.UTC), ), ( True, None, datetime(2024, 9, 5, 8, 56, 0, 0, pytz.UTC), BatchSize.year, 1, datetime(2023, 1, 1, 0, 0, 0, 0, pytz.UTC), ), # BatchSize.month ( False, datetime(2024, 9, 5, 8, 56, 0, 0, pytz.UTC), datetime(2024, 9, 5, 8, 56, 0, 0, pytz.UTC), BatchSize.month, 0, datetime(2024, 9, 1, 0, 0, 0, 0, pytz.UTC), ), ( False, datetime(2024, 9, 5, 8, 56, 0, 0, pytz.UTC), datetime(2024, 9, 5, 8, 56, 0, 0, pytz.UTC), BatchSize.month, # Offset not applied when event_time_start provided 1, datetime(2024, 9, 1, 0, 0, 0, 0, pytz.UTC), ), ( False, None, datetime(2024, 9, 5, 8, 56, 0, 0, pytz.UTC), BatchSize.month, 0, # is_incremental=False + no start_time -> model.config.begin MODEL_CONFIG_BEGIN, ), ( True, None, datetime(2024, 9, 5, 8, 56, 0, 0, pytz.UTC), BatchSize.month, 0, datetime(2024, 9, 1, 0, 0, 0, 0, pytz.UTC), ), ( True, None, datetime(2024, 9, 5, 8, 56, 0, 0, pytz.UTC), BatchSize.month, 1, datetime(2024, 8, 1, 0, 0, 0, 0, pytz.UTC), ), # BatchSize.day ( False, datetime(2024, 9, 5, 8, 56, 0, 0, pytz.UTC), datetime(2024, 9, 5, 8, 56, 0, 0, pytz.UTC), BatchSize.day, 0, datetime(2024, 9, 5, 0, 0, 0, 0, pytz.UTC), ), ( False, datetime(2024, 9, 5, 8, 56, 0, 0, pytz.UTC), datetime(2024, 9, 5, 8, 56, 0, 0, pytz.UTC), BatchSize.day, # Offset not applied when event_time_start provided 1, datetime(2024, 9, 5, 0, 0, 0, 0, pytz.UTC), ), ( False, None, datetime(2024, 9, 5, 8, 56, 0, 0, pytz.UTC), BatchSize.day, 0, # is_incremental=False + no start_time -> model.config.begin MODEL_CONFIG_BEGIN, ), ( True, None, datetime(2024, 9, 5, 8, 56, 0, 0, pytz.UTC), BatchSize.day, 0, datetime(2024, 9, 5, 0, 0, 0, 0, pytz.UTC), ), ( True, None, datetime(2024, 9, 5, 8, 56, 0, 0, pytz.UTC), BatchSize.day, 1, datetime(2024, 9, 4, 0, 0, 0, 0, pytz.UTC), ), # BatchSize.hour ( False, datetime(2024, 9, 5, 8, 56, 0, 0, pytz.UTC), datetime(2024, 9, 5, 8, 56, 0, 0, pytz.UTC), BatchSize.hour, 0, datetime(2024, 9, 5, 8, 0, 0, 0, pytz.UTC), ), ( False, datetime(2024, 9, 5, 8, 56, 0, 0, pytz.UTC), datetime(2024, 9, 5, 8, 56, 0, 0, pytz.UTC), BatchSize.hour, # Offset not applied when event_time_start provided 1, datetime(2024, 9, 5, 8, 0, 0, 0, pytz.UTC), ), ( False, None, datetime(2024, 9, 5, 8, 56, 0, 0, pytz.UTC), BatchSize.hour, 0, # is_incremental=False + no start_time -> model.config.begin MODEL_CONFIG_BEGIN, ), ( True, None, datetime(2024, 9, 5, 8, 56, 0, 0, pytz.UTC), BatchSize.hour, 0, datetime(2024, 9, 5, 8, 0, 0, 0, pytz.UTC), ), ( True, None, datetime(2024, 9, 5, 8, 56, 0, 0, pytz.UTC), BatchSize.hour, 1, datetime(2024, 9, 5, 7, 0, 0, 0, pytz.UTC), ), ( True, None, datetime(2024, 9, 5, 0, 0, 0, 0, pytz.UTC), BatchSize.hour, 0, datetime(2024, 9, 4, 23, 0, 0, 0, pytz.UTC), ), ( True, None, datetime(2024, 9, 5, 0, 0, 0, 0, pytz.UTC), BatchSize.hour, 1, datetime(2024, 9, 4, 22, 0, 0, 0, pytz.UTC), ), ( True, None, datetime(2024, 9, 5, 0, 0, 0, 0, pytz.UTC), BatchSize.day, 0, datetime(2024, 9, 4, 0, 0, 0, 0, pytz.UTC), ), ( True, None, datetime(2024, 9, 5, 0, 0, 0, 0, pytz.UTC), BatchSize.day, 1, datetime(2024, 9, 3, 0, 0, 0, 0, pytz.UTC), ), ( True, None, datetime(2024, 9, 1, 0, 0, 0, 0, pytz.UTC), BatchSize.month, 0, datetime(2024, 8, 1, 0, 0, 0, 0, pytz.UTC), ), ( True, None, datetime(2024, 9, 1, 0, 0, 0, 0, pytz.UTC), BatchSize.month, 1, datetime(2024, 7, 1, 0, 0, 0, 0, pytz.UTC), ), ( True, None, datetime(2024, 1, 1, 0, 0, 0, 0, pytz.UTC), BatchSize.year, 0, datetime(2023, 1, 1, 0, 0, 0, 0, pytz.UTC), ), ( True, None, datetime(2024, 1, 1, 0, 0, 0, 0, pytz.UTC), BatchSize.year, 1, datetime(2022, 1, 1, 0, 0, 0, 0, pytz.UTC), ), ], ) def test_build_start_time( self, microbatch_model, is_incremental, event_time_start, checkpoint, batch_size, lookback, expected_start_time, ): microbatch_model.config.batch_size = batch_size microbatch_model.config.lookback = lookback microbatch_builder = MicrobatchBuilder( model=microbatch_model, is_incremental=is_incremental, event_time_start=event_time_start, event_time_end=None, ) assert microbatch_builder.build_start_time(checkpoint) == expected_start_time @pytest.mark.parametrize( "start,end,batch_size,expected_batches", [ # BatchSize.year ( datetime(2024, 1, 1, 0, 0, 0, 0, pytz.UTC), datetime(2026, 1, 7, 3, 56, 0, 0, pytz.UTC), BatchSize.year, [ ( datetime(2024, 1, 1, 0, 0, 0, 0, pytz.UTC), datetime(2025, 1, 1, 0, 0, 0, 0, pytz.UTC), ), ( datetime(2025, 1, 1, 0, 0, 0, 0, pytz.UTC), datetime(2026, 1, 1, 0, 0, 0, 0, pytz.UTC), ), ( datetime(2026, 1, 1, 0, 0, 0, 0, pytz.UTC), datetime(2026, 1, 7, 3, 56, 0, 0, pytz.UTC), ), ], ), # BatchSize.month ( datetime(2024, 9, 1, 0, 0, 0, 0, pytz.UTC), datetime(2024, 11, 7, 3, 56, 0, 0, pytz.UTC), BatchSize.month, [ ( datetime(2024, 9, 1, 0, 0, 0, 0, pytz.UTC), datetime(2024, 10, 1, 0, 0, 0, 0, pytz.UTC), ), ( datetime(2024, 10, 1, 0, 0, 0, 0, pytz.UTC), datetime(2024, 11, 1, 0, 0, 0, 0, pytz.UTC), ), ( datetime(2024, 11, 1, 0, 0, 0, 0, pytz.UTC), datetime(2024, 11, 7, 3, 56, 0, 0, pytz.UTC), ), ], ), # BatchSize.day ( datetime(2024, 9, 5, 0, 0, 0, 0, pytz.UTC), datetime(2024, 9, 7, 3, 56, 0, 0, pytz.UTC), BatchSize.day, [ ( datetime(2024, 9, 5, 0, 0, 0, 0, pytz.UTC), datetime(2024, 9, 6, 0, 0, 0, 0, pytz.UTC), ), ( datetime(2024, 9, 6, 0, 0, 0, 0, pytz.UTC), datetime(2024, 9, 7, 0, 0, 0, 0, pytz.UTC), ), ( datetime(2024, 9, 7, 0, 0, 0, 0, pytz.UTC), datetime(2024, 9, 7, 3, 56, 0, 0, pytz.UTC), ), ], ), # BatchSize.hour ( datetime(2024, 9, 5, 1, 0, 0, 0, pytz.UTC), datetime(2024, 9, 5, 3, 56, 0, 0, pytz.UTC), BatchSize.hour, [ ( datetime(2024, 9, 5, 1, 0, 0, 0, pytz.UTC), datetime(2024, 9, 5, 2, 0, 0, 0, pytz.UTC), ), ( datetime(2024, 9, 5, 2, 0, 0, 0, pytz.UTC), datetime(2024, 9, 5, 3, 0, 0, 0, pytz.UTC), ), ( datetime(2024, 9, 5, 3, 0, 0, 0, pytz.UTC), datetime(2024, 9, 5, 3, 56, 0, 0, pytz.UTC), ), ], ), # Test when event_time_end matches the truncated batch size ( datetime(2024, 1, 1, 0, 0, 0, 0, pytz.UTC), datetime(2026, 1, 1, 0, 0, 0, 0, pytz.UTC), BatchSize.year, [ ( datetime(2024, 1, 1, 0, 0, 0, 0, pytz.UTC), datetime(2025, 1, 1, 0, 0, 0, 0, pytz.UTC), ), ( datetime(2025, 1, 1, 0, 0, 0, 0, pytz.UTC), datetime(2026, 1, 1, 0, 0, 0, 0, pytz.UTC), ), ], ), ( datetime(2024, 9, 1, 0, 0, 0, 0, pytz.UTC), datetime(2024, 11, 1, 0, 0, 0, 0, pytz.UTC), BatchSize.month, [ ( datetime(2024, 9, 1, 0, 0, 0, 0, pytz.UTC), datetime(2024, 10, 1, 0, 0, 0, 0, pytz.UTC), ), ( datetime(2024, 10, 1, 0, 0, 0, 0, pytz.UTC), datetime(2024, 11, 1, 0, 0, 0, 0, pytz.UTC), ), ], ), ( datetime(2024, 9, 5, 0, 0, 0, 0, pytz.UTC), datetime(2024, 9, 7, 0, 0, 0, 0, pytz.UTC), BatchSize.day, [ ( datetime(2024, 9, 5, 0, 0, 0, 0, pytz.UTC), datetime(2024, 9, 6, 0, 0, 0, 0, pytz.UTC), ), ( datetime(2024, 9, 6, 0, 0, 0, 0, pytz.UTC), datetime(2024, 9, 7, 0, 0, 0, 0, pytz.UTC), ), ], ), ( datetime(2024, 9, 5, 1, 0, 0, 0, pytz.UTC), datetime(2024, 9, 5, 3, 0, 0, 0, pytz.UTC), BatchSize.hour, [ ( datetime(2024, 9, 5, 1, 0, 0, 0, pytz.UTC), datetime(2024, 9, 5, 2, 0, 0, 0, pytz.UTC), ), ( datetime(2024, 9, 5, 2, 0, 0, 0, pytz.UTC), datetime(2024, 9, 5, 3, 0, 0, 0, pytz.UTC), ), ], ), ], ) def test_build_batches(self, microbatch_model, start, end, batch_size, expected_batches): microbatch_model.config.batch_size = batch_size microbatch_builder = MicrobatchBuilder( model=microbatch_model, is_incremental=True, event_time_start=None, event_time_end=None ) actual_batches = microbatch_builder.build_batches(start, end) assert len(actual_batches) == len(expected_batches) assert actual_batches == expected_batches def test_build_jinja_context_for_incremental_batch(self, microbatch_model): context = MicrobatchBuilder.build_jinja_context_for_batch( model=microbatch_model, incremental_batch=True, ) assert context["model"] == microbatch_model.to_dict() assert context["sql"] == microbatch_model.compiled_code assert context["compiled_code"] == microbatch_model.compiled_code assert context["is_incremental"]() is True assert context["should_full_refresh"]() is False def test_build_jinja_context_for_incremental_batch_false(self, microbatch_model): context = MicrobatchBuilder.build_jinja_context_for_batch( model=microbatch_model, incremental_batch=False, ) assert context["model"] == microbatch_model.to_dict() assert context["sql"] == microbatch_model.compiled_code assert context["compiled_code"] == microbatch_model.compiled_code # Only build is_incremental callables when not first batch assert "is_incremental" not in context assert "should_full_refresh" not in context @pytest.mark.parametrize( "timestamp,batch_size,offset,expected_timestamp", [ ( datetime(2024, 9, 5, 3, 56, 1, 1, pytz.UTC), BatchSize.year, 1, datetime(2025, 1, 1, 0, 0, 0, 0, pytz.UTC), ), ( datetime(2024, 9, 5, 3, 56, 1, 1, pytz.UTC), BatchSize.year, -1, datetime(2023, 1, 1, 0, 0, 0, 0, pytz.UTC), ), ( datetime(2024, 9, 5, 3, 56, 1, 1, pytz.UTC), BatchSize.month, 1, datetime(2024, 10, 1, 0, 0, 0, 0, pytz.UTC), ), ( datetime(2024, 9, 5, 3, 56, 1, 1, pytz.UTC), BatchSize.month, -1, datetime(2024, 8, 1, 0, 0, 0, 0, pytz.UTC), ), ( datetime(2024, 9, 5, 3, 56, 1, 1, pytz.UTC), BatchSize.day, 1, datetime(2024, 9, 6, 0, 0, 0, 0, pytz.UTC), ), ( datetime(2024, 9, 5, 3, 56, 1, 1, pytz.UTC), BatchSize.day, -1, datetime(2024, 9, 4, 0, 0, 0, 0, pytz.UTC), ), ( datetime(2024, 9, 5, 3, 56, 1, 1, pytz.UTC), BatchSize.hour, 1, datetime(2024, 9, 5, 4, 0, 0, 0, pytz.UTC), ), ( datetime(2024, 9, 5, 3, 56, 1, 1, pytz.UTC), BatchSize.hour, -1, datetime(2024, 9, 5, 2, 0, 0, 0, pytz.UTC), ), ], ) def test_offset_timestamp(self, timestamp, batch_size, offset, expected_timestamp): assert ( MicrobatchBuilder.offset_timestamp(timestamp, batch_size, offset) == expected_timestamp ) @pytest.mark.parametrize( "timestamp,batch_size,expected_timestamp", [ ( datetime(2024, 9, 5, 3, 56, 1, 1, pytz.UTC), BatchSize.year, datetime(2024, 1, 1, 0, 0, 0, 0, pytz.UTC), ), ( datetime(2024, 9, 5, 3, 56, 1, 1, pytz.UTC), BatchSize.month, datetime(2024, 9, 1, 0, 0, 0, 0, pytz.UTC), ), ( datetime(2024, 9, 5, 3, 56, 1, 1, pytz.UTC), BatchSize.day, datetime(2024, 9, 5, 0, 0, 0, 0, pytz.UTC), ), ( datetime(2024, 9, 5, 3, 56, 1, 1, pytz.UTC), BatchSize.hour, datetime(2024, 9, 5, 3, 0, 0, 0, pytz.UTC), ), ], ) def test_truncate_timestamp(self, timestamp, batch_size, expected_timestamp): assert MicrobatchBuilder.truncate_timestamp(timestamp, batch_size) == expected_timestamp @pytest.mark.parametrize( "batch_size,start_time,expected_formatted_start_time", [ (BatchSize.year, datetime(2020, 1, 1, 1), "2020"), (BatchSize.month, datetime(2020, 1, 1, 1), "202001"), (BatchSize.day, datetime(2020, 1, 1, 1), "20200101"), (BatchSize.hour, datetime(2020, 1, 1, 1), "20200101T01"), ], ) def test_batch_id( self, batch_size: BatchSize, start_time: datetime, expected_formatted_start_time: str ) -> None: assert MicrobatchBuilder.batch_id(start_time, batch_size) == expected_formatted_start_time @pytest.mark.parametrize( "batch_size,batch_start,expected_formatted_batch_start", [ (BatchSize.year, datetime(2020, 1, 1, 1), "2020"), (BatchSize.month, datetime(2020, 1, 1, 1), "2020-01"), (BatchSize.day, datetime(2020, 1, 1, 1), "2020-01-01"), (BatchSize.hour, datetime(2020, 1, 1, 1), "2020-01-01T01"), ], ) def test_format_batch_start( self, batch_size: BatchSize, batch_start: datetime, expected_formatted_batch_start: str ) -> None: assert ( MicrobatchBuilder.format_batch_start(batch_start, batch_size) == expected_formatted_batch_start ) @pytest.mark.parametrize( "timestamp,batch_size,expected_datetime", [ ( datetime(2024, 9, 17, 16, 6, 0, 0, pytz.UTC), BatchSize.hour, datetime(2024, 9, 17, 17, 0, 0, 0, pytz.UTC), ), ( datetime(2024, 9, 17, 16, 0, 0, 0, pytz.UTC), BatchSize.hour, datetime(2024, 9, 17, 16, 0, 0, 0, pytz.UTC), ), ( datetime(2024, 9, 17, 16, 6, 0, 0, pytz.UTC), BatchSize.day, datetime(2024, 9, 18, 0, 0, 0, 0, pytz.UTC), ), ( datetime(2024, 9, 17, 0, 0, 0, 0, pytz.UTC), BatchSize.day, datetime(2024, 9, 17, 0, 0, 0, 0, pytz.UTC), ), ( datetime(2024, 9, 17, 16, 6, 0, 0, pytz.UTC), BatchSize.month, datetime(2024, 10, 1, 0, 0, 0, 0, pytz.UTC), ), ( datetime(2024, 9, 1, 0, 0, 0, 0, pytz.UTC), BatchSize.month, datetime(2024, 9, 1, 0, 0, 0, 0, pytz.UTC), ), ( datetime(2024, 9, 17, 16, 6, 0, 0, pytz.UTC), BatchSize.year, datetime(2025, 1, 1, 0, 0, 0, 0, pytz.UTC), ), ( datetime(2024, 1, 1, 0, 0, 0, 0, pytz.UTC), BatchSize.year, datetime(2024, 1, 1, 0, 0, 0, 0, pytz.UTC), ), ], ) def test_ceiling_timestamp( self, timestamp: datetime, batch_size: BatchSize, expected_datetime: datetime ) -> None: ceilinged = MicrobatchBuilder.ceiling_timestamp(timestamp, batch_size) assert ceilinged == expected_datetime ================================================ FILE: tests/unit/mock_adapter.py ================================================ from contextlib import contextmanager from unittest import mock from dbt.adapters.base import BaseAdapter def adapter_factory(): class MockAdapter(BaseAdapter): ConnectionManager = mock.MagicMock(TYPE="mock") responder = mock.MagicMock() # some convenient defaults responder.quote.side_effect = lambda identifier: '"{}"'.format(identifier) responder.date_function.side_effect = lambda: "unitdate()" responder.is_cancelable.side_effect = lambda: False @contextmanager def exception_handler(self, *args, **kwargs): self.responder.exception_handler(*args, **kwargs) yield def execute(self, *args, **kwargs): return self.responder.execute(*args, **kwargs) def drop_relation(self, *args, **kwargs): return self.responder.drop_relation(*args, **kwargs) def truncate_relation(self, *args, **kwargs): return self.responder.truncate_relation(*args, **kwargs) def rename_relation(self, *args, **kwargs): return self.responder.rename_relation(*args, **kwargs) def get_columns_in_relation(self, *args, **kwargs): return self.responder.get_columns_in_relation(*args, **kwargs) def get_catalog_for_single_relation(self, *args, **kwargs): return self.responder.get_catalog_for_single_relation(*args, **kwargs) def expand_column_types(self, *args, **kwargs): return self.responder.expand_column_types(*args, **kwargs) def list_relations_without_caching(self, *args, **kwargs): return self.responder.list_relations_without_caching(*args, **kwargs) def create_schema(self, *args, **kwargs): return self.responder.create_schema(*args, **kwargs) def drop_schema(self, *args, **kwargs): return self.responder.drop_schema(*args, **kwargs) @classmethod def quote(cls, identifier): return cls.responder.quote(identifier) def convert_text_type(self, *args, **kwargs): return self.responder.convert_text_type(*args, **kwargs) def convert_number_type(self, *args, **kwargs): return self.responder.convert_number_type(*args, **kwargs) def convert_integer_type(self, *args, **kwargs): return self.responder.convert_integer_type(*args, **kwargs) def convert_boolean_type(self, *args, **kwargs): return self.responder.convert_boolean_type(*args, **kwargs) def convert_datetime_type(self, *args, **kwargs): return self.responder.convert_datetime_type(*args, **kwargs) def convert_date_type(self, *args, **kwargs): return self.responder.convert_date_type(*args, **kwargs) def convert_time_type(self, *args, **kwargs): return self.responder.convert_time_type(*args, **kwargs) def list_schemas(self, *args, **kwargs): return self.responder.list_schemas(*args, **kwargs) @classmethod def date_function(cls): return cls.responder.date_function() @classmethod def is_cancelable(cls): return cls.responder.is_cancelable() return MockAdapter ================================================ FILE: tests/unit/parser/__init__.py ================================================ ================================================ FILE: tests/unit/parser/test_docs.py ================================================ import os import unittest from argparse import Namespace from dbt.contracts.files import FileHash, FilePath, SourceFile from dbt.contracts.graph.manifest import Manifest from dbt.contracts.graph.nodes import Documentation from dbt.flags import set_from_args from dbt.node_types import NodeType from dbt.parser import docs from dbt.parser.search import FileBlock from tests.unit.utils import config_from_parts_or_dicts set_from_args(Namespace(WARN_ERROR=False), None) SNOWPLOW_SESSIONS_DOCS = r""" This table contains one record for every session recorded by Snowplow. A session is itself comprised of pageviews that all occur within 30 minutes of each other. If more than 30 minutes elapse between pageviews, then a new session is created. Given the following pageviews: | session_id | page_view_id | page_title | | ---------- | ------------ | ---------- | | abc | 123 | Home | | abc | 456 | About | | abc | 789 | Home | The following sessions will be created: | session_id | first_page_title | count_pageviews | | ---------- | ---------------- | --------------- | | abc | 123 | 2 | | abc | 789 | 1 | """ SNOWPLOW_SESSIONS_SESSION_ID_DOCS = r""" This column is the unique identifier for a Snowplow session. It is generated by a cookie then expires after 30 minutes of inactivity. """ SNOWPLOW_SESSIONS_BLOCK = r""" {{% docs snowplow_sessions %}} {snowplow_sessions_docs} {{% enddocs %}} """.format( snowplow_sessions_docs=SNOWPLOW_SESSIONS_DOCS ).strip() SNOWPLOW_SESSIONS_SESSION_ID_BLOCK = r""" {{% docs snowplow_sessions__session_id %}} {snowplow_sessions_session_id_docs} {{% enddocs %}} """.format( snowplow_sessions_session_id_docs=SNOWPLOW_SESSIONS_SESSION_ID_DOCS ).strip() TEST_DOCUMENTATION_FILE = r""" {sessions_block} {session_id_block} """.format( sessions_block=SNOWPLOW_SESSIONS_BLOCK, session_id_block=SNOWPLOW_SESSIONS_SESSION_ID_BLOCK, ) MULTIPLE_RAW_BLOCKS = r""" {% docs some_doc %} {% raw %} ``` {% docs %}some doc{% enddocs %} ``` {% endraw %} {% enddocs %} {% docs other_doc %} {% raw %} ``` {% docs %}other doc{% enddocs %} ``` {% endraw %} {% enddocs %} """ class DocumentationParserTest(unittest.TestCase): def setUp(self): if os.name == "nt": self.root_path = "C:\\test_root" self.subdir_path = "C:\\test_root\\test_subdir" self.testfile_path = "C:\\test_root\\test_subdir\\test_file.md" else: self.root_path = "/test_root" self.subdir_path = "/test_root/test_subdir" self.testfile_path = "/test_root/test_subdir/test_file.md" profile_data = { "outputs": { "test": { "type": "postgres", "host": "localhost", "schema": "analytics", "user": "test", "pass": "test", "dbname": "test", "port": 1, } }, "target": "test", } root_project = { "name": "root", "version": "0.1", "profile": "test", "project-root": self.root_path, "config-version": 2, } subdir_project = { "name": "some_package", "version": "0.1", "profile": "test", "project-root": self.subdir_path, "quoting": {}, "config-version": 2, } self.root_project_config = config_from_parts_or_dicts( project=root_project, profile=profile_data ) self.subdir_project_config = config_from_parts_or_dicts( project=subdir_project, profile=profile_data ) def _build_file(self, contents, relative_path) -> FileBlock: match = FilePath( relative_path=relative_path, project_root=self.root_path, searched_path=self.subdir_path, modification_time=0.0, ) source_file = SourceFile(path=match, checksum=FileHash.empty()) source_file.contents = contents return FileBlock(file=source_file) def test_load_file(self): parser = docs.DocumentationParser( root_project=self.root_project_config, manifest=Manifest(), project=self.subdir_project_config, ) file_block = self._build_file(TEST_DOCUMENTATION_FILE, "test_file.md") parser.parse_file(file_block) docs_values = sorted(parser.manifest.docs.values(), key=lambda n: n.name) self.assertEqual(len(docs_values), 2) for result in docs_values: self.assertIsInstance(result, Documentation) self.assertEqual(result.package_name, "some_package") self.assertEqual(result.original_file_path, self.testfile_path) self.assertEqual(result.resource_type, NodeType.Documentation) self.assertEqual(result.path, "test_file.md") self.assertEqual(docs_values[0].name, "snowplow_sessions") self.assertEqual(docs_values[1].name, "snowplow_sessions__session_id") def test_load_file_extras(self): TEST_DOCUMENTATION_FILE + "{% model foo %}select 1 as id{% endmodel %}" parser = docs.DocumentationParser( root_project=self.root_project_config, manifest=Manifest(), project=self.subdir_project_config, ) file_block = self._build_file(TEST_DOCUMENTATION_FILE, "test_file.md") parser.parse_file(file_block) docs_values = sorted(parser.manifest.docs.values(), key=lambda n: n.name) self.assertEqual(len(docs_values), 2) for result in docs_values: self.assertIsInstance(result, Documentation) self.assertEqual(docs_values[0].name, "snowplow_sessions") self.assertEqual(docs_values[1].name, "snowplow_sessions__session_id") def test_multiple_raw_blocks(self): parser = docs.DocumentationParser( root_project=self.root_project_config, manifest=Manifest(), project=self.subdir_project_config, ) file_block = self._build_file(MULTIPLE_RAW_BLOCKS, "test_file.md") parser.parse_file(file_block) docs_values = sorted(parser.manifest.docs.values(), key=lambda n: n.name) self.assertEqual(len(docs_values), 2) for result in docs_values: self.assertIsInstance(result, Documentation) self.assertEqual(result.package_name, "some_package") self.assertEqual(result.original_file_path, self.testfile_path) self.assertEqual(result.resource_type, NodeType.Documentation) self.assertEqual(result.path, "test_file.md") self.assertEqual(docs_values[0].name, "other_doc") self.assertEqual( docs_values[0].block_contents, "```\n {% docs %}other doc{% enddocs %}\n ```" ) self.assertEqual(docs_values[1].name, "some_doc") self.assertEqual( docs_values[1].block_contents, "```\n {% docs %}some doc{% enddocs %}\n ```", ) ================================================ FILE: tests/unit/parser/test_get_doc_blocks.py ================================================ from unittest.mock import MagicMock from dbt.parser.manifest import _get_doc_blocks class TestGetDocBlocks: def test_name_node_arg_does_not_raise(self): """doc() with a variable reference (Name node, no .value) must not raise AttributeError.""" manifest = MagicMock() # {{ doc(my_variable) }} produces a jinja2.nodes.Name arg, not a Const result = _get_doc_blocks("{{ doc(my_variable) }}", manifest, "my_package") assert result == [] def test_const_arg_resolves_doc_block(self): """doc() with a string literal still resolves to a doc_block unique_id.""" manifest = MagicMock() manifest.metadata.project_name = "my_project" resolved = MagicMock() resolved.unique_id = "doc.my_project.my_doc" manifest.resolve_doc.return_value = resolved result = _get_doc_blocks("{{ doc('my_doc') }}", manifest, "my_package") assert result == ["doc.my_project.my_doc"] manifest.resolve_doc.assert_called_once_with("my_doc", None, "my_project", "my_package") def test_mixed_args_skips_name_nodes(self): """When one arg is a Name node and others are Const, only Const values are used.""" manifest = MagicMock() manifest.metadata.project_name = "my_project" manifest.resolve_doc.return_value = None # doc(my_var) — one Name arg → doc_args becomes [] → falls through to else: continue result = _get_doc_blocks("{{ doc(my_var) }}", manifest, "my_package") assert result == [] manifest.resolve_doc.assert_not_called() ================================================ FILE: tests/unit/parser/test_manifest.py ================================================ from argparse import Namespace from typing import Optional from unittest.mock import MagicMock, patch import jinja2 import msgpack import pytest from pytest_mock import MockerFixture from dbt.adapters.postgres import PostgresAdapter from dbt.artifacts.resources.base import FileHash from dbt.artifacts.resources.v1.semantic_model import NodeRelation from dbt.config import RuntimeConfig from dbt.contracts.graph.manifest import Manifest, ManifestStateCheck from dbt.events.types import InvalidConcurrentBatchesConfig, UnusedResourceConfigPath from dbt.flags import set_from_args from dbt.parser.manifest import ( ManifestLoader, _warn_for_unused_resource_config_paths, extended_mashumaro_encoder, extended_msgpack_encoder, ) from dbt.parser.read_files import FileDiff from dbt.tracking import User from dbt_common.events.event_catcher import EventCatcher from dbt_common.events.event_manager_client import add_callback_to_manager from tests.unit.fixtures import model_node class TestPartialParse: @patch("dbt.parser.manifest.ManifestLoader.build_manifest_state_check") @patch("dbt.parser.manifest.os.path.exists") @patch("dbt.parser.manifest.open") def test_partial_parse_file_path(self, patched_open, patched_os_exist, patched_state_check): mock_project = MagicMock(RuntimeConfig) mock_project.project_target_path = "mock_target_path" patched_os_exist.return_value = True ManifestLoader(mock_project, {}) # by default we use the project_target_path patched_open.assert_called_with("mock_target_path/partial_parse.msgpack", "rb") set_from_args(Namespace(partial_parse_file_path="specified_partial_parse_path"), {}) ManifestLoader(mock_project, {}) # if specified in flags, we use the specified path patched_open.assert_called_with("specified_partial_parse_path", "rb") def test_profile_hash_change(self, mock_project): # This test validate that the profile_hash is updated when the connection keys change profile_hash = "750bc99c1d64ca518536ead26b28465a224be5ffc918bf2a490102faa5a1bcf5" mock_project.credentials.connection_info.return_value = "test" manifest = ManifestLoader(mock_project, {}) assert manifest.manifest.state_check.profile_hash.checksum == profile_hash mock_project.credentials.connection_info.return_value = "test1" manifest = ManifestLoader(mock_project, {}) assert manifest.manifest.state_check.profile_hash.checksum != profile_hash @patch("dbt.parser.manifest.ManifestLoader.build_manifest_state_check") @patch("dbt.parser.manifest.os.path.exists") @patch("dbt.parser.manifest.open") def test_partial_parse_by_version( self, patched_open, patched_os_exist, patched_state_check, runtime_config: RuntimeConfig, manifest: Manifest, ): file_hash = FileHash.from_contents("test contests") manifest.state_check = ManifestStateCheck( vars_hash=file_hash, profile_hash=file_hash, profile_env_vars_hash=file_hash, project_env_vars_hash=file_hash, ) # we need a loader to compare the two manifests loader = ManifestLoader(runtime_config, {runtime_config.project_name: runtime_config}) loader.manifest = manifest.deepcopy() is_partial_parsable, _ = loader.is_partial_parsable(manifest) assert is_partial_parsable manifest.metadata.dbt_version = "0.0.1a1" is_partial_parsable, _ = loader.is_partial_parsable(manifest) assert not is_partial_parsable manifest.metadata.dbt_version = "99999.99.99" is_partial_parsable, _ = loader.is_partial_parsable(manifest) assert not is_partial_parsable class TestFailedPartialParse: @patch("dbt.tracking.track_partial_parser") @patch("dbt.tracking.active_user") @patch("dbt.parser.manifest.PartialParsing") @patch("dbt.parser.manifest.ManifestLoader.read_manifest_for_partial_parse") @patch("dbt.parser.manifest.ManifestLoader.build_manifest_state_check") def test_partial_parse_safe_update_project_parser_files_partially( self, patched_state_check, patched_read_manifest_for_partial_parse, patched_partial_parsing, patched_active_user, patched_track_partial_parser, ): mock_instance = MagicMock() mock_instance.skip_parsing.return_value = False mock_instance.get_parsing_files.side_effect = KeyError("Whoopsie!") patched_partial_parsing.return_value = mock_instance mock_project = MagicMock(RuntimeConfig) mock_project.project_target_path = "mock_target_path" mock_saved_manifest = MagicMock(Manifest) mock_saved_manifest.files = {} patched_read_manifest_for_partial_parse.return_value = mock_saved_manifest loader = ManifestLoader(mock_project, {}) loader.safe_update_project_parser_files_partially({}) patched_track_partial_parser.assert_called_once() exc_info = patched_track_partial_parser.call_args[0][0] assert "traceback" in exc_info assert "exception" in exc_info assert "code" in exc_info assert "location" in exc_info assert "full_reparse_reason" in exc_info assert "KeyError: 'Whoopsie!'" == exc_info["exception"] assert isinstance(exc_info["code"], str) or isinstance(exc_info["code"], type(None)) class TestGetFullManifest: @pytest.fixture def set_required_mocks( self, mocker: MockerFixture, manifest: Manifest, mock_adapter: MagicMock ): mocker.patch("dbt.parser.manifest.get_adapter").return_value = mock_adapter mocker.patch("dbt.parser.manifest.ManifestLoader.load").return_value = manifest mocker.patch("dbt.parser.manifest._check_manifest").return_value = None mocker.patch("dbt.parser.manifest.ManifestLoader.save_macros_to_adapter").return_value = ( None ) mocker.patch("dbt.tracking.active_user").return_value = User(None) def test_write_perf_info( self, mock_project: MagicMock, mocker: MockerFixture, set_required_mocks, ) -> None: write_perf_info = mocker.patch("dbt.parser.manifest.ManifestLoader.write_perf_info") ManifestLoader.get_full_manifest( config=mock_project, # write_perf_info=False let it default instead ) assert not write_perf_info.called ManifestLoader.get_full_manifest(config=mock_project, write_perf_info=False) assert not write_perf_info.called ManifestLoader.get_full_manifest(config=mock_project, write_perf_info=True) assert write_perf_info.called def test_reset( self, mock_project: MagicMock, mock_adapter: MagicMock, set_required_mocks, ) -> None: ManifestLoader.get_full_manifest( config=mock_project, # reset=False let it default instead ) assert not mock_project.clear_dependencies.called assert not mock_adapter.clear_macro_resolver.called ManifestLoader.get_full_manifest(config=mock_project, reset=False) assert not mock_project.clear_dependencies.called assert not mock_adapter.clear_macro_resolver.called ManifestLoader.get_full_manifest(config=mock_project, reset=True) assert mock_project.clear_dependencies.called assert mock_adapter.clear_macro_resolver.called def test_partial_parse_file_diff_flag( self, mock_project: MagicMock, mocker: MockerFixture, set_required_mocks, ) -> None: # FileDiff.from_dict is only called if PARTIAL_PARSE_FILE_DIFF == False # So we can track this function call to check if setting PARTIAL_PARSE_FILE_DIFF # works appropriately mock_file_diff = mocker.patch("dbt.parser.read_files.FileDiff.from_dict") mock_file_diff.return_value = FileDiff([], [], []) ManifestLoader.get_full_manifest(config=mock_project) assert not mock_file_diff.called set_from_args(Namespace(PARTIAL_PARSE_FILE_DIFF=True), {}) ManifestLoader.get_full_manifest(config=mock_project) assert not mock_file_diff.called set_from_args(Namespace(PARTIAL_PARSE_FILE_DIFF=False), {}) ManifestLoader.get_full_manifest(config=mock_project) assert mock_file_diff.called class TestWarnUnusedConfigs: @pytest.mark.parametrize( "resource_type,path,expect_used", [ ("data_tests", "unused_path", False), ("data_tests", "minimal", True), ("metrics", "unused_path", False), ("metrics", "test", True), ("models", "unused_path", False), ("models", "pkg", True), ("saved_queries", "unused_path", False), ("saved_queries", "test", True), ("seeds", "unused_path", False), ("seeds", "pkg", True), ("semantic_models", "unused_path", False), ("semantic_models", "test", True), ("sources", "unused_path", False), ("sources", "pkg", True), ("unit_tests", "unused_path", False), ("unit_tests", "pkg", True), ], ) def test_warn_for_unused_resource_config_paths( self, resource_type: str, path: str, expect_used: bool, manifest: Manifest, runtime_config: RuntimeConfig, ) -> None: catcher = EventCatcher(UnusedResourceConfigPath) add_callback_to_manager(catcher.catch) setattr(runtime_config, resource_type, {path: {"+materialized": "table"}}) _warn_for_unused_resource_config_paths(manifest=manifest, config=runtime_config) if expect_used: assert len(catcher.caught_events) == 0 else: assert len(catcher.caught_events) == 1 assert f"{resource_type}.{path}" in str(catcher.caught_events[0].data) class TestCheckForcingConcurrentBatches: @pytest.fixture @patch("dbt.parser.manifest.ManifestLoader.build_manifest_state_check") @patch("dbt.parser.manifest.os.path.exists") @patch("dbt.parser.manifest.open") def manifest_loader( self, patched_open, patched_os_exist, patched_state_check ) -> ManifestLoader: mock_project = MagicMock(RuntimeConfig) mock_project.project_target_path = "mock_target_path" mock_project.project_name = "mock_project_name" return ManifestLoader(mock_project, {}) @pytest.fixture def event_catcher(self) -> EventCatcher: return EventCatcher(InvalidConcurrentBatchesConfig) @pytest.mark.parametrize( "adapter_support,concurrent_batches_config,expect_warning", [ (False, True, True), (False, False, False), (False, None, False), (True, True, False), (True, False, False), (True, None, False), ], ) def test_check_forcing_concurrent_batches( self, mocker: MockerFixture, manifest_loader: ManifestLoader, postgres_adapter: PostgresAdapter, event_catcher: EventCatcher, adapter_support: bool, concurrent_batches_config: Optional[bool], expect_warning: bool, ): add_callback_to_manager(event_catcher.catch) model = model_node() model.config.concurrent_batches = concurrent_batches_config mocker.patch.object(postgres_adapter, "supports").return_value = adapter_support mocker.patch("dbt.parser.manifest.get_adapter").return_value = postgres_adapter mocker.patch.object(manifest_loader.manifest, "use_microbatch_batches").return_value = True manifest_loader.manifest.add_node_nofile(model) manifest_loader.check_forcing_batch_concurrency() if expect_warning: assert len(event_catcher.caught_events) == 1 assert "Batches will be run sequentially" in event_catcher.caught_events[0].info.msg # type: ignore else: assert len(event_catcher.caught_events) == 0 class TestUpdateSemanticModel: """Tests for ManifestLoader.update_semantic_model.""" @pytest.fixture @patch("dbt.parser.manifest.ManifestLoader.build_manifest_state_check") @patch("dbt.parser.manifest.os.path.exists") @patch("dbt.parser.manifest.open") def loader(self, patched_open, patched_os_exist, patched_state_check): mock_project = MagicMock(RuntimeConfig) mock_project.project_target_path = "mock_target_path" return ManifestLoader(mock_project, {}) def test_no_index_error_when_depends_on_nodes_is_empty(self, loader): """Regression: update_semantic_model must not raise IndexError when depends_on_nodes is empty (e.g. referenced model is disabled).""" semantic_model = MagicMock() semantic_model.depends_on_nodes = [] # Before the fix this raised: IndexError: list index out of range loader.update_semantic_model(semantic_model) # node_relation must not have been assigned assert "node_relation" not in semantic_model.__dict__ def test_node_relation_set_when_depends_on_nodes_has_entry(self, loader): """When depends_on_nodes is non-empty, node_relation is populated.""" refd_node = MagicMock() refd_node.relation_name = '"db"."schema"."my_model"' refd_node.alias = "my_model" refd_node.schema = "schema" refd_node.database = "db" loader.manifest.nodes["model.pkg.my_model"] = refd_node semantic_model = MagicMock() semantic_model.depends_on_nodes = ["model.pkg.my_model"] loader.update_semantic_model(semantic_model) assert semantic_model.node_relation == NodeRelation( relation_name='"db"."schema"."my_model"', alias="my_model", schema_name="schema", database="db", ) class TestExtendedMsgpackEncoder: """ Unit tests for extended_msgpack_encoder and extended_mashumaro_encoder. The key regression being guarded: jinja2.Undefined objects that end up in manifest node fields (e.g. meta values rendered from schema.yml with an undefined Jinja variable) must be serialisable to msgpack. Without the isinstance(obj, jinja2.Undefined) branch in extended_msgpack_encoder the packer raises: TypeError: can not serialize 'Undefined' object """ def test_undefined_returns_none(self): """extended_msgpack_encoder converts jinja2.Undefined to None.""" undefined = jinja2.Undefined(name="some_undefined_var") result = extended_msgpack_encoder(undefined) assert result is None def test_undefined_subclass_returns_none(self): """extended_msgpack_encoder handles jinja2.Undefined subclasses.""" class MyUndefined(jinja2.Undefined): pass result = extended_msgpack_encoder(MyUndefined()) assert result is None def test_non_undefined_passthrough(self): """extended_msgpack_encoder passes through unknown types unchanged.""" # msgpack itself will raise for truly unserializable objects; the # encoder just returns them so msgpack can raise its own error. obj = object() assert extended_msgpack_encoder(obj) is obj def test_mashumaro_encoder_with_undefined_in_dict(self): """ extended_mashumaro_encoder can pack a dict containing jinja2.Undefined. This is the direct reproduction of the production traceback: a manifest node with a meta dict like {"key": } must serialise without raising TypeError. """ undefined = jinja2.Undefined(name="undefined_jinja_var") data = {"meta": {"key": undefined}} # Must not raise TypeError packed = extended_mashumaro_encoder(data) unpacked = msgpack.unpackb(packed, raw=False) assert unpacked == {"meta": {"key": None}} def test_mashumaro_encoder_without_undefined_unchanged(self): """extended_mashumaro_encoder round-trips plain data correctly.""" data = {"meta": {"key": "value"}, "count": 42} packed = extended_mashumaro_encoder(data) unpacked = msgpack.unpackb(packed, raw=False) assert unpacked == data ================================================ FILE: tests/unit/parser/test_parser.py ================================================ import os import unittest from argparse import Namespace from copy import deepcopy from unittest import mock import yaml import dbt_common.events.functions from dbt import tracking from dbt.artifacts.resources import ModelConfig, RefArgs from dbt.artifacts.resources.v1.model import ( ModelBuildAfter, ModelFreshnessUpdatesOnOptions, ) from dbt.context.context_config import ContextConfig from dbt.contracts.files import FileHash, FilePath, SchemaSourceFile, SourceFile from dbt.contracts.graph.manifest import Manifest from dbt.contracts.graph.model_config import NodeConfig, SnapshotConfig, TestConfig from dbt.contracts.graph.nodes import ( AnalysisNode, DependsOn, Macro, ModelNode, SingularTestNode, SnapshotNode, UnpatchedSourceDefinition, ) from dbt.events.types import FreshnessConfigProblem from dbt.exceptions import CompilationError, ParsingError, SchemaConfigError from dbt.flags import set_from_args from dbt.node_types import NodeType from dbt.parser import ( AnalysisParser, GenericTestParser, MacroParser, ModelParser, SchemaParser, SingularTestParser, SnapshotParser, ) from dbt.parser.common import YamlBlock from dbt.parser.models import ( _get_config_call_dict, _get_exp_sample_result, _get_sample_result, _get_stable_sample_result, _shift_sources, ) from dbt.parser.schemas import ( AnalysisPatchParser, MacroPatchParser, ModelPatchParser, SourceParser, TestablePatchParser, yaml_from_file, ) from dbt.parser.search import FileBlock from dbt.parser.sources import SourcePatcher from dbt.tests.util import safe_set_invocation_context from dbt_common.events.event_catcher import EventCatcher from dbt_common.events.event_manager_client import add_callback_to_manager from tests.unit.utils import ( MockNode, config_from_parts_or_dicts, generate_name_macros, normalize, ) set_from_args( Namespace( warn_error=False, state_modified_compare_more_unrendered_values=False, require_generic_test_arguments_property=False, ), None, ) def get_abs_os_path(unix_path): return normalize(os.path.abspath(unix_path)) class BaseParserTest(unittest.TestCase): maxDiff = None def _generate_macros(self): name_sql = {} for component in ("database", "schema", "alias"): if component == "alias": source = "node.name" else: source = f"target.{component}" name = f"generate_{component}_name" sql = f"{{% macro {name}(value, node) %}} {{% if value %}} {{{{ value }}}} {{% else %}} {{{{ {source} }}}} {{% endif %}} {{% endmacro %}}" name_sql[name] = sql for name, sql in name_sql.items(): pm = Macro( name=name, resource_type=NodeType.Macro, unique_id=f"macro.root.{name}", package_name="root", original_file_path=normalize("macros/macro.sql"), path=normalize("macros/macro.sql"), macro_sql=sql, ) yield pm def setUp(self): set_from_args( Namespace( warn_error=True, state_modified_compare_more_unrendered_values=False, require_generic_test_arguments_property=False, ), None, ) safe_set_invocation_context() # HACK: this is needed since tracking events can # be sent when using the model parser tracking.do_not_track() self.maxDiff = None profile_data = { "target": "test", "quoting": {}, "outputs": { "test": { "type": "postgres", "host": "localhost", "schema": "analytics", "user": "test", "pass": "test", "dbname": "test", "port": 1, } }, } root_project = { "name": "root", "version": "0.1", "profile": "test", "project-root": normalize("/usr/src/app"), "config-version": 2, } self.root_project_config = config_from_parts_or_dicts( project=root_project, profile=profile_data, cli_vars={"test_schema_name": "foo"} ) snowplow_project = { "name": "snowplow", "version": "0.1", "profile": "test", "project-root": get_abs_os_path("./dbt_packages/snowplow"), "config-version": 2, } self.snowplow_project_config = config_from_parts_or_dicts( project=snowplow_project, profile=profile_data ) self.all_projects = { "root": self.root_project_config, "snowplow": self.snowplow_project_config, } self.root_project_config.dependencies = self.all_projects self.snowplow_project_config.dependencies = self.all_projects self.patcher = mock.patch("dbt.context.providers.get_adapter") self.factory = self.patcher.start() self.parser_patcher = mock.patch("dbt.parser.base.get_adapter") self.factory_parser = self.parser_patcher.start() self.manifest = Manifest( macros={m.unique_id: m for m in generate_name_macros("root")}, ) def tearDown(self): self.parser_patcher.stop() self.patcher.stop() def source_file_for(self, data: str, filename: str, searched: str): root_dir = get_abs_os_path("./dbt_packages/snowplow") filename = normalize(filename) path = FilePath( searched_path=searched, relative_path=filename, project_root=root_dir, modification_time=0.0, ) sf_cls = SchemaSourceFile if filename.endswith(".yml") else SourceFile source_file = sf_cls( path=path, checksum=FileHash.from_contents(data), project_name="snowplow", ) source_file.contents = data return source_file def file_block_for(self, data: str, filename: str, searched: str): source_file = self.source_file_for(data, filename, searched) return FileBlock(file=source_file) def assert_has_manifest_lengths( self, manifest, macros=3, nodes=0, sources=0, docs=0, disabled=0, unit_tests=0 ): self.assertEqual(len(manifest.macros), macros) self.assertEqual(len(manifest.nodes), nodes) self.assertEqual(len(manifest.sources), sources) self.assertEqual(len(manifest.docs), docs) self.assertEqual(len(manifest.disabled), disabled) self.assertEqual(len(manifest.unit_tests), unit_tests) def assertEqualNodes(node_one, node_two): node_one_dict = node_one.to_dict() if "created_at" in node_one_dict: del node_one_dict["created_at"] if "relation_name" in node_one_dict: del node_one_dict["relation_name"] node_two_dict = node_two.to_dict() if "created_at" in node_two_dict: del node_two_dict["created_at"] if "relation_name" in node_two_dict: del node_two_dict["relation_name"] # we don't reall care the order of packages, doing this because it is hard to # make config.packages a set instead of a list if "config" in node_one_dict and "packages" in node_one_dict["config"]: if "config" not in node_two_dict and "packages" in node_two_dict["config"]: return False node_one_dict["config"]["packages"] = set(node_one_dict["config"]["packages"]) node_two_dict["config"]["packages"] = set(node_two_dict["config"]["packages"]) node_one_dict["unrendered_config"]["packages"] = set(node_one_dict["config"]["packages"]) node_two_dict["unrendered_config"]["packages"] = set(node_two_dict["config"]["packages"]) if "packages" in node_one_dict["config_call_dict"]: node_one_dict["config_call_dict"]["packages"] = set( node_one_dict["config_call_dict"]["packages"] ) node_two_dict["config_call_dict"]["packages"] = set( node_two_dict["config_call_dict"]["packages"] ) assert node_one_dict == node_two_dict SINGLE_TABLE_SOURCE = """ sources: - name: my_source tables: - name: my_table """ MULTIPLE_TABLE_SOURCE_META = """ sources: - name: my_source meta: source_field: source_value shared_field: shared_field_default tables: - name: my_table_shared_field_default meta: table_field: table_value - name: my_table_shared_field_override meta: shared_field: shared_field_table_override table_field: table_value """ SINGLE_TABLE_SOURCE_TESTS = """ sources: - name: my_source tables: - name: my_table description: A description of my table columns: - name: color data_tests: - not_null: severity: WARN - accepted_values: values: ['red', 'blue', 'green'] """ SINGLE_TABLE_MODEL_TESTS = """ models: - name: my_model description: A description of my model columns: - name: color description: The color value data_tests: - not_null: severity: WARN - accepted_values: description: Only primary colors are allowed in here values: ['red', 'blue', 'green'] - foreign_package.test_case: arg: 100 """ SINGLE_TABLE_MODEL_TESTS_WRONG_SEVERITY = """ models: - name: my_model description: A description of my model columns: - name: color description: The color value data_tests: - not_null: severity: WARNING - accepted_values: values: ['red', 'blue', 'green'] - foreign_package.test_case: arg: 100 """ SINGLE_TABLE_MODEL_FRESHNESS = """ models: - name: my_model description: A description of my model config: freshness: build_after: {count: 1, period: day, updates_on: all} """ SINGLE_TABLE_MODEL_TOP_LEVEL_FRESHNESS = """ models: - name: my_model description: A description of my model freshness: build_after: {count: 1, period: day, updates_on: any} """ SINGLE_TABLE_MODEL_FRESHNESS_ONLY_DEPEND_ON = """ models: - name: my_model description: A description of my model config: freshness: build_after: updates_on: all period: hour count: 5 """ MULTIPLE_TABLE_VERSIONED_MODEL_TESTS = """ models: - name: my_model description: A description of my model data_tests: - unique: column_name: color columns: - name: color description: The color value data_tests: - not_null: severity: WARN - name: location_id data_type: int versions: - v: 1 defined_in: arbitrary_file_name data_tests: [] columns: - include: '*' - name: extra - v: 2 columns: - include: '*' exclude: ['location_id'] - name: extra """ MULTIPLE_TABLE_VERSIONED_MODEL = """ models: - name: my_model description: A description of my model config: materialized: table sql_header: test_sql_header columns: - name: color description: The color value - name: location_id data_type: int versions: - v: 1 defined_in: arbitrary_file_name columns: - include: '*' - name: extra - v: 2 config: materialized: view columns: - include: '*' exclude: ['location_id'] - name: extra """ MULTIPLE_TABLE_VERSIONED_MODEL_CONTRACT_ENFORCED = """ models: - name: my_model config: contract: enforced: true versions: - v: 0 defined_in: arbitrary_file_name - v: 2 """ MULTIPLE_TABLE_VERSIONED_MODEL_V0 = """ models: - name: my_model versions: - v: 0 defined_in: arbitrary_file_name - v: 2 """ MULTIPLE_TABLE_VERSIONED_MODEL_V0_LATEST_VERSION = """ models: - name: my_model latest_version: 0 versions: - v: 0 defined_in: arbitrary_file_name - v: 2 """ SINGLE_TABLE_SOURCE_PATCH = """ sources: - name: my_source overrides: snowplow tables: - name: my_table columns: - name: id data_tests: - not_null - unique """ SOURCE_CUSTOM_FRESHNESS_AT_SOURCE = """ sources: - name: my_source loaded_at_query: "select 1 as id" tables: - name: my_table """ SOURCE_CUSTOM_FRESHNESS_AT_SOURCE_FIELD_AT_TABLE = """ sources: - name: my_source loaded_at_query: "select 1 as id" tables: - name: my_table loaded_at_field: test """ SOURCE_FIELD_AT_SOURCE_CUSTOM_FRESHNESS_AT_TABLE = """ sources: - name: my_source loaded_at_field: test tables: - name: my_table loaded_at_query: "select 1 as id" """ SOURCE_FRESHNESS_AT_TABLE_AND_CONFIG = """ sources: - name: my_source loaded_at_field: test tables: - name: my_table freshness: warn_after: {count: 1, period: hour} error_after: {count: 1, period: day} config: freshness: warn_after: {count: 2, period: hour} error_after: {count: 2, period: day} """ SOURCE_FIELD_AT_CUSTOM_FRESHNESS_BOTH_AT_TABLE = """ sources: - name: my_source loaded_at_field: test tables: - name: my_table loaded_at_query: "select 1 as id" loaded_at_field: test """ SOURCE_FIELD_AT_CUSTOM_FRESHNESS_BOTH_AT_SOURCE = """ sources: - name: my_source loaded_at_field: test loaded_at_query: "select 1 as id" tables: - name: my_table loaded_at_field: test """ SOURCE_FRESHNESS_WITH_LOADED_AT_QUERY = """ sources: - name: my_source tables: - name: my_table config: loaded_at_query: "select 1 as id" freshness: warn_after: {count: 1, period: hour} error_after: {count: 1, period: day} """ class SchemaParserTest(BaseParserTest): def setUp(self): super().setUp() # Reset `warn_error` to False so we don't raise warnigns about top level freshness as errors set_from_args( Namespace( warn_error=False, state_modified_compare_more_unrendered_values=False, require_generic_test_arguments_property=False, ), None, ) self.parser = SchemaParser( project=self.snowplow_project_config, manifest=self.manifest, root_project=self.root_project_config, ) self.source_patcher = SourcePatcher( root_project=self.root_project_config, manifest=self.manifest, ) def file_block_for(self, data, filename, searched="models"): return super().file_block_for(data, filename, searched) def yaml_block_for(self, test_yml: str, filename: str): file_block = self.file_block_for(data=test_yml, filename=filename) return YamlBlock.from_file_block( src=file_block, data=yaml.safe_load(test_yml), ) class SchemaParserSourceTest(SchemaParserTest): def test__read_basic_source(self): block = self.yaml_block_for(SINGLE_TABLE_SOURCE, "test_one.yml") analysis_blocks = AnalysisPatchParser(self.parser, block, "analyses").parse().test_blocks model_blocks = ModelPatchParser(self.parser, block, "models").parse().test_blocks source_blocks = SourceParser(self.parser, block, "sources").parse().test_blocks macro_blocks = MacroPatchParser(self.parser, block, "macros").parse().test_blocks self.assertEqual(len(analysis_blocks), 0) self.assertEqual(len(model_blocks), 0) self.assertEqual(len(source_blocks), 0) self.assertEqual(len(macro_blocks), 0) self.assertEqual(len(list(self.parser.manifest.nodes)), 0) source_values = list(self.parser.manifest.sources.values()) self.assertEqual(len(source_values), 1) self.assertEqual(source_values[0].source.name, "my_source") self.assertEqual(source_values[0].table.name, "my_table") self.assertEqual(source_values[0].table.description, "") self.assertEqual(len(source_values[0].table.columns), 0) @mock.patch("dbt.parser.sources.get_adapter") def test_parse_source_custom_freshness_at_source(self, _): block = self.file_block_for(SOURCE_CUSTOM_FRESHNESS_AT_SOURCE, "test_one.yml") dct = yaml_from_file(block.file, validate=True) self.parser.parse_file(block, dct) unpatched_src_default = self.parser.manifest.sources["source.snowplow.my_source.my_table"] src_default = self.source_patcher.parse_source(unpatched_src_default) assert src_default.loaded_at_query == "select 1 as id" @mock.patch("dbt.parser.sources.get_adapter") def test_parse_source_custom_freshness_at_source_field_at_table(self, _): block = self.file_block_for( SOURCE_CUSTOM_FRESHNESS_AT_SOURCE_FIELD_AT_TABLE, "test_one.yml" ) dct = yaml_from_file(block.file, validate=True) self.parser.parse_file(block, dct) unpatched_src_default = self.parser.manifest.sources["source.snowplow.my_source.my_table"] src_default = self.source_patcher.parse_source(unpatched_src_default) # source loaded_at_query not propagate to table since there's loaded_at_field defined assert src_default.loaded_at_query is None @mock.patch("dbt.parser.sources.get_adapter") def test_parse_source_field_at_source_custom_freshness_at_table(self, _): block = self.file_block_for( SOURCE_FIELD_AT_SOURCE_CUSTOM_FRESHNESS_AT_TABLE, "test_one.yml" ) dct = yaml_from_file(block.file, validate=True) self.parser.parse_file(block, dct) unpatched_src_default = self.parser.manifest.sources["source.snowplow.my_source.my_table"] src_default = self.source_patcher.parse_source(unpatched_src_default) assert src_default.loaded_at_query == "select 1 as id" @mock.patch("dbt.parser.sources.get_adapter") def test_parse_source_field_at_custom_freshness_both_at_table_fails(self, _): block = self.file_block_for(SOURCE_FIELD_AT_CUSTOM_FRESHNESS_BOTH_AT_TABLE, "test_one.yml") dct = yaml_from_file(block.file, validate=True) self.parser.parse_file(block, dct) unpatched_src_default = self.parser.manifest.sources["source.snowplow.my_source.my_table"] with self.assertRaises(ParsingError): self.source_patcher.parse_source(unpatched_src_default) @mock.patch("dbt.parser.sources.get_adapter") def test_parse_source_resulting_node_freshness_matches_config_freshness(self, _): block = self.file_block_for(SOURCE_FRESHNESS_AT_TABLE_AND_CONFIG, "test_one.yml") dct = yaml_from_file(block.file, validate=True) self.parser.parse_file(block, dct) unpatched_src_default = self.parser.manifest.sources["source.snowplow.my_source.my_table"] src_default = self.source_patcher.parse_source(unpatched_src_default) assert src_default.freshness == src_default.config.freshness assert src_default.freshness.warn_after.count == 2 assert src_default.freshness.warn_after.period == "hour" assert src_default.freshness.error_after.count == 2 assert src_default.freshness.error_after.period == "day" @mock.patch("dbt.parser.sources.get_adapter") def test_parse_source_field_at_custom_freshness_both_at_source_fails(self, _): block = self.file_block_for( SOURCE_FIELD_AT_CUSTOM_FRESHNESS_BOTH_AT_SOURCE, "test_one.yml" ) dct = yaml_from_file(block.file, validate=True) self.parser.parse_file(block, dct) unpatched_src_default = self.parser.manifest.sources["source.snowplow.my_source.my_table"] with self.assertRaises(ParsingError): self.source_patcher.parse_source(unpatched_src_default) def test__parse_basic_source(self): block = self.file_block_for(SINGLE_TABLE_SOURCE, "test_one.yml") dct = yaml_from_file(block.file, validate=True) self.parser.parse_file(block, dct) self.assert_has_manifest_lengths(self.parser.manifest, sources=1) src = list(self.parser.manifest.sources.values())[0] assert isinstance(src, UnpatchedSourceDefinition) assert src.package_name == "snowplow" assert src.source.name == "my_source" assert src.table.name == "my_table" assert src.resource_type == NodeType.Source assert src.fqn == ["snowplow", "my_source", "my_table"] @mock.patch("dbt.parser.sources.get_adapter") def test__parse_basic_source_meta(self, mock_get_adapter): block = self.file_block_for(MULTIPLE_TABLE_SOURCE_META, "test_one.yml") dct = yaml_from_file(block.file, validate=True) self.parser.parse_file(block, dct) self.assert_has_manifest_lengths(self.parser.manifest, sources=2) unpatched_src_default = self.parser.manifest.sources[ "source.snowplow.my_source.my_table_shared_field_default" ] src_default = self.source_patcher.parse_source(unpatched_src_default) assert src_default.meta == { "source_field": "source_value", "shared_field": "shared_field_default", "table_field": "table_value", } assert src_default.source_meta == { "source_field": "source_value", "shared_field": "shared_field_default", } unpatched_src_override = self.parser.manifest.sources[ "source.snowplow.my_source.my_table_shared_field_override" ] src_override = self.source_patcher.parse_source(unpatched_src_override) assert src_override.meta == { "source_field": "source_value", "shared_field": "shared_field_table_override", "table_field": "table_value", } assert src_override.source_meta == { "source_field": "source_value", "shared_field": "shared_field_default", } def test__read_basic_source_tests(self): block = self.yaml_block_for(SINGLE_TABLE_SOURCE_TESTS, "test_one.yml") analysis_tests = AnalysisPatchParser(self.parser, block, "analyses").parse().test_blocks model_tests = ModelPatchParser(self.parser, block, "models").parse().test_blocks source_tests = SourceParser(self.parser, block, "sources").parse().test_blocks macro_tests = MacroPatchParser(self.parser, block, "macros").parse().test_blocks self.assertEqual(len(analysis_tests), 0) self.assertEqual(len(model_tests), 0) self.assertEqual(len(source_tests), 0) self.assertEqual(len(macro_tests), 0) self.assertEqual(len(list(self.parser.manifest.nodes)), 0) self.assertEqual(len(list(self.parser.manifest.source_patches)), 0) source_values = list(self.parser.manifest.sources.values()) self.assertEqual(len(source_values), 1) self.assertEqual(source_values[0].source.name, "my_source") self.assertEqual(source_values[0].table.name, "my_table") self.assertEqual(source_values[0].table.description, "A description of my table") self.assertEqual(len(source_values[0].table.columns), 1) def test__parse_basic_source_tests(self): block = self.file_block_for(SINGLE_TABLE_SOURCE_TESTS, "test_one.yml") self.parser.manifest.files[block.file.file_id] = block.file dct = yaml_from_file(block.file, validate=True) self.parser.parse_file(block, dct) self.assertEqual(len(self.parser.manifest.nodes), 0) self.assertEqual(len(self.parser.manifest.sources), 1) src = list(self.parser.manifest.sources.values())[0] self.assertEqual(src.source.name, "my_source") self.assertEqual(src.source.schema, None) self.assertEqual(src.table.name, "my_table") self.assertEqual(src.table.description, "A description of my table") tests = [ self.source_patcher.parse_source_test(src, test, col) for test, col in src.get_tests() ] tests.sort(key=lambda n: n.unique_id) self.assertEqual(tests[0].config.severity, "ERROR") self.assertEqual(tests[0].tags, []) self.assertEqual(tests[0].sources, [["my_source", "my_table"]]) self.assertEqual(tests[0].column_name, "color") self.assertEqual(tests[0].fqn, ["snowplow", tests[0].name]) self.assertEqual(tests[1].config.severity, "WARN") self.assertEqual(tests[1].tags, []) self.assertEqual(tests[1].sources, [["my_source", "my_table"]]) self.assertEqual(tests[1].column_name, "color") self.assertEqual(tests[1].fqn, ["snowplow", tests[1].name]) file_id = "snowplow://" + normalize("models/test_one.yml") self.assertIn(file_id, self.parser.manifest.files) self.assertEqual(self.parser.manifest.files[file_id].data_tests, {}) self.assertEqual( self.parser.manifest.files[file_id].sources, ["source.snowplow.my_source.my_table"] ) self.assertEqual(self.parser.manifest.files[file_id].source_patches, []) def test__read_source_patch(self): # TODO: There needs to be a better way to disable warnings being fired # during unit tests, but all we have today is this global variable. dbt_common.events.functions.WARN_ERROR = False block = self.yaml_block_for(SINGLE_TABLE_SOURCE_PATCH, "test_one.yml") analysis_tests = AnalysisPatchParser(self.parser, block, "analyses").parse().test_blocks model_tests = TestablePatchParser(self.parser, block, "models").parse().test_blocks source_tests = SourceParser(self.parser, block, "sources").parse().test_blocks macro_tests = MacroPatchParser(self.parser, block, "macros").parse().test_blocks self.assertEqual(len(analysis_tests), 0) self.assertEqual(len(model_tests), 0) self.assertEqual(len(source_tests), 0) self.assertEqual(len(macro_tests), 0) self.assertEqual(len(list(self.parser.manifest.nodes)), 0) self.assertEqual(len(list(self.parser.manifest.sources)), 0) source_patches = list(self.parser.manifest.source_patches.values()) self.assertEqual(len(source_patches), 1) self.assertEqual(source_patches[0].name, "my_source") self.assertEqual(source_patches[0].overrides, "snowplow") self.assertIsNone(source_patches[0].description) self.assertEqual(len(source_patches[0].tables), 1) table = source_patches[0].tables[0] self.assertEqual(table.name, "my_table") self.assertIsNone(table.description) self.assertEqual(len(table.columns), 1) self.assertEqual(len(table.columns[0].data_tests), 2) @mock.patch( "dbt.parser.sources.get_adapter", return_value=mock.MagicMock(supports=mock.MagicMock(return_value=False)), ) def test__loaded_at_query_does_not_fire_config_problem_event(self, _): """Test where we have a loaded_at_query defined but no loaded_at_field and the adapter does not support metadata-based freshness. We should not fire a config problem event. """ catcher = EventCatcher(FreshnessConfigProblem) add_callback_to_manager(catcher.catch) block = self.file_block_for(SOURCE_FRESHNESS_WITH_LOADED_AT_QUERY, "test_one.yml") dct = yaml_from_file(block.file, validate=True) self.parser.parse_file(block, dct) unpatched_src_default = self.parser.manifest.sources["source.snowplow.my_source.my_table"] self.source_patcher.parse_source(unpatched_src_default) assert len(catcher.caught_events) == 0 class SchemaParserModelsTest(SchemaParserTest): def setUp(self): super().setUp() my_model_node = MockNode( package="root", name="my_model", config=ModelConfig(enabled=True), refs=[], sources=[], patch_path=None, ) source_file = self.source_file_for("", "my_model.sql", "models") nodes = {my_model_node.unique_id: my_model_node} macros = {m.unique_id: m for m in generate_name_macros("root")} self.manifest = Manifest(nodes=nodes, macros=macros) self.manifest.files[source_file.file_id] = source_file self.manifest.ref_lookup self.parser = SchemaParser( project=self.snowplow_project_config, manifest=self.manifest, root_project=self.root_project_config, ) def test__read_basic_model_tests(self): block = self.yaml_block_for(SINGLE_TABLE_MODEL_TESTS, "test_one.yml") dct = yaml_from_file(block.file, validate=True) self.parser.parse_file(block, dct) self.assertEqual(len(list(self.parser.manifest.sources)), 0) self.assertEqual(len(list(self.parser.manifest.nodes)), 4) def test__parse_model_freshness(self): block = self.file_block_for(SINGLE_TABLE_MODEL_FRESHNESS, "test_one.yml") self.parser.manifest.files[block.file.file_id] = block.file dct = yaml_from_file(block.file, validate=True) self.parser.parse_file(block, dct) self.assert_has_manifest_lengths(self.parser.manifest, nodes=1) assert self.parser.manifest.nodes[ "model.root.my_model" ].config.freshness.build_after == ModelBuildAfter( count=1, period="day", updates_on=ModelFreshnessUpdatesOnOptions.all ) def test__parse_model_ignores_top_level_freshness(self): block = self.file_block_for(SINGLE_TABLE_MODEL_TOP_LEVEL_FRESHNESS, "test_one.yml") self.parser.manifest.files[block.file.file_id] = block.file dct = yaml_from_file(block.file, validate=True) self.parser.parse_file(block, dct) self.assert_has_manifest_lengths(self.parser.manifest, nodes=1) # we can't use hasattr because the model node is a mock, and checking with hasattr will add it to the mock assert "freshness" not in self.parser.manifest.nodes["model.root.my_model"].__dir__() # should be None because nothing set it assert self.parser.manifest.nodes["model.root.my_model"].config.freshness is None def test__parse_model_freshness_depend_on(self): block = self.file_block_for(SINGLE_TABLE_MODEL_FRESHNESS_ONLY_DEPEND_ON, "test_one.yml") self.parser.manifest.files[block.file.file_id] = block.file dct = yaml_from_file(block.file, validate=True) self.parser.parse_file(block, dct) self.assert_has_manifest_lengths(self.parser.manifest, nodes=1) assert self.parser.manifest.nodes[ "model.root.my_model" ].config.freshness.build_after == ModelBuildAfter( count=5, period="hour", updates_on=ModelFreshnessUpdatesOnOptions.all ) def test__read_basic_model_tests_wrong_severity(self): block = self.yaml_block_for(SINGLE_TABLE_MODEL_TESTS_WRONG_SEVERITY, "test_one.yml") dct = yaml_from_file(block.file, validate=True) with self.assertRaisesRegex( SchemaConfigError, "Severity must be either 'warn' or 'error'. Got 'WARNING'" ): self.parser.parse_file(block, dct) def test__parse_basic_model_tests(self): block = self.file_block_for(SINGLE_TABLE_MODEL_TESTS, "test_one.yml") self.parser.manifest.files[block.file.file_id] = block.file dct = yaml_from_file(block.file, validate=True) self.parser.parse_file(block, dct) self.assert_has_manifest_lengths(self.parser.manifest, nodes=4) all_nodes = sorted(self.parser.manifest.nodes.values(), key=lambda n: n.unique_id) tests = [] for node in all_nodes: if node.resource_type != NodeType.Test: continue tests.append(node) self.assertEqual(tests[0].config.severity, "ERROR") self.assertEqual(tests[0].tags, []) self.assertEqual(tests[0].refs, [RefArgs(name="my_model")]) self.assertEqual(tests[0].column_name, "color") self.assertEqual(tests[0].description, "Only primary colors are allowed in here") self.assertEqual(tests[0].package_name, "snowplow") self.assertTrue(tests[0].name.startswith("accepted_values_")) self.assertEqual(tests[0].fqn, ["snowplow", tests[0].name]) self.assertEqual( tests[0].unique_id.split("."), ["test", "snowplow", tests[0].name, "9d4814efde"] ) self.assertEqual(tests[0].test_metadata.name, "accepted_values") self.assertIsNone(tests[0].test_metadata.namespace) self.assertEqual( tests[0].test_metadata.kwargs, { "column_name": "color", "model": "{{ get_where_subquery(ref('my_model')) }}", "values": ["red", "blue", "green"], }, ) # foreign packages are a bit weird, they include the macro package # name in the test name self.assertEqual(tests[1].config.severity, "ERROR") self.assertEqual(tests[1].tags, []) self.assertEqual(tests[1].refs, [RefArgs(name="my_model")]) self.assertEqual(tests[1].column_name, "color") self.assertEqual(tests[1].description, "") self.assertEqual(tests[1].fqn, ["snowplow", tests[1].name]) self.assertTrue(tests[1].name.startswith("foreign_package_test_case_")) self.assertEqual(tests[1].package_name, "snowplow") self.assertEqual( tests[1].unique_id.split("."), ["test", "snowplow", tests[1].name, "13958f62f7"] ) self.assertEqual(tests[1].test_metadata.name, "test_case") self.assertEqual(tests[1].test_metadata.namespace, "foreign_package") self.assertEqual( tests[1].test_metadata.kwargs, { "column_name": "color", "model": "{{ get_where_subquery(ref('my_model')) }}", "arg": 100, }, ) self.assertEqual(tests[2].config.severity, "WARN") self.assertEqual(tests[2].tags, []) self.assertEqual(tests[2].refs, [RefArgs(name="my_model")]) self.assertEqual(tests[2].column_name, "color") self.assertEqual(tests[2].package_name, "snowplow") self.assertTrue(tests[2].name.startswith("not_null_")) self.assertEqual(tests[2].fqn, ["snowplow", tests[2].name]) self.assertEqual( tests[2].unique_id.split("."), ["test", "snowplow", tests[2].name, "2f61818750"] ) self.assertEqual(tests[2].test_metadata.name, "not_null") self.assertIsNone(tests[2].test_metadata.namespace) self.assertEqual( tests[2].test_metadata.kwargs, { "column_name": "color", "model": "{{ get_where_subquery(ref('my_model')) }}", }, ) file_id = "snowplow://" + normalize("models/test_one.yml") self.assertIn(file_id, self.parser.manifest.files) schema_file_test_ids = self.parser.manifest.files[file_id].get_all_test_ids() self.assertEqual(sorted(schema_file_test_ids), [t.unique_id for t in tests]) self.assertEqual(self.parser.manifest.files[file_id].node_patches, ["model.root.my_model"]) class SchemaParserVersionedModels(SchemaParserTest): def setUp(self): super().setUp() my_model_v1_node = MockNode( package="snowplow", name="arbitrary_file_name", config=mock.MagicMock(enabled=True), refs=[], sources=[], patch_path=None, file_id="snowplow://models/arbitrary_file_name.sql", ) my_model_v1_source_file = self.source_file_for("", "arbitrary_file_name.sql", "models") my_model_v2_node = MockNode( package="snowplow", name="my_model_v2", config=mock.MagicMock(enabled=True), refs=[], sources=[], patch_path=None, file_id="snowplow://models/my_model_v2.sql", ) my_model_v2_source_file = self.source_file_for("", "my_model_v2.sql", "models") nodes = { my_model_v1_node.unique_id: my_model_v1_node, my_model_v2_node.unique_id: my_model_v2_node, } macros = {m.unique_id: m for m in generate_name_macros("root")} files = { my_model_v1_source_file.file_id: my_model_v1_source_file, my_model_v2_source_file.file_id: my_model_v2_source_file, } self.manifest = Manifest(nodes=nodes, macros=macros, files=files) self.manifest.ref_lookup self.parser = SchemaParser( project=self.snowplow_project_config, manifest=self.manifest, root_project=self.root_project_config, ) def test__read_versioned_model_tests(self): block = self.yaml_block_for(MULTIPLE_TABLE_VERSIONED_MODEL_TESTS, "test_one.yml") dct = yaml_from_file(block.file, validate=True) self.parser.parse_file(block, dct) self.assertEqual(len(list(self.parser.manifest.sources)), 0) self.assertEqual(len(list(self.parser.manifest.nodes)), 5) def test__parse_versioned_model_tests(self): block = self.file_block_for(MULTIPLE_TABLE_VERSIONED_MODEL_TESTS, "test_one.yml") self.parser.manifest.files[block.file.file_id] = block.file dct = yaml_from_file(block.file, validate=True) self.parser.parse_file(block, dct) self.assert_has_manifest_lengths(self.parser.manifest, nodes=5) all_nodes = sorted(self.parser.manifest.nodes.values(), key=lambda n: n.unique_id) tests = [node for node in all_nodes if node.resource_type == NodeType.Test] # test on color column on my_model v1 self.assertEqual(tests[0].config.severity, "WARN") self.assertEqual(tests[0].tags, []) self.assertEqual(tests[0].refs, [RefArgs(name="my_model", version="1")]) self.assertEqual(tests[0].column_name, "color") self.assertEqual(tests[0].package_name, "snowplow") self.assertTrue(tests[0].name.startswith("not_null")) self.assertEqual(tests[0].fqn, ["snowplow", tests[0].name]) self.assertEqual( tests[0].unique_id.split("."), ["test", "snowplow", tests[0].name, "b704420587"] ) self.assertEqual(tests[0].test_metadata.name, "not_null") self.assertIsNone(tests[0].test_metadata.namespace) self.assertEqual( tests[0].test_metadata.kwargs, { "column_name": "color", "model": "{{ get_where_subquery(ref('my_model', version='1')) }}", }, ) # test on color column on my_model v2 self.assertEqual(tests[1].config.severity, "WARN") self.assertEqual(tests[1].tags, []) self.assertEqual(tests[1].refs, [RefArgs(name="my_model", version="2")]) self.assertEqual(tests[1].column_name, "color") self.assertEqual(tests[1].fqn, ["snowplow", tests[1].name]) self.assertTrue(tests[1].name.startswith("not_null")) self.assertEqual(tests[1].package_name, "snowplow") self.assertEqual( tests[1].unique_id.split("."), ["test", "snowplow", tests[1].name, "3375708d04"] ) self.assertEqual(tests[1].test_metadata.name, "not_null") self.assertIsNone(tests[0].test_metadata.namespace) self.assertEqual( tests[1].test_metadata.kwargs, { "column_name": "color", "model": "{{ get_where_subquery(ref('my_model', version='2')) }}", }, ) # model uniqueness test on column on my_model v2 self.assertEqual(tests[2].config.severity, "ERROR") self.assertEqual(tests[2].tags, []) self.assertEqual(tests[2].refs, [RefArgs(name="my_model", version="2")]) self.assertIsNone(tests[2].column_name) self.assertEqual(tests[2].package_name, "snowplow") self.assertTrue(tests[2].name.startswith("unique")) self.assertEqual(tests[2].fqn, ["snowplow", tests[2].name]) self.assertEqual( tests[2].unique_id.split("."), ["test", "snowplow", tests[2].name, "29b09359d1"] ) self.assertEqual(tests[2].test_metadata.name, "unique") self.assertIsNone(tests[2].test_metadata.namespace) self.assertEqual( tests[2].test_metadata.kwargs, { "column_name": "color", "model": "{{ get_where_subquery(ref('my_model', version='2')) }}", }, ) file_id = "snowplow://" + normalize("models/test_one.yml") self.assertIn(file_id, self.parser.manifest.files) schema_file_test_ids = self.parser.manifest.files[file_id].get_all_test_ids() self.assertEqual(sorted(schema_file_test_ids), [t.unique_id for t in tests]) self.assertEqual( self.parser.manifest.files[file_id].node_patches, ["model.snowplow.my_model.v1", "model.snowplow.my_model.v2"], ) def test__parsed_versioned_models(self): block = self.file_block_for(MULTIPLE_TABLE_VERSIONED_MODEL, "test_one.yml") self.parser.manifest.files[block.file.file_id] = block.file dct = yaml_from_file(block.file, validate=True) self.parser.parse_file(block, dct) self.assert_has_manifest_lengths(self.parser.manifest, nodes=2) def test__parsed_versioned_models_contract_enforced(self): block = self.file_block_for( MULTIPLE_TABLE_VERSIONED_MODEL_CONTRACT_ENFORCED, "test_one.yml" ) self.parser.manifest.files[block.file.file_id] = block.file dct = yaml_from_file(block.file, validate=True) self.parser.parse_file(block, dct) self.assert_has_manifest_lengths(self.parser.manifest, nodes=2) for node in self.parser.manifest.nodes.values(): assert node.contract.enforced node.build_contract_checksum.assert_called() def test__parsed_versioned_models_v0(self): block = self.file_block_for(MULTIPLE_TABLE_VERSIONED_MODEL_V0, "test_one.yml") self.parser.manifest.files[block.file.file_id] = block.file dct = yaml_from_file(block.file, validate=True) self.parser.parse_file(block, dct) self.assert_has_manifest_lengths(self.parser.manifest, nodes=2) def test__parsed_versioned_models_v0_latest_version(self): block = self.file_block_for( MULTIPLE_TABLE_VERSIONED_MODEL_V0_LATEST_VERSION, "test_one.yml" ) self.parser.manifest.files[block.file.file_id] = block.file dct = yaml_from_file(block.file, validate=True) self.parser.parse_file(block, dct) self.assert_has_manifest_lengths(self.parser.manifest, nodes=2) sql_model = """ {{ config(materialized="table") }} select 1 as id """ sql_model_parse_error = "{{ SYNTAX ERROR }}" python_model = """ import textblob import text as a from torch import b import textblob.text import sklearn def model(dbt, session): dbt.config( materialized='table', packages=['sklearn==0.1.0'] ) df0 = dbt.ref("a_model").to_pandas() df1 = dbt.ref("my_sql_model").task.limit(2) df2 = dbt.ref("my_sql_model_1") df3 = dbt.ref("my_sql_model_2") df4 = dbt.source("test", 'table1').limit(max=[max(dbt.ref('something'))]) df5 = [dbt.ref('test1')] a_dict = {'test2': dbt.ref('test2')} df5 = {'test2': dbt.ref('test3')} df6 = [dbt.ref("test4")] f"{dbt.ref('test5')}" df = df0.limit(2) return df """ python_model_config = """ def model(dbt, session): dbt.config.get("param_1") dbt.config.get("param_2") return dbt.ref("some_model") """ python_model_config_with_defaults = """ def model(dbt, session): dbt.config.get("param_None", None) dbt.config.get("param_Str", "default") dbt.config.get("param_List", [1, 2]) return dbt.ref("some_model") """ python_model_single_argument = """ def model(dbt): dbt.config(materialized="table") return dbt.ref("some_model") """ python_model_no_argument = """ import pandas as pd def model(): return pd.dataframe([1, 2]) """ python_model_incorrect_argument_name = """ def model(tbd, session): tbd.config(materialized="table") return tbd.ref("some_model") """ python_model_multiple_models = """ def model(dbt, session): dbt.config(materialized='table') return dbt.ref("some_model") def model(dbt, session): dbt.config(materialized='table') return dbt.ref("some_model") """ python_model_incorrect_function_name = """ def model1(dbt, session): dbt.config(materialized='table') return dbt.ref("some_model") """ python_model_empty_file = """ """ python_model_multiple_returns = """ def model(dbt, session): dbt.config(materialized='table') return dbt.ref("some_model"), dbt.ref("some_other_model") """ python_model_f_string = """ # my_python_model.py import pandas as pd def model(dbt, fal): dbt.config(materialized="table") print(f"my var: {dbt.config.get('my_var')}") # Prints "my var: None" df: pd.DataFrame = dbt.ref("some_model") return df """ python_model_no_return = """ def model(dbt, session): dbt.config(materialized='table') """ python_model_single_return = """ import pandas as pd def model(dbt, session): dbt.config(materialized='table') return pd.dataframe([1, 2]) """ python_model_incorrect_ref = """ def model(dbt, session): model_names = ["orders", "customers"] models = [] for model_name in model_names: models.extend(dbt.ref(model_name)) return models[0] """ python_model_default_materialization = """ import pandas as pd def model(dbt, session): return pd.dataframe([1, 2]) """ python_model_custom_materialization = """ import pandas as pd def model(dbt, session): dbt.config(materialized="incremental") return pd.dataframe([1, 2]) """ python_model_meta_get = """ def model(dbt, session): dbt.config( materialized='table', meta={'owner': 'data-team', 'priority': 'high'} ) owner = dbt.config.meta_get('owner') priority = dbt.config.meta_get('priority') return dbt.ref("some_model") """ python_model_meta_get_with_defaults = """ def model(dbt, session): dbt.config( materialized='table', meta={'owner': 'data-team'} ) owner = dbt.config.meta_get('owner', 'default-owner') priority = dbt.config.meta_get('priority', 'low') env = dbt.config.meta_get('environment', 'dev') return dbt.ref("some_model") """ python_model_mixed_config_and_meta_get = """ def model(dbt, session): dbt.config( materialized='table', meta={'owner': 'data-team', 'priority': 'high'} ) # Regular config access mat = dbt.config.get('materialized') # Meta access owner = dbt.config.meta_get('owner') priority = dbt.config.meta_get('priority', 'low') return dbt.ref("some_model") """ python_model_meta_get_no_args = """ def model(dbt, session): dbt.config(materialized='table') value = dbt.config.meta_get() return dbt.ref("some_model") """ python_model_meta_get_too_many_args = """ def model(dbt, session): dbt.config(materialized='table') value = dbt.config.meta_get('key', 'default', 'extra') return dbt.ref("some_model") """ class ModelParserTest(BaseParserTest): def setUp(self): super().setUp() set_from_args( Namespace( warn_error=False, state_modified_compare_more_unrendered_values=False, require_generic_test_arguments_property=False, ), None, ) self.parser = ModelParser( project=self.snowplow_project_config, manifest=self.manifest, root_project=self.root_project_config, ) def file_block_for(self, data, filename): return super().file_block_for(data, filename, "models") def test_basic(self): block = self.file_block_for(sql_model, "nested/model_1.sql") self.parser.manifest.files[block.file.file_id] = block.file self.parser.parse_file(block) self.assert_has_manifest_lengths(self.parser.manifest, nodes=1) node = list(self.parser.manifest.nodes.values())[0] expected = ModelNode( alias="model_1", name="model_1", database="test", schema="analytics", resource_type=NodeType.Model, unique_id="model.snowplow.model_1", fqn=["snowplow", "nested", "model_1"], package_name="snowplow", original_file_path=normalize("models/nested/model_1.sql"), config=ModelConfig(materialized="table"), path=normalize("nested/model_1.sql"), language="sql", raw_code=sql_model, checksum=block.file.checksum, unrendered_config={"materialized": "table"}, config_call_dict={ "materialized": "table", }, ) assertEqualNodes(node, expected) file_id = "snowplow://" + normalize("models/nested/model_1.sql") self.assertIn(file_id, self.parser.manifest.files) self.assertEqual(self.parser.manifest.files[file_id].nodes, ["model.snowplow.model_1"]) def test_sql_model_parse_error(self): block = self.file_block_for(sql_model_parse_error, "nested/model_1.sql") with self.assertRaises(CompilationError): self.parser.parse_file(block) def test_python_model_parse(self): block = self.file_block_for(python_model, "nested/py_model.py") self.parser.manifest.files[block.file.file_id] = block.file self.parser.parse_file(block) self.assert_has_manifest_lengths(self.parser.manifest, nodes=1) node = list(self.parser.manifest.nodes.values())[0] # we decided to not detect and auto supply for now since import name doesn't always match library name python_packages = ["sklearn==0.1.0"] expected = ModelNode( alias="py_model", name="py_model", database="test", schema="analytics", resource_type=NodeType.Model, unique_id="model.snowplow.py_model", fqn=["snowplow", "nested", "py_model"], package_name="snowplow", original_file_path=normalize("models/nested/py_model.py"), config=ModelConfig(materialized="table", packages=python_packages), # config.packages = ['textblob'] path=normalize("nested/py_model.py"), language="python", raw_code=python_model, checksum=block.file.checksum, unrendered_config={"materialized": "table", "packages": python_packages}, config_call_dict={"materialized": "table", "packages": python_packages}, refs=[ RefArgs(name="a_model"), RefArgs("my_sql_model"), RefArgs("my_sql_model_1"), RefArgs("my_sql_model_2"), RefArgs("something"), RefArgs("test1"), RefArgs("test2"), RefArgs("test3"), RefArgs("test4"), RefArgs("test5"), ], sources=[["test", "table1"]], ) assertEqualNodes(node, expected) file_id = "snowplow://" + normalize("models/nested/py_model.py") self.assertIn(file_id, self.parser.manifest.files) self.assertEqual(self.parser.manifest.files[file_id].nodes, ["model.snowplow.py_model"]) def test_python_model_config(self): block = self.file_block_for(python_model_config, "nested/py_model.py") self.parser.manifest.files[block.file.file_id] = block.file self.parser.parse_file(block) node = list(self.parser.manifest.nodes.values())[0] self.assertEqual(node.config.to_dict()["config_keys_used"], ["param_1", "param_2"]) def test_python_model_f_string_config(self): block = self.file_block_for(python_model_f_string, "nested/py_model.py") self.parser.manifest.files[block.file.file_id] = block.file self.parser.parse_file(block) node = list(self.parser.manifest.nodes.values())[0] self.assertEqual(node.config.to_dict()["config_keys_used"], ["my_var"]) def test_python_model_config_with_defaults(self): block = self.file_block_for(python_model_config_with_defaults, "nested/py_model.py") self.parser.manifest.files[block.file.file_id] = block.file self.parser.parse_file(block) node = list(self.parser.manifest.nodes.values())[0] default_values = node.config.to_dict()["config_keys_defaults"] self.assertIsNone(default_values[0]) self.assertEqual(default_values[1], "default") self.assertEqual(default_values[2], [1, 2]) def test_python_model_single_argument(self): block = self.file_block_for(python_model_single_argument, "nested/py_model.py") self.parser.manifest.files[block.file.file_id] = block.file with self.assertRaises(ParsingError): self.parser.parse_file(block) def test_python_model_no_argument(self): block = self.file_block_for(python_model_no_argument, "nested/py_model.py") self.parser.manifest.files[block.file.file_id] = block.file with self.assertRaises(ParsingError): self.parser.parse_file(block) def test_python_model_incorrect_argument_name(self): block = self.file_block_for(python_model_incorrect_argument_name, "nested/py_model.py") self.parser.manifest.files[block.file.file_id] = block.file with self.assertRaises(ParsingError): self.parser.parse_file(block) def test_python_model_multiple_models(self): block = self.file_block_for(python_model_multiple_models, "nested/py_model.py") self.parser.manifest.files[block.file.file_id] = block.file with self.assertRaises(ParsingError): self.parser.parse_file(block) def test_python_model_incorrect_function_name(self): block = self.file_block_for(python_model_incorrect_function_name, "nested/py_model.py") self.parser.manifest.files[block.file.file_id] = block.file with self.assertRaises(ParsingError): self.parser.parse_file(block) def test_python_model_empty_file(self): block = self.file_block_for(python_model_empty_file, "nested/py_model.py") self.parser.manifest.files[block.file.file_id] = block.file self.assertIsNone(self.parser.parse_file(block)) def test_python_model_multiple_returns(self): block = self.file_block_for(python_model_multiple_returns, "nested/py_model.py") self.parser.manifest.files[block.file.file_id] = block.file with self.assertRaises(ParsingError): self.parser.parse_file(block) def test_python_model_no_return(self): block = self.file_block_for(python_model_no_return, "nested/py_model.py") self.parser.manifest.files[block.file.file_id] = block.file with self.assertRaises(ParsingError): self.parser.parse_file(block) def test_python_model_single_return(self): block = self.file_block_for(python_model_single_return, "nested/py_model.py") self.parser.manifest.files[block.file.file_id] = block.file self.assertIsNone(self.parser.parse_file(block)) def test_python_model_incorrect_ref(self): block = self.file_block_for(python_model_incorrect_ref, "nested/py_model.py") self.parser.manifest.files[block.file.file_id] = block.file with self.assertRaises(ParsingError): self.parser.parse_file(block) def test_python_model_default_materialization(self): block = self.file_block_for(python_model_default_materialization, "nested/py_model.py") self.parser.manifest.files[block.file.file_id] = block.file self.parser.parse_file(block) node = list(self.parser.manifest.nodes.values())[0] self.assertEqual(node.get_materialization(), "table") def test_python_model_custom_materialization(self): block = self.file_block_for(python_model_custom_materialization, "nested/py_model.py") self.parser.manifest.files[block.file.file_id] = block.file self.parser.parse_file(block) node = list(self.parser.manifest.nodes.values())[0] self.assertEqual(node.get_materialization(), "incremental") def test_python_model_meta_get(self): """Test that dbt.config.meta_get() calls are tracked correctly""" block = self.file_block_for(python_model_meta_get, "nested/py_model.py") self.parser.manifest.files[block.file.file_id] = block.file self.parser.parse_file(block) node = list(self.parser.manifest.nodes.values())[0] config_dict = node.config.to_dict() self.assertEqual(config_dict["meta_keys_used"], ["owner", "priority"]) # Both keys should have None as default since no explicit default was provided self.assertIsNone(config_dict["meta_keys_defaults"][0]) self.assertIsNone(config_dict["meta_keys_defaults"][1]) def test_python_model_meta_get_with_defaults(self): """Test that dbt.config.meta_get() default values are tracked correctly""" block = self.file_block_for(python_model_meta_get_with_defaults, "nested/py_model.py") self.parser.manifest.files[block.file.file_id] = block.file self.parser.parse_file(block) node = list(self.parser.manifest.nodes.values())[0] config_dict = node.config.to_dict() self.assertEqual(config_dict["meta_keys_used"], ["owner", "priority", "environment"]) default_values = config_dict["meta_keys_defaults"] self.assertEqual(default_values[0], "default-owner") self.assertEqual(default_values[1], "low") self.assertEqual(default_values[2], "dev") def test_python_model_mixed_config_and_meta_get(self): """Test that config.get() and config.meta_get() can be used together""" block = self.file_block_for(python_model_mixed_config_and_meta_get, "nested/py_model.py") self.parser.manifest.files[block.file.file_id] = block.file self.parser.parse_file(block) node = list(self.parser.manifest.nodes.values())[0] config_dict = node.config.to_dict() # Should track both config.get and meta_get calls self.assertEqual(config_dict["config_keys_used"], ["materialized"]) self.assertEqual(config_dict["meta_keys_used"], ["owner", "priority"]) self.assertIsNone(config_dict["config_keys_defaults"][0]) self.assertIsNone(config_dict["meta_keys_defaults"][0]) self.assertEqual(config_dict["meta_keys_defaults"][1], "low") def test_python_model_meta_get_no_args(self): """Test that meta_get() with no arguments raises an error""" block = self.file_block_for(python_model_meta_get_no_args, "nested/py_model.py") self.parser.manifest.files[block.file.file_id] = block.file with self.assertRaises(ParsingError) as exc: self.parser.parse_file(block) self.assertIn("dbt.config.meta_get() requires at least one argument", str(exc.exception)) def test_python_model_meta_get_too_many_args(self): """Test that meta_get() with more than 2 arguments raises an error""" block = self.file_block_for(python_model_meta_get_too_many_args, "nested/py_model.py") self.parser.manifest.files[block.file.file_id] = block.file with self.assertRaises(ParsingError) as exc: self.parser.parse_file(block) self.assertIn("dbt.config.meta_get() takes at most 2 arguments", str(exc.exception)) class StaticModelParserTest(BaseParserTest): def setUp(self): super().setUp() self.parser = ModelParser( project=self.snowplow_project_config, manifest=self.manifest, root_project=self.root_project_config, ) def file_block_for(self, data, filename): return super().file_block_for(data, filename, "models") # tests that when the ref built-in is overriden with a macro definition # that the ModelParser can detect it. This does not test that the static # parser does not run in this case. That test is in integration test suite 072 def test_built_in_macro_override_detection(self): macro_unique_id = "macro.root.ref" self.parser.manifest.macros[macro_unique_id] = Macro( name="ref", resource_type=NodeType.Macro, unique_id=macro_unique_id, package_name="root", original_file_path=normalize("macros/macro.sql"), path=normalize("macros/macro.sql"), macro_sql='{% macro ref(model_name) %}{% set x = raise("boom") %}{% endmacro %}', ) raw_code = '{{ config(materialized="table") }}select 1 as id' block = self.file_block_for(raw_code, "nested/model_1.sql") node = ModelNode( alias="model_1", name="model_1", database="test", schema="analytics", resource_type=NodeType.Model, unique_id="model.snowplow.model_1", fqn=["snowplow", "nested", "model_1"], package_name="snowplow", original_file_path=normalize("models/nested/model_1.sql"), config=ModelConfig(materialized="table"), path=normalize("nested/model_1.sql"), language="sql", raw_code=raw_code, checksum=block.file.checksum, unrendered_config={"materialized": "table"}, ) assert self.parser._has_banned_macro(node) # TODO class StaticModelParserUnitTest(BaseParserTest): # _get_config_call_dict # _shift_sources # _get_exp_sample_result # _get_stable_sample_result # _get_sample_result def setUp(self): super().setUp() self.parser = ModelParser( project=self.snowplow_project_config, manifest=self.manifest, root_project=self.root_project_config, ) self.example_node = ModelNode( alias="model_1", name="model_1", database="test", schema="analytics", resource_type=NodeType.Model, unique_id="model.snowplow.model_1", fqn=["snowplow", "nested", "model_1"], package_name="snowplow", original_file_path=normalize("models/nested/model_1.sql"), config=ModelConfig(materialized="table"), path=normalize("nested/model_1.sql"), language="sql", raw_code='{{ config(materialized="table") }}select 1 as id', checksum=None, unrendered_config={"materialized": "table"}, ) self.example_config = ContextConfig( self.root_project_config, self.example_node.fqn, self.example_node.resource_type, self.snowplow_project_config, ) def file_block_for(self, data, filename): return super().file_block_for(data, filename, "models") # tests that configs get extracted properly. the function should respect merge behavior, # but becuase it's only reading from one dictionary it won't matter except in edge cases # like this example with tags changing type to a list. def test_config_shifting(self): static_parser_result = { "configs": [("hello", "world"), ("flag", True), ("tags", "tag1"), ("tags", "tag2")] } expected = {"hello": "world", "flag": True, "tags": ["tag1", "tag2"]} got = _get_config_call_dict(static_parser_result) self.assertEqual(expected, got) def test_source_shifting(self): static_parser_result = {"sources": [("abc", "def"), ("x", "y")]} expected = {"sources": [["abc", "def"], ["x", "y"]]} got = _shift_sources(static_parser_result) self.assertEqual(expected, got) def test_sample_results(self): # --- missed ref --- # node = deepcopy(self.example_node) config = deepcopy(self.example_config) sample_node = deepcopy(self.example_node) sample_config = deepcopy(self.example_config) sample_node.refs = [] node.refs = ["myref"] result = _get_sample_result(sample_node, sample_config, node, config) self.assertEqual([(7, "missed_ref_value")], result) # --- false positive ref --- # node = deepcopy(self.example_node) config = deepcopy(self.example_config) sample_node = deepcopy(self.example_node) sample_config = deepcopy(self.example_config) sample_node.refs = ["myref"] node.refs = [] result = _get_sample_result(sample_node, sample_config, node, config) self.assertEqual([(6, "false_positive_ref_value")], result) # --- missed source --- # node = deepcopy(self.example_node) config = deepcopy(self.example_config) sample_node = deepcopy(self.example_node) sample_config = deepcopy(self.example_config) sample_node.sources = [] node.sources = [["abc", "def"]] result = _get_sample_result(sample_node, sample_config, node, config) self.assertEqual([(5, "missed_source_value")], result) # --- false positive source --- # node = deepcopy(self.example_node) config = deepcopy(self.example_config) sample_node = deepcopy(self.example_node) sample_config = deepcopy(self.example_config) sample_node.sources = [["abc", "def"]] node.sources = [] result = _get_sample_result(sample_node, sample_config, node, config) self.assertEqual([(4, "false_positive_source_value")], result) # --- missed config --- # node = deepcopy(self.example_node) config = deepcopy(self.example_config) sample_node = deepcopy(self.example_node) sample_config = deepcopy(self.example_config) sample_config._config_call_dict = {} config._config_call_dict = {"key": "value"} result = _get_sample_result(sample_node, sample_config, node, config) self.assertEqual([(3, "missed_config_value")], result) # --- false positive config --- # node = deepcopy(self.example_node) config = deepcopy(self.example_config) sample_node = deepcopy(self.example_node) sample_config = deepcopy(self.example_config) sample_config._config_call_dict = {"key": "value"} config._config_call_dict = {} result = _get_sample_result(sample_node, sample_config, node, config) self.assertEqual([(2, "false_positive_config_value")], result) def test_exp_sample_results(self): node = deepcopy(self.example_node) config = deepcopy(self.example_config) sample_node = deepcopy(self.example_node) sample_config = deepcopy(self.example_config) result = _get_exp_sample_result(sample_node, sample_config, node, config) self.assertEqual(["00_experimental_exact_match"], result) def test_stable_sample_results(self): node = deepcopy(self.example_node) config = deepcopy(self.example_config) sample_node = deepcopy(self.example_node) sample_config = deepcopy(self.example_config) result = _get_stable_sample_result(sample_node, sample_config, node, config) self.assertEqual(["80_stable_exact_match"], result) class SnapshotParserTest(BaseParserTest): def setUp(self): super().setUp() self.parser = SnapshotParser( project=self.snowplow_project_config, manifest=self.manifest, root_project=self.root_project_config, ) def file_block_for(self, data, filename): return super().file_block_for(data, filename, "snapshots") def test_parse_error(self): block = self.file_block_for( "{% snapshot foo %}select 1 as id{%snapshot bar %}{% endsnapshot %}", "nested/snap_1.sql", ) with self.assertRaises(CompilationError): self.parser.parse_file(block) def test_single_block(self): raw_code = """{{ config(unique_key="id", target_schema="analytics", target_database="dbt", strategy="timestamp", updated_at="last_update") }} select 1 as id, now() as last_update""" full_file = """ {{% snapshot foo %}}{}{{% endsnapshot %}} """.format( raw_code ) block = self.file_block_for(full_file, "nested/snap_1.sql") self.parser.manifest.files[block.file.file_id] = block.file self.parser.parse_file(block) self.assert_has_manifest_lengths(self.parser.manifest, nodes=1) node = list(self.parser.manifest.nodes.values())[0] expected = SnapshotNode( alias="foo", name="foo", # the `database` entry is overrridden by the target_database config database="dbt", schema="analytics", resource_type=NodeType.Snapshot, unique_id="snapshot.snowplow.foo", fqn=["snowplow", "nested", "snap_1", "foo"], package_name="snowplow", original_file_path=normalize("snapshots/nested/snap_1.sql"), config=SnapshotConfig( strategy="timestamp", updated_at="last_update", target_database="dbt", target_schema="analytics", unique_key="id", materialized="snapshot", ), path="foo.sql", language="sql", raw_code=raw_code, checksum=block.file.checksum, unrendered_config={ "unique_key": "id", "target_schema": "analytics", "target_database": "dbt", "strategy": "timestamp", "updated_at": "last_update", }, config_call_dict={ "strategy": "timestamp", "target_database": "dbt", "target_schema": "analytics", "unique_key": "id", "updated_at": "last_update", }, unrendered_config_call_dict={}, ) assertEqualNodes(expected, node) file_id = "snowplow://" + normalize("snapshots/nested/snap_1.sql") self.assertIn(file_id, self.parser.manifest.files) self.assertEqual(self.parser.manifest.files[file_id].nodes, ["snapshot.snowplow.foo"]) def test_multi_block(self): raw_1 = """ {{ config(unique_key="id", target_schema="analytics", target_database="dbt", strategy="timestamp", updated_at="last_update") }} select 1 as id, now() as last_update """ raw_2 = """ {{ config(unique_key="id", target_schema="analytics", target_database="dbt", strategy="timestamp", updated_at="last_update") }} select 2 as id, now() as last_update """ full_file = """ {{% snapshot foo %}}{}{{% endsnapshot %}} {{% snapshot bar %}}{}{{% endsnapshot %}} """.format( raw_1, raw_2 ) block = self.file_block_for(full_file, "nested/snap_1.sql") self.parser.manifest.files[block.file.file_id] = block.file self.parser.parse_file(block) self.assert_has_manifest_lengths(self.parser.manifest, nodes=2) nodes = sorted(self.parser.manifest.nodes.values(), key=lambda n: n.name) expect_foo = SnapshotNode( alias="foo", name="foo", database="dbt", schema="analytics", resource_type=NodeType.Snapshot, unique_id="snapshot.snowplow.foo", fqn=["snowplow", "nested", "snap_1", "foo"], package_name="snowplow", original_file_path=normalize("snapshots/nested/snap_1.sql"), config=SnapshotConfig( strategy="timestamp", updated_at="last_update", target_database="dbt", target_schema="analytics", unique_key="id", materialized="snapshot", ), path="foo.sql", language="sql", raw_code=raw_1, checksum=block.file.checksum, unrendered_config={ "unique_key": "id", "target_schema": "analytics", "target_database": "dbt", "strategy": "timestamp", "updated_at": "last_update", }, config_call_dict={ "strategy": "timestamp", "target_database": "dbt", "target_schema": "analytics", "unique_key": "id", "updated_at": "last_update", }, # Empty until state_modified_compare_more_unrendered_values=True unrendered_config_call_dict={}, ) expect_bar = SnapshotNode( alias="bar", name="bar", database="dbt", schema="analytics", resource_type=NodeType.Snapshot, unique_id="snapshot.snowplow.bar", fqn=["snowplow", "nested", "snap_1", "bar"], package_name="snowplow", original_file_path=normalize("snapshots/nested/snap_1.sql"), config=SnapshotConfig( strategy="timestamp", updated_at="last_update", target_database="dbt", target_schema="analytics", unique_key="id", materialized="snapshot", ), path="bar.sql", language="sql", raw_code=raw_2, checksum=block.file.checksum, unrendered_config={ "unique_key": "id", "target_schema": "analytics", "target_database": "dbt", "strategy": "timestamp", "updated_at": "last_update", }, config_call_dict={ "strategy": "timestamp", "target_database": "dbt", "target_schema": "analytics", "unique_key": "id", "updated_at": "last_update", }, # Empty until state_modified_compare_more_unrendered_values=True unrendered_config_call_dict={}, ) assertEqualNodes(nodes[0], expect_bar) assertEqualNodes(nodes[1], expect_foo) file_id = "snowplow://" + normalize("snapshots/nested/snap_1.sql") self.assertIn(file_id, self.parser.manifest.files) self.assertEqual( sorted(self.parser.manifest.files[file_id].nodes), ["snapshot.snowplow.bar", "snapshot.snowplow.foo"], ) class MacroParserTest(BaseParserTest): def setUp(self): super().setUp() self.parser = MacroParser(project=self.snowplow_project_config, manifest=Manifest()) def file_block_for(self, data, filename): return super().file_block_for(data, filename, "macros") def test_single_block(self): raw_code = "{% macro foo(a, b) %}a ~ b{% endmacro %}" block = self.file_block_for(raw_code, "macro.sql") self.parser.manifest.files[block.file.file_id] = block.file self.parser.parse_file(block) self.assertEqual(len(self.parser.manifest.macros), 1) macro = list(self.parser.manifest.macros.values())[0] expected = Macro( name="foo", resource_type=NodeType.Macro, unique_id="macro.snowplow.foo", package_name="snowplow", original_file_path=normalize("macros/macro.sql"), path=normalize("macros/macro.sql"), macro_sql=raw_code, ) assertEqualNodes(macro, expected) file_id = "snowplow://" + normalize("macros/macro.sql") self.assertIn(file_id, self.parser.manifest.files) self.assertEqual(self.parser.manifest.files[file_id].macros, ["macro.snowplow.foo"]) def test_multiple_blocks(self): raw_code = ( "{% macro foo(a, b) %}a ~ b{% endmacro %}\n{% macro bar(c, d) %}c + d{% endmacro %}" ) block = self.file_block_for(raw_code, "macro.sql") self.parser.manifest.files[block.file.file_id] = block.file self.parser.parse_file(block) self.assertEqual(len(self.parser.manifest.macros), 2) macros = sorted(self.parser.manifest.macros.values(), key=lambda m: m.name) expected_bar = Macro( name="bar", resource_type=NodeType.Macro, unique_id="macro.snowplow.bar", package_name="snowplow", original_file_path=normalize("macros/macro.sql"), path=normalize("macros/macro.sql"), macro_sql="{% macro bar(c, d) %}c + d{% endmacro %}", ) expected_foo = Macro( name="foo", resource_type=NodeType.Macro, unique_id="macro.snowplow.foo", package_name="snowplow", original_file_path=normalize("macros/macro.sql"), path=normalize("macros/macro.sql"), macro_sql="{% macro foo(a, b) %}a ~ b{% endmacro %}", ) assertEqualNodes(macros[0], expected_bar) assertEqualNodes(macros[1], expected_foo) file_id = "snowplow://" + normalize("macros/macro.sql") self.assertIn(file_id, self.parser.manifest.files) self.assertEqual( sorted(self.parser.manifest.files[file_id].macros), ["macro.snowplow.bar", "macro.snowplow.foo"], ) class SingularTestParserTest(BaseParserTest): def setUp(self): super().setUp() self.parser = SingularTestParser( project=self.snowplow_project_config, manifest=self.manifest, root_project=self.root_project_config, ) def file_block_for(self, data, filename): return super().file_block_for(data, filename, "tests") def test_basic(self): raw_code = 'select * from {{ ref("blah") }} limit 0' block = self.file_block_for(raw_code, "test_1.sql") self.manifest.files[block.file.file_id] = block.file self.parser.parse_file(block) self.assert_has_manifest_lengths(self.parser.manifest, nodes=1) node = list(self.parser.manifest.nodes.values())[0] expected = SingularTestNode( alias="test_1", name="test_1", database="test", schema="dbt_test__audit", resource_type=NodeType.Test, unique_id="test.snowplow.test_1", fqn=["snowplow", "test_1"], package_name="snowplow", original_file_path=normalize("tests/test_1.sql"), refs=[RefArgs(name="blah")], config=TestConfig(severity="ERROR"), tags=[], path=normalize("test_1.sql"), language="sql", raw_code=raw_code, checksum=block.file.checksum, unrendered_config={}, ) assertEqualNodes(node, expected) file_id = "snowplow://" + normalize("tests/test_1.sql") self.assertIn(file_id, self.parser.manifest.files) self.assertEqual(self.parser.manifest.files[file_id].nodes, ["test.snowplow.test_1"]) class GenericTestParserTest(BaseParserTest): # generic tests in the test-paths directory currently leverage the macro parser def setUp(self): super().setUp() self.parser = GenericTestParser(project=self.snowplow_project_config, manifest=Manifest()) def file_block_for(self, data, filename): return super().file_block_for(data, filename, "tests/generic") def test_basic(self): raw_code = "{% test not_null(model, column_name) %}select * from {{ model }} where {{ column_name }} is null {% endtest %}" block = self.file_block_for(raw_code, "test_1.sql") self.parser.manifest.files[block.file.file_id] = block.file self.parser.parse_file(block) node = list(self.parser.manifest.macros.values())[0] expected = Macro( name="test_not_null", resource_type=NodeType.Macro, unique_id="macro.snowplow.test_not_null", package_name="snowplow", original_file_path=normalize("tests/generic/test_1.sql"), path=normalize("tests/generic/test_1.sql"), macro_sql=raw_code, ) assertEqualNodes(node, expected) file_id = "snowplow://" + normalize("tests/generic/test_1.sql") self.assertIn(file_id, self.parser.manifest.files) self.assertEqual( self.parser.manifest.files[file_id].macros, ["macro.snowplow.test_not_null"] ) class AnalysisParserTest(BaseParserTest): def setUp(self): super().setUp() self.parser = AnalysisParser( project=self.snowplow_project_config, manifest=self.manifest, root_project=self.root_project_config, ) def file_block_for(self, data, filename): return super().file_block_for(data, filename, "analyses") def test_basic(self): raw_code = "select 1 as id" block = self.file_block_for(raw_code, "nested/analysis_1.sql") self.manifest.files[block.file.file_id] = block.file self.parser.parse_file(block) self.assert_has_manifest_lengths(self.parser.manifest, nodes=1) node = list(self.parser.manifest.nodes.values())[0] expected = AnalysisNode( alias="analysis_1", name="analysis_1", database="test", schema="analytics", resource_type=NodeType.Analysis, unique_id="analysis.snowplow.analysis_1", fqn=["snowplow", "analysis", "nested", "analysis_1"], package_name="snowplow", original_file_path=normalize("analyses/nested/analysis_1.sql"), depends_on=DependsOn(), config=NodeConfig(), path=normalize("analysis/nested/analysis_1.sql"), language="sql", raw_code=raw_code, checksum=block.file.checksum, unrendered_config={}, relation_name=None, ) assertEqualNodes(node, expected) file_id = "snowplow://" + normalize("analyses/nested/analysis_1.sql") self.assertIn(file_id, self.parser.manifest.files) self.assertEqual( self.parser.manifest.files[file_id].nodes, ["analysis.snowplow.analysis_1"] ) ================================================ FILE: tests/unit/parser/test_partial.py ================================================ import time from copy import deepcopy from typing import Dict, List from unittest import mock import pytest from dbt.contracts.files import ( BaseSourceFile, FileHash, FilePath, ParseFileType, SchemaSourceFile, SourceFile, ) from dbt.contracts.graph.manifest import Manifest from dbt.contracts.graph.nodes import SnapshotNode, SourceDefinition from dbt.node_types import NodeType from dbt.parser.partial import PartialParsing from dbt.tests.util import safe_set_invocation_context from tests.unit.utils import normalize from tests.unit.utils.manifest import ( make_generic_test, make_manifest, make_model, make_singular_test, make_source, make_source_snapshot, ) PROJECT_NAME = "my_test" @pytest.fixture def singular_test(): st = make_singular_test( PROJECT_NAME, "my_singular_test", "select 1 where false", path="tests/my_singular_test.sql" ) st.patch_path = "my_test://tests/tests.yml" return st @pytest.fixture def source() -> SourceDefinition: return make_source(PROJECT_NAME, "my_source", "my_source_table", path="models/schema.yml") @pytest.fixture def source_snapshot(source) -> SnapshotNode: return make_source_snapshot( PROJECT_NAME, "my_test_source_snapshot", source, path="models/schema.yml" ) @pytest.fixture def files(singular_test, source, source_snapshot) -> Dict[str, BaseSourceFile]: project_root = "/users/root" sql_model_file = SourceFile( path=FilePath( project_root=project_root, searched_path="models", relative_path="my_model.sql", modification_time=time.time(), ), checksum=FileHash.from_contents("abcdef"), project_name=PROJECT_NAME, parse_file_type=ParseFileType.Model, nodes=["model.my_test.my_model"], env_vars=[], ) sql_model_file_untouched = SourceFile( path=FilePath( project_root=project_root, searched_path="models", relative_path="my_model_untouched.sql", modification_time=time.time(), ), checksum=FileHash.from_contents("abcdef"), project_name=PROJECT_NAME, parse_file_type=ParseFileType.Model, nodes=["model.my_test.my_model_untouched"], env_vars=[], ) python_model_file = SourceFile( path=FilePath( project_root=project_root, searched_path="models", relative_path="python_model.py", modification_time=time.time(), ), checksum=FileHash.from_contents("lalala"), project_name=PROJECT_NAME, parse_file_type=ParseFileType.Model, nodes=["model.my_test.python_model"], env_vars=[], ) python_model_file_untouched = SourceFile( path=FilePath( project_root=project_root, searched_path="models", relative_path="python_model_untouched.py", modification_time=time.time(), ), checksum=FileHash.from_contents("lalala"), project_name=PROJECT_NAME, parse_file_type=ParseFileType.Model, nodes=["model.my_test.python_model_untouched"], env_vars=[], ) schema_file = SchemaSourceFile( path=FilePath( project_root=project_root, searched_path="models", relative_path="schema.yml", modification_time=time.time(), ), checksum=FileHash.from_contents("ghijkl"), project_name=PROJECT_NAME, parse_file_type=ParseFileType.Schema, dfy={ "version": 2, "models": [ {"name": "my_model", "description": "Test model"}, {"name": "python_model", "description": "python"}, {"name": "not_null", "model": "test.my_test.test_my_model"}, ], "sources": [ {"name": source.source_name, "tables": [{"name": source.name}]}, ], "snapshots": [ { "name": source_snapshot.name, "relation": f"source('{source.source_name}', '{source.name}')", }, ], }, ndp=["model.my_test.my_model"], env_vars={}, data_tests={"models": {"not_null": {"test.my_test.test_my_model": []}}}, snapshots=[f"snapshot.{PROJECT_NAME}.{source_snapshot.name}"], sources=[f"source.{PROJECT_NAME}.{source.source_name}.{source.name}"], ) singular_test_sql_file = SourceFile( path=FilePath( project_root=project_root, searched_path="tests", relative_path=f"{singular_test.name}.sql", modification_time=time.time(), ), checksum=FileHash.from_contents("a test"), project_name=PROJECT_NAME, parse_file_type=ParseFileType.SingularTest, nodes=[f"test.{PROJECT_NAME}.{singular_test.name}"], env_vars=[], ) singular_test_yml_file = SourceFile( path=FilePath( project_root=project_root, searched_path="tests", relative_path="tests.yml", modification_time=time.time(), ), checksum=FileHash.from_contents("a test"), project_name=PROJECT_NAME, parse_file_type=ParseFileType.Schema, env_vars=[], ) return { schema_file.file_id: schema_file, sql_model_file.file_id: sql_model_file, sql_model_file_untouched.file_id: sql_model_file_untouched, python_model_file.file_id: python_model_file, python_model_file_untouched.file_id: python_model_file_untouched, singular_test_sql_file.file_id: singular_test_sql_file, singular_test_yml_file.file_id: singular_test_yml_file, } @pytest.fixture def nodes(singular_test, source, source_snapshot) -> List[NodeType]: patch_path = "my_test://" + normalize("models/schema.yml") my_model = make_model(PROJECT_NAME, "my_model", "", patch_path=patch_path) return [ my_model, make_model(PROJECT_NAME, "my_model_untouched", "", patch_path=patch_path), make_model(PROJECT_NAME, "python_model", "", language="python", patch_path=patch_path), make_model( PROJECT_NAME, "python_model_untouched", "", language="python", patch_path=patch_path ), make_generic_test(PROJECT_NAME, "test", my_model, {}), singular_test, source, source_snapshot, ] @pytest.fixture def manifest(files, nodes, source) -> Manifest: return make_manifest(files=files, nodes=nodes, sources=[source]) @pytest.fixture def partial_parsing(manifest, files): safe_set_invocation_context() return PartialParsing(manifest, deepcopy(files)) def test_simple(partial_parsing, files, nodes): # Nothing has changed assert partial_parsing is not None assert partial_parsing.skip_parsing() is True # Change a model file sql_model_file_id = "my_test://" + normalize("models/my_model.sql") partial_parsing.new_files[sql_model_file_id].checksum = FileHash.from_contents("xyzabc") python_model_file_id = "my_test://" + normalize("models/python_model.py") partial_parsing.new_files[python_model_file_id].checksum = FileHash.from_contents("ohohoh") partial_parsing.build_file_diff() assert partial_parsing.skip_parsing() is False pp_files = partial_parsing.get_parsing_files() pp_files["my_test"]["ModelParser"] = set(pp_files["my_test"]["ModelParser"]) # models has 'patch_path' so we expect to see a SchemaParser file listed schema_file_id = "my_test://" + normalize("models/schema.yml") expected_pp_files = { "my_test": { "ModelParser": set([sql_model_file_id, python_model_file_id]), "SchemaParser": [schema_file_id], } } assert pp_files == expected_pp_files schema_file = files[schema_file_id] schema_file_model_names = set([model["name"] for model in schema_file.pp_dict["models"]]) expected_model_names = set(["python_model", "my_model"]) assert schema_file_model_names == expected_model_names schema_file_model_descriptions = set( [model["description"] for model in schema_file.pp_dict["models"]] ) expected_model_descriptions = set(["Test model", "python"]) assert schema_file_model_descriptions == expected_model_descriptions def test_schedule_nodes_for_parsing_basic(partial_parsing, nodes): assert partial_parsing.file_diff["deleted"] == [] assert partial_parsing.project_parser_files == {} partial_parsing.schedule_nodes_for_parsing([nodes[0].unique_id]) assert partial_parsing.project_parser_files == { "my_test": { "ModelParser": ["my_test://models/my_model.sql"], "SchemaParser": ["my_test://models/schema.yml"], } } def test_schedule_macro_nodes_for_parsing_basic(partial_parsing): # XXX it seems kind of confusing what exactly this function does. # Whoever Changes this function please add more comment. # this rely on the dfy and data_tests fields in schema node to add schema file to reparse partial_parsing.schedule_macro_nodes_for_parsing(["test.my_test.test_my_model"]) assert partial_parsing.project_parser_files == { "my_test": {"SchemaParser": ["my_test://models/schema.yml"]} } def test_schedule_nodes_for_parsing_versioning(partial_parsing, source) -> None: # Modify schema file to add versioning schema_file_id = "my_test://" + normalize("models/schema.yml") partial_parsing.new_files[schema_file_id].checksum = FileHash.from_contents("changed") partial_parsing.new_files[schema_file_id].dfy = { "version": 2, "models": [ { "name": "my_model", "description": "Test model", "latest_version": 1, "versions": [{"v": 1}], }, {"name": "python_model", "description": "python"}, {"name": "not_null", "model": "test.my_test.test_my_model"}, ], "sources": [ {"name": source.source_name, "tables": [{"name": source.name}]}, ], } with mock.patch.object( partial_parsing, "schedule_referencing_nodes_for_parsing" ) as mock_schedule_referencing_nodes_for_parsing: partial_parsing.build_file_diff() partial_parsing.get_parsing_files() mock_schedule_referencing_nodes_for_parsing.assert_called_once_with( "model.my_test.my_model" ) class TestFileDiff: @pytest.fixture def partial_parsing(self, manifest, files): safe_set_invocation_context() saved_files = deepcopy(files) saved_files["my_test://models/python_model_untouched.py"].checksum = ( FileHash.from_contents("something new") ) saved_files["my_test://models/schema.yml"].dfy["sources"][0]["tables"][0][ "description" ] = "test" saved_files["my_test://models/schema.yml"].checksum = FileHash.from_contents( "something new" ) saved_files["my_test://tests/my_singular_test.sql"].checksum = FileHash.from_contents( "something new" ) return PartialParsing(manifest, saved_files) def test_build_file_diff_basic(self, partial_parsing): partial_parsing.build_file_diff() assert set(partial_parsing.file_diff["unchanged"]) == { "my_test://models/my_model_untouched.sql", "my_test://models/my_model.sql", "my_test://models/python_model.py", "my_test://tests/tests.yml", } assert set(partial_parsing.file_diff["changed"]) == { "my_test://models/python_model_untouched.py", "my_test://tests/my_singular_test.sql", } assert partial_parsing.file_diff["changed_schema_files"] == [ "my_test://models/schema.yml", ] def test_get_parsing_files(self, partial_parsing): project_parser_files = partial_parsing.get_parsing_files() assert project_parser_files == { PROJECT_NAME: { "ModelParser": ["my_test://models/python_model_untouched.py"], "SchemaParser": ["my_test://models/schema.yml"], "SingularTestParser": ["my_test://tests/my_singular_test.sql"], }, } ================================================ FILE: tests/unit/parser/test_read_files.py ================================================ import pytest from dbt.parser.read_files import normalize_file_contents @pytest.mark.parametrize( "file_contents,expected_normalized_file_contents", [ ("", ""), (" ", ""), (" ", ""), ("\n", ""), ("a b", "a b"), ("a b", "a b"), ("a\nb", "a b"), ("a\n b", "a b"), ("a b ", "a b"), (" a b ", "a b"), ("\na b\n", "a b"), ("\n\na b\n\n", "a b"), ], ) def test_normalize_file_contents(file_contents: str, expected_normalized_file_contents: str): assert normalize_file_contents(file_contents) == expected_normalized_file_contents ================================================ FILE: tests/unit/parser/test_schema_renderer.py ================================================ import unittest from dbt.parser.schema_renderer import SchemaYamlRenderer class TestYamlRendering(unittest.TestCase): def test__models(self): context = { "test_var": "1234", "alt_var": "replaced", } renderer = SchemaYamlRenderer(context, "models") # Verify description is not rendered and misc attribute is rendered dct = { "name": "my_model", "description": "{{ test_var }}", "attribute": "{{ test_var }}", } expected = { "name": "my_model", "description": "{{ test_var }}", "attribute": "1234", } dct = renderer.render_data(dct) self.assertEqual(expected, dct) # Verify description in columns is not rendered dct = { "name": "my_test", "attribute": "{{ test_var }}", "columns": [ {"description": "{{ test_var }}", "name": "id"}, ], } expected = { "name": "my_test", "attribute": "1234", "columns": [ {"description": "{{ test_var }}", "name": "id"}, ], } dct = renderer.render_data(dct) self.assertEqual(expected, dct) def test__sources(self): context = { "test_var": "1234", "alt_var": "replaced", } renderer = SchemaYamlRenderer(context, "sources") # Only descriptions have jinja, none should be rendered dct = { "name": "my_source", "description": "{{ alt_var }}", "loaded_at_query": "select max(ordered_at) from {{ this }}", "tables": [ { "name": "my_table", "description": "{{ alt_var }}", "loaded_at_query": "select max(ordered_at) from {{ this }}", "columns": [ { "name": "id", "description": "{{ alt_var }}", } ], } ], } rendered = renderer.render_data(dct) self.assertEqual(dct, rendered) def test__macros(self): context = { "test_var": "1234", "alt_var": "replaced", } renderer = SchemaYamlRenderer(context, "macros") # Look for description in arguments dct = { "name": "my_macro", "arguments": [ {"name": "my_arg", "attr": "{{ alt_var }}"}, {"name": "an_arg", "description": "{{ alt_var}}"}, ], } expected = { "name": "my_macro", "arguments": [ {"name": "my_arg", "attr": "replaced"}, {"name": "an_arg", "description": "{{ alt_var}}"}, ], } dct = renderer.render_data(dct) self.assertEqual(dct, expected) def test__metrics(self): context = {"metric_name_end": "_metric"} renderer = SchemaYamlRenderer(context, "metrics") dct = { "name": "test{{ metric_name_end }}", "description": "{{ docs('my_doc') }}", "filter": "{{ Dimension('my_entity__my_dim') }} = false", } # We expect the expression and description will not be rendered, but # other fields will be expected = { "name": "test_metric", "description": "{{ docs('my_doc') }}", "filter": "{{ Dimension('my_entity__my_dim') }} = false", } dct = renderer.render_data(dct) self.assertEqual(dct, expected) def test__metrics_list_filter(self): """Test that list-type filters with Dimension jinja are not rendered for standalone metrics.""" context = {"metric_name_end": "_metric"} renderer = SchemaYamlRenderer(context, "metrics") dct = { "name": "test{{ metric_name_end }}", "filter": [ "{{ Dimension('my_entity__is_fraud') }} = false", "{{ Dimension('my_entity__is_employee') }} = false", ], } expected = { "name": "test_metric", "filter": [ "{{ Dimension('my_entity__is_fraud') }} = false", "{{ Dimension('my_entity__is_employee') }} = false", ], } dct = renderer.render_data(dct) self.assertEqual(dct, expected) def test__metrics_nested_filter(self): """Test that deeply nested filters in standalone metrics (type_params) are not rendered.""" context = {"metric_name_end": "_metric"} renderer = SchemaYamlRenderer(context, "metrics") # type_params.numerator.filter as string dct = { "name": "test{{ metric_name_end }}", "type_params": { "numerator": { "name": "some_metric", "filter": "{{ Dimension('my_entity__my_dim') }} > 0", }, }, } expected = { "name": "test_metric", "type_params": { "numerator": { "name": "some_metric", "filter": "{{ Dimension('my_entity__my_dim') }} > 0", }, }, } dct = renderer.render_data(dct) self.assertEqual(dct, expected) # type_params.numerator.filter as list dct = { "name": "test{{ metric_name_end }}", "type_params": { "numerator": { "name": "some_metric", "filter": [ "{{ Dimension('my_entity__is_fraud') }} = false", "{{ Dimension('my_entity__is_employee') }} = false", ], }, }, } expected = { "name": "test_metric", "type_params": { "numerator": { "name": "some_metric", "filter": [ "{{ Dimension('my_entity__is_fraud') }} = false", "{{ Dimension('my_entity__is_employee') }} = false", ], }, }, } dct = renderer.render_data(dct) self.assertEqual(dct, expected) def test__models_v2_metric_filter_string(self): """Test that string filters on v2 metrics (models key) are not rendered.""" context = {"test_var": "1234"} renderer = SchemaYamlRenderer(context, "models") dct = { "name": "my_model", "attribute": "{{ test_var }}", "metrics": [ { "name": "my_metric", "filter": "{{ Dimension('my_entity__my_dim') }} > 0", }, ], } expected = { "name": "my_model", "attribute": "1234", "metrics": [ { "name": "my_metric", "filter": "{{ Dimension('my_entity__my_dim') }} > 0", }, ], } dct = renderer.render_data(dct) self.assertEqual(dct, expected) def test__models_v2_metric_filter_list(self): """Test that list-type filters on v2 metrics (models key) are not rendered. This is the critical regression test: before the fix, list filters like filter: ["{{ Dimension(...) }} = False", "{{ Dimension(...) }} = False"] would be rendered at parse time, causing 'Dimension is undefined' errors. """ context = {"test_var": "1234"} renderer = SchemaYamlRenderer(context, "models") dct = { "name": "my_model", "attribute": "{{ test_var }}", "metrics": [ { "name": "my_metric", "filter": [ "{{ Dimension('my_entity__is_fraud') }} = false", "{{ Dimension('my_entity__is_employee') }} = false", ], }, ], } expected = { "name": "my_model", "attribute": "1234", "metrics": [ { "name": "my_metric", "filter": [ "{{ Dimension('my_entity__is_fraud') }} = false", "{{ Dimension('my_entity__is_employee') }} = false", ], }, ], } dct = renderer.render_data(dct) self.assertEqual(dct, expected) def test__models_v2_nested_metric_filters(self): """Test that nested metric filters (numerator, denominator, input_metrics, etc.) are not rendered for both string and list forms under the models key.""" context = {"test_var": "1234"} renderer = SchemaYamlRenderer(context, "models") # numerator.filter (string) dct = { "name": "my_model", "metrics": [ { "name": "ratio_metric", "numerator": { "name": "base", "filter": "{{ Dimension('my_entity__my_dim') }} > 0", }, }, ], } expected_filter = "{{ Dimension('my_entity__my_dim') }} > 0" rendered = renderer.render_data(dct) self.assertEqual(rendered["metrics"][0]["numerator"]["filter"], expected_filter) # numerator.filter (list) dct = { "name": "my_model", "metrics": [ { "name": "ratio_metric", "numerator": { "name": "base", "filter": [ "{{ Dimension('my_entity__is_fraud') }} = false", "{{ Dimension('my_entity__is_employee') }} = false", ], }, }, ], } expected_filters = [ "{{ Dimension('my_entity__is_fraud') }} = false", "{{ Dimension('my_entity__is_employee') }} = false", ] rendered = renderer.render_data(dct) self.assertEqual(rendered["metrics"][0]["numerator"]["filter"], expected_filters) # input_metrics[].filter (list) dct = { "name": "my_model", "metrics": [ { "name": "derived_metric", "input_metrics": [ { "name": "base", "filter": [ "{{ Dimension('my_entity__is_fraud') }} = false", ], }, ], }, ], } rendered = renderer.render_data(dct) self.assertEqual( rendered["metrics"][0]["input_metrics"][0]["filter"], ["{{ Dimension('my_entity__is_fraud') }} = false"], ) # base_metric.filter (list, conversion metric) dct = { "name": "my_model", "metrics": [ { "name": "conversion_metric", "base_metric": { "name": "base", "filter": [ "{{ Dimension('my_entity__is_fraud') }} = false", ], }, }, ], } rendered = renderer.render_data(dct) self.assertEqual( rendered["metrics"][0]["base_metric"]["filter"], ["{{ Dimension('my_entity__is_fraud') }} = false"], ) def test__derived_semantics_descriptions(self): context = { "test_var": "1234", } renderer = SchemaYamlRenderer(context, "models") # Verify descriptions inside derived_semantics.dimensions and # derived_semantics.entities are not rendered (they contain doc() # Jinja that is resolved later in process_docs) dct = { "name": "my_model", "attribute": "{{ test_var }}", "derived_semantics": { "dimensions": [ { "name": "my_dim", "description": "{{ test_var }}", "type": "categorical", }, ], "entities": [ { "name": "my_entity", "description": "{{ test_var }}", "type": "foreign", }, ], }, } expected = { "name": "my_model", "attribute": "1234", "derived_semantics": { "dimensions": [ { "name": "my_dim", "description": "{{ test_var }}", "type": "categorical", }, ], "entities": [ { "name": "my_entity", "description": "{{ test_var }}", "type": "foreign", }, ], }, } dct = renderer.render_data(dct) self.assertEqual(expected, dct) ================================================ FILE: tests/unit/parser/test_sources.py ================================================ from typing import List, Optional import pytest from core.dbt.artifacts.resources.v1.components import FreshnessThreshold, Time from core.dbt.parser.sources import merge_source_freshness class TestMergeSourceFreshness: @pytest.mark.parametrize( "thresholds,expected_result", [ ([None, None], None), ( [ FreshnessThreshold( warn_after=Time(count=1, period="hour"), error_after=Time(count=1, period="day"), ), None, ], None, ), ( [ FreshnessThreshold( warn_after=Time(count=1, period="hour"), error_after=Time(count=1, period="day"), ), None, FreshnessThreshold(), ], None, ), ( [ FreshnessThreshold(warn_after=Time(count=1, period="hour")), FreshnessThreshold(error_after=Time(count=1, period="day")), ], FreshnessThreshold( warn_after=Time(count=1, period="hour"), error_after=Time(count=1, period="day"), ), ), ( [ None, FreshnessThreshold(warn_after=Time(count=1, period="hour")), FreshnessThreshold(error_after=Time(count=1, period="day")), ], FreshnessThreshold( warn_after=Time(count=1, period="hour"), error_after=Time(count=1, period="day"), ), ), ( [ FreshnessThreshold( warn_after=Time(count=1, period="hour"), error_after=Time(count=1, period="day"), ), FreshnessThreshold(error_after=Time(count=48, period="hour")), ], FreshnessThreshold( warn_after=Time(count=1, period="hour"), error_after=Time(count=48, period="hour"), ), ), ], ) def test_merge_source_freshness( self, thresholds: List[Optional[FreshnessThreshold]], expected_result: Optional[FreshnessThreshold], ): result = merge_source_freshness(*thresholds) assert result == expected_result ================================================ FILE: tests/unit/parser/test_unit_tests.py ================================================ from unittest import mock from dbt.artifacts.resources import DependsOn, UnitTestConfig, UnitTestFormat from dbt.contracts.graph.nodes import NodeType, UnitTestDefinition from dbt.contracts.graph.unparsed import UnitTestOutputFixture from dbt.parser import SchemaParser from dbt.parser.unit_tests import UnitTestParser from dbt_common.events.event_catcher import EventCatcher from dbt_common.events.event_manager_client import add_callback_to_manager from dbt_common.events.types import SystemStdErr from tests.unit.parser.test_parser import SchemaParserTest, assertEqualNodes from tests.unit.utils import MockNode UNIT_TEST_MODEL_NOT_FOUND_SOURCE = """ unit_tests: - name: test_my_model_doesnt_exist model: my_model_doesnt_exist description: "unit test description" given: [] expect: rows: - {a: 1} """ UNIT_TEST_SOURCE = """ unit_tests: - name: test_my_model model: my_model description: "unit test description" given: [] expect: rows: - {a: 1} """ UNIT_TEST_VERSIONED_MODEL_SOURCE = """ unit_tests: - name: test_my_model_versioned model: my_model_versioned.v1 description: "unit test description" given: [] expect: rows: - {a: 1} """ UNIT_TEST_CONFIG_SOURCE = """ unit_tests: - name: test_my_model model: my_model config: tags: "schema_tag" meta: meta_key: meta_value meta_jinja_key: '{{ 1 + 1 }}' description: "unit test description" given: [] expect: rows: - {a: 1} """ UNIT_TEST_MULTIPLE_SOURCE = """ unit_tests: - name: test_my_model model: my_model description: "unit test description" given: [] expect: rows: - {a: 1} - name: test_my_model2 model: my_model description: "unit test description" given: [] expect: rows: - {a: 1} """ UNIT_TEST_NONE_ROWS_SORT = """ unit_tests: - name: test_my_model_null_handling model: my_model description: "unit test description" given: [] expect: rows: - {"id": , "col1": "d"} - {"id": , "col1": "e"} - {"id": 6, "col1": "f"} """ UNIT_TEST_NONE_ROWS_SORT_CSV = """ unit_tests: - name: test_my_model_null_handling model: my_model description: "unit test description" given: [] expect: format: csv rows: | id,col1 ,d ,e 6,f """ UNIT_TEST_NONE_ROWS_SORT_SQL = """ unit_tests: - name: test_my_model_null_handling model: my_model description: "unit test description" given: [] expect: format: sql rows: | select null select 1 """ UNIT_TEST_NONE_ROWS_SORT_FAILS = """ unit_tests: - name: test_my_model_null_handling model: my_model description: "this unit test needs one non-None value row" given: [] expect: rows: - {"id": , "col1": "d"} - {"id": , "col1": "e"} """ UNIT_TEST_DISABLED = """ unit_tests: - name: test_my_model_disabled model: my_model description: "this unit test is disabled" config: enabled: false given: [] expect: rows: - {a: 1} """ class UnitTestParserTest(SchemaParserTest): def setUp(self): super().setUp() my_model_node = MockNode( package="snowplow", name="my_model", config=mock.MagicMock(enabled=True), schema="test_schema", refs=[], sources=[], patch_path=None, ) self.manifest.nodes = {my_model_node.unique_id: my_model_node} self.parser = SchemaParser( project=self.snowplow_project_config, manifest=self.manifest, root_project=self.root_project_config, ) def file_block_for(self, data, filename): return super().file_block_for(data, filename, "unit_tests") def test_basic(self): block = self.yaml_block_for(UNIT_TEST_SOURCE, "test_my_model.yml") UnitTestParser(self.parser, block).parse() self.assert_has_manifest_lengths(self.parser.manifest, nodes=1, unit_tests=1) unit_test = list(self.parser.manifest.unit_tests.values())[0] expected = UnitTestDefinition( name="test_my_model", model="my_model", resource_type=NodeType.Unit, package_name="snowplow", path=block.path.relative_path, original_file_path=block.path.original_file_path, unique_id="unit_test.snowplow.my_model.test_my_model", given=[], expect=UnitTestOutputFixture(rows=[{"a": 1}]), description="unit test description", overrides=None, depends_on=DependsOn(nodes=["model.snowplow.my_model"]), fqn=["snowplow", "my_model", "test_my_model"], config=UnitTestConfig(), schema="test_schema", ) expected.build_unit_test_checksum() assertEqualNodes(unit_test, expected) def test_unit_test_config(self): block = self.yaml_block_for(UNIT_TEST_CONFIG_SOURCE, "test_my_model.yml") self.root_project_config.unit_tests = { "snowplow": {"my_model": {"+tags": ["project_tag"]}} } UnitTestParser(self.parser, block).parse() self.assert_has_manifest_lengths(self.parser.manifest, nodes=1, unit_tests=1) unit_test = self.parser.manifest.unit_tests["unit_test.snowplow.my_model.test_my_model"] self.assertEqual(sorted(unit_test.config.tags), sorted(["schema_tag", "project_tag"])) self.assertEqual(unit_test.config.meta, {"meta_key": "meta_value", "meta_jinja_key": "2"}) def test_unit_test_disabled(self): block = self.yaml_block_for(UNIT_TEST_DISABLED, "test_my_model.yml") self.root_project_config.unit_tests = { "snowplow": {"my_model": {"+tags": ["project_tag"]}} } UnitTestParser(self.parser, block).parse() self.assert_has_manifest_lengths(self.parser.manifest, nodes=1, unit_tests=0, disabled=1) unit_test_disabled_list = self.parser.manifest.disabled[ "unit_test.snowplow.my_model.test_my_model_disabled" ] self.assertEqual(len(unit_test_disabled_list), 1) unit_test_disabled = unit_test_disabled_list[0] self.assertEqual(unit_test_disabled.config.enabled, False) def test_unit_test_versioned_model(self): block = self.yaml_block_for(UNIT_TEST_VERSIONED_MODEL_SOURCE, "test_my_model.yml") my_model_versioned_node = MockNode( package="snowplow", name="my_model_versioned", config=mock.MagicMock(enabled=True), refs=[], sources=[], patch_path=None, version=1, ) self.manifest.nodes[my_model_versioned_node.unique_id] = my_model_versioned_node UnitTestParser(self.parser, block).parse() self.assert_has_manifest_lengths(self.parser.manifest, nodes=2, unit_tests=1) unit_test = self.parser.manifest.unit_tests[ "unit_test.snowplow.my_model_versioned.v1.test_my_model_versioned" ] self.assertEqual(len(unit_test.depends_on.nodes), 1) self.assertEqual(unit_test.depends_on.nodes[0], "model.snowplow.my_model_versioned.v1") def test_multiple_unit_tests(self): block = self.yaml_block_for(UNIT_TEST_MULTIPLE_SOURCE, "test_my_model.yml") UnitTestParser(self.parser, block).parse() self.assert_has_manifest_lengths(self.parser.manifest, nodes=1, unit_tests=2) for unit_test in self.parser.manifest.unit_tests.values(): self.assertEqual(len(unit_test.depends_on.nodes), 1) self.assertEqual(unit_test.depends_on.nodes[0], "model.snowplow.my_model") def _assert_fixture_yml_reorders_to_expected_rows( self, unit_test_fixture_yml, fixture_expected_field_format, expected_rows ): block = self.yaml_block_for(unit_test_fixture_yml, "test_my_model.yml") UnitTestParser(self.parser, block).parse() self.assert_has_manifest_lengths(self.parser.manifest, nodes=1, unit_tests=1) unit_test = list(self.parser.manifest.unit_tests.values())[0] expected = UnitTestDefinition( name="test_my_model_null_handling", model="my_model", resource_type=NodeType.Unit, package_name="snowplow", path=block.path.relative_path, original_file_path=block.path.original_file_path, unique_id="unit_test.snowplow.my_model.test_my_model_null_handling", given=[], expect=UnitTestOutputFixture(format=fixture_expected_field_format, rows=expected_rows), description="unit test description", overrides=None, depends_on=DependsOn(nodes=["model.snowplow.my_model"]), fqn=["snowplow", "my_model", "test_my_model_null_handling"], config=UnitTestConfig(), schema="test_schema", ) expected.build_unit_test_checksum() assertEqualNodes(unit_test, expected) def test_expected_promote_non_none_row_dct(self): expected_rows = [ {"id": 6, "col1": "f"}, {"id": None, "col1": "e"}, {"id": None, "col1": "d"}, ] self._assert_fixture_yml_reorders_to_expected_rows( UNIT_TEST_NONE_ROWS_SORT, UnitTestFormat.Dict, expected_rows ) def test_expected_promote_non_none_row_csv(self): expected_rows = [ {"id": "6", "col1": "f"}, {"id": None, "col1": "e"}, {"id": None, "col1": "d"}, ] self._assert_fixture_yml_reorders_to_expected_rows( UNIT_TEST_NONE_ROWS_SORT_CSV, UnitTestFormat.CSV, expected_rows ) def test_expected_promote_non_none_row_sql(self): expected_rows = "select null\n" + "select 1" self._assert_fixture_yml_reorders_to_expected_rows( UNIT_TEST_NONE_ROWS_SORT_SQL, UnitTestFormat.SQL, expected_rows ) def test_no_full_row_does_not_raise_exception(self): catcher = EventCatcher(SystemStdErr) add_callback_to_manager(catcher.catch) block = self.yaml_block_for(UNIT_TEST_NONE_ROWS_SORT_FAILS, "test_my_model.yml") UnitTestParser(self.parser, block).parse() assert len(catcher.caught_events) == 1 ================================================ FILE: tests/unit/parser/test_v2_column_semantic_parsing.py ================================================ """Unit tests for _parse_v2_column_dimensions and _parse_v2_column_entities. These methods transform column-level dimension/entity definitions into Dimension and Entity objects for the semantic manifest. When a dimension or entity name differs from the column name, the resulting object must have expr set to the column name so that MetricFlow generates SQL against the correct warehouse column. """ from collections import OrderedDict from dbt.artifacts.resources import ColumnDimension, ColumnEntity, ColumnInfo from dbt.parser.schema_yaml_readers import SemanticModelParser from dbt_semantic_interfaces.type_enums import DimensionType, EntityType def _make_column(name, description="", dimension=None, entity=None, granularity=None, config=None): """Helper to construct a ColumnInfo with only the fields we need.""" return ColumnInfo( name=name, description=description, dimension=dimension, entity=entity, granularity=granularity, config=config or {}, ) def _make_columns_dict(*columns): """Build an OrderedDict of ColumnInfo keyed by name, as the parser expects.""" return OrderedDict((col.name, col) for col in columns) class TestParseV2ColumnDimensionsExpr: """Test that _parse_v2_column_dimensions sets expr correctly.""" def test_dimension_name_override_sets_expr_to_column_name(self): """When dimension.name differs from column.name, expr must be the column name.""" columns = _make_columns_dict( _make_column( name="afe_number", description="AFE identifier", dimension=ColumnDimension( name="approval_afe_number", type=DimensionType.CATEGORICAL, ), ), ) dimensions = SemanticModelParser._parse_v2_column_dimensions(None, columns) assert len(dimensions) == 1 dim = dimensions[0] assert dim.name == "approval_afe_number" assert dim.expr == "afe_number" def test_dimension_name_matches_column_name_expr_is_none(self): """When dimension.name matches column.name, expr should be None.""" columns = _make_columns_dict( _make_column( name="status", dimension=ColumnDimension( name="status", type=DimensionType.CATEGORICAL, ), ), ) dimensions = SemanticModelParser._parse_v2_column_dimensions(None, columns) assert len(dimensions) == 1 assert dimensions[0].name == "status" assert dimensions[0].expr is None def test_dimension_empty_name_defaults_to_column_name(self): """When dimension.name is empty string, name defaults to column.name and expr is None.""" columns = _make_columns_dict( _make_column( name="category", dimension=ColumnDimension( name="", type=DimensionType.CATEGORICAL, ), ), ) dimensions = SemanticModelParser._parse_v2_column_dimensions(None, columns) assert len(dimensions) == 1 assert dimensions[0].name == "category" assert dimensions[0].expr is None def test_dimension_shorthand_type_expr_is_none(self): """When dimension is just a DimensionType (shorthand), expr should be None.""" columns = _make_columns_dict( _make_column( name="color", dimension=DimensionType.CATEGORICAL, ), ) dimensions = SemanticModelParser._parse_v2_column_dimensions(None, columns) assert len(dimensions) == 1 assert dimensions[0].name == "color" assert dimensions[0].expr is None class TestParseV2ColumnEntitiesExpr: """Test that _parse_v2_column_entities sets expr correctly.""" def test_entity_name_override_sets_expr_to_column_name(self): """When entity.name differs from column.name, expr must be the column name.""" columns = _make_columns_dict( _make_column( name="id", description="Primary key", entity=ColumnEntity( name="id_entity", type=EntityType.PRIMARY, ), ), ) entities = SemanticModelParser._parse_v2_column_entities(None, columns) assert len(entities) == 1 ent = entities[0] assert ent.name == "id_entity" assert ent.expr == "id" def test_entity_name_matches_column_name_expr_is_none(self): """When entity.name matches column.name, expr should be None.""" columns = _make_columns_dict( _make_column( name="user_id", entity=ColumnEntity( name="user_id", type=EntityType.FOREIGN, ), ), ) entities = SemanticModelParser._parse_v2_column_entities(None, columns) assert len(entities) == 1 assert entities[0].name == "user_id" assert entities[0].expr is None def test_entity_shorthand_type_expr_is_none(self): """When entity is just an EntityType (shorthand), name defaults to column.name and expr is None.""" columns = _make_columns_dict( _make_column( name="foreign_id_col", entity=EntityType.FOREIGN, ), ) entities = SemanticModelParser._parse_v2_column_entities(None, columns) assert len(entities) == 1 assert entities[0].name == "foreign_id_col" assert entities[0].expr is None ================================================ FILE: tests/unit/plugins/test_manager.py ================================================ from unittest import mock import pytest from dbt.exceptions import DbtRuntimeError from dbt.plugins import PluginManager, dbt_hook, dbtPlugin from dbt.plugins.contracts import PluginArtifact, PluginArtifacts from dbt.plugins.exceptions import dbtPluginError from dbt.plugins.manifest import ModelNodeArgs, PluginNodes class ExceptionInitializePlugin(dbtPlugin): def initialize(self) -> None: raise Exception("plugin error message") class dbtRuntimeErrorInitializePlugin(dbtPlugin): def initialize(self) -> None: raise dbtPluginError("plugin error message") class GetNodesPlugin(dbtPlugin): @dbt_hook def get_nodes(self) -> PluginNodes: nodes = PluginNodes() nodes.add_model( ModelNodeArgs( name="test_name", package_name=self.project_name, identifier="test_identifier", schema="test_schema", ) ) return nodes class GetArtifactsPlugin(dbtPlugin): @dbt_hook def get_manifest_artifacts(self, manifest) -> PluginArtifacts: return {self.project_name: PluginArtifact()} class TestPluginManager: @pytest.fixture def get_nodes_plugin(self): return GetNodesPlugin(project_name="test") @pytest.fixture def get_nodes_plugins(self, get_nodes_plugin): return [get_nodes_plugin, GetNodesPlugin(project_name="test2")] @pytest.fixture def get_artifacts_plugin(self): return GetArtifactsPlugin(project_name="test") @pytest.fixture def get_artifacts_plugins(self, get_artifacts_plugin): return [get_artifacts_plugin, GetArtifactsPlugin(project_name="test2")] def test_plugin_manager_init_exception(self): with pytest.raises(DbtRuntimeError, match="plugin error message"): PluginManager(plugins=[ExceptionInitializePlugin(project_name="test")]) def test_plugin_manager_init_plugin_exception(self): with pytest.raises(DbtRuntimeError, match="^Runtime Error\n plugin error message"): PluginManager(plugins=[dbtRuntimeErrorInitializePlugin(project_name="test")]) def test_plugin_manager_init_single_hook(self, get_nodes_plugin): pm = PluginManager(plugins=[get_nodes_plugin]) assert len(pm.hooks) == 1 assert "get_nodes" in pm.hooks assert len(pm.hooks["get_nodes"]) == 1 assert pm.hooks["get_nodes"][0] == get_nodes_plugin.get_nodes def test_plugin_manager_init_single_hook_multiple_methods(self, get_nodes_plugins): pm = PluginManager(plugins=get_nodes_plugins) assert len(pm.hooks) == 1 assert "get_nodes" in pm.hooks assert len(pm.hooks["get_nodes"]) == 2 assert pm.hooks["get_nodes"][0] == get_nodes_plugins[0].get_nodes assert pm.hooks["get_nodes"][1] == get_nodes_plugins[1].get_nodes def test_plugin_manager_init_multiple_hooks(self, get_nodes_plugin, get_artifacts_plugin): pm = PluginManager(plugins=[get_nodes_plugin, get_artifacts_plugin]) assert len(pm.hooks) == 2 assert "get_nodes" in pm.hooks assert len(pm.hooks["get_nodes"]) == 1 assert pm.hooks["get_nodes"][0] == get_nodes_plugin.get_nodes assert "get_manifest_artifacts" in pm.hooks assert len(pm.hooks["get_manifest_artifacts"]) == 1 assert pm.hooks["get_manifest_artifacts"][0] == get_artifacts_plugin.get_manifest_artifacts @mock.patch("dbt.tracking") def test_get_nodes(self, tracking, get_nodes_plugins): tracking.active_user = mock.Mock() pm = PluginManager(plugins=get_nodes_plugins) nodes = pm.get_nodes() assert len(nodes.models) == 2 expected_calls = [ mock.call( { "plugin_name": get_nodes_plugins[0].name, "num_model_nodes": 1, "num_model_packages": 1, } ), mock.call( { "plugin_name": get_nodes_plugins[1].name, "num_model_nodes": 1, "num_model_packages": 1, } ), ] tracking.track_plugin_get_nodes.assert_has_calls(expected_calls) def test_get_manifest_artifact(self, get_artifacts_plugins): pm = PluginManager(plugins=get_artifacts_plugins) artifacts = pm.get_manifest_artifacts(None) assert len(artifacts) == 2 ================================================ FILE: tests/unit/task/__init__.py ================================================ ================================================ FILE: tests/unit/task/docs/__init__.py ================================================ ================================================ FILE: tests/unit/task/docs/test_serve.py ================================================ from http.server import SimpleHTTPRequestHandler from unittest.mock import MagicMock, patch import pytest from dbt.task.docs.serve import ServeTask @pytest.fixture def serve_task(): # Set up task = ServeTask(config=MagicMock(), args=MagicMock()) task.config.project_target_path = "." task.args.port = 8000 task.args.host = "127.0.0.1" return task def test_serve_bind_to_127(serve_task): serve_task.args.browser = False with patch("dbt.task.docs.serve.socketserver.TCPServer") as patched_TCPServer: patched_TCPServer.return_value = MagicMock() serve_task.run() patched_TCPServer.assert_called_once_with(("127.0.0.1", 8000), SimpleHTTPRequestHandler) def test_serve_bind_to_all(serve_task): serve_task.args.browser = False serve_task.args.host = "" with patch("dbt.task.docs.serve.socketserver.TCPServer") as patched_TCPServer: patched_TCPServer.return_value = MagicMock() serve_task.run() patched_TCPServer.assert_called_once_with(("", 8000), SimpleHTTPRequestHandler) ================================================ FILE: tests/unit/task/test_base.py ================================================ import os import dbt_common.exceptions from dbt.contracts.graph.nodes import SourceDefinition from dbt.task.base import BaseRunner, ConfiguredTask from tests.unit.config import BaseConfigTest INITIAL_ROOT = os.getcwd() class MockRunner(BaseRunner): def compile(self): pass class TestBaseRunner: def test_handle_generic_exception_handles_nodes_without_build_path( self, basic_parsed_source_definition_object: SourceDefinition ): # Source definition nodes don't have `build_path` attributes. Thus, this # test will fail if _handle_generic_exception doesn't account for this runner = MockRunner( config=None, adapter=None, node=basic_parsed_source_definition_object, node_index=None, num_nodes=None, ) assert not hasattr(basic_parsed_source_definition_object, "build_path") runner._handle_generic_exception(Exception("bad thing happened"), ctx=None) class InheritsFromConfiguredTask(ConfiguredTask): def run(self): pass class TestConfiguredTask(BaseConfigTest): def tearDown(self): super().tearDown() # These tests will change the directory to the project path, # so it's necessary to change it back at the end. os.chdir(INITIAL_ROOT) def test_configured_task_dir_change(self): self.assertEqual(os.getcwd(), INITIAL_ROOT) self.assertNotEqual(INITIAL_ROOT, self.project_dir) InheritsFromConfiguredTask.from_args(self.args) self.assertEqual(os.path.realpath(os.getcwd()), os.path.realpath(self.project_dir)) def test_configured_task_dir_change_with_bad_path(self): self.args.project_dir = "bad_path" with self.assertRaises(dbt_common.exceptions.DbtRuntimeError): InheritsFromConfiguredTask.from_args(self.args) ================================================ FILE: tests/unit/task/test_build.py ================================================ from dbt.contracts.graph.nodes import SavedQuery from dbt.runners import SavedQueryRunner def test_saved_query_runner_on_skip(saved_query: SavedQuery): runner = SavedQueryRunner( config=None, adapter=None, node=saved_query, node_index=None, num_nodes=None, ) # on_skip would work runner.on_skip() ================================================ FILE: tests/unit/task/test_clone.py ================================================ from unittest.mock import MagicMock, patch from dbt.flags import get_flags from dbt.task.clone import CloneTask def test_clone_task_not_preserve_edges(): mock_node_selector = MagicMock() mock_spec = MagicMock() with patch.object( CloneTask, "get_node_selector", return_value=mock_node_selector ), patch.object(CloneTask, "get_selection_spec", return_value=mock_spec): task = CloneTask(get_flags(), None, None) task.get_graph_queue() # when we get the graph queue, preserve_edges is False mock_node_selector.get_graph_queue.assert_called_with(mock_spec, False) ================================================ FILE: tests/unit/task/test_docs.py ================================================ import unittest from decimal import Decimal from unittest import mock from dbt.task.docs import generate class GenerateTest(unittest.TestCase): def setUp(self): self.maxDiff = None self.manifest = mock.MagicMock() self.patcher = mock.patch("dbt.task.docs.generate.get_unique_id_mapping") self.mock_get_unique_id_mapping = self.patcher.start() def tearDown(self): self.patcher.stop() def map_uids(self, effects): results = {generate.CatalogKey(db, sch, tbl): uid for db, sch, tbl, uid in effects} self.mock_get_unique_id_mapping.return_value = results, {} def generate_catalog_dict(self, columns): nodes, sources = generate.Catalog(columns).make_unique_id_map(self.manifest) result = generate.CatalogResults( nodes=nodes, sources=sources, errors=None, ) return result.to_dict(omit_none=False)["nodes"] def test__unflatten_empty(self): columns = {} expected = {} self.map_uids([]) result = self.generate_catalog_dict(columns) self.mock_get_unique_id_mapping.assert_called_once_with(self.manifest) self.assertEqual(result, expected) def test__unflatten_one_column(self): columns = [ { "column_comment": None, "column_index": Decimal("1"), "column_name": "id", "column_type": "integer", "table_comment": None, "table_name": "test_table", "table_schema": "test_schema", "table_type": "BASE TABLE", "table_database": "test_database", } ] expected = { "test.model.test_table": { "metadata": { "owner": None, "comment": None, "name": "test_table", "type": "BASE TABLE", "schema": "test_schema", "database": "test_database", }, "columns": { "id": {"type": "integer", "comment": None, "index": 1, "name": "id"}, }, "stats": { "has_stats": { "id": "has_stats", "label": "Has Stats?", "value": False, "description": "Indicates whether there are statistics for this table", "include": False, }, }, "unique_id": "test.model.test_table", }, } self.map_uids([("test_database", "test_schema", "test_table", "test.model.test_table")]) result = self.generate_catalog_dict(columns) self.mock_get_unique_id_mapping.assert_called_once_with(self.manifest) self.assertEqual(result, expected) def test__unflatten_multiple_schemas_dbs(self): columns = [ { "column_comment": None, "column_index": Decimal("1"), "column_name": "id", "column_type": "integer", "table_comment": None, "table_name": "test_table", "table_schema": "test_schema", "table_type": "BASE TABLE", "table_database": "test_database", "table_owner": None, }, { "column_comment": None, "column_index": Decimal("2"), "column_name": "name", "column_type": "text", "table_comment": None, "table_name": "test_table", "table_schema": "test_schema", "table_type": "BASE TABLE", "table_database": "test_database", "table_owner": None, }, { "column_comment": None, "column_index": Decimal("1"), "column_name": "id", "column_type": "integer", "table_comment": None, "table_name": "other_test_table", "table_schema": "test_schema", "table_type": "BASE TABLE", "table_database": "test_database", "table_owner": None, }, { "column_comment": None, "column_index": Decimal("2"), "column_name": "email", "column_type": "character varying", "table_comment": None, "table_name": "other_test_table", "table_schema": "test_schema", "table_type": "BASE TABLE", "table_database": "test_database", "table_owner": None, }, { "column_comment": None, "column_index": Decimal("1"), "column_name": "id", "column_type": "integer", "table_comment": None, "table_name": "test_table", "table_schema": "other_test_schema", "table_type": "BASE TABLE", "table_database": "test_database", "table_owner": None, }, { "column_comment": None, "column_index": Decimal("2"), "column_name": "name", "column_type": "text", "table_comment": None, "table_name": "test_table", "table_schema": "other_test_schema", "table_type": "BASE TABLE", "table_database": "test_database", "table_owner": None, }, { "column_comment": None, "column_index": Decimal("1"), "column_name": "id", "column_type": "integer", "table_comment": None, "table_name": "test_table", "table_schema": "test_schema", "table_type": "BASE TABLE", "table_database": "other_test_database", "table_owner": None, }, { "column_comment": None, "column_index": Decimal("2"), "column_name": "name", "column_type": "text", "table_comment": None, "table_name": "test_table", "table_schema": "test_schema", "table_type": "BASE TABLE", "table_database": "other_test_database", "table_owner": None, }, ] expected = { "test.model.test_table": { "metadata": { "owner": None, "comment": None, "name": "test_table", "type": "BASE TABLE", "schema": "test_schema", "database": "test_database", }, "columns": { "id": {"type": "integer", "comment": None, "index": 1, "name": "id"}, "name": { "type": "text", "comment": None, "index": 2, "name": "name", }, }, "stats": { "has_stats": { "id": "has_stats", "label": "Has Stats?", "value": False, "description": "Indicates whether there are statistics for this table", "include": False, }, }, "unique_id": "test.model.test_table", }, "test.model.other_test_table": { "metadata": { "owner": None, "comment": None, "name": "other_test_table", "type": "BASE TABLE", "schema": "test_schema", "database": "test_database", }, "columns": { "id": {"type": "integer", "comment": None, "index": 1, "name": "id"}, "email": { "type": "character varying", "comment": None, "index": 2, "name": "email", }, }, "stats": { "has_stats": { "id": "has_stats", "label": "Has Stats?", "value": False, "description": "Indicates whether there are statistics for this table", "include": False, }, }, "unique_id": "test.model.other_test_table", }, "test.model.test_table_otherschema": { "metadata": { "owner": None, "comment": None, "name": "test_table", "type": "BASE TABLE", "schema": "other_test_schema", "database": "test_database", }, "columns": { "id": {"type": "integer", "comment": None, "index": 1, "name": "id"}, "name": { "type": "text", "comment": None, "index": 2, "name": "name", }, }, "stats": { "has_stats": { "id": "has_stats", "label": "Has Stats?", "value": False, "description": "Indicates whether there are statistics for this table", "include": False, }, }, "unique_id": "test.model.test_table_otherschema", }, "test.model.test_table_otherdb": { "metadata": { "owner": None, "comment": None, "name": "test_table", "type": "BASE TABLE", "schema": "test_schema", "database": "other_test_database", }, "columns": { "id": {"type": "integer", "comment": None, "index": 1, "name": "id"}, "name": { "type": "text", "comment": None, "index": 2, "name": "name", }, }, "stats": { "has_stats": { "id": "has_stats", "label": "Has Stats?", "value": False, "description": "Indicates whether there are statistics for this table", "include": False, }, }, "unique_id": "test.model.test_table_otherdb", }, } self.map_uids( [ ("test_database", "test_schema", "test_table", "test.model.test_table"), ( "test_database", "test_schema", "other_test_table", "test.model.other_test_table", ), ( "test_database", "other_test_schema", "test_table", "test.model.test_table_otherschema", ), ( "other_test_database", "test_schema", "test_table", "test.model.test_table_otherdb", ), ] ) result = self.generate_catalog_dict(columns) self.mock_get_unique_id_mapping.assert_called_once_with(self.manifest) self.assertEqual(result, expected) ================================================ FILE: tests/unit/task/test_freshness.py ================================================ import datetime from unittest import mock import pytest from dbt.task.freshness import FreshnessResponse, FreshnessTask class TestFreshnessTaskMetadataCache: @pytest.fixture(scope="class") def args(self): mock_args = mock.Mock() mock_args.state = None mock_args.defer_state = None mock_args.write_json = None return mock_args @pytest.fixture(scope="class") def config(self): mock_config = mock.Mock() mock_config.threads = 1 mock_config.target_name = "mock_config_target_name" @pytest.fixture(scope="class") def manifest(self): return mock.Mock() @pytest.fixture(scope="class") def source_with_loaded_at_field(self): mock_source = mock.Mock() mock_source.unique_id = "source_with_loaded_at_field" mock_source.loaded_at_field = "loaded_at_field" return mock_source @pytest.fixture(scope="class") def source_no_loaded_at_field(self): mock_source = mock.Mock() mock_source.unique_id = "source_no_loaded_at_field" return mock_source @pytest.fixture(scope="class") def source_no_loaded_at_field2(self): mock_source = mock.Mock() mock_source.unique_id = "source_no_loaded_at_field2" return mock_source @pytest.fixture(scope="class") def adapter(self): return mock.Mock() @pytest.fixture(scope="class") def freshness_response(self): return FreshnessResponse( max_loaded_at=datetime.datetime(2020, 5, 2), snapshotted_at=datetime.datetime(2020, 5, 4), age=2, ) def test_populate_metadata_freshness_cache( self, args, config, manifest, adapter, source_no_loaded_at_field, freshness_response ): manifest.sources = {source_no_loaded_at_field.unique_id: source_no_loaded_at_field} adapter.Relation.create_from.return_value = "source_relation" adapter.calculate_freshness_from_metadata_batch.return_value = ( [], {"source_relation": freshness_response}, ) task = FreshnessTask(args=args, config=config, manifest=manifest) task.populate_metadata_freshness_cache(adapter, {source_no_loaded_at_field.unique_id}) assert task.get_freshness_metadata_cache() == {"source_relation": freshness_response} def test_populate_metadata_freshness_cache_multiple_sources( self, args, config, manifest, adapter, source_no_loaded_at_field, source_no_loaded_at_field2, freshness_response, ): manifest.sources = { source_no_loaded_at_field.unique_id: source_no_loaded_at_field, source_no_loaded_at_field2.unique_id: source_no_loaded_at_field2, } adapter.Relation.create_from.side_effect = ["source_relation1", "source_relation2"] adapter.calculate_freshness_from_metadata_batch.return_value = ( [], {"source_relation1": freshness_response, "source_relation2": freshness_response}, ) task = FreshnessTask(args=args, config=config, manifest=manifest) task.populate_metadata_freshness_cache(adapter, {source_no_loaded_at_field.unique_id}) assert task.get_freshness_metadata_cache() == { "source_relation1": freshness_response, "source_relation2": freshness_response, } def test_populate_metadata_freshness_cache_with_loaded_at_field( self, args, config, manifest, adapter, source_with_loaded_at_field, freshness_response ): manifest.sources = { source_with_loaded_at_field.unique_id: source_with_loaded_at_field, } adapter.Relation.create_from.return_value = "source_relation" adapter.calculate_freshness_from_metadata_batch.return_value = ( [], {"source_relation": freshness_response}, ) task = FreshnessTask(args=args, config=config, manifest=manifest) task.populate_metadata_freshness_cache(adapter, {source_with_loaded_at_field.unique_id}) assert task.get_freshness_metadata_cache() == {"source_relation": freshness_response} def test_populate_metadata_freshness_cache_multiple_sources_mixed( self, args, config, manifest, adapter, source_no_loaded_at_field, source_with_loaded_at_field, freshness_response, ): manifest.sources = { source_no_loaded_at_field.unique_id: source_no_loaded_at_field, source_with_loaded_at_field.unique_id: source_with_loaded_at_field, } adapter.Relation.create_from.return_value = "source_relation" adapter.calculate_freshness_from_metadata_batch.return_value = ( [], {"source_relation": freshness_response}, ) task = FreshnessTask(args=args, config=config, manifest=manifest) task.populate_metadata_freshness_cache(adapter, {source_no_loaded_at_field.unique_id}) assert task.get_freshness_metadata_cache() == {"source_relation": freshness_response} def test_populate_metadata_freshness_cache_adapter_exception( self, args, config, manifest, adapter, source_no_loaded_at_field, freshness_response ): manifest.sources = {source_no_loaded_at_field.unique_id: source_no_loaded_at_field} adapter.Relation.create_from.return_value = "source_relation" adapter.calculate_freshness_from_metadata_batch.side_effect = Exception() task = FreshnessTask(args=args, config=config, manifest=manifest) task.populate_metadata_freshness_cache(adapter, {source_no_loaded_at_field.unique_id}) assert task.get_freshness_metadata_cache() == {} ================================================ FILE: tests/unit/task/test_list.py ================================================ from argparse import Namespace from unittest.mock import patch from dbt.flags import get_flags, set_from_args from dbt.task.list import ListTask from dbt_common.events.types import PrintEvent def test_list_output_results(): set_from_args(Namespace(models=None), {}) task = ListTask(get_flags(), None, None) results = ["node1", "node2", "node3"] expected_node_results = ["node1", "node2", "node3"] with patch("dbt.task.list.fire_event") as mock_fire_event: node_results = task.output_results(results) assert node_results == expected_node_results # assert called with PrintEvent type object and message 'node1', 'node2', 'node3' for call_args in mock_fire_event.call_args_list: assert isinstance(call_args[0][0], PrintEvent) assert call_args[0][0].msg in expected_node_results class TestGetNestedValue: """Unit tests for the _get_nested_value method""" def setup_method(self): set_from_args(Namespace(models=None), {}) self.task = ListTask(get_flags(), None, None) def test_get_nested_value_simple_key(self): """Test getting a simple top-level key""" data = {"name": "test_model", "type": "model"} result = self.task._get_nested_value(data, "name") assert result == "test_model" def test_get_nested_value_nested_key(self): """Test getting a nested key""" data = {"name": "test_model", "config": {"materialized": "table", "tags": ["important"]}} result = self.task._get_nested_value(data, "config.materialized") assert result == "table" def test_get_nested_value_deep_nested_key(self): """Test getting a deeply nested key""" data = { "name": "test_model", "config": {"meta": {"owner": "data-team", "contact": {"email": "team@company.com"}}}, } result = self.task._get_nested_value(data, "config.meta.owner") assert result == "data-team" result = self.task._get_nested_value(data, "config.meta.contact.email") assert result == "team@company.com" def test_get_nested_value_nonexistent_top_level(self): """Test getting a non-existent top-level key returns None""" data = {"name": "test_model", "type": "model"} result = self.task._get_nested_value(data, "nonexistent") assert result is None def test_get_nested_value_nonexistent_nested(self): """Test getting a non-existent nested key returns None""" data = {"name": "test_model", "config": {"materialized": "table"}} result = self.task._get_nested_value(data, "config.nonexistent") assert result is None def test_get_nested_value_nonexistent_parent(self): """Test getting a nested key where parent doesn't exist returns None""" data = {"name": "test_model", "type": "model"} result = self.task._get_nested_value(data, "nonexistent.child") assert result is None def test_get_nested_value_non_dict_parent(self): """Test getting a nested key where parent is not a dict returns None""" data = {"name": "test_model", "config": "not_a_dict"} result = self.task._get_nested_value(data, "config.materialized") assert result is None def test_get_nested_value_none_parent(self): """Test getting a nested key where parent is None returns None""" data = {"name": "test_model", "config": None} result = self.task._get_nested_value(data, "config.materialized") assert result is None def test_get_nested_value_list_value(self): """Test getting a nested key that contains a list""" data = {"name": "test_model", "config": {"tags": ["important", "daily"]}} result = self.task._get_nested_value(data, "config.tags") assert result == ["important", "daily"] def test_get_nested_value_dict_value(self): """Test getting a nested key that contains a dict""" data = { "name": "test_model", "config": {"meta": {"owner": "data-team", "criticality": "high"}}, } result = self.task._get_nested_value(data, "config.meta") assert result == {"owner": "data-team", "criticality": "high"} class TestGenerateJson: """Unit tests for the generate_json method to ensure coverage of nested key logic""" def setup_method(self): set_from_args(Namespace(models=None, output_keys=None), {}) self.task = ListTask(get_flags(), None, None) def create_mock_node(self, name="test_model", materialized="table", meta=None): """Helper to create a mock node with specified properties""" from unittest.mock import Mock node = Mock() node.to_dict.return_value = { "name": name, "resource_type": "model", "unique_id": f"model.test.{name}", "config": { "materialized": materialized, "tags": [], "meta": meta or {}, }, "original_file_path": f"models/{name}.sql", } return node def test_generate_json_with_nested_keys(self): """Test generate_json with nested output keys""" # Mock args to have nested output keys with patch.object(self.task, "args") as mock_args: mock_args.output_keys = ["name", "config.materialized"] # Mock _iterate_selected_nodes to return our test node with patch.object(self.task, "_iterate_selected_nodes") as mock_iterate: node = self.create_mock_node("test_model", "table") mock_iterate.return_value = [node] # Get the result results = list(self.task.generate_json()) assert len(results) == 1 import json result_data = json.loads(results[0]) assert result_data["name"] == "test_model" assert result_data["config.materialized"] == "table" def test_generate_json_with_nonexistent_nested_keys(self): """Test generate_json with non-existent nested keys""" # Mock args to have non-existent nested key with patch.object(self.task, "args") as mock_args: mock_args.output_keys = ["name", "config.nonexistent"] # Mock _iterate_selected_nodes to return our test node with patch.object(self.task, "_iterate_selected_nodes") as mock_iterate: node = self.create_mock_node("test_model", "table") mock_iterate.return_value = [node] # Get the result results = list(self.task.generate_json()) assert len(results) == 1 import json result_data = json.loads(results[0]) assert result_data["name"] == "test_model" # Non-existent key should not be in result assert "config.nonexistent" not in result_data def test_generate_json_with_mixed_keys(self): """Test generate_json with mix of regular and nested keys""" # Mock args to have mixed keys with patch.object(self.task, "args") as mock_args: mock_args.output_keys = [ "name", "resource_type", "config.materialized", "config.meta.owner", ] # Mock _iterate_selected_nodes to return our test node with patch.object(self.task, "_iterate_selected_nodes") as mock_iterate: node = self.create_mock_node("test_model", "incremental", {"owner": "data-team"}) mock_iterate.return_value = [node] # Get the result results = list(self.task.generate_json()) assert len(results) == 1 import json result_data = json.loads(results[0]) assert result_data["name"] == "test_model" assert result_data["resource_type"] == "model" assert result_data["config.materialized"] == "incremental" assert result_data["config.meta.owner"] == "data-team" def test_generate_json_with_no_output_keys(self): """Test generate_json without output_keys (default behavior)""" # Mock args to have None output_keys with patch.object(self.task, "args") as mock_args: mock_args.output_keys = None # Mock _iterate_selected_nodes to return our test node with patch.object(self.task, "_iterate_selected_nodes") as mock_iterate: node = self.create_mock_node("test_model", "table") mock_iterate.return_value = [node] # Get the result results = list(self.task.generate_json()) assert len(results) == 1 import json result_data = json.loads(results[0]) # Should contain ALLOWED_KEYS for allowed_key in self.task.ALLOWED_KEYS: if allowed_key in node.to_dict.return_value: assert allowed_key in result_data def test_generate_json_deep_nested_path(self): """Test generate_json with deeply nested paths""" # Mock args to have deep nested key with patch.object(self.task, "args") as mock_args: mock_args.output_keys = ["config.meta.contact.email"] # Mock _iterate_selected_nodes to return our test node with patch.object(self.task, "_iterate_selected_nodes") as mock_iterate: node = self.create_mock_node( "test_model", "table", {"contact": {"email": "team@company.com"}} ) mock_iterate.return_value = [node] # Get the result results = list(self.task.generate_json()) assert len(results) == 1 import json result_data = json.loads(results[0]) assert result_data["config.meta.contact.email"] == "team@company.com" ================================================ FILE: tests/unit/task/test_retry.py ================================================ from dbt.cli.types import Command from dbt.task.retry import CMD_DICT, TASK_DICT EXCLUDED_COMMANDS = { "clean", "debug", "deps", "freshness", "init", "list", "parse", "retry", "show", "serve", } def test_task_cmd_dicts(): assert TASK_DICT.keys() == CMD_DICT.keys() def test_exhaustive_commands(): assert set(TASK_DICT.keys()).union(EXCLUDED_COMMANDS) == set(i.value.lower() for i in Command) ================================================ FILE: tests/unit/task/test_run.py ================================================ from argparse import Namespace from dataclasses import dataclass from datetime import datetime from importlib import import_module from typing import Optional, Type, Union from unittest import mock from unittest.mock import MagicMock, patch import pytest from psycopg2 import DatabaseError from pytest_mock import MockerFixture from core.dbt.task.run import MicrobatchBatchRunner from dbt.adapters.contracts.connection import AdapterResponse from dbt.adapters.postgres import PostgresAdapter from dbt.artifacts.resources.base import FileHash from dbt.artifacts.resources.types import NodeType, RunHookType from dbt.artifacts.resources.v1.components import DependsOn from dbt.artifacts.resources.v1.config import NodeConfig from dbt.artifacts.resources.v1.model import ModelConfig from dbt.artifacts.schemas.results import RunStatus from dbt.artifacts.schemas.run import RunResult from dbt.config.runtime import RuntimeConfig from dbt.contracts.graph.manifest import Manifest from dbt.contracts.graph.nodes import HookNode, ModelNode from dbt.events.types import LogModelResult from dbt.exceptions import DbtRuntimeError from dbt.flags import get_flags, set_from_args from dbt.task.run import MicrobatchModelRunner, ModelRunner, RunTask, _get_adapter_info from dbt.tests.util import safe_set_invocation_context from dbt_common.events.base_types import EventLevel from dbt_common.events.event_catcher import EventCatcher from dbt_common.events.event_manager_client import add_callback_to_manager @pytest.mark.parametrize( "exception_to_raise, expected_cancel_connections", [ (SystemExit, True), (KeyboardInterrupt, True), (Exception, False), ], ) def test_run_task_cancel_connections( exception_to_raise, expected_cancel_connections, runtime_config: RuntimeConfig ): safe_set_invocation_context() def mock_run_queue(*args, **kwargs): raise exception_to_raise("Test exception") with patch.object(RunTask, "run_queue", mock_run_queue), patch.object( RunTask, "_cancel_connections" ) as mock_cancel_connections: set_from_args(Namespace(write_json=False), None) task = RunTask( get_flags(), runtime_config, None, ) with pytest.raises(exception_to_raise): task.execute_nodes() assert mock_cancel_connections.called == expected_cancel_connections def test_run_task_preserve_edges(): mock_node_selector = MagicMock() mock_spec = MagicMock() with patch.object(RunTask, "get_node_selector", return_value=mock_node_selector), patch.object( RunTask, "get_selection_spec", return_value=mock_spec ): task = RunTask(get_flags(), None, None) task.get_graph_queue() # when we get the graph queue, preserve_edges is True mock_node_selector.get_graph_queue.assert_called_with(mock_spec, True) def test_tracking_fails_safely_for_missing_adapter(): assert {} == _get_adapter_info(None, {}) def test_adapter_info_tracking(): mock_run_result = MagicMock() mock_run_result.node = MagicMock() mock_run_result.node.config = {} assert _get_adapter_info(PostgresAdapter, mock_run_result) == { "model_adapter_details": {}, "adapter_name": PostgresAdapter.__name__.split("Adapter")[0].lower(), "adapter_version": import_module("dbt.adapters.postgres.__version__").version, "base_adapter_version": import_module("dbt.adapters.__about__").version, } class TestModelRunner: @pytest.fixture def log_model_result_catcher(self) -> EventCatcher: catcher = EventCatcher(event_to_catch=LogModelResult) add_callback_to_manager(catcher.catch) return catcher @pytest.fixture def model_runner( self, postgres_adapter: PostgresAdapter, table_model: ModelNode, runtime_config: RuntimeConfig, ) -> ModelRunner: return ModelRunner( config=runtime_config, adapter=postgres_adapter, node=table_model, node_index=1, num_nodes=1, ) @pytest.fixture def run_result(self, table_model: ModelNode) -> RunResult: return RunResult( status=RunStatus.Success, timing=[], thread_id="an_id", execution_time=0, adapter_response={}, message="It did it", failures=None, batch_results=None, node=table_model, ) def test_print_result_line( self, log_model_result_catcher: EventCatcher, model_runner: ModelRunner, run_result: RunResult, ) -> None: # Check `print_result_line` with "successful" RunResult model_runner.print_result_line(run_result) assert len(log_model_result_catcher.caught_events) == 1 assert log_model_result_catcher.caught_events[0].info.level == EventLevel.INFO assert log_model_result_catcher.caught_events[0].data.status == run_result.message # reset event catcher log_model_result_catcher.flush() # Check `print_result_line` with "error" RunResult run_result.status = RunStatus.Error model_runner.print_result_line(run_result) assert len(log_model_result_catcher.caught_events) == 1 assert log_model_result_catcher.caught_events[0].info.level == EventLevel.ERROR assert log_model_result_catcher.caught_events[0].data.status == EventLevel.ERROR @pytest.mark.skip( reason="Default and adapter macros aren't being appropriately populated, leading to a runtime error" ) def test_execute( self, table_model: ModelNode, manifest: Manifest, model_runner: ModelRunner ) -> None: model_runner.execute(model=table_model, manifest=manifest) # TODO: Assert that the model was executed class TestMicrobatchModelRunner: @pytest.fixture def model_runner( self, postgres_adapter: PostgresAdapter, table_model: ModelNode, runtime_config: RuntimeConfig, ) -> MicrobatchModelRunner: return MicrobatchModelRunner( config=runtime_config, adapter=postgres_adapter, node=table_model, node_index=1, num_nodes=1, ) @pytest.fixture def batch_runner( self, postgres_adapter: PostgresAdapter, table_model: ModelNode, runtime_config: RuntimeConfig, ) -> MicrobatchBatchRunner: return MicrobatchBatchRunner( config=runtime_config, adapter=postgres_adapter, node=table_model, node_index=1, num_nodes=1, batch_idx=0, batches=[], relation_exists=False, incremental_batch=False, ) @pytest.mark.parametrize( "has_relation,relation_type,materialized,full_refresh_config,full_refresh_flag,expectation", [ (False, "table", "incremental", None, False, False), (True, "other", "incremental", None, False, False), (True, "table", "other", None, False, False), # model config takes precendence (True, "table", "incremental", True, False, False), # model config takes precendence (True, "table", "incremental", True, True, False), # model config takes precendence (True, "table", "incremental", False, False, True), # model config takes precendence (True, "table", "incremental", False, True, True), # model config is none, so opposite flag value (True, "table", "incremental", None, True, False), # model config is none, so opposite flag value (True, "table", "incremental", None, False, True), ], ) def test__is_incremental( self, mocker: MockerFixture, model_runner: MicrobatchModelRunner, has_relation: bool, relation_type: str, materialized: str, full_refresh_config: Optional[bool], full_refresh_flag: bool, expectation: bool, ) -> None: # Setup adapter relation getting @dataclass class RelationInfo: database: str = "database" schema: str = "schema" name: str = "name" @dataclass class Relation: type: str model_runner.adapter = mocker.Mock() model_runner.adapter.Relation.create_from.return_value = RelationInfo() if has_relation: model_runner.adapter.get_relation.return_value = Relation(type=relation_type) else: model_runner.adapter.get_relation.return_value = None # Set ModelRunner configs model_runner.config.args = Namespace(FULL_REFRESH=full_refresh_flag) # Create model with configs model = model_runner.node model.config = ModelConfig(materialized=materialized, full_refresh=full_refresh_config) # Assert result of _is_incremental assert model_runner._is_incremental(model) == expectation @pytest.mark.parametrize( "adapter_microbatch_concurrency,has_relation,concurrent_batches,has_this,expectation", [ (True, True, None, False, True), (True, True, None, True, False), (True, True, True, False, True), (True, True, True, True, True), (True, True, False, False, False), (True, True, False, True, False), (True, False, None, False, False), (True, False, None, True, False), (True, False, True, False, False), (True, False, True, True, False), (True, False, False, False, False), (True, False, False, True, False), (False, True, None, False, False), (False, True, None, True, False), (False, True, True, False, False), (False, True, True, True, False), (False, True, False, False, False), (False, True, False, True, False), (False, False, None, False, False), (False, False, None, True, False), (False, False, True, False, False), (False, False, True, True, False), (False, False, False, False, False), (False, False, False, True, False), ], ) def test_should_run_in_parallel( self, mocker: MockerFixture, batch_runner: MicrobatchBatchRunner, adapter_microbatch_concurrency: bool, has_relation: bool, concurrent_batches: Optional[bool], has_this: bool, expectation: bool, ) -> None: batch_runner.node._has_this = has_this batch_runner.node.config = ModelConfig(concurrent_batches=concurrent_batches) batch_runner.relation_exists = has_relation mocked_supports = mocker.patch.object(batch_runner.adapter, "supports") mocked_supports.return_value = adapter_microbatch_concurrency # Assert result of should_run_in_parallel assert batch_runner.should_run_in_parallel() == expectation def test_get_microbatch_builder_uses_original_invocation_time_on_retry( self, mocker: MockerFixture, model_runner: MicrobatchModelRunner, ) -> None: """When retrying, get_microbatch_builder should use the original invocation time from the previous run rather than the current invocation time.""" original_time = datetime(2025, 3, 21, 7, 55, 0) current_time = datetime(2025, 3, 25, 1, 4, 0) # Set up a mock parent task with original_invocation_started_at mock_parent = mocker.Mock(spec=RunTask) mock_parent.original_invocation_started_at = original_time model_runner._parent_task = mock_parent # Mock _is_incremental to avoid adapter calls mocker.patch.object(model_runner, "_is_incremental", return_value=True) # Mock get_invocation_started_at to return the "current" (retry) time mocker.patch( "dbt.task.run.get_invocation_started_at", return_value=current_time, ) model = model_runner.node model.config.materialized = "incremental" model.config.incremental_strategy = "microbatch" model.config.batch_size = "day" model.config.begin = "2024-12-01" model.config.event_time = "_event_date" model_runner.config.args = Namespace( EVENT_TIME_START=None, EVENT_TIME_END=None, SAMPLE=None ) builder = model_runner.get_microbatch_builder(model) assert builder.default_end_time == original_time def test_get_microbatch_builder_uses_current_time_without_retry( self, mocker: MockerFixture, model_runner: MicrobatchModelRunner, ) -> None: """When not retrying (normal run), get_microbatch_builder should use the current invocation time.""" current_time = datetime(2025, 3, 25, 1, 4, 0) # Set up a mock parent task with no original_invocation_started_at mock_parent = mocker.Mock(spec=RunTask) mock_parent.original_invocation_started_at = None model_runner._parent_task = mock_parent # Mock _is_incremental to avoid adapter calls mocker.patch.object(model_runner, "_is_incremental", return_value=True) mocker.patch( "dbt.task.run.get_invocation_started_at", return_value=current_time, ) model = model_runner.node model.config.materialized = "incremental" model.config.incremental_strategy = "microbatch" model.config.batch_size = "day" model.config.begin = "2024-12-01" model.config.event_time = "_event_date" model_runner.config.args = Namespace( EVENT_TIME_START=None, EVENT_TIME_END=None, SAMPLE=None ) builder = model_runner.get_microbatch_builder(model) assert builder.default_end_time == current_time class TestRunTask: @pytest.fixture def hook_node(self) -> HookNode: return HookNode( package_name="test", path="/root/x/path.sql", original_file_path="/root/path.sql", language="sql", raw_code="select * from wherever", name="foo", resource_type=NodeType.Operation, unique_id="model.test.foo", fqn=["test", "models", "foo"], refs=[], sources=[], metrics=[], depends_on=DependsOn(), description="", database="test_db", schema="test_schema", alias="bar", tags=[], config=NodeConfig(), index=None, checksum=FileHash.from_contents(""), unrendered_config={}, ) @pytest.mark.parametrize( "error_to_raise,expected_result", [ (None, RunStatus.Success), (DbtRuntimeError, RunStatus.Error), (DatabaseError, RunStatus.Error), (KeyboardInterrupt, KeyboardInterrupt), ], ) def test_safe_run_hooks( self, mocker: MockerFixture, runtime_config: RuntimeConfig, manifest: Manifest, hook_node: HookNode, error_to_raise: Optional[Type[Exception]], expected_result: Union[RunStatus, Type[Exception]], ): mocker.patch("dbt.task.run.RunTask.get_hooks_by_type").return_value = [hook_node] mocker.patch("dbt.task.run.RunTask.get_hook_sql").return_value = hook_node.raw_code flags = mock.Mock() flags.state = None flags.defer_state = None run_task = RunTask( args=flags, config=runtime_config, manifest=manifest, ) adapter = mock.Mock() adapter_execute = mock.Mock() adapter_execute.return_value = (AdapterResponse(_message="Success"), None) if error_to_raise: adapter_execute.side_effect = error_to_raise("Oh no!") adapter.execute = adapter_execute try: result = run_task.safe_run_hooks( adapter=adapter, hook_type=RunHookType.End, extra_context={}, ) assert isinstance(expected_result, RunStatus) assert result == expected_result except BaseException as e: assert not isinstance(expected_result, RunStatus) assert issubclass(expected_result, BaseException) assert type(e) == expected_result ================================================ FILE: tests/unit/task/test_test.py ================================================ import agate import pytest from dbt.task.test import list_rows_from_table class TestListRowsFromTable: @pytest.mark.parametrize( "agate_table_cols,agate_table_rows,expected_list_rows", [ (["a", "b", "c"], [], [["a", "b", "c"]]), # no rows (["a", "b", "c"], [[1, 2, 3]], [["a", "b", "c"], [1, 2, 3]]), # single row, no nulls ( ["a", "b", "c"], [[1, 2, 3], [2, 3, 4]], [["a", "b", "c"], [1, 2, 3], [2, 3, 4]], ), # multiple rows ( ["a", "b", "c"], [[None, 2, 3], [2, None, 4]], [["a", "b", "c"], [None, 2, 3], [2, None, 4]], ), # multiple rows, with nulls ], ) def test_list_rows_from_table_no_sort( self, agate_table_cols, agate_table_rows, expected_list_rows ): table = agate.Table(rows=agate_table_rows, column_names=agate_table_cols) list_rows = list_rows_from_table(table) assert list_rows == expected_list_rows @pytest.mark.parametrize( "agate_table_cols,agate_table_rows,expected_list_rows", [ (["a", "b", "c"], [], [["a", "b", "c"]]), # no rows (["a", "b", "c"], [[1, 2, 3]], [["a", "b", "c"], [1, 2, 3]]), # single row, no nulls ( ["a", "b", "c"], [[1, 2, 3], [2, 3, 4]], [["a", "b", "c"], [1, 2, 3], [2, 3, 4]], ), # multiple rows, in order ( ["a", "b", "c"], [[2, 3, 4], [1, 2, 3]], [["a", "b", "c"], [1, 2, 3], [2, 3, 4]], ), # multiple rows, out of order ( ["a", "b", "c"], [[None, 2, 3], [2, 3, 4]], [["a", "b", "c"], [2, 3, 4], [None, 2, 3]], ), # multiple rows, out of order with nulls in first position ( ["a", "b", "c"], [[4, 5, 6], [1, None, 3]], [["a", "b", "c"], [1, None, 3], [4, 5, 6]], ), # multiple rows, out of order with null in non-first position ( ["a", "b", "c"], [[None, 5, 6], [1, None, 3]], [["a", "b", "c"], [1, None, 3], [None, 5, 6]], ), # multiple rows, out of order with nulls in many positions ], ) def test_list_rows_from_table_with_sort( self, agate_table_cols, agate_table_rows, expected_list_rows ): table = agate.Table(rows=agate_table_rows, column_names=agate_table_cols) list_rows = list_rows_from_table(table, sort=True) assert list_rows == expected_list_rows ================================================ FILE: tests/unit/test_artifact_upload.py ================================================ import os import unittest import uuid from unittest import mock from unittest.mock import MagicMock, call, patch from dbt.constants import MANIFEST_FILE_NAME, RUN_RESULTS_FILE_NAME from dbt.exceptions import DbtProjectError from dbt.utils.artifact_upload import ( ArtifactUploadConfig, _retry_with_backoff, add_artifact_produced, upload_artifacts, ) from dbt_common.exceptions import DbtBaseException class TestArtifactUploadConfig(unittest.TestCase): def setUp(self): self.config = ArtifactUploadConfig( tenant_hostname="test-tenant.dbt.com", DBT_CLOUD_TOKEN="test-token", DBT_CLOUD_ACCOUNT_ID="1234", DBT_CLOUD_ENVIRONMENT_ID="5678", ) self.test_invocation_id = str(uuid.uuid4()) def test_get_ingest_url(self): expected_url = ( "https://test-tenant.dbt.com/api/private/accounts/1234/environments/5678/ingests/" ) self.assertEqual(self.config.get_ingest_url(), expected_url) def test_get_complete_url(self): ingest_id = "9012" expected_url = ( "https://test-tenant.dbt.com/api/private/accounts/1234/environments/5678/ingests/9012/" ) self.assertEqual(self.config.get_complete_url(ingest_id), expected_url) def test_get_headers_with_invocation_id(self): expected_headers = { "Accept": "application/json", "X-Invocation-Id": self.test_invocation_id, "Authorization": "Token test-token", } self.assertEqual( self.config.get_headers(invocation_id=self.test_invocation_id), expected_headers, ) def test_get_headers_without_invocation_id(self): with mock.patch("uuid.uuid4") as mock_uuid: mock_uuid.return_value = uuid.UUID("12345678-1234-1234-1234-123456789012") expected_headers = { "Accept": "application/json", "X-Invocation-Id": "12345678-1234-1234-1234-123456789012", "Authorization": "Token test-token", } self.assertEqual(self.config.get_headers(), expected_headers) class TestRetryWithBackoff(unittest.TestCase): def setUp(self): self.time_sleep_patcher = patch("dbt.utils.artifact_upload.time.sleep") self.mock_sleep = self.time_sleep_patcher.start() def tearDown(self): self.time_sleep_patcher.stop() def test_successful_first_try(self): """Test that function returns immediately on success.""" func = MagicMock(return_value=(True, "success")) result = _retry_with_backoff("operation", func) self.assertEqual(result, "success") func.assert_called_once() self.mock_sleep.assert_not_called() def test_successful_after_retry(self): """Test that function retries and succeeds.""" mock_response = MagicMock() mock_response.status_code = 503 func = MagicMock(side_effect=[(False, mock_response), (True, "success")]) result = _retry_with_backoff("operation", func) self.assertEqual(result, "success") self.assertEqual(func.call_count, 2) self.mock_sleep.assert_called_once_with(1) # First retry delay def test_failure_after_max_retries(self): """Test that function raises exception after max retries.""" mock_response = MagicMock() mock_response.status_code = 503 func = MagicMock(return_value=(False, mock_response)) with self.assertRaises(DbtBaseException) as context: _retry_with_backoff("operation", func, max_retries=3) self.assertIn("Error operation", str(context.exception)) self.assertEqual(func.call_count, 4) # Sleep should be called twice (after first and second attempts) self.assertEqual(self.mock_sleep.call_count, 3) self.assertEqual( self.mock_sleep.call_args_list, [call(1), call(2), call(4)] ) # Exponential backoff def test_non_retryable_status_code(self): """Test that non-retryable status codes raise immediately.""" mock_response = MagicMock() mock_response.status_code = 400 # Not in retry_codes func = MagicMock(return_value=(False, mock_response)) with self.assertRaises(DbtBaseException) as context: _retry_with_backoff("operation", func) self.assertIn("Error operation", str(context.exception)) self.assertEqual(func.call_count, 1) self.mock_sleep.assert_not_called() def test_request_exception_handling(self): """Test that RequestException is caught and retried.""" import requests func = MagicMock( side_effect=[requests.RequestException("Network error"), (True, "success")] ) result = _retry_with_backoff("operation", func) self.assertEqual(result, "success") self.assertEqual(func.call_count, 2) self.mock_sleep.assert_called_once_with(1) def test_request_exception_max_retries(self): """Test that RequestException raises after max retries.""" import requests func = MagicMock(side_effect=requests.RequestException("Network error")) with self.assertRaises(DbtBaseException) as context: _retry_with_backoff("operation", func, max_retries=3) self.assertIn("Error operation: Network error", str(context.exception)) self.assertEqual(func.call_count, 4) self.assertEqual(self.mock_sleep.call_count, 3) def test_custom_retry_codes(self): """Test that custom retry codes are respected.""" mock_response = MagicMock() mock_response.status_code = 429 # Too Many Requests func = MagicMock(side_effect=[(False, mock_response), (True, "success")]) result = _retry_with_backoff("operation", func, retry_codes=[429, 503]) self.assertEqual(result, "success") self.assertEqual(func.call_count, 2) self.mock_sleep.assert_called_once_with(1) class TestUploadArtifacts(unittest.TestCase): def setUp(self): self.project_dir = "/fake/project/dir" self.target_path = "/fake/project/dir/target" self.command = "run" # Create patchers self.load_project_patcher = patch("dbt.utils.artifact_upload.load_project") self.zipfile_patcher = patch("dbt.utils.artifact_upload.zipfile.ZipFile") self.requests_post_patcher = patch("dbt.utils.artifact_upload.requests.post") self.requests_put_patcher = patch("dbt.utils.artifact_upload.requests.put") self.requests_patch_patcher = patch("dbt.utils.artifact_upload.requests.patch") self.open_patcher = patch("builtins.open", mock.mock_open(read_data=b"test data")) self.fire_event_patcher = patch("dbt.utils.artifact_upload.fire_event") self.retry_patcher = patch("dbt.utils.artifact_upload._retry_with_backoff") self.time_sleep_patcher = patch("dbt.utils.artifact_upload.time.sleep") # Start patchers self.mock_load_project = self.load_project_patcher.start() self.mock_zipfile = self.zipfile_patcher.start() self.mock_requests_post = self.requests_post_patcher.start() self.mock_requests_put = self.requests_put_patcher.start() self.mock_requests_patch = self.requests_patch_patcher.start() self.mock_open = self.open_patcher.start() self.mock_fire_event = self.fire_event_patcher.start() self.mock_retry = self.retry_patcher.start() self.mock_sleep = self.time_sleep_patcher.start() # Configure mocks self.mock_project = MagicMock() self.mock_project.dbt_cloud = {"tenant_hostname": "test-tenant"} self.mock_load_project.return_value = self.mock_project # Mock response for POST request (create ingest) self.mock_post_response = MagicMock() self.mock_post_response.status_code = 200 self.mock_post_response.json.return_value = { "data": {"id": "ingest123", "upload_url": "https://test-upload-url.com"} } self.mock_requests_post.return_value = self.mock_post_response # Mock response for PUT request (upload artifacts) self.mock_put_response = MagicMock() self.mock_put_response.status_code = 200 self.mock_requests_put.return_value = self.mock_put_response # Mock response for PATCH request (complete ingest) self.mock_patch_response = MagicMock() self.mock_patch_response.status_code = 204 self.mock_requests_patch.return_value = self.mock_patch_response # Mock retry to pass through to the first result self.mock_retry.side_effect = lambda operation_name, func, max_retries=3: func()[1] # Setup the env var for the test self.original_token = os.environ.get("DBT_CLOUD_TOKEN") self.original_account_id = os.environ.get("DBT_CLOUD_ACCOUNT_ID") self.original_environment_id = os.environ.get("DBT_CLOUD_ENVIRONMENT_ID") os.environ["DBT_CLOUD_TOKEN"] = "test-token" os.environ["DBT_CLOUD_ACCOUNT_ID"] = "1234" os.environ["DBT_CLOUD_ENVIRONMENT_ID"] = "5678" def tearDown(self): self.load_project_patcher.stop() self.zipfile_patcher.stop() self.requests_post_patcher.stop() self.requests_put_patcher.stop() self.requests_patch_patcher.stop() self.open_patcher.stop() self.fire_event_patcher.stop() self.retry_patcher.stop() self.time_sleep_patcher.stop() if self.original_token: os.environ["DBT_CLOUD_TOKEN"] = self.original_token if self.original_account_id: os.environ["DBT_CLOUD_ACCOUNT_ID"] = self.original_account_id if self.original_environment_id: os.environ["DBT_CLOUD_ENVIRONMENT_ID"] = self.original_environment_id def test_upload_artifacts_successful_upload(self): # Set up mock for ZipFile context manager mock_zipfile_instance = MagicMock() self.mock_zipfile.return_value.__enter__.return_value = mock_zipfile_instance add_artifact_produced(os.path.join(self.target_path, MANIFEST_FILE_NAME)) add_artifact_produced(os.path.join(self.target_path, RUN_RESULTS_FILE_NAME)) # Call the function upload_artifacts(self.project_dir, self.target_path, self.command) # Verify the project was loaded self.mock_load_project.assert_called_once_with( self.project_dir, version_check=False, profile=mock.ANY, cli_vars=None ) # Verify zip file was created and artifacts were added self.mock_zipfile.assert_called_once_with("target.zip", "w") expected_artifact_calls = [ call(f"{self.target_path}/{RUN_RESULTS_FILE_NAME}", RUN_RESULTS_FILE_NAME), call(f"{self.target_path}/{MANIFEST_FILE_NAME}", MANIFEST_FILE_NAME), ] mock_zipfile_instance.write.assert_has_calls(expected_artifact_calls, any_order=True) # Verify retry was called for each step self.assertEqual(self.mock_retry.call_count, 3) retry_calls = [call_args[0][0] for call_args in self.mock_retry.call_args_list] self.assertIn("creating ingest request", retry_calls) self.assertIn("uploading artifacts", retry_calls) self.assertIn("completing ingest", retry_calls) # Verify fire_event was called with ArtifactUploadSuccess success_event_call = [ call for call in self.mock_fire_event.call_args_list if "completed successfully" in call[0][0].msg ] self.assertTrue(len(success_event_call) > 0) def test_upload_artifacts_default_target_path(self): # Call the function with target_path=None mock_zipfile_instance = MagicMock() self.mock_zipfile.return_value.__enter__.return_value = mock_zipfile_instance add_artifact_produced(os.path.join(self.target_path, MANIFEST_FILE_NAME)) add_artifact_produced(os.path.join(self.target_path, RUN_RESULTS_FILE_NAME)) upload_artifacts(self.project_dir, None, self.command) # Verify the default target path was used expected_artifact_calls = [ call(f"{self.target_path}/{RUN_RESULTS_FILE_NAME}", RUN_RESULTS_FILE_NAME), call(f"{self.target_path}/{MANIFEST_FILE_NAME}", MANIFEST_FILE_NAME), ] mock_zipfile_instance.write.assert_has_calls(expected_artifact_calls, any_order=True) def test_upload_artifacts_missing_tenant_config(self): # Set up project without dbt_cloud config self.mock_project.dbt_cloud = {} add_artifact_produced(os.path.join(self.target_path, MANIFEST_FILE_NAME)) # Verify that the function raises an exception with self.assertRaises(DbtProjectError) as context: upload_artifacts(self.project_dir, self.target_path, self.command) self.assertIn("tenant_hostname not found", str(context.exception)) self.mock_retry.assert_not_called() def test_upload_artifacts_with_retry_failures(self): # Set up mock for ZipFile context manager mock_zipfile_instance = MagicMock() self.mock_zipfile.return_value.__enter__.return_value = mock_zipfile_instance # Make retry raise exceptions for each step self.mock_retry.side_effect = [ DbtBaseException("Error creating ingest request: Mock failure"), DbtBaseException("Error uploading artifacts: Mock failure"), DbtBaseException("Error completing ingest: Mock failure"), ] add_artifact_produced(os.path.join(self.target_path, MANIFEST_FILE_NAME)) # Test each step failing # 1. Create ingest failure with self.assertRaises(DbtBaseException) as context: upload_artifacts(self.project_dir, self.target_path, self.command) self.assertIn("Error creating ingest request", str(context.exception)) self.mock_retry.reset_mock() # Reset the side effect for the next test self.mock_retry.side_effect = [ self.mock_post_response, # First call succeeds DbtBaseException("Error uploading artifacts: Mock failure"), DbtBaseException("Error completing ingest: Mock failure"), ] # 2. Upload failure with self.assertRaises(DbtBaseException) as context: upload_artifacts(self.project_dir, self.target_path, self.command) self.assertIn("Error uploading artifacts", str(context.exception)) self.mock_retry.reset_mock() # Reset the side effect for the next test self.mock_retry.side_effect = [ self.mock_post_response, # First call succeeds self.mock_put_response, # Second call succeeds DbtBaseException("Error completing ingest: Mock failure"), ] # 3. Complete failure with self.assertRaises(DbtBaseException) as context: upload_artifacts(self.project_dir, self.target_path, self.command) self.assertIn("Error completing ingest", str(context.exception)) if __name__ == "__main__": unittest.main() ================================================ FILE: tests/unit/test_behavior_flags.py ================================================ import pytest from dbt.tracking import ( disable_tracking, initialize_from_flags, track_behavior_change_warn, ) from dbt_common.behavior_flags import Behavior from dbt_common.events.event_manager_client import ( add_callback_to_manager, cleanup_event_logger, ) @pytest.fixture def snowplow_tracker(mocker): # initialize `active_user` without writing the cookie to disk initialize_from_flags(True, "") mocker.patch("dbt.tracking.User.set_cookie").return_value = {"id": 42} # add the relevant callback to the event manager add_callback_to_manager(track_behavior_change_warn) # don't make a call, catch the request # to avoid confusion, this is snowplow_tracker's track, not our wrapper that's also named track snowplow_tracker = mocker.patch("dbt.tracking.tracker.track") yield snowplow_tracker # teardown cleanup_event_logger() disable_tracking() def test_false_evaluation_triggers_snowplow_tracking(snowplow_tracker): behavior = Behavior( [{"name": "my_flag", "default": False, "description": "This is a false flag."}], {} ) if behavior.my_flag: # trigger a False evaluation assert False, "This flag should evaluate to false and skip this line" assert snowplow_tracker.called def test_true_evaluation_does_not_trigger_snowplow_tracking(snowplow_tracker): behavior = Behavior( [{"name": "my_flag", "default": True, "description": "This is a true flag."}], {} ) if behavior.my_flag: pass else: # trigger a True evaluation assert False, "This flag should evaluate to false and skip this line" assert not snowplow_tracker.called def test_false_evaluation_does_not_trigger_snowplow_tracking_when_disabled(snowplow_tracker): disable_tracking() behavior = Behavior( [{"name": "my_flag", "default": False, "description": "This is a false flag."}], {} ) if behavior.my_flag: # trigger a False evaluation assert False, "This flag should evaluate to false and skip this line" assert not snowplow_tracker.called ================================================ FILE: tests/unit/test_compilation.py ================================================ import os import tempfile from queue import Empty from unittest import mock import pytest from dbt.compilation import Graph, Linker from dbt.graph.cli import parse_difference from dbt.graph.queue import GraphQueue from dbt.graph.selector import NodeSelector def _mock_manifest(nodes): config = mock.MagicMock(enabled=True) manifest = mock.MagicMock( nodes={ n: mock.MagicMock( unique_id=n, package_name="pkg", name=n, empty=False, config=config, fqn=["pkg", n], is_versioned=False, ) for n in nodes } ) manifest.expect.side_effect = lambda n: mock.MagicMock(unique_id=n) return manifest class TestLinker: @pytest.fixture def linker(self) -> Linker: return Linker() def test_linker_add_node(self, linker: Linker) -> None: expected_nodes = ["A", "B", "C"] for node in expected_nodes: linker.add_node(node) actual_nodes = linker.nodes() for node in expected_nodes: assert node in actual_nodes assert len(actual_nodes) == len(expected_nodes) def test_linker_write_graph(self, linker: Linker) -> None: expected_nodes = ["A", "B", "C"] for node in expected_nodes: linker.add_node(node) manifest = _mock_manifest("ABC") (fd, fname) = tempfile.mkstemp() os.close(fd) try: linker.write_graph(fname, manifest) assert os.path.exists(fname) finally: os.unlink(fname) def assert_would_join(self, queue: GraphQueue) -> None: """test join() without timeout risk""" assert queue.inner.unfinished_tasks == 0 def _get_graph_queue( self, manifest, linker: Linker, include=None, exclude=None, ) -> GraphQueue: graph = Graph(linker.graph) selector = NodeSelector(graph, manifest) # TODO: The "eager" string below needs to be replaced with programatic access # to the default value for the indirect selection parameter in # dbt.cli.params.indirect_selection # # Doing that is actually a little tricky, so I'm punting it to a new ticket GH #6397 spec = parse_difference(include, exclude) return selector.get_graph_queue(spec) def test_linker_add_dependency(self, linker: Linker) -> None: actual_deps = [("A", "B"), ("A", "C"), ("B", "C")] for l, r in actual_deps: linker.dependency(l, r) queue = self._get_graph_queue(_mock_manifest("ABC"), linker) got = queue.get(block=False) assert got.unique_id == "C" with pytest.raises(Empty): queue.get(block=False) assert not queue.empty() queue.mark_done("C") assert not queue.empty() got = queue.get(block=False) assert got.unique_id == "B" with pytest.raises(Empty): queue.get(block=False) assert not queue.empty() queue.mark_done("B") assert not queue.empty() got = queue.get(block=False) assert got.unique_id == "A" with pytest.raises(Empty): queue.get(block=False) assert queue.empty() queue.mark_done("A") self.assert_would_join(queue) assert queue.empty() def test_linker_add_disjoint_dependencies(self, linker: Linker) -> None: actual_deps = [("A", "B")] additional_node = "Z" for l, r in actual_deps: linker.dependency(l, r) linker.add_node(additional_node) queue = self._get_graph_queue(_mock_manifest("ABCZ"), linker) # the first one we get must be B, it has the longest dep chain first = queue.get(block=False) assert first.unique_id == "B" assert not queue.empty() queue.mark_done("B") assert not queue.empty() second = queue.get(block=False) assert second.unique_id in {"A", "Z"} assert not queue.empty() queue.mark_done(second.unique_id) assert not queue.empty() third = queue.get(block=False) assert third.unique_id in {"A", "Z"} with pytest.raises(Empty): queue.get(block=False) assert second.unique_id != third.unique_id assert queue.empty() queue.mark_done(third.unique_id) self.assert_would_join(queue) assert queue.empty() def test_linker_dependencies_limited_to_some_nodes(self, linker: Linker) -> None: actual_deps = [("A", "B"), ("B", "C"), ("C", "D")] for l, r in actual_deps: linker.dependency(l, r) queue = self._get_graph_queue(_mock_manifest("ABCD"), linker, ["B"]) got = queue.get(block=False) assert got.unique_id == "B" assert queue.empty() queue.mark_done("B") self.assert_would_join(queue) queue_2 = queue = self._get_graph_queue(_mock_manifest("ABCD"), linker, ["A", "B"]) got = queue_2.get(block=False) assert got.unique_id == "B" assert not queue_2.empty() with pytest.raises(Empty): queue_2.get(block=False) queue_2.mark_done("B") assert not queue_2.empty() got = queue_2.get(block=False) assert got.unique_id == "A" assert queue_2.empty() with pytest.raises(Empty): queue_2.get(block=False) assert queue_2.empty() queue_2.mark_done("A") self.assert_would_join(queue_2) def test__find_cycles__cycles(self, linker: Linker) -> None: actual_deps = [("A", "B"), ("B", "C"), ("C", "A")] for l, r in actual_deps: linker.dependency(l, r) assert linker.find_cycles() is not None def test__find_cycles__no_cycles(self, linker: Linker) -> None: actual_deps = [("A", "B"), ("B", "C"), ("C", "D")] for l, r in actual_deps: linker.dependency(l, r) assert linker.find_cycles() is None ================================================ FILE: tests/unit/test_compilation_threading.py ================================================ import threading from dbt.artifacts.resources import Contract from dbt.contracts.files import FileHash from dbt.contracts.graph.nodes import DependsOn, ModelConfig, ModelNode from dbt.node_types import NodeType def _make_model_node(unique_id="model.test.foo", compiled=False, extra_ctes=None): return ModelNode( package_name="test", path="/root/models/foo.sql", original_file_path="models/foo.sql", language="sql", raw_code="select 1", name=unique_id.split(".")[-1], resource_type=NodeType.Model, unique_id=unique_id, fqn=["test", "models", unique_id.split(".")[-1]], refs=[], sources=[], metrics=[], depends_on=DependsOn(), description="", primary_key=[], database="test_db", schema="test_schema", alias=unique_id.split(".")[-1], tags=[], config=ModelConfig(), contract=Contract(), meta={}, compiled=compiled, extra_ctes=extra_ctes or [], extra_ctes_injected=False, compiled_code="select 1" if compiled else None, checksum=FileHash.from_contents(""), unrendered_config={}, ) class TestConcurrentEphemeralCompilation: def test_concurrent_ephemeral_compilation(self): """Two threads compile nodes sharing an ephemeral dep. The ephemeral should only be compiled once (compiled flag set once).""" ephemeral = _make_model_node(unique_id="model.test.ephemeral", compiled=False) compile_count = {"count": 0} lock = threading.Lock() barrier = threading.Barrier(2) def simulate_compile(): barrier.wait(timeout=5) with ephemeral._lock: if ephemeral.compiled is True and ephemeral.extra_ctes_injected is True: return # Simulate compilation with lock: compile_count["count"] += 1 ephemeral.compiled = True ephemeral.compiled_code = "select 1" ephemeral.extra_ctes_injected = True threads = [threading.Thread(target=simulate_compile) for _ in range(2)] for t in threads: t.start() for t in threads: t.join(timeout=10) assert ( compile_count["count"] == 1 ), f"Ephemeral compiled {compile_count['count']} times, expected 1" assert ephemeral.compiled is True assert ephemeral.extra_ctes_injected is True class TestNoDeadlockOnRecursiveLocking: def test_recursive_lock_pattern_does_not_deadlock(self): """Simulate the lock acquisition pattern from _recursively_prepend_ctes for an ephemeral chain: model_a refs ephemeral_b. The real code acquires ephemeral_b._lock for compilation, releases it, then re-acquires ephemeral_b._lock for CTE injection during the recursive call. If the recursion happened *inside* the first lock, this would deadlock with a non-reentrant Lock. This test verifies the current code structure avoids that.""" model_a = _make_model_node(unique_id="model.test.a", compiled=True) ephemeral_b = _make_model_node(unique_id="model.test.b", compiled=False) deadlocked = {"value": False} def simulate_recursively_prepend_ctes(): # -- Processing model_a's extra_ctes, found ephemeral_b -- # Critical section A: compile ephemeral_b (scoped to just compilation) needs_recursion = False with ephemeral_b._lock: if not ephemeral_b.compiled: ephemeral_b.compiled = True ephemeral_b.compiled_code = "select 1" needs_recursion = True # Recursive call for ephemeral_b happens OUTSIDE the lock above. # Inside that recursive call, we'd hit critical section B for # ephemeral_b (CTE injection). This would deadlock if we were # still inside the `with ephemeral_b._lock` above. if needs_recursion: # Simulate the CTE injection lock from the recursive call acquired = ephemeral_b._lock.acquire(timeout=2) if not acquired: deadlocked["value"] = True return ephemeral_b.extra_ctes_injected = True ephemeral_b._lock.release() # Critical section B for model_a (CTE injection) acquired = model_a._lock.acquire(timeout=2) if not acquired: deadlocked["value"] = True return model_a.extra_ctes_injected = True model_a._lock.release() t = threading.Thread(target=simulate_recursively_prepend_ctes) t.start() t.join(timeout=5) assert not t.is_alive(), "Thread is still alive — deadlock detected" assert not deadlocked["value"], "Failed to acquire lock — possible deadlock" assert ephemeral_b.compiled is True assert ephemeral_b.extra_ctes_injected is True assert model_a.extra_ctes_injected is True class TestLockSerialization: def test_lock_excluded_from_serialization(self): """Serialize a node and assert _lock is not in the output.""" node = _make_model_node(compiled=True) node.extra_ctes_injected = True dct = node.to_dict() assert "_lock" not in dct def test_lock_restored_after_deserialization(self): """Round-trip serialize/deserialize, assert _lock is a working Lock.""" node = _make_model_node(compiled=True) node.extra_ctes_injected = True dct = node.to_dict() restored = ModelNode.from_dict(dct) assert hasattr(restored, "_lock") assert isinstance(restored._lock, type(threading.Lock())) # Verify the lock actually works assert restored._lock.acquire(timeout=1) restored._lock.release() def test_lock_survives_pickle_roundtrip(self): """Pickle/unpickle a node, assert _lock is a working Lock.""" import pickle node = _make_model_node(compiled=True) node.extra_ctes_injected = True data = pickle.dumps(node) restored = pickle.loads(data) assert hasattr(restored, "_lock") assert isinstance(restored._lock, type(threading.Lock())) assert restored._lock.acquire(timeout=1) restored._lock.release() ================================================ FILE: tests/unit/test_deprecations.py ================================================ import pytest import dbt.deprecations as deprecations from dbt.events.types import ProjectFlagsMovedDeprecation from dbt_common.events.event_catcher import EventCatcher from dbt_common.events.event_manager_client import add_callback_to_manager from dbt_common.events.types import Note @pytest.fixture(scope="function") def active_deprecations(): deprecations.reset_deprecations() assert len(deprecations.active_deprecations) == 0 yield deprecations.active_deprecations deprecations.reset_deprecations() @pytest.fixture(scope="function") def buffered_deprecations(): deprecations.buffered_deprecations.clear() assert not deprecations.buffered_deprecations yield deprecations.buffered_deprecations deprecations.buffered_deprecations.clear() def test_buffer_deprecation(active_deprecations, buffered_deprecations): deprecations.buffer("project-flags-moved") assert len(active_deprecations) == 0 assert len(buffered_deprecations) == 1 def test_fire_buffered_deprecations(active_deprecations, buffered_deprecations): deprecations.buffer("project-flags-moved") deprecations.fire_buffered_deprecations() assert "project-flags-moved" in active_deprecations assert len(buffered_deprecations) == 0 def test_can_reset_active_deprecations(): deprecations.warn("project-flags-moved") assert "project-flags-moved" in deprecations.active_deprecations deprecations.reset_deprecations() assert "project-flags-moved" not in deprecations.active_deprecations def test_number_of_occurances_is_tracked(): assert "project-flags-moved" not in deprecations.active_deprecations deprecations.warn("project-flags-moved") assert "project-flags-moved" in deprecations.active_deprecations assert deprecations.active_deprecations["project-flags-moved"] == 1 deprecations.warn("project-flags-moved") assert "project-flags-moved" in deprecations.active_deprecations assert deprecations.active_deprecations["project-flags-moved"] == 2 class PreviewedDeprecation(deprecations.DBTDeprecation): _name = "previewed-deprecation" _event = "ProjectFlagsMovedDeprecation" _is_preview = True class TestPreviewDeprecation: @pytest.fixture(scope="class", autouse=True) def deprecations_list_and_deprecations(self): deprecations.deprecations_list.append(PreviewedDeprecation()) deprecations.deprecations["previewed-deprecation"] = PreviewedDeprecation() yield for dep in deprecations.deprecations_list: if dep._name == "previewed-deprecation": deprecations.deprecations_list.remove(dep) break deprecations.deprecations.pop("previewed-deprecation") def test_preview_deprecation(self): pfmd_catcher = EventCatcher(event_to_catch=ProjectFlagsMovedDeprecation) add_callback_to_manager(pfmd_catcher.catch) note_catcher = EventCatcher(event_to_catch=Note) add_callback_to_manager(note_catcher.catch) deprecations.warn( "previewed-deprecation", ) assert "previewed-deprecation" not in deprecations.active_deprecations assert len(pfmd_catcher.caught_events) == 0 assert len(note_catcher.caught_events) == 1 ================================================ FILE: tests/unit/test_env_vars.py ================================================ import os from unittest import mock from dbt.deprecations import EnvironmentVariableNamespaceDeprecation as EVND from dbt.deprecations import active_deprecations from dbt.env_vars import KNOWN_ENGINE_ENV_VARS, validate_engine_env_vars from dbt.events.types import EnvironmentVariableNamespaceDeprecation from dbt.tests.util import safe_set_invocation_context from dbt_common.events.event_catcher import EventCatcher from dbt_common.events.event_manager_client import add_callback_to_manager @mock.patch.dict( os.environ, { "DBT_ENGINE_PARTIAL_PARSE": "False", "DBT_ENGINE_MY_CUSTOM_ENV_VAR_FOR_TESTING": "True", }, ) def test_validate_engine_env_vars(): safe_set_invocation_context() event_catcher = EventCatcher(event_to_catch=EnvironmentVariableNamespaceDeprecation) add_callback_to_manager(event_catcher.catch) validate_engine_env_vars() # If it's zero, then we _failed_ to notice the deprecation instance (and we should look why the custom engine env var wasn't noticed) # If it's more than one, then we're getting too many deprecation instances (and we should check what the other env vars identified were) assert active_deprecations[EVND().name] == 1 assert ( "DBT_ENGINE_MY_CUSTOM_ENV_VAR_FOR_TESTING" == event_catcher.caught_events[0].data.env_var ) def test_engine_env_vars_with_old_names_has_not_increased(): engine_env_vars_with_old_names = sum( 1 for env_var in KNOWN_ENGINE_ENV_VARS if env_var.old_name is not None ) # This failing means we either: # 1. incorrectly created a new engine environment variable without using the `DBT_ENGINE` prefix # 2. we've identified, and added, an existing but previously unknown engine env var to the _ADDITIONAL_ENGINE_ENV_VARS list. # 3. we've _removed_ an existing engine env var with an old name (unlikely) # # In the case of (1), we should correct the new engine environent variable name # In the case of (2), we should increase the number here. # In the case of (3), we should decrease the number here. assert ( engine_env_vars_with_old_names == 65 ), "We've added a new engine env var _without_ using the new naming scheme" ================================================ FILE: tests/unit/test_events.py ================================================ import logging import re from typing import TypeVar import pytest from dbt.adapters.events import types as adapter_types from dbt.adapters.events.logging import AdapterLogger from dbt.artifacts.schemas.results import RunStatus, TimingInfo from dbt.artifacts.schemas.run import RunResult from dbt.events import types as core_types from dbt.events.base_types import ( CoreBaseEvent, DebugLevel, DynamicLevel, ErrorLevel, InfoLevel, TestLevel, WarnLevel, ) from dbt.task.printer import print_run_end_messages from dbt_common.events import types from dbt_common.events.base_types import msg_from_base_event from dbt_common.events.event_manager import EventManager, TestEventManager from dbt_common.events.event_manager_client import ctx_set_event_manager from dbt_common.events.functions import msg_to_dict, msg_to_json from dbt_common.events.helpers import get_json_string_utcnow # takes in a class and finds any subclasses for it def get_all_subclasses(cls): all_subclasses = [] for subclass in cls.__subclasses__(): if subclass not in [TestLevel, DebugLevel, WarnLevel, InfoLevel, ErrorLevel, DynamicLevel]: all_subclasses.append(subclass) all_subclasses.extend(get_all_subclasses(subclass)) return set(all_subclasses) class TestAdapterLogger: # this interface is documented for adapter maintainers to plug into # so we should test that it at the very least doesn't explode. def test_basic_adapter_logging_interface(self): logger = AdapterLogger("dbt_tests") logger.debug("debug message") logger.info("info message") logger.warning("warning message") logger.error("error message") logger.exception("exception message") logger.critical("exception message") # python loggers allow deferring string formatting via this signature: def test_formatting(self): logger = AdapterLogger("dbt_tests") # tests that it doesn't throw logger.debug("hello {}", "world") # enters lower in the call stack to test that it formats correctly event = adapter_types.AdapterEventDebug( name="dbt_tests", base_msg="hello {}", args=["world"] ) assert "hello world" in event.message() # tests that it doesn't throw logger.debug("1 2 {}", "3") # enters lower in the call stack to test that it formats correctly event = adapter_types.AdapterEventDebug(name="dbt_tests", base_msg="1 2 {}", args=[3]) assert "1 2 3" in event.message() # tests that it doesn't throw logger.debug("boop{x}boop") # enters lower in the call stack to test that it formats correctly # in this case it's that we didn't attempt to replace anything since there # were no args passed after the initial message event = adapter_types.AdapterEventDebug(name="dbt_tests", base_msg="boop{x}boop", args=[]) assert "boop{x}boop" in event.message() # ensure AdapterLogger and subclasses makes all base_msg members # of type string; when someone writes logger.debug(a) where a is # any non-string object event = adapter_types.AdapterEventDebug(name="dbt_tests", base_msg=[1, 2, 3], args=[3]) assert isinstance(event.base_msg, str) event = core_types.JinjaLogDebug(msg=[1, 2, 3]) assert isinstance(event.msg, str) def test_set_adapter_dependency_log_level(self): logger = AdapterLogger("dbt_tests") package_log = logging.getLogger("test_package_log") logger.set_adapter_dependency_log_level("test_package_log", "DEBUG") package_log.debug("debug message") class TestEventCodes: # checks to see if event codes are duplicated to keep codes singluar and clear. # also checks that event codes follow correct namming convention ex. E001 def test_event_codes(self): all_concrete = get_all_subclasses(CoreBaseEvent) all_codes = set() for event_cls in all_concrete: code = event_cls.code(event_cls) # must be in the form 1 capital letter, 3 digits assert re.match("^[A-Z][0-9]{3}", code) # cannot have been used already assert ( code not in all_codes ), f"{code} is assigned more than once. Check types.py for duplicates." all_codes.add(code) sample_values = [ # N.B. Events instantiated here include the module prefix in order to # avoid having the entire list twice in the code. # A - pre-project loading core_types.MainReportVersion(version=""), core_types.MainReportArgs(args={}), core_types.MainTrackingUserState(user_state=""), core_types.MissingProfileTarget(profile_name="", target_name=""), core_types.InvalidOptionYAML(option_name="vars"), core_types.LogDbtProjectError(), core_types.LogDbtProfileError(), core_types.StarterProjectPath(dir=""), core_types.ConfigFolderDirectory(dir=""), core_types.NoSampleProfileFound(adapter=""), core_types.ProfileWrittenWithSample(name="", path=""), core_types.ProfileWrittenWithTargetTemplateYAML(name="", path=""), core_types.ProfileWrittenWithProjectTemplateYAML(name="", path=""), core_types.SettingUpProfile(), core_types.InvalidProfileTemplateYAML(), core_types.ProjectNameAlreadyExists(name=""), core_types.ProjectCreated(project_name=""), # D - Deprecations ====================== core_types.PackageRedirectDeprecation(old_name="", new_name=""), core_types.PackageInstallPathDeprecation(), core_types.ConfigSourcePathDeprecation(deprecated_path="", exp_path=""), core_types.ConfigDataPathDeprecation(deprecated_path="", exp_path=""), adapter_types.AdapterDeprecationWarning(old_name="", new_name=""), core_types.MetricAttributesRenamed(metric_name=""), core_types.ExposureNameDeprecation(exposure=""), core_types.InternalDeprecation(name="", reason="", suggested_action="", version=""), core_types.EnvironmentVariableRenamed(old_name="", new_name=""), core_types.ConfigLogPathDeprecation(deprecated_path=""), core_types.ConfigTargetPathDeprecation(deprecated_path=""), adapter_types.CollectFreshnessReturnSignature(), core_types.TestsConfigDeprecation(deprecated_path="", exp_path=""), core_types.ProjectFlagsMovedDeprecation(), core_types.SpacesInResourceNameDeprecation(unique_id="", level=""), core_types.ResourceNamesWithSpacesDeprecation( count_invalid_names=1, show_debug_hint=True, level="" ), core_types.PackageMaterializationOverrideDeprecation( package_name="my_package", materialization_name="view" ), core_types.SourceFreshnessProjectHooksNotRun(), core_types.MFTimespineWithoutYamlConfigurationDeprecation(), core_types.MFCumulativeTypeParamsDeprecation(), core_types.MicrobatchMacroOutsideOfBatchesDeprecation(), core_types.GenericJSONSchemaValidationDeprecation(violation="", key_path="", file=""), core_types.UnexpectedJinjaBlockDeprecation(msg="", file=""), core_types.DuplicateYAMLKeysDeprecation(duplicate_description="", file=""), core_types.CustomTopLevelKeyDeprecation(msg="", file=""), core_types.CustomKeyInConfigDeprecation(key="", key_path="", file=""), core_types.CustomKeyInObjectDeprecation(key="", key_path="", file=""), core_types.DeprecationsSummary(summaries=[], show_all_hint=True), core_types.CustomOutputPathInSourceFreshnessDeprecation(path=""), core_types.SourceOverrideDeprecation(file="", source_name=""), core_types.PropertyMovedToConfigDeprecation(key="", key_path="", file=""), core_types.WEOIncludeExcludeDeprecation(found_include=True, found_exclude=True), core_types.ModelParamUsageDeprecation(), core_types.EnvironmentVariableNamespaceDeprecation(env_var="", reserved_prefix=""), core_types.MissingPlusPrefixDeprecation(key="", key_path="", file=""), core_types.ArgumentsPropertyInGenericTestDeprecation(test_name=""), core_types.MissingArgumentsPropertyInGenericTestDeprecation(test_name=""), core_types.ModulesItertoolsUsageDeprecation(), core_types.DuplicateNameDistinctNodeTypesDeprecation( resource_name="", package_name="", unique_id1="", unique_id2="" ), core_types.TimeDimensionsRequireGranularityDeprecation(msg=""), core_types.GenericSemanticLayerDeprecation(msg=""), core_types.GenerateSchemaNameNullValueDeprecation(resource_unique_id=""), # E - DB Adapter ====================== adapter_types.AdapterEventDebug(), adapter_types.AdapterEventInfo(), adapter_types.AdapterEventWarning(), adapter_types.AdapterEventError(), adapter_types.AdapterRegistered(adapter_name="dbt-awesome", adapter_version="1.2.3"), adapter_types.NewConnection(conn_type="", conn_name=""), adapter_types.ConnectionReused(conn_name=""), adapter_types.ConnectionLeftOpenInCleanup(conn_name=""), adapter_types.ConnectionClosedInCleanup(conn_name=""), adapter_types.RollbackFailed(conn_name=""), adapter_types.ConnectionClosed(conn_name=""), adapter_types.ConnectionLeftOpen(conn_name=""), adapter_types.Rollback(conn_name=""), adapter_types.CacheMiss(conn_name="", database="", schema=""), adapter_types.ListRelations(database="", schema=""), adapter_types.ConnectionUsed(conn_type="", conn_name=""), adapter_types.SQLQuery(conn_name="", sql=""), adapter_types.SQLQueryStatus(status="", elapsed=0.1), adapter_types.SQLCommit(conn_name=""), adapter_types.ColTypeChange( orig_type="", new_type="", table={"database": "", "schema": "", "identifier": ""}, ), adapter_types.SchemaCreation(relation={"database": "", "schema": "", "identifier": ""}), adapter_types.SchemaDrop(relation={"database": "", "schema": "", "identifier": ""}), adapter_types.CacheAction( action="adding_relation", ref_key={"database": "", "schema": "", "identifier": ""}, ref_key_2={"database": "", "schema": "", "identifier": ""}, ), adapter_types.CacheDumpGraph(before_after="before", action="rename", dump=dict()), adapter_types.AdapterImportError(exc=""), adapter_types.PluginLoadError(exc_info=""), adapter_types.NewConnectionOpening(connection_state=""), adapter_types.CodeExecution(conn_name="", code_content=""), adapter_types.CodeExecutionStatus(status="", elapsed=0.1), adapter_types.CatalogGenerationError(exc=""), adapter_types.WriteCatalogFailure(num_exceptions=0), adapter_types.CatalogWritten(path=""), adapter_types.CannotGenerateDocs(), adapter_types.BuildingCatalog(), adapter_types.DatabaseErrorRunningHook(hook_type=""), adapter_types.HooksRunning(num_hooks=0, hook_type=""), adapter_types.FinishedRunningStats(stat_line="", execution="", execution_time=0), adapter_types.ConstraintNotEnforced(constraint="", adapter=""), adapter_types.ConstraintNotSupported(constraint="", adapter=""), # I - Project parsing ====================== core_types.InputFileDiffError(category="testing", file_id="my_file"), core_types.InvalidValueForField(field_name="test", field_value="test"), core_types.ValidationWarning(resource_type="model", field_name="access", node_name="my_macro"), core_types.ParsePerfInfoPath(path=""), core_types.PartialParsingErrorProcessingFile(file=""), core_types.PartialParsingFile(file_id=""), core_types.PartialParsingError(exc_info={}), core_types.PartialParsingSkipParsing(), core_types.UnableToPartialParse(reason="something went wrong"), core_types.StateCheckVarsHash(vars="testing", target="testing", profile="testing"), core_types.PartialParsingNotEnabled(), core_types.ParsedFileLoadFailed(path="", exc="", exc_info=""), core_types.PartialParsingEnabled(deleted=0, added=0, changed=0), core_types.PartialParsingFile(file_id=""), core_types.InvalidDisabledTargetInTestNode( resource_type_title="", unique_id="", original_file_path="", target_kind="", target_name="", target_package="", ), core_types.UnusedResourceConfigPath(unused_config_paths=[]), core_types.SeedIncreased(package_name="", name=""), core_types.SeedExceedsLimitSamePath(package_name="", name=""), core_types.SeedExceedsLimitAndPathChanged(package_name="", name=""), core_types.SeedExceedsLimitChecksumChanged(package_name="", name="", checksum_name=""), core_types.UnusedTables(unused_tables=[]), core_types.WrongResourceSchemaFile( patch_name="", resource_type="", file_path="", plural_resource_type="" ), core_types.NoNodeForYamlKey(patch_name="", yaml_key="", file_path=""), core_types.MacroNotFoundForPatch(patch_name=""), core_types.NodeNotFoundOrDisabled( original_file_path="", unique_id="", resource_type_title="", target_name="", target_kind="", target_package="", disabled="", ), core_types.JinjaLogWarning(), core_types.JinjaLogInfo(msg=""), core_types.JinjaLogDebug(msg=""), core_types.UnpinnedRefNewVersionAvailable( ref_node_name="", ref_node_package="", ref_node_version="", ref_max_version="" ), core_types.DeprecatedModel(model_name="", model_version="", deprecation_date=""), core_types.DeprecatedReference( model_name="", ref_model_name="", ref_model_package="", ref_model_deprecation_date="", ref_model_latest_version="", ), core_types.UpcomingReferenceDeprecation( model_name="", ref_model_name="", ref_model_package="", ref_model_deprecation_date="", ref_model_latest_version="", ), core_types.UnsupportedConstraintMaterialization(materialized=""), core_types.ParseInlineNodeError(exc=""), core_types.SemanticValidationFailure(msg=""), core_types.UnversionedBreakingChange( breaking_changes=[], model_name="", model_file_path="", contract_enforced_disabled=True, columns_removed=[], column_type_changes=[], enforced_column_constraint_removed=[], enforced_model_constraint_removed=[], materialization_changed=[], ), core_types.WarnStateTargetEqual(state_path=""), core_types.FreshnessConfigProblem(msg=""), core_types.SemanticValidationFailure(msg=""), core_types.MicrobatchModelNoEventTimeInputs(model_name=""), core_types.InvalidConcurrentBatchesConfig(num_models=1, adapter_type=""), core_types.InvalidMacroAnnotation(msg="", macro_file_path="", macro_unique_id=""), core_types.PackageNodeDependsOnRootProjectNode( node_name="", node_package="", root_project_unique_id="" ), # M - Deps generation ====================== core_types.GitSparseCheckoutSubdirectory(subdir=""), core_types.GitProgressCheckoutRevision(revision=""), core_types.GitProgressUpdatingExistingDependency(dir=""), core_types.GitProgressPullingNewDependency(dir=""), core_types.GitNothingToDo(sha=""), core_types.GitProgressUpdatedCheckoutRange(start_sha="", end_sha=""), core_types.GitProgressCheckedOutAt(end_sha=""), core_types.RegistryProgressGETRequest(url=""), core_types.RegistryProgressGETResponse(url="", resp_code=1234), core_types.SelectorReportInvalidSelector(valid_selectors="", spec_method="", raw_spec=""), core_types.DepsNoPackagesFound(), core_types.DepsStartPackageInstall(package_name=""), core_types.DepsInstallInfo(version_name=""), core_types.DepsUpdateAvailable(version_latest=""), core_types.DepsUpToDate(), core_types.DepsListSubdirectory(subdirectory=""), core_types.DepsNotifyUpdatesAvailable(packages=["my_pkg", "other_pkg"]), types.RetryExternalCall(attempt=0, max=0), types.RecordRetryException(exc=""), core_types.RegistryIndexProgressGETRequest(url=""), core_types.RegistryIndexProgressGETResponse(url="", resp_code=1234), core_types.RegistryResponseUnexpectedType(response=""), core_types.RegistryResponseMissingTopKeys(response=""), core_types.RegistryResponseMissingNestedKeys(response=""), core_types.RegistryResponseExtraNestedKeys(response=""), core_types.DepsSetDownloadDirectory(path=""), core_types.DepsLockUpdating(lock_filepath=""), core_types.DepsAddPackage(package_name="", version="", packages_filepath=""), core_types.DepsFoundDuplicatePackage(removed_package={}), core_types.DepsScrubbedPackageName(package_name=""), core_types.DepsUnpinned(revision="", git=""), core_types.NoNodesForSelectionCriteria(spec_raw=""), # P - Artifacts ====================== core_types.ArtifactWritten(artifact_type="manifest", artifact_path="path/to/artifact.json"), # Q - Node execution ====================== core_types.RunningOperationCaughtError(exc=""), core_types.CompileComplete(), core_types.FreshnessCheckComplete(), core_types.SeedHeader(header=""), core_types.SQLRunnerException(exc=""), core_types.LogTestResult( name="", index=0, num_models=0, execution_time=0, num_failures=0, ), core_types.LogNodeResult( description="", status="", index=0, total=0, execution_time=0, msg="", ), core_types.LogStartLine(description="", index=0, total=0), core_types.LogModelResult( description="", status="", index=0, total=0, execution_time=0, ), core_types.LogSnapshotResult( status="", description="", cfg={}, index=0, total=0, execution_time=0, ), core_types.LogSeedResult( status="", index=0, total=0, execution_time=0, schema="", relation="", ), core_types.LogFreshnessResult( source_name="", table_name="", index=0, total=0, execution_time=0, ), core_types.LogNodeNoOpResult( description="", status="", index=0, total=0, execution_time=0, ), core_types.LogCancelLine(conn_name=""), core_types.DefaultSelector(name=""), core_types.NodeStart(), core_types.NodeFinished(), core_types.QueryCancelationUnsupported(type=""), core_types.ConcurrencyLine(num_threads=0, target_name=""), core_types.WritingInjectedSQLForNode(), core_types.NodeCompiling(), core_types.NodeExecuting(), core_types.LogHookStartLine( statement="", index=0, total=0, ), core_types.LogHookEndLine( statement="", status="", index=0, total=0, execution_time=0, ), core_types.SkippingDetails( resource_type="", schema="", node_name="", index=0, total=0, ), core_types.NothingToDo(), core_types.RunningOperationUncaughtError(exc=""), core_types.EndRunResult(), core_types.NoNodesSelected(), core_types.CommandCompleted( command="", success=True, elapsed=0.1, completed_at=get_json_string_utcnow(), ), core_types.ShowNode(node_name="", preview="", is_inline=True, unique_id="model.test.my_model"), core_types.CompiledNode( node_name="", compiled="", is_inline=True, unique_id="model.test.my_model" ), core_types.SnapshotTimestampWarning( snapshot_time_data_type="DATETIME", updated_at_data_type="DATETIMEZ" ), core_types.MicrobatchExecutionDebug(msg=""), core_types.LogStartBatch(description="", batch_index=0, total_batches=0), core_types.LogBatchResult( description="", status="", batch_index=0, total_batches=0, execution_time=0, ), core_types.LogFunctionResult( description="", status="", index=0, total=0, execution_time=0, ), # W - Node testing ====================== core_types.CatchableExceptionOnRun(exc=""), core_types.InternalErrorOnRun(build_path="", exc=""), core_types.GenericExceptionOnRun(build_path="", unique_id="", exc=""), core_types.NodeConnectionReleaseError(node_name="", exc=""), core_types.FoundStats(stat_line=""), # Z - misc ====================== core_types.MainKeyboardInterrupt(), core_types.MainEncounteredError(exc=""), core_types.MainStackTrace(stack_trace=""), types.SystemCouldNotWrite(path="", reason="", exc=""), types.SystemExecutingCmd(cmd=[""]), types.SystemStdOut(bmsg=str(b"")), types.SystemStdErr(bmsg=str(b"")), types.SystemReportReturnCode(returncode=0), core_types.TimingInfoCollected(), core_types.LogDebugStackTrace(), core_types.CheckCleanPath(path=""), core_types.ConfirmCleanPath(path=""), core_types.ProtectedCleanPath(path=""), core_types.FinishedCleanPaths(), core_types.OpenCommand(open_cmd="", profiles_dir=""), core_types.RunResultWarning(resource_type="", node_name="", path=""), core_types.RunResultFailure(resource_type="", node_name="", path=""), core_types.StatsLine( stats={"error": 0, "skip": 0, "pass": 0, "warn": 0, "noop": 0, "total": 0} ), core_types.RunResultError(msg=""), core_types.RunResultErrorNoMessage(status=""), core_types.SQLCompiledPath(path=""), core_types.CheckNodeTestFailure(relation_name=""), core_types.EndOfRunSummary(num_errors=0, num_warnings=0, keyboard_interrupt=False), core_types.MarkSkippedChildren(unique_id="", status="skipped"), core_types.LogSkipBecauseError(schema="", relation="", index=0, total=0), core_types.EnsureGitInstalled(), core_types.DepsCreatingLocalSymlink(), core_types.DepsSymlinkNotAvailable(), core_types.DisableTracking(), core_types.SendingEvent(kwargs=""), core_types.SendEventFailure(), core_types.FlushEvents(), core_types.FlushEventsFailure(), types.Formatting(), core_types.TrackingInitializeFailure(), core_types.RunResultWarningMessage(), core_types.DebugCmdOut(), core_types.DebugCmdResult(), core_types.ListCmdOut(), types.Note(msg="This is a note."), core_types.ResourceReport(), core_types.ArtifactUploadSuccess(), core_types.ArtifactUploadError(), core_types.ArtifactUploadSkipped(), core_types.SelectExcludeIgnoredWithSelectorWarning(), ] class TestEventJSONSerialization: # attempts to test that every event is serializable to json. # event types that take `Any` are not possible to test in this way since some will serialize # just fine and others won't. def test_all_serializable(self): all_non_abstract_events = set( get_all_subclasses(CoreBaseEvent), ) all_event_values_list = list(map(lambda x: x.__class__, sample_values)) diff = all_non_abstract_events.difference(set(all_event_values_list)) assert ( not diff ), f"{diff}test is missing concrete values in `sample_values`. Please add the values for the aforementioned event classes" # make sure everything in the list is a value not a type for event in sample_values: assert type(event) != type # if we have everything we need to test, try to serialize everything count = 0 for event in sample_values: msg = msg_from_base_event(event) print(f"--- msg: {msg.info.name}") # Serialize to dictionary try: msg_to_dict(msg) except Exception as e: raise Exception( f"{event} can not be converted to a dict. Originating exception: {e}" ) # Serialize to json try: msg_to_json(msg) except Exception as e: raise Exception(f"{event} is not serializable to json. Originating exception: {e}") # Serialize to binary try: msg.SerializeToString() except Exception as e: raise Exception( f"{event} is not serializable to binary protobuf. Originating exception: {e}" ) count += 1 print(f"--- Found {count} events") T = TypeVar("T") def test_date_serialization(): ti = TimingInfo("compile") ti.begin() ti.end() ti_dict = ti.to_dict() assert ti_dict["started_at"].endswith("Z") assert ti_dict["completed_at"].endswith("Z") def test_bad_serialization(): """Tests that bad serialization enters the proper exception handling When pytest is in use the exception handling of `BaseEvent` raises an exception. When pytest isn't present, it fires a Note event. Thus to test that bad serializations are properly handled, the best we can do is test that the exception handling path is used. """ with pytest.raises(Exception) as excinfo: types.Note(param_event_doesnt_have="This should break") assert 'has no field named "param_event_doesnt_have" at "Note"' in str(excinfo.value) def test_single_run_error(): try: # Add a recording event manager to the context, so we can test events. event_mgr = TestEventManager() ctx_set_event_manager(event_mgr) class MockNode: unique_id: str = "" node_info = None resource_type: str = "model" name: str = "my_model" original_file_path: str = "path/to/model.sql" error_result = RunResult( status=RunStatus.Error, timing=[], thread_id="", execution_time=0.0, node=MockNode(), adapter_response=dict(), message="oh no!", failures=1, batch_results=None, ) results = [error_result] print_run_end_messages(results) summary_event = [ e for e in event_mgr.event_history if isinstance(e[0], core_types.EndOfRunSummary) ] run_result_error_events = [ e for e in event_mgr.event_history if isinstance(e[0], core_types.RunResultError) ] # expect correct plural assert "partial successes" in summary_event[0][0].message() # expect one error to show up assert len(run_result_error_events) == 1 assert run_result_error_events[0][0].msg == "oh no!" finally: # Set an empty event manager unconditionally on exit. This is an early # attempt at unit testing events, and we need to think about how it # could be done in a thread safe way in the long run. ctx_set_event_manager(EventManager()) ================================================ FILE: tests/unit/test_functions.py ================================================ from argparse import Namespace import pytest import dbt.flags as flags from dbt.adapters.events.types import AdapterDeprecationWarning from dbt.events.types import NoNodesForSelectionCriteria from dbt_common.events.functions import msg_to_dict, warn_or_error from dbt_common.events.types import InfoLevel, RetryExternalCall from dbt_common.exceptions import EventCompilationError @pytest.mark.parametrize( "warn_error_options,expect_compilation_exception", [ ({"error": "all"}, True), ({"error": ["NoNodesForSelectionCriteria"]}, True), ({"error": []}, False), ({}, False), ({"error": ["MainTrackingUserState"]}, False), ({"error": "all", "warn": ["NoNodesForSelectionCriteria"]}, False), ], ) def test_warn_or_error_warn_error_options(warn_error_options, expect_compilation_exception): args = Namespace(warn_error_options=warn_error_options) flags.set_from_args(args, {}) if expect_compilation_exception: with pytest.raises(EventCompilationError): warn_or_error(NoNodesForSelectionCriteria()) else: warn_or_error(NoNodesForSelectionCriteria()) @pytest.mark.parametrize( "error_cls", [ NoNodesForSelectionCriteria, # core event AdapterDeprecationWarning, # adapter event RetryExternalCall, # common event ], ) def test_warn_error_options_captures_all_events(error_cls): args = Namespace(warn_error_options={"error": [error_cls.__name__]}) flags.set_from_args(args, {}) with pytest.raises(EventCompilationError): warn_or_error(error_cls()) args = Namespace(warn_error_options={"error": "*", "warn": [error_cls.__name__]}) flags.set_from_args(args, {}) warn_or_error(error_cls()) @pytest.mark.parametrize( "warn_error,expect_compilation_exception", [ (True, True), (False, False), ], ) def test_warn_or_error_warn_error(warn_error, expect_compilation_exception): args = Namespace(warn_error=warn_error) flags.set_from_args(args, {}) if expect_compilation_exception: with pytest.raises(EventCompilationError): warn_or_error(NoNodesForSelectionCriteria()) else: warn_or_error(NoNodesForSelectionCriteria()) def test_msg_to_dict_handles_exceptions_gracefully(): class BadEvent(InfoLevel): """A spoof Note event which breaks dictification""" def __init__(self): self.__class__.__name__ = "Note" event = BadEvent() try: msg_to_dict(event) except Exception as exc: assert ( False ), f"We expect `msg_to_dict` to gracefully handle exceptions, but it raised {exc}" ================================================ FILE: tests/unit/test_graph_selection.py ================================================ import string from unittest import mock import networkx as nx import pytest import dbt.graph.cli as graph_cli import dbt.graph.selector as graph_selector import dbt_common.exceptions from dbt.node_types import NodeType def _get_graph(): integer_graph = nx.balanced_tree(2, 2, nx.DiGraph()) package_mapping = { i: "m." + ("X" if i % 2 == 0 else "Y") + "." + letter for (i, letter) in enumerate(string.ascii_lowercase) } # Edges: [(X.a, Y.b), (X.a, X.c), (Y.b, Y.d), (Y.b, X.e), (X.c, Y.f), (X.c, X.g)] return graph_selector.Graph(nx.relabel_nodes(integer_graph, package_mapping)) def _get_manifest(graph): nodes = {} for unique_id in graph: fqn = unique_id.split(".") node = mock.MagicMock( unique_id=unique_id, fqn=fqn, package_name=fqn[0], tags=[], resource_type=NodeType.Model, empty=False, config=mock.MagicMock(enabled=True), is_versioned=False, ) nodes[unique_id] = node nodes["m.X.a"].tags = ["abc"] nodes["m.Y.b"].tags = ["abc", "bcef"] nodes["m.X.c"].tags = ["abc", "bcef"] nodes["m.Y.d"].tags = [] nodes["m.X.e"].tags = ["efg", "bcef"] nodes["m.Y.f"].tags = ["efg", "bcef"] nodes["m.X.g"].tags = ["efg"] return mock.MagicMock(nodes=nodes) @pytest.fixture def graph(): return _get_graph() @pytest.fixture def manifest(graph): return _get_manifest(graph) def id_macro(arg): if isinstance(arg, str): return arg try: return "_".join(arg) except TypeError: return arg run_specs = [ # include by fqn (["X.a"], [], {"m.X.a"}), # include by tag (["tag:abc"], [], {"m.X.a", "m.Y.b", "m.X.c"}), # exclude by tag (["*"], ["tag:abc"], {"m.Y.d", "m.X.e", "m.Y.f", "m.X.g"}), # tag + fqn (["tag:abc", "a"], [], {"m.X.a", "m.Y.b", "m.X.c"}), (["tag:abc", "d"], [], {"m.X.a", "m.Y.b", "m.X.c", "m.Y.d"}), # multiple node selection across packages (["X.a", "b"], [], {"m.X.a", "m.Y.b"}), (["X.a+"], ["b"], {"m.X.a", "m.X.c", "m.Y.d", "m.X.e", "m.Y.f", "m.X.g"}), # children (["X.c+"], [], {"m.X.c", "m.Y.f", "m.X.g"}), (["X.a+1"], [], {"m.X.a", "m.Y.b", "m.X.c"}), (["X.a+"], ["tag:efg"], {"m.X.a", "m.Y.b", "m.X.c", "m.Y.d"}), # parents (["+Y.f"], [], {"m.X.c", "m.Y.f", "m.X.a"}), (["1+Y.f"], [], {"m.X.c", "m.Y.f"}), # childrens parents (["@X.c"], [], {"m.X.a", "m.X.c", "m.Y.f", "m.X.g"}), # multiple selection/exclusion (["tag:abc", "tag:bcef"], [], {"m.X.a", "m.Y.b", "m.X.c", "m.X.e", "m.Y.f"}), (["tag:abc", "tag:bcef"], ["tag:efg"], {"m.X.a", "m.Y.b", "m.X.c"}), (["tag:abc", "tag:bcef"], ["tag:efg", "a"], {"m.Y.b", "m.X.c"}), # intersections (["a,a"], [], {"m.X.a"}), (["+c,c+"], [], {"m.X.c"}), (["a,b"], [], set()), (["tag:abc,tag:bcef"], [], {"m.Y.b", "m.X.c"}), (["*,tag:abc,a"], [], {"m.X.a"}), (["a,tag:abc,*"], [], {"m.X.a"}), (["tag:abc,tag:bcef"], ["c"], {"m.Y.b"}), (["tag:bcef,tag:efg"], ["tag:bcef,@b"], {"m.Y.f"}), (["tag:bcef,tag:efg"], ["tag:bcef,@a"], set()), (["*,@a,+b"], ["*,tag:abc,tag:bcef"], {"m.X.a"}), (["tag:bcef,tag:efg", "*,tag:abc"], [], {"m.X.a", "m.Y.b", "m.X.c", "m.X.e", "m.Y.f"}), (["tag:bcef,tag:efg", "*,tag:abc"], ["e"], {"m.X.a", "m.Y.b", "m.X.c", "m.Y.f"}), (["tag:bcef,tag:efg", "*,tag:abc"], ["e"], {"m.X.a", "m.Y.b", "m.X.c", "m.Y.f"}), (["tag:bcef,tag:efg", "*,tag:abc"], ["e", "f"], {"m.X.a", "m.Y.b", "m.X.c"}), (["tag:bcef,tag:efg", "*,tag:abc"], ["tag:abc,tag:bcef"], {"m.X.a", "m.X.e", "m.Y.f"}), (["tag:bcef,tag:efg", "*,tag:abc"], ["tag:abc,tag:bcef", "tag:abc,a"], {"m.X.e", "m.Y.f"}), ] @pytest.mark.parametrize("include,exclude,expected", run_specs, ids=id_macro) def test_run_specs(include, exclude, expected, graph, manifest): selector = graph_selector.NodeSelector(graph, manifest) spec = graph_cli.parse_difference(include, exclude) selected, _ = selector.select_nodes(spec) assert selected == expected param_specs = [ ("a", False, None, False, None, "fqn", "a", False), ("+a", True, None, False, None, "fqn", "a", False), ("256+a", True, 256, False, None, "fqn", "a", False), ("a+", False, None, True, None, "fqn", "a", False), ("a+256", False, None, True, 256, "fqn", "a", False), ("+a+", True, None, True, None, "fqn", "a", False), ("16+a+32", True, 16, True, 32, "fqn", "a", False), ("@a", False, None, False, None, "fqn", "a", True), ("a.b", False, None, False, None, "fqn", "a.b", False), ("+a.b", True, None, False, None, "fqn", "a.b", False), ("256+a.b", True, 256, False, None, "fqn", "a.b", False), ("a.b+", False, None, True, None, "fqn", "a.b", False), ("a.b+256", False, None, True, 256, "fqn", "a.b", False), ("+a.b+", True, None, True, None, "fqn", "a.b", False), ("16+a.b+32", True, 16, True, 32, "fqn", "a.b", False), ("@a.b", False, None, False, None, "fqn", "a.b", True), ("a.b.*", False, None, False, None, "fqn", "a.b.*", False), ("+a.b.*", True, None, False, None, "fqn", "a.b.*", False), ("256+a.b.*", True, 256, False, None, "fqn", "a.b.*", False), ("a.b.*+", False, None, True, None, "fqn", "a.b.*", False), ("a.b.*+256", False, None, True, 256, "fqn", "a.b.*", False), ("+a.b.*+", True, None, True, None, "fqn", "a.b.*", False), ("16+a.b.*+32", True, 16, True, 32, "fqn", "a.b.*", False), ("@a.b.*", False, None, False, None, "fqn", "a.b.*", True), ("tag:a", False, None, False, None, "tag", "a", False), ("+tag:a", True, None, False, None, "tag", "a", False), ("256+tag:a", True, 256, False, None, "tag", "a", False), ("tag:a+", False, None, True, None, "tag", "a", False), ("tag:a+256", False, None, True, 256, "tag", "a", False), ("+tag:a+", True, None, True, None, "tag", "a", False), ("16+tag:a+32", True, 16, True, 32, "tag", "a", False), ("@tag:a", False, None, False, None, "tag", "a", True), ("source:a", False, None, False, None, "source", "a", False), ("source:a+", False, None, True, None, "source", "a", False), ("source:a+1", False, None, True, 1, "source", "a", False), ("source:a+32", False, None, True, 32, "source", "a", False), ("@source:a", False, None, False, None, "source", "a", True), ] @pytest.mark.parametrize( "spec,parents,parents_depth,children,children_depth,filter_type,filter_value,childrens_parents", param_specs, ids=id_macro, ) def test_parse_specs( spec, parents, parents_depth, children, children_depth, filter_type, filter_value, childrens_parents, ): parsed = graph_selector.SelectionCriteria.from_single_spec(spec) assert parsed.parents == parents assert parsed.parents_depth == parents_depth assert parsed.children == children assert parsed.children_depth == children_depth assert parsed.method == filter_type assert parsed.value == filter_value assert parsed.childrens_parents == childrens_parents invalid_specs = [ "@a+", "@a.b+", "@a.b*+", "@tag:a+", "@source:a+", ] @pytest.mark.parametrize("invalid", invalid_specs, ids=lambda k: str(k)) def test_invalid_specs(invalid): with pytest.raises(dbt_common.exceptions.DbtRuntimeError): graph_selector.SelectionCriteria.from_single_spec(invalid) ================================================ FILE: tests/unit/test_internal_deprecations.py ================================================ from dbt.internal_deprecations import deprecated @deprecated(reason="just because", version="1.23.0", suggested_action="Make some updates") def to_be_decorated(): return 5 # simple test that the return value is not modified def test_deprecated_func(): assert hasattr(to_be_decorated, "__wrapped__") assert to_be_decorated() == 5 ================================================ FILE: tests/unit/test_invocation_id.py ================================================ from dbt.cli.main import dbtRunner class TestInvocationId: def test_invocation_id(self): """Runs dbt programmatically twice, checking that invocation_id is consistent within an invocation, but changes for the second invocation.""" runner = dbtRunner() # Run once... first_ids = set() runner.callbacks = [lambda e: first_ids.add(e.info.invocation_id)] runner.invoke(["debug"]) # ...run twice... second_ids = set() runner.callbacks = [lambda e: second_ids.add(e.info.invocation_id)] runner.invoke(["debug"]) # ...check that the results were nice. assert len(first_ids) == 1 # There was one consistent invocation_id for first run... assert len(second_ids) == 1 # ...as well as the second... assert len(first_ids.intersection(second_ids)) == 0 ================================================ FILE: tests/unit/test_jsonschemas.py ================================================ import pytest from dbt.deprecations import ( CustomKeyInConfigDeprecation, CustomKeyInObjectDeprecation, GenericJSONSchemaValidationDeprecation, active_deprecations, reset_deprecations, ) from dbt.jsonschemas.jsonschemas import ( jsonschema_validate, resources_schema, validate_model_config, ) from dbt.tests.util import safe_set_invocation_context from dbt_common.context import get_invocation_context from dbt_common.events.event_catcher import EventCatcher from dbt_common.events.event_manager_client import add_callback_to_manager class TestValidateModelConfigNoError: def test_validate_model_config_no_error(self): safe_set_invocation_context() get_invocation_context().uses_adapter("snowflake") caught_events = [] add_callback_to_manager(caught_events.append) config = { "enabled": True, } validate_model_config(config, "test.yml") assert len(caught_events) == 0 def test_validate_model_config_error(self): safe_set_invocation_context() get_invocation_context().uses_adapter("snowflake") ckiod_catcher = EventCatcher(CustomKeyInObjectDeprecation) ckicd_catcher = EventCatcher(CustomKeyInConfigDeprecation) gjsvd_catcher = EventCatcher(GenericJSONSchemaValidationDeprecation) add_callback_to_manager(ckiod_catcher.catch) add_callback_to_manager(ckicd_catcher.catch) add_callback_to_manager(gjsvd_catcher.catch) config = { "non_existent_config": True, # this config key doesn't exist "docs": { "show": True, # this is a valid config key "color": "red", # this is an invalid config key, as it should be `node_color` }, } validate_model_config(config, "test.yml") assert len(ckiod_catcher.caught_events) == 1 assert ckiod_catcher.caught_events[0].data.key == "color" assert len(ckicd_catcher.caught_events) == 1 assert ckicd_catcher.caught_events[0].data.key == "non_existent_config" assert len(gjsvd_catcher.caught_events) == 0 class TestValidateJsonSchema: @pytest.fixture(scope="class") def model_bigquery_alias_config_contents(self): return { "models": [ { "name": "model_1", "config": { "dataset": "dataset_1", "project": "project_1", }, } ], } def test_validate_json_schema_no_error_aliases(self, model_bigquery_alias_config_contents): reset_deprecations() safe_set_invocation_context() get_invocation_context().uses_adapter("bigquery") jsonschema_validate(resources_schema(), model_bigquery_alias_config_contents, "test.yml") assert active_deprecations == {} def test_validate_json_schema_has_error_aliases(self, model_bigquery_alias_config_contents): reset_deprecations() safe_set_invocation_context() # Set to adapter that doesn't support aliases specified get_invocation_context().uses_adapter("snowflake") jsonschema_validate(resources_schema(), model_bigquery_alias_config_contents, "test.yml") assert active_deprecations == {"custom-key-in-config-deprecation": 2} class TestSourceBigQueryAliases: @pytest.fixture(scope="class") def source_with_dataset(self): return { "sources": [ { "name": "my_source", "dataset": "my_dataset", "tables": [{"name": "my_table"}], } ] } @pytest.fixture(scope="class") def source_with_project(self): return { "sources": [ { "name": "my_source", "project": "my-gcp-project", "tables": [{"name": "my_table"}], } ] } @pytest.fixture(scope="class") def source_with_both_aliases(self): return { "sources": [ { "name": "my_source", "project": "my-gcp-project", "dataset": "my_dataset", "tables": [{"name": "my_table"}], } ] } def test_bigquery_source_dataset_no_warning(self, source_with_dataset): reset_deprecations() safe_set_invocation_context() get_invocation_context().uses_adapter("bigquery") jsonschema_validate(resources_schema(), source_with_dataset, "test.yml") assert active_deprecations == {} def test_bigquery_source_project_no_warning(self, source_with_project): reset_deprecations() safe_set_invocation_context() get_invocation_context().uses_adapter("bigquery") jsonschema_validate(resources_schema(), source_with_project, "test.yml") assert active_deprecations == {} def test_bigquery_source_both_aliases_no_warning(self, source_with_both_aliases): reset_deprecations() safe_set_invocation_context() get_invocation_context().uses_adapter("bigquery") jsonschema_validate(resources_schema(), source_with_both_aliases, "test.yml") assert active_deprecations == {} def test_snowflake_source_dataset_warns(self, source_with_dataset): reset_deprecations() safe_set_invocation_context() get_invocation_context().uses_adapter("snowflake") jsonschema_validate(resources_schema(), source_with_dataset, "test.yml") assert active_deprecations == {"custom-key-in-object-deprecation": 1} ================================================ FILE: tests/unit/test_macro_types.py ================================================ from dbt.parser.schemas import is_valid_type def test_valid_names() -> None: valid_names = [ "str", "string", "bool", "int", "integer", "float", "any", "list[int]", "dict[str, int]", "optional[any]", "relation", "column", "list[dict[str, any]]", "dict[str, list[int]]", ] for name in valid_names: assert is_valid_type(name) def test_invalid_names() -> None: invalid_names = [ "strang", # Not a valid name "int int", # Repeat not allowed "intint", # No repeat, even with no space "list[dict[any]]", # dict needs two args "dict[str, list[int]]]", # Can't have extra closing brace "dict[str, list[[int]]", # Can't have extra opening brace "dict[str,]", # Can't have blank nexted type "dict[str,,str]", # Can't have extra comma ] for name in invalid_names: assert not is_valid_type(name) ================================================ FILE: tests/unit/test_node_types.py ================================================ import pytest from dbt.node_types import NodeType node_type_pluralizations = { NodeType.Model: "models", NodeType.Analysis: "analyses", NodeType.Test: "data_tests", NodeType.Snapshot: "snapshots", NodeType.Operation: "operations", NodeType.Seed: "seeds", NodeType.RPCCall: "rpcs", NodeType.SqlOperation: "sql_operations", NodeType.Documentation: "docs", NodeType.Source: "sources", NodeType.Macro: "macros", NodeType.Exposure: "exposures", NodeType.Metric: "metrics", NodeType.Group: "groups", NodeType.SemanticModel: "semantic_models", NodeType.Unit: "unit_tests", NodeType.SavedQuery: "saved_queries", NodeType.Fixture: "fixtures", NodeType.Function: "functions", } def test_all_types_have_pluralizations(): assert set(NodeType) == set(node_type_pluralizations) @pytest.mark.parametrize("node_type, pluralization", node_type_pluralizations.items()) def test_pluralizes_correctly(node_type, pluralization): assert node_type.pluralize() == pluralization ================================================ FILE: tests/unit/test_semantic_layer_nodes_satisfy_protocols.py ================================================ import copy from typing import Protocol, runtime_checkable import pytest from hypothesis import HealthCheck, given, settings from hypothesis.strategies import builds, none, text from dbt.artifacts.resources import ( ConstantPropertyInput, ConversionTypeParams, CumulativeTypeParams, Defaults, Dimension, DimensionTypeParams, DimensionValidityParams, Entity, FileSlice, Measure, MeasureAggregationParameters, MetricInput, MetricInputMeasure, MetricTimeWindow, MetricTypeParams, NodeRelation, NonAdditiveDimension, SourceFileMetadata, WhereFilter, ) from dbt.contracts.graph.nodes import Metric, SavedQuery, SemanticModel from dbt.node_types import NodeType from dbt_semantic_interfaces.protocols import WhereFilter as WhereFilterProtocol from dbt_semantic_interfaces.protocols import dimension as DimensionProtocols from dbt_semantic_interfaces.protocols import entity as EntityProtocols from dbt_semantic_interfaces.protocols import measure as MeasureProtocols from dbt_semantic_interfaces.protocols import metadata as MetadataProtocols from dbt_semantic_interfaces.protocols import metric as MetricProtocols from dbt_semantic_interfaces.protocols import saved_query as SavedQueryProtocols from dbt_semantic_interfaces.protocols import semantic_model as SemanticModelProtocols from dbt_semantic_interfaces.type_enums import ( AggregationType, DimensionType, EntityType, MetricType, TimeGranularity, ) @runtime_checkable class RuntimeCheckableSemanticModel(SemanticModelProtocols.SemanticModel, Protocol): pass @runtime_checkable class RuntimeCheckableDimension(DimensionProtocols.Dimension, Protocol): pass @runtime_checkable class RuntimeCheckableEntity(EntityProtocols.Entity, Protocol): pass @runtime_checkable class RuntimeCheckableMeasure(MeasureProtocols.Measure, Protocol): pass @runtime_checkable class RuntimeCheckableMetric(MetricProtocols.Metric, Protocol): pass @runtime_checkable class RuntimeCheckableMetricInput(MetricProtocols.MetricInput, Protocol): pass @runtime_checkable class RuntimeCheckableMetricInputMeasure(MetricProtocols.MetricInputMeasure, Protocol): pass @runtime_checkable class RuntimeCheckableMetricTypeParams(MetricProtocols.MetricTypeParams, Protocol): pass @runtime_checkable class RuntimeCheckableWhereFilter(WhereFilterProtocol, Protocol): pass @runtime_checkable class RuntimeCheckableNonAdditiveDimension( MeasureProtocols.NonAdditiveDimensionParameters, Protocol ): pass @runtime_checkable class RuntimeCheckableFileSlice(MetadataProtocols.FileSlice, Protocol): pass @runtime_checkable class RuntimeCheckableSourceFileMetadata(MetadataProtocols.Metadata, Protocol): pass @runtime_checkable class RuntimeCheckableSemanticModelDefaults( SemanticModelProtocols.SemanticModelDefaults, Protocol ): pass @runtime_checkable class RuntimeCheckableDimensionValidityParams( DimensionProtocols.DimensionValidityParams, Protocol ): pass @runtime_checkable class RuntimeCheckableDimensionTypeParams(DimensionProtocols.DimensionTypeParams, Protocol): pass @runtime_checkable class RuntimeCheckableMeasureAggregationParams( MeasureProtocols.MeasureAggregationParameters, Protocol ): pass @runtime_checkable class RuntimeCheckableMetricTimeWindow(MetricProtocols.MetricTimeWindow, Protocol): pass @runtime_checkable class RuntimeCheckableSavedQuery(SavedQueryProtocols.SavedQuery, Protocol): pass @pytest.fixture(scope="session") def file_slice() -> FileSlice: return FileSlice( filename="test_filename", content="test content", start_line_number=0, end_line_number=1 ) @pytest.fixture(scope="session") def source_file_metadata(file_slice) -> SourceFileMetadata: return SourceFileMetadata( repo_file_path="test/file/path.yml", file_slice=file_slice, ) @pytest.fixture(scope="session") def semantic_model_defaults() -> Defaults: return Defaults(agg_time_dimension="test_time_dimension") @pytest.fixture(scope="session") def dimension_validity_params() -> DimensionValidityParams: return DimensionValidityParams() @pytest.fixture(scope="session") def dimension_type_params() -> DimensionTypeParams: return DimensionTypeParams(time_granularity=TimeGranularity.DAY) @pytest.fixture(scope="session") def measure_agg_params() -> MeasureAggregationParameters: return MeasureAggregationParameters() @pytest.fixture(scope="session") def non_additive_dimension() -> NonAdditiveDimension: return NonAdditiveDimension( name="dimension_name", window_choice=AggregationType.MIN, window_groupings=["entity_name"], ) @pytest.fixture(scope="session") def where_filter() -> WhereFilter: return WhereFilter( where_sql_template="{{ Dimension('enity_name__dimension_name') }} AND {{ TimeDimension('entity_name__time_dimension_name', 'month') }} AND {{ Entity('entity_name') }}" ) @pytest.fixture(scope="session") def metric_time_window() -> MetricTimeWindow: return MetricTimeWindow(count=1, granularity=TimeGranularity.DAY.value) @pytest.fixture(scope="session") def simple_metric_input() -> MetricInput: return MetricInput(name="test_simple_metric_input") @pytest.fixture(scope="session") def complex_metric_input(metric_time_window, where_filter) -> MetricInput: return MetricInput( name="test_complex_metric_input", filter=where_filter, alias="aliased_metric_input", offset_window=metric_time_window, offset_to_grain=TimeGranularity.DAY.value, ) @pytest.fixture(scope="session") def simple_metric_input_measure() -> MetricInputMeasure: return MetricInputMeasure(name="test_simple_metric_input_measure") @pytest.fixture(scope="session") def complex_metric_input_measure(where_filter) -> MetricInputMeasure: return MetricInputMeasure( name="test_complex_metric_input_measure", filter=where_filter, alias="complex_alias", join_to_timespine=True, fill_nulls_with=0, ) @pytest.fixture(scope="session") def conversion_type_params( simple_metric_input_measure, metric_time_window ) -> ConversionTypeParams: return ConversionTypeParams( base_measure=simple_metric_input_measure, conversion_measure=simple_metric_input_measure, entity="entity", window=metric_time_window, constant_properties=[ ConstantPropertyInput(base_property="base", conversion_property="conversion") ], ) @pytest.fixture(scope="session") def cumulative_type_params() -> CumulativeTypeParams: return CumulativeTypeParams() @pytest.fixture(scope="session") def complex_metric_type_params( metric_time_window, simple_metric_input, simple_metric_input_measure, conversion_type_params, cumulative_type_params, ) -> MetricTypeParams: return MetricTypeParams( measure=simple_metric_input_measure, numerator=simple_metric_input, denominator=simple_metric_input, expr="1 = 1", window=metric_time_window, grain_to_date=TimeGranularity.DAY, metrics=[simple_metric_input], conversion_type_params=conversion_type_params, cumulative_type_params=cumulative_type_params, ) def test_file_slice_obj_satisfies_protocol(file_slice): assert isinstance(file_slice, RuntimeCheckableFileSlice) def test_metadata_obj_satisfies_protocol(source_file_metadata): assert isinstance(source_file_metadata, RuntimeCheckableSourceFileMetadata) def test_defaults_obj_satisfies_protocol(semantic_model_defaults): assert isinstance(semantic_model_defaults, RuntimeCheckableSemanticModelDefaults) assert isinstance(Defaults(), RuntimeCheckableSemanticModelDefaults) def test_dimension_validity_params_satisfies_protocol(dimension_validity_params): assert isinstance(dimension_validity_params, RuntimeCheckableDimensionValidityParams) def test_dimension_type_params_satisfies_protocol( dimension_type_params, dimension_validity_params ): assert isinstance(dimension_type_params, RuntimeCheckableDimensionTypeParams) # check with validity params specified optionals_specified_type_params = copy.deepcopy(dimension_type_params) optionals_specified_type_params.validity_params = dimension_validity_params assert isinstance(optionals_specified_type_params, RuntimeCheckableDimensionTypeParams) def test_measure_aggregation_params_satisfies_protocol(measure_agg_params): assert isinstance(measure_agg_params, RuntimeCheckableMeasureAggregationParams) # check with optionals specified optionals_specified_measure_agg_params = copy.deepcopy(measure_agg_params) optionals_specified_measure_agg_params.percentile = 0.5 assert isinstance( optionals_specified_measure_agg_params, RuntimeCheckableMeasureAggregationParams ) def test_semantic_model_node_satisfies_protocol_optionals_unspecified(): test_semantic_model = SemanticModel( name="test_semantic_model", resource_type=NodeType.SemanticModel, package_name="package_name", path="path.to.semantic_model", original_file_path="path/to/file", unique_id="not_like_the_other_semantic_models", fqn=["fully", "qualified", "name"], model="ref('a_model')", # Technically NodeRelation is optional on our SemanticModel implementation # however, it's functionally always loaded, it's just delayed. # This will type/state mismatch will likely bite us at some point node_relation=NodeRelation( alias="test_alias", schema_name="test_schema_name", ), ) assert isinstance(test_semantic_model, RuntimeCheckableSemanticModel) def test_semantic_model_node_satisfies_protocol_optionals_specified( semantic_model_defaults, source_file_metadata ): test_semantic_model = SemanticModel( name="test_semantic_model", resource_type=NodeType.SemanticModel, package_name="package_name", path="path.to.semantic_model", original_file_path="path/to/file", unique_id="not_like_the_other_semantic_models", fqn=["fully", "qualified", "name"], model="ref('a_model')", node_relation=NodeRelation( alias="test_alias", schema_name="test_schema_name", ), description="test_description", label="test label", defaults=semantic_model_defaults, metadata=source_file_metadata, primary_entity="test_primary_entity", ) assert isinstance(test_semantic_model, RuntimeCheckableSemanticModel) def test_dimension_satisfies_protocol_optionals_unspecified(): dimension = Dimension( name="test_dimension", type=DimensionType.TIME, ) assert isinstance(dimension, RuntimeCheckableDimension) def test_dimension_satisfies_protocol_optionals_specified( dimension_type_params, source_file_metadata ): dimension = Dimension( name="test_dimension", type=DimensionType.TIME, description="test_description", label="test_label", type_params=dimension_type_params, expr="1", metadata=source_file_metadata, ) assert isinstance(dimension, RuntimeCheckableDimension) def test_entity_satisfies_protocol_optionals_unspecified(): entity = Entity( name="test_entity", type=EntityType.PRIMARY, ) assert isinstance(entity, RuntimeCheckableEntity) def test_entity_satisfies_protocol_optionals_specified(): entity = Entity( name="test_entity", description="a test entity", label="A test entity", type=EntityType.PRIMARY, expr="id", role="a_role", ) assert isinstance(entity, RuntimeCheckableEntity) def test_measure_satisfies_protocol_optionals_unspecified(): measure = Measure( name="test_measure", agg="sum", ) assert isinstance(measure, RuntimeCheckableMeasure) def test_measure_satisfies_protocol_optionals_specified( measure_agg_params, non_additive_dimension ): measure = Measure( name="test_measure", description="a test measure", label="A test measure", agg="sum", create_metric=True, expr="amount", agg_params=measure_agg_params, non_additive_dimension=non_additive_dimension, agg_time_dimension="a_time_dimension", ) assert isinstance(measure, RuntimeCheckableMeasure) @pytest.mark.skip(reason="Overly sensitive to non-breaking changes") def test_metric_node_satisfies_protocol_optionals_unspecified(): metric = Metric( name="a_metric", resource_type=NodeType.Metric, package_name="package_name", path="path.to.semantic_model", original_file_path="path/to/file", unique_id="not_like_the_other_semantic_models", fqn=["fully", "qualified", "name"], description="a test metric", label="A test metric", type=MetricType.SIMPLE, type_params=MetricTypeParams( measure=MetricInputMeasure( name="a_test_measure", filter=WhereFilter(where_sql_template="a_dimension is true") ) ), ) assert isinstance(metric, RuntimeCheckableMetric) @pytest.mark.skip(reason="Overly sensitive to non-breaking changes") def test_metric_node_satisfies_protocol_optionals_specified( complex_metric_type_params, source_file_metadata, where_filter ): metric = Metric( name="a_metric", resource_type=NodeType.Metric, package_name="package_name", path="path.to.semantic_model", original_file_path="path/to/file", unique_id="not_like_the_other_semantic_models", fqn=["fully", "qualified", "name"], description="a test metric", label="A test metric", type=MetricType.SIMPLE, type_params=complex_metric_type_params, filter=where_filter, metadata=source_file_metadata, group="test_group", ) assert isinstance(metric, RuntimeCheckableMetric) def test_where_filter_satisfies_protocol(where_filter): assert isinstance(where_filter, RuntimeCheckableWhereFilter) def test_metric_time_window(metric_time_window): assert isinstance(metric_time_window, RuntimeCheckableMetricTimeWindow) def test_metric_input(simple_metric_input, complex_metric_input): assert isinstance(simple_metric_input, RuntimeCheckableMetricInput) assert isinstance(complex_metric_input, RuntimeCheckableMetricInput) def test_metric_input_measure(simple_metric_input_measure, complex_metric_input_measure): assert isinstance(simple_metric_input_measure, RuntimeCheckableMetricInputMeasure) assert isinstance(complex_metric_input_measure, RuntimeCheckableMetricInputMeasure) @pytest.mark.skip(reason="Overly sensitive to non-breaking changes") def test_metric_type_params_satisfies_protocol(complex_metric_type_params): assert isinstance(MetricTypeParams(), RuntimeCheckableMetricTypeParams) assert isinstance(complex_metric_type_params, RuntimeCheckableMetricTypeParams) def test_non_additive_dimension_satisfies_protocol(non_additive_dimension): assert isinstance(non_additive_dimension, RuntimeCheckableNonAdditiveDimension) @given( builds( SavedQuery, description=text() | none(), label=text() | none(), metadata=builds(SourceFileMetadata) | none(), ) ) @settings(suppress_health_check=(HealthCheck.too_slow,)) def test_saved_query_satisfies_protocol(saved_query: SavedQuery): assert isinstance(saved_query, SavedQuery) ================================================ FILE: tests/unit/test_tracking.py ================================================ import datetime import tempfile import pytest import dbt.tracking @pytest.fixture(scope="function") def active_user_none() -> None: dbt.tracking.active_user = None @pytest.fixture(scope="function") def tempdir(active_user_none) -> str: return tempfile.mkdtemp() class TestTracking: def test_tracking_initial(self, tempdir): assert dbt.tracking.active_user is None dbt.tracking.initialize_from_flags(True, tempdir) assert isinstance(dbt.tracking.active_user, dbt.tracking.User) invocation_id = dbt.tracking.active_user.invocation_id run_started_at = dbt.tracking.active_user.run_started_at assert dbt.tracking.active_user.do_not_track is False assert isinstance(dbt.tracking.active_user.id, str) assert isinstance(invocation_id, str) assert isinstance(run_started_at, datetime.datetime) dbt.tracking.disable_tracking() assert isinstance(dbt.tracking.active_user, dbt.tracking.User) assert dbt.tracking.active_user.do_not_track is True assert dbt.tracking.active_user.id is None assert dbt.tracking.active_user.invocation_id == invocation_id assert dbt.tracking.active_user.run_started_at == run_started_at # this should generate a whole new user object -> new run_started_at dbt.tracking.do_not_track() assert isinstance(dbt.tracking.active_user, dbt.tracking.User) assert dbt.tracking.active_user.do_not_track is True assert dbt.tracking.active_user.id is None assert isinstance(dbt.tracking.active_user.invocation_id, str) assert isinstance(dbt.tracking.active_user.run_started_at, datetime.datetime) # invocation_id no longer only linked to active_user so it doesn't change assert dbt.tracking.active_user.invocation_id == invocation_id # if you use `!=`, you might hit a race condition (especially on windows) assert dbt.tracking.active_user.run_started_at is not run_started_at def test_tracking_never_ok(self, active_user_none): assert dbt.tracking.active_user is None # this should generate a whole new user object -> new invocation_id/run_started_at dbt.tracking.do_not_track() assert isinstance(dbt.tracking.active_user, dbt.tracking.User) assert dbt.tracking.active_user.do_not_track is True assert dbt.tracking.active_user.id is None assert isinstance(dbt.tracking.active_user.invocation_id, str) assert isinstance(dbt.tracking.active_user.run_started_at, datetime.datetime) def test_disable_never_enabled(self, active_user_none): assert dbt.tracking.active_user is None # this should generate a whole new user object -> new invocation_id/run_started_at dbt.tracking.disable_tracking() assert isinstance(dbt.tracking.active_user, dbt.tracking.User) assert dbt.tracking.active_user.do_not_track is True assert dbt.tracking.active_user.id is None assert isinstance(dbt.tracking.active_user.invocation_id, str) assert isinstance(dbt.tracking.active_user.run_started_at, datetime.datetime) @pytest.mark.parametrize("send_anonymous_usage_stats", [True, False]) def test_initialize_from_flags(self, tempdir, send_anonymous_usage_stats): dbt.tracking.initialize_from_flags(send_anonymous_usage_stats, tempdir) assert dbt.tracking.active_user.do_not_track != send_anonymous_usage_stats ================================================ FILE: tests/unit/test_utils.py ================================================ import unittest import dbt.exceptions import dbt.utils class TestMultiDict(unittest.TestCase): def test_one_member(self): dct = {"a": 1, "b": 2, "c": 3} md = dbt.utils.MultiDict([dct]) assert len(md) == 3 for key in "abc": assert key in md assert md["a"] == 1 assert md["b"] == 2 assert md["c"] == 3 def test_two_members_no_overlap(self): first = {"a": 1, "b": 2, "c": 3} second = {"d": 1, "e": 2, "f": 3} md = dbt.utils.MultiDict([first, second]) assert len(md) == 6 for key in "abcdef": assert key in md assert md["a"] == 1 assert md["b"] == 2 assert md["c"] == 3 assert md["d"] == 1 assert md["e"] == 2 assert md["f"] == 3 def test_two_members_overlap(self): first = {"a": 1, "b": 2, "c": 3} second = {"c": 1, "d": 2, "e": 3} md = dbt.utils.MultiDict([first, second]) assert len(md) == 5 for key in "abcde": assert key in md assert md["a"] == 1 assert md["b"] == 2 assert md["c"] == 1 assert md["d"] == 2 assert md["e"] == 3 class TestHumanizeExecutionTime(unittest.TestCase): def test_humanzing_execution_time_with_integer(self): result = dbt.utils.humanize_execution_time(execution_time=9460) assert result == " in 2 hours 37 minutes and 40.00 seconds" def test_humanzing_execution_time_with_two_decimal_place_float(self): result = dbt.utils.humanize_execution_time(execution_time=0.32) assert result == " in 0 hours 0 minutes and 0.32 seconds" def test_humanzing_execution_time_with_four_decimal_place_float(self): result = dbt.utils.humanize_execution_time(execution_time=0.3254) assert result == " in 0 hours 0 minutes and 0.33 seconds" ================================================ FILE: tests/unit/test_version.py ================================================ import dbt.version from dbt_common.ui import green, yellow class TestGetVersionInformation: def test_all_versions_equal(self, mocker): mock_versions( mocker, installed="1.0.0", latest="1.0.0", plugins={ "foobar": ("1.0.0", "1.0.0"), }, ) actual = dbt.version.get_version_information() expected = "\n".join( [ "Core:", " - installed: 1.0.0", f" - latest: 1.0.0 - {green('Up to date!')}", "", "Plugins:", f" - foobar: 1.0.0 - {green('Up to date!')}", "", "", ] ) assert expected == actual def test_core_ahead(self, mocker): mock_versions( mocker, installed="1.0.1", latest="1.0.0", plugins={ "foobar": ("1.0.0", "1.0.0"), }, ) actual = dbt.version.get_version_information() expected = "\n".join( [ "Core:", " - installed: 1.0.1", f" - latest: 1.0.0 - {yellow('Ahead of latest version!')}", "", "Plugins:", f" - foobar: 1.0.0 - {green('Up to date!')}", "", "", ] ) assert expected == actual def test_core_behind(self, mocker): mock_versions( mocker, installed="1.0.0", latest="1.0.1", plugins={ "foobar": ("1.0.0", "1.0.0"), }, ) actual = dbt.version.get_version_information() expected = "\n".join( [ "Core:", " - installed: 1.0.0", f" - latest: 1.0.1 - {yellow('Update available!')}", "", " Your version of dbt-core is out of date!", " You can find instructions for upgrading here:", " https://docs.getdbt.com/docs/installation", "", "Plugins:", f" - foobar: 1.0.0 - {green('Up to date!')}", "", "", ] ) assert expected == actual def test_core_no_latest(self, mocker): mock_versions( mocker, installed="1.0.0", plugins={ "foobar": ("1.0.0", "1.0.0"), }, ) actual = dbt.version.get_version_information() expected = "\n".join( [ "Core:", " - installed: 1.0.0", "", " The latest version of dbt-core could not be determined!", " Make sure that the following URL is accessible:", " https://pypi.org/pypi/dbt-core/json", "", "Plugins:", f" - foobar: 1.0.0 - {green('Up to date!')}", "", "", ] ) assert expected == actual def test_plugins_none(self, mocker): mock_versions( mocker, installed="1.0.0", latest="1.0.0", ) actual = dbt.version.get_version_information() expected = "\n".join( [ "Core:", " - installed: 1.0.0", f" - latest: 1.0.0 - {green('Up to date!')}", "", "Plugins:", "", "", ] ) assert expected == actual def test_plugins_multiple(self, mocker): mock_versions( mocker, installed="1.0.0", latest="1.0.0", plugins={ "foobar": ("1.0.0", "1.0.0"), "bazqux": ("1.0.0", "1.0.0"), "quuxcorge": ("1.0.0", "1.0.0"), "graultgarply": ("1.0.0", "1.0.0"), }, ) actual = dbt.version.get_version_information() expected = "\n".join( [ "Core:", " - installed: 1.0.0", f" - latest: 1.0.0 - {green('Up to date!')}", "", "Plugins:", f" - foobar: 1.0.0 - {green('Up to date!')}", f" - bazqux: 1.0.0 - {green('Up to date!')}", f" - quuxcorge: 1.0.0 - {green('Up to date!')}", f" - graultgarply: 1.0.0 - {green('Up to date!')}", "", "", ] ) assert expected == actual def test_plugin_match_core_match_latest(self, mocker): mock_versions( mocker, installed="1.0.0", latest="1.0.0", plugins={ "foobar": ("1.0.0", "1.0.0"), }, ) actual = dbt.version.get_version_information() expected = "\n".join( [ "Core:", " - installed: 1.0.0", f" - latest: 1.0.0 - {green('Up to date!')}", "", "Plugins:", f" - foobar: 1.0.0 - {green('Up to date!')}", "", "", ] ) assert expected == actual def test_plugin_match_core_no_latest(self, mocker): mock_versions( mocker, installed="1.0.0", latest="1.0.0", plugins={ "foobar": ("1.0.0", None), }, ) actual = dbt.version.get_version_information() expected = "\n".join( [ "Core:", " - installed: 1.0.0", f" - latest: 1.0.0 - {green('Up to date!')}", "", "Plugins:", f" - foobar: 1.0.0 - {yellow('Could not determine latest version')}", "", "", ] ) assert expected == actual def test_plugin_match_core_behind_latest(self, mocker): mock_versions( mocker, installed="1.0.0", latest="1.0.0", plugins={ "foobar": ("1.0.0", "2.0.0"), }, ) actual = dbt.version.get_version_information() expected = "\n".join( [ "Core:", " - installed: 1.0.0", f" - latest: 1.0.0 - {green('Up to date!')}", "", "Plugins:", f" - foobar: 1.0.0 - {yellow('Update available!')}", "", " At least one plugin is out of date with dbt-core.", " You can find instructions for upgrading here:", " https://docs.getdbt.com/docs/installation", "", "", ] ) assert expected == actual def test_plugin_match_core_ahead_latest(self, mocker): mock_versions( mocker, installed="1.0.0", latest="1.0.0", plugins={ "foobar": ("1.0.0", "2.0.0"), }, ) actual = dbt.version.get_version_information() expected = "\n".join( [ "Core:", " - installed: 1.0.0", f" - latest: 1.0.0 - {green('Up to date!')}", "", "Plugins:", f" - foobar: 1.0.0 - {yellow('Update available!')}", "", " At least one plugin is out of date with dbt-core.", " You can find instructions for upgrading here:", " https://docs.getdbt.com/docs/installation", "", "", ] ) assert expected == actual def test_plugin_diff_core_minor_match_latest(self, mocker): mock_versions( mocker, installed="1.1.0", latest="1.1.0", plugins={ "foobar": ("1.0.0", "1.0.0"), }, ) actual = dbt.version.get_version_information() expected = "\n".join( [ "Core:", " - installed: 1.1.0", f" - latest: 1.1.0 - {green('Up to date!')}", "", "Plugins:", f" - foobar: 1.0.0 - {green('Up to date!')}", "", "", ] ) assert expected == actual def test_plugin_diff_core_minor_no_latest(self, mocker): mock_versions( mocker, installed="1.1.0", latest="1.1.0", plugins={ "foobar": ("1.0.0", None), }, ) actual = dbt.version.get_version_information() expected = "\n".join( [ "Core:", " - installed: 1.1.0", f" - latest: 1.1.0 - {green('Up to date!')}", "", "Plugins:", f" - foobar: 1.0.0 - {yellow('Could not determine latest version')}", "", "", ] ) assert expected == actual def test_plugin_diff_core_minor_ahead_latest(self, mocker): mock_versions( mocker, installed="1.1.0", latest="1.1.0", plugins={ "foobar": ("1.0.0", "0.0.1"), }, ) actual = dbt.version.get_version_information() expected = "\n".join( [ "Core:", " - installed: 1.1.0", f" - latest: 1.1.0 - {green('Up to date!')}", "", "Plugins:", f" - foobar: 1.0.0 - {yellow('Ahead of latest version!')}", "", "", ] ) assert expected == actual def test_plugin_diff_plugin_minor_ahead_latest(self, mocker): """ Now that adapters are decoupled from core, a higher minor version of a plugin is compatible with a lower minor version of core. """ mock_versions( mocker, installed="1.8.0", latest="1.8.0", plugins={ "foobar": ("1.9.0", "1.9.0"), }, ) actual = dbt.version.get_version_information() expected = "\n".join( [ "Core:", " - installed: 1.8.0", f" - latest: 1.8.0 - {green('Up to date!')}", "", "Plugins:", f" - foobar: 1.9.0 - {green('Up to date!')}", "", "", ] ) assert expected == actual def test_plugin_diff_plugin_patch_ahead_latest(self, mocker): """ Now that adapters are decoupled from core, a higher minor version of a plugin is compatible with a lower minor version of core. """ mock_versions( mocker, installed="1.8.0", latest="1.8.0", plugins={ "foobar": ("1.8.4", "1.8.4"), }, ) actual = dbt.version.get_version_information() expected = "\n".join( [ "Core:", " - installed: 1.8.0", f" - latest: 1.8.0 - {green('Up to date!')}", "", "Plugins:", f" - foobar: 1.8.4 - {green('Up to date!')}", "", "", ] ) assert expected == actual def test_plugin_diff_plugin_minor_ahead_no_latest(self, mocker): """ Now that adapters are decoupled from core, a higher minor version of a plugin is compatible with a lower minor version of core. """ mock_versions( mocker, installed="1.8.0", latest="1.8.0", plugins={ "foobar": ("1.9.0", None), }, ) actual = dbt.version.get_version_information() expected = "\n".join( [ "Core:", " - installed: 1.8.0", f" - latest: 1.8.0 - {green('Up to date!')}", "", "Plugins:", f" - foobar: 1.9.0 - {yellow('Could not determine latest version')}", "", "", ] ) assert expected == actual def test_plugin_diff_plugin_minor_behind_core_no_latest(self, mocker): """ Now that adapters are decoupled from core, a lower minor version of a plugin (1.8) is compatible with a higher minor version of core. (1.9) """ mock_versions( mocker, installed="1.9.0", latest="1.9.0", plugins={ "foobar": ("1.8.0", "1.8.0"), }, ) actual = dbt.version.get_version_information() expected = "\n".join( [ "Core:", " - installed: 1.9.0", f" - latest: 1.9.0 - {green('Up to date!')}", "", "Plugins:", f" - foobar: 1.8.0 - {green('Up to date!')}", "", "", ] ) assert expected == actual def test_plugin_diff_core_minor_behind_latest(self, mocker): mock_versions( mocker, installed="1.1.0", latest="1.1.0", plugins={ "foobar": ("1.0.0", "1.0.1"), }, ) actual = dbt.version.get_version_information() expected = "\n".join( [ "Core:", " - installed: 1.1.0", f" - latest: 1.1.0 - {green('Up to date!')}", "", "Plugins:", f" - foobar: 1.0.0 - {yellow('Update available!')}", "", " At least one plugin is out of date with dbt-core.", " You can find instructions for upgrading here:", " https://docs.getdbt.com/docs/installation", "", "", ] ) assert expected == actual def test_plugins_various(self, mocker): mock_versions( mocker, installed="2.1.0", latest="2.1.0", plugins={ "foobar": ("2.1.0", "2.1.0"), "bazqux": ("2.1.0", None), "quuux": ("2.1.0", "2.1.0"), "grault": ("2.1.0", "2.1.1"), "garply": ("2.1.0-b1", None), }, ) actual = dbt.version.get_version_information() expected = "\n".join( [ "Core:", " - installed: 2.1.0", f" - latest: 2.1.0 - {green('Up to date!')}", "", "Plugins:", f" - foobar: 2.1.0 - {green('Up to date!')}", f" - bazqux: 2.1.0 - {yellow('Could not determine latest version')}", f" - quuux: 2.1.0 - {green('Up to date!')}", f" - grault: 2.1.0 - {yellow('Update available!')}", f" - garply: 2.1.0-b1 - {yellow('Could not determine latest version')}", "", " At least one plugin is out of date with dbt-core.", " You can find instructions for upgrading here:", " https://docs.getdbt.com/docs/installation", "", "", ] ) assert expected == actual def test_plugins_alignment(self, mocker): mock_versions( mocker, installed="1.1.1-b123", latest="1.1.1-b123", plugins={ "foobar": ("1.1.0-b1", "1.1.0-b1"), "bazqux": ("1.1.1-b123", "1.1.1-b123"), "quuux": ("1.1.0", "1.1.0"), }, ) actual = dbt.version.get_version_information() expected = "\n".join( [ "Core:", " - installed: 1.1.1-b123", f" - latest: 1.1.1-b123 - {green('Up to date!')}", "", "Plugins:", f" - foobar: 1.1.0-b1 - {green('Up to date!')}", f" - bazqux: 1.1.1-b123 - {green('Up to date!')}", f" - quuux: 1.1.0 - {green('Up to date!')}", "", "", ] ) assert expected == actual def test_file_globbing(self, mocker): mocker.patch("dbt.version.__version__", "1.0.0") mock_latest_versions(mocker, core_latest="1.0.0") mocked_spec = mocker.Mock() mocker.patch("importlib.util.find_spec").return_value = mocked_spec paths = [ "/some/sort/of/path", "/another/path", "/yet/another/type/of/path", ] mocked_spec.submodule_search_locations = paths plugins = [ "foobar", "bazqux", "quuux", "corge", "grault", "garply", ] version_paths = [ [ f"/some/sort/of/path/{plugins[0]}/__version__.py", f"/some/sort/of/path/{plugins[1]}/__version__.py", ], [ f"/another/path/{plugins[2]}/__version__.py", f"/another/path/{plugins[3]}/__version__.py", ], [ f"/yet/another/type/of/path/{plugins[4]}/__version__.py", f"/yet/another/type/of/path/{plugins[5]}/__version__.py", ], ] def mock_glob(*args, **kwargs): version_glob = args[0] for i, path in enumerate(paths): if path in version_glob: return version_paths[i] mocker.patch("glob.glob").side_effect = mock_glob def mock_import(*args, **kwargs): mock_module = mocker.Mock() mock_module.version = "1.0.0" import_path = args[0] for plugin in plugins: if plugin in import_path: return mock_module raise ImportError mocker.patch("importlib.import_module").side_effect = mock_import actual = dbt.version.get_version_information() expected = "\n".join( [ "Core:", " - installed: 1.0.0", f" - latest: 1.0.0 - {green('Up to date!')}", "", "Plugins:", f" - foobar: 1.0.0 - {yellow('Could not determine latest version')}", f" - bazqux: 1.0.0 - {yellow('Could not determine latest version')}", f" - quuux: 1.0.0 - {yellow('Could not determine latest version')}", f" - corge: 1.0.0 - {yellow('Could not determine latest version')}", f" - grault: 1.0.0 - {yellow('Could not determine latest version')}", f" - garply: 1.0.0 - {yellow('Could not determine latest version')}", "", "", ] ) assert expected == actual def mock_versions(mocker, installed="1.0.0", latest=None, plugins={}): mocker.patch("dbt.version.__version__", installed) mock_latest_versions(mocker, latest, plugins) # mock_plugins must be called last to avoid erronously raising an ImportError. mock_plugins(mocker, plugins) # NOTE: mock_plugins patches importlib.import_module, and should always be the last # patch to be mocked in order to avoid erronously raising an ImportError. # Explanation: As of Python 3.11, mock.patch indirectly uses importlib.import_module # and thus uses the mocked object (in this case, mock_import) instead of the real # implementation in subsequent mock.patch calls. Issue: https://github.com/python/cpython/issues/98771 def mock_plugins(mocker, plugins): mock_find_spec = mocker.patch("importlib.util.find_spec") path = "/tmp/dbt/adapters" mock_find_spec.return_value.submodule_search_locations = [path] mocker.patch("glob.glob").return_value = [f"{path}/{name}/__version__.py" for name in plugins] def mock_import(*args, **kwargs): import_path = args[0] for plugin_name in plugins: if plugin_name in import_path: plugin_version = plugins.get(plugin_name)[0] module_mock = mocker.Mock() module_mock.version = plugin_version return module_mock raise ImportError mocker.patch("importlib.import_module").side_effect = mock_import def mock_latest_versions(mocker, core_latest=None, plugins={}): def mock_get(*args, **kwargs): mocked_response = mocker.Mock() mocked_response.json.return_value = {"bad object": None} version_url = args[0] if version_url is dbt.version.PYPI_VERSION_URL: if core_latest: mocked_response.json.return_value = {"info": {"version": core_latest}} return mocked_response for plugin_name in plugins: if plugin_name in version_url: plugin_latest = plugins.get(plugin_name)[1] if plugin_latest: mocked_response.json.return_value = {"info": {"version": plugin_latest}} return mocked_response return mocked_response mocker.patch("requests.get").side_effect = mock_get ================================================ FILE: tests/unit/utils/__init__.py ================================================ """Unit test utility functions. Note that all imports should be inside the functions to avoid import/mocking issues. """ import os import string from unittest import TestCase, mock import agate import pytest from dbt.config.project import PartialProject from dbt.contracts.graph.manifest import Manifest from dbt_common.dataclass_schema import ValidationError def normalize(path): """On windows, neither is enough on its own: >>> normcase('C:\\documents/ALL CAPS/subdir\\..') 'c:\\documents\\all caps\\subdir\\..' >>> normpath('C:\\documents/ALL CAPS/subdir\\..') 'C:\\documents\\ALL CAPS' >>> normpath(normcase('C:\\documents/ALL CAPS/subdir\\..')) 'c:\\documents\\all caps' """ return os.path.normcase(os.path.normpath(path)) class Obj: which = "blah" single_threaded = False def mock_connection(name, state="open"): conn = mock.MagicMock() conn.name = name conn.state = state return conn def profile_from_dict(profile, profile_name, cli_vars="{}"): from dbt.config import Profile from dbt.config.renderer import ProfileRenderer from dbt.config.utils import parse_cli_vars if not isinstance(cli_vars, dict): cli_vars = parse_cli_vars(cli_vars) renderer = ProfileRenderer(cli_vars) return Profile.from_raw_profile_info( profile, profile_name, renderer, ) def project_from_dict(project, profile, packages=None, selectors=None, cli_vars="{}"): from dbt.config.renderer import DbtProjectYamlRenderer from dbt.config.utils import parse_cli_vars if not isinstance(cli_vars, dict): cli_vars = parse_cli_vars(cli_vars) renderer = DbtProjectYamlRenderer(profile, cli_vars) project_root = project.pop("project-root", os.getcwd()) partial = PartialProject.from_dicts( project_root=project_root, project_dict=project, packages_dict=packages, selectors_dict=selectors, ) return partial.render(renderer) def config_from_parts_or_dicts(project, profile, packages=None, selectors=None, cli_vars={}): from copy import deepcopy from dbt.config import Profile, Project, RuntimeConfig if isinstance(project, Project): profile_name = project.profile_name else: profile_name = project.get("profile") if not isinstance(profile, Profile): profile = profile_from_dict( deepcopy(profile), profile_name, cli_vars, ) if not isinstance(project, Project): project = project_from_dict( deepcopy(project), profile, packages, selectors, cli_vars, ) args = Obj() args.vars = cli_vars args.profile_dir = "/dev/null" return RuntimeConfig.from_parts(project=project, profile=profile, args=args) def inject_plugin(plugin): from dbt.adapters.factory import FACTORY key = plugin.adapter.type() FACTORY.plugins[key] = plugin def inject_plugin_for(config): # from dbt.adapters.postgres import Plugin, PostgresAdapter from dbt.adapters.factory import FACTORY FACTORY.load_plugin(config.credentials.type) adapter = FACTORY.get_adapter(config) return adapter def inject_adapter(value, plugin): """Inject the given adapter into the adapter factory, so your hand-crafted artisanal adapter will be available from get_adapter() as if dbt loaded it. """ inject_plugin(plugin) from dbt.adapters.factory import FACTORY key = value.type() FACTORY.adapters[key] = value def clear_plugin(plugin): from dbt.adapters.factory import FACTORY key = plugin.adapter.type() FACTORY.plugins.pop(key, None) FACTORY.adapters.pop(key, None) class ContractTestCase(TestCase): ContractType = None def setUp(self): self.maxDiff = None super().setUp() def assert_to_dict(self, obj, dct): self.assertEqual(obj.to_dict(omit_none=True), dct) def assert_from_dict(self, obj, dct, cls=None): if cls is None: cls = self.ContractType cls.validate(dct) self.assertEqual(cls.from_dict(dct), obj) def assert_symmetric(self, obj, dct, cls=None): self.assert_to_dict(obj, dct) self.assert_from_dict(obj, dct, cls) def assert_fails_validation(self, dct, cls=None): if cls is None: cls = self.ContractType with self.assertRaises(ValidationError): cls.validate(dct) cls.from_dict(dct) def compare_dicts(dict1, dict2): first_set = set(dict1.keys()) second_set = set(dict2.keys()) print(f"--- Difference between first and second keys: {first_set.difference(second_set)}") print(f"--- Difference between second and first keys: {second_set.difference(first_set)}") common_keys = set(first_set).intersection(set(second_set)) found_differences = False for key in common_keys: if dict1[key] != dict2[key]: print(f"--- --- first dict: {key}: {str(dict1[key])}") print(f"--- --- second dict: {key}: {str(dict2[key])}") found_differences = True if found_differences: print("--- Found differences in dictionaries") else: print("--- Found no differences in dictionaries") def assert_from_dict(obj, dct, cls=None): if cls is None: cls = obj.__class__ cls.validate(dct) obj_from_dict = cls.from_dict(dct) if hasattr(obj, "created_at"): obj_from_dict.created_at = 1 obj.created_at = 1 assert obj_from_dict == obj def assert_to_dict(obj, dct): obj_to_dict = obj.to_dict(omit_none=True) if "created_at" in obj_to_dict: obj_to_dict["created_at"] = 1 if "created_at" in dct: dct["created_at"] = 1 if obj_to_dict != dct: compare_dicts(obj_to_dict, dct) assert obj_to_dict == dct def assert_symmetric(obj, dct, cls=None): assert_to_dict(obj, dct) assert_from_dict(obj, dct, cls) def assert_fails_validation(dct, cls): with pytest.raises(ValidationError): cls.validate(dct) cls.from_dict(dct) def generate_name_macros(package): from dbt.contracts.graph.nodes import Macro from dbt.node_types import NodeType name_sql = {} for component in ("database", "schema", "alias"): if component == "alias": source = "node.name" else: source = f"target.{component}" name = f"generate_{component}_name" sql = f"{{% macro {name}(value, node) %}} {{% if value %}} {{{{ value }}}} {{% else %}} {{{{ {source} }}}} {{% endif %}} {{% endmacro %}}" name_sql[name] = sql for name, sql in name_sql.items(): pm = Macro( name=name, resource_type=NodeType.Macro, unique_id=f"macro.{package}.{name}", package_name=package, original_file_path=normalize("macros/macro.sql"), path=normalize("macros/macro.sql"), macro_sql=sql, ) yield pm class TestAdapterConversions(TestCase): def _get_tester_for(self, column_type): from dbt_common.clients import agate_helper if column_type is agate.TimeDelta: # dbt never makes this! return agate.TimeDelta() for instance in agate_helper.DEFAULT_TYPE_TESTER._possible_types: if isinstance(instance, column_type): # include child types return instance raise ValueError(f"no tester for {column_type}") def _make_table_of(self, rows, column_types): column_names = list(string.ascii_letters[: len(rows[0])]) if isinstance(column_types, type): column_types = [self._get_tester_for(column_types) for _ in column_names] else: column_types = [self._get_tester_for(typ) for typ in column_types] table = agate.Table(rows, column_names=column_names, column_types=column_types) return table def MockMacro(package, name="my_macro", **kwargs): from dbt.contracts.graph.nodes import Macro from dbt.node_types import NodeType mock_kwargs = dict( resource_type=NodeType.Macro, package_name=package, unique_id=f"macro.{package}.{name}", original_file_path="/dev/null", ) mock_kwargs.update(kwargs) macro = mock.MagicMock(spec=Macro, **mock_kwargs) macro.name = name return macro def MockMaterialization(package, name="my_materialization", adapter_type=None, **kwargs): if adapter_type is None: adapter_type = "default" kwargs["adapter_type"] = adapter_type return MockMacro(package, f"materialization_{name}_{adapter_type}", **kwargs) def MockGenerateMacro(package, component="some_component", **kwargs): name = f"generate_{component}_name" return MockMacro(package, name=name, **kwargs) def MockSource(package, source_name, name, **kwargs): from dbt.contracts.graph.nodes import SourceDefinition from dbt.node_types import NodeType src = mock.MagicMock( __class__=SourceDefinition, resource_type=NodeType.Source, source_name=source_name, package_name=package, unique_id=f"source.{package}.{source_name}.{name}", search_name=f"{source_name}.{name}", **kwargs, ) src.name = name return src def MockNode(package, name, resource_type=None, **kwargs): from dbt.contracts.graph.nodes import ModelNode, SeedNode from dbt.node_types import NodeType if resource_type is None: resource_type = NodeType.Model if resource_type == NodeType.Model: cls = ModelNode elif resource_type == NodeType.Seed: cls = SeedNode else: raise ValueError(f"I do not know how to handle {resource_type}") version = kwargs.get("version") search_name = name if version is None else f"{name}.v{version}" unique_id = f"{str(resource_type)}.{package}.{search_name}" node = mock.MagicMock( __class__=cls, resource_type=resource_type, package_name=package, unique_id=unique_id, search_name=search_name, **kwargs, ) node.name = name node.is_versioned = resource_type is NodeType.Model and version is not None return node def MockDocumentation(package, name, **kwargs): from dbt.contracts.graph.nodes import Documentation from dbt.node_types import NodeType doc = mock.MagicMock( __class__=Documentation, resource_type=NodeType.Documentation, package_name=package, search_name=name, unique_id=f"{package}.{name}", **kwargs, ) doc.name = name return doc def load_internal_manifest_macros(config, macro_hook=lambda m: None): from dbt.parser.manifest import ManifestLoader return ManifestLoader.load_macros(config, macro_hook) def dict_replace(dct, **kwargs): dct = dct.copy() dct.update(kwargs) return dct def replace_config(n, **kwargs): from dataclasses import replace return replace( n, config=n.config.replace(**kwargs), unrendered_config=dict_replace(n.unrendered_config, **kwargs), ) def make_manifest(nodes=[], sources=[], macros=[], docs=[]) -> Manifest: return Manifest( nodes={n.unique_id: n for n in nodes}, macros={m.unique_id: m for m in macros}, sources={s.unique_id: s for s in sources}, docs={d.unique_id: d for d in docs}, disabled={}, files={}, exposures={}, metrics={}, selectors={}, ) ================================================ FILE: tests/unit/utils/adapter.py ================================================ import sys from unittest.mock import MagicMock import pytest from pytest_mock import MockerFixture from dbt.adapters.factory import get_adapter, register_adapter, reset_adapters from dbt.adapters.postgres import PostgresAdapter from dbt.adapters.sql import SQLConnectionManager from dbt.config.runtime import RuntimeConfig from dbt.context.providers import generate_runtime_macro_context from dbt.contracts.graph.manifest import ManifestStateCheck from dbt.mp_context import get_mp_context from dbt.parser.manifest import ManifestLoader if sys.version_info < (3, 9): from typing import Generator else: from collections.abc import Generator @pytest.fixture def mock_connection_manager() -> MagicMock: mock_connection_manager = MagicMock(SQLConnectionManager) mock_connection_manager.set_query_header = lambda query_header_context: None return mock_connection_manager @pytest.fixture def mock_adapter(mock_connection_manager: MagicMock) -> MagicMock: mock_adapter = MagicMock(PostgresAdapter) mock_adapter.connections = mock_connection_manager mock_adapter.clear_macro_resolver = MagicMock() return mock_adapter @pytest.fixture def postgres_adapter( mocker: MockerFixture, runtime_config: RuntimeConfig ) -> Generator[PostgresAdapter, None, None]: register_adapter(runtime_config, get_mp_context()) adapter = get_adapter(runtime_config) assert isinstance(adapter, PostgresAdapter) mocker.patch("dbt.parser.manifest.ManifestLoader.build_manifest_state_check").return_value = ( ManifestStateCheck() ) manifest = ManifestLoader.load_macros( runtime_config, adapter.connections.set_query_header, base_macros_only=True, ) adapter.set_macro_resolver(manifest) adapter.set_macro_context_generator(generate_runtime_macro_context) yield adapter adapter.cleanup_connections() reset_adapters() ================================================ FILE: tests/unit/utils/config.py ================================================ import pytest from dbt.adapters.postgres.connections import PostgresCredentials from dbt.config.profile import Profile from dbt.config.project import Project from dbt.config.renderer import ProfileRenderer from dbt.config.runtime import RuntimeConfig from dbt.flags import get_flags @pytest.fixture def credentials() -> PostgresCredentials: return PostgresCredentials( database="test_database", schema="test_schema", host="test_host", user="test_user", port=1337, password="test_password", ) @pytest.fixture def profile() -> Profile: profile_yaml = { "target": "postgres", "outputs": { "postgres": { "type": "postgres", "host": "postgres-db-hostname", "port": 5555, "user": "db_user", "pass": "db_pass", "dbname": "postgres-db-name", "schema": "postgres-schema", "threads": 7, }, }, } return Profile.from_raw_profile_info( raw_profile=profile_yaml, profile_name="test_profile", renderer=ProfileRenderer({}) ) @pytest.fixture def runtime_config(project: Project, profile: Profile, set_test_flags) -> RuntimeConfig: args = get_flags() return RuntimeConfig.from_parts( project=project, profile=profile, args=args, ) ================================================ FILE: tests/unit/utils/event_manager.py ================================================ import pytest from dbt_common.events.event_manager_client import cleanup_event_logger @pytest.fixture(autouse=True) def always_clean_event_manager() -> None: cleanup_event_logger() ================================================ FILE: tests/unit/utils/flags.py ================================================ import sys from argparse import Namespace if sys.version_info < (3, 9): from typing import Generator else: from collections.abc import Generator import pytest from dbt.flags import set_from_args @pytest.fixture def args_for_flags() -> Namespace: """Defines the namespace args to be used in `set_from_args` of `set_test_flags` fixture. This fixture is meant to be overrided by tests that need specific flags to be set. """ return Namespace() @pytest.fixture(autouse=True) def set_test_flags(args_for_flags: Namespace) -> Generator[None, None, None]: """Sets up and tears down the global flags for every pytest unit test Override `args_for_flags` fixture as needed to set any specific flags. """ set_from_args(args_for_flags, {}) # fixtures stop setup upon yield yield None # everything after yield is run at test teardown set_from_args(Namespace(), {}) ================================================ FILE: tests/unit/utils/manifest.py ================================================ from typing import Any, Dict, List import pytest from dbt.artifacts.resources import ( ExposureType, MacroDependsOn, MetricInputMeasure, MetricTypeParams, NodeRelation, Owner, QueryParams, RefArgs, SnapshotConfig, TestConfig, TestMetadata, WhereFilter, WhereFilterIntersection, ) from dbt.artifacts.resources.types import ModelLanguage from dbt.artifacts.resources.v1.model import ModelConfig from dbt.artifacts.resources.v1.semantic_model import ( Defaults, Dimension, DimensionTypeParams, Measure, ) from dbt.contracts.files import AnySourceFile, FileHash from dbt.contracts.graph.manifest import Manifest, ManifestMetadata from dbt.contracts.graph.nodes import ( AccessType, DependsOn, Documentation, Exposure, GenericTestNode, GraphMemberNode, Group, Macro, ManifestNode, Metric, ModelNode, NodeConfig, SavedQuery, SeedNode, SemanticModel, SingularTestNode, SnapshotNode, SourceDefinition, UnitTestDefinition, ) from dbt.contracts.graph.unparsed import UnitTestInputFixture, UnitTestOutputFixture from dbt.node_types import NodeType from dbt_semantic_interfaces.type_enums import ( AggregationType, DimensionType, MetricType, TimeGranularity, ) def make_model( pkg, name, code, language="sql", refs=None, sources=None, tags=None, path=None, alias=None, config_kwargs=None, fqn_extras=None, depends_on_macros=None, version=None, latest_version=None, access=None, patch_path=None, ): if refs is None: refs = [] if sources is None: sources = [] if tags is None: tags = [] if path is None: if language == "sql": path = f"{name}.sql" elif language == "python": path = f"{name}.py" else: raise ValueError(f"Unknown language: {language}") if alias is None: alias = name if config_kwargs is None: config_kwargs = {} if depends_on_macros is None: depends_on_macros = [] if fqn_extras is None: fqn_extras = [] fqn = [pkg] + fqn_extras + [name] if version: fqn.append(f"v{version}") depends_on_nodes = [] source_values = [] ref_values = [] for ref in refs: ref_version = ref.version if hasattr(ref, "version") else None ref_values.append(RefArgs(name=ref.name, package=ref.package_name, version=ref_version)) depends_on_nodes.append(ref.unique_id) for src in sources: source_values.append([src.source_name, src.name]) depends_on_nodes.append(src.unique_id) return ModelNode( language="sql", raw_code=code, database="dbt", schema="dbt_schema", alias=alias, name=name, fqn=fqn, unique_id=f"model.{pkg}.{name}" if not version else f"model.{pkg}.{name}.v{version}", package_name=pkg, path=path, original_file_path=f"models/{path}", config=NodeConfig(**config_kwargs), tags=tags, refs=ref_values, sources=source_values, depends_on=DependsOn( nodes=depends_on_nodes, macros=depends_on_macros, ), resource_type=NodeType.Model, checksum=FileHash.from_contents(""), version=version, latest_version=latest_version, access=access or AccessType.Protected, patch_path=patch_path, ) def make_seed( pkg, name, path=None, loader=None, alias=None, tags=None, fqn_extras=None, checksum=None ): if alias is None: alias = name if tags is None: tags = [] if path is None: path = f"{name}.csv" if fqn_extras is None: fqn_extras = [] if checksum is None: checksum = FileHash.from_contents("") fqn = [pkg] + fqn_extras + [name] return SeedNode( database="dbt", schema="dbt_schema", alias=alias, name=name, fqn=fqn, unique_id=f"seed.{pkg}.{name}", package_name=pkg, path=path, original_file_path=f"data/{path}", tags=tags, resource_type=NodeType.Seed, checksum=FileHash.from_contents(""), ) def make_source( pkg, source_name, table_name, path=None, loader=None, identifier=None, fqn_extras=None ): if path is None: path = "models/schema.yml" if loader is None: loader = "my_loader" if identifier is None: identifier = table_name if fqn_extras is None: fqn_extras = [] fqn = [pkg] + fqn_extras + [source_name, table_name] return SourceDefinition( fqn=fqn, database="dbt", schema="dbt_schema", unique_id=f"source.{pkg}.{source_name}.{table_name}", package_name=pkg, path=path, original_file_path=path, name=table_name, source_name=source_name, loader="my_loader", identifier=identifier, resource_type=NodeType.Source, loaded_at_field="loaded_at", tags=[], source_description="", ) def make_macro(pkg, name, macro_sql, path=None, depends_on_macros=None): if path is None: path = "macros/macros.sql" if depends_on_macros is None: depends_on_macros = [] return Macro( name=name, macro_sql=macro_sql, unique_id=f"macro.{pkg}.{name}", package_name=pkg, path=path, original_file_path=path, resource_type=NodeType.Macro, depends_on=MacroDependsOn(macros=depends_on_macros), ) def make_unique_test(pkg, test_model, column_name, path=None, refs=None, sources=None, tags=None): return make_generic_test(pkg, "unique", test_model, {}, column_name=column_name) def make_not_null_test( pkg, test_model, column_name, path=None, refs=None, sources=None, tags=None ): return make_generic_test(pkg, "not_null", test_model, {}, column_name=column_name) def make_generic_test( pkg, test_name, test_model, test_kwargs, path=None, refs=None, sources=None, tags=None, column_name=None, ): kwargs = test_kwargs.copy() ref_values = [] source_values = [] # this doesn't really have to be correct if isinstance(test_model, SourceDefinition): kwargs["model"] = ( "{{ source('" + test_model.source_name + "', '" + test_model.name + "') }}" ) source_values.append([test_model.source_name, test_model.name]) else: kwargs["model"] = "{{ ref('" + test_model.name + "')}}" ref_values.append( RefArgs( name=test_model.name, package=test_model.package_name, version=test_model.version ) ) if column_name is not None: kwargs["column_name"] = column_name # whatever args_name = test_model.search_name.replace(".", "_") if column_name is not None: args_name += "_" + column_name node_name = f"{test_name}_{args_name}" raw_code = ( '{{ config(severity="ERROR") }}{{ test_' + test_name + "(**dbt_schema_test_kwargs) }}" ) name_parts = test_name.split(".") if len(name_parts) == 2: namespace, test_name = name_parts macro_depends = f"macro.{namespace}.test_{test_name}" elif len(name_parts) == 1: namespace = None macro_depends = f"macro.dbt.test_{test_name}" else: assert False, f"invalid test name: {test_name}" if path is None: path = "schema.yml" if tags is None: tags = ["schema"] if refs is None: refs = [] if sources is None: sources = [] depends_on_nodes = [] for ref in refs: ref_version = ref.version if hasattr(ref, "version") else None ref_values.append(RefArgs(name=ref.name, package=ref.package_name, version=ref_version)) depends_on_nodes.append(ref.unique_id) for source in sources: source_values.append([source.source_name, source.name]) depends_on_nodes.append(source.unique_id) return GenericTestNode( language="sql", raw_code=raw_code, test_metadata=TestMetadata( namespace=namespace, name=test_name, kwargs=kwargs, ), database="dbt", schema="dbt_postgres", name=node_name, alias=node_name, fqn=["minimal", "schema_test", node_name], unique_id=f"test.{pkg}.{node_name}", package_name=pkg, path=f"schema_test/{node_name}.sql", original_file_path=f"models/{path}", resource_type=NodeType.Test, tags=tags, refs=ref_values, sources=[], depends_on=DependsOn(macros=[macro_depends], nodes=depends_on_nodes), column_name=column_name, checksum=FileHash.from_contents(""), ) def make_unit_test( pkg, test_name, test_model, ): input_fixture = UnitTestInputFixture( input="ref('table_model')", rows=[{"id": 1, "string_a": "a"}], ) output_fixture = UnitTestOutputFixture( rows=[{"id": 1, "string_a": "a"}], ) return UnitTestDefinition( name=test_name, model=test_model, package_name=pkg, resource_type=NodeType.Unit, path="unit_tests.yml", original_file_path="models/unit_tests.yml", unique_id=f"unit.{pkg}.{test_model.name}__{test_name}", given=[input_fixture], expect=output_fixture, fqn=[pkg, test_model.name, test_name], ) def make_singular_test( pkg, name, sql, refs=None, sources=None, tags=None, path=None, config_kwargs=None ): if refs is None: refs = [] if sources is None: sources = [] if tags is None: tags = ["data"] if path is None: path = f"{name}.sql" if config_kwargs is None: config_kwargs = {} fqn = ["minimal", "data_test", name] depends_on_nodes = [] source_values = [] ref_values = [] for ref in refs: ref_version = ref.version if hasattr(ref, "version") else None ref_values.append(RefArgs(name=ref.name, package=ref.package_name, version=ref_version)) depends_on_nodes.append(ref.unique_id) for src in sources: source_values.append([src.source_name, src.name]) depends_on_nodes.append(src.unique_id) return SingularTestNode( language="sql", raw_code=sql, database="dbt", schema="dbt_schema", name=name, alias=name, fqn=fqn, unique_id=f"test.{pkg}.{name}", package_name=pkg, path=path, original_file_path=f"tests/{path}", config=TestConfig(**config_kwargs), tags=tags, refs=ref_values, sources=source_values, depends_on=DependsOn(nodes=depends_on_nodes, macros=[]), resource_type=NodeType.Test, checksum=FileHash.from_contents(""), ) def make_exposure(pkg, name, path=None, fqn_extras=None, owner=None): if path is None: path = "schema.yml" if fqn_extras is None: fqn_extras = [] if owner is None: owner = Owner(email="test@example.com") fqn = [pkg, "exposures"] + fqn_extras + [name] return Exposure( name=name, resource_type=NodeType.Exposure, type=ExposureType.Notebook, fqn=fqn, unique_id=f"exposure.{pkg}.{name}", package_name=pkg, path=path, original_file_path=path, owner=owner, ) def make_metric(pkg, name, path=None): if path is None: path = "schema.yml" return Metric( name=name, resource_type=NodeType.Metric, path=path, package_name=pkg, original_file_path=path, unique_id=f"metric.{pkg}.{name}", fqn=[pkg, "metrics", name], label="New Customers", description="New customers", type=MetricType.SIMPLE, type_params=MetricTypeParams(measure=MetricInputMeasure(name="count_cats")), meta={"is_okr": True}, tags=["okrs"], ) def make_group(pkg, name, path=None): if path is None: path = "schema.yml" return Group( name=name, resource_type=NodeType.Group, path=path, package_name=pkg, original_file_path=path, unique_id=f"group.{pkg}.{name}", owner="email@gmail.com", ) def make_semantic_model( pkg: str, name: str, model, path=None, ): if path is None: path = "schema.yml" return SemanticModel( name=name, resource_type=NodeType.SemanticModel, model=model, node_relation=NodeRelation( alias=model.alias, schema_name="dbt", relation_name=model.name, ), package_name=pkg, path=path, description="Customer entity", primary_entity="customer", unique_id=f"semantic_model.{pkg}.{name}", original_file_path=path, fqn=[pkg, "semantic_models", name], defaults=Defaults(agg_time_dimension="created_at"), dimensions=[ Dimension( name="created_at", type=DimensionType.TIME, type_params=DimensionTypeParams(time_granularity=TimeGranularity.DAY), ) ], measures=[ Measure( name="a_measure", agg=AggregationType.COUNT, expr="1", ) ], ) def make_saved_query(pkg: str, name: str, metric: str, path=None): if path is None: path = "schema.yml" return SavedQuery( name=name, resource_type=NodeType.SavedQuery, package_name=pkg, path=path, description="Test Saved Query", query_params=QueryParams( metrics=[metric], group_by=[], where=None, ), exports=[], unique_id=f"saved_query.{pkg}.{name}", original_file_path=path, fqn=[pkg, "saved_queries", name], ) def make_source_snapshot(pkg: str, name: str, source: SourceDefinition, path=None) -> SnapshotNode: if path is None: path = "schema.yml" return SnapshotNode( database=source.database, schema=source.schema, name=name, resource_type=NodeType.Snapshot, package_name=source.package_name, path=path, original_file_path=path, unique_id=f"snapshot.{pkg}.{name}", fqn=[pkg, "snapshot", name], alias=None, checksum="", sources=[[source.source_name, source.name]], depends_on=DependsOn( nodes=[source.unique_id], macros=[], ), config=SnapshotConfig( strategy="check", unique_key="'dummy'", check_cols="all", ), ) @pytest.fixture def macro_test_unique() -> Macro: return make_macro( "dbt", "test_unique", "blablabla", depends_on_macros=["macro.dbt.default__test_unique"] ) @pytest.fixture def macro_default_test_unique() -> Macro: return make_macro("dbt", "default__test_unique", "blablabla") @pytest.fixture def macro_test_not_null() -> Macro: return make_macro( "dbt", "test_not_null", "blablabla", depends_on_macros=["macro.dbt.default__test_not_null"] ) @pytest.fixture def macro_materialization_table_default() -> Macro: macro = make_macro("dbt", "materialization_table_default", "SELECT 1") macro.supported_languages = [ModelLanguage.sql] return macro @pytest.fixture def macro_default_test_not_null() -> Macro: return make_macro("dbt", "default__test_not_null", "blabla") @pytest.fixture def seed() -> SeedNode: return make_seed("pkg", "seed") @pytest.fixture def source() -> SourceDefinition: return make_source("pkg", "raw", "seed", identifier="seed") @pytest.fixture def ephemeral_model(source) -> ModelNode: return make_model( "pkg", "ephemeral_model", 'select * from {{ source("raw", "seed") }}', config_kwargs={"materialized": "ephemeral"}, sources=[source], ) @pytest.fixture def view_model(ephemeral_model) -> ModelNode: return make_model( "pkg", "view_model", 'select * from {{ ref("ephemeral_model") }}', config_kwargs={"materialized": "view"}, refs=[ephemeral_model], tags=["uses_ephemeral"], ) @pytest.fixture def table_model(ephemeral_model) -> ModelNode: return make_model( "pkg", "table_model", 'select * from {{ ref("ephemeral_model") }}', config_kwargs={ "materialized": "table", "meta": { # Other properties to test in test_select_config_meta "string_property": "some_string", "truthy_bool_property": True, "falsy_bool_property": False, "list_property": ["some_value", True, False], }, }, refs=[ephemeral_model], tags=["uses_ephemeral"], path="subdirectory/table_model.sql", ) @pytest.fixture def table_model_py(seed) -> ModelNode: return make_model( "pkg", "table_model_py", 'select * from {{ ref("seed") }}', config_kwargs={"materialized": "table"}, refs=[seed], tags=[], path="subdirectory/table_model.py", ) @pytest.fixture def table_model_csv(seed) -> ModelNode: return make_model( "pkg", "table_model_csv", 'select * from {{ ref("seed") }}', config_kwargs={"materialized": "table"}, refs=[seed], tags=[], path="subdirectory/table_model.csv", ) @pytest.fixture def ext_source() -> SourceDefinition: return make_source( "ext", "ext_raw", "ext_source", ) @pytest.fixture def ext_source_2() -> SourceDefinition: return make_source( "ext", "ext_raw", "ext_source_2", ) @pytest.fixture def ext_source_other() -> SourceDefinition: return make_source( "ext", "raw", "ext_source", ) @pytest.fixture def ext_source_other_2() -> SourceDefinition: return make_source( "ext", "raw", "ext_source_2", ) @pytest.fixture def ext_model(ext_source) -> ModelNode: return make_model( "ext", "ext_model", 'select * from {{ source("ext_raw", "ext_source") }}', sources=[ext_source], ) @pytest.fixture def union_model(seed, ext_source) -> ModelNode: return make_model( "pkg", "union_model", 'select * from {{ ref("seed") }} union all select * from {{ source("ext_raw", "ext_source") }}', config_kwargs={"materialized": "table"}, refs=[seed], sources=[ext_source], fqn_extras=["unions"], path="subdirectory/union_model.sql", tags=["unions"], ) @pytest.fixture def versioned_model_v1(seed) -> ModelNode: return make_model( "pkg", "versioned_model", 'select * from {{ ref("seed") }}', config_kwargs={"materialized": "table"}, refs=[seed], sources=[], path="subdirectory/versioned_model_v1.sql", version=1, latest_version=2, ) @pytest.fixture def versioned_model_v2(seed) -> ModelNode: return make_model( "pkg", "versioned_model", 'select * from {{ ref("seed") }}', config_kwargs={"materialized": "table"}, refs=[seed], sources=[], path="subdirectory/versioned_model_v2.sql", version=2, latest_version=2, ) @pytest.fixture def versioned_model_v3(seed) -> ModelNode: return make_model( "pkg", "versioned_model", 'select * from {{ ref("seed") }}', config_kwargs={"materialized": "table"}, refs=[seed], sources=[], path="subdirectory/versioned_model_v3.sql", version="3", latest_version=2, ) @pytest.fixture def versioned_model_v12_string(seed) -> ModelNode: return make_model( "pkg", "versioned_model", 'select * from {{ ref("seed") }}', config_kwargs={"materialized": "table"}, refs=[seed], sources=[], path="subdirectory/versioned_model_v12.sql", version="12", latest_version=2, ) @pytest.fixture def versioned_model_v4_nested_dir(seed) -> ModelNode: return make_model( "pkg", "versioned_model", 'select * from {{ ref("seed") }}', config_kwargs={"materialized": "table"}, refs=[seed], sources=[], path="subdirectory/nested_dir/versioned_model_v3.sql", version="4", latest_version=2, fqn_extras=["nested_dir"], ) @pytest.fixture def table_id_unique(table_model) -> GenericTestNode: return make_unique_test("pkg", table_model, "id") @pytest.fixture def table_id_not_null(table_model) -> GenericTestNode: return make_not_null_test("pkg", table_model, "id") @pytest.fixture def view_id_unique(view_model) -> GenericTestNode: return make_unique_test("pkg", view_model, "id") @pytest.fixture def ext_source_id_unique(ext_source) -> GenericTestNode: return make_unique_test("ext", ext_source, "id") @pytest.fixture def view_test_nothing(view_model) -> SingularTestNode: return make_singular_test( "pkg", "view_test_nothing", 'select * from {{ ref("view_model") }} limit 0', refs=[view_model], ) @pytest.fixture def unit_test_table_model(table_model) -> UnitTestDefinition: return make_unit_test( "pkg", "unit_test_table_model", table_model, ) # Support dots as namespace separators @pytest.fixture def namespaced_seed() -> SeedNode: return make_seed("pkg", "mynamespace.seed") @pytest.fixture def namespace_model(source) -> ModelNode: return make_model( "pkg", "mynamespace.ephemeral_model", 'select * from {{ source("raw", "seed") }}', config_kwargs={"materialized": "ephemeral"}, sources=[source], ) @pytest.fixture def namespaced_union_model(seed, ext_source) -> ModelNode: return make_model( "pkg", "mynamespace.union_model", 'select * from {{ ref("mynamespace.seed") }} union all select * from {{ ref("mynamespace.ephemeral_model") }}', config_kwargs={"materialized": "table"}, refs=[seed], sources=[ext_source], fqn_extras=["unions"], path="subdirectory/union_model.sql", tags=["unions"], ) @pytest.fixture def metric() -> Metric: return Metric( name="my_metric", resource_type=NodeType.Metric, type=MetricType.SIMPLE, type_params=MetricTypeParams(measure=MetricInputMeasure(name="a_measure")), fqn=["test", "metrics", "myq_metric"], unique_id="metric.test.my_metric", package_name="test", path="models/metric.yml", original_file_path="models/metric.yml", description="", meta={}, tags=[], label="test_label", ) @pytest.fixture def saved_query() -> SavedQuery: pkg = "test" name = "test_saved_query" path = "test_path" return SavedQuery( name=name, resource_type=NodeType.SavedQuery, package_name=pkg, path=path, description="Test Saved Query", query_params=QueryParams( metrics=["my_metric"], group_by=[], where=WhereFilterIntersection( where_filters=[ WhereFilter(where_sql_template="1=1"), ] ), ), exports=[], unique_id=f"saved_query.{pkg}.{name}", original_file_path=path, fqn=[pkg, "saved_queries", name], ) @pytest.fixture def semantic_model(table_model) -> SemanticModel: return make_semantic_model("test", "test_semantic_model", model=table_model) @pytest.fixture def metricflow_time_spine_model() -> ModelNode: return ModelNode( name="metricflow_time_spine", database="dbt", schema="analytics", alias="events", resource_type=NodeType.Model, unique_id="model.test.metricflow_time_spine", fqn=["snowplow", "events"], package_name="snowplow", refs=[], sources=[], metrics=[], depends_on=DependsOn(), config=ModelConfig(), tags=[], path="events.sql", original_file_path="events.sql", meta={}, language="sql", raw_code="does not matter", checksum=FileHash.empty(), relation_name="events", ) @pytest.fixture def nodes( seed, ephemeral_model, view_model, table_model, table_model_py, table_model_csv, union_model, versioned_model_v1, versioned_model_v2, versioned_model_v3, versioned_model_v4_nested_dir, versioned_model_v12_string, ext_model, table_id_unique, table_id_not_null, view_id_unique, ext_source_id_unique, view_test_nothing, namespaced_seed, namespace_model, namespaced_union_model, ) -> List[ManifestNode]: return [ seed, ephemeral_model, view_model, table_model, table_model_py, table_model_csv, union_model, versioned_model_v1, versioned_model_v2, versioned_model_v3, versioned_model_v4_nested_dir, versioned_model_v12_string, ext_model, table_id_unique, table_id_not_null, view_id_unique, ext_source_id_unique, view_test_nothing, namespaced_seed, namespace_model, namespaced_union_model, ] @pytest.fixture def sources( source, ext_source, ext_source_2, ext_source_other, ext_source_other_2, ) -> list: return [source, ext_source, ext_source_2, ext_source_other, ext_source_other_2] @pytest.fixture def macros( macro_test_unique, macro_default_test_unique, macro_test_not_null, macro_default_test_not_null, macro_materialization_table_default, ) -> List[Macro]: return [ macro_test_unique, macro_default_test_unique, macro_test_not_null, macro_default_test_not_null, macro_materialization_table_default, ] @pytest.fixture def unit_tests(unit_test_table_model) -> List[UnitTestDefinition]: return [unit_test_table_model] @pytest.fixture def metrics(metric: Metric) -> List[Metric]: return [metric] @pytest.fixture def semantic_models(semantic_model: SemanticModel) -> List[SemanticModel]: return [semantic_model] @pytest.fixture def saved_queries(saved_query: SavedQuery) -> List[SavedQuery]: return [saved_query] @pytest.fixture def files() -> Dict[str, AnySourceFile]: return {} def make_manifest( disabled: Dict[str, List[GraphMemberNode]] = {}, docs: List[Documentation] = [], exposures: List[Exposure] = [], files: Dict[str, AnySourceFile] = {}, groups: List[Group] = [], macros: List[Macro] = [], metrics: List[Metric] = [], nodes: List[ManifestNode] = [], saved_queries: List[SavedQuery] = [], selectors: Dict[str, Any] = {}, semantic_models: List[SemanticModel] = [], sources: List[SourceDefinition] = [], unit_tests: List[UnitTestDefinition] = [], ) -> Manifest: manifest = Manifest( nodes={n.unique_id: n for n in nodes}, sources={s.unique_id: s for s in sources}, macros={m.unique_id: m for m in macros}, unit_tests={t.unique_id: t for t in unit_tests}, semantic_models={s.unique_id: s for s in semantic_models}, docs={d.unique_id: d for d in docs}, files=files, exposures={e.unique_id: e for e in exposures}, metrics={m.unique_id: m for m in metrics}, disabled=disabled, selectors=selectors, groups={g.unique_id: g for g in groups}, metadata=ManifestMetadata(adapter_type="postgres", project_name="pkg"), saved_queries={s.unique_id: s for s in saved_queries}, ) manifest.build_parent_and_child_maps() return manifest @pytest.fixture def manifest( metric, semantic_model, nodes, sources, macros, unit_tests, metrics, semantic_models, files, saved_queries, ) -> Manifest: return make_manifest( nodes=nodes, sources=sources, macros=macros, unit_tests=unit_tests, semantic_models=semantic_models, files=files, metrics=metrics, saved_queries=saved_queries, ) ================================================ FILE: tests/unit/utils/project.py ================================================ from unittest.mock import MagicMock import pytest from dbt.adapters.contracts.connection import DEFAULT_QUERY_COMMENT, QueryComment from dbt.config import RuntimeConfig from dbt.config.project import Project, RenderComponents, VarProvider from dbt.config.selectors import SelectorConfig from dbt.contracts.project import PackageConfig from dbt_common.semver import VersionSpecifier @pytest.fixture(scope="function") def selector_config() -> SelectorConfig: return SelectorConfig.selectors_from_dict( data={ "selectors": [ { "name": "my_selector", "definition": "give me cats", "default": True, } ] } ) @pytest.fixture(scope="function") def project(selector_config: SelectorConfig) -> Project: return Project( project_name="test_project", version=1.0, project_root="doesnt/actually/exist", profile_name="test_profile", model_paths=["models"], macro_paths=["macros"], seed_paths=["seeds"], function_paths=["functions"], test_paths=["tests"], analysis_paths=["analyses"], docs_paths=["docs"], asset_paths=["assets"], target_path="target", snapshot_paths=["snapshots"], clean_targets=["target"], log_path="path/to/project/logs", packages_install_path="dbt_packages", packages_specified_path="packages.yml", quoting={}, models={}, on_run_start=[], on_run_end=[], dispatch=[{"macro_namespace": "dbt_utils", "search_order": ["test_project", "dbt_utils"]}], seeds={}, snapshots={}, sources={}, data_tests={}, unit_tests={}, metrics={}, semantic_models={}, saved_queries={}, exposures={}, functions={}, vars=VarProvider({}), dbt_version=[VersionSpecifier.from_version_string("0.0.0")], packages=PackageConfig([]), manifest_selectors={}, selectors=selector_config, # QueryComment contract defaults are defined by dbt-adapters, so not hard-coding this fixture to rely on particular settings of their defaults. query_comment=QueryComment(comment=DEFAULT_QUERY_COMMENT, append=False, job_label=False), config_version=1, unrendered=RenderComponents({}, {}, {}), project_env_vars={}, restrict_access=False, dbt_cloud={}, flags={}, vars_from_file={}, ) @pytest.fixture def mock_project(): mock_project = MagicMock(RuntimeConfig) mock_project.cli_vars = {} mock_project.args = MagicMock() mock_project.profile_name = "test" mock_project.target_name = "test" mock_project.args.profile = "test" mock_project.args.target = "test" mock_project.project_env_vars = {} mock_project.profile_env_vars = {} mock_project.project_target_path = "mock_target_path" mock_project.credentials = MagicMock() mock_project.clear_dependencies = MagicMock() mock_project.vars_from_file = {} return mock_project ================================================ FILE: third-party-stubs/agate/__init__.pyi ================================================ from collections.abc import Sequence from typing import Any, Optional, Callable, Iterable, Dict, Union from . import data_types as data_types from .data_types import ( Text as Text, Number as Number, Boolean as Boolean, DateTime as DateTime, Date as Date, TimeDelta as TimeDelta, ) class MappedSequence(Sequence): def __init__(self, values: Any, keys: Optional[Any] = ...) -> None: ... def __unicode__(self): ... def __getitem__(self, key: Any): ... def __setitem__(self, key: Any, value: Any) -> None: ... def __iter__(self): ... def __len__(self): ... def __eq__(self, other: Any): ... def __ne__(self, other: Any): ... def __contains__(self, value: Any): ... def keys(self): ... def values(self): ... def items(self): ... def get(self, key: Any, default: Optional[Any] = ...): ... def dict(self): ... class Row(MappedSequence): ... class Table: def __init__( self, rows: Any, column_names: Optional[Any] = ..., column_types: Optional[Any] = ..., row_names: Optional[Any] = ..., _is_fork: bool = ..., ) -> None: ... def __len__(self): ... def __iter__(self): ... def __getitem__(self, key: Any): ... @property def column_types(self): ... @property def column_names(self): ... @property def row_names(self): ... @property def columns(self): ... @property def rows(self): ... def print_csv(self, **kwargs: Any) -> None: ... def print_json(self, **kwargs: Any) -> None: ... def where(self, test: Callable[[Row], bool]) -> "Table": ... def select(self, key: Union[Iterable[str], str]) -> "Table": ... # these definitions are much narrower than what's actually accepted @classmethod def from_object( cls, obj: Iterable[Dict[str, Any]], *, column_types: Optional["TypeTester"] = None ) -> "Table": ... @classmethod def from_csv( cls, path: Iterable[str], *, column_types: Optional["TypeTester"] = None ) -> "Table": ... @classmethod def merge(cls, tables: Iterable["Table"]) -> "Table": ... def rename( self, column_names: Optional[Iterable[str]] = None, row_names: Optional[Any] = None, slug_columns: bool = False, slug_rows: bool = False, **kwargs: Any, ) -> "Table": ... class TypeTester: def __init__( self, force: Any = ..., limit: Optional[Any] = ..., types: Optional[Any] = ... ) -> None: ... def run(self, rows: Any, column_names: Any): ... class MaxPrecision: def __init__(self, column_name: Any) -> None: ... # this is not strictly true, but it's all we care about. def aggregate(self, aggregations: MaxPrecision) -> int: ... ================================================ FILE: third-party-stubs/agate/data_types.pyi ================================================ from typing import Any, Optional DEFAULT_NULL_VALUES: Any class DataType: null_values: Any = ... def __init__(self, null_values: Any = ...) -> None: ... def test(self, d: Any): ... def cast(self, d: Any) -> None: ... def csvify(self, d: Any): ... def jsonify(self, d: Any): ... DEFAULT_TRUE_VALUES: Any DEFAULT_FALSE_VALUES: Any class Boolean(DataType): true_values: Any = ... false_values: Any = ... def __init__( self, true_values: Any = ..., false_values: Any = ..., null_values: Any = ... ) -> None: ... def cast(self, d: Any): ... def jsonify(self, d: Any): ... ZERO_DT: Any class Date(DataType): date_format: Any = ... parser: Any = ... def __init__(self, date_format: Optional[Any] = ..., **kwargs: Any) -> None: ... def cast(self, d: Any): ... def csvify(self, d: Any): ... def jsonify(self, d: Any): ... class DateTime(DataType): datetime_format: Any = ... timezone: Any = ... def __init__( self, datetime_format: Optional[Any] = ..., timezone: Optional[Any] = ..., **kwargs: Any ) -> None: ... def cast(self, d: Any): ... def csvify(self, d: Any): ... def jsonify(self, d: Any): ... DEFAULT_CURRENCY_SYMBOLS: Any POSITIVE: Any NEGATIVE: Any class Number(DataType): locale: Any = ... currency_symbols: Any = ... group_symbol: Any = ... decimal_symbol: Any = ... def __init__( self, locale: str = ..., group_symbol: Optional[Any] = ..., decimal_symbol: Optional[Any] = ..., currency_symbols: Any = ..., **kwargs: Any, ) -> None: ... def cast(self, d: Any): ... def jsonify(self, d: Any): ... class TimeDelta(DataType): def cast(self, d: Any): ... class Text(DataType): cast_nulls: Any = ... def __init__(self, cast_nulls: bool = ..., **kwargs: Any) -> None: ... def cast(self, d: Any): ... ================================================ FILE: third-party-stubs/cdecimal/__init__.pyi ================================================ class Decimal: pass ================================================ FILE: third-party-stubs/daff/__init__.pyi ================================================ import __builtin__ # type: ignore import builtins import functools from _typeshed import Incomplete builtins = __builtin__ hxunicode: Incomplete hxunichr: Incomplete def hxnext(x): ... hx_cmp_to_key = functools.cmp_to_key hxunicode = str hxrange = range hxunichr = chr unichr = chr unicode = str hx_cmp_to_key = functools.cmp_to_key python_lib_Builtin = builtins String = builtins.str python_lib_Dict = builtins.dict python_lib_Set = builtins.set imap: Incomplete ifilter: Incomplete class _hx_AnonObject: __dict__: Incomplete def __init__(self, fields) -> None: ... def __contains__(self, item) -> bool: ... def __getitem__(self, item): ... def __getattr__(self, name) -> None: ... class Enum: tag: Incomplete index: Incomplete params: Incomplete def __init__(self, tag, index, params) -> None: ... class Alignment: has_removal: Incomplete has_addition: Incomplete index_columns: Incomplete order_cache: Incomplete tb: Incomplete ta: Incomplete map_a2b: Incomplete map_b2a: Incomplete hb: int ha: Incomplete map_count: int reference: Incomplete meta: Incomplete comp: Incomplete order_cache_has_reference: bool ia: int ib: int marked_as_identical: bool def __init__(self) -> None: ... def range(self, ha, hb) -> None: ... def tables(self, ta, tb) -> None: ... def headers(self, ia, ib) -> None: ... def setRowlike(self, flag) -> None: ... def link(self, a, b) -> None: ... def addIndexColumns(self, unit) -> None: ... def getIndexColumns(self): ... def a2b(self, a): ... def b2a(self, b): ... def count(self): ... def toString(self): ... def toOrder(self): ... def addToOrder(self, l, r, p: Incomplete | None = ...) -> None: ... def getSource(self): ... def getTarget(self): ... def getSourceHeader(self): ... def getTargetHeader(self): ... def toOrder3(self): ... def markIdentical(self) -> None: ... def isMarkedAsIdentical(self): ... class CellBuilder: ... class CellInfo: meta: Incomplete rvalue: Incomplete lvalue: Incomplete pvalue: Incomplete conflicted: Incomplete updated: Incomplete pretty_separator: Incomplete separator: Incomplete category_given_tr: Incomplete category: Incomplete pretty_value: Incomplete value: Incomplete raw: Incomplete def __init__(self) -> None: ... def toString(self): ... class Class: ... class ColumnChange: props: Incomplete name: Incomplete prevName: Incomplete def __init__(self) -> None: ... class Table: ... class CombinedTable: meta: Incomplete body: Incomplete t: Incomplete dx: int dy: int core: Incomplete head: Incomplete def __init__(self, t) -> None: ... def all(self): ... def getTable(self): ... def get_width(self): ... def get_height(self): ... def getCell(self, x, y): ... def setCell(self, x, y, c) -> None: ... def toString(self): ... def getCellView(self): ... def isResizable(self): ... def resize(self, w, h): ... def clear(self) -> None: ... def insertOrDeleteRows(self, fate, hfate): ... def insertOrDeleteColumns(self, fate, wfate): ... def trimBlank(self): ... def getData(self) -> None: ... def clone(self): ... def create(self): ... def getMeta(self): ... class CombinedTableBody: meta: Incomplete parent: Incomplete dx: Incomplete dy: Incomplete all: Incomplete def __init__(self, parent, dx, dy) -> None: ... def getTable(self): ... def get_width(self): ... def get_height(self): ... def getCell(self, x, y): ... def setCell(self, x, y, c) -> None: ... def toString(self): ... def getCellView(self): ... def isResizable(self): ... def resize(self, w, h): ... def clear(self) -> None: ... def insertOrDeleteRows(self, fate, hfate): ... def insertOrDeleteColumns(self, fate, wfate): ... def trimBlank(self): ... def getData(self) -> None: ... def clone(self): ... def create(self): ... def getMeta(self): ... class CombinedTableHead: parent: Incomplete dx: Incomplete dy: Incomplete all: Incomplete def __init__(self, parent, dx, dy) -> None: ... def getTable(self): ... def get_width(self): ... def get_height(self): ... def getCell(self, x, y): ... def setCell(self, x, y, c) -> None: ... def toString(self): ... def getCellView(self): ... def isResizable(self): ... def resize(self, w, h): ... def clear(self) -> None: ... def insertOrDeleteRows(self, fate, hfate): ... def insertOrDeleteColumns(self, fate, wfate): ... def trimBlank(self): ... def getData(self) -> None: ... def clone(self) -> None: ... def create(self) -> None: ... def getMeta(self) -> None: ... class CompareFlags: padding_strategy: Incomplete ordered: bool show_unchanged: bool unchanged_context: int always_show_order: bool never_show_order: bool show_unchanged_columns: bool unchanged_column_context: int always_show_header: bool acts: Incomplete ids: Incomplete columns_to_ignore: Incomplete allow_nested_cells: bool warnings: Incomplete diff_strategy: Incomplete show_meta: bool show_unchanged_meta: bool tables: Incomplete parent: Incomplete count_like_a_spreadsheet: bool ignore_whitespace: bool ignore_case: bool ignore_epsilon: int terminal_format: Incomplete use_glyphs: bool quote_html: bool def __init__(self) -> None: ... def filter(self, act, allow): ... def allowUpdate(self): ... def allowInsert(self): ... def allowDelete(self): ... def allowColumn(self): ... def getIgnoredColumns(self): ... def addPrimaryKey(self, column) -> None: ... def ignoreColumn(self, column) -> None: ... def addTable(self, table) -> None: ... def addWarning(self, warn) -> None: ... def getWarning(self): ... def getNameByRole(self, name, role): ... def getCanonicalName(self, name): ... def getIdsByRole(self, role): ... class CompareTable: indexes: Incomplete comp: Incomplete def __init__(self, comp) -> None: ... def run(self): ... def align(self): ... def getComparisonState(self): ... def alignCore(self, align) -> None: ... def alignCore2(self, align, a, b): ... def alignColumns(self, align, a, b) -> None: ... def testHasSameColumns(self): ... def hasSameColumns2(self, a, b): ... def testIsEqual(self): ... def isEqual2(self, a, b): ... def compareCore(self): ... def storeIndexes(self) -> None: ... def getIndexes(self): ... def useSql(self): ... class ConflictInfo: row: Incomplete col: Incomplete pvalue: Incomplete lvalue: Incomplete rvalue: Incomplete def __init__(self, row, col, pvalue, lvalue, rvalue) -> None: ... class Coopy: daff_cmd: Incomplete status: Incomplete mv: Incomplete diffs_found: Incomplete fail_if_diff: Incomplete cache_txt: Incomplete flags: Incomplete fragment: Incomplete css_output: Incomplete strategy: Incomplete io: Incomplete order_preference: Incomplete order_set: Incomplete nested_output: Incomplete output_format_set: Incomplete output_format: Incomplete extern_preference: Incomplete csv_eol_preference: Incomplete delim_preference: Incomplete format_preference: Incomplete def __init__(self, io: Incomplete | None = ...) -> None: ... def init(self) -> None: ... def checkFormat(self, name): ... def setFormat(self, name) -> None: ... def getRenderer(self): ... def applyRenderer(self, name, renderer): ... def renderTable(self, name, t): ... def renderTables(self, name, t): ... def saveTable(self, name, t, render: Incomplete | None = ...): ... def encodeTable(self, name, t, render: Incomplete | None = ...): ... def saveTables(self, name, os, use_color, is_diff): ... def saveText(self, name, txt): ... def jsonToTables(self, json): ... def jsonToTable(self, json): ... def useColor(self, flags, output): ... def runDiff(self, parent, a, b, flags, output) -> None: ... def loadTable(self, name, role): ... def command(self, io, cmd, args): ... def installGitDriver(self, io, formats): ... def run(self, args, io: Incomplete | None = ...): ... def coopyhx(self, io): ... @staticmethod def diffAsHtml(local, remote, flags: Incomplete | None = ...): ... @staticmethod def diffAsAnsi(local, remote, flags: Incomplete | None = ...): ... @staticmethod def diff(local, remote, flags: Incomplete | None = ...): ... @staticmethod def getBlankTable(td, comp): ... @staticmethod def align(local, remote, flags, comp): ... @staticmethod def patch(local, patch, flags: Incomplete | None = ...): ... @staticmethod def compareTables(local, remote, flags: Incomplete | None = ...): ... @staticmethod def compareTables3(parent, local, remote, flags: Incomplete | None = ...): ... @staticmethod def keepAround(): ... @staticmethod def cellFor(x): ... @staticmethod def main(): ... @staticmethod def show(t) -> None: ... @staticmethod def jsonify(t): ... @staticmethod def tablify(data): ... class CrossMatch: item_b: Incomplete item_a: Incomplete spot_b: Incomplete spot_a: Incomplete def __init__(self) -> None: ... class Csv: has_structure: Incomplete cursor: int row_ended: bool delim: Incomplete discovered_eol: Incomplete preferred_eol: Incomplete def __init__(self, delim: Incomplete | None = ..., eol: Incomplete | None = ...) -> None: ... def renderTable(self, t): ... def renderCell(self, v, d, force_quote: Incomplete | None = ...): ... def parseTable(self, txt, tab): ... def makeTable(self, txt): ... def parseCellPart(self, txt): ... def parseCell(self, txt): ... def getDiscoveredEol(self): ... def setPreferredEol(self, eol) -> None: ... class Date: dateUTC: Incomplete date: Incomplete def __init__(self, year, month, day, hour, _hx_min, sec) -> None: ... def toString(self): ... @staticmethod def makeLocal(date): ... class DiffRender: section: Incomplete td_close: Incomplete td_open: Incomplete text_to_insert: Incomplete open: bool pretty_arrows: bool quote_html: bool def __init__(self) -> None: ... def usePrettyArrows(self, flag) -> None: ... def quoteHtml(self, flag) -> None: ... def insert(self, _hx_str) -> None: ... def beginTable(self) -> None: ... def setSection(self, _hx_str) -> None: ... def beginRow(self, mode) -> None: ... def insertCell(self, txt, mode) -> None: ... def endRow(self) -> None: ... def endTable(self) -> None: ... def html(self): ... def toString(self): ... def render(self, tab): ... def renderTables(self, tabs): ... def sampleCss(self): ... def completeHtml(self) -> None: ... @staticmethod def examineCell( x, y, view, raw, vcol, vrow, vcorner, cell, offset: Incomplete | None = ... ): ... @staticmethod def markSpaces(sl, sr): ... @staticmethod def renderCell(tab, view, x, y): ... class DiffSummary: different: Incomplete col_count_final: Incomplete col_count_initial: Incomplete row_count_final: Incomplete row_count_initial: Incomplete row_count_final_with_header: Incomplete row_count_initial_with_header: Incomplete col_reorders: Incomplete col_renames: Incomplete col_updates: Incomplete col_inserts: Incomplete col_deletes: Incomplete row_reorders: Incomplete row_updates: Incomplete row_inserts: Incomplete row_deletes: Incomplete def __init__(self) -> None: ... class FlatCellBuilder: conflict_separator: Incomplete separator: Incomplete view: Incomplete flags: Incomplete def __init__(self, flags) -> None: ... def needSeparator(self): ... def setSeparator(self, separator) -> None: ... def setConflictSeparator(self, separator) -> None: ... def setView(self, view) -> None: ... def update(self, local, remote): ... def conflict(self, parent, local, remote): ... def marker(self, label): ... def links(self, unit, row_like): ... @staticmethod def quoteForDiff(v, d): ... class Row: ... class HighlightPatch: finished_columns: Incomplete next_meta: Incomplete prev_meta: Incomplete process_meta: Incomplete meta_change: Incomplete preambleRow: Incomplete headerRow: Incomplete haveDroppedColumns: Incomplete colPermutationRev: Incomplete colPermutation: Incomplete rowPermutationRev: Incomplete rowPermutation: Incomplete actions: Incomplete lastSourceRow: Incomplete patchInSourceRow: Incomplete patchInDestCol: Incomplete destInPatchCol: Incomplete patchInSourceCol: Incomplete sourceInPatchCol: Incomplete indexes: Incomplete rcOffset: Incomplete cellInfo: Incomplete rowInfo: Incomplete cmods: Incomplete mods: Incomplete payloadTop: Incomplete payloadCol: Incomplete currentRow: Incomplete modifier: Incomplete headerMove: Incomplete headerRename: Incomplete headerPost: Incomplete headerPre: Incomplete header: Incomplete csv: Incomplete source: Incomplete patch: Incomplete flags: Incomplete view: Incomplete sourceView: Incomplete meta: Incomplete def __init__(self, source, patch, flags: Incomplete | None = ...) -> None: ... def reset(self): ... def processMeta(self) -> None: ... def apply(self): ... def needSourceColumns(self) -> None: ... def needDestColumns(self) -> None: ... def needSourceIndex(self) -> None: ... def setMetaProp(self, target, column_name, prop_name, value) -> None: ... def applyMetaRow(self, code) -> None: ... def applyRow(self, r): ... def getDatum(self, c): ... def getString(self, c): ... def getStringNull(self, c): ... def applyMeta(self) -> None: ... def applyHeader(self) -> None: ... def lookUp(self, _hx_del: Incomplete | None = ...): ... def applyActionExternal(self, code) -> None: ... def applyAction(self, code): ... def checkAct(self) -> None: ... def getPreString(self, txt): ... def getRowString(self, c): ... def isPreamble(self): ... def sortMods(self, a, b): ... def processMods(self, rmods, fate, _hx_len): ... def useMetaForColumnChanges(self): ... def useMetaForRowChanges(self): ... def computeOrdering(self, mods, permutation, permutationRev, dim) -> None: ... def permuteRows(self) -> None: ... def fillInNewColumns(self) -> None: ... def finishRows(self) -> None: ... def permuteColumns(self) -> None: ... def finishColumns(self): ... class HighlightPatchUnit: add: bool rem: bool update: bool sourceRow: int sourceRowOffset: int sourcePrevRow: int sourceNextRow: int destRow: int patchRow: int code: str def __init__(self) -> None: ... def toString(self): ... class Index: indexed_table: Incomplete v: Incomplete items: Incomplete cols: Incomplete keys: Incomplete top_freq: int height: int hdr: int ignore_whitespace: bool ignore_case: bool def __init__(self, flags) -> None: ... def addColumn(self, i) -> None: ... def indexTable(self, t, hdr) -> None: ... def toKey(self, t, i): ... def toKeyByContent(self, row): ... def getTable(self): ... class IndexItem: lst: Incomplete def __init__(self) -> None: ... def add(self, i): ... def length(self): ... def value(self): ... def asList(self): ... class IndexPair: flags: Incomplete ia: Incomplete ib: Incomplete quality: int hdr: int def __init__(self, flags) -> None: ... def addColumns(self, ca, cb) -> None: ... def indexTables(self, a, b, hdr) -> None: ... def queryByKey(self, ka): ... def queryByContent(self, row): ... def queryLocal(self, row): ... def localKey(self, row): ... def remoteKey(self, row): ... def getTopFreq(self): ... def getQuality(self): ... class Meta: ... class JsonTable: name: Incomplete idx2col: Incomplete h: Incomplete w: Incomplete data: Incomplete columns: Incomplete rows: Incomplete def __init__(self, data, name) -> None: ... def getTable(self): ... def get_width(self): ... def get_height(self): ... def getCell(self, x, y): ... def setCell(self, x, y, c) -> None: ... def toString(self): ... def getCellView(self): ... def isResizable(self): ... def resize(self, w, h): ... def clear(self) -> None: ... def insertOrDeleteRows(self, fate, hfate): ... def insertOrDeleteColumns(self, fate, wfate): ... def trimBlank(self): ... def getData(self) -> None: ... def clone(self) -> None: ... def setMeta(self, meta) -> None: ... def getMeta(self): ... def create(self) -> None: ... def alterColumns(self, columns): ... def changeRow(self, rc): ... def applyFlags(self, flags): ... def asTable(self) -> None: ... def cloneMeta(self, table: Incomplete | None = ...) -> None: ... def useForColumnChanges(self): ... def useForRowChanges(self): ... def getRowStream(self) -> None: ... def isNested(self): ... def isSql(self): ... def getName(self): ... class JsonTables: flags: Incomplete db: Incomplete t: Incomplete def __init__(self, json, flags) -> None: ... def getCell(self, x, y): ... def setCell(self, x, y, c) -> None: ... def getCellView(self): ... def isResizable(self): ... def resize(self, w, h): ... def clear(self) -> None: ... def insertOrDeleteRows(self, fate, hfate): ... def insertOrDeleteColumns(self, fate, wfate): ... def trimBlank(self): ... def get_width(self): ... def get_height(self): ... def getData(self) -> None: ... def clone(self) -> None: ... def getMeta(self): ... def create(self) -> None: ... class Lambda: @staticmethod def array(it): ... @staticmethod def has(it, elt): ... class Merger: conflict_infos: Incomplete conflicts: Incomplete column_mix_remote: Incomplete column_mix_local: Incomplete row_mix_remote: Incomplete row_mix_local: Incomplete column_units: Incomplete column_order: Incomplete units: Incomplete order: Incomplete parent: Incomplete local: Incomplete remote: Incomplete flags: Incomplete def __init__(self, parent, local, remote, flags) -> None: ... def shuffleDimension(self, dim_units, _hx_len, fate, cl, cr): ... def shuffleColumns(self) -> None: ... def shuffleRows(self) -> None: ... def apply(self): ... def getConflictInfos(self): ... def addConflictInfo(self, row, col, view, pcell, lcell, rcell) -> None: ... @staticmethod def makeConflictedCell(view, pcell, lcell, rcell): ... class Mover: @staticmethod def moveUnits(units): ... @staticmethod def move(isrc, idest): ... @staticmethod def moveWithoutExtras(src, dest): ... class Ndjson: columns: Incomplete tab: Incomplete view: Incomplete header_row: int def __init__(self, tab) -> None: ... def renderRow(self, r): ... def render(self): ... def addRow(self, r, txt) -> None: ... def addHeaderRow(self, r) -> None: ... def parse(self, txt) -> None: ... class NestedCellBuilder: view: Incomplete def __init__(self) -> None: ... def needSeparator(self): ... def setSeparator(self, separator) -> None: ... def setConflictSeparator(self, separator) -> None: ... def setView(self, view) -> None: ... def update(self, local, remote): ... def conflict(self, parent, local, remote): ... def marker(self, label): ... def negToNull(self, x): ... def links(self, unit, row_like): ... class Ordering: order: Incomplete ignore_parent: bool def __init__(self) -> None: ... def add(self, l, r, p: Incomplete | None = ...) -> None: ... def getList(self): ... def setList(self, lst) -> None: ... def toString(self): ... def ignoreParent(self) -> None: ... class PropertyChange: val: Incomplete name: Incomplete prevName: Incomplete def __init__(self) -> None: ... class Reflect: @staticmethod def field(o, field): ... @staticmethod def isFunction(f): ... @staticmethod def compare(a, b): ... class RowChange: action: Incomplete is_key: Incomplete conflicted: Incomplete conflicting_parent_val: Incomplete conflicting_val: Incomplete val: Incomplete cond: Incomplete def __init__(self) -> None: ... def showMap(self, m): ... def toString(self): ... class RowStream: ... class SimpleMeta: may_be_nested: Incomplete row_change_cache: Incomplete row_active: Incomplete keys: Incomplete metadata: Incomplete has_properties: Incomplete name2col: Incomplete name2row: Incomplete t: Incomplete def __init__( self, t, has_properties: Incomplete | None = ..., may_be_nested: Incomplete | None = ... ) -> None: ... def storeRowChanges(self, changes) -> None: ... def rowChange(self) -> None: ... def colChange(self) -> None: ... def col(self, key): ... def row(self, key): ... def alterColumns(self, columns): ... def setCell(self, c, r, val): ... def addMetaData(self, column, property, val) -> None: ... def asTable(self): ... def cloneMeta(self, table: Incomplete | None = ...): ... def useForColumnChanges(self): ... def useForRowChanges(self): ... def changeRow(self, rc): ... def applyFlags(self, flags): ... def getRowStream(self): ... def isNested(self): ... def isSql(self): ... def getName(self) -> None: ... class SimpleTable: data: Incomplete w: Incomplete h: Incomplete meta: Incomplete def __init__(self, w, h) -> None: ... def getTable(self): ... def get_width(self): ... def get_height(self): ... def getCell(self, x, y): ... def setCell(self, x, y, c) -> None: ... def toString(self): ... def getCellView(self): ... def isResizable(self): ... def resize(self, w, h): ... def clear(self) -> None: ... def insertOrDeleteRows(self, fate, hfate): ... def insertOrDeleteColumns(self, fate, wfate): ... def trimBlank(self): ... def getData(self) -> None: ... def clone(self): ... def create(self): ... def setMeta(self, meta) -> None: ... def getMeta(self): ... @staticmethod def tableToString(tab): ... @staticmethod def tableIsSimilar(tab1, tab2): ... class View: ... class SimpleView: def __init__(self) -> None: ... def toString(self, d): ... def equals(self, d1, d2): ... def toDatum(self, x): ... def makeHash(self): ... def hashSet(self, h, _hx_str, d) -> None: ... def hashExists(self, h, _hx_str): ... def hashGet(self, h, _hx_str): ... def isHash(self, h): ... def isTable(self, t): ... def getTable(self, t): ... def wrapTable(self, t): ... class SparseSheet: zero: Incomplete row: Incomplete w: int h: Incomplete def __init__(self) -> None: ... def resize(self, w, h, zero) -> None: ... def nonDestructiveResize(self, w, h, zero) -> None: ... def get(self, x, y): ... def set(self, x, y, val) -> None: ... class SqlColumn: name: str primary: bool type_value: Incomplete type_family: Incomplete def __init__(self) -> None: ... def setName(self, name) -> None: ... def setPrimaryKey(self, primary) -> None: ... def setType(self, value, family) -> None: ... def getName(self): ... def isPrimaryKey(self): ... def toString(self): ... class SqlCompare: needed: Incomplete alt_peered: Incomplete peered: Incomplete diff_ct: Incomplete at2: Incomplete at1: Incomplete at0: Incomplete db: Incomplete local: Incomplete remote: Incomplete alt: Incomplete align: Incomplete flags: Incomplete def __init__( self, db, local, remote, alt, align: Incomplete | None = ..., flags: Incomplete | None = ..., ) -> None: ... def equalArray(self, a1, a2): ... def validateSchema(self): ... def denull(self, x): ... def link(self) -> None: ... def linkQuery(self, query, order) -> None: ... def where(self, txt): ... def scanColumns(self, all_cols1, all_cols2, key_cols, present1, present2, align) -> None: ... def apply(self): ... class SqlDatabase: ... class SqlHelper: ... class SqlTable: columnNames: Incomplete quotedTableName: Incomplete columns: Incomplete db: Incomplete name: Incomplete helper: Incomplete cache: Incomplete h: int id2rid: Incomplete def __init__(self, db, name, helper: Incomplete | None = ...) -> None: ... def getColumns(self) -> None: ... def getPrimaryKey(self): ... def getAllButPrimaryKey(self): ... def getColumnNames(self): ... def getQuotedTableName(self): ... def getQuotedColumnName(self, name): ... def getCell(self, x, y): ... def setCellCache(self, x, y, c) -> None: ... def setCell(self, x, y, c) -> None: ... def getCellView(self): ... def isResizable(self): ... def resize(self, w, h): ... def clear(self) -> None: ... def insertOrDeleteRows(self, fate, hfate): ... def insertOrDeleteColumns(self, fate, wfate): ... def trimBlank(self): ... def get_width(self): ... def get_height(self): ... def getData(self) -> None: ... def clone(self) -> None: ... def create(self) -> None: ... def getMeta(self): ... def alterColumns(self, columns): ... def changeRow(self, rc): ... def asTable(self): ... def useForColumnChanges(self): ... def useForRowChanges(self): ... def cloneMeta(self, table: Incomplete | None = ...) -> None: ... def applyFlags(self, flags): ... def getDatabase(self): ... def getRowStream(self): ... def isNested(self): ... def isSql(self): ... def fetchRow(self): ... def fetchColumns(self): ... def getName(self): ... class SqlTableName: name: Incomplete prefix: Incomplete def __init__(self, name: Incomplete | None = ..., prefix: Incomplete | None = ...) -> None: ... def toString(self): ... class SqlTables: flags: Incomplete t: Incomplete db: Incomplete def __init__(self, db, flags, role) -> None: ... def getCell(self, x, y): ... def setCell(self, x, y, c) -> None: ... def getCellView(self): ... def isResizable(self): ... def resize(self, w, h): ... def clear(self) -> None: ... def insertOrDeleteRows(self, fate, hfate): ... def insertOrDeleteColumns(self, fate, wfate): ... def trimBlank(self): ... def get_width(self): ... def get_height(self): ... def getData(self) -> None: ... def clone(self) -> None: ... def create(self) -> None: ... def getMeta(self): ... class SqliteHelper: def __init__(self) -> None: ... def getTableNames(self, db): ... def countRows(self, db, name): ... def getRowIDs(self, db, name): ... def update(self, db, name, conds, vals): ... def delete(self, db, name, conds): ... def insert(self, db, name, vals): ... def attach(self, db, tag, resource_name): ... def columnListSql(self, x): ... def fetchSchema(self, db, name): ... def splitSchema(self, db, name, sql): ... def alterColumns(self, db, name, columns): ... class Std: @staticmethod def isOfType(v, t): ... @staticmethod def string(s): ... @staticmethod def parseInt(x): ... @staticmethod def shortenPossibleNumber(x): ... @staticmethod def parseFloat(x): ... class Float: ... class Int: ... class Bool: ... class Dynamic: ... class StringBuf: b: Incomplete def __init__(self) -> None: ... def get_length(self): ... class StringTools: @staticmethod def htmlEscape(s, quotes: Incomplete | None = ...): ... @staticmethod def isSpace(s, pos): ... @staticmethod def ltrim(s): ... @staticmethod def rtrim(s): ... @staticmethod def trim(s): ... @staticmethod def lpad(s, c, l): ... @staticmethod def replace(s, sub, by): ... class sys_FileSystem: @staticmethod def exists(path): ... class haxe_IMap: ... class haxe_ds_StringMap: h: Incomplete def __init__(self) -> None: ... def keys(self): ... def iterator(self): ... class python_HaxeIterator: checked: bool has: bool x: Incomplete it: Incomplete def __init__(self, it) -> None: ... def __next__(self): ... def next(self): ... def hasNext(self): ... class Sys: @staticmethod def exit(code) -> None: ... @staticmethod def args(): ... @staticmethod def getEnv(s): ... @staticmethod def command(cmd, args: Incomplete | None = ...): ... @staticmethod def stdout(): ... @staticmethod def stderr(): ... class TableComparisonState: child_order: Incomplete children: Incomplete alignment: Incomplete b_meta: Incomplete a_meta: Incomplete p_meta: Incomplete compare_flags: Incomplete has_same_columns_known: Incomplete has_same_columns: Incomplete is_equal_known: Incomplete is_equal: Incomplete run_to_completion: Incomplete completed: Incomplete b: Incomplete a: Incomplete p: Incomplete def __init__(self) -> None: ... def reset(self) -> None: ... def getMeta(self) -> None: ... class TableDiff: nesting_present: Incomplete nested: Incomplete column_units_updated: Incomplete col_reorders: Incomplete col_renames: Incomplete col_updates: Incomplete col_inserts: Incomplete col_deletes: Incomplete row_reorders: Incomplete row_updates: Incomplete row_inserts: Incomplete row_deletes: Incomplete schema_diff_found: Incomplete diff_found: Incomplete publish: Incomplete act: Incomplete have_addition: Incomplete top_line_done: Incomplete have_schema: Incomplete schema: Incomplete conflict_sep: Incomplete sep: Incomplete v: Incomplete allow_column: Incomplete allow_update: Incomplete allow_delete: Incomplete allow_insert: Incomplete active_column: Incomplete active_row: Incomplete col_moves: Incomplete row_moves: Incomplete show_rc_numbers: Incomplete column_units: Incomplete row_units: Incomplete order: Incomplete is_index_b: Incomplete is_index_a: Incomplete is_index_p: Incomplete rb_header: Incomplete ra_header: Incomplete rp_header: Incomplete p: Incomplete b: Incomplete a: Incomplete has_parent: Incomplete col_map: Incomplete row_map: Incomplete align: Incomplete flags: Incomplete builder: Incomplete preserve_columns: bool def __init__(self, align, flags) -> None: ... def setCellBuilder(self, builder) -> None: ... def getSeparator(self, t, t2, root): ... def isReordered(self, m, ct): ... def spreadContext(self, units, _hx_del, active) -> None: ... def setIgnore(self, ignore, idx_ignore, tab, r_header) -> None: ... def countActive(self, active): ... def reset(self): ... def setupTables(self) -> None: ... def scanActivity(self) -> None: ... def setupColumns(self) -> None: ... def setupMoves(self) -> None: ... def scanSchema(self) -> None: ... def checkRcNumbers(self, w, h) -> None: ... def addRcNumbers(self, output): ... def elideColumns(self, output, admin_w) -> None: ... def addSchema(self, output) -> None: ... def addHeader(self, output) -> None: ... def checkMeta(self, t, meta): ... def getMetaTable(self, t): ... def addMeta(self, output): ... def refineActivity(self) -> None: ... def normalizeString(self, v, _hx_str): ... def isEqual(self, v, aa, bb): ... def checkNesting(self, v, have_ll, ll, have_rr, rr, have_pp, pp, x, y): ... def scanRow(self, unit, output, at, i, out) -> None: ... def hilite(self, output): ... def hiliteSingle(self, output): ... def hiliteWithNesting(self, output): ... def hasDifference(self): ... def hasSchemaDifference(self): ... def isNested(self): ... def getComparisonState(self): ... def getSummary(self): ... class TableIO: def __init__(self) -> None: ... def valid(self): ... def getContent(self, name): ... def saveContent(self, name, txt): ... def args(self): ... def writeStdout(self, txt) -> None: ... def writeStderr(self, txt) -> None: ... def command(self, cmd, args): ... def hasAsync(self): ... def exists(self, path): ... def isTtyKnown(self): ... def isTty(self): ... def openSqliteDatabase(self, path): ... def sendToBrowser(self, html) -> None: ... class TableModifier: t: Incomplete def __init__(self, t) -> None: ... def removeColumn(self, at): ... class TableStream: row: Incomplete columns: Incomplete t: Incomplete at: int h: Incomplete src: Incomplete def __init__(self, t) -> None: ... def fetchColumns(self): ... def fetchRow(self): ... def fetch(self): ... def getCell(self, x): ... def width(self): ... class Tables: alignment: Incomplete template: Incomplete tables: Incomplete table_order: Incomplete def __init__(self, template) -> None: ... def add(self, name): ... def getOrder(self): ... def get(self, name): ... def one(self): ... def hasInsDel(self): ... class TerminalDiffRender: v: Incomplete csv: Incomplete t: Incomplete codes: Incomplete align_columns: bool wide_columns: bool use_glyphs: bool flags: Incomplete delim: Incomplete diff: Incomplete def __init__( self, flags: Incomplete | None = ..., delim: Incomplete | None = ..., diff: Incomplete | None = ..., ) -> None: ... def alignColumns(self, enable) -> None: ... def render(self, t): ... def getText(self, x, y, color): ... def pickSizes(self, t): ... class ValueType(Enum): @staticmethod def TClass(c): ... @staticmethod def TEnum(e): ... class Type: @staticmethod def getClass(o): ... @staticmethod def typeof(v): ... class Unit: l: Incomplete r: Incomplete p: Incomplete def __init__( self, l: Incomplete | None = ..., r: Incomplete | None = ..., p: Incomplete | None = ... ) -> None: ... def lp(self): ... def toString(self): ... def fromString(self, txt): ... def base26(self, num): ... def toBase26String(self): ... @staticmethod def describe(i): ... class Viterbi: path: Incomplete src: Incomplete cost: Incomplete best_cost: Incomplete path_valid: Incomplete mode: Incomplete index: Incomplete T: int K: Incomplete def __init__(self) -> None: ... def reset(self) -> None: ... def setSize(self, states, sequence_length) -> None: ... def assertMode(self, next) -> None: ... def addTransition(self, s0, s1, c) -> None: ... def endTransitions(self) -> None: ... def beginTransitions(self) -> None: ... def calculatePath(self) -> None: ... def toString(self): ... def length(self): ... def get(self, i): ... def getCost(self): ... class haxe_Exception(Exception): def __init__( self, message, previous: Incomplete | None = ..., native: Incomplete | None = ... ) -> None: ... def unwrap(self): ... def get_native(self): ... @staticmethod def caught(value): ... @staticmethod def thrown(value): ... class haxe_NativeStackTrace: @staticmethod def saveStack(exception) -> None: ... @staticmethod def exceptionStack(): ... class haxe_ValueException(haxe_Exception): value: Incomplete def __init__( self, value, previous: Incomplete | None = ..., native: Incomplete | None = ... ) -> None: ... def unwrap(self): ... class haxe_ds_IntMap: h: Incomplete def __init__(self) -> None: ... def set(self, key, value) -> None: ... def remove(self, key): ... def keys(self): ... def toString(self): ... class haxe_format_JsonPrinter: replacer: Incomplete indent: Incomplete pretty: Incomplete nind: int buf: Incomplete def __init__(self, replacer, space) -> None: ... def write(self, k, v) -> None: ... def classString(self, v) -> None: ... def fieldsString(self, v, fields) -> None: ... def quote(self, s) -> None: ... @staticmethod def print(o, replacer: Incomplete | None = ..., space: Incomplete | None = ...): ... class haxe_io_Bytes: length: Incomplete b: Incomplete def __init__(self, length, b) -> None: ... @staticmethod def ofString(s, encoding: Incomplete | None = ...): ... class haxe_io_Encoding(Enum): ... class haxe_io_Error(Enum): @staticmethod def Custom(e): ... class haxe_io_Output: def writeByte(self, c) -> None: ... def writeBytes(self, s, pos, _hx_len): ... bigEndian: Incomplete def set_bigEndian(self, b): ... def writeFullBytes(self, s, pos, _hx_len) -> None: ... def writeString(self, s, encoding: Incomplete | None = ...) -> None: ... class haxe_iterators_ArrayIterator: current: int array: Incomplete def __init__(self, array) -> None: ... def hasNext(self): ... def __next__(self): ... def next(self): ... class haxe_iterators_ArrayKeyValueIterator: current: int array: Incomplete def __init__(self, array) -> None: ... def hasNext(self): ... def __next__(self): ... def next(self): ... class python_Boot: @staticmethod def toString1(o, s): ... @staticmethod def fields(o): ... @staticmethod def simpleField(o, field): ... @staticmethod def hasField(o, field): ... @staticmethod def field(o, field): ... @staticmethod def getInstanceFields(c): ... @staticmethod def getSuperClass(c): ... @staticmethod def getClassFields(c): ... @staticmethod def unhandleKeywords(name): ... class python__KwArgs_KwArgs_Impl_: @staticmethod def fromT(d): ... class python_Lib: @staticmethod def dictToAnon(v): ... @staticmethod def anonToDict(o): ... @staticmethod def anonAsDict(o): ... class python_internal_ArrayImpl: @staticmethod def concat(a1, a2): ... @staticmethod def copy(x): ... @staticmethod def iterator(x): ... @staticmethod def keyValueIterator(x): ... @staticmethod def indexOf(a, x, fromIndex: Incomplete | None = ...): ... @staticmethod def lastIndexOf(a, x, fromIndex: Incomplete | None = ...): ... @staticmethod def join(x, sep): ... @staticmethod def toString(x): ... @staticmethod def pop(x): ... @staticmethod def push(x, e): ... @staticmethod def unshift(x, e) -> None: ... @staticmethod def remove(x, e): ... @staticmethod def contains(x, e): ... @staticmethod def shift(x): ... @staticmethod def slice(x, pos, end: Incomplete | None = ...): ... @staticmethod def sort(x, f) -> None: ... @staticmethod def splice(x, pos, _hx_len): ... @staticmethod def map(x, f): ... @staticmethod def filter(x, f): ... @staticmethod def insert(a, pos, x) -> None: ... @staticmethod def reverse(a) -> None: ... class HxOverrides: @staticmethod def iterator(x): ... @staticmethod def eq(a, b): ... @staticmethod def stringOrNull(s): ... @staticmethod def modf(a, b): ... @staticmethod def mod(a, b): ... @staticmethod def mapKwArgs(a, v): ... class python_internal_MethodClosure: obj: Incomplete func: Incomplete def __init__(self, obj, func) -> None: ... def __call__(self, *args): ... class HxString: @staticmethod def split(s, d): ... @staticmethod def charCodeAt(s, index): ... @staticmethod def charAt(s, index): ... @staticmethod def lastIndexOf(s, _hx_str, startIndex: Incomplete | None = ...): ... @staticmethod def toUpperCase(s): ... @staticmethod def toLowerCase(s): ... @staticmethod def indexOf(s, _hx_str, startIndex: Incomplete | None = ...): ... @staticmethod def indexOfImpl(s, _hx_str, startIndex): ... @staticmethod def toString(s): ... @staticmethod def substring(s, startIndex, endIndex: Incomplete | None = ...): ... @staticmethod def substr(s, startIndex, _hx_len: Incomplete | None = ...): ... class python_io_NativeOutput(haxe_io_Output): stream: Incomplete def __init__(self, stream) -> None: ... class python_io_IOutput: ... class python_io_IFileOutput: ... class python_io_NativeTextOutput(python_io_NativeOutput): def __init__(self, stream) -> None: ... def writeBytes(self, s, pos, _hx_len): ... def writeByte(self, c) -> None: ... class python_io_FileTextOutput(python_io_NativeTextOutput): def __init__(self, stream) -> None: ... class python_io_IoTools: @staticmethod def createFileOutputFromText(t): ... class sys_io_File: @staticmethod def getContent(path): ... @staticmethod def saveContent(path, content) -> None: ... class sys_io_FileOutput(haxe_io_Output): impl: Incomplete def __init__(self, impl) -> None: ... def set_bigEndian(self, b): ... def writeByte(self, c) -> None: ... def writeBytes(self, s, pos, _hx_len): ... def writeFullBytes(self, s, pos, _hx_len) -> None: ... def writeString(self, s, encoding: Incomplete | None = ...) -> None: ... class PythonCellView(View): def __init__(self) -> None: ... def toString(self, d): ... def equals(self, d1, d2): ... def toDatum(self, d): ... def makeHash(self): ... def isHash(self, d): ... def hashSet(self, d, k, v) -> None: ... def hashGet(self, d, k): ... def hashExists(self, d, k): ... class PythonTableView(Table): data: Incomplete height: Incomplete width: int def __init__(self, data) -> None: ... def get_width(self): ... def get_height(self): ... def getCell(self, x, y): ... def setCell(self, x, y, c) -> None: ... def toString(self): ... def getCellView(self): ... def isResizable(self): ... def resize(self, w, h): ... def clear(self) -> None: ... def trimBlank(self): ... def getData(self): ... def insertOrDeleteRows(self, fate, hfate): ... def insertOrDeleteColumns(self, fate, wfate): ... def isSimilar(self, alt): ... def clone(self): ... def create(self): ... def getMeta(self) -> None: ... class SqliteDatabase(SqlDatabase): db: Incomplete fname: Incomplete cursor: Incomplete row: Incomplete quoter: Incomplete view: Incomplete def __init__(self, db, fname) -> None: ... def getQuotedColumnName(self, name): ... def getQuotedTableName(self, name): ... def getColumns(self, name): ... def begin(self, query, args=..., order=...): ... def beginRow(self, tab, row, order=...): ... def read(self): ... def get(self, index): ... def end(self) -> None: ... def rowid(self): ... def getHelper(self): ... def getNameForAttachment(self): ... def get_stdout(): ... def stream_write(s): ... def main() -> None: ... ================================================ FILE: third-party-stubs/mashumaro/__init__.pyi ================================================ from mashumaro.exceptions import MissingField as MissingField from mashumaro.helper import field_options as field_options, pass_through as pass_through from mashumaro.mixins.dict import DataClassDictMixin as DataClassDictMixin ================================================ FILE: third-party-stubs/mashumaro/config.pyi ================================================ from mashumaro.core.const import Sentinel from mashumaro.dialect import Dialect from mashumaro.types import Discriminator, SerializationStrategy from typing import Any, Callable, Dict, List, Optional, Type, Union from typing_extensions import Literal TO_DICT_ADD_BY_ALIAS_FLAG: str TO_DICT_ADD_OMIT_NONE_FLAG: str ADD_DIALECT_SUPPORT: str ADD_SERIALIZATION_CONTEXT: str SerializationStrategyValueType = Union[SerializationStrategy, Dict[str, Union[str, Callable]]] class BaseConfig: debug: bool code_generation_options: List[str] serialization_strategy: Dict[Any, SerializationStrategyValueType] aliases: Dict[str, str] serialize_by_alias: bool namedtuple_as_dict: bool allow_postponed_evaluation: bool dialect: Optional[Type[Dialect]] omit_none: Union[bool, Sentinel.MISSING] orjson_options: Optional[int] json_schema: Dict[str, Any] discriminator: Optional[Discriminator] lazy_compilation: bool ================================================ FILE: third-party-stubs/mashumaro/core/__init__.pyi ================================================ ================================================ FILE: third-party-stubs/mashumaro/core/const.pyi ================================================ import enum from _typeshed import Incomplete PY_37: Incomplete PY_38: Incomplete PY_39: Incomplete PY_310: Incomplete PY_311_MIN: Incomplete PY_310_MIN: Incomplete PY_39_MIN: Incomplete PY_38_MIN: Incomplete PY_37_MIN: Incomplete PEP_585_COMPATIBLE = PY_39_MIN PEP_586_COMPATIBLE = PY_38_MIN class Sentinel(enum.Enum): MISSING: Incomplete ================================================ FILE: third-party-stubs/mashumaro/core/helpers.pyi ================================================ import datetime from _typeshed import Incomplete UTC_OFFSET_PATTERN: str def parse_timezone(s: str) -> datetime.timezone: ... class ConfigValue: name: Incomplete def __init__(self, name: str) -> None: ... ================================================ FILE: third-party-stubs/mashumaro/core/meta/__init__.pyi ================================================ ================================================ FILE: third-party-stubs/mashumaro/core/meta/code/__init__.pyi ================================================ ================================================ FILE: third-party-stubs/mashumaro/core/meta/code/builder.pyi ================================================ import types import typing from _typeshed import Incomplete from collections.abc import Generator from dataclasses import Field from mashumaro.config import ( ADD_DIALECT_SUPPORT as ADD_DIALECT_SUPPORT, ADD_SERIALIZATION_CONTEXT as ADD_SERIALIZATION_CONTEXT, BaseConfig as BaseConfig, SerializationStrategyValueType as SerializationStrategyValueType, TO_DICT_ADD_BY_ALIAS_FLAG as TO_DICT_ADD_BY_ALIAS_FLAG, TO_DICT_ADD_OMIT_NONE_FLAG as TO_DICT_ADD_OMIT_NONE_FLAG, ) from mashumaro.core.const import Sentinel as Sentinel from mashumaro.core.helpers import ConfigValue as ConfigValue from mashumaro.core.meta.code.lines import CodeLines as CodeLines from mashumaro.core.meta.helpers import ( get_args as get_args, get_class_that_defines_field as get_class_that_defines_field, get_class_that_defines_method as get_class_that_defines_method, get_literal_values as get_literal_values, get_name_error_name as get_name_error_name, hash_type_args as hash_type_args, is_class_var as is_class_var, is_dataclass_dict_mixin as is_dataclass_dict_mixin, is_dialect_subclass as is_dialect_subclass, is_init_var as is_init_var, is_literal as is_literal, is_optional as is_optional, is_type_var_any as is_type_var_any, resolve_type_params as resolve_type_params, substitute_type_params as substitute_type_params, type_name as type_name, ) from mashumaro.core.meta.types.common import FieldContext as FieldContext, ValueSpec as ValueSpec from mashumaro.core.meta.types.pack import PackerRegistry as PackerRegistry from mashumaro.core.meta.types.unpack import ( SubtypeUnpackerBuilder as SubtypeUnpackerBuilder, UnpackerRegistry as UnpackerRegistry, ) from mashumaro.dialect import Dialect as Dialect from mashumaro.exceptions import ( BadDialect as BadDialect, BadHookSignature as BadHookSignature, InvalidFieldValue as InvalidFieldValue, MissingDiscriminatorError as MissingDiscriminatorError, MissingField as MissingField, SuitableVariantNotFoundError as SuitableVariantNotFoundError, ThirdPartyModuleNotFoundError as ThirdPartyModuleNotFoundError, UnresolvedTypeReferenceError as UnresolvedTypeReferenceError, UnserializableDataError as UnserializableDataError, UnserializableField as UnserializableField, UnsupportedDeserializationEngine as UnsupportedDeserializationEngine, UnsupportedSerializationEngine as UnsupportedSerializationEngine, ) from mashumaro.types import Discriminator as Discriminator __PRE_SERIALIZE__: str __PRE_DESERIALIZE__: str __POST_SERIALIZE__: str __POST_DESERIALIZE__: str class CodeBuilder: cls: Incomplete lines: Incomplete globals: Incomplete resolved_type_params: Incomplete field_classes: Incomplete initial_type_args: Incomplete dialect: Incomplete default_dialect: Incomplete allow_postponed_evaluation: Incomplete format_name: Incomplete decoder: Incomplete encoder: Incomplete encoder_kwargs: Incomplete def __init__( self, cls: typing.Type, type_args: typing.Tuple[typing.Type, ...] = ..., dialect: typing.Optional[typing.Type[Dialect]] = ..., first_method: str = ..., allow_postponed_evaluation: bool = ..., format_name: str = ..., decoder: typing.Optional[typing.Any] = ..., encoder: typing.Optional[typing.Any] = ..., encoder_kwargs: typing.Optional[typing.Dict[str, typing.Any]] = ..., default_dialect: typing.Optional[typing.Type[Dialect]] = ..., ) -> None: ... def reset(self) -> None: ... @property def namespace(self) -> typing.Mapping[typing.Any, typing.Any]: ... @property def annotations(self) -> typing.Dict[str, typing.Any]: ... def get_field_resolved_type_params( self, field_name: str ) -> typing.Dict[typing.Type, typing.Type]: ... def get_field_types(self, include_extras: bool = ...) -> typing.Dict[str, typing.Any]: ... @property def dataclass_fields(self) -> typing.Dict[str, Field]: ... @property def metadatas(self) -> typing.Dict[str, typing.Mapping[str, typing.Any]]: ... def get_field_default(self, name: str) -> typing.Any: ... def add_type_modules(self, *types_: typing.Type) -> None: ... def ensure_module_imported(self, module: types.ModuleType) -> None: ... def ensure_object_imported( self, obj: typing.Any, name: typing.Optional[str] = ... ) -> None: ... def add_line(self, line: str) -> None: ... def indent(self, expr: typing.Optional[str] = ...) -> typing.Generator[None, None, None]: ... def compile(self) -> None: ... def get_declared_hook(self, method_name: str) -> typing.Any: ... def add_unpack_method(self) -> None: ... def get_config(self, cls: Incomplete | None = ..., look_in_parents: bool = ...): ... def get_discriminator(self) -> typing.Optional[Discriminator]: ... def get_pack_method_flags( self, cls: typing.Optional[typing.Type] = ..., pass_encoder: bool = ... ) -> str: ... def get_unpack_method_flags( self, cls: typing.Optional[typing.Type] = ..., pass_decoder: bool = ... ) -> str: ... def get_pack_method_default_flag_values( self, cls: typing.Optional[typing.Type] = ..., pass_encoder: bool = ... ) -> str: ... def get_unpack_method_default_flag_values(self, pass_decoder: bool = ...) -> str: ... def is_code_generation_option_enabled( self, option: str, cls: typing.Optional[typing.Type] = ... ) -> bool: ... @classmethod def get_unpack_method_name( cls, type_args: typing.Iterable = ..., format_name: str = ..., decoder: typing.Optional[typing.Any] = ..., ) -> str: ... @classmethod def get_pack_method_name( cls, type_args: typing.Tuple[typing.Type, ...] = ..., format_name: str = ..., encoder: typing.Optional[typing.Any] = ..., ) -> str: ... def add_pack_method(self) -> None: ... def iter_serialization_strategies( self, metadata, ftype ) -> Generator[Incomplete, None, None]: ... ================================================ FILE: third-party-stubs/mashumaro/core/meta/code/lines.pyi ================================================ from typing import Generator, Optional class CodeLines: def __init__(self) -> None: ... def append(self, line: str) -> None: ... def indent(self, expr: Optional[str] = ...) -> Generator[None, None, None]: ... def as_text(self) -> str: ... def reset(self) -> None: ... ================================================ FILE: third-party-stubs/mashumaro/core/meta/helpers.pyi ================================================ import typing from typing import Any, Dict, Optional, Sequence, Tuple, Type def get_type_origin(typ: Type) -> Type: ... def get_generic_name(typ: Type, short: bool = ...) -> str: ... def get_args(typ: Optional[Type]) -> Tuple[Type, ...]: ... def get_literal_values(typ: Type) -> Tuple[Any, ...]: ... def type_name( typ: Optional[Type], short: bool = ..., resolved_type_params: Optional[Dict[Type, Type]] = ..., is_type_origin: bool = ..., none_type_as_none: bool = ..., ) -> str: ... def is_special_typing_primitive(typ: Any) -> bool: ... def is_generic(typ: Type) -> bool: ... def is_typed_dict(typ: Type) -> bool: ... def is_named_tuple(typ: Type) -> bool: ... def is_new_type(typ: Type) -> bool: ... def is_union(typ: Type) -> bool: ... def is_optional(typ: Type, resolved_type_params: Optional[Dict[Type, Type]] = ...) -> bool: ... def is_annotated(typ: Type) -> bool: ... def get_type_annotations(typ: Type) -> Sequence[Any]: ... def is_literal(typ: Type) -> bool: ... def not_none_type_arg( type_args: Tuple[Type, ...], resolved_type_params: Optional[Dict[Type, Type]] = ... ) -> Optional[Type]: ... def is_type_var(typ: Type) -> bool: ... def is_type_var_any(typ: Type) -> bool: ... def is_class_var(typ: Type) -> bool: ... def is_final(typ: Type) -> bool: ... def is_init_var(typ: Type) -> bool: ... def get_class_that_defines_method(method_name: str, cls: Type) -> Optional[Type]: ... def get_class_that_defines_field(field_name: str, cls: Type) -> Optional[Type]: ... def is_dataclass_dict_mixin(typ: Type) -> bool: ... def is_dataclass_dict_mixin_subclass(typ: Type) -> bool: ... def collect_type_params(typ: Type) -> Sequence[Type]: ... def resolve_type_params( typ: Type, type_args: Sequence[Type] = ..., include_bases: bool = ... ) -> Dict[Type, Dict[Type, Type]]: ... def substitute_type_params(typ: Type, substitutions: Dict[Type, Type]) -> Type: ... def get_name_error_name(e: NameError) -> str: ... def is_dialect_subclass(typ: Type) -> bool: ... def is_self(typ: Type) -> bool: ... def is_required(typ: Type) -> bool: ... def is_not_required(typ: Type) -> bool: ... def get_function_arg_annotation( function: typing.Callable[[Any], Any], arg_name: typing.Optional[str] = ..., arg_pos: typing.Optional[int] = ..., ) -> typing.Type: ... def get_function_return_annotation( function: typing.Callable[[typing.Any], typing.Any] ) -> typing.Type: ... def is_unpack(typ: Type) -> bool: ... def is_type_var_tuple(typ: Type) -> bool: ... def hash_type_args(type_args: typing.Iterable[typing.Type]) -> str: ... def iter_all_subclasses(cls) -> typing.Iterator[Type]: ... ================================================ FILE: third-party-stubs/mashumaro/core/meta/mixin.pyi ================================================ from mashumaro.dialect import Dialect from typing import Any, Dict, Optional, Tuple, Type def compile_mixin_packer( cls, format_name: str = ..., dialect: Optional[Type[Dialect]] = ..., encoder: Any = ..., encoder_kwargs: Optional[Dict[str, Dict[str, Tuple[str, Any]]]] = ..., ) -> None: ... def compile_mixin_unpacker( cls, format_name: str = ..., dialect: Optional[Type[Dialect]] = ..., decoder: Any = ... ) -> None: ... ================================================ FILE: third-party-stubs/mashumaro/core/meta/types/__init__.pyi ================================================ ================================================ FILE: third-party-stubs/mashumaro/core/meta/types/common.pyi ================================================ from _typeshed import Incomplete from functools import cached_property from mashumaro.core.const import PEP_585_COMPATIBLE as PEP_585_COMPATIBLE from mashumaro.core.meta.code.builder import CodeBuilder as CodeBuilder from mashumaro.core.meta.helpers import ( get_type_origin as get_type_origin, is_annotated as is_annotated, is_generic as is_generic, type_name as type_name, ) from mashumaro.exceptions import ( UnserializableDataError as UnserializableDataError, UnserializableField as UnserializableField, ) from typing import Any, Dict, Mapping, Optional, Sequence, Type, TypeVar from typing_extensions import TypeAlias cached_property = property NoneType: Incomplete Expression: TypeAlias P: Incomplete T = TypeVar("T") class ExpressionWrapper: expression: Incomplete def __init__(self, expression: str) -> None: ... PROPER_COLLECTION_TYPES: Dict[Type, str] class FieldContext: name: str metadata: Mapping def copy(self, **changes: Any) -> FieldContext: ... def __init__(self, name, metadata) -> None: ... class ValueSpec: type: Type origin_type: Type expression: Expression builder: CodeBuilder field_ctx: FieldContext could_be_none: bool annotated_type: Optional[Type] def __setattr__(self, key: str, value: Any) -> None: ... def copy(self, **changes: Any) -> ValueSpec: ... @cached_property def annotations(self) -> Sequence[str]: ... def __init__( self, type, expression, builder, field_ctx, could_be_none, annotated_type ) -> None: ... ValueSpecExprCreator: TypeAlias class Registry: def register(self, function: ValueSpecExprCreator) -> ValueSpecExprCreator: ... def get(self, spec: ValueSpec) -> Expression: ... def __init__(self, _registry) -> None: ... def ensure_generic_collection(spec: ValueSpec) -> bool: ... def ensure_collection_type_args_supported( collection_type: Type, type_args: Sequence[Type] ) -> bool: ... def ensure_generic_collection_subclass(spec: ValueSpec, *checked_types: Type) -> bool: ... def ensure_generic_mapping(spec: ValueSpec, args: Sequence[Type], checked_type: Type) -> bool: ... def expr_or_maybe_none(spec: ValueSpec, new_expr: Expression) -> Expression: ... def random_hex() -> str: ... def clean_id(value: str) -> str: ... ================================================ FILE: third-party-stubs/mashumaro/core/meta/types/pack.pyi ================================================ from _typeshed import Incomplete PackerRegistry: Incomplete ================================================ FILE: third-party-stubs/mashumaro/core/meta/types/unpack.pyi ================================================ import abc from _typeshed import Incomplete from abc import ABC, abstractmethod from mashumaro.core.meta.types.common import ValueSpec from mashumaro.types import Discriminator from typing import Optional, Tuple, Type UnpackerRegistry: Incomplete class AbstractUnpackerBuilder(ABC, metaclass=abc.ABCMeta): @abstractmethod def get_method_prefix(self) -> str: ... def build(self, spec: ValueSpec) -> str: ... class UnionUnpackerBuilder(AbstractUnpackerBuilder): union_args: Incomplete def __init__(self, args: Tuple[Type, ...]) -> None: ... def get_method_prefix(self) -> str: ... class TypeVarUnpackerBuilder(UnionUnpackerBuilder): def get_method_prefix(self) -> str: ... class LiteralUnpackerBuilder(AbstractUnpackerBuilder): def get_method_prefix(self) -> str: ... class DiscriminatedUnionUnpackerBuilder(AbstractUnpackerBuilder): discriminator: Incomplete base_variants: Incomplete def __init__( self, discriminator: Discriminator, base_variants: Optional[Tuple[Type, ...]] = ... ) -> None: ... def get_method_prefix(self) -> str: ... class SubtypeUnpackerBuilder(DiscriminatedUnionUnpackerBuilder): ... ================================================ FILE: third-party-stubs/mashumaro/dialect.pyi ================================================ from mashumaro.core.const import Sentinel from mashumaro.types import SerializationStrategy from typing import Callable, Dict, Union from typing_extensions import Literal SerializationStrategyValueType = Union[SerializationStrategy, Dict[str, Union[str, Callable]]] class Dialect: serialization_strategy: Dict[str, SerializationStrategyValueType] omit_none: Union[bool, Sentinel.MISSING] ================================================ FILE: third-party-stubs/mashumaro/exceptions.pyi ================================================ from _typeshed import Incomplete from mashumaro.core.meta.helpers import type_name as type_name from typing import Any, Optional, Type class MissingField(LookupError): field_name: Incomplete field_type: Incomplete holder_class: Incomplete def __init__(self, field_name: str, field_type: Type, holder_class: Type) -> None: ... @property def field_type_name(self) -> str: ... @property def holder_class_name(self) -> str: ... class UnserializableDataError(TypeError): ... class UnserializableField(UnserializableDataError): field_name: Incomplete field_type: Incomplete holder_class: Incomplete msg: Incomplete def __init__( self, field_name: str, field_type: Type, holder_class: Type, msg: Optional[str] = ... ) -> None: ... @property def field_type_name(self) -> str: ... @property def holder_class_name(self) -> str: ... class UnsupportedSerializationEngine(UnserializableField): def __init__( self, field_name: str, field_type: Type, holder_class: Type, engine: Any ) -> None: ... class UnsupportedDeserializationEngine(UnserializableField): def __init__( self, field_name: str, field_type: Type, holder_class: Type, engine: Any ) -> None: ... class InvalidFieldValue(ValueError): field_name: Incomplete field_type: Incomplete field_value: Incomplete holder_class: Incomplete msg: Incomplete def __init__( self, field_name: str, field_type: Type, field_value: Any, holder_class: Type, msg: Optional[str] = ..., ) -> None: ... @property def field_type_name(self) -> str: ... @property def holder_class_name(self) -> str: ... class MissingDiscriminatorError(LookupError): field_name: Incomplete def __init__(self, field_name: str) -> None: ... class SuitableVariantNotFoundError(ValueError): variants_type: Incomplete discriminator_name: Incomplete discriminator_value: Incomplete def __init__( self, variants_type: Type, discriminator_name: Optional[str] = ..., discriminator_value: Any = ..., ) -> None: ... class BadHookSignature(TypeError): ... class ThirdPartyModuleNotFoundError(ModuleNotFoundError): module_name: Incomplete field_name: Incomplete holder_class: Incomplete def __init__(self, module_name: str, field_name: str, holder_class: Type) -> None: ... @property def holder_class_name(self) -> str: ... class UnresolvedTypeReferenceError(NameError): holder_class: Incomplete name: Incomplete def __init__(self, holder_class: Type, unresolved_type_name: str) -> None: ... @property def holder_class_name(self) -> str: ... class BadDialect(ValueError): ... ================================================ FILE: third-party-stubs/mashumaro/helper.pyi ================================================ from mashumaro.types import SerializationStrategy from typing import Any, Callable, Dict, Optional, TypeVar, Union from typing_extensions import Literal NamedTupleDeserializationEngine = Literal["as_dict", "as_list"] DateTimeDeserializationEngine = Literal["ciso8601", "pendulum"] AnyDeserializationEngine = Literal[NamedTupleDeserializationEngine, DateTimeDeserializationEngine] NamedTupleSerializationEngine = Literal["as_dict", "as_list"] AnySerializationEngine = Union[NamedTupleSerializationEngine, OmitSerializationEngine] OmitSerializationEngine = Literal["omit"] T = TypeVar("T") def field_options( serialize: Optional[Union[AnySerializationEngine, Callable[[Any], Any]]] = ..., deserialize: Optional[Union[AnyDeserializationEngine, Callable[[Any], Any]]] = ..., serialization_strategy: Optional[SerializationStrategy] = ..., alias: Optional[str] = ..., ) -> Dict[str, Any]: ... class _PassThrough(SerializationStrategy): def __call__(self, *args: Any, **kwargs: Any) -> Any: ... def serialize(self, value: T) -> T: ... def deserialize(self, value: T) -> T: ... pass_through: Any ================================================ FILE: third-party-stubs/mashumaro/jsonschema/__init__.pyi ================================================ from .builder import JSONSchemaBuilder as JSONSchemaBuilder, build_json_schema as build_json_schema from .dialects import DRAFT_2020_12 as DRAFT_2020_12, OPEN_API_3_1 as OPEN_API_3_1 ================================================ FILE: third-party-stubs/mashumaro/jsonschema/annotations.pyi ================================================ from mashumaro.jsonschema.models import JSONSchema, Number from typing import Dict, Set class Annotation: ... class Constraint(Annotation): ... class NumberConstraint(Constraint): ... class Minimum(NumberConstraint): value: Number def __init__(self, value) -> None: ... class Maximum(NumberConstraint): value: Number def __init__(self, value) -> None: ... class ExclusiveMinimum(NumberConstraint): value: Number def __init__(self, value) -> None: ... class ExclusiveMaximum(NumberConstraint): value: Number def __init__(self, value) -> None: ... class MultipleOf(NumberConstraint): value: Number def __init__(self, value) -> None: ... class StringConstraint(Constraint): ... class MinLength(StringConstraint): value: int def __init__(self, value) -> None: ... class MaxLength(StringConstraint): value: int def __init__(self, value) -> None: ... class Pattern(StringConstraint): value: str def __init__(self, value) -> None: ... class ArrayConstraint(Constraint): ... class MinItems(ArrayConstraint): value: int def __init__(self, value) -> None: ... class MaxItems(ArrayConstraint): value: int def __init__(self, value) -> None: ... class UniqueItems(ArrayConstraint): value: bool def __init__(self, value) -> None: ... class Contains(ArrayConstraint): value: JSONSchema def __init__(self, value) -> None: ... class MinContains(ArrayConstraint): value: int def __init__(self, value) -> None: ... class MaxContains(ArrayConstraint): value: int def __init__(self, value) -> None: ... class ObjectConstraint(Constraint): ... class MaxProperties(ObjectConstraint): value: int def __init__(self, value) -> None: ... class MinProperties(ObjectConstraint): value: int def __init__(self, value) -> None: ... class DependentRequired(ObjectConstraint): value: Dict[str, Set[str]] def __init__(self, value) -> None: ... ================================================ FILE: third-party-stubs/mashumaro/jsonschema/builder.pyi ================================================ from _typeshed import Incomplete from mashumaro.jsonschema.dialects import JSONSchemaDialect from mashumaro.jsonschema.models import Context, JSONSchema from mashumaro.mixins.json import DataClassJSONMixin from typing import Any, Dict, List, Optional, Type def build_json_schema( instance_type: Type, context: Optional[Context] = ..., with_definitions: bool = ..., all_refs: Optional[bool] = ..., with_dialect_uri: bool = ..., dialect: Optional[JSONSchemaDialect] = ..., ref_prefix: Optional[str] = ..., ) -> JSONSchema: ... class JSONSchemaDefinitions(DataClassJSONMixin): definitions: Dict[str, JSONSchema] def __post_serialize__(self, d: Dict[Any, Any], context: Optional[Dict]) -> List[Dict[str, Any]]: ... # type: ignore def __init__(self, definitions) -> None: ... class JSONSchemaBuilder: context: Incomplete def __init__( self, dialect: JSONSchemaDialect = ..., all_refs: Optional[bool] = ..., ref_prefix: Optional[str] = ..., ) -> None: ... def build(self, instance_type: Type) -> JSONSchema: ... def get_definitions(self) -> JSONSchemaDefinitions: ... ================================================ FILE: third-party-stubs/mashumaro/jsonschema/dialects.pyi ================================================ from _typeshed import Incomplete class JSONSchemaDialect: uri: str definitions_root_pointer: str all_refs: bool def __init__(self, uri, definitions_root_pointer, all_refs) -> None: ... class JSONSchemaDraft202012Dialect(JSONSchemaDialect): uri: str definitions_root_pointer: str all_refs: bool def __init__(self, uri, definitions_root_pointer, all_refs) -> None: ... class OpenAPISchema31Dialect(JSONSchemaDialect): uri: str definitions_root_pointer: str all_refs: bool def __init__(self, uri, definitions_root_pointer, all_refs) -> None: ... DRAFT_2020_12: Incomplete OPEN_API_3_1: Incomplete ================================================ FILE: third-party-stubs/mashumaro/jsonschema/models.pyi ================================================ from _typeshed import Incomplete from enum import Enum from mashumaro.config import BaseConfig as BaseConfig from mashumaro.helper import pass_through as pass_through from mashumaro.jsonschema.dialects import ( DRAFT_2020_12 as DRAFT_2020_12, JSONSchemaDialect as JSONSchemaDialect, ) from mashumaro.mixins.json import DataClassJSONMixin as DataClassJSONMixin from typing import Any, Dict, List, Optional, Set, Union from typing_extensions import TypeAlias Number: TypeAlias = Union[int, float] Null = object() class JSONSchemaInstanceType(Enum): NULL: str BOOLEAN: str OBJECT: str ARRAY: str NUMBER: str STRING: str INTEGER: str class JSONSchemaInstanceFormat(Enum): ... class JSONSchemaStringFormat(JSONSchemaInstanceFormat): DATETIME: str DATE: str TIME: str DURATION: str EMAIL: str IDN_EMAIL: str HOSTNAME: str IDN_HOSTNAME: str IPV4ADDRESS: str IPV6ADDRESS: str URI: str URI_REFERENCE: str IRI: str IRI_REFERENCE: str UUID: str URI_TEMPLATE: str JSON_POINTER: str RELATIVE_JSON_POINTER: str REGEX: str class JSONSchemaInstanceFormatExtension(JSONSchemaInstanceFormat): TIMEDELTA: str TIME_ZONE: str IPV4NETWORK: str IPV6NETWORK: str IPV4INTERFACE: str IPV6INTERFACE: str DECIMAL: str FRACTION: str BASE64: str PATH: str DATETIME_FORMATS: Incomplete IPADDRESS_FORMATS: Incomplete class JSONSchema(DataClassJSONMixin): schema: Optional[str] type: Optional[JSONSchemaInstanceType] enum: Optional[List[Any]] const: Optional[Any] format: Optional[ Union[JSONSchemaInstanceFormat, JSONSchemaStringFormat, JSONSchemaInstanceFormatExtension] ] title: Optional[str] description: Optional[str] anyOf: Optional[List["JSONSchema"]] reference: Optional[str] definitions: Optional[Dict[str, "JSONSchema"]] default: Optional[Any] deprecated: Optional[bool] examples: Optional[List[Any]] properties: Optional[Dict[str, "JSONSchema"]] patternProperties: Optional[Dict[str, "JSONSchema"]] additionalProperties: Union["JSONSchema", bool, None] propertyNames: Optional["JSONSchema"] prefixItems: Optional[List["JSONSchema"]] items: Optional["JSONSchema"] contains: Optional["JSONSchema"] multipleOf: Optional[Number] maximum: Optional[Number] exclusiveMaximum: Optional[Number] minimum: Optional[Number] exclusiveMinimum: Optional[Number] maxLength: Optional[int] minLength: Optional[int] pattern: Optional[str] maxItems: Optional[int] minItems: Optional[int] uniqueItems: Optional[bool] maxContains: Optional[int] minContains: Optional[int] maxProperties: Optional[int] minProperties: Optional[int] required: Optional[List[str]] dependentRequired: Optional[Dict[str, Set[str]]] class Config(BaseConfig): omit_none: bool serialize_by_alias: bool aliases: Incomplete serialization_strategy: Incomplete def __pre_serialize__(self, context: Optional[Dict]) -> JSONSchema: ... def __post_serialize__(self, d: Dict[Any, Any], context: Optional[Dict]) -> Dict[Any, Any]: ... def __init__( self, schema, type, enum, const, format, title, description, anyOf, reference, definitions, default, deprecated, examples, properties, patternProperties, additionalProperties, propertyNames, prefixItems, items, contains, multipleOf, maximum, exclusiveMaximum, minimum, exclusiveMinimum, maxLength, minLength, pattern, maxItems, minItems, uniqueItems, maxContains, minContains, maxProperties, minProperties, required, dependentRequired, ) -> None: ... class JSONObjectSchema(JSONSchema): type: JSONSchemaInstanceType def __init__( self, schema, type, enum, const, format, title, description, anyOf, reference, definitions, default, deprecated, examples, properties, patternProperties, additionalProperties, propertyNames, prefixItems, items, contains, multipleOf, maximum, exclusiveMaximum, minimum, exclusiveMinimum, maxLength, minLength, pattern, maxItems, minItems, uniqueItems, maxContains, minContains, maxProperties, minProperties, required, dependentRequired, ) -> None: ... class JSONArraySchema(JSONSchema): type: JSONSchemaInstanceType def __init__( self, schema, type, enum, const, format, title, description, anyOf, reference, definitions, default, deprecated, examples, properties, patternProperties, additionalProperties, propertyNames, prefixItems, items, contains, multipleOf, maximum, exclusiveMaximum, minimum, exclusiveMinimum, maxLength, minLength, pattern, maxItems, minItems, uniqueItems, maxContains, minContains, maxProperties, minProperties, required, dependentRequired, ) -> None: ... class Context: dialect: JSONSchemaDialect definitions: Dict[str, JSONSchema] all_refs: Optional[bool] ref_prefix: Optional[str] def __init__(self, dialect, definitions, all_refs, ref_prefix) -> None: ... ================================================ FILE: third-party-stubs/mashumaro/jsonschema/schema.pyi ================================================ from mashumaro.config import BaseConfig from mashumaro.jsonschema.annotations import Annotation from mashumaro.jsonschema.models import Context, JSONSchema from typing import Any, Callable, Iterable, List, Mapping, Optional, Tuple, Type, Union class Instance: type: Type name: Optional[str] origin_type: Type annotations: List[Annotation] @property def metadata(self) -> Mapping[str, Any]: ... @property def alias(self) -> Optional[str]: ... @property def holder_class(self) -> Optional[Type]: ... def copy(self, **changes: Any) -> Instance: ... def __post_init__(self) -> None: ... def update_type(self, new_type: Type) -> None: ... def fields(self) -> Iterable[Tuple[str, Type, bool, Any]]: ... def get_overridden_serialization_method(self) -> Optional[Union[Callable, str]]: ... def get_config(self) -> Type[BaseConfig]: ... def __init__(self, type, name, __builder) -> None: ... class InstanceSchemaCreatorRegistry: def register(self, func: InstanceSchemaCreator) -> InstanceSchemaCreator: ... def iter(self) -> Iterable[InstanceSchemaCreator]: ... def __init__(self, _registry) -> None: ... class EmptyJSONSchema(JSONSchema): def __init__( self, schema, type, enum, const, format, title, description, anyOf, reference, definitions, default, deprecated, examples, properties, patternProperties, additionalProperties, propertyNames, prefixItems, items, contains, multipleOf, maximum, exclusiveMaximum, minimum, exclusiveMinimum, maxLength, minLength, pattern, maxItems, minItems, uniqueItems, maxContains, minContains, maxProperties, minProperties, required, dependentRequired, ) -> None: ... def get_schema(instance: Instance, ctx: Context, with_dialect_uri: bool = ...) -> JSONSchema: ... ================================================ FILE: third-party-stubs/mashumaro/mixins/__init__.pyi ================================================ ================================================ FILE: third-party-stubs/mashumaro/mixins/dict.pyi ================================================ from typing import Any, Dict, Mapping, Type, TypeVar, Optional T = TypeVar("T", bound="DataClassDictMixin") class DataClassDictMixin: def __init_subclass__(cls: Type[T], **kwargs: Any) -> None: ... def to_dict(self, **kwargs: Any) -> dict: ... @classmethod def from_dict(cls, d: Mapping, **kwargs: Any) -> Any: ... @classmethod def __pre_deserialize__(cls: Type[T], d: Dict[Any, Any]) -> Dict[Any, Any]: ... @classmethod def __post_deserialize__(cls: Type[T], obj: T) -> T: ... def __pre_serialize__(self: T, context: Optional[Dict]) -> T: ... def __post_serialize__(self, d: Dict[Any, Any], context: Optional[Dict]) -> Dict[Any, Any]: ... ================================================ FILE: third-party-stubs/mashumaro/mixins/json.pyi ================================================ from mashumaro.mixins.dict import DataClassDictMixin as DataClassDictMixin from typing import Any, Callable, Dict, TypeVar, Union, Type EncodedData = Union[str, bytes, bytearray] T = TypeVar("T", bound="DataClassJSONMixin") class Encoder: def __call__(self, obj: Any, **kwargs: Any) -> EncodedData: ... class Decoder: def __call__(self, s: EncodedData, **kwargs: Any) -> Dict[Any, Any]: ... class DataClassJSONMixin(DataClassDictMixin): def to_json(self, encoder: Encoder = ..., **to_dict_kwargs: Any) -> EncodedData: ... @classmethod def from_json( cls: Type[T], data: EncodedData, decoder: Decoder = ..., **from_dict_kwargs: Any ) -> T: ... ================================================ FILE: third-party-stubs/mashumaro/mixins/msgpack.pyi ================================================ from _typeshed import Incomplete from mashumaro.dialect import Dialect as Dialect from mashumaro.helper import pass_through as pass_through from mashumaro.mixins.dict import DataClassDictMixin as DataClassDictMixin from typing import Any, Callable, Dict, TypeVar, Type T = TypeVar("T", bound="DataClassMessagePackMixin") EncodedData = bytes Encoder = Callable[[Any], EncodedData] Decoder = Callable[[EncodedData], Dict[Any, Any]] class MessagePackDialect(Dialect): serialization_strategy: Incomplete def default_encoder(data: Any) -> EncodedData: ... def default_decoder(data: EncodedData) -> Dict[Any, Any]: ... class DataClassMessagePackMixin(DataClassDictMixin): def to_msgpack(self, encoder: Encoder = ..., **to_dict_kwargs: Any) -> EncodedData: ... @classmethod def from_msgpack( cls: Type[T], data: EncodedData, decoder: Decoder = ..., **from_dict_kwargs: Any ) -> T: ... ================================================ FILE: third-party-stubs/mashumaro/mixins/orjson.pyi ================================================ from mashumaro.dialect import Dialect as Dialect from mashumaro.mixins.dict import DataClassDictMixin as DataClassDictMixin from typing import Any, Callable, Dict, TypeVar, Union T = TypeVar("T", bound="DataClassORJSONMixin") EncodedData = Union[str, bytes, bytearray] Encoder = Callable[[Any], EncodedData] Decoder = Callable[[EncodedData], Dict[Any, Any]] class OrjsonDialect(Dialect): serialization_strategy: Any class DataClassORJSONMixin(DataClassDictMixin): def to_jsonb( self, encoder: Encoder = ..., *, orjson_options: int = ..., **to_dict_kwargs: Any ) -> bytes: ... def to_json( self, encoder: Encoder = ..., *, orjson_options: int = ..., **to_dict_kwargs: Any ) -> bytes: ... @classmethod def from_json( cls, data: EncodedData, decoder: Decoder = ..., **from_dict_kwargs: Any ) -> T: ... ================================================ FILE: third-party-stubs/mashumaro/mixins/toml.pyi ================================================ from _typeshed import Incomplete from mashumaro.dialect import Dialect as Dialect from mashumaro.helper import pass_through as pass_through from mashumaro.mixins.dict import DataClassDictMixin as DataClassDictMixin from typing import Any, Callable, Dict, TypeVar T = TypeVar("T", bound="DataClassTOMLMixin") EncodedData = str Encoder = Callable[[Any], EncodedData] Decoder = Callable[[EncodedData], Dict[Any, Any]] class TOMLDialect(Dialect): omit_none: bool serialization_strategy: Incomplete class DataClassTOMLMixin(DataClassDictMixin): def to_toml(self, encoder: Encoder = ..., **to_dict_kwargs: Any) -> EncodedData: ... @classmethod def from_toml( cls, data: EncodedData, decoder: Decoder = ..., **from_dict_kwargs: Any ) -> T: ... ================================================ FILE: third-party-stubs/mashumaro/mixins/yaml.pyi ================================================ from _typeshed import Incomplete from mashumaro.mixins.dict import DataClassDictMixin as DataClassDictMixin from typing import Any, Callable, Dict, TypeVar, Union T = TypeVar("T", bound="DataClassYAMLMixin") EncodedData = Union[str, bytes] Encoder = Callable[[Any], EncodedData] Decoder = Callable[[EncodedData], Dict[Any, Any]] DefaultLoader: Incomplete DefaultDumper: Incomplete def default_encoder(data: Any) -> EncodedData: ... def default_decoder(data: EncodedData) -> Dict[Any, Any]: ... class DataClassYAMLMixin(DataClassDictMixin): def to_yaml(self, encoder: Encoder = ..., **to_dict_kwargs: Any) -> EncodedData: ... @classmethod def from_yaml( cls, data: EncodedData, decoder: Decoder = ..., **from_dict_kwargs: Any ) -> T: ... ================================================ FILE: third-party-stubs/mashumaro/types.pyi ================================================ import decimal from _typeshed import Incomplete from mashumaro.core.const import Sentinel from typing import Any, Optional, Union from typing_extensions import Literal class SerializableType: ... class GenericSerializableType: ... class SerializationStrategy: def serialize(self, value: Any) -> Any: ... def deserialize(self, value: Any) -> Any: ... class RoundedDecimal(SerializationStrategy): exp: Incomplete rounding: Incomplete def __init__(self, places: Optional[int] = ..., rounding: Optional[str] = ...) -> None: ... def serialize(self, value: decimal.Decimal) -> str: ... def deserialize(self, value: str) -> decimal.Decimal: ... class Discriminator: field: Optional[str] include_supertypes: bool include_subtypes: bool def __post_init__(self) -> None: ... def __init__(self, field, include_supertypes, include_subtypes) -> None: ... ================================================ FILE: third-party-stubs/msgpack/__init__.pyi ================================================ from __future__ import annotations from typing import Any, Callable, Dict, List, Optional, Tuple from msgpack.exceptions import ( BufferFull, ExtraData, FormatError, OutOfData, PackException, PackOverflowError, PackValueError, StackError, UnpackException, UnpackValueError, ) from typing_extensions import Protocol from msgpack.fallback import Packer, Unpacker, unpackb from msgpack import exceptions from msgpack.ext import ExtType from msgpack import ext class _Stream(Protocol): def read(self) -> bytes: ... class _FileLike(Protocol): def read(self, n: int) -> bytes: ... def pack( o: Any, stream: _Stream, default: Optional[Callable[[Any], Any]] = ..., use_single_float: bool = ..., autoreset: bool = ..., use_bin_type: bool = ..., strict_types: bool = ..., datetime: bool = ..., unicode_errors: Optional[str] = ..., ) -> None: ... def packb( o: Any, default: Optional[Callable[[Any], Any]] = ..., use_single_float: bool = ..., autoreset: bool = ..., use_bin_type: bool = ..., strict_types: bool = ..., datetime: bool = ..., unicode_errors: Optional[str] = ..., ) -> bytes: ... def unpack( stream: _Stream, file_like: Optional[_FileLike] = ..., read_size: int = ..., use_list: bool = ..., raw: bool = ..., timestamp: int = ..., strict_map_key: bool = ..., object_hook: Optional[Callable[[Dict[Any, Any]], Any]] = ..., object_pairs_hook: Optional[Callable[[List[Tuple[Any, Any]]], Any]] = ..., list_hook: Optional[Callable[[List[Any]], Any]] = ..., unicode_errors: Optional[str] = ..., max_buffer_size: int = ..., ext_hook: Callable[[int, bytes], Any] = ..., max_str_len: int = ..., max_bin_len: int = ..., max_array_len: int = ..., max_map_len: int = ..., max_ext_len: int = ..., ) -> Any: ... load = unpack loads = unpackb dump = pack dumps = packb __all__ = [ "BufferFull", "ExtType", "ExtraData", "FormatError", "OutOfData", "PackException", "PackOverflowError", "PackValueError", "Packer", "StackError", "UnpackException", "UnpackValueError", "Unpacker", "dump", "dumps", "exceptions", "ext", "load", "loads", "pack", "packb", "unpack", "unpackb", ] ================================================ FILE: third-party-stubs/msgpack/_version.pyi ================================================ from typing import Tuple version: Tuple[int, int, int] ================================================ FILE: third-party-stubs/msgpack/exceptions.pyi ================================================ from typing import Any class UnpackException(Exception): ... class BufferFull(UnpackException): ... class OutOfData(UnpackException): ... class FormatError(ValueError, UnpackException): ... class StackError(ValueError, UnpackException): ... UnpackValueError = ValueError class ExtraData(UnpackValueError): def __init__(self, unpacked: Any, exta: Any) -> None: ... PackException = Exception PackValueError = ValueError PackOverflowError = OverflowError ================================================ FILE: third-party-stubs/msgpack/ext.pyi ================================================ from __future__ import annotations from typing import NamedTuple import datetime class _ExtType(NamedTuple): code: int data: bytes class ExtType(_ExtType): ... class TimeStamp: def __init__(self, seconds: int, nanoseconds: int = ...) -> None: ... def __eq__(self, o: object) -> bool: ... def __ne__(self, o: object) -> bool: ... @staticmethod def from_bytes(b: bytes) -> TimeStamp: ... @staticmethod def to_bytes(self) -> bytes: ... @staticmethod def from_unix(self, unix_sec: float) -> TimeStamp: ... def to_unix(self) -> float: ... @staticmethod def from_unix_nano(unix_ns: int) -> TimeStamp: ... @staticmethod def to_unix_nano(self) -> int: ... def to_datetime(self) -> datetime.datetime: ... @staticmethod def from_datetime(dt: datetime.datetime) -> TimeStamp: ... ================================================ FILE: third-party-stubs/msgpack/fallback.pyi ================================================ from __future__ import annotations from typing import Any, Callable, Dict, List, Optional, Tuple from typing_extensions import Protocol class _FileLike(Protocol): def read(self, n: int) -> bytes: ... def unpackb( packed: bytes, file_like: Optional[_FileLike] = ..., read_size: int = ..., use_list: bool = ..., raw: bool = ..., timestamp: int = ..., strict_map_key: bool = ..., object_hook: Optional[Callable[[Dict[Any, Any]], Any]] = ..., object_pairs_hook: Optional[Callable[[List[Tuple[Any, Any]]], Any]] = ..., unicode_errors: Optional[str] = ..., max_buffer_size: int = ..., ext_hook: Callable[[int, bytes], Any] = ..., max_str_len: int = ..., max_bin_len: int = ..., max_array_len: int = ..., max_map_len: int = ..., max_ext_len: int = ..., ) -> Any: ... class Unpacker: def __init__( self, file_like: Optional[_FileLike] = ..., read_size: int = ..., use_list: bool = ..., raw: bool = ..., timestamp: int = ..., strict_map_key: bool = ..., object_hook: Optional[Callable[[Dict[Any, Any]], Any]] = ..., object_pairs_hook: Optional[Callable[[List[Tuple[Any, Any]]], Any]] = ..., unicode_errors: Optional[str] = ..., max_buffer_size: int = ..., ext_hook: Callable[[int, bytes], Any] = ..., max_str_len: int = ..., max_bin_len: int = ..., max_array_len: int = ..., max_map_len: int = ..., max_ext_len: int = ..., ): ... def feed(self, next_bytes: bytes) -> None: ... def read_bytes(self, n: int) -> bytearray: ... def __iter__(self) -> Unpacker: ... def __next__(self) -> Any: ... def next(self) -> Any: ... def skip(self) -> None: ... def unpack(self) -> Any: ... def read_array_header(self) -> Any: ... def read_map_header(self) -> Any: ... def tell(self) -> int: ... class Packer: def __init__( self, default: Optional[Callable[[Any], Any]] = ..., use_single_float: bool = ..., autoreset: bool = ..., use_bin_type: bool = ..., strict_types: bool = ..., datetime: bool = ..., unicode_errors: Optional[str] = ..., ): ... def pack(self, obj: Any) -> bytes: ... def pack_map_pairs(self, pairs: Any) -> bytes: ... def pack_array_header(self, n: int) -> bytes: ... def pack_map_header(self, n: int) -> bytes: ... def pack_ext_type(self, typecode: int, data: bytes) -> None: ... def bytes(self) -> bytes: ... def reset(self) -> None: ... def getbuffer(self) -> memoryview: ... ================================================ FILE: third-party-stubs/snowplow_tracker/__init__.pyi ================================================ import logging from typing import Union, Optional, List, Any, Dict class Subject: def __init__(self) -> None: ... def set_platform(self, value: Any): ... def set_user_id(self, user_id: Any): ... def set_screen_resolution(self, width: Any, height: Any): ... def set_viewport(self, width: Any, height: Any): ... def set_color_depth(self, depth: Any): ... def set_timezone(self, timezone: Any): ... def set_lang(self, lang: Any): ... def set_domain_user_id(self, duid: Any): ... def set_ip_address(self, ip: Any): ... def set_useragent(self, ua: Any): ... def set_network_user_id(self, nuid: Any): ... logger: logging.Logger class Emitter: endpoint: str def __init__( self, endpoint: str, protocol: str = ..., port: Optional[int] = ..., method: str = ..., buffer_size: Optional[int] = ..., on_success: Optional[Any] = ..., on_failure: Optional[Any] = ..., byte_limit: Optional[int] = ..., ) -> None: ... def is_good_status_code(self, status_code: int) -> bool: ... class Tracker: emitters: Union[List[Any], Any] = ... subject: Optional[Subject] = ... namespace: Optional[str] = ... app_id: Optional[str] = ... encode_base64: bool = ... def __init__( self, emitters: Union[List[Any], Any], subject: Optional[Subject] = ..., namespace: Optional[str] = ..., app_id: Optional[str] = ..., encode_base64: bool = ..., ) -> None: ... def set_subject(self, subject: Optional[Subject]): ... def track_struct_event( self, category: str, action: str, label: Optional[str] = None, property_: Optional[str] = None, value: Optional[float] = None, context: Optional[List[Any]] = None, tstamp: Optional[Any] = None, ): ... def flush(self, asynchronous: bool = False): ... class SelfDescribingJson: schema: Any = ... data: Any = ... def __init__(self, schema: Any, data: Any) -> None: ... def to_json(self) -> Dict[str, Any]: ... def to_string(self) -> str: ... ================================================ FILE: third-party-stubs/sqlparse/__init__.pyi ================================================ from typing import Tuple from . import sql from . import tokens def parse(sql: str) -> Tuple[sql.Statement]: ... ================================================ FILE: third-party-stubs/sqlparse/sql.pyi ================================================ from typing import Tuple, Iterable class Token: def __init__(self, ttype, value): ... is_keyword: bool is_whitespace: bool normalized: str class TokenList(Token): tokens: Tuple[Token] def __getitem__(self, key) -> Token: ... def __iter__(self) -> Iterable[Token]: ... def insert_before(self, where, token, skip_ws=True): ... def insert_after(self, where, token, skip_ws=True): ... def token_first(self) -> Token: ... class Statement(TokenList): ... ================================================ FILE: third-party-stubs/sqlparse/tokens.pyi ================================================ class _TokenType(tuple): ... Keyword: _TokenType = _TokenType() Punctuation: _TokenType = _TokenType()